1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
2 /* Copyright (C) 2015-2018 Netronome Systems, Inc. */
3 
4 /*
5  * nfp_target.c
6  * CPP Access Width Decoder
7  * Authors: Jakub Kicinski <jakub.kicinski@netronome.com>
8  *          Jason McMullan <jason.mcmullan@netronome.com>
9  *          Francois H. Theron <francois.theron@netronome.com>
10  */
11 
12 #define pr_fmt(fmt)       "NFP target: " fmt
13 
14 #include <linux/bitops.h>
15 #include <linux/kernel.h>
16 #include <linux/printk.h>
17 
18 #include "nfp_cpp.h"
19 
20 #include "nfp6000/nfp6000.h"
21 
22 #define P32 1
23 #define P64 2
24 
25 /* This structure ONLY includes items that can be done with a read or write of
26  * 32-bit or 64-bit words. All others are not listed.
27  */
28 
29 #define AT(_action, _token, _pull, _push)				\
30 	case NFP_CPP_ID(0, (_action), (_token)):			\
31 		return PUSHPULL((_pull), (_push))
32 
target_rw(u32 cpp_id,int pp,int start,int len)33 static int target_rw(u32 cpp_id, int pp, int start, int len)
34 {
35 	switch (cpp_id & NFP_CPP_ID(0, ~0, ~0)) {
36 	AT(0, 0,  0, pp);
37 	AT(1, 0, pp,  0);
38 	AT(NFP_CPP_ACTION_RW, 0, pp, pp);
39 	default:
40 		return -EINVAL;
41 	}
42 }
43 
nfp6000_nbi_dma(u32 cpp_id)44 static int nfp6000_nbi_dma(u32 cpp_id)
45 {
46 	switch (cpp_id & NFP_CPP_ID(0, ~0, ~0)) {
47 	AT(0, 0,   0, P64);	/* ReadNbiDma */
48 	AT(1, 0,   P64, 0);	/* WriteNbiDma */
49 	AT(NFP_CPP_ACTION_RW, 0, P64, P64);
50 	default:
51 		return -EINVAL;
52 	}
53 }
54 
nfp6000_nbi_stats(u32 cpp_id)55 static int nfp6000_nbi_stats(u32 cpp_id)
56 {
57 	switch (cpp_id & NFP_CPP_ID(0, ~0, ~0)) {
58 	AT(0, 0,   0, P32);	/* ReadNbiStats */
59 	AT(1, 0,   P32, 0);	/* WriteNbiStats */
60 	AT(NFP_CPP_ACTION_RW, 0, P32, P32);
61 	default:
62 		return -EINVAL;
63 	}
64 }
65 
nfp6000_nbi_tm(u32 cpp_id)66 static int nfp6000_nbi_tm(u32 cpp_id)
67 {
68 	switch (cpp_id & NFP_CPP_ID(0, ~0, ~0)) {
69 	AT(0, 0,   0, P64);	/* ReadNbiTM */
70 	AT(1, 0,   P64, 0);	/* WriteNbiTM */
71 	AT(NFP_CPP_ACTION_RW, 0, P64, P64);
72 	default:
73 		return -EINVAL;
74 	}
75 }
76 
nfp6000_nbi_ppc(u32 cpp_id)77 static int nfp6000_nbi_ppc(u32 cpp_id)
78 {
79 	switch (cpp_id & NFP_CPP_ID(0, ~0, ~0)) {
80 	AT(0, 0,   0, P64);	/* ReadNbiPreclassifier */
81 	AT(1, 0,   P64, 0);	/* WriteNbiPreclassifier */
82 	AT(NFP_CPP_ACTION_RW, 0, P64, P64);
83 	default:
84 		return -EINVAL;
85 	}
86 }
87 
nfp6000_nbi(u32 cpp_id,u64 address)88 static int nfp6000_nbi(u32 cpp_id, u64 address)
89 {
90 	u64 rel_addr = address & 0x3fFFFF;
91 
92 	if (rel_addr < (1 << 20))
93 		return nfp6000_nbi_dma(cpp_id);
94 	if (rel_addr < (2 << 20))
95 		return nfp6000_nbi_stats(cpp_id);
96 	if (rel_addr < (3 << 20))
97 		return nfp6000_nbi_tm(cpp_id);
98 	return nfp6000_nbi_ppc(cpp_id);
99 }
100 
101 /* This structure ONLY includes items that can be done with a read or write of
102  * 32-bit or 64-bit words. All others are not listed.
103  */
nfp6000_mu_common(u32 cpp_id)104 static int nfp6000_mu_common(u32 cpp_id)
105 {
106 	switch (cpp_id & NFP_CPP_ID(0, ~0, ~0)) {
107 	AT(NFP_CPP_ACTION_RW, 0, P64, P64);	/* read_be/write_be */
108 	AT(NFP_CPP_ACTION_RW, 1, P64, P64);	/* read_le/write_le */
109 	AT(NFP_CPP_ACTION_RW, 2, P64, P64);	/* read_swap_be/write_swap_be */
110 	AT(NFP_CPP_ACTION_RW, 3, P64, P64);	/* read_swap_le/write_swap_le */
111 	AT(0, 0,   0, P64);	/* read_be */
112 	AT(0, 1,   0, P64);	/* read_le */
113 	AT(0, 2,   0, P64);	/* read_swap_be */
114 	AT(0, 3,   0, P64);	/* read_swap_le */
115 	AT(1, 0, P64,   0);	/* write_be */
116 	AT(1, 1, P64,   0);	/* write_le */
117 	AT(1, 2, P64,   0);	/* write_swap_be */
118 	AT(1, 3, P64,   0);	/* write_swap_le */
119 	AT(3, 0,   0, P32);	/* atomic_read */
120 	AT(3, 2, P32,   0);	/* mask_compare_write */
121 	AT(4, 0, P32,   0);	/* atomic_write */
122 	AT(4, 2,   0,   0);	/* atomic_write_imm */
123 	AT(4, 3,   0, P32);	/* swap_imm */
124 	AT(5, 0, P32,   0);	/* set */
125 	AT(5, 3,   0, P32);	/* test_set_imm */
126 	AT(6, 0, P32,   0);	/* clr */
127 	AT(6, 3,   0, P32);	/* test_clr_imm */
128 	AT(7, 0, P32,   0);	/* add */
129 	AT(7, 3,   0, P32);	/* test_add_imm */
130 	AT(8, 0, P32,   0);	/* addsat */
131 	AT(8, 3,   0, P32);	/* test_subsat_imm */
132 	AT(9, 0, P32,   0);	/* sub */
133 	AT(9, 3,   0, P32);	/* test_sub_imm */
134 	AT(10, 0, P32,   0);	/* subsat */
135 	AT(10, 3,   0, P32);	/* test_subsat_imm */
136 	AT(13, 0,   0, P32);	/* microq128_get */
137 	AT(13, 1,   0, P32);	/* microq128_pop */
138 	AT(13, 2, P32,   0);	/* microq128_put */
139 	AT(15, 0, P32,   0);	/* xor */
140 	AT(15, 3,   0, P32);	/* test_xor_imm */
141 	AT(28, 0,   0, P32);	/* read32_be */
142 	AT(28, 1,   0, P32);	/* read32_le */
143 	AT(28, 2,   0, P32);	/* read32_swap_be */
144 	AT(28, 3,   0, P32);	/* read32_swap_le */
145 	AT(31, 0, P32,   0);	/* write32_be */
146 	AT(31, 1, P32,   0);	/* write32_le */
147 	AT(31, 2, P32,   0);	/* write32_swap_be */
148 	AT(31, 3, P32,   0);	/* write32_swap_le */
149 	default:
150 		return -EINVAL;
151 	}
152 }
153 
nfp6000_mu_ctm(u32 cpp_id)154 static int nfp6000_mu_ctm(u32 cpp_id)
155 {
156 	switch (cpp_id & NFP_CPP_ID(0, ~0, ~0)) {
157 	AT(16, 1,   0, P32);	/* packet_read_packet_status */
158 	AT(17, 1,   0, P32);	/* packet_credit_get */
159 	AT(17, 3,   0, P64);	/* packet_add_thread */
160 	AT(18, 2,   0, P64);	/* packet_free_and_return_pointer */
161 	AT(18, 3,   0, P64);	/* packet_return_pointer */
162 	AT(21, 0,   0, P64);	/* pe_dma_to_memory_indirect */
163 	AT(21, 1,   0, P64);	/* pe_dma_to_memory_indirect_swap */
164 	AT(21, 2,   0, P64);	/* pe_dma_to_memory_indirect_free */
165 	AT(21, 3,   0, P64);	/* pe_dma_to_memory_indirect_free_swap */
166 	default:
167 		return nfp6000_mu_common(cpp_id);
168 	}
169 }
170 
nfp6000_mu_emu(u32 cpp_id)171 static int nfp6000_mu_emu(u32 cpp_id)
172 {
173 	switch (cpp_id & NFP_CPP_ID(0, ~0, ~0)) {
174 	AT(18, 0,   0, P32);	/* read_queue */
175 	AT(18, 1,   0, P32);	/* read_queue_ring */
176 	AT(18, 2, P32,   0);	/* write_queue */
177 	AT(18, 3, P32,   0);	/* write_queue_ring */
178 	AT(20, 2, P32,   0);	/* journal */
179 	AT(21, 0,   0, P32);	/* get */
180 	AT(21, 1,   0, P32);	/* get_eop */
181 	AT(21, 2,   0, P32);	/* get_freely */
182 	AT(22, 0,   0, P32);	/* pop */
183 	AT(22, 1,   0, P32);	/* pop_eop */
184 	AT(22, 2,   0, P32);	/* pop_freely */
185 	default:
186 		return nfp6000_mu_common(cpp_id);
187 	}
188 }
189 
nfp6000_mu_imu(u32 cpp_id)190 static int nfp6000_mu_imu(u32 cpp_id)
191 {
192 	return nfp6000_mu_common(cpp_id);
193 }
194 
nfp6000_mu(u32 cpp_id,u64 address)195 static int nfp6000_mu(u32 cpp_id, u64 address)
196 {
197 	int pp;
198 
199 	if (address < 0x2000000000ULL)
200 		pp = nfp6000_mu_ctm(cpp_id);
201 	else if (address < 0x8000000000ULL)
202 		pp = nfp6000_mu_emu(cpp_id);
203 	else if (address < 0x9800000000ULL)
204 		pp = nfp6000_mu_ctm(cpp_id);
205 	else if (address < 0x9C00000000ULL)
206 		pp = nfp6000_mu_emu(cpp_id);
207 	else if (address < 0xA000000000ULL)
208 		pp = nfp6000_mu_imu(cpp_id);
209 	else
210 		pp = nfp6000_mu_ctm(cpp_id);
211 
212 	return pp;
213 }
214 
nfp6000_ila(u32 cpp_id)215 static int nfp6000_ila(u32 cpp_id)
216 {
217 	switch (cpp_id & NFP_CPP_ID(0, ~0, ~0)) {
218 	AT(0, 1,   0, P32);	/* read_check_error */
219 	AT(2, 0,   0, P32);	/* read_int */
220 	AT(3, 0, P32,   0);	/* write_int */
221 	default:
222 		return target_rw(cpp_id, P32, 48, 4);
223 	}
224 }
225 
nfp6000_pci(u32 cpp_id)226 static int nfp6000_pci(u32 cpp_id)
227 {
228 	switch (cpp_id & NFP_CPP_ID(0, ~0, ~0)) {
229 	AT(2, 0,   0, P32);
230 	AT(3, 0, P32,   0);
231 	default:
232 		return target_rw(cpp_id, P32, 4, 4);
233 	}
234 }
235 
nfp6000_crypto(u32 cpp_id)236 static int nfp6000_crypto(u32 cpp_id)
237 {
238 	switch (cpp_id & NFP_CPP_ID(0, ~0, ~0)) {
239 	AT(2, 0, P64,   0);
240 	default:
241 		return target_rw(cpp_id, P64, 12, 4);
242 	}
243 }
244 
nfp6000_cap_xpb(u32 cpp_id)245 static int nfp6000_cap_xpb(u32 cpp_id)
246 {
247 	switch (cpp_id & NFP_CPP_ID(0, ~0, ~0)) {
248 	AT(0, 1,   0, P32); /* RingGet */
249 	AT(0, 2, P32,   0); /* Interthread Signal */
250 	AT(1, 1, P32,   0); /* RingPut */
251 	AT(1, 2, P32,   0); /* CTNNWr */
252 	AT(2, 0,   0, P32); /* ReflectRd, signal none */
253 	AT(2, 1,   0, P32); /* ReflectRd, signal self */
254 	AT(2, 2,   0, P32); /* ReflectRd, signal remote */
255 	AT(2, 3,   0, P32); /* ReflectRd, signal both */
256 	AT(3, 0, P32,   0); /* ReflectWr, signal none */
257 	AT(3, 1, P32,   0); /* ReflectWr, signal self */
258 	AT(3, 2, P32,   0); /* ReflectWr, signal remote */
259 	AT(3, 3, P32,   0); /* ReflectWr, signal both */
260 	AT(NFP_CPP_ACTION_RW, 1, P32, P32);
261 	default:
262 		return target_rw(cpp_id, P32, 1, 63);
263 	}
264 }
265 
nfp6000_cls(u32 cpp_id)266 static int nfp6000_cls(u32 cpp_id)
267 {
268 	switch (cpp_id & NFP_CPP_ID(0, ~0, ~0)) {
269 	AT(0, 3, P32,  0); /* xor */
270 	AT(2, 0, P32,  0); /* set */
271 	AT(2, 1, P32,  0); /* clr */
272 	AT(4, 0, P32,  0); /* add */
273 	AT(4, 1, P32,  0); /* add64 */
274 	AT(6, 0, P32,  0); /* sub */
275 	AT(6, 1, P32,  0); /* sub64 */
276 	AT(6, 2, P32,  0); /* subsat */
277 	AT(8, 2, P32,  0); /* hash_mask */
278 	AT(8, 3, P32,  0); /* hash_clear */
279 	AT(9, 0,  0, P32); /* ring_get */
280 	AT(9, 1,  0, P32); /* ring_pop */
281 	AT(9, 2,  0, P32); /* ring_get_freely */
282 	AT(9, 3,  0, P32); /* ring_pop_freely */
283 	AT(10, 0, P32,  0); /* ring_put */
284 	AT(10, 2, P32,  0); /* ring_journal */
285 	AT(14, 0,  P32, 0); /* reflect_write_sig_local */
286 	AT(15, 1,  0, P32); /* reflect_read_sig_local */
287 	AT(17, 2, P32,  0); /* statisic */
288 	AT(24, 0,  0, P32); /* ring_read */
289 	AT(24, 1, P32,  0); /* ring_write */
290 	AT(25, 0,  0, P32); /* ring_workq_add_thread */
291 	AT(25, 1, P32,  0); /* ring_workq_add_work */
292 	default:
293 		return target_rw(cpp_id, P32, 0, 64);
294 	}
295 }
296 
nfp_target_pushpull(u32 cpp_id,u64 address)297 int nfp_target_pushpull(u32 cpp_id, u64 address)
298 {
299 	switch (NFP_CPP_ID_TARGET_of(cpp_id)) {
300 	case NFP_CPP_TARGET_NBI:
301 		return nfp6000_nbi(cpp_id, address);
302 	case NFP_CPP_TARGET_QDR:
303 		return target_rw(cpp_id, P32, 24, 4);
304 	case NFP_CPP_TARGET_ILA:
305 		return nfp6000_ila(cpp_id);
306 	case NFP_CPP_TARGET_MU:
307 		return nfp6000_mu(cpp_id, address);
308 	case NFP_CPP_TARGET_PCIE:
309 		return nfp6000_pci(cpp_id);
310 	case NFP_CPP_TARGET_ARM:
311 		if (address < 0x10000)
312 			return target_rw(cpp_id, P64, 1, 1);
313 		else
314 			return target_rw(cpp_id, P32, 1, 1);
315 	case NFP_CPP_TARGET_CRYPTO:
316 		return nfp6000_crypto(cpp_id);
317 	case NFP_CPP_TARGET_CT_XPB:
318 		return nfp6000_cap_xpb(cpp_id);
319 	case NFP_CPP_TARGET_CLS:
320 		return nfp6000_cls(cpp_id);
321 	case 0:
322 		return target_rw(cpp_id, P32, 4, 4);
323 	default:
324 		return -EINVAL;
325 	}
326 }
327 
328 #undef AT
329 #undef P32
330 #undef P64
331 
332 /* All magic NFP-6xxx IMB 'mode' numbers here are from:
333  * Databook (1 August 2013)
334  * - System Overview and Connectivity
335  * -- Internal Connectivity
336  * --- Distributed Switch Fabric - Command Push/Pull (DSF-CPP) Bus
337  * ---- CPP addressing
338  * ----- Table 3.6. CPP Address Translation Mode Commands
339  */
340 
341 #define _NIC_NFP6000_MU_LOCALITY_DIRECT     2
342 
nfp_decode_basic(u64 addr,int * dest_island,int cpp_tgt,int mode,bool addr40,int isld1,int isld0)343 static int nfp_decode_basic(u64 addr, int *dest_island, int cpp_tgt,
344 			    int mode, bool addr40, int isld1, int isld0)
345 {
346 	int iid_lsb, idx_lsb;
347 
348 	/* This function doesn't handle MU or CTXBP */
349 	if (cpp_tgt == NFP_CPP_TARGET_MU || cpp_tgt == NFP_CPP_TARGET_CT_XPB)
350 		return -EINVAL;
351 
352 	switch (mode) {
353 	case 0:
354 		/* For VQDR, in this mode for 32-bit addressing
355 		 * it would be islands 0, 16, 32 and 48 depending on channel
356 		 * and upper address bits.
357 		 * Since those are not all valid islands, most decode
358 		 * cases would result in bad island IDs, but we do them
359 		 * anyway since this is decoding an address that is already
360 		 * assumed to be used as-is to get to sram.
361 		 */
362 		iid_lsb = addr40 ? 34 : 26;
363 		*dest_island = (addr >> iid_lsb) & 0x3F;
364 		return 0;
365 	case 1:
366 		/* For VQDR 32-bit, this would decode as:
367 		 * Channel 0: island#0
368 		 * Channel 1: island#0
369 		 * Channel 2: island#1
370 		 * Channel 3: island#1
371 		 * That would be valid as long as both islands
372 		 * have VQDR. Let's allow this.
373 		 */
374 		idx_lsb = addr40 ? 39 : 31;
375 		if (addr & BIT_ULL(idx_lsb))
376 			*dest_island = isld1;
377 		else
378 			*dest_island = isld0;
379 
380 		return 0;
381 	case 2:
382 		/* For VQDR 32-bit:
383 		 * Channel 0: (island#0 | 0)
384 		 * Channel 1: (island#0 | 1)
385 		 * Channel 2: (island#1 | 0)
386 		 * Channel 3: (island#1 | 1)
387 		 *
388 		 * Make sure we compare against isldN values
389 		 * by clearing the LSB.
390 		 * This is what the silicon does.
391 		 */
392 		isld0 &= ~1;
393 		isld1 &= ~1;
394 
395 		idx_lsb = addr40 ? 39 : 31;
396 		iid_lsb = idx_lsb - 1;
397 
398 		if (addr & BIT_ULL(idx_lsb))
399 			*dest_island = isld1 | (int)((addr >> iid_lsb) & 1);
400 		else
401 			*dest_island = isld0 | (int)((addr >> iid_lsb) & 1);
402 
403 		return 0;
404 	case 3:
405 		/* In this mode the data address starts to affect the island ID
406 		 * so rather not allow it. In some really specific case
407 		 * one could use this to send the upper half of the
408 		 * VQDR channel to another MU, but this is getting very
409 		 * specific.
410 		 * However, as above for mode 0, this is the decoder
411 		 * and the caller should validate the resulting IID.
412 		 * This blindly does what the silicon would do.
413 		 */
414 		isld0 &= ~3;
415 		isld1 &= ~3;
416 
417 		idx_lsb = addr40 ? 39 : 31;
418 		iid_lsb = idx_lsb - 2;
419 
420 		if (addr & BIT_ULL(idx_lsb))
421 			*dest_island = isld1 | (int)((addr >> iid_lsb) & 3);
422 		else
423 			*dest_island = isld0 | (int)((addr >> iid_lsb) & 3);
424 
425 		return 0;
426 	default:
427 		return -EINVAL;
428 	}
429 }
430 
nfp_encode_basic_qdr(u64 addr,int dest_island,int cpp_tgt,int mode,bool addr40,int isld1,int isld0)431 static int nfp_encode_basic_qdr(u64 addr, int dest_island, int cpp_tgt,
432 				int mode, bool addr40, int isld1, int isld0)
433 {
434 	int v, ret;
435 
436 	/* Full Island ID and channel bits overlap? */
437 	ret = nfp_decode_basic(addr, &v, cpp_tgt, mode, addr40, isld1, isld0);
438 	if (ret)
439 		return ret;
440 
441 	/* The current address won't go where expected? */
442 	if (dest_island != -1 && dest_island != v)
443 		return -EINVAL;
444 
445 	/* If dest_island was -1, we don't care where it goes. */
446 	return 0;
447 }
448 
449 /* Try each option, take first one that fits.
450  * Not sure if we would want to do some smarter
451  * searching and prefer 0 or non-0 island IDs.
452  */
nfp_encode_basic_search(u64 * addr,int dest_island,int * isld,int iid_lsb,int idx_lsb,int v_max)453 static int nfp_encode_basic_search(u64 *addr, int dest_island, int *isld,
454 				   int iid_lsb, int idx_lsb, int v_max)
455 {
456 	int i, v;
457 
458 	for (i = 0; i < 2; i++)
459 		for (v = 0; v < v_max; v++) {
460 			if (dest_island != (isld[i] | v))
461 				continue;
462 
463 			*addr &= ~GENMASK_ULL(idx_lsb, iid_lsb);
464 			*addr |= ((u64)i << idx_lsb);
465 			*addr |= ((u64)v << iid_lsb);
466 			return 0;
467 		}
468 
469 	return -ENODEV;
470 }
471 
472 /* For VQDR, we may not modify the Channel bits, which might overlap
473  *  with the Index bit. When it does, we need to ensure that isld0 == isld1.
474  */
nfp_encode_basic(u64 * addr,int dest_island,int cpp_tgt,int mode,bool addr40,int isld1,int isld0)475 static int nfp_encode_basic(u64 *addr, int dest_island, int cpp_tgt,
476 			    int mode, bool addr40, int isld1, int isld0)
477 {
478 	int iid_lsb, idx_lsb;
479 	int isld[2];
480 	u64 v64;
481 
482 	isld[0] = isld0;
483 	isld[1] = isld1;
484 
485 	/* This function doesn't handle MU or CTXBP */
486 	if (cpp_tgt == NFP_CPP_TARGET_MU || cpp_tgt == NFP_CPP_TARGET_CT_XPB)
487 		return -EINVAL;
488 
489 	switch (mode) {
490 	case 0:
491 		if (cpp_tgt == NFP_CPP_TARGET_QDR && !addr40)
492 			/* In this specific mode we'd rather not modify
493 			 * the address but we can verify if the existing
494 			 * contents will point to a valid island.
495 			 */
496 			return nfp_encode_basic_qdr(*addr, cpp_tgt, dest_island,
497 						    mode, addr40, isld1, isld0);
498 
499 		iid_lsb = addr40 ? 34 : 26;
500 		/* <39:34> or <31:26> */
501 		v64 = GENMASK_ULL(iid_lsb + 5, iid_lsb);
502 		*addr &= ~v64;
503 		*addr |= ((u64)dest_island << iid_lsb) & v64;
504 		return 0;
505 	case 1:
506 		if (cpp_tgt == NFP_CPP_TARGET_QDR && !addr40)
507 			return nfp_encode_basic_qdr(*addr, cpp_tgt, dest_island,
508 						    mode, addr40, isld1, isld0);
509 
510 		idx_lsb = addr40 ? 39 : 31;
511 		if (dest_island == isld0) {
512 			/* Only need to clear the Index bit */
513 			*addr &= ~BIT_ULL(idx_lsb);
514 			return 0;
515 		}
516 
517 		if (dest_island == isld1) {
518 			/* Only need to set the Index bit */
519 			*addr |= BIT_ULL(idx_lsb);
520 			return 0;
521 		}
522 
523 		return -ENODEV;
524 	case 2:
525 		/* iid<0> = addr<30> = channel<0>
526 		 * channel<1> = addr<31> = Index
527 		 */
528 		if (cpp_tgt == NFP_CPP_TARGET_QDR && !addr40)
529 			/* Special case where we allow channel bits to
530 			 * be set before hand and with them select an island.
531 			 * So we need to confirm that it's at least plausible.
532 			 */
533 			return nfp_encode_basic_qdr(*addr, cpp_tgt, dest_island,
534 						    mode, addr40, isld1, isld0);
535 
536 		/* Make sure we compare against isldN values
537 		 * by clearing the LSB.
538 		 * This is what the silicon does.
539 		 */
540 		isld[0] &= ~1;
541 		isld[1] &= ~1;
542 
543 		idx_lsb = addr40 ? 39 : 31;
544 		iid_lsb = idx_lsb - 1;
545 
546 		return nfp_encode_basic_search(addr, dest_island, isld,
547 					       iid_lsb, idx_lsb, 2);
548 	case 3:
549 		if (cpp_tgt == NFP_CPP_TARGET_QDR && !addr40)
550 			/* iid<0> = addr<29> = data
551 			 * iid<1> = addr<30> = channel<0>
552 			 * channel<1> = addr<31> = Index
553 			 */
554 			return nfp_encode_basic_qdr(*addr, cpp_tgt, dest_island,
555 						    mode, addr40, isld1, isld0);
556 
557 		isld[0] &= ~3;
558 		isld[1] &= ~3;
559 
560 		idx_lsb = addr40 ? 39 : 31;
561 		iid_lsb = idx_lsb - 2;
562 
563 		return nfp_encode_basic_search(addr, dest_island, isld,
564 					       iid_lsb, idx_lsb, 4);
565 	default:
566 		return -EINVAL;
567 	}
568 }
569 
nfp_encode_mu(u64 * addr,int dest_island,int mode,bool addr40,int isld1,int isld0)570 static int nfp_encode_mu(u64 *addr, int dest_island, int mode,
571 			 bool addr40, int isld1, int isld0)
572 {
573 	int iid_lsb, idx_lsb, locality_lsb;
574 	int isld[2];
575 	u64 v64;
576 	int da;
577 
578 	isld[0] = isld0;
579 	isld[1] = isld1;
580 	locality_lsb = nfp_cppat_mu_locality_lsb(mode, addr40);
581 
582 	if (((*addr >> locality_lsb) & 3) == _NIC_NFP6000_MU_LOCALITY_DIRECT)
583 		da = 1;
584 	else
585 		da = 0;
586 
587 	switch (mode) {
588 	case 0:
589 		iid_lsb = addr40 ? 32 : 24;
590 		v64 = GENMASK_ULL(iid_lsb + 5, iid_lsb);
591 		*addr &= ~v64;
592 		*addr |= (((u64)dest_island) << iid_lsb) & v64;
593 		return 0;
594 	case 1:
595 		if (da) {
596 			iid_lsb = addr40 ? 32 : 24;
597 			v64 = GENMASK_ULL(iid_lsb + 5, iid_lsb);
598 			*addr &= ~v64;
599 			*addr |= (((u64)dest_island) << iid_lsb) & v64;
600 			return 0;
601 		}
602 
603 		idx_lsb = addr40 ? 37 : 29;
604 		if (dest_island == isld0) {
605 			*addr &= ~BIT_ULL(idx_lsb);
606 			return 0;
607 		}
608 
609 		if (dest_island == isld1) {
610 			*addr |= BIT_ULL(idx_lsb);
611 			return 0;
612 		}
613 
614 		return -ENODEV;
615 	case 2:
616 		if (da) {
617 			iid_lsb = addr40 ? 32 : 24;
618 			v64 = GENMASK_ULL(iid_lsb + 5, iid_lsb);
619 			*addr &= ~v64;
620 			*addr |= (((u64)dest_island) << iid_lsb) & v64;
621 			return 0;
622 		}
623 
624 		/* Make sure we compare against isldN values
625 		 * by clearing the LSB.
626 		 * This is what the silicon does.
627 		 */
628 		isld[0] &= ~1;
629 		isld[1] &= ~1;
630 
631 		idx_lsb = addr40 ? 37 : 29;
632 		iid_lsb = idx_lsb - 1;
633 
634 		return nfp_encode_basic_search(addr, dest_island, isld,
635 					       iid_lsb, idx_lsb, 2);
636 	case 3:
637 		/* Only the EMU will use 40 bit addressing. Silently
638 		 * set the direct locality bit for everyone else.
639 		 * The SDK toolchain uses dest_island <= 0 to test
640 		 * for atypical address encodings to support access
641 		 * to local-island CTM with a 32-but address (high-locality
642 		 * is effewctively ignored and just used for
643 		 * routing to island #0).
644 		 */
645 		if (dest_island > 0 && (dest_island < 24 || dest_island > 26)) {
646 			*addr |= ((u64)_NIC_NFP6000_MU_LOCALITY_DIRECT)
647 							<< locality_lsb;
648 			da = 1;
649 		}
650 
651 		if (da) {
652 			iid_lsb = addr40 ? 32 : 24;
653 			v64 = GENMASK_ULL(iid_lsb + 5, iid_lsb);
654 			*addr &= ~v64;
655 			*addr |= (((u64)dest_island) << iid_lsb) & v64;
656 			return 0;
657 		}
658 
659 		isld[0] &= ~3;
660 		isld[1] &= ~3;
661 
662 		idx_lsb = addr40 ? 37 : 29;
663 		iid_lsb = idx_lsb - 2;
664 
665 		return nfp_encode_basic_search(addr, dest_island, isld,
666 					       iid_lsb, idx_lsb, 4);
667 	default:
668 		return -EINVAL;
669 	}
670 }
671 
nfp_cppat_addr_encode(u64 * addr,int dest_island,int cpp_tgt,int mode,bool addr40,int isld1,int isld0)672 static int nfp_cppat_addr_encode(u64 *addr, int dest_island, int cpp_tgt,
673 				 int mode, bool addr40, int isld1, int isld0)
674 {
675 	switch (cpp_tgt) {
676 	case NFP_CPP_TARGET_NBI:
677 	case NFP_CPP_TARGET_QDR:
678 	case NFP_CPP_TARGET_ILA:
679 	case NFP_CPP_TARGET_PCIE:
680 	case NFP_CPP_TARGET_ARM:
681 	case NFP_CPP_TARGET_CRYPTO:
682 	case NFP_CPP_TARGET_CLS:
683 		return nfp_encode_basic(addr, dest_island, cpp_tgt, mode,
684 					addr40, isld1, isld0);
685 
686 	case NFP_CPP_TARGET_MU:
687 		return nfp_encode_mu(addr, dest_island, mode,
688 				     addr40, isld1, isld0);
689 
690 	case NFP_CPP_TARGET_CT_XPB:
691 		if (mode != 1 || addr40)
692 			return -EINVAL;
693 		*addr &= ~GENMASK_ULL(29, 24);
694 		*addr |= ((u64)dest_island << 24) & GENMASK_ULL(29, 24);
695 		return 0;
696 	default:
697 		return -EINVAL;
698 	}
699 }
700 
nfp_target_cpp(u32 cpp_island_id,u64 cpp_island_address,u32 * cpp_target_id,u64 * cpp_target_address,const u32 * imb_table)701 int nfp_target_cpp(u32 cpp_island_id, u64 cpp_island_address,
702 		   u32 *cpp_target_id, u64 *cpp_target_address,
703 		   const u32 *imb_table)
704 {
705 	const int island = NFP_CPP_ID_ISLAND_of(cpp_island_id);
706 	const int target = NFP_CPP_ID_TARGET_of(cpp_island_id);
707 	u32 imb;
708 	int err;
709 
710 	if (target < 0 || target >= 16) {
711 		pr_err("Invalid CPP target: %d\n", target);
712 		return -EINVAL;
713 	}
714 
715 	if (island == 0) {
716 		/* Already translated */
717 		*cpp_target_id = cpp_island_id;
718 		*cpp_target_address = cpp_island_address;
719 		return 0;
720 	}
721 
722 	/* CPP + Island only allowed on systems with IMB tables */
723 	if (!imb_table)
724 		return -EINVAL;
725 
726 	imb = imb_table[target];
727 
728 	*cpp_target_address = cpp_island_address;
729 	err = nfp_cppat_addr_encode(cpp_target_address, island, target,
730 				    ((imb >> 13) & 7), ((imb >> 12) & 1),
731 				    ((imb >> 6)  & 0x3f), ((imb >> 0)  & 0x3f));
732 	if (err) {
733 		pr_err("Can't encode CPP address: %d\n", err);
734 		return err;
735 	}
736 
737 	*cpp_target_id = NFP_CPP_ID(target,
738 				    NFP_CPP_ID_ACTION_of(cpp_island_id),
739 				    NFP_CPP_ID_TOKEN_of(cpp_island_id));
740 
741 	return 0;
742 }
743