1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
2 /* Copyright (C) 2015-2018 Netronome Systems, Inc. */
3 
4 /*
5  * nfp6000_pcie.c
6  * Authors: Jakub Kicinski <jakub.kicinski@netronome.com>
7  *          Jason McMullan <jason.mcmullan@netronome.com>
8  *          Rolf Neugebauer <rolf.neugebauer@netronome.com>
9  *
10  * Multiplexes the NFP BARs between NFP internal resources and
11  * implements the PCIe specific interface for generic CPP bus access.
12  *
13  * The BARs are managed with refcounts and are allocated/acquired
14  * using target, token and offset/size matching.  The generic CPP bus
15  * abstraction builds upon this BAR interface.
16  */
17 
18 #include <asm/unaligned.h>
19 #include <linux/kernel.h>
20 #include <linux/module.h>
21 #include <linux/kref.h>
22 #include <linux/io.h>
23 #include <linux/delay.h>
24 #include <linux/interrupt.h>
25 #include <linux/sort.h>
26 #include <linux/sched.h>
27 #include <linux/types.h>
28 #include <linux/pci.h>
29 
30 #include "nfp_cpp.h"
31 #include "nfp_dev.h"
32 
33 #include "nfp6000/nfp6000.h"
34 
35 #include "nfp6000_pcie.h"
36 
37 #define NFP_PCIE_BAR(_pf)	(0x30000 + ((_pf) & 7) * 0xc0)
38 #define NFP_PCIE_BAR_EXPLICIT_BAR0(_x, _y) \
39 	(0x00000080 + (0x40 * ((_x) & 0x3)) + (0x10 * ((_y) & 0x3)))
40 #define   NFP_PCIE_BAR_EXPLICIT_BAR0_SignalType(_x)     (((_x) & 0x3) << 30)
41 #define   NFP_PCIE_BAR_EXPLICIT_BAR0_SignalType_of(_x)  (((_x) >> 30) & 0x3)
42 #define   NFP_PCIE_BAR_EXPLICIT_BAR0_Token(_x)          (((_x) & 0x3) << 28)
43 #define   NFP_PCIE_BAR_EXPLICIT_BAR0_Token_of(_x)       (((_x) >> 28) & 0x3)
44 #define   NFP_PCIE_BAR_EXPLICIT_BAR0_Address(_x)        (((_x) & 0xffffff) << 0)
45 #define   NFP_PCIE_BAR_EXPLICIT_BAR0_Address_of(_x)     (((_x) >> 0) & 0xffffff)
46 #define NFP_PCIE_BAR_EXPLICIT_BAR1(_x, _y) \
47 	(0x00000084 + (0x40 * ((_x) & 0x3)) + (0x10 * ((_y) & 0x3)))
48 #define   NFP_PCIE_BAR_EXPLICIT_BAR1_SignalRef(_x)      (((_x) & 0x7f) << 24)
49 #define   NFP_PCIE_BAR_EXPLICIT_BAR1_SignalRef_of(_x)   (((_x) >> 24) & 0x7f)
50 #define   NFP_PCIE_BAR_EXPLICIT_BAR1_DataMaster(_x)     (((_x) & 0x3ff) << 14)
51 #define   NFP_PCIE_BAR_EXPLICIT_BAR1_DataMaster_of(_x)  (((_x) >> 14) & 0x3ff)
52 #define   NFP_PCIE_BAR_EXPLICIT_BAR1_DataRef(_x)        (((_x) & 0x3fff) << 0)
53 #define   NFP_PCIE_BAR_EXPLICIT_BAR1_DataRef_of(_x)     (((_x) >> 0) & 0x3fff)
54 #define NFP_PCIE_BAR_EXPLICIT_BAR2(_x, _y) \
55 	(0x00000088 + (0x40 * ((_x) & 0x3)) + (0x10 * ((_y) & 0x3)))
56 #define   NFP_PCIE_BAR_EXPLICIT_BAR2_Target(_x)         (((_x) & 0xf) << 28)
57 #define   NFP_PCIE_BAR_EXPLICIT_BAR2_Target_of(_x)      (((_x) >> 28) & 0xf)
58 #define   NFP_PCIE_BAR_EXPLICIT_BAR2_Action(_x)         (((_x) & 0x1f) << 23)
59 #define   NFP_PCIE_BAR_EXPLICIT_BAR2_Action_of(_x)      (((_x) >> 23) & 0x1f)
60 #define   NFP_PCIE_BAR_EXPLICIT_BAR2_Length(_x)         (((_x) & 0x1f) << 18)
61 #define   NFP_PCIE_BAR_EXPLICIT_BAR2_Length_of(_x)      (((_x) >> 18) & 0x1f)
62 #define   NFP_PCIE_BAR_EXPLICIT_BAR2_ByteMask(_x)       (((_x) & 0xff) << 10)
63 #define   NFP_PCIE_BAR_EXPLICIT_BAR2_ByteMask_of(_x)    (((_x) >> 10) & 0xff)
64 #define   NFP_PCIE_BAR_EXPLICIT_BAR2_SignalMaster(_x)   (((_x) & 0x3ff) << 0)
65 #define   NFP_PCIE_BAR_EXPLICIT_BAR2_SignalMaster_of(_x) (((_x) >> 0) & 0x3ff)
66 
67 #define   NFP_PCIE_BAR_PCIE2CPP_Action_BaseAddress(_x)  (((_x) & 0x1f) << 16)
68 #define   NFP_PCIE_BAR_PCIE2CPP_Action_BaseAddress_of(_x) (((_x) >> 16) & 0x1f)
69 #define   NFP_PCIE_BAR_PCIE2CPP_BaseAddress(_x)         (((_x) & 0xffff) << 0)
70 #define   NFP_PCIE_BAR_PCIE2CPP_BaseAddress_of(_x)      (((_x) >> 0) & 0xffff)
71 #define   NFP_PCIE_BAR_PCIE2CPP_LengthSelect(_x)        (((_x) & 0x3) << 27)
72 #define   NFP_PCIE_BAR_PCIE2CPP_LengthSelect_of(_x)     (((_x) >> 27) & 0x3)
73 #define     NFP_PCIE_BAR_PCIE2CPP_LengthSelect_32BIT    0
74 #define     NFP_PCIE_BAR_PCIE2CPP_LengthSelect_64BIT    1
75 #define     NFP_PCIE_BAR_PCIE2CPP_LengthSelect_0BYTE    3
76 #define   NFP_PCIE_BAR_PCIE2CPP_MapType(_x)             (((_x) & 0x7) << 29)
77 #define   NFP_PCIE_BAR_PCIE2CPP_MapType_of(_x)          (((_x) >> 29) & 0x7)
78 #define     NFP_PCIE_BAR_PCIE2CPP_MapType_FIXED         0
79 #define     NFP_PCIE_BAR_PCIE2CPP_MapType_BULK          1
80 #define     NFP_PCIE_BAR_PCIE2CPP_MapType_TARGET        2
81 #define     NFP_PCIE_BAR_PCIE2CPP_MapType_GENERAL       3
82 #define     NFP_PCIE_BAR_PCIE2CPP_MapType_EXPLICIT0     4
83 #define     NFP_PCIE_BAR_PCIE2CPP_MapType_EXPLICIT1     5
84 #define     NFP_PCIE_BAR_PCIE2CPP_MapType_EXPLICIT2     6
85 #define     NFP_PCIE_BAR_PCIE2CPP_MapType_EXPLICIT3     7
86 #define   NFP_PCIE_BAR_PCIE2CPP_Target_BaseAddress(_x)  (((_x) & 0xf) << 23)
87 #define   NFP_PCIE_BAR_PCIE2CPP_Target_BaseAddress_of(_x) (((_x) >> 23) & 0xf)
88 #define   NFP_PCIE_BAR_PCIE2CPP_Token_BaseAddress(_x)   (((_x) & 0x3) << 21)
89 #define   NFP_PCIE_BAR_PCIE2CPP_Token_BaseAddress_of(_x) (((_x) >> 21) & 0x3)
90 #define NFP_PCIE_EM                                     0x020000
91 #define NFP_PCIE_SRAM                                   0x000000
92 
93 /* Minimal size of the PCIe cfg memory we depend on being mapped,
94  * queue controller and DMA controller don't have to be covered.
95  */
96 #define NFP_PCI_MIN_MAP_SIZE				0x080000
97 
98 #define NFP_PCIE_P2C_FIXED_SIZE(bar)               (1 << (bar)->bitsize)
99 #define NFP_PCIE_P2C_BULK_SIZE(bar)                (1 << (bar)->bitsize)
100 #define NFP_PCIE_P2C_GENERAL_TARGET_OFFSET(bar, x) ((x) << ((bar)->bitsize - 2))
101 #define NFP_PCIE_P2C_GENERAL_TOKEN_OFFSET(bar, x) ((x) << ((bar)->bitsize - 4))
102 #define NFP_PCIE_P2C_GENERAL_SIZE(bar)             (1 << ((bar)->bitsize - 4))
103 
104 #define NFP_PCIE_P2C_EXPBAR_OFFSET(bar_index)		((bar_index) * 4)
105 
106 /* The number of explicit BARs to reserve.
107  * Minimum is 0, maximum is 4 on the NFP6000.
108  * The NFP3800 can have only one per PF.
109  */
110 #define NFP_PCIE_EXPLICIT_BARS		2
111 
112 struct nfp6000_pcie;
113 struct nfp6000_area_priv;
114 
115 /**
116  * struct nfp_bar - describes BAR configuration and usage
117  * @nfp:	backlink to owner
118  * @barcfg:	cached contents of BAR config CSR
119  * @base:	the BAR's base CPP offset
120  * @mask:       mask for the BAR aperture (read only)
121  * @bitsize:	bitsize of BAR aperture (read only)
122  * @index:	index of the BAR
123  * @refcnt:	number of current users
124  * @iomem:	mapped IO memory
125  * @resource:	iomem resource window
126  */
127 struct nfp_bar {
128 	struct nfp6000_pcie *nfp;
129 	u32 barcfg;
130 	u64 base;          /* CPP address base */
131 	u64 mask;          /* Bit mask of the bar */
132 	u32 bitsize;       /* Bit size of the bar */
133 	int index;
134 	atomic_t refcnt;
135 
136 	void __iomem *iomem;
137 	struct resource *resource;
138 };
139 
140 #define NFP_PCI_BAR_MAX    (PCI_64BIT_BAR_COUNT * 8)
141 
142 struct nfp6000_pcie {
143 	struct pci_dev *pdev;
144 	struct device *dev;
145 	const struct nfp_dev_info *dev_info;
146 
147 	/* PCI BAR management */
148 	spinlock_t bar_lock;		/* Protect the PCI2CPP BAR cache */
149 	int bars;
150 	struct nfp_bar bar[NFP_PCI_BAR_MAX];
151 	wait_queue_head_t bar_waiters;
152 
153 	/* Reserved BAR access */
154 	struct {
155 		void __iomem *csr;
156 		void __iomem *em;
157 		void __iomem *expl[4];
158 	} iomem;
159 
160 	/* Explicit IO access */
161 	struct {
162 		struct mutex mutex; /* Lock access to this explicit group */
163 		u8 master_id;
164 		u8 signal_ref;
165 		void __iomem *data;
166 		struct {
167 			void __iomem *addr;
168 			int bitsize;
169 			int free[4];
170 		} group[4];
171 	} expl;
172 };
173 
nfp_bar_maptype(struct nfp_bar * bar)174 static u32 nfp_bar_maptype(struct nfp_bar *bar)
175 {
176 	return NFP_PCIE_BAR_PCIE2CPP_MapType_of(bar->barcfg);
177 }
178 
nfp_bar_resource_len(struct nfp_bar * bar)179 static resource_size_t nfp_bar_resource_len(struct nfp_bar *bar)
180 {
181 	return pci_resource_len(bar->nfp->pdev, (bar->index / 8) * 2) / 8;
182 }
183 
nfp_bar_resource_start(struct nfp_bar * bar)184 static resource_size_t nfp_bar_resource_start(struct nfp_bar *bar)
185 {
186 	return pci_resource_start(bar->nfp->pdev, (bar->index / 8) * 2)
187 		+ nfp_bar_resource_len(bar) * (bar->index & 7);
188 }
189 
190 #define TARGET_WIDTH_32    4
191 #define TARGET_WIDTH_64    8
192 
193 static int
compute_bar(const struct nfp6000_pcie * nfp,const struct nfp_bar * bar,u32 * bar_config,u64 * bar_base,int tgt,int act,int tok,u64 offset,size_t size,int width)194 compute_bar(const struct nfp6000_pcie *nfp, const struct nfp_bar *bar,
195 	    u32 *bar_config, u64 *bar_base,
196 	    int tgt, int act, int tok, u64 offset, size_t size, int width)
197 {
198 	int bitsize;
199 	u32 newcfg;
200 
201 	if (tgt >= NFP_CPP_NUM_TARGETS)
202 		return -EINVAL;
203 
204 	switch (width) {
205 	case 8:
206 		newcfg = NFP_PCIE_BAR_PCIE2CPP_LengthSelect(
207 			NFP_PCIE_BAR_PCIE2CPP_LengthSelect_64BIT);
208 		break;
209 	case 4:
210 		newcfg = NFP_PCIE_BAR_PCIE2CPP_LengthSelect(
211 			NFP_PCIE_BAR_PCIE2CPP_LengthSelect_32BIT);
212 		break;
213 	case 0:
214 		newcfg = NFP_PCIE_BAR_PCIE2CPP_LengthSelect(
215 			NFP_PCIE_BAR_PCIE2CPP_LengthSelect_0BYTE);
216 		break;
217 	default:
218 		return -EINVAL;
219 	}
220 
221 	if (act != NFP_CPP_ACTION_RW && act != 0) {
222 		/* Fixed CPP mapping with specific action */
223 		u64 mask = ~(NFP_PCIE_P2C_FIXED_SIZE(bar) - 1);
224 
225 		newcfg |= NFP_PCIE_BAR_PCIE2CPP_MapType(
226 			  NFP_PCIE_BAR_PCIE2CPP_MapType_FIXED);
227 		newcfg |= NFP_PCIE_BAR_PCIE2CPP_Target_BaseAddress(tgt);
228 		newcfg |= NFP_PCIE_BAR_PCIE2CPP_Action_BaseAddress(act);
229 		newcfg |= NFP_PCIE_BAR_PCIE2CPP_Token_BaseAddress(tok);
230 
231 		if ((offset & mask) != ((offset + size - 1) & mask))
232 			return -EINVAL;
233 		offset &= mask;
234 
235 		bitsize = 40 - 16;
236 	} else {
237 		u64 mask = ~(NFP_PCIE_P2C_BULK_SIZE(bar) - 1);
238 
239 		/* Bulk mapping */
240 		newcfg |= NFP_PCIE_BAR_PCIE2CPP_MapType(
241 			NFP_PCIE_BAR_PCIE2CPP_MapType_BULK);
242 		newcfg |= NFP_PCIE_BAR_PCIE2CPP_Target_BaseAddress(tgt);
243 		newcfg |= NFP_PCIE_BAR_PCIE2CPP_Token_BaseAddress(tok);
244 
245 		if ((offset & mask) != ((offset + size - 1) & mask))
246 			return -EINVAL;
247 
248 		offset &= mask;
249 
250 		bitsize = 40 - 21;
251 	}
252 
253 	if (bar->bitsize < bitsize)
254 		return -EINVAL;
255 
256 	newcfg |= offset >> bitsize;
257 
258 	if (bar_base)
259 		*bar_base = offset;
260 
261 	if (bar_config)
262 		*bar_config = newcfg;
263 
264 	return 0;
265 }
266 
267 static int
nfp6000_bar_write(struct nfp6000_pcie * nfp,struct nfp_bar * bar,u32 newcfg)268 nfp6000_bar_write(struct nfp6000_pcie *nfp, struct nfp_bar *bar, u32 newcfg)
269 {
270 	unsigned int xbar;
271 
272 	xbar = NFP_PCIE_P2C_EXPBAR_OFFSET(bar->index);
273 
274 	if (nfp->iomem.csr) {
275 		writel(newcfg, nfp->iomem.csr + xbar);
276 		/* Readback to ensure BAR is flushed */
277 		readl(nfp->iomem.csr + xbar);
278 	} else {
279 		xbar += nfp->dev_info->pcie_cfg_expbar_offset;
280 		pci_write_config_dword(nfp->pdev, xbar, newcfg);
281 	}
282 
283 	bar->barcfg = newcfg;
284 
285 	return 0;
286 }
287 
288 static int
reconfigure_bar(struct nfp6000_pcie * nfp,struct nfp_bar * bar,int tgt,int act,int tok,u64 offset,size_t size,int width)289 reconfigure_bar(struct nfp6000_pcie *nfp, struct nfp_bar *bar,
290 		int tgt, int act, int tok, u64 offset, size_t size, int width)
291 {
292 	u64 newbase;
293 	u32 newcfg;
294 	int err;
295 
296 	err = compute_bar(nfp, bar, &newcfg, &newbase,
297 			  tgt, act, tok, offset, size, width);
298 	if (err)
299 		return err;
300 
301 	bar->base = newbase;
302 
303 	return nfp6000_bar_write(nfp, bar, newcfg);
304 }
305 
306 /* Check if BAR can be used with the given parameters. */
matching_bar(struct nfp_bar * bar,u32 tgt,u32 act,u32 tok,u64 offset,size_t size,int width)307 static int matching_bar(struct nfp_bar *bar, u32 tgt, u32 act, u32 tok,
308 			u64 offset, size_t size, int width)
309 {
310 	int bartgt, baract, bartok;
311 	int barwidth;
312 	u32 maptype;
313 
314 	maptype = NFP_PCIE_BAR_PCIE2CPP_MapType_of(bar->barcfg);
315 	bartgt = NFP_PCIE_BAR_PCIE2CPP_Target_BaseAddress_of(bar->barcfg);
316 	bartok = NFP_PCIE_BAR_PCIE2CPP_Token_BaseAddress_of(bar->barcfg);
317 	baract = NFP_PCIE_BAR_PCIE2CPP_Action_BaseAddress_of(bar->barcfg);
318 
319 	barwidth = NFP_PCIE_BAR_PCIE2CPP_LengthSelect_of(bar->barcfg);
320 	switch (barwidth) {
321 	case NFP_PCIE_BAR_PCIE2CPP_LengthSelect_32BIT:
322 		barwidth = 4;
323 		break;
324 	case NFP_PCIE_BAR_PCIE2CPP_LengthSelect_64BIT:
325 		barwidth = 8;
326 		break;
327 	case NFP_PCIE_BAR_PCIE2CPP_LengthSelect_0BYTE:
328 		barwidth = 0;
329 		break;
330 	default:
331 		barwidth = -1;
332 		break;
333 	}
334 
335 	switch (maptype) {
336 	case NFP_PCIE_BAR_PCIE2CPP_MapType_TARGET:
337 		bartok = -1;
338 		fallthrough;
339 	case NFP_PCIE_BAR_PCIE2CPP_MapType_BULK:
340 		baract = NFP_CPP_ACTION_RW;
341 		if (act == 0)
342 			act = NFP_CPP_ACTION_RW;
343 		fallthrough;
344 	case NFP_PCIE_BAR_PCIE2CPP_MapType_FIXED:
345 		break;
346 	default:
347 		/* We don't match explicit bars through the area interface */
348 		return 0;
349 	}
350 
351 	/* Make sure to match up the width */
352 	if (barwidth != width)
353 		return 0;
354 
355 	if ((bartgt < 0 || bartgt == tgt) &&
356 	    (bartok < 0 || bartok == tok) &&
357 	    (baract == act) &&
358 	    bar->base <= offset &&
359 	    (bar->base + (1 << bar->bitsize)) >= (offset + size))
360 		return 1;
361 
362 	/* No match */
363 	return 0;
364 }
365 
366 static int
find_matching_bar(struct nfp6000_pcie * nfp,u32 tgt,u32 act,u32 tok,u64 offset,size_t size,int width)367 find_matching_bar(struct nfp6000_pcie *nfp,
368 		  u32 tgt, u32 act, u32 tok, u64 offset, size_t size, int width)
369 {
370 	int n;
371 
372 	for (n = 0; n < nfp->bars; n++) {
373 		struct nfp_bar *bar = &nfp->bar[n];
374 
375 		if (matching_bar(bar, tgt, act, tok, offset, size, width))
376 			return n;
377 	}
378 
379 	return -1;
380 }
381 
382 /* Return EAGAIN if no resource is available */
383 static int
find_unused_bar_noblock(const struct nfp6000_pcie * nfp,int tgt,int act,int tok,u64 offset,size_t size,int width)384 find_unused_bar_noblock(const struct nfp6000_pcie *nfp,
385 			int tgt, int act, int tok,
386 			u64 offset, size_t size, int width)
387 {
388 	int n, busy = 0;
389 
390 	for (n = 0; n < nfp->bars; n++) {
391 		const struct nfp_bar *bar = &nfp->bar[n];
392 		int err;
393 
394 		if (!bar->bitsize)
395 			continue;
396 
397 		/* Just check to see if we can make it fit... */
398 		err = compute_bar(nfp, bar, NULL, NULL,
399 				  tgt, act, tok, offset, size, width);
400 		if (err)
401 			continue;
402 
403 		if (!atomic_read(&bar->refcnt))
404 			return n;
405 
406 		busy++;
407 	}
408 
409 	if (WARN(!busy, "No suitable BAR found for request tgt:0x%x act:0x%x tok:0x%x off:0x%llx size:%zd width:%d\n",
410 		 tgt, act, tok, offset, size, width))
411 		return -EINVAL;
412 
413 	return -EAGAIN;
414 }
415 
416 static int
find_unused_bar_and_lock(struct nfp6000_pcie * nfp,int tgt,int act,int tok,u64 offset,size_t size,int width)417 find_unused_bar_and_lock(struct nfp6000_pcie *nfp,
418 			 int tgt, int act, int tok,
419 			 u64 offset, size_t size, int width)
420 {
421 	unsigned long flags;
422 	int n;
423 
424 	spin_lock_irqsave(&nfp->bar_lock, flags);
425 
426 	n = find_unused_bar_noblock(nfp, tgt, act, tok, offset, size, width);
427 	if (n < 0)
428 		spin_unlock_irqrestore(&nfp->bar_lock, flags);
429 	else
430 		__release(&nfp->bar_lock);
431 
432 	return n;
433 }
434 
nfp_bar_get(struct nfp6000_pcie * nfp,struct nfp_bar * bar)435 static void nfp_bar_get(struct nfp6000_pcie *nfp, struct nfp_bar *bar)
436 {
437 	atomic_inc(&bar->refcnt);
438 }
439 
nfp_bar_put(struct nfp6000_pcie * nfp,struct nfp_bar * bar)440 static void nfp_bar_put(struct nfp6000_pcie *nfp, struct nfp_bar *bar)
441 {
442 	if (atomic_dec_and_test(&bar->refcnt))
443 		wake_up_interruptible(&nfp->bar_waiters);
444 }
445 
446 static int
nfp_wait_for_bar(struct nfp6000_pcie * nfp,int * barnum,u32 tgt,u32 act,u32 tok,u64 offset,size_t size,int width)447 nfp_wait_for_bar(struct nfp6000_pcie *nfp, int *barnum,
448 		 u32 tgt, u32 act, u32 tok, u64 offset, size_t size, int width)
449 {
450 	return wait_event_interruptible(nfp->bar_waiters,
451 		(*barnum = find_unused_bar_and_lock(nfp, tgt, act, tok,
452 						    offset, size, width))
453 					!= -EAGAIN);
454 }
455 
456 static int
nfp_alloc_bar(struct nfp6000_pcie * nfp,u32 tgt,u32 act,u32 tok,u64 offset,size_t size,int width,int nonblocking)457 nfp_alloc_bar(struct nfp6000_pcie *nfp,
458 	      u32 tgt, u32 act, u32 tok,
459 	      u64 offset, size_t size, int width, int nonblocking)
460 {
461 	unsigned long irqflags;
462 	int barnum, retval;
463 
464 	if (size > (1 << 24))
465 		return -EINVAL;
466 
467 	spin_lock_irqsave(&nfp->bar_lock, irqflags);
468 	barnum = find_matching_bar(nfp, tgt, act, tok, offset, size, width);
469 	if (barnum >= 0) {
470 		/* Found a perfect match. */
471 		nfp_bar_get(nfp, &nfp->bar[barnum]);
472 		spin_unlock_irqrestore(&nfp->bar_lock, irqflags);
473 		return barnum;
474 	}
475 
476 	barnum = find_unused_bar_noblock(nfp, tgt, act, tok,
477 					 offset, size, width);
478 	if (barnum < 0) {
479 		if (nonblocking)
480 			goto err_nobar;
481 
482 		/* Wait until a BAR becomes available.  The
483 		 * find_unused_bar function will reclaim the bar_lock
484 		 * if a free BAR is found.
485 		 */
486 		spin_unlock_irqrestore(&nfp->bar_lock, irqflags);
487 		retval = nfp_wait_for_bar(nfp, &barnum, tgt, act, tok,
488 					  offset, size, width);
489 		if (retval)
490 			return retval;
491 		__acquire(&nfp->bar_lock);
492 	}
493 
494 	nfp_bar_get(nfp, &nfp->bar[barnum]);
495 	retval = reconfigure_bar(nfp, &nfp->bar[barnum],
496 				 tgt, act, tok, offset, size, width);
497 	if (retval < 0) {
498 		nfp_bar_put(nfp, &nfp->bar[barnum]);
499 		barnum = retval;
500 	}
501 
502 err_nobar:
503 	spin_unlock_irqrestore(&nfp->bar_lock, irqflags);
504 	return barnum;
505 }
506 
507 static void disable_bars(struct nfp6000_pcie *nfp);
508 
bar_cmp(const void * aptr,const void * bptr)509 static int bar_cmp(const void *aptr, const void *bptr)
510 {
511 	const struct nfp_bar *a = aptr, *b = bptr;
512 
513 	if (a->bitsize == b->bitsize)
514 		return a->index - b->index;
515 	else
516 		return a->bitsize - b->bitsize;
517 }
518 
519 /* Map all PCI bars and fetch the actual BAR configurations from the
520  * board.  We assume that the BAR with the PCIe config block is
521  * already mapped.
522  *
523  * BAR0.0: Reserved for General Mapping (for MSI-X access to PCIe SRAM)
524  * BAR0.1: Reserved for XPB access (for MSI-X access to PCIe PBA)
525  * BAR0.2: --
526  * BAR0.3: --
527  * BAR0.4: Reserved for Explicit 0.0-0.3 access
528  * BAR0.5: Reserved for Explicit 1.0-1.3 access
529  * BAR0.6: Reserved for Explicit 2.0-2.3 access
530  * BAR0.7: Reserved for Explicit 3.0-3.3 access
531  *
532  * BAR1.0-BAR1.7: --
533  * BAR2.0-BAR2.7: --
534  */
enable_bars(struct nfp6000_pcie * nfp,u16 interface)535 static int enable_bars(struct nfp6000_pcie *nfp, u16 interface)
536 {
537 	const u32 barcfg_msix_general =
538 		NFP_PCIE_BAR_PCIE2CPP_MapType(
539 			NFP_PCIE_BAR_PCIE2CPP_MapType_GENERAL) |
540 		NFP_PCIE_BAR_PCIE2CPP_LengthSelect(
541 			NFP_PCIE_BAR_PCIE2CPP_LengthSelect_32BIT);
542 	const u32 barcfg_msix_xpb =
543 		NFP_PCIE_BAR_PCIE2CPP_MapType(
544 			NFP_PCIE_BAR_PCIE2CPP_MapType_BULK) |
545 		NFP_PCIE_BAR_PCIE2CPP_LengthSelect(
546 			NFP_PCIE_BAR_PCIE2CPP_LengthSelect_32BIT) |
547 		NFP_PCIE_BAR_PCIE2CPP_Target_BaseAddress(
548 			NFP_CPP_TARGET_ISLAND_XPB);
549 	const u32 barcfg_explicit[4] = {
550 		NFP_PCIE_BAR_PCIE2CPP_MapType(
551 			NFP_PCIE_BAR_PCIE2CPP_MapType_EXPLICIT0),
552 		NFP_PCIE_BAR_PCIE2CPP_MapType(
553 			NFP_PCIE_BAR_PCIE2CPP_MapType_EXPLICIT1),
554 		NFP_PCIE_BAR_PCIE2CPP_MapType(
555 			NFP_PCIE_BAR_PCIE2CPP_MapType_EXPLICIT2),
556 		NFP_PCIE_BAR_PCIE2CPP_MapType(
557 			NFP_PCIE_BAR_PCIE2CPP_MapType_EXPLICIT3),
558 	};
559 	char status_msg[196] = {};
560 	int i, err, bars_free;
561 	struct nfp_bar *bar;
562 	int expl_groups;
563 	char *msg, *end;
564 
565 	msg = status_msg +
566 		snprintf(status_msg, sizeof(status_msg) - 1, "RESERVED BARs: ");
567 	end = status_msg + sizeof(status_msg) - 1;
568 
569 	bar = &nfp->bar[0];
570 	for (i = 0; i < ARRAY_SIZE(nfp->bar); i++, bar++) {
571 		struct resource *res;
572 
573 		res = &nfp->pdev->resource[(i >> 3) * 2];
574 
575 		/* Skip over BARs that are not IORESOURCE_MEM */
576 		if (!(resource_type(res) & IORESOURCE_MEM)) {
577 			bar--;
578 			continue;
579 		}
580 
581 		bar->resource = res;
582 		bar->barcfg = 0;
583 
584 		bar->nfp = nfp;
585 		bar->index = i;
586 		bar->mask = nfp_bar_resource_len(bar) - 1;
587 		bar->bitsize = fls(bar->mask);
588 		bar->base = 0;
589 		bar->iomem = NULL;
590 	}
591 
592 	nfp->bars = bar - &nfp->bar[0];
593 	if (nfp->bars < 8) {
594 		dev_err(nfp->dev, "No usable BARs found!\n");
595 		return -EINVAL;
596 	}
597 
598 	bars_free = nfp->bars;
599 
600 	/* Convert unit ID (0..3) to signal master/data master ID (0x40..0x70)
601 	 */
602 	mutex_init(&nfp->expl.mutex);
603 
604 	nfp->expl.master_id = ((NFP_CPP_INTERFACE_UNIT_of(interface) & 3) + 4)
605 		<< 4;
606 	nfp->expl.signal_ref = 0x10;
607 
608 	/* Configure, and lock, BAR0.0 for General Target use (MSI-X SRAM) */
609 	bar = &nfp->bar[0];
610 	if (nfp_bar_resource_len(bar) >= NFP_PCI_MIN_MAP_SIZE)
611 		bar->iomem = ioremap(nfp_bar_resource_start(bar),
612 					     nfp_bar_resource_len(bar));
613 	if (bar->iomem) {
614 		int pf;
615 
616 		msg += scnprintf(msg, end - msg, "0.0: General/MSI-X SRAM, ");
617 		atomic_inc(&bar->refcnt);
618 		bars_free--;
619 
620 		nfp6000_bar_write(nfp, bar, barcfg_msix_general);
621 
622 		nfp->expl.data = bar->iomem + NFP_PCIE_SRAM +
623 			nfp->dev_info->pcie_expl_offset;
624 
625 		switch (nfp->pdev->device) {
626 		case PCI_DEVICE_ID_NFP3800:
627 			pf = nfp->pdev->devfn & 7;
628 			nfp->iomem.csr = bar->iomem + NFP_PCIE_BAR(pf);
629 			break;
630 		case PCI_DEVICE_ID_NFP4000:
631 		case PCI_DEVICE_ID_NFP5000:
632 		case PCI_DEVICE_ID_NFP6000:
633 			nfp->iomem.csr = bar->iomem + NFP_PCIE_BAR(0);
634 			break;
635 		default:
636 			dev_err(nfp->dev, "Unsupported device ID: %04hx!\n",
637 				nfp->pdev->device);
638 			err = -EINVAL;
639 			goto err_unmap_bar0;
640 		}
641 		nfp->iomem.em = bar->iomem + NFP_PCIE_EM;
642 	}
643 
644 	switch (nfp->pdev->device) {
645 	case PCI_DEVICE_ID_NFP3800:
646 		expl_groups = 1;
647 		break;
648 	case PCI_DEVICE_ID_NFP4000:
649 	case PCI_DEVICE_ID_NFP5000:
650 	case PCI_DEVICE_ID_NFP6000:
651 		expl_groups = 4;
652 		break;
653 	default:
654 		dev_err(nfp->dev, "Unsupported device ID: %04hx!\n",
655 			nfp->pdev->device);
656 		err = -EINVAL;
657 		goto err_unmap_bar0;
658 	}
659 
660 	/* Configure, and lock, BAR0.1 for PCIe XPB (MSI-X PBA) */
661 	bar = &nfp->bar[1];
662 	msg += scnprintf(msg, end - msg, "0.1: PCIe XPB/MSI-X PBA, ");
663 	atomic_inc(&bar->refcnt);
664 	bars_free--;
665 
666 	nfp6000_bar_write(nfp, bar, barcfg_msix_xpb);
667 
668 	/* Use BAR0.4..BAR0.7 for EXPL IO */
669 	for (i = 0; i < 4; i++) {
670 		int j;
671 
672 		if (i >= NFP_PCIE_EXPLICIT_BARS || i >= expl_groups) {
673 			nfp->expl.group[i].bitsize = 0;
674 			continue;
675 		}
676 
677 		bar = &nfp->bar[4 + i];
678 		bar->iomem = ioremap(nfp_bar_resource_start(bar),
679 					     nfp_bar_resource_len(bar));
680 		if (bar->iomem) {
681 			msg += scnprintf(msg, end - msg,
682 					 "0.%d: Explicit%d, ", 4 + i, i);
683 			atomic_inc(&bar->refcnt);
684 			bars_free--;
685 
686 			nfp->expl.group[i].bitsize = bar->bitsize;
687 			nfp->expl.group[i].addr = bar->iomem;
688 			nfp6000_bar_write(nfp, bar, barcfg_explicit[i]);
689 
690 			for (j = 0; j < 4; j++)
691 				nfp->expl.group[i].free[j] = true;
692 		}
693 		nfp->iomem.expl[i] = bar->iomem;
694 	}
695 
696 	/* Sort bars by bit size - use the smallest possible first. */
697 	sort(&nfp->bar[0], nfp->bars, sizeof(nfp->bar[0]),
698 	     bar_cmp, NULL);
699 
700 	dev_info(nfp->dev, "%sfree: %d/%d\n", status_msg, bars_free, nfp->bars);
701 
702 	return 0;
703 
704 err_unmap_bar0:
705 	if (nfp->bar[0].iomem)
706 		iounmap(nfp->bar[0].iomem);
707 	return err;
708 }
709 
disable_bars(struct nfp6000_pcie * nfp)710 static void disable_bars(struct nfp6000_pcie *nfp)
711 {
712 	struct nfp_bar *bar = &nfp->bar[0];
713 	int n;
714 
715 	for (n = 0; n < nfp->bars; n++, bar++) {
716 		if (bar->iomem) {
717 			iounmap(bar->iomem);
718 			bar->iomem = NULL;
719 		}
720 	}
721 }
722 
723 /*
724  * Generic CPP bus access interface.
725  */
726 
727 struct nfp6000_area_priv {
728 	atomic_t refcnt;
729 
730 	struct nfp_bar *bar;
731 	u32 bar_offset;
732 
733 	u32 target;
734 	u32 action;
735 	u32 token;
736 	u64 offset;
737 	struct {
738 		int read;
739 		int write;
740 		int bar;
741 	} width;
742 	size_t size;
743 
744 	void __iomem *iomem;
745 	phys_addr_t phys;
746 	struct resource resource;
747 };
748 
nfp6000_area_init(struct nfp_cpp_area * area,u32 dest,unsigned long long address,unsigned long size)749 static int nfp6000_area_init(struct nfp_cpp_area *area, u32 dest,
750 			     unsigned long long address, unsigned long size)
751 {
752 	struct nfp6000_area_priv *priv = nfp_cpp_area_priv(area);
753 	u32 target = NFP_CPP_ID_TARGET_of(dest);
754 	u32 action = NFP_CPP_ID_ACTION_of(dest);
755 	u32 token = NFP_CPP_ID_TOKEN_of(dest);
756 	int pp;
757 
758 	pp = nfp_target_pushpull(NFP_CPP_ID(target, action, token), address);
759 	if (pp < 0)
760 		return pp;
761 
762 	priv->width.read = PUSH_WIDTH(pp);
763 	priv->width.write = PULL_WIDTH(pp);
764 	if (priv->width.read > 0 &&
765 	    priv->width.write > 0 &&
766 	    priv->width.read != priv->width.write) {
767 		return -EINVAL;
768 	}
769 
770 	if (priv->width.read > 0)
771 		priv->width.bar = priv->width.read;
772 	else
773 		priv->width.bar = priv->width.write;
774 
775 	atomic_set(&priv->refcnt, 0);
776 	priv->bar = NULL;
777 
778 	priv->target = target;
779 	priv->action = action;
780 	priv->token = token;
781 	priv->offset = address;
782 	priv->size = size;
783 	memset(&priv->resource, 0, sizeof(priv->resource));
784 
785 	return 0;
786 }
787 
nfp6000_area_cleanup(struct nfp_cpp_area * area)788 static void nfp6000_area_cleanup(struct nfp_cpp_area *area)
789 {
790 }
791 
priv_area_get(struct nfp_cpp_area * area)792 static void priv_area_get(struct nfp_cpp_area *area)
793 {
794 	struct nfp6000_area_priv *priv = nfp_cpp_area_priv(area);
795 
796 	atomic_inc(&priv->refcnt);
797 }
798 
priv_area_put(struct nfp_cpp_area * area)799 static int priv_area_put(struct nfp_cpp_area *area)
800 {
801 	struct nfp6000_area_priv *priv = nfp_cpp_area_priv(area);
802 
803 	if (WARN_ON(!atomic_read(&priv->refcnt)))
804 		return 0;
805 
806 	return atomic_dec_and_test(&priv->refcnt);
807 }
808 
nfp6000_area_acquire(struct nfp_cpp_area * area)809 static int nfp6000_area_acquire(struct nfp_cpp_area *area)
810 {
811 	struct nfp6000_pcie *nfp = nfp_cpp_priv(nfp_cpp_area_cpp(area));
812 	struct nfp6000_area_priv *priv = nfp_cpp_area_priv(area);
813 	int barnum, err;
814 
815 	if (priv->bar) {
816 		/* Already allocated. */
817 		priv_area_get(area);
818 		return 0;
819 	}
820 
821 	barnum = nfp_alloc_bar(nfp, priv->target, priv->action, priv->token,
822 			       priv->offset, priv->size, priv->width.bar, 1);
823 
824 	if (barnum < 0) {
825 		err = barnum;
826 		goto err_alloc_bar;
827 	}
828 	priv->bar = &nfp->bar[barnum];
829 
830 	/* Calculate offset into BAR. */
831 	if (nfp_bar_maptype(priv->bar) ==
832 	    NFP_PCIE_BAR_PCIE2CPP_MapType_GENERAL) {
833 		priv->bar_offset = priv->offset &
834 			(NFP_PCIE_P2C_GENERAL_SIZE(priv->bar) - 1);
835 		priv->bar_offset += NFP_PCIE_P2C_GENERAL_TARGET_OFFSET(
836 			priv->bar, priv->target);
837 		priv->bar_offset += NFP_PCIE_P2C_GENERAL_TOKEN_OFFSET(
838 			priv->bar, priv->token);
839 	} else {
840 		priv->bar_offset = priv->offset & priv->bar->mask;
841 	}
842 
843 	/* We don't actually try to acquire the resource area using
844 	 * request_resource.  This would prevent sharing the mapped
845 	 * BAR between multiple CPP areas and prevent us from
846 	 * effectively utilizing the limited amount of BAR resources.
847 	 */
848 	priv->phys = nfp_bar_resource_start(priv->bar) + priv->bar_offset;
849 	priv->resource.name = nfp_cpp_area_name(area);
850 	priv->resource.start = priv->phys;
851 	priv->resource.end = priv->resource.start + priv->size - 1;
852 	priv->resource.flags = IORESOURCE_MEM;
853 
854 	/* If the bar is already mapped in, use its mapping */
855 	if (priv->bar->iomem)
856 		priv->iomem = priv->bar->iomem + priv->bar_offset;
857 	else
858 		/* Must have been too big. Sub-allocate. */
859 		priv->iomem = ioremap(priv->phys, priv->size);
860 
861 	if (IS_ERR_OR_NULL(priv->iomem)) {
862 		dev_err(nfp->dev, "Can't ioremap() a %d byte region of BAR %d\n",
863 			(int)priv->size, priv->bar->index);
864 		err = !priv->iomem ? -ENOMEM : PTR_ERR(priv->iomem);
865 		priv->iomem = NULL;
866 		goto err_iomem_remap;
867 	}
868 
869 	priv_area_get(area);
870 	return 0;
871 
872 err_iomem_remap:
873 	nfp_bar_put(nfp, priv->bar);
874 	priv->bar = NULL;
875 err_alloc_bar:
876 	return err;
877 }
878 
nfp6000_area_release(struct nfp_cpp_area * area)879 static void nfp6000_area_release(struct nfp_cpp_area *area)
880 {
881 	struct nfp6000_pcie *nfp = nfp_cpp_priv(nfp_cpp_area_cpp(area));
882 	struct nfp6000_area_priv *priv = nfp_cpp_area_priv(area);
883 
884 	if (!priv_area_put(area))
885 		return;
886 
887 	if (!priv->bar->iomem)
888 		iounmap(priv->iomem);
889 
890 	nfp_bar_put(nfp, priv->bar);
891 
892 	priv->bar = NULL;
893 	priv->iomem = NULL;
894 }
895 
nfp6000_area_phys(struct nfp_cpp_area * area)896 static phys_addr_t nfp6000_area_phys(struct nfp_cpp_area *area)
897 {
898 	struct nfp6000_area_priv *priv = nfp_cpp_area_priv(area);
899 
900 	return priv->phys;
901 }
902 
nfp6000_area_iomem(struct nfp_cpp_area * area)903 static void __iomem *nfp6000_area_iomem(struct nfp_cpp_area *area)
904 {
905 	struct nfp6000_area_priv *priv = nfp_cpp_area_priv(area);
906 
907 	return priv->iomem;
908 }
909 
nfp6000_area_resource(struct nfp_cpp_area * area)910 static struct resource *nfp6000_area_resource(struct nfp_cpp_area *area)
911 {
912 	/* Use the BAR resource as the resource for the CPP area.
913 	 * This enables us to share the BAR among multiple CPP areas
914 	 * without resource conflicts.
915 	 */
916 	struct nfp6000_area_priv *priv = nfp_cpp_area_priv(area);
917 
918 	return priv->bar->resource;
919 }
920 
nfp6000_area_read(struct nfp_cpp_area * area,void * kernel_vaddr,unsigned long offset,unsigned int length)921 static int nfp6000_area_read(struct nfp_cpp_area *area, void *kernel_vaddr,
922 			     unsigned long offset, unsigned int length)
923 {
924 	u64 __maybe_unused *wrptr64 = kernel_vaddr;
925 	const u64 __iomem __maybe_unused *rdptr64;
926 	struct nfp6000_area_priv *priv;
927 	u32 *wrptr32 = kernel_vaddr;
928 	const u32 __iomem *rdptr32;
929 	int n, width;
930 
931 	priv = nfp_cpp_area_priv(area);
932 	rdptr64 = priv->iomem + offset;
933 	rdptr32 = priv->iomem + offset;
934 
935 	if (offset + length > priv->size)
936 		return -EFAULT;
937 
938 	width = priv->width.read;
939 	if (width <= 0)
940 		return -EINVAL;
941 
942 	/* MU reads via a PCIe2CPP BAR support 32bit (and other) lengths */
943 	if (priv->target == (NFP_CPP_TARGET_MU & NFP_CPP_TARGET_ID_MASK) &&
944 	    priv->action == NFP_CPP_ACTION_RW &&
945 	    (offset % sizeof(u64) == 4 || length % sizeof(u64) == 4))
946 		width = TARGET_WIDTH_32;
947 
948 	/* Unaligned? Translate to an explicit access */
949 	if ((priv->offset + offset) & (width - 1))
950 		return nfp_cpp_explicit_read(nfp_cpp_area_cpp(area),
951 					     NFP_CPP_ID(priv->target,
952 							priv->action,
953 							priv->token),
954 					     priv->offset + offset,
955 					     kernel_vaddr, length, width);
956 
957 	if (WARN_ON(!priv->bar))
958 		return -EFAULT;
959 
960 	switch (width) {
961 	case TARGET_WIDTH_32:
962 		if (offset % sizeof(u32) != 0 || length % sizeof(u32) != 0)
963 			return -EINVAL;
964 
965 		for (n = 0; n < length; n += sizeof(u32))
966 			*wrptr32++ = __raw_readl(rdptr32++);
967 		return n;
968 #ifdef __raw_readq
969 	case TARGET_WIDTH_64:
970 		if (offset % sizeof(u64) != 0 || length % sizeof(u64) != 0)
971 			return -EINVAL;
972 
973 		for (n = 0; n < length; n += sizeof(u64))
974 			*wrptr64++ = __raw_readq(rdptr64++);
975 		return n;
976 #endif
977 	default:
978 		return -EINVAL;
979 	}
980 }
981 
982 static int
nfp6000_area_write(struct nfp_cpp_area * area,const void * kernel_vaddr,unsigned long offset,unsigned int length)983 nfp6000_area_write(struct nfp_cpp_area *area,
984 		   const void *kernel_vaddr,
985 		   unsigned long offset, unsigned int length)
986 {
987 	const u64 __maybe_unused *rdptr64 = kernel_vaddr;
988 	u64 __iomem __maybe_unused *wrptr64;
989 	const u32 *rdptr32 = kernel_vaddr;
990 	struct nfp6000_area_priv *priv;
991 	u32 __iomem *wrptr32;
992 	int n, width;
993 
994 	priv = nfp_cpp_area_priv(area);
995 	wrptr64 = priv->iomem + offset;
996 	wrptr32 = priv->iomem + offset;
997 
998 	if (offset + length > priv->size)
999 		return -EFAULT;
1000 
1001 	width = priv->width.write;
1002 	if (width <= 0)
1003 		return -EINVAL;
1004 
1005 	/* MU writes via a PCIe2CPP BAR support 32bit (and other) lengths */
1006 	if (priv->target == (NFP_CPP_TARGET_ID_MASK & NFP_CPP_TARGET_MU) &&
1007 	    priv->action == NFP_CPP_ACTION_RW &&
1008 	    (offset % sizeof(u64) == 4 || length % sizeof(u64) == 4))
1009 		width = TARGET_WIDTH_32;
1010 
1011 	/* Unaligned? Translate to an explicit access */
1012 	if ((priv->offset + offset) & (width - 1))
1013 		return nfp_cpp_explicit_write(nfp_cpp_area_cpp(area),
1014 					      NFP_CPP_ID(priv->target,
1015 							 priv->action,
1016 							 priv->token),
1017 					      priv->offset + offset,
1018 					      kernel_vaddr, length, width);
1019 
1020 	if (WARN_ON(!priv->bar))
1021 		return -EFAULT;
1022 
1023 	switch (width) {
1024 	case TARGET_WIDTH_32:
1025 		if (offset % sizeof(u32) != 0 || length % sizeof(u32) != 0)
1026 			return -EINVAL;
1027 
1028 		for (n = 0; n < length; n += sizeof(u32)) {
1029 			__raw_writel(*rdptr32++, wrptr32++);
1030 			wmb();
1031 		}
1032 		return n;
1033 #ifdef __raw_writeq
1034 	case TARGET_WIDTH_64:
1035 		if (offset % sizeof(u64) != 0 || length % sizeof(u64) != 0)
1036 			return -EINVAL;
1037 
1038 		for (n = 0; n < length; n += sizeof(u64)) {
1039 			__raw_writeq(*rdptr64++, wrptr64++);
1040 			wmb();
1041 		}
1042 		return n;
1043 #endif
1044 	default:
1045 		return -EINVAL;
1046 	}
1047 }
1048 
1049 struct nfp6000_explicit_priv {
1050 	struct nfp6000_pcie *nfp;
1051 	struct {
1052 		int group;
1053 		int area;
1054 	} bar;
1055 	int bitsize;
1056 	void __iomem *data;
1057 	void __iomem *addr;
1058 };
1059 
nfp6000_explicit_acquire(struct nfp_cpp_explicit * expl)1060 static int nfp6000_explicit_acquire(struct nfp_cpp_explicit *expl)
1061 {
1062 	struct nfp6000_pcie *nfp = nfp_cpp_priv(nfp_cpp_explicit_cpp(expl));
1063 	struct nfp6000_explicit_priv *priv = nfp_cpp_explicit_priv(expl);
1064 	int i, j;
1065 
1066 	mutex_lock(&nfp->expl.mutex);
1067 	for (i = 0; i < ARRAY_SIZE(nfp->expl.group); i++) {
1068 		if (!nfp->expl.group[i].bitsize)
1069 			continue;
1070 
1071 		for (j = 0; j < ARRAY_SIZE(nfp->expl.group[i].free); j++) {
1072 			u16 data_offset;
1073 
1074 			if (!nfp->expl.group[i].free[j])
1075 				continue;
1076 
1077 			priv->nfp = nfp;
1078 			priv->bar.group = i;
1079 			priv->bar.area = j;
1080 			priv->bitsize = nfp->expl.group[i].bitsize - 2;
1081 
1082 			data_offset = (priv->bar.group << 9) +
1083 				(priv->bar.area << 7);
1084 			priv->data = nfp->expl.data + data_offset;
1085 			priv->addr = nfp->expl.group[i].addr +
1086 				(priv->bar.area << priv->bitsize);
1087 			nfp->expl.group[i].free[j] = false;
1088 
1089 			mutex_unlock(&nfp->expl.mutex);
1090 			return 0;
1091 		}
1092 	}
1093 	mutex_unlock(&nfp->expl.mutex);
1094 
1095 	return -EAGAIN;
1096 }
1097 
nfp6000_explicit_release(struct nfp_cpp_explicit * expl)1098 static void nfp6000_explicit_release(struct nfp_cpp_explicit *expl)
1099 {
1100 	struct nfp6000_explicit_priv *priv = nfp_cpp_explicit_priv(expl);
1101 	struct nfp6000_pcie *nfp = priv->nfp;
1102 
1103 	mutex_lock(&nfp->expl.mutex);
1104 	nfp->expl.group[priv->bar.group].free[priv->bar.area] = true;
1105 	mutex_unlock(&nfp->expl.mutex);
1106 }
1107 
nfp6000_explicit_put(struct nfp_cpp_explicit * expl,const void * buff,size_t len)1108 static int nfp6000_explicit_put(struct nfp_cpp_explicit *expl,
1109 				const void *buff, size_t len)
1110 {
1111 	struct nfp6000_explicit_priv *priv = nfp_cpp_explicit_priv(expl);
1112 	const u32 *src = buff;
1113 	size_t i;
1114 
1115 	for (i = 0; i < len; i += sizeof(u32))
1116 		writel(*(src++), priv->data + i);
1117 
1118 	return i;
1119 }
1120 
1121 static int
nfp6000_explicit_do(struct nfp_cpp_explicit * expl,const struct nfp_cpp_explicit_command * cmd,u64 address)1122 nfp6000_explicit_do(struct nfp_cpp_explicit *expl,
1123 		    const struct nfp_cpp_explicit_command *cmd, u64 address)
1124 {
1125 	struct nfp6000_explicit_priv *priv = nfp_cpp_explicit_priv(expl);
1126 	u8 signal_master, signal_ref, data_master;
1127 	struct nfp6000_pcie *nfp = priv->nfp;
1128 	int sigmask = 0;
1129 	u16 data_ref;
1130 	u32 csr[3];
1131 
1132 	if (cmd->siga_mode)
1133 		sigmask |= 1 << cmd->siga;
1134 	if (cmd->sigb_mode)
1135 		sigmask |= 1 << cmd->sigb;
1136 
1137 	signal_master = cmd->signal_master;
1138 	if (!signal_master)
1139 		signal_master = nfp->expl.master_id;
1140 
1141 	signal_ref = cmd->signal_ref;
1142 	if (signal_master == nfp->expl.master_id)
1143 		signal_ref = nfp->expl.signal_ref +
1144 			((priv->bar.group * 4 + priv->bar.area) << 1);
1145 
1146 	data_master = cmd->data_master;
1147 	if (!data_master)
1148 		data_master = nfp->expl.master_id;
1149 
1150 	data_ref = cmd->data_ref;
1151 	if (data_master == nfp->expl.master_id)
1152 		data_ref = 0x1000 +
1153 			(priv->bar.group << 9) + (priv->bar.area << 7);
1154 
1155 	csr[0] = NFP_PCIE_BAR_EXPLICIT_BAR0_SignalType(sigmask) |
1156 		NFP_PCIE_BAR_EXPLICIT_BAR0_Token(
1157 			NFP_CPP_ID_TOKEN_of(cmd->cpp_id)) |
1158 		NFP_PCIE_BAR_EXPLICIT_BAR0_Address(address >> 16);
1159 
1160 	csr[1] = NFP_PCIE_BAR_EXPLICIT_BAR1_SignalRef(signal_ref) |
1161 		NFP_PCIE_BAR_EXPLICIT_BAR1_DataMaster(data_master) |
1162 		NFP_PCIE_BAR_EXPLICIT_BAR1_DataRef(data_ref);
1163 
1164 	csr[2] = NFP_PCIE_BAR_EXPLICIT_BAR2_Target(
1165 			NFP_CPP_ID_TARGET_of(cmd->cpp_id)) |
1166 		NFP_PCIE_BAR_EXPLICIT_BAR2_Action(
1167 			NFP_CPP_ID_ACTION_of(cmd->cpp_id)) |
1168 		NFP_PCIE_BAR_EXPLICIT_BAR2_Length(cmd->len) |
1169 		NFP_PCIE_BAR_EXPLICIT_BAR2_ByteMask(cmd->byte_mask) |
1170 		NFP_PCIE_BAR_EXPLICIT_BAR2_SignalMaster(signal_master);
1171 
1172 	if (nfp->iomem.csr) {
1173 		writel(csr[0], nfp->iomem.csr +
1174 		       NFP_PCIE_BAR_EXPLICIT_BAR0(priv->bar.group,
1175 						  priv->bar.area));
1176 		writel(csr[1], nfp->iomem.csr +
1177 		       NFP_PCIE_BAR_EXPLICIT_BAR1(priv->bar.group,
1178 						  priv->bar.area));
1179 		writel(csr[2], nfp->iomem.csr +
1180 		       NFP_PCIE_BAR_EXPLICIT_BAR2(priv->bar.group,
1181 						  priv->bar.area));
1182 		/* Readback to ensure BAR is flushed */
1183 		readl(nfp->iomem.csr +
1184 		      NFP_PCIE_BAR_EXPLICIT_BAR0(priv->bar.group,
1185 						 priv->bar.area));
1186 		readl(nfp->iomem.csr +
1187 		      NFP_PCIE_BAR_EXPLICIT_BAR1(priv->bar.group,
1188 						 priv->bar.area));
1189 		readl(nfp->iomem.csr +
1190 		      NFP_PCIE_BAR_EXPLICIT_BAR2(priv->bar.group,
1191 						 priv->bar.area));
1192 	} else {
1193 		pci_write_config_dword(nfp->pdev, 0x400 +
1194 				       NFP_PCIE_BAR_EXPLICIT_BAR0(
1195 					       priv->bar.group, priv->bar.area),
1196 				       csr[0]);
1197 
1198 		pci_write_config_dword(nfp->pdev, 0x400 +
1199 				       NFP_PCIE_BAR_EXPLICIT_BAR1(
1200 					       priv->bar.group, priv->bar.area),
1201 				       csr[1]);
1202 
1203 		pci_write_config_dword(nfp->pdev, 0x400 +
1204 				       NFP_PCIE_BAR_EXPLICIT_BAR2(
1205 					       priv->bar.group, priv->bar.area),
1206 				       csr[2]);
1207 	}
1208 
1209 	/* Issue the 'kickoff' transaction */
1210 	readb(priv->addr + (address & ((1 << priv->bitsize) - 1)));
1211 
1212 	return sigmask;
1213 }
1214 
nfp6000_explicit_get(struct nfp_cpp_explicit * expl,void * buff,size_t len)1215 static int nfp6000_explicit_get(struct nfp_cpp_explicit *expl,
1216 				void *buff, size_t len)
1217 {
1218 	struct nfp6000_explicit_priv *priv = nfp_cpp_explicit_priv(expl);
1219 	u32 *dst = buff;
1220 	size_t i;
1221 
1222 	for (i = 0; i < len; i += sizeof(u32))
1223 		*(dst++) = readl(priv->data + i);
1224 
1225 	return i;
1226 }
1227 
nfp6000_init(struct nfp_cpp * cpp)1228 static int nfp6000_init(struct nfp_cpp *cpp)
1229 {
1230 	nfp_cpp_area_cache_add(cpp, SZ_64K);
1231 	nfp_cpp_area_cache_add(cpp, SZ_64K);
1232 	nfp_cpp_area_cache_add(cpp, SZ_256K);
1233 
1234 	return 0;
1235 }
1236 
nfp6000_free(struct nfp_cpp * cpp)1237 static void nfp6000_free(struct nfp_cpp *cpp)
1238 {
1239 	struct nfp6000_pcie *nfp = nfp_cpp_priv(cpp);
1240 
1241 	disable_bars(nfp);
1242 	kfree(nfp);
1243 }
1244 
nfp6000_read_serial(struct device * dev,u8 * serial)1245 static int nfp6000_read_serial(struct device *dev, u8 *serial)
1246 {
1247 	struct pci_dev *pdev = to_pci_dev(dev);
1248 	u64 dsn;
1249 
1250 	dsn = pci_get_dsn(pdev);
1251 	if (!dsn) {
1252 		dev_err(dev, "can't find PCIe Serial Number Capability\n");
1253 		return -EINVAL;
1254 	}
1255 
1256 	put_unaligned_be32((u32)(dsn >> 32), serial);
1257 	put_unaligned_be16((u16)(dsn >> 16), serial + 4);
1258 
1259 	return 0;
1260 }
1261 
nfp6000_get_interface(struct device * dev)1262 static int nfp6000_get_interface(struct device *dev)
1263 {
1264 	struct pci_dev *pdev = to_pci_dev(dev);
1265 	u64 dsn;
1266 
1267 	dsn = pci_get_dsn(pdev);
1268 	if (!dsn) {
1269 		dev_err(dev, "can't find PCIe Serial Number Capability\n");
1270 		return -EINVAL;
1271 	}
1272 
1273 	return dsn & 0xffff;
1274 }
1275 
1276 static const struct nfp_cpp_operations nfp6000_pcie_ops = {
1277 	.owner			= THIS_MODULE,
1278 
1279 	.init			= nfp6000_init,
1280 	.free			= nfp6000_free,
1281 
1282 	.read_serial		= nfp6000_read_serial,
1283 	.get_interface		= nfp6000_get_interface,
1284 
1285 	.area_priv_size		= sizeof(struct nfp6000_area_priv),
1286 	.area_init		= nfp6000_area_init,
1287 	.area_cleanup		= nfp6000_area_cleanup,
1288 	.area_acquire		= nfp6000_area_acquire,
1289 	.area_release		= nfp6000_area_release,
1290 	.area_phys		= nfp6000_area_phys,
1291 	.area_iomem		= nfp6000_area_iomem,
1292 	.area_resource		= nfp6000_area_resource,
1293 	.area_read		= nfp6000_area_read,
1294 	.area_write		= nfp6000_area_write,
1295 
1296 	.explicit_priv_size	= sizeof(struct nfp6000_explicit_priv),
1297 	.explicit_acquire	= nfp6000_explicit_acquire,
1298 	.explicit_release	= nfp6000_explicit_release,
1299 	.explicit_put		= nfp6000_explicit_put,
1300 	.explicit_do		= nfp6000_explicit_do,
1301 	.explicit_get		= nfp6000_explicit_get,
1302 };
1303 
1304 /**
1305  * nfp_cpp_from_nfp6000_pcie() - Build a NFP CPP bus from a NFP6000 PCI device
1306  * @pdev:	NFP6000 PCI device
1307  * @dev_info:	NFP ASIC params
1308  *
1309  * Return: NFP CPP handle
1310  */
1311 struct nfp_cpp *
nfp_cpp_from_nfp6000_pcie(struct pci_dev * pdev,const struct nfp_dev_info * dev_info)1312 nfp_cpp_from_nfp6000_pcie(struct pci_dev *pdev, const struct nfp_dev_info *dev_info)
1313 {
1314 	struct nfp6000_pcie *nfp;
1315 	u16 interface;
1316 	int err;
1317 
1318 	/*  Finished with card initialization. */
1319 	dev_info(&pdev->dev, "Network Flow Processor %s PCIe Card Probe\n",
1320 		 dev_info->chip_names);
1321 	pcie_print_link_status(pdev);
1322 
1323 	nfp = kzalloc(sizeof(*nfp), GFP_KERNEL);
1324 	if (!nfp) {
1325 		err = -ENOMEM;
1326 		goto err_ret;
1327 	}
1328 
1329 	nfp->dev = &pdev->dev;
1330 	nfp->pdev = pdev;
1331 	nfp->dev_info = dev_info;
1332 	init_waitqueue_head(&nfp->bar_waiters);
1333 	spin_lock_init(&nfp->bar_lock);
1334 
1335 	interface = nfp6000_get_interface(&pdev->dev);
1336 
1337 	if (NFP_CPP_INTERFACE_TYPE_of(interface) !=
1338 	    NFP_CPP_INTERFACE_TYPE_PCI) {
1339 		dev_err(&pdev->dev,
1340 			"Interface type %d is not the expected %d\n",
1341 			NFP_CPP_INTERFACE_TYPE_of(interface),
1342 			NFP_CPP_INTERFACE_TYPE_PCI);
1343 		err = -ENODEV;
1344 		goto err_free_nfp;
1345 	}
1346 
1347 	if (NFP_CPP_INTERFACE_CHANNEL_of(interface) !=
1348 	    NFP_CPP_INTERFACE_CHANNEL_PEROPENER) {
1349 		dev_err(&pdev->dev, "Interface channel %d is not the expected %d\n",
1350 			NFP_CPP_INTERFACE_CHANNEL_of(interface),
1351 			NFP_CPP_INTERFACE_CHANNEL_PEROPENER);
1352 		err = -ENODEV;
1353 		goto err_free_nfp;
1354 	}
1355 
1356 	err = enable_bars(nfp, interface);
1357 	if (err)
1358 		goto err_free_nfp;
1359 
1360 	/* Probe for all the common NFP devices */
1361 	return nfp_cpp_from_operations(&nfp6000_pcie_ops, &pdev->dev, nfp);
1362 
1363 err_free_nfp:
1364 	kfree(nfp);
1365 err_ret:
1366 	dev_err(&pdev->dev, "NFP6000 PCI setup failed\n");
1367 	return ERR_PTR(err);
1368 }
1369