1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) 2 /* Copyright (C) 2015-2018 Netronome Systems, Inc. */ 3 4 /* 5 * nfp6000_pcie.c 6 * Authors: Jakub Kicinski <jakub.kicinski@netronome.com> 7 * Jason McMullan <jason.mcmullan@netronome.com> 8 * Rolf Neugebauer <rolf.neugebauer@netronome.com> 9 * 10 * Multiplexes the NFP BARs between NFP internal resources and 11 * implements the PCIe specific interface for generic CPP bus access. 12 * 13 * The BARs are managed with refcounts and are allocated/acquired 14 * using target, token and offset/size matching. The generic CPP bus 15 * abstraction builds upon this BAR interface. 16 */ 17 18 #include <asm/unaligned.h> 19 #include <linux/kernel.h> 20 #include <linux/module.h> 21 #include <linux/kref.h> 22 #include <linux/io.h> 23 #include <linux/delay.h> 24 #include <linux/interrupt.h> 25 #include <linux/sort.h> 26 #include <linux/sched.h> 27 #include <linux/types.h> 28 #include <linux/pci.h> 29 30 #include "nfp_cpp.h" 31 #include "nfp_dev.h" 32 33 #include "nfp6000/nfp6000.h" 34 35 #include "nfp6000_pcie.h" 36 37 #define NFP_PCIE_BAR(_pf) (0x30000 + ((_pf) & 7) * 0xc0) 38 #define NFP_PCIE_BAR_EXPLICIT_BAR0(_x, _y) \ 39 (0x00000080 + (0x40 * ((_x) & 0x3)) + (0x10 * ((_y) & 0x3))) 40 #define NFP_PCIE_BAR_EXPLICIT_BAR0_SignalType(_x) (((_x) & 0x3) << 30) 41 #define NFP_PCIE_BAR_EXPLICIT_BAR0_SignalType_of(_x) (((_x) >> 30) & 0x3) 42 #define NFP_PCIE_BAR_EXPLICIT_BAR0_Token(_x) (((_x) & 0x3) << 28) 43 #define NFP_PCIE_BAR_EXPLICIT_BAR0_Token_of(_x) (((_x) >> 28) & 0x3) 44 #define NFP_PCIE_BAR_EXPLICIT_BAR0_Address(_x) (((_x) & 0xffffff) << 0) 45 #define NFP_PCIE_BAR_EXPLICIT_BAR0_Address_of(_x) (((_x) >> 0) & 0xffffff) 46 #define NFP_PCIE_BAR_EXPLICIT_BAR1(_x, _y) \ 47 (0x00000084 + (0x40 * ((_x) & 0x3)) + (0x10 * ((_y) & 0x3))) 48 #define NFP_PCIE_BAR_EXPLICIT_BAR1_SignalRef(_x) (((_x) & 0x7f) << 24) 49 #define NFP_PCIE_BAR_EXPLICIT_BAR1_SignalRef_of(_x) (((_x) >> 24) & 0x7f) 50 #define NFP_PCIE_BAR_EXPLICIT_BAR1_DataMaster(_x) (((_x) & 0x3ff) << 14) 51 #define NFP_PCIE_BAR_EXPLICIT_BAR1_DataMaster_of(_x) (((_x) >> 14) & 0x3ff) 52 #define NFP_PCIE_BAR_EXPLICIT_BAR1_DataRef(_x) (((_x) & 0x3fff) << 0) 53 #define NFP_PCIE_BAR_EXPLICIT_BAR1_DataRef_of(_x) (((_x) >> 0) & 0x3fff) 54 #define NFP_PCIE_BAR_EXPLICIT_BAR2(_x, _y) \ 55 (0x00000088 + (0x40 * ((_x) & 0x3)) + (0x10 * ((_y) & 0x3))) 56 #define NFP_PCIE_BAR_EXPLICIT_BAR2_Target(_x) (((_x) & 0xf) << 28) 57 #define NFP_PCIE_BAR_EXPLICIT_BAR2_Target_of(_x) (((_x) >> 28) & 0xf) 58 #define NFP_PCIE_BAR_EXPLICIT_BAR2_Action(_x) (((_x) & 0x1f) << 23) 59 #define NFP_PCIE_BAR_EXPLICIT_BAR2_Action_of(_x) (((_x) >> 23) & 0x1f) 60 #define NFP_PCIE_BAR_EXPLICIT_BAR2_Length(_x) (((_x) & 0x1f) << 18) 61 #define NFP_PCIE_BAR_EXPLICIT_BAR2_Length_of(_x) (((_x) >> 18) & 0x1f) 62 #define NFP_PCIE_BAR_EXPLICIT_BAR2_ByteMask(_x) (((_x) & 0xff) << 10) 63 #define NFP_PCIE_BAR_EXPLICIT_BAR2_ByteMask_of(_x) (((_x) >> 10) & 0xff) 64 #define NFP_PCIE_BAR_EXPLICIT_BAR2_SignalMaster(_x) (((_x) & 0x3ff) << 0) 65 #define NFP_PCIE_BAR_EXPLICIT_BAR2_SignalMaster_of(_x) (((_x) >> 0) & 0x3ff) 66 67 #define NFP_PCIE_BAR_PCIE2CPP_Action_BaseAddress(_x) (((_x) & 0x1f) << 16) 68 #define NFP_PCIE_BAR_PCIE2CPP_Action_BaseAddress_of(_x) (((_x) >> 16) & 0x1f) 69 #define NFP_PCIE_BAR_PCIE2CPP_BaseAddress(_x) (((_x) & 0xffff) << 0) 70 #define NFP_PCIE_BAR_PCIE2CPP_BaseAddress_of(_x) (((_x) >> 0) & 0xffff) 71 #define NFP_PCIE_BAR_PCIE2CPP_LengthSelect(_x) (((_x) & 0x3) << 27) 72 #define NFP_PCIE_BAR_PCIE2CPP_LengthSelect_of(_x) (((_x) >> 27) & 0x3) 73 #define NFP_PCIE_BAR_PCIE2CPP_LengthSelect_32BIT 0 74 #define NFP_PCIE_BAR_PCIE2CPP_LengthSelect_64BIT 1 75 #define NFP_PCIE_BAR_PCIE2CPP_LengthSelect_0BYTE 3 76 #define NFP_PCIE_BAR_PCIE2CPP_MapType(_x) (((_x) & 0x7) << 29) 77 #define NFP_PCIE_BAR_PCIE2CPP_MapType_of(_x) (((_x) >> 29) & 0x7) 78 #define NFP_PCIE_BAR_PCIE2CPP_MapType_FIXED 0 79 #define NFP_PCIE_BAR_PCIE2CPP_MapType_BULK 1 80 #define NFP_PCIE_BAR_PCIE2CPP_MapType_TARGET 2 81 #define NFP_PCIE_BAR_PCIE2CPP_MapType_GENERAL 3 82 #define NFP_PCIE_BAR_PCIE2CPP_MapType_EXPLICIT0 4 83 #define NFP_PCIE_BAR_PCIE2CPP_MapType_EXPLICIT1 5 84 #define NFP_PCIE_BAR_PCIE2CPP_MapType_EXPLICIT2 6 85 #define NFP_PCIE_BAR_PCIE2CPP_MapType_EXPLICIT3 7 86 #define NFP_PCIE_BAR_PCIE2CPP_Target_BaseAddress(_x) (((_x) & 0xf) << 23) 87 #define NFP_PCIE_BAR_PCIE2CPP_Target_BaseAddress_of(_x) (((_x) >> 23) & 0xf) 88 #define NFP_PCIE_BAR_PCIE2CPP_Token_BaseAddress(_x) (((_x) & 0x3) << 21) 89 #define NFP_PCIE_BAR_PCIE2CPP_Token_BaseAddress_of(_x) (((_x) >> 21) & 0x3) 90 #define NFP_PCIE_EM 0x020000 91 #define NFP_PCIE_SRAM 0x000000 92 93 /* Minimal size of the PCIe cfg memory we depend on being mapped, 94 * queue controller and DMA controller don't have to be covered. 95 */ 96 #define NFP_PCI_MIN_MAP_SIZE 0x080000 97 98 #define NFP_PCIE_P2C_FIXED_SIZE(bar) (1 << (bar)->bitsize) 99 #define NFP_PCIE_P2C_BULK_SIZE(bar) (1 << (bar)->bitsize) 100 #define NFP_PCIE_P2C_GENERAL_TARGET_OFFSET(bar, x) ((x) << ((bar)->bitsize - 2)) 101 #define NFP_PCIE_P2C_GENERAL_TOKEN_OFFSET(bar, x) ((x) << ((bar)->bitsize - 4)) 102 #define NFP_PCIE_P2C_GENERAL_SIZE(bar) (1 << ((bar)->bitsize - 4)) 103 104 #define NFP_PCIE_P2C_EXPBAR_OFFSET(bar_index) ((bar_index) * 4) 105 106 /* The number of explicit BARs to reserve. 107 * Minimum is 0, maximum is 4 on the NFP6000. 108 * The NFP3800 can have only one per PF. 109 */ 110 #define NFP_PCIE_EXPLICIT_BARS 2 111 112 struct nfp6000_pcie; 113 struct nfp6000_area_priv; 114 115 /** 116 * struct nfp_bar - describes BAR configuration and usage 117 * @nfp: backlink to owner 118 * @barcfg: cached contents of BAR config CSR 119 * @base: the BAR's base CPP offset 120 * @mask: mask for the BAR aperture (read only) 121 * @bitsize: bitsize of BAR aperture (read only) 122 * @index: index of the BAR 123 * @refcnt: number of current users 124 * @iomem: mapped IO memory 125 * @resource: iomem resource window 126 */ 127 struct nfp_bar { 128 struct nfp6000_pcie *nfp; 129 u32 barcfg; 130 u64 base; /* CPP address base */ 131 u64 mask; /* Bit mask of the bar */ 132 u32 bitsize; /* Bit size of the bar */ 133 int index; 134 atomic_t refcnt; 135 136 void __iomem *iomem; 137 struct resource *resource; 138 }; 139 140 #define NFP_PCI_BAR_MAX (PCI_64BIT_BAR_COUNT * 8) 141 142 struct nfp6000_pcie { 143 struct pci_dev *pdev; 144 struct device *dev; 145 const struct nfp_dev_info *dev_info; 146 147 /* PCI BAR management */ 148 spinlock_t bar_lock; /* Protect the PCI2CPP BAR cache */ 149 int bars; 150 struct nfp_bar bar[NFP_PCI_BAR_MAX]; 151 wait_queue_head_t bar_waiters; 152 153 /* Reserved BAR access */ 154 struct { 155 void __iomem *csr; 156 void __iomem *em; 157 void __iomem *expl[4]; 158 } iomem; 159 160 /* Explicit IO access */ 161 struct { 162 struct mutex mutex; /* Lock access to this explicit group */ 163 u8 master_id; 164 u8 signal_ref; 165 void __iomem *data; 166 struct { 167 void __iomem *addr; 168 int bitsize; 169 int free[4]; 170 } group[4]; 171 } expl; 172 }; 173 174 static u32 nfp_bar_maptype(struct nfp_bar *bar) 175 { 176 return NFP_PCIE_BAR_PCIE2CPP_MapType_of(bar->barcfg); 177 } 178 179 static resource_size_t nfp_bar_resource_len(struct nfp_bar *bar) 180 { 181 return pci_resource_len(bar->nfp->pdev, (bar->index / 8) * 2) / 8; 182 } 183 184 static resource_size_t nfp_bar_resource_start(struct nfp_bar *bar) 185 { 186 return pci_resource_start(bar->nfp->pdev, (bar->index / 8) * 2) 187 + nfp_bar_resource_len(bar) * (bar->index & 7); 188 } 189 190 #define TARGET_WIDTH_32 4 191 #define TARGET_WIDTH_64 8 192 193 static int 194 compute_bar(const struct nfp6000_pcie *nfp, const struct nfp_bar *bar, 195 u32 *bar_config, u64 *bar_base, 196 int tgt, int act, int tok, u64 offset, size_t size, int width) 197 { 198 int bitsize; 199 u32 newcfg; 200 201 if (tgt >= NFP_CPP_NUM_TARGETS) 202 return -EINVAL; 203 204 switch (width) { 205 case 8: 206 newcfg = NFP_PCIE_BAR_PCIE2CPP_LengthSelect( 207 NFP_PCIE_BAR_PCIE2CPP_LengthSelect_64BIT); 208 break; 209 case 4: 210 newcfg = NFP_PCIE_BAR_PCIE2CPP_LengthSelect( 211 NFP_PCIE_BAR_PCIE2CPP_LengthSelect_32BIT); 212 break; 213 case 0: 214 newcfg = NFP_PCIE_BAR_PCIE2CPP_LengthSelect( 215 NFP_PCIE_BAR_PCIE2CPP_LengthSelect_0BYTE); 216 break; 217 default: 218 return -EINVAL; 219 } 220 221 if (act != NFP_CPP_ACTION_RW && act != 0) { 222 /* Fixed CPP mapping with specific action */ 223 u64 mask = ~(NFP_PCIE_P2C_FIXED_SIZE(bar) - 1); 224 225 newcfg |= NFP_PCIE_BAR_PCIE2CPP_MapType( 226 NFP_PCIE_BAR_PCIE2CPP_MapType_FIXED); 227 newcfg |= NFP_PCIE_BAR_PCIE2CPP_Target_BaseAddress(tgt); 228 newcfg |= NFP_PCIE_BAR_PCIE2CPP_Action_BaseAddress(act); 229 newcfg |= NFP_PCIE_BAR_PCIE2CPP_Token_BaseAddress(tok); 230 231 if ((offset & mask) != ((offset + size - 1) & mask)) 232 return -EINVAL; 233 offset &= mask; 234 235 bitsize = 40 - 16; 236 } else { 237 u64 mask = ~(NFP_PCIE_P2C_BULK_SIZE(bar) - 1); 238 239 /* Bulk mapping */ 240 newcfg |= NFP_PCIE_BAR_PCIE2CPP_MapType( 241 NFP_PCIE_BAR_PCIE2CPP_MapType_BULK); 242 newcfg |= NFP_PCIE_BAR_PCIE2CPP_Target_BaseAddress(tgt); 243 newcfg |= NFP_PCIE_BAR_PCIE2CPP_Token_BaseAddress(tok); 244 245 if ((offset & mask) != ((offset + size - 1) & mask)) 246 return -EINVAL; 247 248 offset &= mask; 249 250 bitsize = 40 - 21; 251 } 252 253 if (bar->bitsize < bitsize) 254 return -EINVAL; 255 256 newcfg |= offset >> bitsize; 257 258 if (bar_base) 259 *bar_base = offset; 260 261 if (bar_config) 262 *bar_config = newcfg; 263 264 return 0; 265 } 266 267 static int 268 nfp6000_bar_write(struct nfp6000_pcie *nfp, struct nfp_bar *bar, u32 newcfg) 269 { 270 unsigned int xbar; 271 272 xbar = NFP_PCIE_P2C_EXPBAR_OFFSET(bar->index); 273 274 if (nfp->iomem.csr) { 275 writel(newcfg, nfp->iomem.csr + xbar); 276 /* Readback to ensure BAR is flushed */ 277 readl(nfp->iomem.csr + xbar); 278 } else { 279 xbar += nfp->dev_info->pcie_cfg_expbar_offset; 280 pci_write_config_dword(nfp->pdev, xbar, newcfg); 281 } 282 283 bar->barcfg = newcfg; 284 285 return 0; 286 } 287 288 static int 289 reconfigure_bar(struct nfp6000_pcie *nfp, struct nfp_bar *bar, 290 int tgt, int act, int tok, u64 offset, size_t size, int width) 291 { 292 u64 newbase; 293 u32 newcfg; 294 int err; 295 296 err = compute_bar(nfp, bar, &newcfg, &newbase, 297 tgt, act, tok, offset, size, width); 298 if (err) 299 return err; 300 301 bar->base = newbase; 302 303 return nfp6000_bar_write(nfp, bar, newcfg); 304 } 305 306 /* Check if BAR can be used with the given parameters. */ 307 static int matching_bar(struct nfp_bar *bar, u32 tgt, u32 act, u32 tok, 308 u64 offset, size_t size, int width) 309 { 310 int bartgt, baract, bartok; 311 int barwidth; 312 u32 maptype; 313 314 maptype = NFP_PCIE_BAR_PCIE2CPP_MapType_of(bar->barcfg); 315 bartgt = NFP_PCIE_BAR_PCIE2CPP_Target_BaseAddress_of(bar->barcfg); 316 bartok = NFP_PCIE_BAR_PCIE2CPP_Token_BaseAddress_of(bar->barcfg); 317 baract = NFP_PCIE_BAR_PCIE2CPP_Action_BaseAddress_of(bar->barcfg); 318 319 barwidth = NFP_PCIE_BAR_PCIE2CPP_LengthSelect_of(bar->barcfg); 320 switch (barwidth) { 321 case NFP_PCIE_BAR_PCIE2CPP_LengthSelect_32BIT: 322 barwidth = 4; 323 break; 324 case NFP_PCIE_BAR_PCIE2CPP_LengthSelect_64BIT: 325 barwidth = 8; 326 break; 327 case NFP_PCIE_BAR_PCIE2CPP_LengthSelect_0BYTE: 328 barwidth = 0; 329 break; 330 default: 331 barwidth = -1; 332 break; 333 } 334 335 switch (maptype) { 336 case NFP_PCIE_BAR_PCIE2CPP_MapType_TARGET: 337 bartok = -1; 338 fallthrough; 339 case NFP_PCIE_BAR_PCIE2CPP_MapType_BULK: 340 baract = NFP_CPP_ACTION_RW; 341 if (act == 0) 342 act = NFP_CPP_ACTION_RW; 343 fallthrough; 344 case NFP_PCIE_BAR_PCIE2CPP_MapType_FIXED: 345 break; 346 default: 347 /* We don't match explicit bars through the area interface */ 348 return 0; 349 } 350 351 /* Make sure to match up the width */ 352 if (barwidth != width) 353 return 0; 354 355 if ((bartgt < 0 || bartgt == tgt) && 356 (bartok < 0 || bartok == tok) && 357 (baract == act) && 358 bar->base <= offset && 359 (bar->base + (1 << bar->bitsize)) >= (offset + size)) 360 return 1; 361 362 /* No match */ 363 return 0; 364 } 365 366 static int 367 find_matching_bar(struct nfp6000_pcie *nfp, 368 u32 tgt, u32 act, u32 tok, u64 offset, size_t size, int width) 369 { 370 int n; 371 372 for (n = 0; n < nfp->bars; n++) { 373 struct nfp_bar *bar = &nfp->bar[n]; 374 375 if (matching_bar(bar, tgt, act, tok, offset, size, width)) 376 return n; 377 } 378 379 return -1; 380 } 381 382 /* Return EAGAIN if no resource is available */ 383 static int 384 find_unused_bar_noblock(const struct nfp6000_pcie *nfp, 385 int tgt, int act, int tok, 386 u64 offset, size_t size, int width) 387 { 388 int n, busy = 0; 389 390 for (n = 0; n < nfp->bars; n++) { 391 const struct nfp_bar *bar = &nfp->bar[n]; 392 int err; 393 394 if (!bar->bitsize) 395 continue; 396 397 /* Just check to see if we can make it fit... */ 398 err = compute_bar(nfp, bar, NULL, NULL, 399 tgt, act, tok, offset, size, width); 400 if (err) 401 continue; 402 403 if (!atomic_read(&bar->refcnt)) 404 return n; 405 406 busy++; 407 } 408 409 if (WARN(!busy, "No suitable BAR found for request tgt:0x%x act:0x%x tok:0x%x off:0x%llx size:%zd width:%d\n", 410 tgt, act, tok, offset, size, width)) 411 return -EINVAL; 412 413 return -EAGAIN; 414 } 415 416 static int 417 find_unused_bar_and_lock(struct nfp6000_pcie *nfp, 418 int tgt, int act, int tok, 419 u64 offset, size_t size, int width) 420 { 421 unsigned long flags; 422 int n; 423 424 spin_lock_irqsave(&nfp->bar_lock, flags); 425 426 n = find_unused_bar_noblock(nfp, tgt, act, tok, offset, size, width); 427 if (n < 0) 428 spin_unlock_irqrestore(&nfp->bar_lock, flags); 429 else 430 __release(&nfp->bar_lock); 431 432 return n; 433 } 434 435 static void nfp_bar_get(struct nfp6000_pcie *nfp, struct nfp_bar *bar) 436 { 437 atomic_inc(&bar->refcnt); 438 } 439 440 static void nfp_bar_put(struct nfp6000_pcie *nfp, struct nfp_bar *bar) 441 { 442 if (atomic_dec_and_test(&bar->refcnt)) 443 wake_up_interruptible(&nfp->bar_waiters); 444 } 445 446 static int 447 nfp_wait_for_bar(struct nfp6000_pcie *nfp, int *barnum, 448 u32 tgt, u32 act, u32 tok, u64 offset, size_t size, int width) 449 { 450 return wait_event_interruptible(nfp->bar_waiters, 451 (*barnum = find_unused_bar_and_lock(nfp, tgt, act, tok, 452 offset, size, width)) 453 != -EAGAIN); 454 } 455 456 static int 457 nfp_alloc_bar(struct nfp6000_pcie *nfp, 458 u32 tgt, u32 act, u32 tok, 459 u64 offset, size_t size, int width, int nonblocking) 460 { 461 unsigned long irqflags; 462 int barnum, retval; 463 464 if (size > (1 << 24)) 465 return -EINVAL; 466 467 spin_lock_irqsave(&nfp->bar_lock, irqflags); 468 barnum = find_matching_bar(nfp, tgt, act, tok, offset, size, width); 469 if (barnum >= 0) { 470 /* Found a perfect match. */ 471 nfp_bar_get(nfp, &nfp->bar[barnum]); 472 spin_unlock_irqrestore(&nfp->bar_lock, irqflags); 473 return barnum; 474 } 475 476 barnum = find_unused_bar_noblock(nfp, tgt, act, tok, 477 offset, size, width); 478 if (barnum < 0) { 479 if (nonblocking) 480 goto err_nobar; 481 482 /* Wait until a BAR becomes available. The 483 * find_unused_bar function will reclaim the bar_lock 484 * if a free BAR is found. 485 */ 486 spin_unlock_irqrestore(&nfp->bar_lock, irqflags); 487 retval = nfp_wait_for_bar(nfp, &barnum, tgt, act, tok, 488 offset, size, width); 489 if (retval) 490 return retval; 491 __acquire(&nfp->bar_lock); 492 } 493 494 nfp_bar_get(nfp, &nfp->bar[barnum]); 495 retval = reconfigure_bar(nfp, &nfp->bar[barnum], 496 tgt, act, tok, offset, size, width); 497 if (retval < 0) { 498 nfp_bar_put(nfp, &nfp->bar[barnum]); 499 barnum = retval; 500 } 501 502 err_nobar: 503 spin_unlock_irqrestore(&nfp->bar_lock, irqflags); 504 return barnum; 505 } 506 507 static void disable_bars(struct nfp6000_pcie *nfp); 508 509 static int bar_cmp(const void *aptr, const void *bptr) 510 { 511 const struct nfp_bar *a = aptr, *b = bptr; 512 513 if (a->bitsize == b->bitsize) 514 return a->index - b->index; 515 else 516 return a->bitsize - b->bitsize; 517 } 518 519 /* Map all PCI bars and fetch the actual BAR configurations from the 520 * board. We assume that the BAR with the PCIe config block is 521 * already mapped. 522 * 523 * BAR0.0: Reserved for General Mapping (for MSI-X access to PCIe SRAM) 524 * BAR0.1: Reserved for XPB access (for MSI-X access to PCIe PBA) 525 * BAR0.2: -- 526 * BAR0.3: -- 527 * BAR0.4: Reserved for Explicit 0.0-0.3 access 528 * BAR0.5: Reserved for Explicit 1.0-1.3 access 529 * BAR0.6: Reserved for Explicit 2.0-2.3 access 530 * BAR0.7: Reserved for Explicit 3.0-3.3 access 531 * 532 * BAR1.0-BAR1.7: -- 533 * BAR2.0-BAR2.7: -- 534 */ 535 static int enable_bars(struct nfp6000_pcie *nfp, u16 interface) 536 { 537 const u32 barcfg_msix_general = 538 NFP_PCIE_BAR_PCIE2CPP_MapType( 539 NFP_PCIE_BAR_PCIE2CPP_MapType_GENERAL) | 540 NFP_PCIE_BAR_PCIE2CPP_LengthSelect( 541 NFP_PCIE_BAR_PCIE2CPP_LengthSelect_32BIT); 542 const u32 barcfg_msix_xpb = 543 NFP_PCIE_BAR_PCIE2CPP_MapType( 544 NFP_PCIE_BAR_PCIE2CPP_MapType_BULK) | 545 NFP_PCIE_BAR_PCIE2CPP_LengthSelect( 546 NFP_PCIE_BAR_PCIE2CPP_LengthSelect_32BIT) | 547 NFP_PCIE_BAR_PCIE2CPP_Target_BaseAddress( 548 NFP_CPP_TARGET_ISLAND_XPB); 549 const u32 barcfg_explicit[4] = { 550 NFP_PCIE_BAR_PCIE2CPP_MapType( 551 NFP_PCIE_BAR_PCIE2CPP_MapType_EXPLICIT0), 552 NFP_PCIE_BAR_PCIE2CPP_MapType( 553 NFP_PCIE_BAR_PCIE2CPP_MapType_EXPLICIT1), 554 NFP_PCIE_BAR_PCIE2CPP_MapType( 555 NFP_PCIE_BAR_PCIE2CPP_MapType_EXPLICIT2), 556 NFP_PCIE_BAR_PCIE2CPP_MapType( 557 NFP_PCIE_BAR_PCIE2CPP_MapType_EXPLICIT3), 558 }; 559 char status_msg[196] = {}; 560 int i, err, bars_free; 561 struct nfp_bar *bar; 562 int expl_groups; 563 char *msg, *end; 564 565 msg = status_msg + 566 snprintf(status_msg, sizeof(status_msg) - 1, "RESERVED BARs: "); 567 end = status_msg + sizeof(status_msg) - 1; 568 569 bar = &nfp->bar[0]; 570 for (i = 0; i < ARRAY_SIZE(nfp->bar); i++, bar++) { 571 struct resource *res; 572 573 res = &nfp->pdev->resource[(i >> 3) * 2]; 574 575 /* Skip over BARs that are not IORESOURCE_MEM */ 576 if (!(resource_type(res) & IORESOURCE_MEM)) { 577 bar--; 578 continue; 579 } 580 581 bar->resource = res; 582 bar->barcfg = 0; 583 584 bar->nfp = nfp; 585 bar->index = i; 586 bar->mask = nfp_bar_resource_len(bar) - 1; 587 bar->bitsize = fls(bar->mask); 588 bar->base = 0; 589 bar->iomem = NULL; 590 } 591 592 nfp->bars = bar - &nfp->bar[0]; 593 if (nfp->bars < 8) { 594 dev_err(nfp->dev, "No usable BARs found!\n"); 595 return -EINVAL; 596 } 597 598 bars_free = nfp->bars; 599 600 /* Convert unit ID (0..3) to signal master/data master ID (0x40..0x70) 601 */ 602 mutex_init(&nfp->expl.mutex); 603 604 nfp->expl.master_id = ((NFP_CPP_INTERFACE_UNIT_of(interface) & 3) + 4) 605 << 4; 606 nfp->expl.signal_ref = 0x10; 607 608 /* Configure, and lock, BAR0.0 for General Target use (MSI-X SRAM) */ 609 bar = &nfp->bar[0]; 610 if (nfp_bar_resource_len(bar) >= NFP_PCI_MIN_MAP_SIZE) 611 bar->iomem = ioremap(nfp_bar_resource_start(bar), 612 nfp_bar_resource_len(bar)); 613 if (bar->iomem) { 614 int pf; 615 616 msg += scnprintf(msg, end - msg, "0.0: General/MSI-X SRAM, "); 617 atomic_inc(&bar->refcnt); 618 bars_free--; 619 620 nfp6000_bar_write(nfp, bar, barcfg_msix_general); 621 622 nfp->expl.data = bar->iomem + NFP_PCIE_SRAM + 623 nfp->dev_info->pcie_expl_offset; 624 625 switch (nfp->pdev->device) { 626 case PCI_DEVICE_ID_NFP3800: 627 pf = nfp->pdev->devfn & 7; 628 nfp->iomem.csr = bar->iomem + NFP_PCIE_BAR(pf); 629 break; 630 case PCI_DEVICE_ID_NFP4000: 631 case PCI_DEVICE_ID_NFP5000: 632 case PCI_DEVICE_ID_NFP6000: 633 nfp->iomem.csr = bar->iomem + NFP_PCIE_BAR(0); 634 break; 635 default: 636 dev_err(nfp->dev, "Unsupported device ID: %04hx!\n", 637 nfp->pdev->device); 638 err = -EINVAL; 639 goto err_unmap_bar0; 640 } 641 nfp->iomem.em = bar->iomem + NFP_PCIE_EM; 642 } 643 644 switch (nfp->pdev->device) { 645 case PCI_DEVICE_ID_NFP3800: 646 expl_groups = 1; 647 break; 648 case PCI_DEVICE_ID_NFP4000: 649 case PCI_DEVICE_ID_NFP5000: 650 case PCI_DEVICE_ID_NFP6000: 651 expl_groups = 4; 652 break; 653 default: 654 dev_err(nfp->dev, "Unsupported device ID: %04hx!\n", 655 nfp->pdev->device); 656 err = -EINVAL; 657 goto err_unmap_bar0; 658 } 659 660 /* Configure, and lock, BAR0.1 for PCIe XPB (MSI-X PBA) */ 661 bar = &nfp->bar[1]; 662 msg += scnprintf(msg, end - msg, "0.1: PCIe XPB/MSI-X PBA, "); 663 atomic_inc(&bar->refcnt); 664 bars_free--; 665 666 nfp6000_bar_write(nfp, bar, barcfg_msix_xpb); 667 668 /* Use BAR0.4..BAR0.7 for EXPL IO */ 669 for (i = 0; i < 4; i++) { 670 int j; 671 672 if (i >= NFP_PCIE_EXPLICIT_BARS || i >= expl_groups) { 673 nfp->expl.group[i].bitsize = 0; 674 continue; 675 } 676 677 bar = &nfp->bar[4 + i]; 678 bar->iomem = ioremap(nfp_bar_resource_start(bar), 679 nfp_bar_resource_len(bar)); 680 if (bar->iomem) { 681 msg += scnprintf(msg, end - msg, 682 "0.%d: Explicit%d, ", 4 + i, i); 683 atomic_inc(&bar->refcnt); 684 bars_free--; 685 686 nfp->expl.group[i].bitsize = bar->bitsize; 687 nfp->expl.group[i].addr = bar->iomem; 688 nfp6000_bar_write(nfp, bar, barcfg_explicit[i]); 689 690 for (j = 0; j < 4; j++) 691 nfp->expl.group[i].free[j] = true; 692 } 693 nfp->iomem.expl[i] = bar->iomem; 694 } 695 696 /* Sort bars by bit size - use the smallest possible first. */ 697 sort(&nfp->bar[0], nfp->bars, sizeof(nfp->bar[0]), 698 bar_cmp, NULL); 699 700 dev_info(nfp->dev, "%sfree: %d/%d\n", status_msg, bars_free, nfp->bars); 701 702 return 0; 703 704 err_unmap_bar0: 705 if (nfp->bar[0].iomem) 706 iounmap(nfp->bar[0].iomem); 707 return err; 708 } 709 710 static void disable_bars(struct nfp6000_pcie *nfp) 711 { 712 struct nfp_bar *bar = &nfp->bar[0]; 713 int n; 714 715 for (n = 0; n < nfp->bars; n++, bar++) { 716 if (bar->iomem) { 717 iounmap(bar->iomem); 718 bar->iomem = NULL; 719 } 720 } 721 } 722 723 /* 724 * Generic CPP bus access interface. 725 */ 726 727 struct nfp6000_area_priv { 728 atomic_t refcnt; 729 730 struct nfp_bar *bar; 731 u32 bar_offset; 732 733 u32 target; 734 u32 action; 735 u32 token; 736 u64 offset; 737 struct { 738 int read; 739 int write; 740 int bar; 741 } width; 742 size_t size; 743 744 void __iomem *iomem; 745 phys_addr_t phys; 746 struct resource resource; 747 }; 748 749 static int nfp6000_area_init(struct nfp_cpp_area *area, u32 dest, 750 unsigned long long address, unsigned long size) 751 { 752 struct nfp6000_area_priv *priv = nfp_cpp_area_priv(area); 753 u32 target = NFP_CPP_ID_TARGET_of(dest); 754 u32 action = NFP_CPP_ID_ACTION_of(dest); 755 u32 token = NFP_CPP_ID_TOKEN_of(dest); 756 int pp; 757 758 pp = nfp_target_pushpull(NFP_CPP_ID(target, action, token), address); 759 if (pp < 0) 760 return pp; 761 762 priv->width.read = PUSH_WIDTH(pp); 763 priv->width.write = PULL_WIDTH(pp); 764 if (priv->width.read > 0 && 765 priv->width.write > 0 && 766 priv->width.read != priv->width.write) { 767 return -EINVAL; 768 } 769 770 if (priv->width.read > 0) 771 priv->width.bar = priv->width.read; 772 else 773 priv->width.bar = priv->width.write; 774 775 atomic_set(&priv->refcnt, 0); 776 priv->bar = NULL; 777 778 priv->target = target; 779 priv->action = action; 780 priv->token = token; 781 priv->offset = address; 782 priv->size = size; 783 memset(&priv->resource, 0, sizeof(priv->resource)); 784 785 return 0; 786 } 787 788 static void nfp6000_area_cleanup(struct nfp_cpp_area *area) 789 { 790 } 791 792 static void priv_area_get(struct nfp_cpp_area *area) 793 { 794 struct nfp6000_area_priv *priv = nfp_cpp_area_priv(area); 795 796 atomic_inc(&priv->refcnt); 797 } 798 799 static int priv_area_put(struct nfp_cpp_area *area) 800 { 801 struct nfp6000_area_priv *priv = nfp_cpp_area_priv(area); 802 803 if (WARN_ON(!atomic_read(&priv->refcnt))) 804 return 0; 805 806 return atomic_dec_and_test(&priv->refcnt); 807 } 808 809 static int nfp6000_area_acquire(struct nfp_cpp_area *area) 810 { 811 struct nfp6000_pcie *nfp = nfp_cpp_priv(nfp_cpp_area_cpp(area)); 812 struct nfp6000_area_priv *priv = nfp_cpp_area_priv(area); 813 int barnum, err; 814 815 if (priv->bar) { 816 /* Already allocated. */ 817 priv_area_get(area); 818 return 0; 819 } 820 821 barnum = nfp_alloc_bar(nfp, priv->target, priv->action, priv->token, 822 priv->offset, priv->size, priv->width.bar, 1); 823 824 if (barnum < 0) { 825 err = barnum; 826 goto err_alloc_bar; 827 } 828 priv->bar = &nfp->bar[barnum]; 829 830 /* Calculate offset into BAR. */ 831 if (nfp_bar_maptype(priv->bar) == 832 NFP_PCIE_BAR_PCIE2CPP_MapType_GENERAL) { 833 priv->bar_offset = priv->offset & 834 (NFP_PCIE_P2C_GENERAL_SIZE(priv->bar) - 1); 835 priv->bar_offset += NFP_PCIE_P2C_GENERAL_TARGET_OFFSET( 836 priv->bar, priv->target); 837 priv->bar_offset += NFP_PCIE_P2C_GENERAL_TOKEN_OFFSET( 838 priv->bar, priv->token); 839 } else { 840 priv->bar_offset = priv->offset & priv->bar->mask; 841 } 842 843 /* We don't actually try to acquire the resource area using 844 * request_resource. This would prevent sharing the mapped 845 * BAR between multiple CPP areas and prevent us from 846 * effectively utilizing the limited amount of BAR resources. 847 */ 848 priv->phys = nfp_bar_resource_start(priv->bar) + priv->bar_offset; 849 priv->resource.name = nfp_cpp_area_name(area); 850 priv->resource.start = priv->phys; 851 priv->resource.end = priv->resource.start + priv->size - 1; 852 priv->resource.flags = IORESOURCE_MEM; 853 854 /* If the bar is already mapped in, use its mapping */ 855 if (priv->bar->iomem) 856 priv->iomem = priv->bar->iomem + priv->bar_offset; 857 else 858 /* Must have been too big. Sub-allocate. */ 859 priv->iomem = ioremap(priv->phys, priv->size); 860 861 if (IS_ERR_OR_NULL(priv->iomem)) { 862 dev_err(nfp->dev, "Can't ioremap() a %d byte region of BAR %d\n", 863 (int)priv->size, priv->bar->index); 864 err = !priv->iomem ? -ENOMEM : PTR_ERR(priv->iomem); 865 priv->iomem = NULL; 866 goto err_iomem_remap; 867 } 868 869 priv_area_get(area); 870 return 0; 871 872 err_iomem_remap: 873 nfp_bar_put(nfp, priv->bar); 874 priv->bar = NULL; 875 err_alloc_bar: 876 return err; 877 } 878 879 static void nfp6000_area_release(struct nfp_cpp_area *area) 880 { 881 struct nfp6000_pcie *nfp = nfp_cpp_priv(nfp_cpp_area_cpp(area)); 882 struct nfp6000_area_priv *priv = nfp_cpp_area_priv(area); 883 884 if (!priv_area_put(area)) 885 return; 886 887 if (!priv->bar->iomem) 888 iounmap(priv->iomem); 889 890 nfp_bar_put(nfp, priv->bar); 891 892 priv->bar = NULL; 893 priv->iomem = NULL; 894 } 895 896 static phys_addr_t nfp6000_area_phys(struct nfp_cpp_area *area) 897 { 898 struct nfp6000_area_priv *priv = nfp_cpp_area_priv(area); 899 900 return priv->phys; 901 } 902 903 static void __iomem *nfp6000_area_iomem(struct nfp_cpp_area *area) 904 { 905 struct nfp6000_area_priv *priv = nfp_cpp_area_priv(area); 906 907 return priv->iomem; 908 } 909 910 static struct resource *nfp6000_area_resource(struct nfp_cpp_area *area) 911 { 912 /* Use the BAR resource as the resource for the CPP area. 913 * This enables us to share the BAR among multiple CPP areas 914 * without resource conflicts. 915 */ 916 struct nfp6000_area_priv *priv = nfp_cpp_area_priv(area); 917 918 return priv->bar->resource; 919 } 920 921 static int nfp6000_area_read(struct nfp_cpp_area *area, void *kernel_vaddr, 922 unsigned long offset, unsigned int length) 923 { 924 u64 __maybe_unused *wrptr64 = kernel_vaddr; 925 const u64 __iomem __maybe_unused *rdptr64; 926 struct nfp6000_area_priv *priv; 927 u32 *wrptr32 = kernel_vaddr; 928 const u32 __iomem *rdptr32; 929 int n, width; 930 931 priv = nfp_cpp_area_priv(area); 932 rdptr64 = priv->iomem + offset; 933 rdptr32 = priv->iomem + offset; 934 935 if (offset + length > priv->size) 936 return -EFAULT; 937 938 width = priv->width.read; 939 if (width <= 0) 940 return -EINVAL; 941 942 /* MU reads via a PCIe2CPP BAR support 32bit (and other) lengths */ 943 if (priv->target == (NFP_CPP_TARGET_MU & NFP_CPP_TARGET_ID_MASK) && 944 priv->action == NFP_CPP_ACTION_RW && 945 (offset % sizeof(u64) == 4 || length % sizeof(u64) == 4)) 946 width = TARGET_WIDTH_32; 947 948 /* Unaligned? Translate to an explicit access */ 949 if ((priv->offset + offset) & (width - 1)) 950 return nfp_cpp_explicit_read(nfp_cpp_area_cpp(area), 951 NFP_CPP_ID(priv->target, 952 priv->action, 953 priv->token), 954 priv->offset + offset, 955 kernel_vaddr, length, width); 956 957 if (WARN_ON(!priv->bar)) 958 return -EFAULT; 959 960 switch (width) { 961 case TARGET_WIDTH_32: 962 if (offset % sizeof(u32) != 0 || length % sizeof(u32) != 0) 963 return -EINVAL; 964 965 for (n = 0; n < length; n += sizeof(u32)) 966 *wrptr32++ = __raw_readl(rdptr32++); 967 return n; 968 #ifdef __raw_readq 969 case TARGET_WIDTH_64: 970 if (offset % sizeof(u64) != 0 || length % sizeof(u64) != 0) 971 return -EINVAL; 972 973 for (n = 0; n < length; n += sizeof(u64)) 974 *wrptr64++ = __raw_readq(rdptr64++); 975 return n; 976 #endif 977 default: 978 return -EINVAL; 979 } 980 } 981 982 static int 983 nfp6000_area_write(struct nfp_cpp_area *area, 984 const void *kernel_vaddr, 985 unsigned long offset, unsigned int length) 986 { 987 const u64 __maybe_unused *rdptr64 = kernel_vaddr; 988 u64 __iomem __maybe_unused *wrptr64; 989 const u32 *rdptr32 = kernel_vaddr; 990 struct nfp6000_area_priv *priv; 991 u32 __iomem *wrptr32; 992 int n, width; 993 994 priv = nfp_cpp_area_priv(area); 995 wrptr64 = priv->iomem + offset; 996 wrptr32 = priv->iomem + offset; 997 998 if (offset + length > priv->size) 999 return -EFAULT; 1000 1001 width = priv->width.write; 1002 if (width <= 0) 1003 return -EINVAL; 1004 1005 /* MU writes via a PCIe2CPP BAR support 32bit (and other) lengths */ 1006 if (priv->target == (NFP_CPP_TARGET_ID_MASK & NFP_CPP_TARGET_MU) && 1007 priv->action == NFP_CPP_ACTION_RW && 1008 (offset % sizeof(u64) == 4 || length % sizeof(u64) == 4)) 1009 width = TARGET_WIDTH_32; 1010 1011 /* Unaligned? Translate to an explicit access */ 1012 if ((priv->offset + offset) & (width - 1)) 1013 return nfp_cpp_explicit_write(nfp_cpp_area_cpp(area), 1014 NFP_CPP_ID(priv->target, 1015 priv->action, 1016 priv->token), 1017 priv->offset + offset, 1018 kernel_vaddr, length, width); 1019 1020 if (WARN_ON(!priv->bar)) 1021 return -EFAULT; 1022 1023 switch (width) { 1024 case TARGET_WIDTH_32: 1025 if (offset % sizeof(u32) != 0 || length % sizeof(u32) != 0) 1026 return -EINVAL; 1027 1028 for (n = 0; n < length; n += sizeof(u32)) { 1029 __raw_writel(*rdptr32++, wrptr32++); 1030 wmb(); 1031 } 1032 return n; 1033 #ifdef __raw_writeq 1034 case TARGET_WIDTH_64: 1035 if (offset % sizeof(u64) != 0 || length % sizeof(u64) != 0) 1036 return -EINVAL; 1037 1038 for (n = 0; n < length; n += sizeof(u64)) { 1039 __raw_writeq(*rdptr64++, wrptr64++); 1040 wmb(); 1041 } 1042 return n; 1043 #endif 1044 default: 1045 return -EINVAL; 1046 } 1047 } 1048 1049 struct nfp6000_explicit_priv { 1050 struct nfp6000_pcie *nfp; 1051 struct { 1052 int group; 1053 int area; 1054 } bar; 1055 int bitsize; 1056 void __iomem *data; 1057 void __iomem *addr; 1058 }; 1059 1060 static int nfp6000_explicit_acquire(struct nfp_cpp_explicit *expl) 1061 { 1062 struct nfp6000_pcie *nfp = nfp_cpp_priv(nfp_cpp_explicit_cpp(expl)); 1063 struct nfp6000_explicit_priv *priv = nfp_cpp_explicit_priv(expl); 1064 int i, j; 1065 1066 mutex_lock(&nfp->expl.mutex); 1067 for (i = 0; i < ARRAY_SIZE(nfp->expl.group); i++) { 1068 if (!nfp->expl.group[i].bitsize) 1069 continue; 1070 1071 for (j = 0; j < ARRAY_SIZE(nfp->expl.group[i].free); j++) { 1072 u16 data_offset; 1073 1074 if (!nfp->expl.group[i].free[j]) 1075 continue; 1076 1077 priv->nfp = nfp; 1078 priv->bar.group = i; 1079 priv->bar.area = j; 1080 priv->bitsize = nfp->expl.group[i].bitsize - 2; 1081 1082 data_offset = (priv->bar.group << 9) + 1083 (priv->bar.area << 7); 1084 priv->data = nfp->expl.data + data_offset; 1085 priv->addr = nfp->expl.group[i].addr + 1086 (priv->bar.area << priv->bitsize); 1087 nfp->expl.group[i].free[j] = false; 1088 1089 mutex_unlock(&nfp->expl.mutex); 1090 return 0; 1091 } 1092 } 1093 mutex_unlock(&nfp->expl.mutex); 1094 1095 return -EAGAIN; 1096 } 1097 1098 static void nfp6000_explicit_release(struct nfp_cpp_explicit *expl) 1099 { 1100 struct nfp6000_explicit_priv *priv = nfp_cpp_explicit_priv(expl); 1101 struct nfp6000_pcie *nfp = priv->nfp; 1102 1103 mutex_lock(&nfp->expl.mutex); 1104 nfp->expl.group[priv->bar.group].free[priv->bar.area] = true; 1105 mutex_unlock(&nfp->expl.mutex); 1106 } 1107 1108 static int nfp6000_explicit_put(struct nfp_cpp_explicit *expl, 1109 const void *buff, size_t len) 1110 { 1111 struct nfp6000_explicit_priv *priv = nfp_cpp_explicit_priv(expl); 1112 const u32 *src = buff; 1113 size_t i; 1114 1115 for (i = 0; i < len; i += sizeof(u32)) 1116 writel(*(src++), priv->data + i); 1117 1118 return i; 1119 } 1120 1121 static int 1122 nfp6000_explicit_do(struct nfp_cpp_explicit *expl, 1123 const struct nfp_cpp_explicit_command *cmd, u64 address) 1124 { 1125 struct nfp6000_explicit_priv *priv = nfp_cpp_explicit_priv(expl); 1126 u8 signal_master, signal_ref, data_master; 1127 struct nfp6000_pcie *nfp = priv->nfp; 1128 int sigmask = 0; 1129 u16 data_ref; 1130 u32 csr[3]; 1131 1132 if (cmd->siga_mode) 1133 sigmask |= 1 << cmd->siga; 1134 if (cmd->sigb_mode) 1135 sigmask |= 1 << cmd->sigb; 1136 1137 signal_master = cmd->signal_master; 1138 if (!signal_master) 1139 signal_master = nfp->expl.master_id; 1140 1141 signal_ref = cmd->signal_ref; 1142 if (signal_master == nfp->expl.master_id) 1143 signal_ref = nfp->expl.signal_ref + 1144 ((priv->bar.group * 4 + priv->bar.area) << 1); 1145 1146 data_master = cmd->data_master; 1147 if (!data_master) 1148 data_master = nfp->expl.master_id; 1149 1150 data_ref = cmd->data_ref; 1151 if (data_master == nfp->expl.master_id) 1152 data_ref = 0x1000 + 1153 (priv->bar.group << 9) + (priv->bar.area << 7); 1154 1155 csr[0] = NFP_PCIE_BAR_EXPLICIT_BAR0_SignalType(sigmask) | 1156 NFP_PCIE_BAR_EXPLICIT_BAR0_Token( 1157 NFP_CPP_ID_TOKEN_of(cmd->cpp_id)) | 1158 NFP_PCIE_BAR_EXPLICIT_BAR0_Address(address >> 16); 1159 1160 csr[1] = NFP_PCIE_BAR_EXPLICIT_BAR1_SignalRef(signal_ref) | 1161 NFP_PCIE_BAR_EXPLICIT_BAR1_DataMaster(data_master) | 1162 NFP_PCIE_BAR_EXPLICIT_BAR1_DataRef(data_ref); 1163 1164 csr[2] = NFP_PCIE_BAR_EXPLICIT_BAR2_Target( 1165 NFP_CPP_ID_TARGET_of(cmd->cpp_id)) | 1166 NFP_PCIE_BAR_EXPLICIT_BAR2_Action( 1167 NFP_CPP_ID_ACTION_of(cmd->cpp_id)) | 1168 NFP_PCIE_BAR_EXPLICIT_BAR2_Length(cmd->len) | 1169 NFP_PCIE_BAR_EXPLICIT_BAR2_ByteMask(cmd->byte_mask) | 1170 NFP_PCIE_BAR_EXPLICIT_BAR2_SignalMaster(signal_master); 1171 1172 if (nfp->iomem.csr) { 1173 writel(csr[0], nfp->iomem.csr + 1174 NFP_PCIE_BAR_EXPLICIT_BAR0(priv->bar.group, 1175 priv->bar.area)); 1176 writel(csr[1], nfp->iomem.csr + 1177 NFP_PCIE_BAR_EXPLICIT_BAR1(priv->bar.group, 1178 priv->bar.area)); 1179 writel(csr[2], nfp->iomem.csr + 1180 NFP_PCIE_BAR_EXPLICIT_BAR2(priv->bar.group, 1181 priv->bar.area)); 1182 /* Readback to ensure BAR is flushed */ 1183 readl(nfp->iomem.csr + 1184 NFP_PCIE_BAR_EXPLICIT_BAR0(priv->bar.group, 1185 priv->bar.area)); 1186 readl(nfp->iomem.csr + 1187 NFP_PCIE_BAR_EXPLICIT_BAR1(priv->bar.group, 1188 priv->bar.area)); 1189 readl(nfp->iomem.csr + 1190 NFP_PCIE_BAR_EXPLICIT_BAR2(priv->bar.group, 1191 priv->bar.area)); 1192 } else { 1193 pci_write_config_dword(nfp->pdev, 0x400 + 1194 NFP_PCIE_BAR_EXPLICIT_BAR0( 1195 priv->bar.group, priv->bar.area), 1196 csr[0]); 1197 1198 pci_write_config_dword(nfp->pdev, 0x400 + 1199 NFP_PCIE_BAR_EXPLICIT_BAR1( 1200 priv->bar.group, priv->bar.area), 1201 csr[1]); 1202 1203 pci_write_config_dword(nfp->pdev, 0x400 + 1204 NFP_PCIE_BAR_EXPLICIT_BAR2( 1205 priv->bar.group, priv->bar.area), 1206 csr[2]); 1207 } 1208 1209 /* Issue the 'kickoff' transaction */ 1210 readb(priv->addr + (address & ((1 << priv->bitsize) - 1))); 1211 1212 return sigmask; 1213 } 1214 1215 static int nfp6000_explicit_get(struct nfp_cpp_explicit *expl, 1216 void *buff, size_t len) 1217 { 1218 struct nfp6000_explicit_priv *priv = nfp_cpp_explicit_priv(expl); 1219 u32 *dst = buff; 1220 size_t i; 1221 1222 for (i = 0; i < len; i += sizeof(u32)) 1223 *(dst++) = readl(priv->data + i); 1224 1225 return i; 1226 } 1227 1228 static int nfp6000_init(struct nfp_cpp *cpp) 1229 { 1230 nfp_cpp_area_cache_add(cpp, SZ_64K); 1231 nfp_cpp_area_cache_add(cpp, SZ_64K); 1232 nfp_cpp_area_cache_add(cpp, SZ_256K); 1233 1234 return 0; 1235 } 1236 1237 static void nfp6000_free(struct nfp_cpp *cpp) 1238 { 1239 struct nfp6000_pcie *nfp = nfp_cpp_priv(cpp); 1240 1241 disable_bars(nfp); 1242 kfree(nfp); 1243 } 1244 1245 static int nfp6000_read_serial(struct device *dev, u8 *serial) 1246 { 1247 struct pci_dev *pdev = to_pci_dev(dev); 1248 u64 dsn; 1249 1250 dsn = pci_get_dsn(pdev); 1251 if (!dsn) { 1252 dev_err(dev, "can't find PCIe Serial Number Capability\n"); 1253 return -EINVAL; 1254 } 1255 1256 put_unaligned_be32((u32)(dsn >> 32), serial); 1257 put_unaligned_be16((u16)(dsn >> 16), serial + 4); 1258 1259 return 0; 1260 } 1261 1262 static int nfp6000_get_interface(struct device *dev) 1263 { 1264 struct pci_dev *pdev = to_pci_dev(dev); 1265 u64 dsn; 1266 1267 dsn = pci_get_dsn(pdev); 1268 if (!dsn) { 1269 dev_err(dev, "can't find PCIe Serial Number Capability\n"); 1270 return -EINVAL; 1271 } 1272 1273 return dsn & 0xffff; 1274 } 1275 1276 static const struct nfp_cpp_operations nfp6000_pcie_ops = { 1277 .owner = THIS_MODULE, 1278 1279 .init = nfp6000_init, 1280 .free = nfp6000_free, 1281 1282 .read_serial = nfp6000_read_serial, 1283 .get_interface = nfp6000_get_interface, 1284 1285 .area_priv_size = sizeof(struct nfp6000_area_priv), 1286 .area_init = nfp6000_area_init, 1287 .area_cleanup = nfp6000_area_cleanup, 1288 .area_acquire = nfp6000_area_acquire, 1289 .area_release = nfp6000_area_release, 1290 .area_phys = nfp6000_area_phys, 1291 .area_iomem = nfp6000_area_iomem, 1292 .area_resource = nfp6000_area_resource, 1293 .area_read = nfp6000_area_read, 1294 .area_write = nfp6000_area_write, 1295 1296 .explicit_priv_size = sizeof(struct nfp6000_explicit_priv), 1297 .explicit_acquire = nfp6000_explicit_acquire, 1298 .explicit_release = nfp6000_explicit_release, 1299 .explicit_put = nfp6000_explicit_put, 1300 .explicit_do = nfp6000_explicit_do, 1301 .explicit_get = nfp6000_explicit_get, 1302 }; 1303 1304 /** 1305 * nfp_cpp_from_nfp6000_pcie() - Build a NFP CPP bus from a NFP6000 PCI device 1306 * @pdev: NFP6000 PCI device 1307 * @dev_info: NFP ASIC params 1308 * 1309 * Return: NFP CPP handle 1310 */ 1311 struct nfp_cpp * 1312 nfp_cpp_from_nfp6000_pcie(struct pci_dev *pdev, const struct nfp_dev_info *dev_info) 1313 { 1314 struct nfp6000_pcie *nfp; 1315 u16 interface; 1316 int err; 1317 1318 /* Finished with card initialization. */ 1319 dev_info(&pdev->dev, "Network Flow Processor %s PCIe Card Probe\n", 1320 dev_info->chip_names); 1321 pcie_print_link_status(pdev); 1322 1323 nfp = kzalloc(sizeof(*nfp), GFP_KERNEL); 1324 if (!nfp) { 1325 err = -ENOMEM; 1326 goto err_ret; 1327 } 1328 1329 nfp->dev = &pdev->dev; 1330 nfp->pdev = pdev; 1331 nfp->dev_info = dev_info; 1332 init_waitqueue_head(&nfp->bar_waiters); 1333 spin_lock_init(&nfp->bar_lock); 1334 1335 interface = nfp6000_get_interface(&pdev->dev); 1336 1337 if (NFP_CPP_INTERFACE_TYPE_of(interface) != 1338 NFP_CPP_INTERFACE_TYPE_PCI) { 1339 dev_err(&pdev->dev, 1340 "Interface type %d is not the expected %d\n", 1341 NFP_CPP_INTERFACE_TYPE_of(interface), 1342 NFP_CPP_INTERFACE_TYPE_PCI); 1343 err = -ENODEV; 1344 goto err_free_nfp; 1345 } 1346 1347 if (NFP_CPP_INTERFACE_CHANNEL_of(interface) != 1348 NFP_CPP_INTERFACE_CHANNEL_PEROPENER) { 1349 dev_err(&pdev->dev, "Interface channel %d is not the expected %d\n", 1350 NFP_CPP_INTERFACE_CHANNEL_of(interface), 1351 NFP_CPP_INTERFACE_CHANNEL_PEROPENER); 1352 err = -ENODEV; 1353 goto err_free_nfp; 1354 } 1355 1356 err = enable_bars(nfp, interface); 1357 if (err) 1358 goto err_free_nfp; 1359 1360 /* Probe for all the common NFP devices */ 1361 return nfp_cpp_from_operations(&nfp6000_pcie_ops, &pdev->dev, nfp); 1362 1363 err_free_nfp: 1364 kfree(nfp); 1365 err_ret: 1366 dev_err(&pdev->dev, "NFP6000 PCI setup failed\n"); 1367 return ERR_PTR(err); 1368 } 1369