1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2014 Hauke Mehrtens <hauke@hauke-m.de> 4 * Copyright (C) 2015 Broadcom Corporation 5 */ 6 7 #include <linux/kernel.h> 8 #include <linux/pci.h> 9 #include <linux/pci-ecam.h> 10 #include <linux/msi.h> 11 #include <linux/clk.h> 12 #include <linux/module.h> 13 #include <linux/mbus.h> 14 #include <linux/slab.h> 15 #include <linux/delay.h> 16 #include <linux/interrupt.h> 17 #include <linux/irqchip/arm-gic-v3.h> 18 #include <linux/platform_device.h> 19 #include <linux/of_address.h> 20 #include <linux/of_pci.h> 21 #include <linux/of_irq.h> 22 #include <linux/of_platform.h> 23 #include <linux/phy/phy.h> 24 25 #include "pcie-iproc.h" 26 27 #define EP_PERST_SOURCE_SELECT_SHIFT 2 28 #define EP_PERST_SOURCE_SELECT BIT(EP_PERST_SOURCE_SELECT_SHIFT) 29 #define EP_MODE_SURVIVE_PERST_SHIFT 1 30 #define EP_MODE_SURVIVE_PERST BIT(EP_MODE_SURVIVE_PERST_SHIFT) 31 #define RC_PCIE_RST_OUTPUT_SHIFT 0 32 #define RC_PCIE_RST_OUTPUT BIT(RC_PCIE_RST_OUTPUT_SHIFT) 33 #define PAXC_RESET_MASK 0x7f 34 35 #define GIC_V3_CFG_SHIFT 0 36 #define GIC_V3_CFG BIT(GIC_V3_CFG_SHIFT) 37 38 #define MSI_ENABLE_CFG_SHIFT 0 39 #define MSI_ENABLE_CFG BIT(MSI_ENABLE_CFG_SHIFT) 40 41 #define CFG_IND_ADDR_MASK 0x00001ffc 42 43 #define CFG_ADDR_REG_NUM_MASK 0x00000ffc 44 #define CFG_ADDR_CFG_TYPE_1 1 45 46 #define SYS_RC_INTX_MASK 0xf 47 48 #define PCIE_PHYLINKUP_SHIFT 3 49 #define PCIE_PHYLINKUP BIT(PCIE_PHYLINKUP_SHIFT) 50 #define PCIE_DL_ACTIVE_SHIFT 2 51 #define PCIE_DL_ACTIVE BIT(PCIE_DL_ACTIVE_SHIFT) 52 53 #define APB_ERR_EN_SHIFT 0 54 #define APB_ERR_EN BIT(APB_ERR_EN_SHIFT) 55 56 #define CFG_RD_SUCCESS 0 57 #define CFG_RD_UR 1 58 #define CFG_RD_CRS 2 59 #define CFG_RD_CA 3 60 #define CFG_RETRY_STATUS 0xffff0001 61 #define CFG_RETRY_STATUS_TIMEOUT_US 500000 /* 500 milliseconds */ 62 63 /* derive the enum index of the outbound/inbound mapping registers */ 64 #define MAP_REG(base_reg, index) ((base_reg) + (index) * 2) 65 66 /* 67 * Maximum number of outbound mapping window sizes that can be supported by any 68 * OARR/OMAP mapping pair 69 */ 70 #define MAX_NUM_OB_WINDOW_SIZES 4 71 72 #define OARR_VALID_SHIFT 0 73 #define OARR_VALID BIT(OARR_VALID_SHIFT) 74 #define OARR_SIZE_CFG_SHIFT 1 75 76 /* 77 * Maximum number of inbound mapping region sizes that can be supported by an 78 * IARR 79 */ 80 #define MAX_NUM_IB_REGION_SIZES 9 81 82 #define IMAP_VALID_SHIFT 0 83 #define IMAP_VALID BIT(IMAP_VALID_SHIFT) 84 85 #define IPROC_PCI_PM_CAP 0x48 86 #define IPROC_PCI_PM_CAP_MASK 0xffff 87 #define IPROC_PCI_EXP_CAP 0xac 88 89 #define IPROC_PCIE_REG_INVALID 0xffff 90 91 /** 92 * struct iproc_pcie_ob_map - iProc PCIe outbound mapping controller-specific 93 * parameters 94 * @window_sizes: list of supported outbound mapping window sizes in MB 95 * @nr_sizes: number of supported outbound mapping window sizes 96 */ 97 struct iproc_pcie_ob_map { 98 resource_size_t window_sizes[MAX_NUM_OB_WINDOW_SIZES]; 99 unsigned int nr_sizes; 100 }; 101 102 static const struct iproc_pcie_ob_map paxb_ob_map[] = { 103 { 104 /* OARR0/OMAP0 */ 105 .window_sizes = { 128, 256 }, 106 .nr_sizes = 2, 107 }, 108 { 109 /* OARR1/OMAP1 */ 110 .window_sizes = { 128, 256 }, 111 .nr_sizes = 2, 112 }, 113 }; 114 115 static const struct iproc_pcie_ob_map paxb_v2_ob_map[] = { 116 { 117 /* OARR0/OMAP0 */ 118 .window_sizes = { 128, 256 }, 119 .nr_sizes = 2, 120 }, 121 { 122 /* OARR1/OMAP1 */ 123 .window_sizes = { 128, 256 }, 124 .nr_sizes = 2, 125 }, 126 { 127 /* OARR2/OMAP2 */ 128 .window_sizes = { 128, 256, 512, 1024 }, 129 .nr_sizes = 4, 130 }, 131 { 132 /* OARR3/OMAP3 */ 133 .window_sizes = { 128, 256, 512, 1024 }, 134 .nr_sizes = 4, 135 }, 136 }; 137 138 /** 139 * enum iproc_pcie_ib_map_type - iProc PCIe inbound mapping type 140 * @IPROC_PCIE_IB_MAP_MEM: DDR memory 141 * @IPROC_PCIE_IB_MAP_IO: device I/O memory 142 * @IPROC_PCIE_IB_MAP_INVALID: invalid or unused 143 */ 144 enum iproc_pcie_ib_map_type { 145 IPROC_PCIE_IB_MAP_MEM = 0, 146 IPROC_PCIE_IB_MAP_IO, 147 IPROC_PCIE_IB_MAP_INVALID 148 }; 149 150 /** 151 * struct iproc_pcie_ib_map - iProc PCIe inbound mapping controller-specific 152 * parameters 153 * @type: inbound mapping region type 154 * @size_unit: inbound mapping region size unit, could be SZ_1K, SZ_1M, or 155 * SZ_1G 156 * @region_sizes: list of supported inbound mapping region sizes in KB, MB, or 157 * GB, depending on the size unit 158 * @nr_sizes: number of supported inbound mapping region sizes 159 * @nr_windows: number of supported inbound mapping windows for the region 160 * @imap_addr_offset: register offset between the upper and lower 32-bit 161 * IMAP address registers 162 * @imap_window_offset: register offset between each IMAP window 163 */ 164 struct iproc_pcie_ib_map { 165 enum iproc_pcie_ib_map_type type; 166 unsigned int size_unit; 167 resource_size_t region_sizes[MAX_NUM_IB_REGION_SIZES]; 168 unsigned int nr_sizes; 169 unsigned int nr_windows; 170 u16 imap_addr_offset; 171 u16 imap_window_offset; 172 }; 173 174 static const struct iproc_pcie_ib_map paxb_v2_ib_map[] = { 175 { 176 /* IARR0/IMAP0 */ 177 .type = IPROC_PCIE_IB_MAP_IO, 178 .size_unit = SZ_1K, 179 .region_sizes = { 32 }, 180 .nr_sizes = 1, 181 .nr_windows = 8, 182 .imap_addr_offset = 0x40, 183 .imap_window_offset = 0x4, 184 }, 185 { 186 /* IARR1/IMAP1 */ 187 .type = IPROC_PCIE_IB_MAP_MEM, 188 .size_unit = SZ_1M, 189 .region_sizes = { 8 }, 190 .nr_sizes = 1, 191 .nr_windows = 8, 192 .imap_addr_offset = 0x4, 193 .imap_window_offset = 0x8, 194 195 }, 196 { 197 /* IARR2/IMAP2 */ 198 .type = IPROC_PCIE_IB_MAP_MEM, 199 .size_unit = SZ_1M, 200 .region_sizes = { 64, 128, 256, 512, 1024, 2048, 4096, 8192, 201 16384 }, 202 .nr_sizes = 9, 203 .nr_windows = 1, 204 .imap_addr_offset = 0x4, 205 .imap_window_offset = 0x8, 206 }, 207 { 208 /* IARR3/IMAP3 */ 209 .type = IPROC_PCIE_IB_MAP_MEM, 210 .size_unit = SZ_1G, 211 .region_sizes = { 1, 2, 4, 8, 16, 32 }, 212 .nr_sizes = 6, 213 .nr_windows = 8, 214 .imap_addr_offset = 0x4, 215 .imap_window_offset = 0x8, 216 }, 217 { 218 /* IARR4/IMAP4 */ 219 .type = IPROC_PCIE_IB_MAP_MEM, 220 .size_unit = SZ_1G, 221 .region_sizes = { 32, 64, 128, 256, 512 }, 222 .nr_sizes = 5, 223 .nr_windows = 8, 224 .imap_addr_offset = 0x4, 225 .imap_window_offset = 0x8, 226 }, 227 }; 228 229 /* 230 * iProc PCIe host registers 231 */ 232 enum iproc_pcie_reg { 233 /* clock/reset signal control */ 234 IPROC_PCIE_CLK_CTRL = 0, 235 236 /* 237 * To allow MSI to be steered to an external MSI controller (e.g., ARM 238 * GICv3 ITS) 239 */ 240 IPROC_PCIE_MSI_GIC_MODE, 241 242 /* 243 * IPROC_PCIE_MSI_BASE_ADDR and IPROC_PCIE_MSI_WINDOW_SIZE define the 244 * window where the MSI posted writes are written, for the writes to be 245 * interpreted as MSI writes. 246 */ 247 IPROC_PCIE_MSI_BASE_ADDR, 248 IPROC_PCIE_MSI_WINDOW_SIZE, 249 250 /* 251 * To hold the address of the register where the MSI writes are 252 * programmed. When ARM GICv3 ITS is used, this should be programmed 253 * with the address of the GITS_TRANSLATER register. 254 */ 255 IPROC_PCIE_MSI_ADDR_LO, 256 IPROC_PCIE_MSI_ADDR_HI, 257 258 /* enable MSI */ 259 IPROC_PCIE_MSI_EN_CFG, 260 261 /* allow access to root complex configuration space */ 262 IPROC_PCIE_CFG_IND_ADDR, 263 IPROC_PCIE_CFG_IND_DATA, 264 265 /* allow access to device configuration space */ 266 IPROC_PCIE_CFG_ADDR, 267 IPROC_PCIE_CFG_DATA, 268 269 /* enable INTx */ 270 IPROC_PCIE_INTX_EN, 271 272 /* outbound address mapping */ 273 IPROC_PCIE_OARR0, 274 IPROC_PCIE_OMAP0, 275 IPROC_PCIE_OARR1, 276 IPROC_PCIE_OMAP1, 277 IPROC_PCIE_OARR2, 278 IPROC_PCIE_OMAP2, 279 IPROC_PCIE_OARR3, 280 IPROC_PCIE_OMAP3, 281 282 /* inbound address mapping */ 283 IPROC_PCIE_IARR0, 284 IPROC_PCIE_IMAP0, 285 IPROC_PCIE_IARR1, 286 IPROC_PCIE_IMAP1, 287 IPROC_PCIE_IARR2, 288 IPROC_PCIE_IMAP2, 289 IPROC_PCIE_IARR3, 290 IPROC_PCIE_IMAP3, 291 IPROC_PCIE_IARR4, 292 IPROC_PCIE_IMAP4, 293 294 /* config read status */ 295 IPROC_PCIE_CFG_RD_STATUS, 296 297 /* link status */ 298 IPROC_PCIE_LINK_STATUS, 299 300 /* enable APB error for unsupported requests */ 301 IPROC_PCIE_APB_ERR_EN, 302 303 /* total number of core registers */ 304 IPROC_PCIE_MAX_NUM_REG, 305 }; 306 307 /* iProc PCIe PAXB BCMA registers */ 308 static const u16 iproc_pcie_reg_paxb_bcma[IPROC_PCIE_MAX_NUM_REG] = { 309 [IPROC_PCIE_CLK_CTRL] = 0x000, 310 [IPROC_PCIE_CFG_IND_ADDR] = 0x120, 311 [IPROC_PCIE_CFG_IND_DATA] = 0x124, 312 [IPROC_PCIE_CFG_ADDR] = 0x1f8, 313 [IPROC_PCIE_CFG_DATA] = 0x1fc, 314 [IPROC_PCIE_INTX_EN] = 0x330, 315 [IPROC_PCIE_LINK_STATUS] = 0xf0c, 316 }; 317 318 /* iProc PCIe PAXB registers */ 319 static const u16 iproc_pcie_reg_paxb[IPROC_PCIE_MAX_NUM_REG] = { 320 [IPROC_PCIE_CLK_CTRL] = 0x000, 321 [IPROC_PCIE_CFG_IND_ADDR] = 0x120, 322 [IPROC_PCIE_CFG_IND_DATA] = 0x124, 323 [IPROC_PCIE_CFG_ADDR] = 0x1f8, 324 [IPROC_PCIE_CFG_DATA] = 0x1fc, 325 [IPROC_PCIE_INTX_EN] = 0x330, 326 [IPROC_PCIE_OARR0] = 0xd20, 327 [IPROC_PCIE_OMAP0] = 0xd40, 328 [IPROC_PCIE_OARR1] = 0xd28, 329 [IPROC_PCIE_OMAP1] = 0xd48, 330 [IPROC_PCIE_LINK_STATUS] = 0xf0c, 331 [IPROC_PCIE_APB_ERR_EN] = 0xf40, 332 }; 333 334 /* iProc PCIe PAXB v2 registers */ 335 static const u16 iproc_pcie_reg_paxb_v2[IPROC_PCIE_MAX_NUM_REG] = { 336 [IPROC_PCIE_CLK_CTRL] = 0x000, 337 [IPROC_PCIE_CFG_IND_ADDR] = 0x120, 338 [IPROC_PCIE_CFG_IND_DATA] = 0x124, 339 [IPROC_PCIE_CFG_ADDR] = 0x1f8, 340 [IPROC_PCIE_CFG_DATA] = 0x1fc, 341 [IPROC_PCIE_INTX_EN] = 0x330, 342 [IPROC_PCIE_OARR0] = 0xd20, 343 [IPROC_PCIE_OMAP0] = 0xd40, 344 [IPROC_PCIE_OARR1] = 0xd28, 345 [IPROC_PCIE_OMAP1] = 0xd48, 346 [IPROC_PCIE_OARR2] = 0xd60, 347 [IPROC_PCIE_OMAP2] = 0xd68, 348 [IPROC_PCIE_OARR3] = 0xdf0, 349 [IPROC_PCIE_OMAP3] = 0xdf8, 350 [IPROC_PCIE_IARR0] = 0xd00, 351 [IPROC_PCIE_IMAP0] = 0xc00, 352 [IPROC_PCIE_IARR1] = 0xd08, 353 [IPROC_PCIE_IMAP1] = 0xd70, 354 [IPROC_PCIE_IARR2] = 0xd10, 355 [IPROC_PCIE_IMAP2] = 0xcc0, 356 [IPROC_PCIE_IARR3] = 0xe00, 357 [IPROC_PCIE_IMAP3] = 0xe08, 358 [IPROC_PCIE_IARR4] = 0xe68, 359 [IPROC_PCIE_IMAP4] = 0xe70, 360 [IPROC_PCIE_CFG_RD_STATUS] = 0xee0, 361 [IPROC_PCIE_LINK_STATUS] = 0xf0c, 362 [IPROC_PCIE_APB_ERR_EN] = 0xf40, 363 }; 364 365 /* iProc PCIe PAXC v1 registers */ 366 static const u16 iproc_pcie_reg_paxc[IPROC_PCIE_MAX_NUM_REG] = { 367 [IPROC_PCIE_CLK_CTRL] = 0x000, 368 [IPROC_PCIE_CFG_IND_ADDR] = 0x1f0, 369 [IPROC_PCIE_CFG_IND_DATA] = 0x1f4, 370 [IPROC_PCIE_CFG_ADDR] = 0x1f8, 371 [IPROC_PCIE_CFG_DATA] = 0x1fc, 372 }; 373 374 /* iProc PCIe PAXC v2 registers */ 375 static const u16 iproc_pcie_reg_paxc_v2[IPROC_PCIE_MAX_NUM_REG] = { 376 [IPROC_PCIE_MSI_GIC_MODE] = 0x050, 377 [IPROC_PCIE_MSI_BASE_ADDR] = 0x074, 378 [IPROC_PCIE_MSI_WINDOW_SIZE] = 0x078, 379 [IPROC_PCIE_MSI_ADDR_LO] = 0x07c, 380 [IPROC_PCIE_MSI_ADDR_HI] = 0x080, 381 [IPROC_PCIE_MSI_EN_CFG] = 0x09c, 382 [IPROC_PCIE_CFG_IND_ADDR] = 0x1f0, 383 [IPROC_PCIE_CFG_IND_DATA] = 0x1f4, 384 [IPROC_PCIE_CFG_ADDR] = 0x1f8, 385 [IPROC_PCIE_CFG_DATA] = 0x1fc, 386 }; 387 388 /* 389 * List of device IDs of controllers that have corrupted capability list that 390 * require SW fixup 391 */ 392 static const u16 iproc_pcie_corrupt_cap_did[] = { 393 0x16cd, 394 0x16f0, 395 0xd802, 396 0xd804 397 }; 398 399 static inline struct iproc_pcie *iproc_data(struct pci_bus *bus) 400 { 401 struct iproc_pcie *pcie = bus->sysdata; 402 return pcie; 403 } 404 405 static inline bool iproc_pcie_reg_is_invalid(u16 reg_offset) 406 { 407 return !!(reg_offset == IPROC_PCIE_REG_INVALID); 408 } 409 410 static inline u16 iproc_pcie_reg_offset(struct iproc_pcie *pcie, 411 enum iproc_pcie_reg reg) 412 { 413 return pcie->reg_offsets[reg]; 414 } 415 416 static inline u32 iproc_pcie_read_reg(struct iproc_pcie *pcie, 417 enum iproc_pcie_reg reg) 418 { 419 u16 offset = iproc_pcie_reg_offset(pcie, reg); 420 421 if (iproc_pcie_reg_is_invalid(offset)) 422 return 0; 423 424 return readl(pcie->base + offset); 425 } 426 427 static inline void iproc_pcie_write_reg(struct iproc_pcie *pcie, 428 enum iproc_pcie_reg reg, u32 val) 429 { 430 u16 offset = iproc_pcie_reg_offset(pcie, reg); 431 432 if (iproc_pcie_reg_is_invalid(offset)) 433 return; 434 435 writel(val, pcie->base + offset); 436 } 437 438 /* 439 * APB error forwarding can be disabled during access of configuration 440 * registers of the endpoint device, to prevent unsupported requests 441 * (typically seen during enumeration with multi-function devices) from 442 * triggering a system exception. 443 */ 444 static inline void iproc_pcie_apb_err_disable(struct pci_bus *bus, 445 bool disable) 446 { 447 struct iproc_pcie *pcie = iproc_data(bus); 448 u32 val; 449 450 if (bus->number && pcie->has_apb_err_disable) { 451 val = iproc_pcie_read_reg(pcie, IPROC_PCIE_APB_ERR_EN); 452 if (disable) 453 val &= ~APB_ERR_EN; 454 else 455 val |= APB_ERR_EN; 456 iproc_pcie_write_reg(pcie, IPROC_PCIE_APB_ERR_EN, val); 457 } 458 } 459 460 static void __iomem *iproc_pcie_map_ep_cfg_reg(struct iproc_pcie *pcie, 461 unsigned int busno, 462 unsigned int devfn, 463 int where) 464 { 465 u16 offset; 466 u32 val; 467 468 /* EP device access */ 469 val = ALIGN_DOWN(PCIE_ECAM_OFFSET(busno, devfn, where), 4) | 470 CFG_ADDR_CFG_TYPE_1; 471 472 iproc_pcie_write_reg(pcie, IPROC_PCIE_CFG_ADDR, val); 473 offset = iproc_pcie_reg_offset(pcie, IPROC_PCIE_CFG_DATA); 474 475 if (iproc_pcie_reg_is_invalid(offset)) 476 return NULL; 477 478 return (pcie->base + offset); 479 } 480 481 static unsigned int iproc_pcie_cfg_retry(struct iproc_pcie *pcie, 482 void __iomem *cfg_data_p) 483 { 484 int timeout = CFG_RETRY_STATUS_TIMEOUT_US; 485 unsigned int data; 486 u32 status; 487 488 /* 489 * As per PCIe spec r3.1, sec 2.3.2, CRS Software Visibility only 490 * affects config reads of the Vendor ID. For config writes or any 491 * other config reads, the Root may automatically reissue the 492 * configuration request again as a new request. 493 * 494 * For config reads, this hardware returns CFG_RETRY_STATUS data 495 * when it receives a CRS completion, regardless of the address of 496 * the read or the CRS Software Visibility Enable bit. As a 497 * partial workaround for this, we retry in software any read that 498 * returns CFG_RETRY_STATUS. 499 * 500 * Note that a non-Vendor ID config register may have a value of 501 * CFG_RETRY_STATUS. If we read that, we can't distinguish it from 502 * a CRS completion, so we will incorrectly retry the read and 503 * eventually return the wrong data (0xffffffff). 504 */ 505 data = readl(cfg_data_p); 506 while (data == CFG_RETRY_STATUS && timeout--) { 507 /* 508 * CRS state is set in CFG_RD status register 509 * This will handle the case where CFG_RETRY_STATUS is 510 * valid config data. 511 */ 512 status = iproc_pcie_read_reg(pcie, IPROC_PCIE_CFG_RD_STATUS); 513 if (status != CFG_RD_CRS) 514 return data; 515 516 udelay(1); 517 data = readl(cfg_data_p); 518 } 519 520 if (data == CFG_RETRY_STATUS) 521 data = 0xffffffff; 522 523 return data; 524 } 525 526 static void iproc_pcie_fix_cap(struct iproc_pcie *pcie, int where, u32 *val) 527 { 528 u32 i, dev_id; 529 530 switch (where & ~0x3) { 531 case PCI_VENDOR_ID: 532 dev_id = *val >> 16; 533 534 /* 535 * Activate fixup for those controllers that have corrupted 536 * capability list registers 537 */ 538 for (i = 0; i < ARRAY_SIZE(iproc_pcie_corrupt_cap_did); i++) 539 if (dev_id == iproc_pcie_corrupt_cap_did[i]) 540 pcie->fix_paxc_cap = true; 541 break; 542 543 case IPROC_PCI_PM_CAP: 544 if (pcie->fix_paxc_cap) { 545 /* advertise PM, force next capability to PCIe */ 546 *val &= ~IPROC_PCI_PM_CAP_MASK; 547 *val |= IPROC_PCI_EXP_CAP << 8 | PCI_CAP_ID_PM; 548 } 549 break; 550 551 case IPROC_PCI_EXP_CAP: 552 if (pcie->fix_paxc_cap) { 553 /* advertise root port, version 2, terminate here */ 554 *val = (PCI_EXP_TYPE_ROOT_PORT << 4 | 2) << 16 | 555 PCI_CAP_ID_EXP; 556 } 557 break; 558 559 case IPROC_PCI_EXP_CAP + PCI_EXP_RTCTL: 560 /* Don't advertise CRS SV support */ 561 *val &= ~(PCI_EXP_RTCAP_CRSVIS << 16); 562 break; 563 564 default: 565 break; 566 } 567 } 568 569 static int iproc_pcie_config_read(struct pci_bus *bus, unsigned int devfn, 570 int where, int size, u32 *val) 571 { 572 struct iproc_pcie *pcie = iproc_data(bus); 573 unsigned int busno = bus->number; 574 void __iomem *cfg_data_p; 575 unsigned int data; 576 int ret; 577 578 /* root complex access */ 579 if (busno == 0) { 580 ret = pci_generic_config_read32(bus, devfn, where, size, val); 581 if (ret == PCIBIOS_SUCCESSFUL) 582 iproc_pcie_fix_cap(pcie, where, val); 583 584 return ret; 585 } 586 587 cfg_data_p = iproc_pcie_map_ep_cfg_reg(pcie, busno, devfn, where); 588 589 if (!cfg_data_p) 590 return PCIBIOS_DEVICE_NOT_FOUND; 591 592 data = iproc_pcie_cfg_retry(pcie, cfg_data_p); 593 594 *val = data; 595 if (size <= 2) 596 *val = (data >> (8 * (where & 3))) & ((1 << (size * 8)) - 1); 597 598 /* 599 * For PAXC and PAXCv2, the total number of PFs that one can enumerate 600 * depends on the firmware configuration. Unfortunately, due to an ASIC 601 * bug, unconfigured PFs cannot be properly hidden from the root 602 * complex. As a result, write access to these PFs will cause bus lock 603 * up on the embedded processor 604 * 605 * Since all unconfigured PFs are left with an incorrect, staled device 606 * ID of 0x168e (PCI_DEVICE_ID_NX2_57810), we try to catch those access 607 * early here and reject them all 608 */ 609 #define DEVICE_ID_MASK 0xffff0000 610 #define DEVICE_ID_SHIFT 16 611 if (pcie->rej_unconfig_pf && 612 (where & CFG_ADDR_REG_NUM_MASK) == PCI_VENDOR_ID) 613 if ((*val & DEVICE_ID_MASK) == 614 (PCI_DEVICE_ID_NX2_57810 << DEVICE_ID_SHIFT)) 615 return PCIBIOS_FUNC_NOT_SUPPORTED; 616 617 return PCIBIOS_SUCCESSFUL; 618 } 619 620 /* 621 * Note access to the configuration registers are protected at the higher layer 622 * by 'pci_lock' in drivers/pci/access.c 623 */ 624 static void __iomem *iproc_pcie_map_cfg_bus(struct iproc_pcie *pcie, 625 int busno, unsigned int devfn, 626 int where) 627 { 628 u16 offset; 629 630 /* root complex access */ 631 if (busno == 0) { 632 if (PCIE_ECAM_DEVFN(devfn) > 0) 633 return NULL; 634 635 iproc_pcie_write_reg(pcie, IPROC_PCIE_CFG_IND_ADDR, 636 where & CFG_IND_ADDR_MASK); 637 offset = iproc_pcie_reg_offset(pcie, IPROC_PCIE_CFG_IND_DATA); 638 if (iproc_pcie_reg_is_invalid(offset)) 639 return NULL; 640 else 641 return (pcie->base + offset); 642 } 643 644 return iproc_pcie_map_ep_cfg_reg(pcie, busno, devfn, where); 645 } 646 647 static void __iomem *iproc_pcie_bus_map_cfg_bus(struct pci_bus *bus, 648 unsigned int devfn, 649 int where) 650 { 651 return iproc_pcie_map_cfg_bus(iproc_data(bus), bus->number, devfn, 652 where); 653 } 654 655 static int iproc_pci_raw_config_read32(struct iproc_pcie *pcie, 656 unsigned int devfn, int where, 657 int size, u32 *val) 658 { 659 void __iomem *addr; 660 661 addr = iproc_pcie_map_cfg_bus(pcie, 0, devfn, where & ~0x3); 662 if (!addr) { 663 *val = ~0; 664 return PCIBIOS_DEVICE_NOT_FOUND; 665 } 666 667 *val = readl(addr); 668 669 if (size <= 2) 670 *val = (*val >> (8 * (where & 3))) & ((1 << (size * 8)) - 1); 671 672 return PCIBIOS_SUCCESSFUL; 673 } 674 675 static int iproc_pci_raw_config_write32(struct iproc_pcie *pcie, 676 unsigned int devfn, int where, 677 int size, u32 val) 678 { 679 void __iomem *addr; 680 u32 mask, tmp; 681 682 addr = iproc_pcie_map_cfg_bus(pcie, 0, devfn, where & ~0x3); 683 if (!addr) 684 return PCIBIOS_DEVICE_NOT_FOUND; 685 686 if (size == 4) { 687 writel(val, addr); 688 return PCIBIOS_SUCCESSFUL; 689 } 690 691 mask = ~(((1 << (size * 8)) - 1) << ((where & 0x3) * 8)); 692 tmp = readl(addr) & mask; 693 tmp |= val << ((where & 0x3) * 8); 694 writel(tmp, addr); 695 696 return PCIBIOS_SUCCESSFUL; 697 } 698 699 static int iproc_pcie_config_read32(struct pci_bus *bus, unsigned int devfn, 700 int where, int size, u32 *val) 701 { 702 int ret; 703 struct iproc_pcie *pcie = iproc_data(bus); 704 705 iproc_pcie_apb_err_disable(bus, true); 706 if (pcie->iproc_cfg_read) 707 ret = iproc_pcie_config_read(bus, devfn, where, size, val); 708 else 709 ret = pci_generic_config_read32(bus, devfn, where, size, val); 710 iproc_pcie_apb_err_disable(bus, false); 711 712 return ret; 713 } 714 715 static int iproc_pcie_config_write32(struct pci_bus *bus, unsigned int devfn, 716 int where, int size, u32 val) 717 { 718 int ret; 719 720 iproc_pcie_apb_err_disable(bus, true); 721 ret = pci_generic_config_write32(bus, devfn, where, size, val); 722 iproc_pcie_apb_err_disable(bus, false); 723 724 return ret; 725 } 726 727 static struct pci_ops iproc_pcie_ops = { 728 .map_bus = iproc_pcie_bus_map_cfg_bus, 729 .read = iproc_pcie_config_read32, 730 .write = iproc_pcie_config_write32, 731 }; 732 733 static void iproc_pcie_perst_ctrl(struct iproc_pcie *pcie, bool assert) 734 { 735 u32 val; 736 737 /* 738 * PAXC and the internal emulated endpoint device downstream should not 739 * be reset. If firmware has been loaded on the endpoint device at an 740 * earlier boot stage, reset here causes issues. 741 */ 742 if (pcie->ep_is_internal) 743 return; 744 745 if (assert) { 746 val = iproc_pcie_read_reg(pcie, IPROC_PCIE_CLK_CTRL); 747 val &= ~EP_PERST_SOURCE_SELECT & ~EP_MODE_SURVIVE_PERST & 748 ~RC_PCIE_RST_OUTPUT; 749 iproc_pcie_write_reg(pcie, IPROC_PCIE_CLK_CTRL, val); 750 udelay(250); 751 } else { 752 val = iproc_pcie_read_reg(pcie, IPROC_PCIE_CLK_CTRL); 753 val |= RC_PCIE_RST_OUTPUT; 754 iproc_pcie_write_reg(pcie, IPROC_PCIE_CLK_CTRL, val); 755 msleep(100); 756 } 757 } 758 759 int iproc_pcie_shutdown(struct iproc_pcie *pcie) 760 { 761 iproc_pcie_perst_ctrl(pcie, true); 762 msleep(500); 763 764 return 0; 765 } 766 EXPORT_SYMBOL_GPL(iproc_pcie_shutdown); 767 768 static int iproc_pcie_check_link(struct iproc_pcie *pcie) 769 { 770 struct device *dev = pcie->dev; 771 u32 hdr_type, link_ctrl, link_status, class, val; 772 bool link_is_active = false; 773 774 /* 775 * PAXC connects to emulated endpoint devices directly and does not 776 * have a Serdes. Therefore skip the link detection logic here. 777 */ 778 if (pcie->ep_is_internal) 779 return 0; 780 781 val = iproc_pcie_read_reg(pcie, IPROC_PCIE_LINK_STATUS); 782 if (!(val & PCIE_PHYLINKUP) || !(val & PCIE_DL_ACTIVE)) { 783 dev_err(dev, "PHY or data link is INACTIVE!\n"); 784 return -ENODEV; 785 } 786 787 /* make sure we are not in EP mode */ 788 iproc_pci_raw_config_read32(pcie, 0, PCI_HEADER_TYPE, 1, &hdr_type); 789 if ((hdr_type & 0x7f) != PCI_HEADER_TYPE_BRIDGE) { 790 dev_err(dev, "in EP mode, hdr=%#02x\n", hdr_type); 791 return -EFAULT; 792 } 793 794 /* force class to PCI_CLASS_BRIDGE_PCI (0x0604) */ 795 #define PCI_BRIDGE_CTRL_REG_OFFSET 0x43c 796 #define PCI_CLASS_BRIDGE_MASK 0xffff00 797 #define PCI_CLASS_BRIDGE_SHIFT 8 798 iproc_pci_raw_config_read32(pcie, 0, PCI_BRIDGE_CTRL_REG_OFFSET, 799 4, &class); 800 class &= ~PCI_CLASS_BRIDGE_MASK; 801 class |= (PCI_CLASS_BRIDGE_PCI << PCI_CLASS_BRIDGE_SHIFT); 802 iproc_pci_raw_config_write32(pcie, 0, PCI_BRIDGE_CTRL_REG_OFFSET, 803 4, class); 804 805 /* check link status to see if link is active */ 806 iproc_pci_raw_config_read32(pcie, 0, IPROC_PCI_EXP_CAP + PCI_EXP_LNKSTA, 807 2, &link_status); 808 if (link_status & PCI_EXP_LNKSTA_NLW) 809 link_is_active = true; 810 811 if (!link_is_active) { 812 /* try GEN 1 link speed */ 813 #define PCI_TARGET_LINK_SPEED_MASK 0xf 814 #define PCI_TARGET_LINK_SPEED_GEN2 0x2 815 #define PCI_TARGET_LINK_SPEED_GEN1 0x1 816 iproc_pci_raw_config_read32(pcie, 0, 817 IPROC_PCI_EXP_CAP + PCI_EXP_LNKCTL2, 818 4, &link_ctrl); 819 if ((link_ctrl & PCI_TARGET_LINK_SPEED_MASK) == 820 PCI_TARGET_LINK_SPEED_GEN2) { 821 link_ctrl &= ~PCI_TARGET_LINK_SPEED_MASK; 822 link_ctrl |= PCI_TARGET_LINK_SPEED_GEN1; 823 iproc_pci_raw_config_write32(pcie, 0, 824 IPROC_PCI_EXP_CAP + PCI_EXP_LNKCTL2, 825 4, link_ctrl); 826 msleep(100); 827 828 iproc_pci_raw_config_read32(pcie, 0, 829 IPROC_PCI_EXP_CAP + PCI_EXP_LNKSTA, 830 2, &link_status); 831 if (link_status & PCI_EXP_LNKSTA_NLW) 832 link_is_active = true; 833 } 834 } 835 836 dev_info(dev, "link: %s\n", link_is_active ? "UP" : "DOWN"); 837 838 return link_is_active ? 0 : -ENODEV; 839 } 840 841 static void iproc_pcie_enable(struct iproc_pcie *pcie) 842 { 843 iproc_pcie_write_reg(pcie, IPROC_PCIE_INTX_EN, SYS_RC_INTX_MASK); 844 } 845 846 static inline bool iproc_pcie_ob_is_valid(struct iproc_pcie *pcie, 847 int window_idx) 848 { 849 u32 val; 850 851 val = iproc_pcie_read_reg(pcie, MAP_REG(IPROC_PCIE_OARR0, window_idx)); 852 853 return !!(val & OARR_VALID); 854 } 855 856 static inline int iproc_pcie_ob_write(struct iproc_pcie *pcie, int window_idx, 857 int size_idx, u64 axi_addr, u64 pci_addr) 858 { 859 struct device *dev = pcie->dev; 860 u16 oarr_offset, omap_offset; 861 862 /* 863 * Derive the OARR/OMAP offset from the first pair (OARR0/OMAP0) based 864 * on window index. 865 */ 866 oarr_offset = iproc_pcie_reg_offset(pcie, MAP_REG(IPROC_PCIE_OARR0, 867 window_idx)); 868 omap_offset = iproc_pcie_reg_offset(pcie, MAP_REG(IPROC_PCIE_OMAP0, 869 window_idx)); 870 if (iproc_pcie_reg_is_invalid(oarr_offset) || 871 iproc_pcie_reg_is_invalid(omap_offset)) 872 return -EINVAL; 873 874 /* 875 * Program the OARR registers. The upper 32-bit OARR register is 876 * always right after the lower 32-bit OARR register. 877 */ 878 writel(lower_32_bits(axi_addr) | (size_idx << OARR_SIZE_CFG_SHIFT) | 879 OARR_VALID, pcie->base + oarr_offset); 880 writel(upper_32_bits(axi_addr), pcie->base + oarr_offset + 4); 881 882 /* now program the OMAP registers */ 883 writel(lower_32_bits(pci_addr), pcie->base + omap_offset); 884 writel(upper_32_bits(pci_addr), pcie->base + omap_offset + 4); 885 886 dev_dbg(dev, "ob window [%d]: offset 0x%x axi %pap pci %pap\n", 887 window_idx, oarr_offset, &axi_addr, &pci_addr); 888 dev_dbg(dev, "oarr lo 0x%x oarr hi 0x%x\n", 889 readl(pcie->base + oarr_offset), 890 readl(pcie->base + oarr_offset + 4)); 891 dev_dbg(dev, "omap lo 0x%x omap hi 0x%x\n", 892 readl(pcie->base + omap_offset), 893 readl(pcie->base + omap_offset + 4)); 894 895 return 0; 896 } 897 898 /* 899 * Some iProc SoCs require the SW to configure the outbound address mapping 900 * 901 * Outbound address translation: 902 * 903 * iproc_pcie_address = axi_address - axi_offset 904 * OARR = iproc_pcie_address 905 * OMAP = pci_addr 906 * 907 * axi_addr -> iproc_pcie_address -> OARR -> OMAP -> pci_address 908 */ 909 static int iproc_pcie_setup_ob(struct iproc_pcie *pcie, u64 axi_addr, 910 u64 pci_addr, resource_size_t size) 911 { 912 struct iproc_pcie_ob *ob = &pcie->ob; 913 struct device *dev = pcie->dev; 914 int ret = -EINVAL, window_idx, size_idx; 915 916 if (axi_addr < ob->axi_offset) { 917 dev_err(dev, "axi address %pap less than offset %pap\n", 918 &axi_addr, &ob->axi_offset); 919 return -EINVAL; 920 } 921 922 /* 923 * Translate the AXI address to the internal address used by the iProc 924 * PCIe core before programming the OARR 925 */ 926 axi_addr -= ob->axi_offset; 927 928 /* iterate through all OARR/OMAP mapping windows */ 929 for (window_idx = ob->nr_windows - 1; window_idx >= 0; window_idx--) { 930 const struct iproc_pcie_ob_map *ob_map = 931 &pcie->ob_map[window_idx]; 932 933 /* 934 * If current outbound window is already in use, move on to the 935 * next one. 936 */ 937 if (iproc_pcie_ob_is_valid(pcie, window_idx)) 938 continue; 939 940 /* 941 * Iterate through all supported window sizes within the 942 * OARR/OMAP pair to find a match. Go through the window sizes 943 * in a descending order. 944 */ 945 for (size_idx = ob_map->nr_sizes - 1; size_idx >= 0; 946 size_idx--) { 947 resource_size_t window_size = 948 ob_map->window_sizes[size_idx] * SZ_1M; 949 950 /* 951 * Keep iterating until we reach the last window and 952 * with the minimal window size at index zero. In this 953 * case, we take a compromise by mapping it using the 954 * minimum window size that can be supported 955 */ 956 if (size < window_size) { 957 if (size_idx > 0 || window_idx > 0) 958 continue; 959 960 /* 961 * For the corner case of reaching the minimal 962 * window size that can be supported on the 963 * last window 964 */ 965 axi_addr = ALIGN_DOWN(axi_addr, window_size); 966 pci_addr = ALIGN_DOWN(pci_addr, window_size); 967 size = window_size; 968 } 969 970 if (!IS_ALIGNED(axi_addr, window_size) || 971 !IS_ALIGNED(pci_addr, window_size)) { 972 dev_err(dev, 973 "axi %pap or pci %pap not aligned\n", 974 &axi_addr, &pci_addr); 975 return -EINVAL; 976 } 977 978 /* 979 * Match found! Program both OARR and OMAP and mark 980 * them as a valid entry. 981 */ 982 ret = iproc_pcie_ob_write(pcie, window_idx, size_idx, 983 axi_addr, pci_addr); 984 if (ret) 985 goto err_ob; 986 987 size -= window_size; 988 if (size == 0) 989 return 0; 990 991 /* 992 * If we are here, we are done with the current window, 993 * but not yet finished all mappings. Need to move on 994 * to the next window. 995 */ 996 axi_addr += window_size; 997 pci_addr += window_size; 998 break; 999 } 1000 } 1001 1002 err_ob: 1003 dev_err(dev, "unable to configure outbound mapping\n"); 1004 dev_err(dev, 1005 "axi %pap, axi offset %pap, pci %pap, res size %pap\n", 1006 &axi_addr, &ob->axi_offset, &pci_addr, &size); 1007 1008 return ret; 1009 } 1010 1011 static int iproc_pcie_map_ranges(struct iproc_pcie *pcie, 1012 struct list_head *resources) 1013 { 1014 struct device *dev = pcie->dev; 1015 struct resource_entry *window; 1016 int ret; 1017 1018 resource_list_for_each_entry(window, resources) { 1019 struct resource *res = window->res; 1020 u64 res_type = resource_type(res); 1021 1022 switch (res_type) { 1023 case IORESOURCE_IO: 1024 case IORESOURCE_BUS: 1025 break; 1026 case IORESOURCE_MEM: 1027 ret = iproc_pcie_setup_ob(pcie, res->start, 1028 res->start - window->offset, 1029 resource_size(res)); 1030 if (ret) 1031 return ret; 1032 break; 1033 default: 1034 dev_err(dev, "invalid resource %pR\n", res); 1035 return -EINVAL; 1036 } 1037 } 1038 1039 return 0; 1040 } 1041 1042 static inline bool iproc_pcie_ib_is_in_use(struct iproc_pcie *pcie, 1043 int region_idx) 1044 { 1045 const struct iproc_pcie_ib_map *ib_map = &pcie->ib_map[region_idx]; 1046 u32 val; 1047 1048 val = iproc_pcie_read_reg(pcie, MAP_REG(IPROC_PCIE_IARR0, region_idx)); 1049 1050 return !!(val & (BIT(ib_map->nr_sizes) - 1)); 1051 } 1052 1053 static inline bool iproc_pcie_ib_check_type(const struct iproc_pcie_ib_map *ib_map, 1054 enum iproc_pcie_ib_map_type type) 1055 { 1056 return !!(ib_map->type == type); 1057 } 1058 1059 static int iproc_pcie_ib_write(struct iproc_pcie *pcie, int region_idx, 1060 int size_idx, int nr_windows, u64 axi_addr, 1061 u64 pci_addr, resource_size_t size) 1062 { 1063 struct device *dev = pcie->dev; 1064 const struct iproc_pcie_ib_map *ib_map = &pcie->ib_map[region_idx]; 1065 u16 iarr_offset, imap_offset; 1066 u32 val; 1067 int window_idx; 1068 1069 iarr_offset = iproc_pcie_reg_offset(pcie, 1070 MAP_REG(IPROC_PCIE_IARR0, region_idx)); 1071 imap_offset = iproc_pcie_reg_offset(pcie, 1072 MAP_REG(IPROC_PCIE_IMAP0, region_idx)); 1073 if (iproc_pcie_reg_is_invalid(iarr_offset) || 1074 iproc_pcie_reg_is_invalid(imap_offset)) 1075 return -EINVAL; 1076 1077 dev_dbg(dev, "ib region [%d]: offset 0x%x axi %pap pci %pap\n", 1078 region_idx, iarr_offset, &axi_addr, &pci_addr); 1079 1080 /* 1081 * Program the IARR registers. The upper 32-bit IARR register is 1082 * always right after the lower 32-bit IARR register. 1083 */ 1084 writel(lower_32_bits(pci_addr) | BIT(size_idx), 1085 pcie->base + iarr_offset); 1086 writel(upper_32_bits(pci_addr), pcie->base + iarr_offset + 4); 1087 1088 dev_dbg(dev, "iarr lo 0x%x iarr hi 0x%x\n", 1089 readl(pcie->base + iarr_offset), 1090 readl(pcie->base + iarr_offset + 4)); 1091 1092 /* 1093 * Now program the IMAP registers. Each IARR region may have one or 1094 * more IMAP windows. 1095 */ 1096 size >>= ilog2(nr_windows); 1097 for (window_idx = 0; window_idx < nr_windows; window_idx++) { 1098 val = readl(pcie->base + imap_offset); 1099 val |= lower_32_bits(axi_addr) | IMAP_VALID; 1100 writel(val, pcie->base + imap_offset); 1101 writel(upper_32_bits(axi_addr), 1102 pcie->base + imap_offset + ib_map->imap_addr_offset); 1103 1104 dev_dbg(dev, "imap window [%d] lo 0x%x hi 0x%x\n", 1105 window_idx, readl(pcie->base + imap_offset), 1106 readl(pcie->base + imap_offset + 1107 ib_map->imap_addr_offset)); 1108 1109 imap_offset += ib_map->imap_window_offset; 1110 axi_addr += size; 1111 } 1112 1113 return 0; 1114 } 1115 1116 static int iproc_pcie_setup_ib(struct iproc_pcie *pcie, 1117 struct resource_entry *entry, 1118 enum iproc_pcie_ib_map_type type) 1119 { 1120 struct device *dev = pcie->dev; 1121 struct iproc_pcie_ib *ib = &pcie->ib; 1122 int ret; 1123 unsigned int region_idx, size_idx; 1124 u64 axi_addr = entry->res->start; 1125 u64 pci_addr = entry->res->start - entry->offset; 1126 resource_size_t size = resource_size(entry->res); 1127 1128 /* iterate through all IARR mapping regions */ 1129 for (region_idx = 0; region_idx < ib->nr_regions; region_idx++) { 1130 const struct iproc_pcie_ib_map *ib_map = 1131 &pcie->ib_map[region_idx]; 1132 1133 /* 1134 * If current inbound region is already in use or not a 1135 * compatible type, move on to the next. 1136 */ 1137 if (iproc_pcie_ib_is_in_use(pcie, region_idx) || 1138 !iproc_pcie_ib_check_type(ib_map, type)) 1139 continue; 1140 1141 /* iterate through all supported region sizes to find a match */ 1142 for (size_idx = 0; size_idx < ib_map->nr_sizes; size_idx++) { 1143 resource_size_t region_size = 1144 ib_map->region_sizes[size_idx] * ib_map->size_unit; 1145 1146 if (size != region_size) 1147 continue; 1148 1149 if (!IS_ALIGNED(axi_addr, region_size) || 1150 !IS_ALIGNED(pci_addr, region_size)) { 1151 dev_err(dev, 1152 "axi %pap or pci %pap not aligned\n", 1153 &axi_addr, &pci_addr); 1154 return -EINVAL; 1155 } 1156 1157 /* Match found! Program IARR and all IMAP windows. */ 1158 ret = iproc_pcie_ib_write(pcie, region_idx, size_idx, 1159 ib_map->nr_windows, axi_addr, 1160 pci_addr, size); 1161 if (ret) 1162 goto err_ib; 1163 else 1164 return 0; 1165 1166 } 1167 } 1168 ret = -EINVAL; 1169 1170 err_ib: 1171 dev_err(dev, "unable to configure inbound mapping\n"); 1172 dev_err(dev, "axi %pap, pci %pap, res size %pap\n", 1173 &axi_addr, &pci_addr, &size); 1174 1175 return ret; 1176 } 1177 1178 static int iproc_pcie_map_dma_ranges(struct iproc_pcie *pcie) 1179 { 1180 struct pci_host_bridge *host = pci_host_bridge_from_priv(pcie); 1181 struct resource_entry *entry; 1182 int ret = 0; 1183 1184 resource_list_for_each_entry(entry, &host->dma_ranges) { 1185 /* Each range entry corresponds to an inbound mapping region */ 1186 ret = iproc_pcie_setup_ib(pcie, entry, IPROC_PCIE_IB_MAP_MEM); 1187 if (ret) 1188 break; 1189 } 1190 1191 return ret; 1192 } 1193 1194 static void iproc_pcie_invalidate_mapping(struct iproc_pcie *pcie) 1195 { 1196 struct iproc_pcie_ib *ib = &pcie->ib; 1197 struct iproc_pcie_ob *ob = &pcie->ob; 1198 int idx; 1199 1200 if (pcie->ep_is_internal) 1201 return; 1202 1203 if (pcie->need_ob_cfg) { 1204 /* iterate through all OARR mapping regions */ 1205 for (idx = ob->nr_windows - 1; idx >= 0; idx--) { 1206 iproc_pcie_write_reg(pcie, 1207 MAP_REG(IPROC_PCIE_OARR0, idx), 0); 1208 } 1209 } 1210 1211 if (pcie->need_ib_cfg) { 1212 /* iterate through all IARR mapping regions */ 1213 for (idx = 0; idx < ib->nr_regions; idx++) { 1214 iproc_pcie_write_reg(pcie, 1215 MAP_REG(IPROC_PCIE_IARR0, idx), 0); 1216 } 1217 } 1218 } 1219 1220 static int iproce_pcie_get_msi(struct iproc_pcie *pcie, 1221 struct device_node *msi_node, 1222 u64 *msi_addr) 1223 { 1224 struct device *dev = pcie->dev; 1225 int ret; 1226 struct resource res; 1227 1228 /* 1229 * Check if 'msi-map' points to ARM GICv3 ITS, which is the only 1230 * supported external MSI controller that requires steering. 1231 */ 1232 if (!of_device_is_compatible(msi_node, "arm,gic-v3-its")) { 1233 dev_err(dev, "unable to find compatible MSI controller\n"); 1234 return -ENODEV; 1235 } 1236 1237 /* derive GITS_TRANSLATER address from GICv3 */ 1238 ret = of_address_to_resource(msi_node, 0, &res); 1239 if (ret < 0) { 1240 dev_err(dev, "unable to obtain MSI controller resources\n"); 1241 return ret; 1242 } 1243 1244 *msi_addr = res.start + GITS_TRANSLATER; 1245 return 0; 1246 } 1247 1248 static int iproc_pcie_paxb_v2_msi_steer(struct iproc_pcie *pcie, u64 msi_addr) 1249 { 1250 int ret; 1251 struct resource_entry entry; 1252 1253 memset(&entry, 0, sizeof(entry)); 1254 entry.res = &entry.__res; 1255 1256 msi_addr &= ~(SZ_32K - 1); 1257 entry.res->start = msi_addr; 1258 entry.res->end = msi_addr + SZ_32K - 1; 1259 1260 ret = iproc_pcie_setup_ib(pcie, &entry, IPROC_PCIE_IB_MAP_IO); 1261 return ret; 1262 } 1263 1264 static void iproc_pcie_paxc_v2_msi_steer(struct iproc_pcie *pcie, u64 msi_addr, 1265 bool enable) 1266 { 1267 u32 val; 1268 1269 if (!enable) { 1270 /* 1271 * Disable PAXC MSI steering. All write transfers will be 1272 * treated as non-MSI transfers 1273 */ 1274 val = iproc_pcie_read_reg(pcie, IPROC_PCIE_MSI_EN_CFG); 1275 val &= ~MSI_ENABLE_CFG; 1276 iproc_pcie_write_reg(pcie, IPROC_PCIE_MSI_EN_CFG, val); 1277 return; 1278 } 1279 1280 /* 1281 * Program bits [43:13] of address of GITS_TRANSLATER register into 1282 * bits [30:0] of the MSI base address register. In fact, in all iProc 1283 * based SoCs, all I/O register bases are well below the 32-bit 1284 * boundary, so we can safely assume bits [43:32] are always zeros. 1285 */ 1286 iproc_pcie_write_reg(pcie, IPROC_PCIE_MSI_BASE_ADDR, 1287 (u32)(msi_addr >> 13)); 1288 1289 /* use a default 8K window size */ 1290 iproc_pcie_write_reg(pcie, IPROC_PCIE_MSI_WINDOW_SIZE, 0); 1291 1292 /* steering MSI to GICv3 ITS */ 1293 val = iproc_pcie_read_reg(pcie, IPROC_PCIE_MSI_GIC_MODE); 1294 val |= GIC_V3_CFG; 1295 iproc_pcie_write_reg(pcie, IPROC_PCIE_MSI_GIC_MODE, val); 1296 1297 /* 1298 * Program bits [43:2] of address of GITS_TRANSLATER register into the 1299 * iProc MSI address registers. 1300 */ 1301 msi_addr >>= 2; 1302 iproc_pcie_write_reg(pcie, IPROC_PCIE_MSI_ADDR_HI, 1303 upper_32_bits(msi_addr)); 1304 iproc_pcie_write_reg(pcie, IPROC_PCIE_MSI_ADDR_LO, 1305 lower_32_bits(msi_addr)); 1306 1307 /* enable MSI */ 1308 val = iproc_pcie_read_reg(pcie, IPROC_PCIE_MSI_EN_CFG); 1309 val |= MSI_ENABLE_CFG; 1310 iproc_pcie_write_reg(pcie, IPROC_PCIE_MSI_EN_CFG, val); 1311 } 1312 1313 static int iproc_pcie_msi_steer(struct iproc_pcie *pcie, 1314 struct device_node *msi_node) 1315 { 1316 struct device *dev = pcie->dev; 1317 int ret; 1318 u64 msi_addr; 1319 1320 ret = iproce_pcie_get_msi(pcie, msi_node, &msi_addr); 1321 if (ret < 0) { 1322 dev_err(dev, "msi steering failed\n"); 1323 return ret; 1324 } 1325 1326 switch (pcie->type) { 1327 case IPROC_PCIE_PAXB_V2: 1328 ret = iproc_pcie_paxb_v2_msi_steer(pcie, msi_addr); 1329 if (ret) 1330 return ret; 1331 break; 1332 case IPROC_PCIE_PAXC_V2: 1333 iproc_pcie_paxc_v2_msi_steer(pcie, msi_addr, true); 1334 break; 1335 default: 1336 return -EINVAL; 1337 } 1338 1339 return 0; 1340 } 1341 1342 static int iproc_pcie_msi_enable(struct iproc_pcie *pcie) 1343 { 1344 struct device_node *msi_node; 1345 int ret; 1346 1347 /* 1348 * Either the "msi-parent" or the "msi-map" phandle needs to exist 1349 * for us to obtain the MSI node. 1350 */ 1351 1352 msi_node = of_parse_phandle(pcie->dev->of_node, "msi-parent", 0); 1353 if (!msi_node) { 1354 const __be32 *msi_map = NULL; 1355 int len; 1356 u32 phandle; 1357 1358 msi_map = of_get_property(pcie->dev->of_node, "msi-map", &len); 1359 if (!msi_map) 1360 return -ENODEV; 1361 1362 phandle = be32_to_cpup(msi_map + 1); 1363 msi_node = of_find_node_by_phandle(phandle); 1364 if (!msi_node) 1365 return -ENODEV; 1366 } 1367 1368 /* 1369 * Certain revisions of the iProc PCIe controller require additional 1370 * configurations to steer the MSI writes towards an external MSI 1371 * controller. 1372 */ 1373 if (pcie->need_msi_steer) { 1374 ret = iproc_pcie_msi_steer(pcie, msi_node); 1375 if (ret) 1376 goto out_put_node; 1377 } 1378 1379 /* 1380 * If another MSI controller is being used, the call below should fail 1381 * but that is okay 1382 */ 1383 ret = iproc_msi_init(pcie, msi_node); 1384 1385 out_put_node: 1386 of_node_put(msi_node); 1387 return ret; 1388 } 1389 1390 static void iproc_pcie_msi_disable(struct iproc_pcie *pcie) 1391 { 1392 iproc_msi_exit(pcie); 1393 } 1394 1395 static int iproc_pcie_rev_init(struct iproc_pcie *pcie) 1396 { 1397 struct device *dev = pcie->dev; 1398 unsigned int reg_idx; 1399 const u16 *regs; 1400 1401 switch (pcie->type) { 1402 case IPROC_PCIE_PAXB_BCMA: 1403 regs = iproc_pcie_reg_paxb_bcma; 1404 break; 1405 case IPROC_PCIE_PAXB: 1406 regs = iproc_pcie_reg_paxb; 1407 pcie->has_apb_err_disable = true; 1408 if (pcie->need_ob_cfg) { 1409 pcie->ob_map = paxb_ob_map; 1410 pcie->ob.nr_windows = ARRAY_SIZE(paxb_ob_map); 1411 } 1412 break; 1413 case IPROC_PCIE_PAXB_V2: 1414 regs = iproc_pcie_reg_paxb_v2; 1415 pcie->iproc_cfg_read = true; 1416 pcie->has_apb_err_disable = true; 1417 if (pcie->need_ob_cfg) { 1418 pcie->ob_map = paxb_v2_ob_map; 1419 pcie->ob.nr_windows = ARRAY_SIZE(paxb_v2_ob_map); 1420 } 1421 pcie->ib.nr_regions = ARRAY_SIZE(paxb_v2_ib_map); 1422 pcie->ib_map = paxb_v2_ib_map; 1423 pcie->need_msi_steer = true; 1424 dev_warn(dev, "reads of config registers that contain %#x return incorrect data\n", 1425 CFG_RETRY_STATUS); 1426 break; 1427 case IPROC_PCIE_PAXC: 1428 regs = iproc_pcie_reg_paxc; 1429 pcie->ep_is_internal = true; 1430 pcie->iproc_cfg_read = true; 1431 pcie->rej_unconfig_pf = true; 1432 break; 1433 case IPROC_PCIE_PAXC_V2: 1434 regs = iproc_pcie_reg_paxc_v2; 1435 pcie->ep_is_internal = true; 1436 pcie->iproc_cfg_read = true; 1437 pcie->rej_unconfig_pf = true; 1438 pcie->need_msi_steer = true; 1439 break; 1440 default: 1441 dev_err(dev, "incompatible iProc PCIe interface\n"); 1442 return -EINVAL; 1443 } 1444 1445 pcie->reg_offsets = devm_kcalloc(dev, IPROC_PCIE_MAX_NUM_REG, 1446 sizeof(*pcie->reg_offsets), 1447 GFP_KERNEL); 1448 if (!pcie->reg_offsets) 1449 return -ENOMEM; 1450 1451 /* go through the register table and populate all valid registers */ 1452 pcie->reg_offsets[0] = (pcie->type == IPROC_PCIE_PAXC_V2) ? 1453 IPROC_PCIE_REG_INVALID : regs[0]; 1454 for (reg_idx = 1; reg_idx < IPROC_PCIE_MAX_NUM_REG; reg_idx++) 1455 pcie->reg_offsets[reg_idx] = regs[reg_idx] ? 1456 regs[reg_idx] : IPROC_PCIE_REG_INVALID; 1457 1458 return 0; 1459 } 1460 1461 int iproc_pcie_setup(struct iproc_pcie *pcie, struct list_head *res) 1462 { 1463 struct device *dev; 1464 int ret; 1465 struct pci_dev *pdev; 1466 struct pci_host_bridge *host = pci_host_bridge_from_priv(pcie); 1467 1468 dev = pcie->dev; 1469 1470 ret = iproc_pcie_rev_init(pcie); 1471 if (ret) { 1472 dev_err(dev, "unable to initialize controller parameters\n"); 1473 return ret; 1474 } 1475 1476 ret = phy_init(pcie->phy); 1477 if (ret) { 1478 dev_err(dev, "unable to initialize PCIe PHY\n"); 1479 return ret; 1480 } 1481 1482 ret = phy_power_on(pcie->phy); 1483 if (ret) { 1484 dev_err(dev, "unable to power on PCIe PHY\n"); 1485 goto err_exit_phy; 1486 } 1487 1488 iproc_pcie_perst_ctrl(pcie, true); 1489 iproc_pcie_perst_ctrl(pcie, false); 1490 1491 iproc_pcie_invalidate_mapping(pcie); 1492 1493 if (pcie->need_ob_cfg) { 1494 ret = iproc_pcie_map_ranges(pcie, res); 1495 if (ret) { 1496 dev_err(dev, "map failed\n"); 1497 goto err_power_off_phy; 1498 } 1499 } 1500 1501 if (pcie->need_ib_cfg) { 1502 ret = iproc_pcie_map_dma_ranges(pcie); 1503 if (ret && ret != -ENOENT) 1504 goto err_power_off_phy; 1505 } 1506 1507 ret = iproc_pcie_check_link(pcie); 1508 if (ret) { 1509 dev_err(dev, "no PCIe EP device detected\n"); 1510 goto err_power_off_phy; 1511 } 1512 1513 iproc_pcie_enable(pcie); 1514 1515 if (IS_ENABLED(CONFIG_PCI_MSI)) 1516 if (iproc_pcie_msi_enable(pcie)) 1517 dev_info(dev, "not using iProc MSI\n"); 1518 1519 host->ops = &iproc_pcie_ops; 1520 host->sysdata = pcie; 1521 host->map_irq = pcie->map_irq; 1522 1523 ret = pci_host_probe(host); 1524 if (ret < 0) { 1525 dev_err(dev, "failed to scan host: %d\n", ret); 1526 goto err_power_off_phy; 1527 } 1528 1529 for_each_pci_bridge(pdev, host->bus) { 1530 if (pci_pcie_type(pdev) == PCI_EXP_TYPE_ROOT_PORT) 1531 pcie_print_link_status(pdev); 1532 } 1533 1534 return 0; 1535 1536 err_power_off_phy: 1537 phy_power_off(pcie->phy); 1538 err_exit_phy: 1539 phy_exit(pcie->phy); 1540 return ret; 1541 } 1542 EXPORT_SYMBOL(iproc_pcie_setup); 1543 1544 int iproc_pcie_remove(struct iproc_pcie *pcie) 1545 { 1546 struct pci_host_bridge *host = pci_host_bridge_from_priv(pcie); 1547 1548 pci_stop_root_bus(host->bus); 1549 pci_remove_root_bus(host->bus); 1550 1551 iproc_pcie_msi_disable(pcie); 1552 1553 phy_power_off(pcie->phy); 1554 phy_exit(pcie->phy); 1555 1556 return 0; 1557 } 1558 EXPORT_SYMBOL(iproc_pcie_remove); 1559 1560 /* 1561 * The MSI parsing logic in certain revisions of Broadcom PAXC based root 1562 * complex does not work and needs to be disabled 1563 */ 1564 static void quirk_paxc_disable_msi_parsing(struct pci_dev *pdev) 1565 { 1566 struct iproc_pcie *pcie = iproc_data(pdev->bus); 1567 1568 if (pdev->hdr_type == PCI_HEADER_TYPE_BRIDGE) 1569 iproc_pcie_paxc_v2_msi_steer(pcie, 0, false); 1570 } 1571 DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_BROADCOM, 0x16f0, 1572 quirk_paxc_disable_msi_parsing); 1573 DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_BROADCOM, 0xd802, 1574 quirk_paxc_disable_msi_parsing); 1575 DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_BROADCOM, 0xd804, 1576 quirk_paxc_disable_msi_parsing); 1577 1578 static void quirk_paxc_bridge(struct pci_dev *pdev) 1579 { 1580 /* 1581 * The PCI config space is shared with the PAXC root port and the first 1582 * Ethernet device. So, we need to workaround this by telling the PCI 1583 * code that the bridge is not an Ethernet device. 1584 */ 1585 if (pdev->hdr_type == PCI_HEADER_TYPE_BRIDGE) 1586 pdev->class = PCI_CLASS_BRIDGE_PCI << 8; 1587 1588 /* 1589 * MPSS is not being set properly (as it is currently 0). This is 1590 * because that area of the PCI config space is hard coded to zero, and 1591 * is not modifiable by firmware. Set this to 2 (e.g., 512 byte MPS) 1592 * so that the MPS can be set to the real max value. 1593 */ 1594 pdev->pcie_mpss = 2; 1595 } 1596 DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_BROADCOM, 0x16cd, quirk_paxc_bridge); 1597 DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_BROADCOM, 0x16f0, quirk_paxc_bridge); 1598 DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_BROADCOM, 0xd750, quirk_paxc_bridge); 1599 DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_BROADCOM, 0xd802, quirk_paxc_bridge); 1600 DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_BROADCOM, 0xd804, quirk_paxc_bridge); 1601 1602 MODULE_AUTHOR("Ray Jui <rjui@broadcom.com>"); 1603 MODULE_DESCRIPTION("Broadcom iPROC PCIe common driver"); 1604 MODULE_LICENSE("GPL v2"); 1605