1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2014 Hauke Mehrtens <hauke@hauke-m.de> 4 * Copyright (C) 2015 Broadcom Corporation 5 */ 6 7 #include <linux/kernel.h> 8 #include <linux/pci.h> 9 #include <linux/pci-ecam.h> 10 #include <linux/msi.h> 11 #include <linux/clk.h> 12 #include <linux/module.h> 13 #include <linux/mbus.h> 14 #include <linux/slab.h> 15 #include <linux/delay.h> 16 #include <linux/interrupt.h> 17 #include <linux/irqchip/arm-gic-v3.h> 18 #include <linux/platform_device.h> 19 #include <linux/of_address.h> 20 #include <linux/of_pci.h> 21 #include <linux/of_irq.h> 22 #include <linux/of_platform.h> 23 #include <linux/phy/phy.h> 24 25 #include "pcie-iproc.h" 26 27 #define EP_PERST_SOURCE_SELECT_SHIFT 2 28 #define EP_PERST_SOURCE_SELECT BIT(EP_PERST_SOURCE_SELECT_SHIFT) 29 #define EP_MODE_SURVIVE_PERST_SHIFT 1 30 #define EP_MODE_SURVIVE_PERST BIT(EP_MODE_SURVIVE_PERST_SHIFT) 31 #define RC_PCIE_RST_OUTPUT_SHIFT 0 32 #define RC_PCIE_RST_OUTPUT BIT(RC_PCIE_RST_OUTPUT_SHIFT) 33 #define PAXC_RESET_MASK 0x7f 34 35 #define GIC_V3_CFG_SHIFT 0 36 #define GIC_V3_CFG BIT(GIC_V3_CFG_SHIFT) 37 38 #define MSI_ENABLE_CFG_SHIFT 0 39 #define MSI_ENABLE_CFG BIT(MSI_ENABLE_CFG_SHIFT) 40 41 #define CFG_IND_ADDR_MASK 0x00001ffc 42 43 #define CFG_ADDR_REG_NUM_MASK 0x00000ffc 44 #define CFG_ADDR_CFG_TYPE_1 1 45 46 #define SYS_RC_INTX_MASK 0xf 47 48 #define PCIE_PHYLINKUP_SHIFT 3 49 #define PCIE_PHYLINKUP BIT(PCIE_PHYLINKUP_SHIFT) 50 #define PCIE_DL_ACTIVE_SHIFT 2 51 #define PCIE_DL_ACTIVE BIT(PCIE_DL_ACTIVE_SHIFT) 52 53 #define APB_ERR_EN_SHIFT 0 54 #define APB_ERR_EN BIT(APB_ERR_EN_SHIFT) 55 56 #define CFG_RD_SUCCESS 0 57 #define CFG_RD_UR 1 58 #define CFG_RD_CRS 2 59 #define CFG_RD_CA 3 60 #define CFG_RETRY_STATUS 0xffff0001 61 #define CFG_RETRY_STATUS_TIMEOUT_US 500000 /* 500 milliseconds */ 62 63 /* derive the enum index of the outbound/inbound mapping registers */ 64 #define MAP_REG(base_reg, index) ((base_reg) + (index) * 2) 65 66 /* 67 * Maximum number of outbound mapping window sizes that can be supported by any 68 * OARR/OMAP mapping pair 69 */ 70 #define MAX_NUM_OB_WINDOW_SIZES 4 71 72 #define OARR_VALID_SHIFT 0 73 #define OARR_VALID BIT(OARR_VALID_SHIFT) 74 #define OARR_SIZE_CFG_SHIFT 1 75 76 /* 77 * Maximum number of inbound mapping region sizes that can be supported by an 78 * IARR 79 */ 80 #define MAX_NUM_IB_REGION_SIZES 9 81 82 #define IMAP_VALID_SHIFT 0 83 #define IMAP_VALID BIT(IMAP_VALID_SHIFT) 84 85 #define IPROC_PCI_PM_CAP 0x48 86 #define IPROC_PCI_PM_CAP_MASK 0xffff 87 #define IPROC_PCI_EXP_CAP 0xac 88 89 #define IPROC_PCIE_REG_INVALID 0xffff 90 91 /** 92 * struct iproc_pcie_ob_map - iProc PCIe outbound mapping controller-specific 93 * parameters 94 * @window_sizes: list of supported outbound mapping window sizes in MB 95 * @nr_sizes: number of supported outbound mapping window sizes 96 */ 97 struct iproc_pcie_ob_map { 98 resource_size_t window_sizes[MAX_NUM_OB_WINDOW_SIZES]; 99 unsigned int nr_sizes; 100 }; 101 102 static const struct iproc_pcie_ob_map paxb_ob_map[] = { 103 { 104 /* OARR0/OMAP0 */ 105 .window_sizes = { 128, 256 }, 106 .nr_sizes = 2, 107 }, 108 { 109 /* OARR1/OMAP1 */ 110 .window_sizes = { 128, 256 }, 111 .nr_sizes = 2, 112 }, 113 }; 114 115 static const struct iproc_pcie_ob_map paxb_v2_ob_map[] = { 116 { 117 /* OARR0/OMAP0 */ 118 .window_sizes = { 128, 256 }, 119 .nr_sizes = 2, 120 }, 121 { 122 /* OARR1/OMAP1 */ 123 .window_sizes = { 128, 256 }, 124 .nr_sizes = 2, 125 }, 126 { 127 /* OARR2/OMAP2 */ 128 .window_sizes = { 128, 256, 512, 1024 }, 129 .nr_sizes = 4, 130 }, 131 { 132 /* OARR3/OMAP3 */ 133 .window_sizes = { 128, 256, 512, 1024 }, 134 .nr_sizes = 4, 135 }, 136 }; 137 138 /** 139 * enum iproc_pcie_ib_map_type - iProc PCIe inbound mapping type 140 * @IPROC_PCIE_IB_MAP_MEM: DDR memory 141 * @IPROC_PCIE_IB_MAP_IO: device I/O memory 142 * @IPROC_PCIE_IB_MAP_INVALID: invalid or unused 143 */ 144 enum iproc_pcie_ib_map_type { 145 IPROC_PCIE_IB_MAP_MEM = 0, 146 IPROC_PCIE_IB_MAP_IO, 147 IPROC_PCIE_IB_MAP_INVALID 148 }; 149 150 /** 151 * struct iproc_pcie_ib_map - iProc PCIe inbound mapping controller-specific 152 * parameters 153 * @type: inbound mapping region type 154 * @size_unit: inbound mapping region size unit, could be SZ_1K, SZ_1M, or 155 * SZ_1G 156 * @region_sizes: list of supported inbound mapping region sizes in KB, MB, or 157 * GB, depending on the size unit 158 * @nr_sizes: number of supported inbound mapping region sizes 159 * @nr_windows: number of supported inbound mapping windows for the region 160 * @imap_addr_offset: register offset between the upper and lower 32-bit 161 * IMAP address registers 162 * @imap_window_offset: register offset between each IMAP window 163 */ 164 struct iproc_pcie_ib_map { 165 enum iproc_pcie_ib_map_type type; 166 unsigned int size_unit; 167 resource_size_t region_sizes[MAX_NUM_IB_REGION_SIZES]; 168 unsigned int nr_sizes; 169 unsigned int nr_windows; 170 u16 imap_addr_offset; 171 u16 imap_window_offset; 172 }; 173 174 static const struct iproc_pcie_ib_map paxb_v2_ib_map[] = { 175 { 176 /* IARR0/IMAP0 */ 177 .type = IPROC_PCIE_IB_MAP_IO, 178 .size_unit = SZ_1K, 179 .region_sizes = { 32 }, 180 .nr_sizes = 1, 181 .nr_windows = 8, 182 .imap_addr_offset = 0x40, 183 .imap_window_offset = 0x4, 184 }, 185 { 186 /* IARR1/IMAP1 */ 187 .type = IPROC_PCIE_IB_MAP_MEM, 188 .size_unit = SZ_1M, 189 .region_sizes = { 8 }, 190 .nr_sizes = 1, 191 .nr_windows = 8, 192 .imap_addr_offset = 0x4, 193 .imap_window_offset = 0x8, 194 195 }, 196 { 197 /* IARR2/IMAP2 */ 198 .type = IPROC_PCIE_IB_MAP_MEM, 199 .size_unit = SZ_1M, 200 .region_sizes = { 64, 128, 256, 512, 1024, 2048, 4096, 8192, 201 16384 }, 202 .nr_sizes = 9, 203 .nr_windows = 1, 204 .imap_addr_offset = 0x4, 205 .imap_window_offset = 0x8, 206 }, 207 { 208 /* IARR3/IMAP3 */ 209 .type = IPROC_PCIE_IB_MAP_MEM, 210 .size_unit = SZ_1G, 211 .region_sizes = { 1, 2, 4, 8, 16, 32 }, 212 .nr_sizes = 6, 213 .nr_windows = 8, 214 .imap_addr_offset = 0x4, 215 .imap_window_offset = 0x8, 216 }, 217 { 218 /* IARR4/IMAP4 */ 219 .type = IPROC_PCIE_IB_MAP_MEM, 220 .size_unit = SZ_1G, 221 .region_sizes = { 32, 64, 128, 256, 512 }, 222 .nr_sizes = 5, 223 .nr_windows = 8, 224 .imap_addr_offset = 0x4, 225 .imap_window_offset = 0x8, 226 }, 227 }; 228 229 /* 230 * iProc PCIe host registers 231 */ 232 enum iproc_pcie_reg { 233 /* clock/reset signal control */ 234 IPROC_PCIE_CLK_CTRL = 0, 235 236 /* 237 * To allow MSI to be steered to an external MSI controller (e.g., ARM 238 * GICv3 ITS) 239 */ 240 IPROC_PCIE_MSI_GIC_MODE, 241 242 /* 243 * IPROC_PCIE_MSI_BASE_ADDR and IPROC_PCIE_MSI_WINDOW_SIZE define the 244 * window where the MSI posted writes are written, for the writes to be 245 * interpreted as MSI writes. 246 */ 247 IPROC_PCIE_MSI_BASE_ADDR, 248 IPROC_PCIE_MSI_WINDOW_SIZE, 249 250 /* 251 * To hold the address of the register where the MSI writes are 252 * programmed. When ARM GICv3 ITS is used, this should be programmed 253 * with the address of the GITS_TRANSLATER register. 254 */ 255 IPROC_PCIE_MSI_ADDR_LO, 256 IPROC_PCIE_MSI_ADDR_HI, 257 258 /* enable MSI */ 259 IPROC_PCIE_MSI_EN_CFG, 260 261 /* allow access to root complex configuration space */ 262 IPROC_PCIE_CFG_IND_ADDR, 263 IPROC_PCIE_CFG_IND_DATA, 264 265 /* allow access to device configuration space */ 266 IPROC_PCIE_CFG_ADDR, 267 IPROC_PCIE_CFG_DATA, 268 269 /* enable INTx */ 270 IPROC_PCIE_INTX_EN, 271 272 /* outbound address mapping */ 273 IPROC_PCIE_OARR0, 274 IPROC_PCIE_OMAP0, 275 IPROC_PCIE_OARR1, 276 IPROC_PCIE_OMAP1, 277 IPROC_PCIE_OARR2, 278 IPROC_PCIE_OMAP2, 279 IPROC_PCIE_OARR3, 280 IPROC_PCIE_OMAP3, 281 282 /* inbound address mapping */ 283 IPROC_PCIE_IARR0, 284 IPROC_PCIE_IMAP0, 285 IPROC_PCIE_IARR1, 286 IPROC_PCIE_IMAP1, 287 IPROC_PCIE_IARR2, 288 IPROC_PCIE_IMAP2, 289 IPROC_PCIE_IARR3, 290 IPROC_PCIE_IMAP3, 291 IPROC_PCIE_IARR4, 292 IPROC_PCIE_IMAP4, 293 294 /* config read status */ 295 IPROC_PCIE_CFG_RD_STATUS, 296 297 /* link status */ 298 IPROC_PCIE_LINK_STATUS, 299 300 /* enable APB error for unsupported requests */ 301 IPROC_PCIE_APB_ERR_EN, 302 303 /* total number of core registers */ 304 IPROC_PCIE_MAX_NUM_REG, 305 }; 306 307 /* iProc PCIe PAXB BCMA registers */ 308 static const u16 iproc_pcie_reg_paxb_bcma[IPROC_PCIE_MAX_NUM_REG] = { 309 [IPROC_PCIE_CLK_CTRL] = 0x000, 310 [IPROC_PCIE_CFG_IND_ADDR] = 0x120, 311 [IPROC_PCIE_CFG_IND_DATA] = 0x124, 312 [IPROC_PCIE_CFG_ADDR] = 0x1f8, 313 [IPROC_PCIE_CFG_DATA] = 0x1fc, 314 [IPROC_PCIE_INTX_EN] = 0x330, 315 [IPROC_PCIE_LINK_STATUS] = 0xf0c, 316 }; 317 318 /* iProc PCIe PAXB registers */ 319 static const u16 iproc_pcie_reg_paxb[IPROC_PCIE_MAX_NUM_REG] = { 320 [IPROC_PCIE_CLK_CTRL] = 0x000, 321 [IPROC_PCIE_CFG_IND_ADDR] = 0x120, 322 [IPROC_PCIE_CFG_IND_DATA] = 0x124, 323 [IPROC_PCIE_CFG_ADDR] = 0x1f8, 324 [IPROC_PCIE_CFG_DATA] = 0x1fc, 325 [IPROC_PCIE_INTX_EN] = 0x330, 326 [IPROC_PCIE_OARR0] = 0xd20, 327 [IPROC_PCIE_OMAP0] = 0xd40, 328 [IPROC_PCIE_OARR1] = 0xd28, 329 [IPROC_PCIE_OMAP1] = 0xd48, 330 [IPROC_PCIE_LINK_STATUS] = 0xf0c, 331 [IPROC_PCIE_APB_ERR_EN] = 0xf40, 332 }; 333 334 /* iProc PCIe PAXB v2 registers */ 335 static const u16 iproc_pcie_reg_paxb_v2[IPROC_PCIE_MAX_NUM_REG] = { 336 [IPROC_PCIE_CLK_CTRL] = 0x000, 337 [IPROC_PCIE_CFG_IND_ADDR] = 0x120, 338 [IPROC_PCIE_CFG_IND_DATA] = 0x124, 339 [IPROC_PCIE_CFG_ADDR] = 0x1f8, 340 [IPROC_PCIE_CFG_DATA] = 0x1fc, 341 [IPROC_PCIE_INTX_EN] = 0x330, 342 [IPROC_PCIE_OARR0] = 0xd20, 343 [IPROC_PCIE_OMAP0] = 0xd40, 344 [IPROC_PCIE_OARR1] = 0xd28, 345 [IPROC_PCIE_OMAP1] = 0xd48, 346 [IPROC_PCIE_OARR2] = 0xd60, 347 [IPROC_PCIE_OMAP2] = 0xd68, 348 [IPROC_PCIE_OARR3] = 0xdf0, 349 [IPROC_PCIE_OMAP3] = 0xdf8, 350 [IPROC_PCIE_IARR0] = 0xd00, 351 [IPROC_PCIE_IMAP0] = 0xc00, 352 [IPROC_PCIE_IARR1] = 0xd08, 353 [IPROC_PCIE_IMAP1] = 0xd70, 354 [IPROC_PCIE_IARR2] = 0xd10, 355 [IPROC_PCIE_IMAP2] = 0xcc0, 356 [IPROC_PCIE_IARR3] = 0xe00, 357 [IPROC_PCIE_IMAP3] = 0xe08, 358 [IPROC_PCIE_IARR4] = 0xe68, 359 [IPROC_PCIE_IMAP4] = 0xe70, 360 [IPROC_PCIE_CFG_RD_STATUS] = 0xee0, 361 [IPROC_PCIE_LINK_STATUS] = 0xf0c, 362 [IPROC_PCIE_APB_ERR_EN] = 0xf40, 363 }; 364 365 /* iProc PCIe PAXC v1 registers */ 366 static const u16 iproc_pcie_reg_paxc[IPROC_PCIE_MAX_NUM_REG] = { 367 [IPROC_PCIE_CLK_CTRL] = 0x000, 368 [IPROC_PCIE_CFG_IND_ADDR] = 0x1f0, 369 [IPROC_PCIE_CFG_IND_DATA] = 0x1f4, 370 [IPROC_PCIE_CFG_ADDR] = 0x1f8, 371 [IPROC_PCIE_CFG_DATA] = 0x1fc, 372 }; 373 374 /* iProc PCIe PAXC v2 registers */ 375 static const u16 iproc_pcie_reg_paxc_v2[IPROC_PCIE_MAX_NUM_REG] = { 376 [IPROC_PCIE_MSI_GIC_MODE] = 0x050, 377 [IPROC_PCIE_MSI_BASE_ADDR] = 0x074, 378 [IPROC_PCIE_MSI_WINDOW_SIZE] = 0x078, 379 [IPROC_PCIE_MSI_ADDR_LO] = 0x07c, 380 [IPROC_PCIE_MSI_ADDR_HI] = 0x080, 381 [IPROC_PCIE_MSI_EN_CFG] = 0x09c, 382 [IPROC_PCIE_CFG_IND_ADDR] = 0x1f0, 383 [IPROC_PCIE_CFG_IND_DATA] = 0x1f4, 384 [IPROC_PCIE_CFG_ADDR] = 0x1f8, 385 [IPROC_PCIE_CFG_DATA] = 0x1fc, 386 }; 387 388 /* 389 * List of device IDs of controllers that have corrupted capability list that 390 * require SW fixup 391 */ 392 static const u16 iproc_pcie_corrupt_cap_did[] = { 393 0x16cd, 394 0x16f0, 395 0xd802, 396 0xd804 397 }; 398 399 static inline struct iproc_pcie *iproc_data(struct pci_bus *bus) 400 { 401 struct iproc_pcie *pcie = bus->sysdata; 402 return pcie; 403 } 404 405 static inline bool iproc_pcie_reg_is_invalid(u16 reg_offset) 406 { 407 return !!(reg_offset == IPROC_PCIE_REG_INVALID); 408 } 409 410 static inline u16 iproc_pcie_reg_offset(struct iproc_pcie *pcie, 411 enum iproc_pcie_reg reg) 412 { 413 return pcie->reg_offsets[reg]; 414 } 415 416 static inline u32 iproc_pcie_read_reg(struct iproc_pcie *pcie, 417 enum iproc_pcie_reg reg) 418 { 419 u16 offset = iproc_pcie_reg_offset(pcie, reg); 420 421 if (iproc_pcie_reg_is_invalid(offset)) 422 return 0; 423 424 return readl(pcie->base + offset); 425 } 426 427 static inline void iproc_pcie_write_reg(struct iproc_pcie *pcie, 428 enum iproc_pcie_reg reg, u32 val) 429 { 430 u16 offset = iproc_pcie_reg_offset(pcie, reg); 431 432 if (iproc_pcie_reg_is_invalid(offset)) 433 return; 434 435 writel(val, pcie->base + offset); 436 } 437 438 /* 439 * APB error forwarding can be disabled during access of configuration 440 * registers of the endpoint device, to prevent unsupported requests 441 * (typically seen during enumeration with multi-function devices) from 442 * triggering a system exception. 443 */ 444 static inline void iproc_pcie_apb_err_disable(struct pci_bus *bus, 445 bool disable) 446 { 447 struct iproc_pcie *pcie = iproc_data(bus); 448 u32 val; 449 450 if (bus->number && pcie->has_apb_err_disable) { 451 val = iproc_pcie_read_reg(pcie, IPROC_PCIE_APB_ERR_EN); 452 if (disable) 453 val &= ~APB_ERR_EN; 454 else 455 val |= APB_ERR_EN; 456 iproc_pcie_write_reg(pcie, IPROC_PCIE_APB_ERR_EN, val); 457 } 458 } 459 460 static void __iomem *iproc_pcie_map_ep_cfg_reg(struct iproc_pcie *pcie, 461 unsigned int busno, 462 unsigned int devfn, 463 int where) 464 { 465 u16 offset; 466 u32 val; 467 468 /* EP device access */ 469 val = ALIGN_DOWN(PCIE_ECAM_OFFSET(busno, devfn, where), 4) | 470 CFG_ADDR_CFG_TYPE_1; 471 472 iproc_pcie_write_reg(pcie, IPROC_PCIE_CFG_ADDR, val); 473 offset = iproc_pcie_reg_offset(pcie, IPROC_PCIE_CFG_DATA); 474 475 if (iproc_pcie_reg_is_invalid(offset)) 476 return NULL; 477 478 return (pcie->base + offset); 479 } 480 481 static unsigned int iproc_pcie_cfg_retry(struct iproc_pcie *pcie, 482 void __iomem *cfg_data_p) 483 { 484 int timeout = CFG_RETRY_STATUS_TIMEOUT_US; 485 unsigned int data; 486 u32 status; 487 488 /* 489 * As per PCIe spec r3.1, sec 2.3.2, CRS Software Visibility only 490 * affects config reads of the Vendor ID. For config writes or any 491 * other config reads, the Root may automatically reissue the 492 * configuration request again as a new request. 493 * 494 * For config reads, this hardware returns CFG_RETRY_STATUS data 495 * when it receives a CRS completion, regardless of the address of 496 * the read or the CRS Software Visibility Enable bit. As a 497 * partial workaround for this, we retry in software any read that 498 * returns CFG_RETRY_STATUS. 499 * 500 * Note that a non-Vendor ID config register may have a value of 501 * CFG_RETRY_STATUS. If we read that, we can't distinguish it from 502 * a CRS completion, so we will incorrectly retry the read and 503 * eventually return the wrong data (0xffffffff). 504 */ 505 data = readl(cfg_data_p); 506 while (data == CFG_RETRY_STATUS && timeout--) { 507 /* 508 * CRS state is set in CFG_RD status register 509 * This will handle the case where CFG_RETRY_STATUS is 510 * valid config data. 511 */ 512 status = iproc_pcie_read_reg(pcie, IPROC_PCIE_CFG_RD_STATUS); 513 if (status != CFG_RD_CRS) 514 return data; 515 516 udelay(1); 517 data = readl(cfg_data_p); 518 } 519 520 if (data == CFG_RETRY_STATUS) 521 data = 0xffffffff; 522 523 return data; 524 } 525 526 static void iproc_pcie_fix_cap(struct iproc_pcie *pcie, int where, u32 *val) 527 { 528 u32 i, dev_id; 529 530 switch (where & ~0x3) { 531 case PCI_VENDOR_ID: 532 dev_id = *val >> 16; 533 534 /* 535 * Activate fixup for those controllers that have corrupted 536 * capability list registers 537 */ 538 for (i = 0; i < ARRAY_SIZE(iproc_pcie_corrupt_cap_did); i++) 539 if (dev_id == iproc_pcie_corrupt_cap_did[i]) 540 pcie->fix_paxc_cap = true; 541 break; 542 543 case IPROC_PCI_PM_CAP: 544 if (pcie->fix_paxc_cap) { 545 /* advertise PM, force next capability to PCIe */ 546 *val &= ~IPROC_PCI_PM_CAP_MASK; 547 *val |= IPROC_PCI_EXP_CAP << 8 | PCI_CAP_ID_PM; 548 } 549 break; 550 551 case IPROC_PCI_EXP_CAP: 552 if (pcie->fix_paxc_cap) { 553 /* advertise root port, version 2, terminate here */ 554 *val = (PCI_EXP_TYPE_ROOT_PORT << 4 | 2) << 16 | 555 PCI_CAP_ID_EXP; 556 } 557 break; 558 559 case IPROC_PCI_EXP_CAP + PCI_EXP_RTCTL: 560 /* Don't advertise CRS SV support */ 561 *val &= ~(PCI_EXP_RTCAP_CRSVIS << 16); 562 break; 563 564 default: 565 break; 566 } 567 } 568 569 static int iproc_pcie_config_read(struct pci_bus *bus, unsigned int devfn, 570 int where, int size, u32 *val) 571 { 572 struct iproc_pcie *pcie = iproc_data(bus); 573 unsigned int busno = bus->number; 574 void __iomem *cfg_data_p; 575 unsigned int data; 576 int ret; 577 578 /* root complex access */ 579 if (busno == 0) { 580 ret = pci_generic_config_read32(bus, devfn, where, size, val); 581 if (ret == PCIBIOS_SUCCESSFUL) 582 iproc_pcie_fix_cap(pcie, where, val); 583 584 return ret; 585 } 586 587 cfg_data_p = iproc_pcie_map_ep_cfg_reg(pcie, busno, devfn, where); 588 589 if (!cfg_data_p) 590 return PCIBIOS_DEVICE_NOT_FOUND; 591 592 data = iproc_pcie_cfg_retry(pcie, cfg_data_p); 593 594 *val = data; 595 if (size <= 2) 596 *val = (data >> (8 * (where & 3))) & ((1 << (size * 8)) - 1); 597 598 /* 599 * For PAXC and PAXCv2, the total number of PFs that one can enumerate 600 * depends on the firmware configuration. Unfortunately, due to an ASIC 601 * bug, unconfigured PFs cannot be properly hidden from the root 602 * complex. As a result, write access to these PFs will cause bus lock 603 * up on the embedded processor 604 * 605 * Since all unconfigured PFs are left with an incorrect, staled device 606 * ID of 0x168e (PCI_DEVICE_ID_NX2_57810), we try to catch those access 607 * early here and reject them all 608 */ 609 #define DEVICE_ID_MASK 0xffff0000 610 #define DEVICE_ID_SHIFT 16 611 if (pcie->rej_unconfig_pf && 612 (where & CFG_ADDR_REG_NUM_MASK) == PCI_VENDOR_ID) 613 if ((*val & DEVICE_ID_MASK) == 614 (PCI_DEVICE_ID_NX2_57810 << DEVICE_ID_SHIFT)) 615 return PCIBIOS_FUNC_NOT_SUPPORTED; 616 617 return PCIBIOS_SUCCESSFUL; 618 } 619 620 /* 621 * Note access to the configuration registers are protected at the higher layer 622 * by 'pci_lock' in drivers/pci/access.c 623 */ 624 static void __iomem *iproc_pcie_map_cfg_bus(struct iproc_pcie *pcie, 625 int busno, unsigned int devfn, 626 int where) 627 { 628 u16 offset; 629 630 /* root complex access */ 631 if (busno == 0) { 632 if (PCIE_ECAM_DEVFN(devfn) > 0) 633 return NULL; 634 635 iproc_pcie_write_reg(pcie, IPROC_PCIE_CFG_IND_ADDR, 636 where & CFG_IND_ADDR_MASK); 637 offset = iproc_pcie_reg_offset(pcie, IPROC_PCIE_CFG_IND_DATA); 638 if (iproc_pcie_reg_is_invalid(offset)) 639 return NULL; 640 else 641 return (pcie->base + offset); 642 } 643 644 return iproc_pcie_map_ep_cfg_reg(pcie, busno, devfn, where); 645 } 646 647 static void __iomem *iproc_pcie_bus_map_cfg_bus(struct pci_bus *bus, 648 unsigned int devfn, 649 int where) 650 { 651 return iproc_pcie_map_cfg_bus(iproc_data(bus), bus->number, devfn, 652 where); 653 } 654 655 static int iproc_pci_raw_config_read32(struct iproc_pcie *pcie, 656 unsigned int devfn, int where, 657 int size, u32 *val) 658 { 659 void __iomem *addr; 660 661 addr = iproc_pcie_map_cfg_bus(pcie, 0, devfn, where & ~0x3); 662 if (!addr) 663 return PCIBIOS_DEVICE_NOT_FOUND; 664 665 *val = readl(addr); 666 667 if (size <= 2) 668 *val = (*val >> (8 * (where & 3))) & ((1 << (size * 8)) - 1); 669 670 return PCIBIOS_SUCCESSFUL; 671 } 672 673 static int iproc_pci_raw_config_write32(struct iproc_pcie *pcie, 674 unsigned int devfn, int where, 675 int size, u32 val) 676 { 677 void __iomem *addr; 678 u32 mask, tmp; 679 680 addr = iproc_pcie_map_cfg_bus(pcie, 0, devfn, where & ~0x3); 681 if (!addr) 682 return PCIBIOS_DEVICE_NOT_FOUND; 683 684 if (size == 4) { 685 writel(val, addr); 686 return PCIBIOS_SUCCESSFUL; 687 } 688 689 mask = ~(((1 << (size * 8)) - 1) << ((where & 0x3) * 8)); 690 tmp = readl(addr) & mask; 691 tmp |= val << ((where & 0x3) * 8); 692 writel(tmp, addr); 693 694 return PCIBIOS_SUCCESSFUL; 695 } 696 697 static int iproc_pcie_config_read32(struct pci_bus *bus, unsigned int devfn, 698 int where, int size, u32 *val) 699 { 700 int ret; 701 struct iproc_pcie *pcie = iproc_data(bus); 702 703 iproc_pcie_apb_err_disable(bus, true); 704 if (pcie->iproc_cfg_read) 705 ret = iproc_pcie_config_read(bus, devfn, where, size, val); 706 else 707 ret = pci_generic_config_read32(bus, devfn, where, size, val); 708 iproc_pcie_apb_err_disable(bus, false); 709 710 return ret; 711 } 712 713 static int iproc_pcie_config_write32(struct pci_bus *bus, unsigned int devfn, 714 int where, int size, u32 val) 715 { 716 int ret; 717 718 iproc_pcie_apb_err_disable(bus, true); 719 ret = pci_generic_config_write32(bus, devfn, where, size, val); 720 iproc_pcie_apb_err_disable(bus, false); 721 722 return ret; 723 } 724 725 static struct pci_ops iproc_pcie_ops = { 726 .map_bus = iproc_pcie_bus_map_cfg_bus, 727 .read = iproc_pcie_config_read32, 728 .write = iproc_pcie_config_write32, 729 }; 730 731 static void iproc_pcie_perst_ctrl(struct iproc_pcie *pcie, bool assert) 732 { 733 u32 val; 734 735 /* 736 * PAXC and the internal emulated endpoint device downstream should not 737 * be reset. If firmware has been loaded on the endpoint device at an 738 * earlier boot stage, reset here causes issues. 739 */ 740 if (pcie->ep_is_internal) 741 return; 742 743 if (assert) { 744 val = iproc_pcie_read_reg(pcie, IPROC_PCIE_CLK_CTRL); 745 val &= ~EP_PERST_SOURCE_SELECT & ~EP_MODE_SURVIVE_PERST & 746 ~RC_PCIE_RST_OUTPUT; 747 iproc_pcie_write_reg(pcie, IPROC_PCIE_CLK_CTRL, val); 748 udelay(250); 749 } else { 750 val = iproc_pcie_read_reg(pcie, IPROC_PCIE_CLK_CTRL); 751 val |= RC_PCIE_RST_OUTPUT; 752 iproc_pcie_write_reg(pcie, IPROC_PCIE_CLK_CTRL, val); 753 msleep(100); 754 } 755 } 756 757 int iproc_pcie_shutdown(struct iproc_pcie *pcie) 758 { 759 iproc_pcie_perst_ctrl(pcie, true); 760 msleep(500); 761 762 return 0; 763 } 764 EXPORT_SYMBOL_GPL(iproc_pcie_shutdown); 765 766 static int iproc_pcie_check_link(struct iproc_pcie *pcie) 767 { 768 struct device *dev = pcie->dev; 769 u32 hdr_type, link_ctrl, link_status, class, val; 770 bool link_is_active = false; 771 772 /* 773 * PAXC connects to emulated endpoint devices directly and does not 774 * have a Serdes. Therefore skip the link detection logic here. 775 */ 776 if (pcie->ep_is_internal) 777 return 0; 778 779 val = iproc_pcie_read_reg(pcie, IPROC_PCIE_LINK_STATUS); 780 if (!(val & PCIE_PHYLINKUP) || !(val & PCIE_DL_ACTIVE)) { 781 dev_err(dev, "PHY or data link is INACTIVE!\n"); 782 return -ENODEV; 783 } 784 785 /* make sure we are not in EP mode */ 786 iproc_pci_raw_config_read32(pcie, 0, PCI_HEADER_TYPE, 1, &hdr_type); 787 if ((hdr_type & 0x7f) != PCI_HEADER_TYPE_BRIDGE) { 788 dev_err(dev, "in EP mode, hdr=%#02x\n", hdr_type); 789 return -EFAULT; 790 } 791 792 /* force class to PCI_CLASS_BRIDGE_PCI (0x0604) */ 793 #define PCI_BRIDGE_CTRL_REG_OFFSET 0x43c 794 #define PCI_CLASS_BRIDGE_MASK 0xffff00 795 #define PCI_CLASS_BRIDGE_SHIFT 8 796 iproc_pci_raw_config_read32(pcie, 0, PCI_BRIDGE_CTRL_REG_OFFSET, 797 4, &class); 798 class &= ~PCI_CLASS_BRIDGE_MASK; 799 class |= (PCI_CLASS_BRIDGE_PCI << PCI_CLASS_BRIDGE_SHIFT); 800 iproc_pci_raw_config_write32(pcie, 0, PCI_BRIDGE_CTRL_REG_OFFSET, 801 4, class); 802 803 /* check link status to see if link is active */ 804 iproc_pci_raw_config_read32(pcie, 0, IPROC_PCI_EXP_CAP + PCI_EXP_LNKSTA, 805 2, &link_status); 806 if (link_status & PCI_EXP_LNKSTA_NLW) 807 link_is_active = true; 808 809 if (!link_is_active) { 810 /* try GEN 1 link speed */ 811 #define PCI_TARGET_LINK_SPEED_MASK 0xf 812 #define PCI_TARGET_LINK_SPEED_GEN2 0x2 813 #define PCI_TARGET_LINK_SPEED_GEN1 0x1 814 iproc_pci_raw_config_read32(pcie, 0, 815 IPROC_PCI_EXP_CAP + PCI_EXP_LNKCTL2, 816 4, &link_ctrl); 817 if ((link_ctrl & PCI_TARGET_LINK_SPEED_MASK) == 818 PCI_TARGET_LINK_SPEED_GEN2) { 819 link_ctrl &= ~PCI_TARGET_LINK_SPEED_MASK; 820 link_ctrl |= PCI_TARGET_LINK_SPEED_GEN1; 821 iproc_pci_raw_config_write32(pcie, 0, 822 IPROC_PCI_EXP_CAP + PCI_EXP_LNKCTL2, 823 4, link_ctrl); 824 msleep(100); 825 826 iproc_pci_raw_config_read32(pcie, 0, 827 IPROC_PCI_EXP_CAP + PCI_EXP_LNKSTA, 828 2, &link_status); 829 if (link_status & PCI_EXP_LNKSTA_NLW) 830 link_is_active = true; 831 } 832 } 833 834 dev_info(dev, "link: %s\n", link_is_active ? "UP" : "DOWN"); 835 836 return link_is_active ? 0 : -ENODEV; 837 } 838 839 static void iproc_pcie_enable(struct iproc_pcie *pcie) 840 { 841 iproc_pcie_write_reg(pcie, IPROC_PCIE_INTX_EN, SYS_RC_INTX_MASK); 842 } 843 844 static inline bool iproc_pcie_ob_is_valid(struct iproc_pcie *pcie, 845 int window_idx) 846 { 847 u32 val; 848 849 val = iproc_pcie_read_reg(pcie, MAP_REG(IPROC_PCIE_OARR0, window_idx)); 850 851 return !!(val & OARR_VALID); 852 } 853 854 static inline int iproc_pcie_ob_write(struct iproc_pcie *pcie, int window_idx, 855 int size_idx, u64 axi_addr, u64 pci_addr) 856 { 857 struct device *dev = pcie->dev; 858 u16 oarr_offset, omap_offset; 859 860 /* 861 * Derive the OARR/OMAP offset from the first pair (OARR0/OMAP0) based 862 * on window index. 863 */ 864 oarr_offset = iproc_pcie_reg_offset(pcie, MAP_REG(IPROC_PCIE_OARR0, 865 window_idx)); 866 omap_offset = iproc_pcie_reg_offset(pcie, MAP_REG(IPROC_PCIE_OMAP0, 867 window_idx)); 868 if (iproc_pcie_reg_is_invalid(oarr_offset) || 869 iproc_pcie_reg_is_invalid(omap_offset)) 870 return -EINVAL; 871 872 /* 873 * Program the OARR registers. The upper 32-bit OARR register is 874 * always right after the lower 32-bit OARR register. 875 */ 876 writel(lower_32_bits(axi_addr) | (size_idx << OARR_SIZE_CFG_SHIFT) | 877 OARR_VALID, pcie->base + oarr_offset); 878 writel(upper_32_bits(axi_addr), pcie->base + oarr_offset + 4); 879 880 /* now program the OMAP registers */ 881 writel(lower_32_bits(pci_addr), pcie->base + omap_offset); 882 writel(upper_32_bits(pci_addr), pcie->base + omap_offset + 4); 883 884 dev_dbg(dev, "ob window [%d]: offset 0x%x axi %pap pci %pap\n", 885 window_idx, oarr_offset, &axi_addr, &pci_addr); 886 dev_dbg(dev, "oarr lo 0x%x oarr hi 0x%x\n", 887 readl(pcie->base + oarr_offset), 888 readl(pcie->base + oarr_offset + 4)); 889 dev_dbg(dev, "omap lo 0x%x omap hi 0x%x\n", 890 readl(pcie->base + omap_offset), 891 readl(pcie->base + omap_offset + 4)); 892 893 return 0; 894 } 895 896 /* 897 * Some iProc SoCs require the SW to configure the outbound address mapping 898 * 899 * Outbound address translation: 900 * 901 * iproc_pcie_address = axi_address - axi_offset 902 * OARR = iproc_pcie_address 903 * OMAP = pci_addr 904 * 905 * axi_addr -> iproc_pcie_address -> OARR -> OMAP -> pci_address 906 */ 907 static int iproc_pcie_setup_ob(struct iproc_pcie *pcie, u64 axi_addr, 908 u64 pci_addr, resource_size_t size) 909 { 910 struct iproc_pcie_ob *ob = &pcie->ob; 911 struct device *dev = pcie->dev; 912 int ret = -EINVAL, window_idx, size_idx; 913 914 if (axi_addr < ob->axi_offset) { 915 dev_err(dev, "axi address %pap less than offset %pap\n", 916 &axi_addr, &ob->axi_offset); 917 return -EINVAL; 918 } 919 920 /* 921 * Translate the AXI address to the internal address used by the iProc 922 * PCIe core before programming the OARR 923 */ 924 axi_addr -= ob->axi_offset; 925 926 /* iterate through all OARR/OMAP mapping windows */ 927 for (window_idx = ob->nr_windows - 1; window_idx >= 0; window_idx--) { 928 const struct iproc_pcie_ob_map *ob_map = 929 &pcie->ob_map[window_idx]; 930 931 /* 932 * If current outbound window is already in use, move on to the 933 * next one. 934 */ 935 if (iproc_pcie_ob_is_valid(pcie, window_idx)) 936 continue; 937 938 /* 939 * Iterate through all supported window sizes within the 940 * OARR/OMAP pair to find a match. Go through the window sizes 941 * in a descending order. 942 */ 943 for (size_idx = ob_map->nr_sizes - 1; size_idx >= 0; 944 size_idx--) { 945 resource_size_t window_size = 946 ob_map->window_sizes[size_idx] * SZ_1M; 947 948 /* 949 * Keep iterating until we reach the last window and 950 * with the minimal window size at index zero. In this 951 * case, we take a compromise by mapping it using the 952 * minimum window size that can be supported 953 */ 954 if (size < window_size) { 955 if (size_idx > 0 || window_idx > 0) 956 continue; 957 958 /* 959 * For the corner case of reaching the minimal 960 * window size that can be supported on the 961 * last window 962 */ 963 axi_addr = ALIGN_DOWN(axi_addr, window_size); 964 pci_addr = ALIGN_DOWN(pci_addr, window_size); 965 size = window_size; 966 } 967 968 if (!IS_ALIGNED(axi_addr, window_size) || 969 !IS_ALIGNED(pci_addr, window_size)) { 970 dev_err(dev, 971 "axi %pap or pci %pap not aligned\n", 972 &axi_addr, &pci_addr); 973 return -EINVAL; 974 } 975 976 /* 977 * Match found! Program both OARR and OMAP and mark 978 * them as a valid entry. 979 */ 980 ret = iproc_pcie_ob_write(pcie, window_idx, size_idx, 981 axi_addr, pci_addr); 982 if (ret) 983 goto err_ob; 984 985 size -= window_size; 986 if (size == 0) 987 return 0; 988 989 /* 990 * If we are here, we are done with the current window, 991 * but not yet finished all mappings. Need to move on 992 * to the next window. 993 */ 994 axi_addr += window_size; 995 pci_addr += window_size; 996 break; 997 } 998 } 999 1000 err_ob: 1001 dev_err(dev, "unable to configure outbound mapping\n"); 1002 dev_err(dev, 1003 "axi %pap, axi offset %pap, pci %pap, res size %pap\n", 1004 &axi_addr, &ob->axi_offset, &pci_addr, &size); 1005 1006 return ret; 1007 } 1008 1009 static int iproc_pcie_map_ranges(struct iproc_pcie *pcie, 1010 struct list_head *resources) 1011 { 1012 struct device *dev = pcie->dev; 1013 struct resource_entry *window; 1014 int ret; 1015 1016 resource_list_for_each_entry(window, resources) { 1017 struct resource *res = window->res; 1018 u64 res_type = resource_type(res); 1019 1020 switch (res_type) { 1021 case IORESOURCE_IO: 1022 case IORESOURCE_BUS: 1023 break; 1024 case IORESOURCE_MEM: 1025 ret = iproc_pcie_setup_ob(pcie, res->start, 1026 res->start - window->offset, 1027 resource_size(res)); 1028 if (ret) 1029 return ret; 1030 break; 1031 default: 1032 dev_err(dev, "invalid resource %pR\n", res); 1033 return -EINVAL; 1034 } 1035 } 1036 1037 return 0; 1038 } 1039 1040 static inline bool iproc_pcie_ib_is_in_use(struct iproc_pcie *pcie, 1041 int region_idx) 1042 { 1043 const struct iproc_pcie_ib_map *ib_map = &pcie->ib_map[region_idx]; 1044 u32 val; 1045 1046 val = iproc_pcie_read_reg(pcie, MAP_REG(IPROC_PCIE_IARR0, region_idx)); 1047 1048 return !!(val & (BIT(ib_map->nr_sizes) - 1)); 1049 } 1050 1051 static inline bool iproc_pcie_ib_check_type(const struct iproc_pcie_ib_map *ib_map, 1052 enum iproc_pcie_ib_map_type type) 1053 { 1054 return !!(ib_map->type == type); 1055 } 1056 1057 static int iproc_pcie_ib_write(struct iproc_pcie *pcie, int region_idx, 1058 int size_idx, int nr_windows, u64 axi_addr, 1059 u64 pci_addr, resource_size_t size) 1060 { 1061 struct device *dev = pcie->dev; 1062 const struct iproc_pcie_ib_map *ib_map = &pcie->ib_map[region_idx]; 1063 u16 iarr_offset, imap_offset; 1064 u32 val; 1065 int window_idx; 1066 1067 iarr_offset = iproc_pcie_reg_offset(pcie, 1068 MAP_REG(IPROC_PCIE_IARR0, region_idx)); 1069 imap_offset = iproc_pcie_reg_offset(pcie, 1070 MAP_REG(IPROC_PCIE_IMAP0, region_idx)); 1071 if (iproc_pcie_reg_is_invalid(iarr_offset) || 1072 iproc_pcie_reg_is_invalid(imap_offset)) 1073 return -EINVAL; 1074 1075 dev_dbg(dev, "ib region [%d]: offset 0x%x axi %pap pci %pap\n", 1076 region_idx, iarr_offset, &axi_addr, &pci_addr); 1077 1078 /* 1079 * Program the IARR registers. The upper 32-bit IARR register is 1080 * always right after the lower 32-bit IARR register. 1081 */ 1082 writel(lower_32_bits(pci_addr) | BIT(size_idx), 1083 pcie->base + iarr_offset); 1084 writel(upper_32_bits(pci_addr), pcie->base + iarr_offset + 4); 1085 1086 dev_dbg(dev, "iarr lo 0x%x iarr hi 0x%x\n", 1087 readl(pcie->base + iarr_offset), 1088 readl(pcie->base + iarr_offset + 4)); 1089 1090 /* 1091 * Now program the IMAP registers. Each IARR region may have one or 1092 * more IMAP windows. 1093 */ 1094 size >>= ilog2(nr_windows); 1095 for (window_idx = 0; window_idx < nr_windows; window_idx++) { 1096 val = readl(pcie->base + imap_offset); 1097 val |= lower_32_bits(axi_addr) | IMAP_VALID; 1098 writel(val, pcie->base + imap_offset); 1099 writel(upper_32_bits(axi_addr), 1100 pcie->base + imap_offset + ib_map->imap_addr_offset); 1101 1102 dev_dbg(dev, "imap window [%d] lo 0x%x hi 0x%x\n", 1103 window_idx, readl(pcie->base + imap_offset), 1104 readl(pcie->base + imap_offset + 1105 ib_map->imap_addr_offset)); 1106 1107 imap_offset += ib_map->imap_window_offset; 1108 axi_addr += size; 1109 } 1110 1111 return 0; 1112 } 1113 1114 static int iproc_pcie_setup_ib(struct iproc_pcie *pcie, 1115 struct resource_entry *entry, 1116 enum iproc_pcie_ib_map_type type) 1117 { 1118 struct device *dev = pcie->dev; 1119 struct iproc_pcie_ib *ib = &pcie->ib; 1120 int ret; 1121 unsigned int region_idx, size_idx; 1122 u64 axi_addr = entry->res->start; 1123 u64 pci_addr = entry->res->start - entry->offset; 1124 resource_size_t size = resource_size(entry->res); 1125 1126 /* iterate through all IARR mapping regions */ 1127 for (region_idx = 0; region_idx < ib->nr_regions; region_idx++) { 1128 const struct iproc_pcie_ib_map *ib_map = 1129 &pcie->ib_map[region_idx]; 1130 1131 /* 1132 * If current inbound region is already in use or not a 1133 * compatible type, move on to the next. 1134 */ 1135 if (iproc_pcie_ib_is_in_use(pcie, region_idx) || 1136 !iproc_pcie_ib_check_type(ib_map, type)) 1137 continue; 1138 1139 /* iterate through all supported region sizes to find a match */ 1140 for (size_idx = 0; size_idx < ib_map->nr_sizes; size_idx++) { 1141 resource_size_t region_size = 1142 ib_map->region_sizes[size_idx] * ib_map->size_unit; 1143 1144 if (size != region_size) 1145 continue; 1146 1147 if (!IS_ALIGNED(axi_addr, region_size) || 1148 !IS_ALIGNED(pci_addr, region_size)) { 1149 dev_err(dev, 1150 "axi %pap or pci %pap not aligned\n", 1151 &axi_addr, &pci_addr); 1152 return -EINVAL; 1153 } 1154 1155 /* Match found! Program IARR and all IMAP windows. */ 1156 ret = iproc_pcie_ib_write(pcie, region_idx, size_idx, 1157 ib_map->nr_windows, axi_addr, 1158 pci_addr, size); 1159 if (ret) 1160 goto err_ib; 1161 else 1162 return 0; 1163 1164 } 1165 } 1166 ret = -EINVAL; 1167 1168 err_ib: 1169 dev_err(dev, "unable to configure inbound mapping\n"); 1170 dev_err(dev, "axi %pap, pci %pap, res size %pap\n", 1171 &axi_addr, &pci_addr, &size); 1172 1173 return ret; 1174 } 1175 1176 static int iproc_pcie_map_dma_ranges(struct iproc_pcie *pcie) 1177 { 1178 struct pci_host_bridge *host = pci_host_bridge_from_priv(pcie); 1179 struct resource_entry *entry; 1180 int ret = 0; 1181 1182 resource_list_for_each_entry(entry, &host->dma_ranges) { 1183 /* Each range entry corresponds to an inbound mapping region */ 1184 ret = iproc_pcie_setup_ib(pcie, entry, IPROC_PCIE_IB_MAP_MEM); 1185 if (ret) 1186 break; 1187 } 1188 1189 return ret; 1190 } 1191 1192 static void iproc_pcie_invalidate_mapping(struct iproc_pcie *pcie) 1193 { 1194 struct iproc_pcie_ib *ib = &pcie->ib; 1195 struct iproc_pcie_ob *ob = &pcie->ob; 1196 int idx; 1197 1198 if (pcie->ep_is_internal) 1199 return; 1200 1201 if (pcie->need_ob_cfg) { 1202 /* iterate through all OARR mapping regions */ 1203 for (idx = ob->nr_windows - 1; idx >= 0; idx--) { 1204 iproc_pcie_write_reg(pcie, 1205 MAP_REG(IPROC_PCIE_OARR0, idx), 0); 1206 } 1207 } 1208 1209 if (pcie->need_ib_cfg) { 1210 /* iterate through all IARR mapping regions */ 1211 for (idx = 0; idx < ib->nr_regions; idx++) { 1212 iproc_pcie_write_reg(pcie, 1213 MAP_REG(IPROC_PCIE_IARR0, idx), 0); 1214 } 1215 } 1216 } 1217 1218 static int iproce_pcie_get_msi(struct iproc_pcie *pcie, 1219 struct device_node *msi_node, 1220 u64 *msi_addr) 1221 { 1222 struct device *dev = pcie->dev; 1223 int ret; 1224 struct resource res; 1225 1226 /* 1227 * Check if 'msi-map' points to ARM GICv3 ITS, which is the only 1228 * supported external MSI controller that requires steering. 1229 */ 1230 if (!of_device_is_compatible(msi_node, "arm,gic-v3-its")) { 1231 dev_err(dev, "unable to find compatible MSI controller\n"); 1232 return -ENODEV; 1233 } 1234 1235 /* derive GITS_TRANSLATER address from GICv3 */ 1236 ret = of_address_to_resource(msi_node, 0, &res); 1237 if (ret < 0) { 1238 dev_err(dev, "unable to obtain MSI controller resources\n"); 1239 return ret; 1240 } 1241 1242 *msi_addr = res.start + GITS_TRANSLATER; 1243 return 0; 1244 } 1245 1246 static int iproc_pcie_paxb_v2_msi_steer(struct iproc_pcie *pcie, u64 msi_addr) 1247 { 1248 int ret; 1249 struct resource_entry entry; 1250 1251 memset(&entry, 0, sizeof(entry)); 1252 entry.res = &entry.__res; 1253 1254 msi_addr &= ~(SZ_32K - 1); 1255 entry.res->start = msi_addr; 1256 entry.res->end = msi_addr + SZ_32K - 1; 1257 1258 ret = iproc_pcie_setup_ib(pcie, &entry, IPROC_PCIE_IB_MAP_IO); 1259 return ret; 1260 } 1261 1262 static void iproc_pcie_paxc_v2_msi_steer(struct iproc_pcie *pcie, u64 msi_addr, 1263 bool enable) 1264 { 1265 u32 val; 1266 1267 if (!enable) { 1268 /* 1269 * Disable PAXC MSI steering. All write transfers will be 1270 * treated as non-MSI transfers 1271 */ 1272 val = iproc_pcie_read_reg(pcie, IPROC_PCIE_MSI_EN_CFG); 1273 val &= ~MSI_ENABLE_CFG; 1274 iproc_pcie_write_reg(pcie, IPROC_PCIE_MSI_EN_CFG, val); 1275 return; 1276 } 1277 1278 /* 1279 * Program bits [43:13] of address of GITS_TRANSLATER register into 1280 * bits [30:0] of the MSI base address register. In fact, in all iProc 1281 * based SoCs, all I/O register bases are well below the 32-bit 1282 * boundary, so we can safely assume bits [43:32] are always zeros. 1283 */ 1284 iproc_pcie_write_reg(pcie, IPROC_PCIE_MSI_BASE_ADDR, 1285 (u32)(msi_addr >> 13)); 1286 1287 /* use a default 8K window size */ 1288 iproc_pcie_write_reg(pcie, IPROC_PCIE_MSI_WINDOW_SIZE, 0); 1289 1290 /* steering MSI to GICv3 ITS */ 1291 val = iproc_pcie_read_reg(pcie, IPROC_PCIE_MSI_GIC_MODE); 1292 val |= GIC_V3_CFG; 1293 iproc_pcie_write_reg(pcie, IPROC_PCIE_MSI_GIC_MODE, val); 1294 1295 /* 1296 * Program bits [43:2] of address of GITS_TRANSLATER register into the 1297 * iProc MSI address registers. 1298 */ 1299 msi_addr >>= 2; 1300 iproc_pcie_write_reg(pcie, IPROC_PCIE_MSI_ADDR_HI, 1301 upper_32_bits(msi_addr)); 1302 iproc_pcie_write_reg(pcie, IPROC_PCIE_MSI_ADDR_LO, 1303 lower_32_bits(msi_addr)); 1304 1305 /* enable MSI */ 1306 val = iproc_pcie_read_reg(pcie, IPROC_PCIE_MSI_EN_CFG); 1307 val |= MSI_ENABLE_CFG; 1308 iproc_pcie_write_reg(pcie, IPROC_PCIE_MSI_EN_CFG, val); 1309 } 1310 1311 static int iproc_pcie_msi_steer(struct iproc_pcie *pcie, 1312 struct device_node *msi_node) 1313 { 1314 struct device *dev = pcie->dev; 1315 int ret; 1316 u64 msi_addr; 1317 1318 ret = iproce_pcie_get_msi(pcie, msi_node, &msi_addr); 1319 if (ret < 0) { 1320 dev_err(dev, "msi steering failed\n"); 1321 return ret; 1322 } 1323 1324 switch (pcie->type) { 1325 case IPROC_PCIE_PAXB_V2: 1326 ret = iproc_pcie_paxb_v2_msi_steer(pcie, msi_addr); 1327 if (ret) 1328 return ret; 1329 break; 1330 case IPROC_PCIE_PAXC_V2: 1331 iproc_pcie_paxc_v2_msi_steer(pcie, msi_addr, true); 1332 break; 1333 default: 1334 return -EINVAL; 1335 } 1336 1337 return 0; 1338 } 1339 1340 static int iproc_pcie_msi_enable(struct iproc_pcie *pcie) 1341 { 1342 struct device_node *msi_node; 1343 int ret; 1344 1345 /* 1346 * Either the "msi-parent" or the "msi-map" phandle needs to exist 1347 * for us to obtain the MSI node. 1348 */ 1349 1350 msi_node = of_parse_phandle(pcie->dev->of_node, "msi-parent", 0); 1351 if (!msi_node) { 1352 const __be32 *msi_map = NULL; 1353 int len; 1354 u32 phandle; 1355 1356 msi_map = of_get_property(pcie->dev->of_node, "msi-map", &len); 1357 if (!msi_map) 1358 return -ENODEV; 1359 1360 phandle = be32_to_cpup(msi_map + 1); 1361 msi_node = of_find_node_by_phandle(phandle); 1362 if (!msi_node) 1363 return -ENODEV; 1364 } 1365 1366 /* 1367 * Certain revisions of the iProc PCIe controller require additional 1368 * configurations to steer the MSI writes towards an external MSI 1369 * controller. 1370 */ 1371 if (pcie->need_msi_steer) { 1372 ret = iproc_pcie_msi_steer(pcie, msi_node); 1373 if (ret) 1374 goto out_put_node; 1375 } 1376 1377 /* 1378 * If another MSI controller is being used, the call below should fail 1379 * but that is okay 1380 */ 1381 ret = iproc_msi_init(pcie, msi_node); 1382 1383 out_put_node: 1384 of_node_put(msi_node); 1385 return ret; 1386 } 1387 1388 static void iproc_pcie_msi_disable(struct iproc_pcie *pcie) 1389 { 1390 iproc_msi_exit(pcie); 1391 } 1392 1393 static int iproc_pcie_rev_init(struct iproc_pcie *pcie) 1394 { 1395 struct device *dev = pcie->dev; 1396 unsigned int reg_idx; 1397 const u16 *regs; 1398 1399 switch (pcie->type) { 1400 case IPROC_PCIE_PAXB_BCMA: 1401 regs = iproc_pcie_reg_paxb_bcma; 1402 break; 1403 case IPROC_PCIE_PAXB: 1404 regs = iproc_pcie_reg_paxb; 1405 pcie->has_apb_err_disable = true; 1406 if (pcie->need_ob_cfg) { 1407 pcie->ob_map = paxb_ob_map; 1408 pcie->ob.nr_windows = ARRAY_SIZE(paxb_ob_map); 1409 } 1410 break; 1411 case IPROC_PCIE_PAXB_V2: 1412 regs = iproc_pcie_reg_paxb_v2; 1413 pcie->iproc_cfg_read = true; 1414 pcie->has_apb_err_disable = true; 1415 if (pcie->need_ob_cfg) { 1416 pcie->ob_map = paxb_v2_ob_map; 1417 pcie->ob.nr_windows = ARRAY_SIZE(paxb_v2_ob_map); 1418 } 1419 pcie->ib.nr_regions = ARRAY_SIZE(paxb_v2_ib_map); 1420 pcie->ib_map = paxb_v2_ib_map; 1421 pcie->need_msi_steer = true; 1422 dev_warn(dev, "reads of config registers that contain %#x return incorrect data\n", 1423 CFG_RETRY_STATUS); 1424 break; 1425 case IPROC_PCIE_PAXC: 1426 regs = iproc_pcie_reg_paxc; 1427 pcie->ep_is_internal = true; 1428 pcie->iproc_cfg_read = true; 1429 pcie->rej_unconfig_pf = true; 1430 break; 1431 case IPROC_PCIE_PAXC_V2: 1432 regs = iproc_pcie_reg_paxc_v2; 1433 pcie->ep_is_internal = true; 1434 pcie->iproc_cfg_read = true; 1435 pcie->rej_unconfig_pf = true; 1436 pcie->need_msi_steer = true; 1437 break; 1438 default: 1439 dev_err(dev, "incompatible iProc PCIe interface\n"); 1440 return -EINVAL; 1441 } 1442 1443 pcie->reg_offsets = devm_kcalloc(dev, IPROC_PCIE_MAX_NUM_REG, 1444 sizeof(*pcie->reg_offsets), 1445 GFP_KERNEL); 1446 if (!pcie->reg_offsets) 1447 return -ENOMEM; 1448 1449 /* go through the register table and populate all valid registers */ 1450 pcie->reg_offsets[0] = (pcie->type == IPROC_PCIE_PAXC_V2) ? 1451 IPROC_PCIE_REG_INVALID : regs[0]; 1452 for (reg_idx = 1; reg_idx < IPROC_PCIE_MAX_NUM_REG; reg_idx++) 1453 pcie->reg_offsets[reg_idx] = regs[reg_idx] ? 1454 regs[reg_idx] : IPROC_PCIE_REG_INVALID; 1455 1456 return 0; 1457 } 1458 1459 int iproc_pcie_setup(struct iproc_pcie *pcie, struct list_head *res) 1460 { 1461 struct device *dev; 1462 int ret; 1463 struct pci_dev *pdev; 1464 struct pci_host_bridge *host = pci_host_bridge_from_priv(pcie); 1465 1466 dev = pcie->dev; 1467 1468 ret = iproc_pcie_rev_init(pcie); 1469 if (ret) { 1470 dev_err(dev, "unable to initialize controller parameters\n"); 1471 return ret; 1472 } 1473 1474 ret = phy_init(pcie->phy); 1475 if (ret) { 1476 dev_err(dev, "unable to initialize PCIe PHY\n"); 1477 return ret; 1478 } 1479 1480 ret = phy_power_on(pcie->phy); 1481 if (ret) { 1482 dev_err(dev, "unable to power on PCIe PHY\n"); 1483 goto err_exit_phy; 1484 } 1485 1486 iproc_pcie_perst_ctrl(pcie, true); 1487 iproc_pcie_perst_ctrl(pcie, false); 1488 1489 iproc_pcie_invalidate_mapping(pcie); 1490 1491 if (pcie->need_ob_cfg) { 1492 ret = iproc_pcie_map_ranges(pcie, res); 1493 if (ret) { 1494 dev_err(dev, "map failed\n"); 1495 goto err_power_off_phy; 1496 } 1497 } 1498 1499 if (pcie->need_ib_cfg) { 1500 ret = iproc_pcie_map_dma_ranges(pcie); 1501 if (ret && ret != -ENOENT) 1502 goto err_power_off_phy; 1503 } 1504 1505 ret = iproc_pcie_check_link(pcie); 1506 if (ret) { 1507 dev_err(dev, "no PCIe EP device detected\n"); 1508 goto err_power_off_phy; 1509 } 1510 1511 iproc_pcie_enable(pcie); 1512 1513 if (IS_ENABLED(CONFIG_PCI_MSI)) 1514 if (iproc_pcie_msi_enable(pcie)) 1515 dev_info(dev, "not using iProc MSI\n"); 1516 1517 host->ops = &iproc_pcie_ops; 1518 host->sysdata = pcie; 1519 host->map_irq = pcie->map_irq; 1520 1521 ret = pci_host_probe(host); 1522 if (ret < 0) { 1523 dev_err(dev, "failed to scan host: %d\n", ret); 1524 goto err_power_off_phy; 1525 } 1526 1527 for_each_pci_bridge(pdev, host->bus) { 1528 if (pci_pcie_type(pdev) == PCI_EXP_TYPE_ROOT_PORT) 1529 pcie_print_link_status(pdev); 1530 } 1531 1532 return 0; 1533 1534 err_power_off_phy: 1535 phy_power_off(pcie->phy); 1536 err_exit_phy: 1537 phy_exit(pcie->phy); 1538 return ret; 1539 } 1540 EXPORT_SYMBOL(iproc_pcie_setup); 1541 1542 int iproc_pcie_remove(struct iproc_pcie *pcie) 1543 { 1544 struct pci_host_bridge *host = pci_host_bridge_from_priv(pcie); 1545 1546 pci_stop_root_bus(host->bus); 1547 pci_remove_root_bus(host->bus); 1548 1549 iproc_pcie_msi_disable(pcie); 1550 1551 phy_power_off(pcie->phy); 1552 phy_exit(pcie->phy); 1553 1554 return 0; 1555 } 1556 EXPORT_SYMBOL(iproc_pcie_remove); 1557 1558 /* 1559 * The MSI parsing logic in certain revisions of Broadcom PAXC based root 1560 * complex does not work and needs to be disabled 1561 */ 1562 static void quirk_paxc_disable_msi_parsing(struct pci_dev *pdev) 1563 { 1564 struct iproc_pcie *pcie = iproc_data(pdev->bus); 1565 1566 if (pdev->hdr_type == PCI_HEADER_TYPE_BRIDGE) 1567 iproc_pcie_paxc_v2_msi_steer(pcie, 0, false); 1568 } 1569 DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_BROADCOM, 0x16f0, 1570 quirk_paxc_disable_msi_parsing); 1571 DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_BROADCOM, 0xd802, 1572 quirk_paxc_disable_msi_parsing); 1573 DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_BROADCOM, 0xd804, 1574 quirk_paxc_disable_msi_parsing); 1575 1576 static void quirk_paxc_bridge(struct pci_dev *pdev) 1577 { 1578 /* 1579 * The PCI config space is shared with the PAXC root port and the first 1580 * Ethernet device. So, we need to workaround this by telling the PCI 1581 * code that the bridge is not an Ethernet device. 1582 */ 1583 if (pdev->hdr_type == PCI_HEADER_TYPE_BRIDGE) 1584 pdev->class = PCI_CLASS_BRIDGE_PCI << 8; 1585 1586 /* 1587 * MPSS is not being set properly (as it is currently 0). This is 1588 * because that area of the PCI config space is hard coded to zero, and 1589 * is not modifiable by firmware. Set this to 2 (e.g., 512 byte MPS) 1590 * so that the MPS can be set to the real max value. 1591 */ 1592 pdev->pcie_mpss = 2; 1593 } 1594 DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_BROADCOM, 0x16cd, quirk_paxc_bridge); 1595 DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_BROADCOM, 0x16f0, quirk_paxc_bridge); 1596 DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_BROADCOM, 0xd750, quirk_paxc_bridge); 1597 DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_BROADCOM, 0xd802, quirk_paxc_bridge); 1598 DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_BROADCOM, 0xd804, quirk_paxc_bridge); 1599 1600 MODULE_AUTHOR("Ray Jui <rjui@broadcom.com>"); 1601 MODULE_DESCRIPTION("Broadcom iPROC PCIe common driver"); 1602 MODULE_LICENSE("GPL v2"); 1603