1 // SPDX-License-Identifier: GPL-2.0 2 // Copyright (c) 2017 Cadence 3 // Cadence PCIe host controller driver. 4 // Author: Cyrille Pitchen <cyrille.pitchen@free-electrons.com> 5 6 #include <linux/delay.h> 7 #include <linux/kernel.h> 8 #include <linux/list_sort.h> 9 #include <linux/of_address.h> 10 #include <linux/of_pci.h> 11 #include <linux/platform_device.h> 12 13 #include "pcie-cadence.h" 14 15 static u64 bar_max_size[] = { 16 [RP_BAR0] = _ULL(128 * SZ_2G), 17 [RP_BAR1] = SZ_2G, 18 [RP_NO_BAR] = _BITULL(63), 19 }; 20 21 static u8 bar_aperture_mask[] = { 22 [RP_BAR0] = 0x1F, 23 [RP_BAR1] = 0xF, 24 }; 25 26 void __iomem *cdns_pci_map_bus(struct pci_bus *bus, unsigned int devfn, 27 int where) 28 { 29 struct pci_host_bridge *bridge = pci_find_host_bridge(bus); 30 struct cdns_pcie_rc *rc = pci_host_bridge_priv(bridge); 31 struct cdns_pcie *pcie = &rc->pcie; 32 unsigned int busn = bus->number; 33 u32 addr0, desc0; 34 35 if (pci_is_root_bus(bus)) { 36 /* 37 * Only the root port (devfn == 0) is connected to this bus. 38 * All other PCI devices are behind some bridge hence on another 39 * bus. 40 */ 41 if (devfn) 42 return NULL; 43 44 return pcie->reg_base + (where & 0xfff); 45 } 46 /* Check that the link is up */ 47 if (!(cdns_pcie_readl(pcie, CDNS_PCIE_LM_BASE) & 0x1)) 48 return NULL; 49 /* Clear AXI link-down status */ 50 cdns_pcie_writel(pcie, CDNS_PCIE_AT_LINKDOWN, 0x0); 51 52 /* Update Output registers for AXI region 0. */ 53 addr0 = CDNS_PCIE_AT_OB_REGION_PCI_ADDR0_NBITS(12) | 54 CDNS_PCIE_AT_OB_REGION_PCI_ADDR0_DEVFN(devfn) | 55 CDNS_PCIE_AT_OB_REGION_PCI_ADDR0_BUS(busn); 56 cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_PCI_ADDR0(0), addr0); 57 58 /* Configuration Type 0 or Type 1 access. */ 59 desc0 = CDNS_PCIE_AT_OB_REGION_DESC0_HARDCODED_RID | 60 CDNS_PCIE_AT_OB_REGION_DESC0_DEVFN(0); 61 /* 62 * The bus number was already set once for all in desc1 by 63 * cdns_pcie_host_init_address_translation(). 64 */ 65 if (busn == bridge->busnr + 1) 66 desc0 |= CDNS_PCIE_AT_OB_REGION_DESC0_TYPE_CONF_TYPE0; 67 else 68 desc0 |= CDNS_PCIE_AT_OB_REGION_DESC0_TYPE_CONF_TYPE1; 69 cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_DESC0(0), desc0); 70 71 return rc->cfg_base + (where & 0xfff); 72 } 73 74 static struct pci_ops cdns_pcie_host_ops = { 75 .map_bus = cdns_pci_map_bus, 76 .read = pci_generic_config_read, 77 .write = pci_generic_config_write, 78 }; 79 80 static int cdns_pcie_host_wait_for_link(struct cdns_pcie *pcie) 81 { 82 struct device *dev = pcie->dev; 83 int retries; 84 85 /* Check if the link is up or not */ 86 for (retries = 0; retries < LINK_WAIT_MAX_RETRIES; retries++) { 87 if (cdns_pcie_link_up(pcie)) { 88 dev_info(dev, "Link up\n"); 89 return 0; 90 } 91 usleep_range(LINK_WAIT_USLEEP_MIN, LINK_WAIT_USLEEP_MAX); 92 } 93 94 return -ETIMEDOUT; 95 } 96 97 static int cdns_pcie_retrain(struct cdns_pcie *pcie) 98 { 99 u32 lnk_cap_sls, pcie_cap_off = CDNS_PCIE_RP_CAP_OFFSET; 100 u16 lnk_stat, lnk_ctl; 101 int ret = 0; 102 103 /* 104 * Set retrain bit if current speed is 2.5 GB/s, 105 * but the PCIe root port support is > 2.5 GB/s. 106 */ 107 108 lnk_cap_sls = cdns_pcie_readl(pcie, (CDNS_PCIE_RP_BASE + pcie_cap_off + 109 PCI_EXP_LNKCAP)); 110 if ((lnk_cap_sls & PCI_EXP_LNKCAP_SLS) <= PCI_EXP_LNKCAP_SLS_2_5GB) 111 return ret; 112 113 lnk_stat = cdns_pcie_rp_readw(pcie, pcie_cap_off + PCI_EXP_LNKSTA); 114 if ((lnk_stat & PCI_EXP_LNKSTA_CLS) == PCI_EXP_LNKSTA_CLS_2_5GB) { 115 lnk_ctl = cdns_pcie_rp_readw(pcie, 116 pcie_cap_off + PCI_EXP_LNKCTL); 117 lnk_ctl |= PCI_EXP_LNKCTL_RL; 118 cdns_pcie_rp_writew(pcie, pcie_cap_off + PCI_EXP_LNKCTL, 119 lnk_ctl); 120 121 ret = cdns_pcie_host_wait_for_link(pcie); 122 } 123 return ret; 124 } 125 126 static void cdns_pcie_host_enable_ptm_response(struct cdns_pcie *pcie) 127 { 128 u32 val; 129 130 val = cdns_pcie_readl(pcie, CDNS_PCIE_LM_PTM_CTRL); 131 cdns_pcie_writel(pcie, CDNS_PCIE_LM_PTM_CTRL, val | CDNS_PCIE_LM_TPM_CTRL_PTMRSEN); 132 } 133 134 static int cdns_pcie_host_start_link(struct cdns_pcie_rc *rc) 135 { 136 struct cdns_pcie *pcie = &rc->pcie; 137 int ret; 138 139 ret = cdns_pcie_host_wait_for_link(pcie); 140 141 /* 142 * Retrain link for Gen2 training defect 143 * if quirk flag is set. 144 */ 145 if (!ret && rc->quirk_retrain_flag) 146 ret = cdns_pcie_retrain(pcie); 147 148 return ret; 149 } 150 151 static int cdns_pcie_host_init_root_port(struct cdns_pcie_rc *rc) 152 { 153 struct cdns_pcie *pcie = &rc->pcie; 154 u32 value, ctrl; 155 u32 id; 156 157 /* 158 * Set the root complex BAR configuration register: 159 * - disable both BAR0 and BAR1. 160 * - enable Prefetchable Memory Base and Limit registers in type 1 161 * config space (64 bits). 162 * - enable IO Base and Limit registers in type 1 config 163 * space (32 bits). 164 */ 165 ctrl = CDNS_PCIE_LM_BAR_CFG_CTRL_DISABLED; 166 value = CDNS_PCIE_LM_RC_BAR_CFG_BAR0_CTRL(ctrl) | 167 CDNS_PCIE_LM_RC_BAR_CFG_BAR1_CTRL(ctrl) | 168 CDNS_PCIE_LM_RC_BAR_CFG_PREFETCH_MEM_ENABLE | 169 CDNS_PCIE_LM_RC_BAR_CFG_PREFETCH_MEM_64BITS | 170 CDNS_PCIE_LM_RC_BAR_CFG_IO_ENABLE | 171 CDNS_PCIE_LM_RC_BAR_CFG_IO_32BITS; 172 cdns_pcie_writel(pcie, CDNS_PCIE_LM_RC_BAR_CFG, value); 173 174 /* Set root port configuration space */ 175 if (rc->vendor_id != 0xffff) { 176 id = CDNS_PCIE_LM_ID_VENDOR(rc->vendor_id) | 177 CDNS_PCIE_LM_ID_SUBSYS(rc->vendor_id); 178 cdns_pcie_writel(pcie, CDNS_PCIE_LM_ID, id); 179 } 180 181 if (rc->device_id != 0xffff) 182 cdns_pcie_rp_writew(pcie, PCI_DEVICE_ID, rc->device_id); 183 184 cdns_pcie_rp_writeb(pcie, PCI_CLASS_REVISION, 0); 185 cdns_pcie_rp_writeb(pcie, PCI_CLASS_PROG, 0); 186 cdns_pcie_rp_writew(pcie, PCI_CLASS_DEVICE, PCI_CLASS_BRIDGE_PCI); 187 188 return 0; 189 } 190 191 static int cdns_pcie_host_bar_ib_config(struct cdns_pcie_rc *rc, 192 enum cdns_pcie_rp_bar bar, 193 u64 cpu_addr, u64 size, 194 unsigned long flags) 195 { 196 struct cdns_pcie *pcie = &rc->pcie; 197 u32 addr0, addr1, aperture, value; 198 199 if (!rc->avail_ib_bar[bar]) 200 return -EBUSY; 201 202 rc->avail_ib_bar[bar] = false; 203 204 aperture = ilog2(size); 205 addr0 = CDNS_PCIE_AT_IB_RP_BAR_ADDR0_NBITS(aperture) | 206 (lower_32_bits(cpu_addr) & GENMASK(31, 8)); 207 addr1 = upper_32_bits(cpu_addr); 208 cdns_pcie_writel(pcie, CDNS_PCIE_AT_IB_RP_BAR_ADDR0(bar), addr0); 209 cdns_pcie_writel(pcie, CDNS_PCIE_AT_IB_RP_BAR_ADDR1(bar), addr1); 210 211 if (bar == RP_NO_BAR) 212 return 0; 213 214 value = cdns_pcie_readl(pcie, CDNS_PCIE_LM_RC_BAR_CFG); 215 value &= ~(LM_RC_BAR_CFG_CTRL_MEM_64BITS(bar) | 216 LM_RC_BAR_CFG_CTRL_PREF_MEM_64BITS(bar) | 217 LM_RC_BAR_CFG_CTRL_MEM_32BITS(bar) | 218 LM_RC_BAR_CFG_CTRL_PREF_MEM_32BITS(bar) | 219 LM_RC_BAR_CFG_APERTURE(bar, bar_aperture_mask[bar] + 2)); 220 if (size + cpu_addr >= SZ_4G) { 221 if (!(flags & IORESOURCE_PREFETCH)) 222 value |= LM_RC_BAR_CFG_CTRL_MEM_64BITS(bar); 223 value |= LM_RC_BAR_CFG_CTRL_PREF_MEM_64BITS(bar); 224 } else { 225 if (!(flags & IORESOURCE_PREFETCH)) 226 value |= LM_RC_BAR_CFG_CTRL_MEM_32BITS(bar); 227 value |= LM_RC_BAR_CFG_CTRL_PREF_MEM_32BITS(bar); 228 } 229 230 value |= LM_RC_BAR_CFG_APERTURE(bar, aperture); 231 cdns_pcie_writel(pcie, CDNS_PCIE_LM_RC_BAR_CFG, value); 232 233 return 0; 234 } 235 236 static enum cdns_pcie_rp_bar 237 cdns_pcie_host_find_min_bar(struct cdns_pcie_rc *rc, u64 size) 238 { 239 enum cdns_pcie_rp_bar bar, sel_bar; 240 241 sel_bar = RP_BAR_UNDEFINED; 242 for (bar = RP_BAR0; bar <= RP_NO_BAR; bar++) { 243 if (!rc->avail_ib_bar[bar]) 244 continue; 245 246 if (size <= bar_max_size[bar]) { 247 if (sel_bar == RP_BAR_UNDEFINED) { 248 sel_bar = bar; 249 continue; 250 } 251 252 if (bar_max_size[bar] < bar_max_size[sel_bar]) 253 sel_bar = bar; 254 } 255 } 256 257 return sel_bar; 258 } 259 260 static enum cdns_pcie_rp_bar 261 cdns_pcie_host_find_max_bar(struct cdns_pcie_rc *rc, u64 size) 262 { 263 enum cdns_pcie_rp_bar bar, sel_bar; 264 265 sel_bar = RP_BAR_UNDEFINED; 266 for (bar = RP_BAR0; bar <= RP_NO_BAR; bar++) { 267 if (!rc->avail_ib_bar[bar]) 268 continue; 269 270 if (size >= bar_max_size[bar]) { 271 if (sel_bar == RP_BAR_UNDEFINED) { 272 sel_bar = bar; 273 continue; 274 } 275 276 if (bar_max_size[bar] > bar_max_size[sel_bar]) 277 sel_bar = bar; 278 } 279 } 280 281 return sel_bar; 282 } 283 284 static int cdns_pcie_host_bar_config(struct cdns_pcie_rc *rc, 285 struct resource_entry *entry) 286 { 287 u64 cpu_addr, pci_addr, size, winsize; 288 struct cdns_pcie *pcie = &rc->pcie; 289 struct device *dev = pcie->dev; 290 enum cdns_pcie_rp_bar bar; 291 unsigned long flags; 292 int ret; 293 294 cpu_addr = entry->res->start; 295 pci_addr = entry->res->start - entry->offset; 296 flags = entry->res->flags; 297 size = resource_size(entry->res); 298 299 if (entry->offset) { 300 dev_err(dev, "PCI addr: %llx must be equal to CPU addr: %llx\n", 301 pci_addr, cpu_addr); 302 return -EINVAL; 303 } 304 305 while (size > 0) { 306 /* 307 * Try to find a minimum BAR whose size is greater than 308 * or equal to the remaining resource_entry size. This will 309 * fail if the size of each of the available BARs is less than 310 * the remaining resource_entry size. 311 * If a minimum BAR is found, IB ATU will be configured and 312 * exited. 313 */ 314 bar = cdns_pcie_host_find_min_bar(rc, size); 315 if (bar != RP_BAR_UNDEFINED) { 316 ret = cdns_pcie_host_bar_ib_config(rc, bar, cpu_addr, 317 size, flags); 318 if (ret) 319 dev_err(dev, "IB BAR: %d config failed\n", bar); 320 return ret; 321 } 322 323 /* 324 * If the control reaches here, it would mean the remaining 325 * resource_entry size cannot be fitted in a single BAR. So we 326 * find a maximum BAR whose size is less than or equal to the 327 * remaining resource_entry size and split the resource entry 328 * so that part of resource entry is fitted inside the maximum 329 * BAR. The remaining size would be fitted during the next 330 * iteration of the loop. 331 * If a maximum BAR is not found, there is no way we can fit 332 * this resource_entry, so we error out. 333 */ 334 bar = cdns_pcie_host_find_max_bar(rc, size); 335 if (bar == RP_BAR_UNDEFINED) { 336 dev_err(dev, "No free BAR to map cpu_addr %llx\n", 337 cpu_addr); 338 return -EINVAL; 339 } 340 341 winsize = bar_max_size[bar]; 342 ret = cdns_pcie_host_bar_ib_config(rc, bar, cpu_addr, winsize, 343 flags); 344 if (ret) { 345 dev_err(dev, "IB BAR: %d config failed\n", bar); 346 return ret; 347 } 348 349 size -= winsize; 350 cpu_addr += winsize; 351 } 352 353 return 0; 354 } 355 356 static int cdns_pcie_host_dma_ranges_cmp(void *priv, const struct list_head *a, 357 const struct list_head *b) 358 { 359 struct resource_entry *entry1, *entry2; 360 361 entry1 = container_of(a, struct resource_entry, node); 362 entry2 = container_of(b, struct resource_entry, node); 363 364 return resource_size(entry2->res) - resource_size(entry1->res); 365 } 366 367 static int cdns_pcie_host_map_dma_ranges(struct cdns_pcie_rc *rc) 368 { 369 struct cdns_pcie *pcie = &rc->pcie; 370 struct device *dev = pcie->dev; 371 struct device_node *np = dev->of_node; 372 struct pci_host_bridge *bridge; 373 struct resource_entry *entry; 374 u32 no_bar_nbits = 32; 375 int err; 376 377 bridge = pci_host_bridge_from_priv(rc); 378 if (!bridge) 379 return -ENOMEM; 380 381 if (list_empty(&bridge->dma_ranges)) { 382 of_property_read_u32(np, "cdns,no-bar-match-nbits", 383 &no_bar_nbits); 384 err = cdns_pcie_host_bar_ib_config(rc, RP_NO_BAR, 0x0, 385 (u64)1 << no_bar_nbits, 0); 386 if (err) 387 dev_err(dev, "IB BAR: %d config failed\n", RP_NO_BAR); 388 return err; 389 } 390 391 list_sort(NULL, &bridge->dma_ranges, cdns_pcie_host_dma_ranges_cmp); 392 393 resource_list_for_each_entry(entry, &bridge->dma_ranges) { 394 err = cdns_pcie_host_bar_config(rc, entry); 395 if (err) { 396 dev_err(dev, "Fail to configure IB using dma-ranges\n"); 397 return err; 398 } 399 } 400 401 return 0; 402 } 403 404 static int cdns_pcie_host_init_address_translation(struct cdns_pcie_rc *rc) 405 { 406 struct cdns_pcie *pcie = &rc->pcie; 407 struct pci_host_bridge *bridge = pci_host_bridge_from_priv(rc); 408 struct resource *cfg_res = rc->cfg_res; 409 struct resource_entry *entry; 410 u64 cpu_addr = cfg_res->start; 411 u32 addr0, addr1, desc1; 412 int r, busnr = 0; 413 414 entry = resource_list_first_type(&bridge->windows, IORESOURCE_BUS); 415 if (entry) 416 busnr = entry->res->start; 417 418 /* 419 * Reserve region 0 for PCI configure space accesses: 420 * OB_REGION_PCI_ADDR0 and OB_REGION_DESC0 are updated dynamically by 421 * cdns_pci_map_bus(), other region registers are set here once for all. 422 */ 423 addr1 = 0; /* Should be programmed to zero. */ 424 desc1 = CDNS_PCIE_AT_OB_REGION_DESC1_BUS(busnr); 425 cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_PCI_ADDR1(0), addr1); 426 cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_DESC1(0), desc1); 427 428 if (pcie->ops->cpu_addr_fixup) 429 cpu_addr = pcie->ops->cpu_addr_fixup(pcie, cpu_addr); 430 431 addr0 = CDNS_PCIE_AT_OB_REGION_CPU_ADDR0_NBITS(12) | 432 (lower_32_bits(cpu_addr) & GENMASK(31, 8)); 433 addr1 = upper_32_bits(cpu_addr); 434 cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_CPU_ADDR0(0), addr0); 435 cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_CPU_ADDR1(0), addr1); 436 437 r = 1; 438 resource_list_for_each_entry(entry, &bridge->windows) { 439 struct resource *res = entry->res; 440 u64 pci_addr = res->start - entry->offset; 441 442 if (resource_type(res) == IORESOURCE_IO) 443 cdns_pcie_set_outbound_region(pcie, busnr, 0, r, 444 true, 445 pci_pio_to_address(res->start), 446 pci_addr, 447 resource_size(res)); 448 else 449 cdns_pcie_set_outbound_region(pcie, busnr, 0, r, 450 false, 451 res->start, 452 pci_addr, 453 resource_size(res)); 454 455 r++; 456 } 457 458 return cdns_pcie_host_map_dma_ranges(rc); 459 } 460 461 static int cdns_pcie_host_init(struct device *dev, 462 struct cdns_pcie_rc *rc) 463 { 464 int err; 465 466 err = cdns_pcie_host_init_root_port(rc); 467 if (err) 468 return err; 469 470 return cdns_pcie_host_init_address_translation(rc); 471 } 472 473 int cdns_pcie_host_setup(struct cdns_pcie_rc *rc) 474 { 475 struct device *dev = rc->pcie.dev; 476 struct platform_device *pdev = to_platform_device(dev); 477 struct device_node *np = dev->of_node; 478 struct pci_host_bridge *bridge; 479 enum cdns_pcie_rp_bar bar; 480 struct cdns_pcie *pcie; 481 struct resource *res; 482 int ret; 483 484 bridge = pci_host_bridge_from_priv(rc); 485 if (!bridge) 486 return -ENOMEM; 487 488 pcie = &rc->pcie; 489 pcie->is_rc = true; 490 491 rc->vendor_id = 0xffff; 492 of_property_read_u32(np, "vendor-id", &rc->vendor_id); 493 494 rc->device_id = 0xffff; 495 of_property_read_u32(np, "device-id", &rc->device_id); 496 497 pcie->reg_base = devm_platform_ioremap_resource_byname(pdev, "reg"); 498 if (IS_ERR(pcie->reg_base)) { 499 dev_err(dev, "missing \"reg\"\n"); 500 return PTR_ERR(pcie->reg_base); 501 } 502 503 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "cfg"); 504 rc->cfg_base = devm_pci_remap_cfg_resource(dev, res); 505 if (IS_ERR(rc->cfg_base)) 506 return PTR_ERR(rc->cfg_base); 507 rc->cfg_res = res; 508 509 if (rc->quirk_detect_quiet_flag) 510 cdns_pcie_detect_quiet_min_delay_set(&rc->pcie); 511 512 cdns_pcie_host_enable_ptm_response(pcie); 513 514 ret = cdns_pcie_start_link(pcie); 515 if (ret) { 516 dev_err(dev, "Failed to start link\n"); 517 return ret; 518 } 519 520 ret = cdns_pcie_host_start_link(rc); 521 if (ret) 522 dev_dbg(dev, "PCIe link never came up\n"); 523 524 for (bar = RP_BAR0; bar <= RP_NO_BAR; bar++) 525 rc->avail_ib_bar[bar] = true; 526 527 ret = cdns_pcie_host_init(dev, rc); 528 if (ret) 529 return ret; 530 531 if (!bridge->ops) 532 bridge->ops = &cdns_pcie_host_ops; 533 534 ret = pci_host_probe(bridge); 535 if (ret < 0) 536 goto err_init; 537 538 return 0; 539 540 err_init: 541 pm_runtime_put_sync(dev); 542 543 return ret; 544 } 545