1 // SPDX-License-Identifier: GPL-2.0+ 2 /* 3 * APM X-Gene PCIe Driver 4 * 5 * Copyright (c) 2014 Applied Micro Circuits Corporation. 6 * 7 * Author: Tanmay Inamdar <tinamdar@apm.com>. 8 */ 9 #include <linux/clk.h> 10 #include <linux/delay.h> 11 #include <linux/io.h> 12 #include <linux/jiffies.h> 13 #include <linux/memblock.h> 14 #include <linux/init.h> 15 #include <linux/of.h> 16 #include <linux/of_address.h> 17 #include <linux/of_pci.h> 18 #include <linux/pci.h> 19 #include <linux/pci-acpi.h> 20 #include <linux/pci-ecam.h> 21 #include <linux/platform_device.h> 22 #include <linux/slab.h> 23 24 #include "../pci.h" 25 26 #define PCIECORE_CTLANDSTATUS 0x50 27 #define PIM1_1L 0x80 28 #define IBAR2 0x98 29 #define IR2MSK 0x9c 30 #define PIM2_1L 0xa0 31 #define IBAR3L 0xb4 32 #define IR3MSKL 0xbc 33 #define PIM3_1L 0xc4 34 #define OMR1BARL 0x100 35 #define OMR2BARL 0x118 36 #define OMR3BARL 0x130 37 #define CFGBARL 0x154 38 #define CFGBARH 0x158 39 #define CFGCTL 0x15c 40 #define RTDID 0x160 41 #define BRIDGE_CFG_0 0x2000 42 #define BRIDGE_CFG_4 0x2010 43 #define BRIDGE_STATUS_0 0x2600 44 45 #define LINK_UP_MASK 0x00000100 46 #define AXI_EP_CFG_ACCESS 0x10000 47 #define EN_COHERENCY 0xF0000000 48 #define EN_REG 0x00000001 49 #define OB_LO_IO 0x00000002 50 #define XGENE_PCIE_DEVICEID 0xE004 51 #define PIPE_PHY_RATE_RD(src) ((0xc000 & (u32)(src)) >> 0xe) 52 53 #define XGENE_V1_PCI_EXP_CAP 0x40 54 55 /* PCIe IP version */ 56 #define XGENE_PCIE_IP_VER_UNKN 0 57 #define XGENE_PCIE_IP_VER_1 1 58 #define XGENE_PCIE_IP_VER_2 2 59 60 #if defined(CONFIG_PCI_XGENE) || (defined(CONFIG_ACPI) && defined(CONFIG_PCI_QUIRKS)) 61 struct xgene_pcie { 62 struct device_node *node; 63 struct device *dev; 64 struct clk *clk; 65 void __iomem *csr_base; 66 void __iomem *cfg_base; 67 unsigned long cfg_addr; 68 bool link_up; 69 u32 version; 70 }; 71 72 static u32 xgene_pcie_readl(struct xgene_pcie *port, u32 reg) 73 { 74 return readl(port->csr_base + reg); 75 } 76 77 static void xgene_pcie_writel(struct xgene_pcie *port, u32 reg, u32 val) 78 { 79 writel(val, port->csr_base + reg); 80 } 81 82 static inline u32 pcie_bar_low_val(u32 addr, u32 flags) 83 { 84 return (addr & PCI_BASE_ADDRESS_MEM_MASK) | flags; 85 } 86 87 static inline struct xgene_pcie *pcie_bus_to_port(struct pci_bus *bus) 88 { 89 struct pci_config_window *cfg; 90 91 if (acpi_disabled) 92 return (struct xgene_pcie *)(bus->sysdata); 93 94 cfg = bus->sysdata; 95 return (struct xgene_pcie *)(cfg->priv); 96 } 97 98 /* 99 * When the address bit [17:16] is 2'b01, the Configuration access will be 100 * treated as Type 1 and it will be forwarded to external PCIe device. 101 */ 102 static void __iomem *xgene_pcie_get_cfg_base(struct pci_bus *bus) 103 { 104 struct xgene_pcie *port = pcie_bus_to_port(bus); 105 106 if (bus->number >= (bus->primary + 1)) 107 return port->cfg_base + AXI_EP_CFG_ACCESS; 108 109 return port->cfg_base; 110 } 111 112 /* 113 * For Configuration request, RTDID register is used as Bus Number, 114 * Device Number and Function number of the header fields. 115 */ 116 static void xgene_pcie_set_rtdid_reg(struct pci_bus *bus, uint devfn) 117 { 118 struct xgene_pcie *port = pcie_bus_to_port(bus); 119 unsigned int b, d, f; 120 u32 rtdid_val = 0; 121 122 b = bus->number; 123 d = PCI_SLOT(devfn); 124 f = PCI_FUNC(devfn); 125 126 if (!pci_is_root_bus(bus)) 127 rtdid_val = (b << 8) | (d << 3) | f; 128 129 xgene_pcie_writel(port, RTDID, rtdid_val); 130 /* read the register back to ensure flush */ 131 xgene_pcie_readl(port, RTDID); 132 } 133 134 /* 135 * X-Gene PCIe port uses BAR0-BAR1 of RC's configuration space as 136 * the translation from PCI bus to native BUS. Entire DDR region 137 * is mapped into PCIe space using these registers, so it can be 138 * reached by DMA from EP devices. The BAR0/1 of bridge should be 139 * hidden during enumeration to avoid the sizing and resource allocation 140 * by PCIe core. 141 */ 142 static bool xgene_pcie_hide_rc_bars(struct pci_bus *bus, int offset) 143 { 144 if (pci_is_root_bus(bus) && ((offset == PCI_BASE_ADDRESS_0) || 145 (offset == PCI_BASE_ADDRESS_1))) 146 return true; 147 148 return false; 149 } 150 151 static void __iomem *xgene_pcie_map_bus(struct pci_bus *bus, unsigned int devfn, 152 int offset) 153 { 154 if ((pci_is_root_bus(bus) && devfn != 0) || 155 xgene_pcie_hide_rc_bars(bus, offset)) 156 return NULL; 157 158 xgene_pcie_set_rtdid_reg(bus, devfn); 159 return xgene_pcie_get_cfg_base(bus) + offset; 160 } 161 162 static int xgene_pcie_config_read32(struct pci_bus *bus, unsigned int devfn, 163 int where, int size, u32 *val) 164 { 165 struct xgene_pcie *port = pcie_bus_to_port(bus); 166 167 if (pci_generic_config_read32(bus, devfn, where & ~0x3, 4, val) != 168 PCIBIOS_SUCCESSFUL) 169 return PCIBIOS_DEVICE_NOT_FOUND; 170 171 /* 172 * The v1 controller has a bug in its Configuration Request Retry 173 * Status (CRS) logic: when CRS Software Visibility is enabled and 174 * we read the Vendor and Device ID of a non-existent device, the 175 * controller fabricates return data of 0xFFFF0001 ("device exists 176 * but is not ready") instead of 0xFFFFFFFF (PCI_ERROR_RESPONSE) 177 * ("device does not exist"). This causes the PCI core to retry 178 * the read until it times out. Avoid this by not claiming to 179 * support CRS SV. 180 */ 181 if (pci_is_root_bus(bus) && (port->version == XGENE_PCIE_IP_VER_1) && 182 ((where & ~0x3) == XGENE_V1_PCI_EXP_CAP + PCI_EXP_RTCTL)) 183 *val &= ~(PCI_EXP_RTCAP_CRSVIS << 16); 184 185 if (size <= 2) 186 *val = (*val >> (8 * (where & 3))) & ((1 << (size * 8)) - 1); 187 188 return PCIBIOS_SUCCESSFUL; 189 } 190 #endif 191 192 #if defined(CONFIG_ACPI) && defined(CONFIG_PCI_QUIRKS) 193 static int xgene_get_csr_resource(struct acpi_device *adev, 194 struct resource *res) 195 { 196 struct device *dev = &adev->dev; 197 struct resource_entry *entry; 198 struct list_head list; 199 unsigned long flags; 200 int ret; 201 202 INIT_LIST_HEAD(&list); 203 flags = IORESOURCE_MEM; 204 ret = acpi_dev_get_resources(adev, &list, 205 acpi_dev_filter_resource_type_cb, 206 (void *) flags); 207 if (ret < 0) { 208 dev_err(dev, "failed to parse _CRS method, error code %d\n", 209 ret); 210 return ret; 211 } 212 213 if (ret == 0) { 214 dev_err(dev, "no IO and memory resources present in _CRS\n"); 215 return -EINVAL; 216 } 217 218 entry = list_first_entry(&list, struct resource_entry, node); 219 *res = *entry->res; 220 acpi_dev_free_resource_list(&list); 221 return 0; 222 } 223 224 static int xgene_pcie_ecam_init(struct pci_config_window *cfg, u32 ipversion) 225 { 226 struct device *dev = cfg->parent; 227 struct acpi_device *adev = to_acpi_device(dev); 228 struct xgene_pcie *port; 229 struct resource csr; 230 int ret; 231 232 port = devm_kzalloc(dev, sizeof(*port), GFP_KERNEL); 233 if (!port) 234 return -ENOMEM; 235 236 ret = xgene_get_csr_resource(adev, &csr); 237 if (ret) { 238 dev_err(dev, "can't get CSR resource\n"); 239 return ret; 240 } 241 port->csr_base = devm_pci_remap_cfg_resource(dev, &csr); 242 if (IS_ERR(port->csr_base)) 243 return PTR_ERR(port->csr_base); 244 245 port->cfg_base = cfg->win; 246 port->version = ipversion; 247 248 cfg->priv = port; 249 return 0; 250 } 251 252 static int xgene_v1_pcie_ecam_init(struct pci_config_window *cfg) 253 { 254 return xgene_pcie_ecam_init(cfg, XGENE_PCIE_IP_VER_1); 255 } 256 257 const struct pci_ecam_ops xgene_v1_pcie_ecam_ops = { 258 .init = xgene_v1_pcie_ecam_init, 259 .pci_ops = { 260 .map_bus = xgene_pcie_map_bus, 261 .read = xgene_pcie_config_read32, 262 .write = pci_generic_config_write, 263 } 264 }; 265 266 static int xgene_v2_pcie_ecam_init(struct pci_config_window *cfg) 267 { 268 return xgene_pcie_ecam_init(cfg, XGENE_PCIE_IP_VER_2); 269 } 270 271 const struct pci_ecam_ops xgene_v2_pcie_ecam_ops = { 272 .init = xgene_v2_pcie_ecam_init, 273 .pci_ops = { 274 .map_bus = xgene_pcie_map_bus, 275 .read = xgene_pcie_config_read32, 276 .write = pci_generic_config_write, 277 } 278 }; 279 #endif 280 281 #if defined(CONFIG_PCI_XGENE) 282 static u64 xgene_pcie_set_ib_mask(struct xgene_pcie *port, u32 addr, 283 u32 flags, u64 size) 284 { 285 u64 mask = (~(size - 1) & PCI_BASE_ADDRESS_MEM_MASK) | flags; 286 u32 val32 = 0; 287 u32 val; 288 289 val32 = xgene_pcie_readl(port, addr); 290 val = (val32 & 0x0000ffff) | (lower_32_bits(mask) << 16); 291 xgene_pcie_writel(port, addr, val); 292 293 val32 = xgene_pcie_readl(port, addr + 0x04); 294 val = (val32 & 0xffff0000) | (lower_32_bits(mask) >> 16); 295 xgene_pcie_writel(port, addr + 0x04, val); 296 297 val32 = xgene_pcie_readl(port, addr + 0x04); 298 val = (val32 & 0x0000ffff) | (upper_32_bits(mask) << 16); 299 xgene_pcie_writel(port, addr + 0x04, val); 300 301 val32 = xgene_pcie_readl(port, addr + 0x08); 302 val = (val32 & 0xffff0000) | (upper_32_bits(mask) >> 16); 303 xgene_pcie_writel(port, addr + 0x08, val); 304 305 return mask; 306 } 307 308 static void xgene_pcie_linkup(struct xgene_pcie *port, 309 u32 *lanes, u32 *speed) 310 { 311 u32 val32; 312 313 port->link_up = false; 314 val32 = xgene_pcie_readl(port, PCIECORE_CTLANDSTATUS); 315 if (val32 & LINK_UP_MASK) { 316 port->link_up = true; 317 *speed = PIPE_PHY_RATE_RD(val32); 318 val32 = xgene_pcie_readl(port, BRIDGE_STATUS_0); 319 *lanes = val32 >> 26; 320 } 321 } 322 323 static int xgene_pcie_init_port(struct xgene_pcie *port) 324 { 325 struct device *dev = port->dev; 326 int rc; 327 328 port->clk = clk_get(dev, NULL); 329 if (IS_ERR(port->clk)) { 330 dev_err(dev, "clock not available\n"); 331 return -ENODEV; 332 } 333 334 rc = clk_prepare_enable(port->clk); 335 if (rc) { 336 dev_err(dev, "clock enable failed\n"); 337 return rc; 338 } 339 340 return 0; 341 } 342 343 static int xgene_pcie_map_reg(struct xgene_pcie *port, 344 struct platform_device *pdev) 345 { 346 struct device *dev = port->dev; 347 struct resource *res; 348 349 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "csr"); 350 port->csr_base = devm_pci_remap_cfg_resource(dev, res); 351 if (IS_ERR(port->csr_base)) 352 return PTR_ERR(port->csr_base); 353 354 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "cfg"); 355 port->cfg_base = devm_ioremap_resource(dev, res); 356 if (IS_ERR(port->cfg_base)) 357 return PTR_ERR(port->cfg_base); 358 port->cfg_addr = res->start; 359 360 return 0; 361 } 362 363 static void xgene_pcie_setup_ob_reg(struct xgene_pcie *port, 364 struct resource *res, u32 offset, 365 u64 cpu_addr, u64 pci_addr) 366 { 367 struct device *dev = port->dev; 368 resource_size_t size = resource_size(res); 369 u64 restype = resource_type(res); 370 u64 mask = 0; 371 u32 min_size; 372 u32 flag = EN_REG; 373 374 if (restype == IORESOURCE_MEM) { 375 min_size = SZ_128M; 376 } else { 377 min_size = 128; 378 flag |= OB_LO_IO; 379 } 380 381 if (size >= min_size) 382 mask = ~(size - 1) | flag; 383 else 384 dev_warn(dev, "res size 0x%llx less than minimum 0x%x\n", 385 (u64)size, min_size); 386 387 xgene_pcie_writel(port, offset, lower_32_bits(cpu_addr)); 388 xgene_pcie_writel(port, offset + 0x04, upper_32_bits(cpu_addr)); 389 xgene_pcie_writel(port, offset + 0x08, lower_32_bits(mask)); 390 xgene_pcie_writel(port, offset + 0x0c, upper_32_bits(mask)); 391 xgene_pcie_writel(port, offset + 0x10, lower_32_bits(pci_addr)); 392 xgene_pcie_writel(port, offset + 0x14, upper_32_bits(pci_addr)); 393 } 394 395 static void xgene_pcie_setup_cfg_reg(struct xgene_pcie *port) 396 { 397 u64 addr = port->cfg_addr; 398 399 xgene_pcie_writel(port, CFGBARL, lower_32_bits(addr)); 400 xgene_pcie_writel(port, CFGBARH, upper_32_bits(addr)); 401 xgene_pcie_writel(port, CFGCTL, EN_REG); 402 } 403 404 static int xgene_pcie_map_ranges(struct xgene_pcie *port) 405 { 406 struct pci_host_bridge *bridge = pci_host_bridge_from_priv(port); 407 struct resource_entry *window; 408 struct device *dev = port->dev; 409 410 resource_list_for_each_entry(window, &bridge->windows) { 411 struct resource *res = window->res; 412 u64 restype = resource_type(res); 413 414 dev_dbg(dev, "%pR\n", res); 415 416 switch (restype) { 417 case IORESOURCE_IO: 418 xgene_pcie_setup_ob_reg(port, res, OMR3BARL, 419 pci_pio_to_address(res->start), 420 res->start - window->offset); 421 break; 422 case IORESOURCE_MEM: 423 if (res->flags & IORESOURCE_PREFETCH) 424 xgene_pcie_setup_ob_reg(port, res, OMR2BARL, 425 res->start, 426 res->start - 427 window->offset); 428 else 429 xgene_pcie_setup_ob_reg(port, res, OMR1BARL, 430 res->start, 431 res->start - 432 window->offset); 433 break; 434 case IORESOURCE_BUS: 435 break; 436 default: 437 dev_err(dev, "invalid resource %pR\n", res); 438 return -EINVAL; 439 } 440 } 441 xgene_pcie_setup_cfg_reg(port); 442 return 0; 443 } 444 445 static void xgene_pcie_setup_pims(struct xgene_pcie *port, u32 pim_reg, 446 u64 pim, u64 size) 447 { 448 xgene_pcie_writel(port, pim_reg, lower_32_bits(pim)); 449 xgene_pcie_writel(port, pim_reg + 0x04, 450 upper_32_bits(pim) | EN_COHERENCY); 451 xgene_pcie_writel(port, pim_reg + 0x10, lower_32_bits(size)); 452 xgene_pcie_writel(port, pim_reg + 0x14, upper_32_bits(size)); 453 } 454 455 /* 456 * X-Gene PCIe support maximum 3 inbound memory regions 457 * This function helps to select a region based on size of region 458 */ 459 static int xgene_pcie_select_ib_reg(u8 *ib_reg_mask, u64 size) 460 { 461 if ((size > 4) && (size < SZ_16M) && !(*ib_reg_mask & (1 << 1))) { 462 *ib_reg_mask |= (1 << 1); 463 return 1; 464 } 465 466 if ((size > SZ_1K) && (size < SZ_1T) && !(*ib_reg_mask & (1 << 0))) { 467 *ib_reg_mask |= (1 << 0); 468 return 0; 469 } 470 471 if ((size > SZ_1M) && (size < SZ_1T) && !(*ib_reg_mask & (1 << 2))) { 472 *ib_reg_mask |= (1 << 2); 473 return 2; 474 } 475 476 return -EINVAL; 477 } 478 479 static void xgene_pcie_setup_ib_reg(struct xgene_pcie *port, 480 struct of_pci_range *range, u8 *ib_reg_mask) 481 { 482 void __iomem *cfg_base = port->cfg_base; 483 struct device *dev = port->dev; 484 void __iomem *bar_addr; 485 u32 pim_reg; 486 u64 cpu_addr = range->cpu_addr; 487 u64 pci_addr = range->pci_addr; 488 u64 size = range->size; 489 u64 mask = ~(size - 1) | EN_REG; 490 u32 flags = PCI_BASE_ADDRESS_MEM_TYPE_64; 491 u32 bar_low; 492 int region; 493 494 region = xgene_pcie_select_ib_reg(ib_reg_mask, range->size); 495 if (region < 0) { 496 dev_warn(dev, "invalid pcie dma-range config\n"); 497 return; 498 } 499 500 if (range->flags & IORESOURCE_PREFETCH) 501 flags |= PCI_BASE_ADDRESS_MEM_PREFETCH; 502 503 bar_low = pcie_bar_low_val((u32)cpu_addr, flags); 504 switch (region) { 505 case 0: 506 xgene_pcie_set_ib_mask(port, BRIDGE_CFG_4, flags, size); 507 bar_addr = cfg_base + PCI_BASE_ADDRESS_0; 508 writel(bar_low, bar_addr); 509 writel(upper_32_bits(cpu_addr), bar_addr + 0x4); 510 pim_reg = PIM1_1L; 511 break; 512 case 1: 513 xgene_pcie_writel(port, IBAR2, bar_low); 514 xgene_pcie_writel(port, IR2MSK, lower_32_bits(mask)); 515 pim_reg = PIM2_1L; 516 break; 517 case 2: 518 xgene_pcie_writel(port, IBAR3L, bar_low); 519 xgene_pcie_writel(port, IBAR3L + 0x4, upper_32_bits(cpu_addr)); 520 xgene_pcie_writel(port, IR3MSKL, lower_32_bits(mask)); 521 xgene_pcie_writel(port, IR3MSKL + 0x4, upper_32_bits(mask)); 522 pim_reg = PIM3_1L; 523 break; 524 } 525 526 xgene_pcie_setup_pims(port, pim_reg, pci_addr, ~(size - 1)); 527 } 528 529 static int xgene_pcie_parse_map_dma_ranges(struct xgene_pcie *port) 530 { 531 struct device_node *np = port->node; 532 struct of_pci_range range; 533 struct of_pci_range_parser parser; 534 struct device *dev = port->dev; 535 u8 ib_reg_mask = 0; 536 537 if (of_pci_dma_range_parser_init(&parser, np)) { 538 dev_err(dev, "missing dma-ranges property\n"); 539 return -EINVAL; 540 } 541 542 /* Get the dma-ranges from DT */ 543 for_each_of_pci_range(&parser, &range) { 544 u64 end = range.cpu_addr + range.size - 1; 545 546 dev_dbg(dev, "0x%08x 0x%016llx..0x%016llx -> 0x%016llx\n", 547 range.flags, range.cpu_addr, end, range.pci_addr); 548 xgene_pcie_setup_ib_reg(port, &range, &ib_reg_mask); 549 } 550 return 0; 551 } 552 553 /* clear BAR configuration which was done by firmware */ 554 static void xgene_pcie_clear_config(struct xgene_pcie *port) 555 { 556 int i; 557 558 for (i = PIM1_1L; i <= CFGCTL; i += 4) 559 xgene_pcie_writel(port, i, 0); 560 } 561 562 static int xgene_pcie_setup(struct xgene_pcie *port) 563 { 564 struct device *dev = port->dev; 565 u32 val, lanes = 0, speed = 0; 566 int ret; 567 568 xgene_pcie_clear_config(port); 569 570 /* setup the vendor and device IDs correctly */ 571 val = (XGENE_PCIE_DEVICEID << 16) | PCI_VENDOR_ID_AMCC; 572 xgene_pcie_writel(port, BRIDGE_CFG_0, val); 573 574 ret = xgene_pcie_map_ranges(port); 575 if (ret) 576 return ret; 577 578 ret = xgene_pcie_parse_map_dma_ranges(port); 579 if (ret) 580 return ret; 581 582 xgene_pcie_linkup(port, &lanes, &speed); 583 if (!port->link_up) 584 dev_info(dev, "(rc) link down\n"); 585 else 586 dev_info(dev, "(rc) x%d gen-%d link up\n", lanes, speed + 1); 587 return 0; 588 } 589 590 static struct pci_ops xgene_pcie_ops = { 591 .map_bus = xgene_pcie_map_bus, 592 .read = xgene_pcie_config_read32, 593 .write = pci_generic_config_write32, 594 }; 595 596 static int xgene_pcie_probe(struct platform_device *pdev) 597 { 598 struct device *dev = &pdev->dev; 599 struct device_node *dn = dev->of_node; 600 struct xgene_pcie *port; 601 struct pci_host_bridge *bridge; 602 int ret; 603 604 bridge = devm_pci_alloc_host_bridge(dev, sizeof(*port)); 605 if (!bridge) 606 return -ENOMEM; 607 608 port = pci_host_bridge_priv(bridge); 609 610 port->node = of_node_get(dn); 611 port->dev = dev; 612 613 port->version = XGENE_PCIE_IP_VER_UNKN; 614 if (of_device_is_compatible(port->node, "apm,xgene-pcie")) 615 port->version = XGENE_PCIE_IP_VER_1; 616 617 ret = xgene_pcie_map_reg(port, pdev); 618 if (ret) 619 return ret; 620 621 ret = xgene_pcie_init_port(port); 622 if (ret) 623 return ret; 624 625 ret = xgene_pcie_setup(port); 626 if (ret) 627 return ret; 628 629 bridge->sysdata = port; 630 bridge->ops = &xgene_pcie_ops; 631 632 return pci_host_probe(bridge); 633 } 634 635 static const struct of_device_id xgene_pcie_match_table[] = { 636 {.compatible = "apm,xgene-pcie",}, 637 {}, 638 }; 639 640 static struct platform_driver xgene_pcie_driver = { 641 .driver = { 642 .name = "xgene-pcie", 643 .of_match_table = xgene_pcie_match_table, 644 .suppress_bind_attrs = true, 645 }, 646 .probe = xgene_pcie_probe, 647 }; 648 builtin_platform_driver(xgene_pcie_driver); 649 #endif 650