1 // SPDX-License-Identifier: GPL-2.0+ 2 /* 3 * Rockchip AXI PCIe endpoint controller driver 4 * 5 * Copyright (c) 2018 Rockchip, Inc. 6 * 7 * Author: Shawn Lin <shawn.lin@rock-chips.com> 8 * Simon Xue <xxm@rock-chips.com> 9 */ 10 11 #include <linux/configfs.h> 12 #include <linux/delay.h> 13 #include <linux/kernel.h> 14 #include <linux/of.h> 15 #include <linux/pci-epc.h> 16 #include <linux/platform_device.h> 17 #include <linux/pci-epf.h> 18 #include <linux/sizes.h> 19 20 #include "pcie-rockchip.h" 21 22 /** 23 * struct rockchip_pcie_ep - private data for PCIe endpoint controller driver 24 * @rockchip: Rockchip PCIe controller 25 * @epc: PCI EPC device 26 * @max_regions: maximum number of regions supported by hardware 27 * @ob_region_map: bitmask of mapped outbound regions 28 * @ob_addr: base addresses in the AXI bus where the outbound regions start 29 * @irq_phys_addr: base address on the AXI bus where the MSI/legacy IRQ 30 * dedicated outbound regions is mapped. 31 * @irq_cpu_addr: base address in the CPU space where a write access triggers 32 * the sending of a memory write (MSI) / normal message (legacy 33 * IRQ) TLP through the PCIe bus. 34 * @irq_pci_addr: used to save the current mapping of the MSI/legacy IRQ 35 * dedicated outbound region. 36 * @irq_pci_fn: the latest PCI function that has updated the mapping of 37 * the MSI/legacy IRQ dedicated outbound region. 38 * @irq_pending: bitmask of asserted legacy IRQs. 39 */ 40 struct rockchip_pcie_ep { 41 struct rockchip_pcie rockchip; 42 struct pci_epc *epc; 43 u32 max_regions; 44 unsigned long ob_region_map; 45 phys_addr_t *ob_addr; 46 phys_addr_t irq_phys_addr; 47 void __iomem *irq_cpu_addr; 48 u64 irq_pci_addr; 49 u8 irq_pci_fn; 50 u8 irq_pending; 51 }; 52 53 static void rockchip_pcie_clear_ep_ob_atu(struct rockchip_pcie *rockchip, 54 u32 region) 55 { 56 rockchip_pcie_write(rockchip, 0, 57 ROCKCHIP_PCIE_AT_OB_REGION_PCI_ADDR0(region)); 58 rockchip_pcie_write(rockchip, 0, 59 ROCKCHIP_PCIE_AT_OB_REGION_PCI_ADDR1(region)); 60 rockchip_pcie_write(rockchip, 0, 61 ROCKCHIP_PCIE_AT_OB_REGION_DESC0(region)); 62 rockchip_pcie_write(rockchip, 0, 63 ROCKCHIP_PCIE_AT_OB_REGION_DESC1(region)); 64 } 65 66 static void rockchip_pcie_prog_ep_ob_atu(struct rockchip_pcie *rockchip, u8 fn, 67 u32 r, u64 cpu_addr, u64 pci_addr, 68 size_t size) 69 { 70 int num_pass_bits = fls64(size - 1); 71 u32 addr0, addr1, desc0; 72 73 if (num_pass_bits < 8) 74 num_pass_bits = 8; 75 76 addr0 = ((num_pass_bits - 1) & PCIE_CORE_OB_REGION_ADDR0_NUM_BITS) | 77 (lower_32_bits(pci_addr) & PCIE_CORE_OB_REGION_ADDR0_LO_ADDR); 78 addr1 = upper_32_bits(pci_addr); 79 desc0 = ROCKCHIP_PCIE_AT_OB_REGION_DESC0_DEVFN(fn) | AXI_WRAPPER_MEM_WRITE; 80 81 /* PCI bus address region */ 82 rockchip_pcie_write(rockchip, addr0, 83 ROCKCHIP_PCIE_AT_OB_REGION_PCI_ADDR0(r)); 84 rockchip_pcie_write(rockchip, addr1, 85 ROCKCHIP_PCIE_AT_OB_REGION_PCI_ADDR1(r)); 86 rockchip_pcie_write(rockchip, desc0, 87 ROCKCHIP_PCIE_AT_OB_REGION_DESC0(r)); 88 rockchip_pcie_write(rockchip, 0, 89 ROCKCHIP_PCIE_AT_OB_REGION_DESC1(r)); 90 } 91 92 static int rockchip_pcie_ep_write_header(struct pci_epc *epc, u8 fn, u8 vfn, 93 struct pci_epf_header *hdr) 94 { 95 u32 reg; 96 struct rockchip_pcie_ep *ep = epc_get_drvdata(epc); 97 struct rockchip_pcie *rockchip = &ep->rockchip; 98 99 /* All functions share the same vendor ID with function 0 */ 100 if (fn == 0) { 101 rockchip_pcie_write(rockchip, 102 hdr->vendorid | hdr->subsys_vendor_id << 16, 103 PCIE_CORE_CONFIG_VENDOR); 104 } 105 106 reg = rockchip_pcie_read(rockchip, PCIE_EP_CONFIG_DID_VID); 107 reg = (reg & 0xFFFF) | (hdr->deviceid << 16); 108 rockchip_pcie_write(rockchip, reg, PCIE_EP_CONFIG_DID_VID); 109 110 rockchip_pcie_write(rockchip, 111 hdr->revid | 112 hdr->progif_code << 8 | 113 hdr->subclass_code << 16 | 114 hdr->baseclass_code << 24, 115 ROCKCHIP_PCIE_EP_FUNC_BASE(fn) + PCI_REVISION_ID); 116 rockchip_pcie_write(rockchip, hdr->cache_line_size, 117 ROCKCHIP_PCIE_EP_FUNC_BASE(fn) + 118 PCI_CACHE_LINE_SIZE); 119 rockchip_pcie_write(rockchip, hdr->subsys_id << 16, 120 ROCKCHIP_PCIE_EP_FUNC_BASE(fn) + 121 PCI_SUBSYSTEM_VENDOR_ID); 122 rockchip_pcie_write(rockchip, hdr->interrupt_pin << 8, 123 ROCKCHIP_PCIE_EP_FUNC_BASE(fn) + 124 PCI_INTERRUPT_LINE); 125 126 return 0; 127 } 128 129 static int rockchip_pcie_ep_set_bar(struct pci_epc *epc, u8 fn, u8 vfn, 130 struct pci_epf_bar *epf_bar) 131 { 132 struct rockchip_pcie_ep *ep = epc_get_drvdata(epc); 133 struct rockchip_pcie *rockchip = &ep->rockchip; 134 dma_addr_t bar_phys = epf_bar->phys_addr; 135 enum pci_barno bar = epf_bar->barno; 136 int flags = epf_bar->flags; 137 u32 addr0, addr1, reg, cfg, b, aperture, ctrl; 138 u64 sz; 139 140 /* BAR size is 2^(aperture + 7) */ 141 sz = max_t(size_t, epf_bar->size, MIN_EP_APERTURE); 142 143 /* 144 * roundup_pow_of_two() returns an unsigned long, which is not suited 145 * for 64bit values. 146 */ 147 sz = 1ULL << fls64(sz - 1); 148 aperture = ilog2(sz) - 7; /* 128B -> 0, 256B -> 1, 512B -> 2, ... */ 149 150 if ((flags & PCI_BASE_ADDRESS_SPACE) == PCI_BASE_ADDRESS_SPACE_IO) { 151 ctrl = ROCKCHIP_PCIE_CORE_BAR_CFG_CTRL_IO_32BITS; 152 } else { 153 bool is_prefetch = !!(flags & PCI_BASE_ADDRESS_MEM_PREFETCH); 154 bool is_64bits = sz > SZ_2G; 155 156 if (is_64bits && (bar & 1)) 157 return -EINVAL; 158 159 if (is_64bits && is_prefetch) 160 ctrl = 161 ROCKCHIP_PCIE_CORE_BAR_CFG_CTRL_PREFETCH_MEM_64BITS; 162 else if (is_prefetch) 163 ctrl = 164 ROCKCHIP_PCIE_CORE_BAR_CFG_CTRL_PREFETCH_MEM_32BITS; 165 else if (is_64bits) 166 ctrl = ROCKCHIP_PCIE_CORE_BAR_CFG_CTRL_MEM_64BITS; 167 else 168 ctrl = ROCKCHIP_PCIE_CORE_BAR_CFG_CTRL_MEM_32BITS; 169 } 170 171 if (bar < BAR_4) { 172 reg = ROCKCHIP_PCIE_CORE_EP_FUNC_BAR_CFG0(fn); 173 b = bar; 174 } else { 175 reg = ROCKCHIP_PCIE_CORE_EP_FUNC_BAR_CFG1(fn); 176 b = bar - BAR_4; 177 } 178 179 addr0 = lower_32_bits(bar_phys); 180 addr1 = upper_32_bits(bar_phys); 181 182 cfg = rockchip_pcie_read(rockchip, reg); 183 cfg &= ~(ROCKCHIP_PCIE_CORE_EP_FUNC_BAR_CFG_BAR_APERTURE_MASK(b) | 184 ROCKCHIP_PCIE_CORE_EP_FUNC_BAR_CFG_BAR_CTRL_MASK(b)); 185 cfg |= (ROCKCHIP_PCIE_CORE_EP_FUNC_BAR_CFG_BAR_APERTURE(b, aperture) | 186 ROCKCHIP_PCIE_CORE_EP_FUNC_BAR_CFG_BAR_CTRL(b, ctrl)); 187 188 rockchip_pcie_write(rockchip, cfg, reg); 189 rockchip_pcie_write(rockchip, addr0, 190 ROCKCHIP_PCIE_AT_IB_EP_FUNC_BAR_ADDR0(fn, bar)); 191 rockchip_pcie_write(rockchip, addr1, 192 ROCKCHIP_PCIE_AT_IB_EP_FUNC_BAR_ADDR1(fn, bar)); 193 194 return 0; 195 } 196 197 static void rockchip_pcie_ep_clear_bar(struct pci_epc *epc, u8 fn, u8 vfn, 198 struct pci_epf_bar *epf_bar) 199 { 200 struct rockchip_pcie_ep *ep = epc_get_drvdata(epc); 201 struct rockchip_pcie *rockchip = &ep->rockchip; 202 u32 reg, cfg, b, ctrl; 203 enum pci_barno bar = epf_bar->barno; 204 205 if (bar < BAR_4) { 206 reg = ROCKCHIP_PCIE_CORE_EP_FUNC_BAR_CFG0(fn); 207 b = bar; 208 } else { 209 reg = ROCKCHIP_PCIE_CORE_EP_FUNC_BAR_CFG1(fn); 210 b = bar - BAR_4; 211 } 212 213 ctrl = ROCKCHIP_PCIE_CORE_BAR_CFG_CTRL_DISABLED; 214 cfg = rockchip_pcie_read(rockchip, reg); 215 cfg &= ~(ROCKCHIP_PCIE_CORE_EP_FUNC_BAR_CFG_BAR_APERTURE_MASK(b) | 216 ROCKCHIP_PCIE_CORE_EP_FUNC_BAR_CFG_BAR_CTRL_MASK(b)); 217 cfg |= ROCKCHIP_PCIE_CORE_EP_FUNC_BAR_CFG_BAR_CTRL(b, ctrl); 218 219 rockchip_pcie_write(rockchip, cfg, reg); 220 rockchip_pcie_write(rockchip, 0x0, 221 ROCKCHIP_PCIE_AT_IB_EP_FUNC_BAR_ADDR0(fn, bar)); 222 rockchip_pcie_write(rockchip, 0x0, 223 ROCKCHIP_PCIE_AT_IB_EP_FUNC_BAR_ADDR1(fn, bar)); 224 } 225 226 static inline u32 rockchip_ob_region(phys_addr_t addr) 227 { 228 return (addr >> ilog2(SZ_1M)) & 0x1f; 229 } 230 231 static int rockchip_pcie_ep_map_addr(struct pci_epc *epc, u8 fn, u8 vfn, 232 phys_addr_t addr, u64 pci_addr, 233 size_t size) 234 { 235 struct rockchip_pcie_ep *ep = epc_get_drvdata(epc); 236 struct rockchip_pcie *pcie = &ep->rockchip; 237 u32 r = rockchip_ob_region(addr); 238 239 rockchip_pcie_prog_ep_ob_atu(pcie, fn, r, addr, pci_addr, size); 240 241 set_bit(r, &ep->ob_region_map); 242 ep->ob_addr[r] = addr; 243 244 return 0; 245 } 246 247 static void rockchip_pcie_ep_unmap_addr(struct pci_epc *epc, u8 fn, u8 vfn, 248 phys_addr_t addr) 249 { 250 struct rockchip_pcie_ep *ep = epc_get_drvdata(epc); 251 struct rockchip_pcie *rockchip = &ep->rockchip; 252 u32 r; 253 254 for (r = 0; r < ep->max_regions; r++) 255 if (ep->ob_addr[r] == addr) 256 break; 257 258 if (r == ep->max_regions) 259 return; 260 261 rockchip_pcie_clear_ep_ob_atu(rockchip, r); 262 263 ep->ob_addr[r] = 0; 264 clear_bit(r, &ep->ob_region_map); 265 } 266 267 static int rockchip_pcie_ep_set_msi(struct pci_epc *epc, u8 fn, u8 vfn, 268 u8 multi_msg_cap) 269 { 270 struct rockchip_pcie_ep *ep = epc_get_drvdata(epc); 271 struct rockchip_pcie *rockchip = &ep->rockchip; 272 u32 flags; 273 274 flags = rockchip_pcie_read(rockchip, 275 ROCKCHIP_PCIE_EP_FUNC_BASE(fn) + 276 ROCKCHIP_PCIE_EP_MSI_CTRL_REG); 277 flags &= ~ROCKCHIP_PCIE_EP_MSI_CTRL_MMC_MASK; 278 flags |= 279 (multi_msg_cap << ROCKCHIP_PCIE_EP_MSI_CTRL_MMC_OFFSET) | 280 (PCI_MSI_FLAGS_64BIT << ROCKCHIP_PCIE_EP_MSI_FLAGS_OFFSET); 281 flags &= ~ROCKCHIP_PCIE_EP_MSI_CTRL_MASK_MSI_CAP; 282 rockchip_pcie_write(rockchip, flags, 283 ROCKCHIP_PCIE_EP_FUNC_BASE(fn) + 284 ROCKCHIP_PCIE_EP_MSI_CTRL_REG); 285 return 0; 286 } 287 288 static int rockchip_pcie_ep_get_msi(struct pci_epc *epc, u8 fn, u8 vfn) 289 { 290 struct rockchip_pcie_ep *ep = epc_get_drvdata(epc); 291 struct rockchip_pcie *rockchip = &ep->rockchip; 292 u32 flags; 293 294 flags = rockchip_pcie_read(rockchip, 295 ROCKCHIP_PCIE_EP_FUNC_BASE(fn) + 296 ROCKCHIP_PCIE_EP_MSI_CTRL_REG); 297 if (!(flags & ROCKCHIP_PCIE_EP_MSI_CTRL_ME)) 298 return -EINVAL; 299 300 return ((flags & ROCKCHIP_PCIE_EP_MSI_CTRL_MME_MASK) >> 301 ROCKCHIP_PCIE_EP_MSI_CTRL_MME_OFFSET); 302 } 303 304 static void rockchip_pcie_ep_assert_intx(struct rockchip_pcie_ep *ep, u8 fn, 305 u8 intx, bool do_assert) 306 { 307 struct rockchip_pcie *rockchip = &ep->rockchip; 308 309 intx &= 3; 310 311 if (do_assert) { 312 ep->irq_pending |= BIT(intx); 313 rockchip_pcie_write(rockchip, 314 PCIE_CLIENT_INT_IN_ASSERT | 315 PCIE_CLIENT_INT_PEND_ST_PEND, 316 PCIE_CLIENT_LEGACY_INT_CTRL); 317 } else { 318 ep->irq_pending &= ~BIT(intx); 319 rockchip_pcie_write(rockchip, 320 PCIE_CLIENT_INT_IN_DEASSERT | 321 PCIE_CLIENT_INT_PEND_ST_NORMAL, 322 PCIE_CLIENT_LEGACY_INT_CTRL); 323 } 324 } 325 326 static int rockchip_pcie_ep_send_legacy_irq(struct rockchip_pcie_ep *ep, u8 fn, 327 u8 intx) 328 { 329 u16 cmd; 330 331 cmd = rockchip_pcie_read(&ep->rockchip, 332 ROCKCHIP_PCIE_EP_FUNC_BASE(fn) + 333 ROCKCHIP_PCIE_EP_CMD_STATUS); 334 335 if (cmd & PCI_COMMAND_INTX_DISABLE) 336 return -EINVAL; 337 338 /* 339 * Should add some delay between toggling INTx per TRM vaguely saying 340 * it depends on some cycles of the AHB bus clock to function it. So 341 * add sufficient 1ms here. 342 */ 343 rockchip_pcie_ep_assert_intx(ep, fn, intx, true); 344 mdelay(1); 345 rockchip_pcie_ep_assert_intx(ep, fn, intx, false); 346 return 0; 347 } 348 349 static int rockchip_pcie_ep_send_msi_irq(struct rockchip_pcie_ep *ep, u8 fn, 350 u8 interrupt_num) 351 { 352 struct rockchip_pcie *rockchip = &ep->rockchip; 353 u32 flags, mme, data, data_mask; 354 u8 msi_count; 355 u64 pci_addr; 356 u32 r; 357 358 /* Check MSI enable bit */ 359 flags = rockchip_pcie_read(&ep->rockchip, 360 ROCKCHIP_PCIE_EP_FUNC_BASE(fn) + 361 ROCKCHIP_PCIE_EP_MSI_CTRL_REG); 362 if (!(flags & ROCKCHIP_PCIE_EP_MSI_CTRL_ME)) 363 return -EINVAL; 364 365 /* Get MSI numbers from MME */ 366 mme = ((flags & ROCKCHIP_PCIE_EP_MSI_CTRL_MME_MASK) >> 367 ROCKCHIP_PCIE_EP_MSI_CTRL_MME_OFFSET); 368 msi_count = 1 << mme; 369 if (!interrupt_num || interrupt_num > msi_count) 370 return -EINVAL; 371 372 /* Set MSI private data */ 373 data_mask = msi_count - 1; 374 data = rockchip_pcie_read(rockchip, 375 ROCKCHIP_PCIE_EP_FUNC_BASE(fn) + 376 ROCKCHIP_PCIE_EP_MSI_CTRL_REG + 377 PCI_MSI_DATA_64); 378 data = (data & ~data_mask) | ((interrupt_num - 1) & data_mask); 379 380 /* Get MSI PCI address */ 381 pci_addr = rockchip_pcie_read(rockchip, 382 ROCKCHIP_PCIE_EP_FUNC_BASE(fn) + 383 ROCKCHIP_PCIE_EP_MSI_CTRL_REG + 384 PCI_MSI_ADDRESS_HI); 385 pci_addr <<= 32; 386 pci_addr |= rockchip_pcie_read(rockchip, 387 ROCKCHIP_PCIE_EP_FUNC_BASE(fn) + 388 ROCKCHIP_PCIE_EP_MSI_CTRL_REG + 389 PCI_MSI_ADDRESS_LO); 390 391 /* Set the outbound region if needed. */ 392 if (unlikely(ep->irq_pci_addr != (pci_addr & PCIE_ADDR_MASK) || 393 ep->irq_pci_fn != fn)) { 394 r = rockchip_ob_region(ep->irq_phys_addr); 395 rockchip_pcie_prog_ep_ob_atu(rockchip, fn, r, 396 ep->irq_phys_addr, 397 pci_addr & PCIE_ADDR_MASK, 398 ~PCIE_ADDR_MASK + 1); 399 ep->irq_pci_addr = (pci_addr & PCIE_ADDR_MASK); 400 ep->irq_pci_fn = fn; 401 } 402 403 writew(data, ep->irq_cpu_addr + (pci_addr & ~PCIE_ADDR_MASK)); 404 return 0; 405 } 406 407 static int rockchip_pcie_ep_raise_irq(struct pci_epc *epc, u8 fn, u8 vfn, 408 enum pci_epc_irq_type type, 409 u16 interrupt_num) 410 { 411 struct rockchip_pcie_ep *ep = epc_get_drvdata(epc); 412 413 switch (type) { 414 case PCI_EPC_IRQ_LEGACY: 415 return rockchip_pcie_ep_send_legacy_irq(ep, fn, 0); 416 case PCI_EPC_IRQ_MSI: 417 return rockchip_pcie_ep_send_msi_irq(ep, fn, interrupt_num); 418 default: 419 return -EINVAL; 420 } 421 } 422 423 static int rockchip_pcie_ep_start(struct pci_epc *epc) 424 { 425 struct rockchip_pcie_ep *ep = epc_get_drvdata(epc); 426 struct rockchip_pcie *rockchip = &ep->rockchip; 427 struct pci_epf *epf; 428 u32 cfg; 429 430 cfg = BIT(0); 431 list_for_each_entry(epf, &epc->pci_epf, list) 432 cfg |= BIT(epf->func_no); 433 434 rockchip_pcie_write(rockchip, cfg, PCIE_CORE_PHY_FUNC_CFG); 435 436 return 0; 437 } 438 439 static const struct pci_epc_features rockchip_pcie_epc_features = { 440 .linkup_notifier = false, 441 .msi_capable = true, 442 .msix_capable = false, 443 .align = 256, 444 }; 445 446 static const struct pci_epc_features* 447 rockchip_pcie_ep_get_features(struct pci_epc *epc, u8 func_no, u8 vfunc_no) 448 { 449 return &rockchip_pcie_epc_features; 450 } 451 452 static const struct pci_epc_ops rockchip_pcie_epc_ops = { 453 .write_header = rockchip_pcie_ep_write_header, 454 .set_bar = rockchip_pcie_ep_set_bar, 455 .clear_bar = rockchip_pcie_ep_clear_bar, 456 .map_addr = rockchip_pcie_ep_map_addr, 457 .unmap_addr = rockchip_pcie_ep_unmap_addr, 458 .set_msi = rockchip_pcie_ep_set_msi, 459 .get_msi = rockchip_pcie_ep_get_msi, 460 .raise_irq = rockchip_pcie_ep_raise_irq, 461 .start = rockchip_pcie_ep_start, 462 .get_features = rockchip_pcie_ep_get_features, 463 }; 464 465 static int rockchip_pcie_parse_ep_dt(struct rockchip_pcie *rockchip, 466 struct rockchip_pcie_ep *ep) 467 { 468 struct device *dev = rockchip->dev; 469 int err; 470 471 err = rockchip_pcie_parse_dt(rockchip); 472 if (err) 473 return err; 474 475 err = rockchip_pcie_get_phys(rockchip); 476 if (err) 477 return err; 478 479 err = of_property_read_u32(dev->of_node, 480 "rockchip,max-outbound-regions", 481 &ep->max_regions); 482 if (err < 0 || ep->max_regions > MAX_REGION_LIMIT) 483 ep->max_regions = MAX_REGION_LIMIT; 484 485 ep->ob_region_map = 0; 486 487 err = of_property_read_u8(dev->of_node, "max-functions", 488 &ep->epc->max_functions); 489 if (err < 0) 490 ep->epc->max_functions = 1; 491 492 return 0; 493 } 494 495 static const struct of_device_id rockchip_pcie_ep_of_match[] = { 496 { .compatible = "rockchip,rk3399-pcie-ep"}, 497 {}, 498 }; 499 500 static int rockchip_pcie_ep_probe(struct platform_device *pdev) 501 { 502 struct device *dev = &pdev->dev; 503 struct rockchip_pcie_ep *ep; 504 struct rockchip_pcie *rockchip; 505 struct pci_epc *epc; 506 size_t max_regions; 507 struct pci_epc_mem_window *windows = NULL; 508 int err, i; 509 u32 cfg_msi, cfg_msix_cp; 510 511 ep = devm_kzalloc(dev, sizeof(*ep), GFP_KERNEL); 512 if (!ep) 513 return -ENOMEM; 514 515 rockchip = &ep->rockchip; 516 rockchip->is_rc = false; 517 rockchip->dev = dev; 518 519 epc = devm_pci_epc_create(dev, &rockchip_pcie_epc_ops); 520 if (IS_ERR(epc)) { 521 dev_err(dev, "failed to create epc device\n"); 522 return PTR_ERR(epc); 523 } 524 525 ep->epc = epc; 526 epc_set_drvdata(epc, ep); 527 528 err = rockchip_pcie_parse_ep_dt(rockchip, ep); 529 if (err) 530 return err; 531 532 err = rockchip_pcie_enable_clocks(rockchip); 533 if (err) 534 return err; 535 536 err = rockchip_pcie_init_port(rockchip); 537 if (err) 538 goto err_disable_clocks; 539 540 /* Establish the link automatically */ 541 rockchip_pcie_write(rockchip, PCIE_CLIENT_LINK_TRAIN_ENABLE, 542 PCIE_CLIENT_CONFIG); 543 544 max_regions = ep->max_regions; 545 ep->ob_addr = devm_kcalloc(dev, max_regions, sizeof(*ep->ob_addr), 546 GFP_KERNEL); 547 548 if (!ep->ob_addr) { 549 err = -ENOMEM; 550 goto err_uninit_port; 551 } 552 553 /* Only enable function 0 by default */ 554 rockchip_pcie_write(rockchip, BIT(0), PCIE_CORE_PHY_FUNC_CFG); 555 556 windows = devm_kcalloc(dev, ep->max_regions, 557 sizeof(struct pci_epc_mem_window), GFP_KERNEL); 558 if (!windows) { 559 err = -ENOMEM; 560 goto err_uninit_port; 561 } 562 for (i = 0; i < ep->max_regions; i++) { 563 windows[i].phys_base = rockchip->mem_res->start + (SZ_1M * i); 564 windows[i].size = SZ_1M; 565 windows[i].page_size = SZ_1M; 566 } 567 err = pci_epc_multi_mem_init(epc, windows, ep->max_regions); 568 devm_kfree(dev, windows); 569 570 if (err < 0) { 571 dev_err(dev, "failed to initialize the memory space\n"); 572 goto err_uninit_port; 573 } 574 575 ep->irq_cpu_addr = pci_epc_mem_alloc_addr(epc, &ep->irq_phys_addr, 576 SZ_1M); 577 if (!ep->irq_cpu_addr) { 578 dev_err(dev, "failed to reserve memory space for MSI\n"); 579 err = -ENOMEM; 580 goto err_epc_mem_exit; 581 } 582 583 ep->irq_pci_addr = ROCKCHIP_PCIE_EP_DUMMY_IRQ_ADDR; 584 585 /* 586 * MSI-X is not supported but the controller still advertises the MSI-X 587 * capability by default, which can lead to the Root Complex side 588 * allocating MSI-X vectors which cannot be used. Avoid this by skipping 589 * the MSI-X capability entry in the PCIe capabilities linked-list: get 590 * the next pointer from the MSI-X entry and set that in the MSI 591 * capability entry (which is the previous entry). This way the MSI-X 592 * entry is skipped (left out of the linked-list) and not advertised. 593 */ 594 cfg_msi = rockchip_pcie_read(rockchip, PCIE_EP_CONFIG_BASE + 595 ROCKCHIP_PCIE_EP_MSI_CTRL_REG); 596 597 cfg_msi &= ~ROCKCHIP_PCIE_EP_MSI_CP1_MASK; 598 599 cfg_msix_cp = rockchip_pcie_read(rockchip, PCIE_EP_CONFIG_BASE + 600 ROCKCHIP_PCIE_EP_MSIX_CAP_REG) & 601 ROCKCHIP_PCIE_EP_MSIX_CAP_CP_MASK; 602 603 cfg_msi |= cfg_msix_cp; 604 605 rockchip_pcie_write(rockchip, cfg_msi, 606 PCIE_EP_CONFIG_BASE + ROCKCHIP_PCIE_EP_MSI_CTRL_REG); 607 608 rockchip_pcie_write(rockchip, PCIE_CLIENT_CONF_ENABLE, 609 PCIE_CLIENT_CONFIG); 610 611 return 0; 612 err_epc_mem_exit: 613 pci_epc_mem_exit(epc); 614 err_uninit_port: 615 rockchip_pcie_deinit_phys(rockchip); 616 err_disable_clocks: 617 rockchip_pcie_disable_clocks(rockchip); 618 return err; 619 } 620 621 static struct platform_driver rockchip_pcie_ep_driver = { 622 .driver = { 623 .name = "rockchip-pcie-ep", 624 .of_match_table = rockchip_pcie_ep_of_match, 625 }, 626 .probe = rockchip_pcie_ep_probe, 627 }; 628 629 builtin_platform_driver(rockchip_pcie_ep_driver); 630