1 // SPDX-License-Identifier: GPL-2.0+ 2 /* 3 * Rockchip AXI PCIe endpoint controller driver 4 * 5 * Copyright (c) 2018 Rockchip, Inc. 6 * 7 * Author: Shawn Lin <shawn.lin@rock-chips.com> 8 * Simon Xue <xxm@rock-chips.com> 9 */ 10 11 #include <linux/configfs.h> 12 #include <linux/delay.h> 13 #include <linux/kernel.h> 14 #include <linux/of.h> 15 #include <linux/pci-epc.h> 16 #include <linux/platform_device.h> 17 #include <linux/pci-epf.h> 18 #include <linux/sizes.h> 19 20 #include "pcie-rockchip.h" 21 22 /** 23 * struct rockchip_pcie_ep - private data for PCIe endpoint controller driver 24 * @rockchip: Rockchip PCIe controller 25 * @epc: PCI EPC device 26 * @max_regions: maximum number of regions supported by hardware 27 * @ob_region_map: bitmask of mapped outbound regions 28 * @ob_addr: base addresses in the AXI bus where the outbound regions start 29 * @irq_phys_addr: base address on the AXI bus where the MSI/legacy IRQ 30 * dedicated outbound regions is mapped. 31 * @irq_cpu_addr: base address in the CPU space where a write access triggers 32 * the sending of a memory write (MSI) / normal message (legacy 33 * IRQ) TLP through the PCIe bus. 34 * @irq_pci_addr: used to save the current mapping of the MSI/legacy IRQ 35 * dedicated outbound region. 36 * @irq_pci_fn: the latest PCI function that has updated the mapping of 37 * the MSI/legacy IRQ dedicated outbound region. 38 * @irq_pending: bitmask of asserted legacy IRQs. 39 */ 40 struct rockchip_pcie_ep { 41 struct rockchip_pcie rockchip; 42 struct pci_epc *epc; 43 u32 max_regions; 44 unsigned long ob_region_map; 45 phys_addr_t *ob_addr; 46 phys_addr_t irq_phys_addr; 47 void __iomem *irq_cpu_addr; 48 u64 irq_pci_addr; 49 u8 irq_pci_fn; 50 u8 irq_pending; 51 }; 52 53 static void rockchip_pcie_clear_ep_ob_atu(struct rockchip_pcie *rockchip, 54 u32 region) 55 { 56 rockchip_pcie_write(rockchip, 0, 57 ROCKCHIP_PCIE_AT_OB_REGION_PCI_ADDR0(region)); 58 rockchip_pcie_write(rockchip, 0, 59 ROCKCHIP_PCIE_AT_OB_REGION_PCI_ADDR1(region)); 60 rockchip_pcie_write(rockchip, 0, 61 ROCKCHIP_PCIE_AT_OB_REGION_DESC0(region)); 62 rockchip_pcie_write(rockchip, 0, 63 ROCKCHIP_PCIE_AT_OB_REGION_DESC1(region)); 64 } 65 66 static void rockchip_pcie_prog_ep_ob_atu(struct rockchip_pcie *rockchip, u8 fn, 67 u32 r, u64 cpu_addr, u64 pci_addr, 68 size_t size) 69 { 70 int num_pass_bits = fls64(size - 1); 71 u32 addr0, addr1, desc0; 72 73 if (num_pass_bits < 8) 74 num_pass_bits = 8; 75 76 addr0 = ((num_pass_bits - 1) & PCIE_CORE_OB_REGION_ADDR0_NUM_BITS) | 77 (lower_32_bits(pci_addr) & PCIE_CORE_OB_REGION_ADDR0_LO_ADDR); 78 addr1 = upper_32_bits(pci_addr); 79 desc0 = ROCKCHIP_PCIE_AT_OB_REGION_DESC0_DEVFN(fn) | AXI_WRAPPER_MEM_WRITE; 80 81 /* PCI bus address region */ 82 rockchip_pcie_write(rockchip, addr0, 83 ROCKCHIP_PCIE_AT_OB_REGION_PCI_ADDR0(r)); 84 rockchip_pcie_write(rockchip, addr1, 85 ROCKCHIP_PCIE_AT_OB_REGION_PCI_ADDR1(r)); 86 rockchip_pcie_write(rockchip, desc0, 87 ROCKCHIP_PCIE_AT_OB_REGION_DESC0(r)); 88 rockchip_pcie_write(rockchip, 0, 89 ROCKCHIP_PCIE_AT_OB_REGION_DESC1(r)); 90 } 91 92 static int rockchip_pcie_ep_write_header(struct pci_epc *epc, u8 fn, u8 vfn, 93 struct pci_epf_header *hdr) 94 { 95 u32 reg; 96 struct rockchip_pcie_ep *ep = epc_get_drvdata(epc); 97 struct rockchip_pcie *rockchip = &ep->rockchip; 98 99 /* All functions share the same vendor ID with function 0 */ 100 if (fn == 0) { 101 u32 vid_regs = (hdr->vendorid & GENMASK(15, 0)) | 102 (hdr->subsys_vendor_id & GENMASK(31, 16)) << 16; 103 104 rockchip_pcie_write(rockchip, vid_regs, 105 PCIE_CORE_CONFIG_VENDOR); 106 } 107 108 reg = rockchip_pcie_read(rockchip, PCIE_EP_CONFIG_DID_VID); 109 reg = (reg & 0xFFFF) | (hdr->deviceid << 16); 110 rockchip_pcie_write(rockchip, reg, PCIE_EP_CONFIG_DID_VID); 111 112 rockchip_pcie_write(rockchip, 113 hdr->revid | 114 hdr->progif_code << 8 | 115 hdr->subclass_code << 16 | 116 hdr->baseclass_code << 24, 117 ROCKCHIP_PCIE_EP_FUNC_BASE(fn) + PCI_REVISION_ID); 118 rockchip_pcie_write(rockchip, hdr->cache_line_size, 119 ROCKCHIP_PCIE_EP_FUNC_BASE(fn) + 120 PCI_CACHE_LINE_SIZE); 121 rockchip_pcie_write(rockchip, hdr->subsys_id << 16, 122 ROCKCHIP_PCIE_EP_FUNC_BASE(fn) + 123 PCI_SUBSYSTEM_VENDOR_ID); 124 rockchip_pcie_write(rockchip, hdr->interrupt_pin << 8, 125 ROCKCHIP_PCIE_EP_FUNC_BASE(fn) + 126 PCI_INTERRUPT_LINE); 127 128 return 0; 129 } 130 131 static int rockchip_pcie_ep_set_bar(struct pci_epc *epc, u8 fn, u8 vfn, 132 struct pci_epf_bar *epf_bar) 133 { 134 struct rockchip_pcie_ep *ep = epc_get_drvdata(epc); 135 struct rockchip_pcie *rockchip = &ep->rockchip; 136 dma_addr_t bar_phys = epf_bar->phys_addr; 137 enum pci_barno bar = epf_bar->barno; 138 int flags = epf_bar->flags; 139 u32 addr0, addr1, reg, cfg, b, aperture, ctrl; 140 u64 sz; 141 142 /* BAR size is 2^(aperture + 7) */ 143 sz = max_t(size_t, epf_bar->size, MIN_EP_APERTURE); 144 145 /* 146 * roundup_pow_of_two() returns an unsigned long, which is not suited 147 * for 64bit values. 148 */ 149 sz = 1ULL << fls64(sz - 1); 150 aperture = ilog2(sz) - 7; /* 128B -> 0, 256B -> 1, 512B -> 2, ... */ 151 152 if ((flags & PCI_BASE_ADDRESS_SPACE) == PCI_BASE_ADDRESS_SPACE_IO) { 153 ctrl = ROCKCHIP_PCIE_CORE_BAR_CFG_CTRL_IO_32BITS; 154 } else { 155 bool is_prefetch = !!(flags & PCI_BASE_ADDRESS_MEM_PREFETCH); 156 bool is_64bits = sz > SZ_2G; 157 158 if (is_64bits && (bar & 1)) 159 return -EINVAL; 160 161 if (is_64bits && is_prefetch) 162 ctrl = 163 ROCKCHIP_PCIE_CORE_BAR_CFG_CTRL_PREFETCH_MEM_64BITS; 164 else if (is_prefetch) 165 ctrl = 166 ROCKCHIP_PCIE_CORE_BAR_CFG_CTRL_PREFETCH_MEM_32BITS; 167 else if (is_64bits) 168 ctrl = ROCKCHIP_PCIE_CORE_BAR_CFG_CTRL_MEM_64BITS; 169 else 170 ctrl = ROCKCHIP_PCIE_CORE_BAR_CFG_CTRL_MEM_32BITS; 171 } 172 173 if (bar < BAR_4) { 174 reg = ROCKCHIP_PCIE_CORE_EP_FUNC_BAR_CFG0(fn); 175 b = bar; 176 } else { 177 reg = ROCKCHIP_PCIE_CORE_EP_FUNC_BAR_CFG1(fn); 178 b = bar - BAR_4; 179 } 180 181 addr0 = lower_32_bits(bar_phys); 182 addr1 = upper_32_bits(bar_phys); 183 184 cfg = rockchip_pcie_read(rockchip, reg); 185 cfg &= ~(ROCKCHIP_PCIE_CORE_EP_FUNC_BAR_CFG_BAR_APERTURE_MASK(b) | 186 ROCKCHIP_PCIE_CORE_EP_FUNC_BAR_CFG_BAR_CTRL_MASK(b)); 187 cfg |= (ROCKCHIP_PCIE_CORE_EP_FUNC_BAR_CFG_BAR_APERTURE(b, aperture) | 188 ROCKCHIP_PCIE_CORE_EP_FUNC_BAR_CFG_BAR_CTRL(b, ctrl)); 189 190 rockchip_pcie_write(rockchip, cfg, reg); 191 rockchip_pcie_write(rockchip, addr0, 192 ROCKCHIP_PCIE_AT_IB_EP_FUNC_BAR_ADDR0(fn, bar)); 193 rockchip_pcie_write(rockchip, addr1, 194 ROCKCHIP_PCIE_AT_IB_EP_FUNC_BAR_ADDR1(fn, bar)); 195 196 return 0; 197 } 198 199 static void rockchip_pcie_ep_clear_bar(struct pci_epc *epc, u8 fn, u8 vfn, 200 struct pci_epf_bar *epf_bar) 201 { 202 struct rockchip_pcie_ep *ep = epc_get_drvdata(epc); 203 struct rockchip_pcie *rockchip = &ep->rockchip; 204 u32 reg, cfg, b, ctrl; 205 enum pci_barno bar = epf_bar->barno; 206 207 if (bar < BAR_4) { 208 reg = ROCKCHIP_PCIE_CORE_EP_FUNC_BAR_CFG0(fn); 209 b = bar; 210 } else { 211 reg = ROCKCHIP_PCIE_CORE_EP_FUNC_BAR_CFG1(fn); 212 b = bar - BAR_4; 213 } 214 215 ctrl = ROCKCHIP_PCIE_CORE_BAR_CFG_CTRL_DISABLED; 216 cfg = rockchip_pcie_read(rockchip, reg); 217 cfg &= ~(ROCKCHIP_PCIE_CORE_EP_FUNC_BAR_CFG_BAR_APERTURE_MASK(b) | 218 ROCKCHIP_PCIE_CORE_EP_FUNC_BAR_CFG_BAR_CTRL_MASK(b)); 219 cfg |= ROCKCHIP_PCIE_CORE_EP_FUNC_BAR_CFG_BAR_CTRL(b, ctrl); 220 221 rockchip_pcie_write(rockchip, cfg, reg); 222 rockchip_pcie_write(rockchip, 0x0, 223 ROCKCHIP_PCIE_AT_IB_EP_FUNC_BAR_ADDR0(fn, bar)); 224 rockchip_pcie_write(rockchip, 0x0, 225 ROCKCHIP_PCIE_AT_IB_EP_FUNC_BAR_ADDR1(fn, bar)); 226 } 227 228 static inline u32 rockchip_ob_region(phys_addr_t addr) 229 { 230 return (addr >> ilog2(SZ_1M)) & 0x1f; 231 } 232 233 static int rockchip_pcie_ep_map_addr(struct pci_epc *epc, u8 fn, u8 vfn, 234 phys_addr_t addr, u64 pci_addr, 235 size_t size) 236 { 237 struct rockchip_pcie_ep *ep = epc_get_drvdata(epc); 238 struct rockchip_pcie *pcie = &ep->rockchip; 239 u32 r = rockchip_ob_region(addr); 240 241 rockchip_pcie_prog_ep_ob_atu(pcie, fn, r, addr, pci_addr, size); 242 243 set_bit(r, &ep->ob_region_map); 244 ep->ob_addr[r] = addr; 245 246 return 0; 247 } 248 249 static void rockchip_pcie_ep_unmap_addr(struct pci_epc *epc, u8 fn, u8 vfn, 250 phys_addr_t addr) 251 { 252 struct rockchip_pcie_ep *ep = epc_get_drvdata(epc); 253 struct rockchip_pcie *rockchip = &ep->rockchip; 254 u32 r; 255 256 for (r = 0; r < ep->max_regions; r++) 257 if (ep->ob_addr[r] == addr) 258 break; 259 260 if (r == ep->max_regions) 261 return; 262 263 rockchip_pcie_clear_ep_ob_atu(rockchip, r); 264 265 ep->ob_addr[r] = 0; 266 clear_bit(r, &ep->ob_region_map); 267 } 268 269 static int rockchip_pcie_ep_set_msi(struct pci_epc *epc, u8 fn, u8 vfn, 270 u8 multi_msg_cap) 271 { 272 struct rockchip_pcie_ep *ep = epc_get_drvdata(epc); 273 struct rockchip_pcie *rockchip = &ep->rockchip; 274 u32 flags; 275 276 flags = rockchip_pcie_read(rockchip, 277 ROCKCHIP_PCIE_EP_FUNC_BASE(fn) + 278 ROCKCHIP_PCIE_EP_MSI_CTRL_REG); 279 flags &= ~ROCKCHIP_PCIE_EP_MSI_CTRL_MMC_MASK; 280 flags |= 281 (multi_msg_cap << ROCKCHIP_PCIE_EP_MSI_CTRL_MMC_OFFSET) | 282 (PCI_MSI_FLAGS_64BIT << ROCKCHIP_PCIE_EP_MSI_FLAGS_OFFSET); 283 flags &= ~ROCKCHIP_PCIE_EP_MSI_CTRL_MASK_MSI_CAP; 284 rockchip_pcie_write(rockchip, flags, 285 ROCKCHIP_PCIE_EP_FUNC_BASE(fn) + 286 ROCKCHIP_PCIE_EP_MSI_CTRL_REG); 287 return 0; 288 } 289 290 static int rockchip_pcie_ep_get_msi(struct pci_epc *epc, u8 fn, u8 vfn) 291 { 292 struct rockchip_pcie_ep *ep = epc_get_drvdata(epc); 293 struct rockchip_pcie *rockchip = &ep->rockchip; 294 u32 flags; 295 296 flags = rockchip_pcie_read(rockchip, 297 ROCKCHIP_PCIE_EP_FUNC_BASE(fn) + 298 ROCKCHIP_PCIE_EP_MSI_CTRL_REG); 299 if (!(flags & ROCKCHIP_PCIE_EP_MSI_CTRL_ME)) 300 return -EINVAL; 301 302 return ((flags & ROCKCHIP_PCIE_EP_MSI_CTRL_MME_MASK) >> 303 ROCKCHIP_PCIE_EP_MSI_CTRL_MME_OFFSET); 304 } 305 306 static void rockchip_pcie_ep_assert_intx(struct rockchip_pcie_ep *ep, u8 fn, 307 u8 intx, bool do_assert) 308 { 309 struct rockchip_pcie *rockchip = &ep->rockchip; 310 311 intx &= 3; 312 313 if (do_assert) { 314 ep->irq_pending |= BIT(intx); 315 rockchip_pcie_write(rockchip, 316 PCIE_CLIENT_INT_IN_ASSERT | 317 PCIE_CLIENT_INT_PEND_ST_PEND, 318 PCIE_CLIENT_LEGACY_INT_CTRL); 319 } else { 320 ep->irq_pending &= ~BIT(intx); 321 rockchip_pcie_write(rockchip, 322 PCIE_CLIENT_INT_IN_DEASSERT | 323 PCIE_CLIENT_INT_PEND_ST_NORMAL, 324 PCIE_CLIENT_LEGACY_INT_CTRL); 325 } 326 } 327 328 static int rockchip_pcie_ep_send_legacy_irq(struct rockchip_pcie_ep *ep, u8 fn, 329 u8 intx) 330 { 331 u16 cmd; 332 333 cmd = rockchip_pcie_read(&ep->rockchip, 334 ROCKCHIP_PCIE_EP_FUNC_BASE(fn) + 335 ROCKCHIP_PCIE_EP_CMD_STATUS); 336 337 if (cmd & PCI_COMMAND_INTX_DISABLE) 338 return -EINVAL; 339 340 /* 341 * Should add some delay between toggling INTx per TRM vaguely saying 342 * it depends on some cycles of the AHB bus clock to function it. So 343 * add sufficient 1ms here. 344 */ 345 rockchip_pcie_ep_assert_intx(ep, fn, intx, true); 346 mdelay(1); 347 rockchip_pcie_ep_assert_intx(ep, fn, intx, false); 348 return 0; 349 } 350 351 static int rockchip_pcie_ep_send_msi_irq(struct rockchip_pcie_ep *ep, u8 fn, 352 u8 interrupt_num) 353 { 354 struct rockchip_pcie *rockchip = &ep->rockchip; 355 u32 flags, mme, data, data_mask; 356 u8 msi_count; 357 u64 pci_addr; 358 u32 r; 359 360 /* Check MSI enable bit */ 361 flags = rockchip_pcie_read(&ep->rockchip, 362 ROCKCHIP_PCIE_EP_FUNC_BASE(fn) + 363 ROCKCHIP_PCIE_EP_MSI_CTRL_REG); 364 if (!(flags & ROCKCHIP_PCIE_EP_MSI_CTRL_ME)) 365 return -EINVAL; 366 367 /* Get MSI numbers from MME */ 368 mme = ((flags & ROCKCHIP_PCIE_EP_MSI_CTRL_MME_MASK) >> 369 ROCKCHIP_PCIE_EP_MSI_CTRL_MME_OFFSET); 370 msi_count = 1 << mme; 371 if (!interrupt_num || interrupt_num > msi_count) 372 return -EINVAL; 373 374 /* Set MSI private data */ 375 data_mask = msi_count - 1; 376 data = rockchip_pcie_read(rockchip, 377 ROCKCHIP_PCIE_EP_FUNC_BASE(fn) + 378 ROCKCHIP_PCIE_EP_MSI_CTRL_REG + 379 PCI_MSI_DATA_64); 380 data = (data & ~data_mask) | ((interrupt_num - 1) & data_mask); 381 382 /* Get MSI PCI address */ 383 pci_addr = rockchip_pcie_read(rockchip, 384 ROCKCHIP_PCIE_EP_FUNC_BASE(fn) + 385 ROCKCHIP_PCIE_EP_MSI_CTRL_REG + 386 PCI_MSI_ADDRESS_HI); 387 pci_addr <<= 32; 388 pci_addr |= rockchip_pcie_read(rockchip, 389 ROCKCHIP_PCIE_EP_FUNC_BASE(fn) + 390 ROCKCHIP_PCIE_EP_MSI_CTRL_REG + 391 PCI_MSI_ADDRESS_LO); 392 393 /* Set the outbound region if needed. */ 394 if (unlikely(ep->irq_pci_addr != (pci_addr & PCIE_ADDR_MASK) || 395 ep->irq_pci_fn != fn)) { 396 r = rockchip_ob_region(ep->irq_phys_addr); 397 rockchip_pcie_prog_ep_ob_atu(rockchip, fn, r, 398 ep->irq_phys_addr, 399 pci_addr & PCIE_ADDR_MASK, 400 ~PCIE_ADDR_MASK + 1); 401 ep->irq_pci_addr = (pci_addr & PCIE_ADDR_MASK); 402 ep->irq_pci_fn = fn; 403 } 404 405 writew(data, ep->irq_cpu_addr + (pci_addr & ~PCIE_ADDR_MASK)); 406 return 0; 407 } 408 409 static int rockchip_pcie_ep_raise_irq(struct pci_epc *epc, u8 fn, u8 vfn, 410 enum pci_epc_irq_type type, 411 u16 interrupt_num) 412 { 413 struct rockchip_pcie_ep *ep = epc_get_drvdata(epc); 414 415 switch (type) { 416 case PCI_EPC_IRQ_LEGACY: 417 return rockchip_pcie_ep_send_legacy_irq(ep, fn, 0); 418 case PCI_EPC_IRQ_MSI: 419 return rockchip_pcie_ep_send_msi_irq(ep, fn, interrupt_num); 420 default: 421 return -EINVAL; 422 } 423 } 424 425 static int rockchip_pcie_ep_start(struct pci_epc *epc) 426 { 427 struct rockchip_pcie_ep *ep = epc_get_drvdata(epc); 428 struct rockchip_pcie *rockchip = &ep->rockchip; 429 struct pci_epf *epf; 430 u32 cfg; 431 432 cfg = BIT(0); 433 list_for_each_entry(epf, &epc->pci_epf, list) 434 cfg |= BIT(epf->func_no); 435 436 rockchip_pcie_write(rockchip, cfg, PCIE_CORE_PHY_FUNC_CFG); 437 438 return 0; 439 } 440 441 static const struct pci_epc_features rockchip_pcie_epc_features = { 442 .linkup_notifier = false, 443 .msi_capable = true, 444 .msix_capable = false, 445 .align = 256, 446 }; 447 448 static const struct pci_epc_features* 449 rockchip_pcie_ep_get_features(struct pci_epc *epc, u8 func_no, u8 vfunc_no) 450 { 451 return &rockchip_pcie_epc_features; 452 } 453 454 static const struct pci_epc_ops rockchip_pcie_epc_ops = { 455 .write_header = rockchip_pcie_ep_write_header, 456 .set_bar = rockchip_pcie_ep_set_bar, 457 .clear_bar = rockchip_pcie_ep_clear_bar, 458 .map_addr = rockchip_pcie_ep_map_addr, 459 .unmap_addr = rockchip_pcie_ep_unmap_addr, 460 .set_msi = rockchip_pcie_ep_set_msi, 461 .get_msi = rockchip_pcie_ep_get_msi, 462 .raise_irq = rockchip_pcie_ep_raise_irq, 463 .start = rockchip_pcie_ep_start, 464 .get_features = rockchip_pcie_ep_get_features, 465 }; 466 467 static int rockchip_pcie_parse_ep_dt(struct rockchip_pcie *rockchip, 468 struct rockchip_pcie_ep *ep) 469 { 470 struct device *dev = rockchip->dev; 471 int err; 472 473 err = rockchip_pcie_parse_dt(rockchip); 474 if (err) 475 return err; 476 477 err = rockchip_pcie_get_phys(rockchip); 478 if (err) 479 return err; 480 481 err = of_property_read_u32(dev->of_node, 482 "rockchip,max-outbound-regions", 483 &ep->max_regions); 484 if (err < 0 || ep->max_regions > MAX_REGION_LIMIT) 485 ep->max_regions = MAX_REGION_LIMIT; 486 487 ep->ob_region_map = 0; 488 489 err = of_property_read_u8(dev->of_node, "max-functions", 490 &ep->epc->max_functions); 491 if (err < 0) 492 ep->epc->max_functions = 1; 493 494 return 0; 495 } 496 497 static const struct of_device_id rockchip_pcie_ep_of_match[] = { 498 { .compatible = "rockchip,rk3399-pcie-ep"}, 499 {}, 500 }; 501 502 static int rockchip_pcie_ep_probe(struct platform_device *pdev) 503 { 504 struct device *dev = &pdev->dev; 505 struct rockchip_pcie_ep *ep; 506 struct rockchip_pcie *rockchip; 507 struct pci_epc *epc; 508 size_t max_regions; 509 struct pci_epc_mem_window *windows = NULL; 510 int err, i; 511 u32 cfg_msi, cfg_msix_cp; 512 513 ep = devm_kzalloc(dev, sizeof(*ep), GFP_KERNEL); 514 if (!ep) 515 return -ENOMEM; 516 517 rockchip = &ep->rockchip; 518 rockchip->is_rc = false; 519 rockchip->dev = dev; 520 521 epc = devm_pci_epc_create(dev, &rockchip_pcie_epc_ops); 522 if (IS_ERR(epc)) { 523 dev_err(dev, "failed to create epc device\n"); 524 return PTR_ERR(epc); 525 } 526 527 ep->epc = epc; 528 epc_set_drvdata(epc, ep); 529 530 err = rockchip_pcie_parse_ep_dt(rockchip, ep); 531 if (err) 532 return err; 533 534 err = rockchip_pcie_enable_clocks(rockchip); 535 if (err) 536 return err; 537 538 err = rockchip_pcie_init_port(rockchip); 539 if (err) 540 goto err_disable_clocks; 541 542 /* Establish the link automatically */ 543 rockchip_pcie_write(rockchip, PCIE_CLIENT_LINK_TRAIN_ENABLE, 544 PCIE_CLIENT_CONFIG); 545 546 max_regions = ep->max_regions; 547 ep->ob_addr = devm_kcalloc(dev, max_regions, sizeof(*ep->ob_addr), 548 GFP_KERNEL); 549 550 if (!ep->ob_addr) { 551 err = -ENOMEM; 552 goto err_uninit_port; 553 } 554 555 /* Only enable function 0 by default */ 556 rockchip_pcie_write(rockchip, BIT(0), PCIE_CORE_PHY_FUNC_CFG); 557 558 windows = devm_kcalloc(dev, ep->max_regions, 559 sizeof(struct pci_epc_mem_window), GFP_KERNEL); 560 if (!windows) { 561 err = -ENOMEM; 562 goto err_uninit_port; 563 } 564 for (i = 0; i < ep->max_regions; i++) { 565 windows[i].phys_base = rockchip->mem_res->start + (SZ_1M * i); 566 windows[i].size = SZ_1M; 567 windows[i].page_size = SZ_1M; 568 } 569 err = pci_epc_multi_mem_init(epc, windows, ep->max_regions); 570 devm_kfree(dev, windows); 571 572 if (err < 0) { 573 dev_err(dev, "failed to initialize the memory space\n"); 574 goto err_uninit_port; 575 } 576 577 ep->irq_cpu_addr = pci_epc_mem_alloc_addr(epc, &ep->irq_phys_addr, 578 SZ_1M); 579 if (!ep->irq_cpu_addr) { 580 dev_err(dev, "failed to reserve memory space for MSI\n"); 581 err = -ENOMEM; 582 goto err_epc_mem_exit; 583 } 584 585 ep->irq_pci_addr = ROCKCHIP_PCIE_EP_DUMMY_IRQ_ADDR; 586 587 /* 588 * MSI-X is not supported but the controller still advertises the MSI-X 589 * capability by default, which can lead to the Root Complex side 590 * allocating MSI-X vectors which cannot be used. Avoid this by skipping 591 * the MSI-X capability entry in the PCIe capabilities linked-list: get 592 * the next pointer from the MSI-X entry and set that in the MSI 593 * capability entry (which is the previous entry). This way the MSI-X 594 * entry is skipped (left out of the linked-list) and not advertised. 595 */ 596 cfg_msi = rockchip_pcie_read(rockchip, PCIE_EP_CONFIG_BASE + 597 ROCKCHIP_PCIE_EP_MSI_CTRL_REG); 598 599 cfg_msi &= ~ROCKCHIP_PCIE_EP_MSI_CP1_MASK; 600 601 cfg_msix_cp = rockchip_pcie_read(rockchip, PCIE_EP_CONFIG_BASE + 602 ROCKCHIP_PCIE_EP_MSIX_CAP_REG) & 603 ROCKCHIP_PCIE_EP_MSIX_CAP_CP_MASK; 604 605 cfg_msi |= cfg_msix_cp; 606 607 rockchip_pcie_write(rockchip, cfg_msi, 608 PCIE_EP_CONFIG_BASE + ROCKCHIP_PCIE_EP_MSI_CTRL_REG); 609 610 rockchip_pcie_write(rockchip, PCIE_CLIENT_CONF_ENABLE, 611 PCIE_CLIENT_CONFIG); 612 613 return 0; 614 err_epc_mem_exit: 615 pci_epc_mem_exit(epc); 616 err_uninit_port: 617 rockchip_pcie_deinit_phys(rockchip); 618 err_disable_clocks: 619 rockchip_pcie_disable_clocks(rockchip); 620 return err; 621 } 622 623 static struct platform_driver rockchip_pcie_ep_driver = { 624 .driver = { 625 .name = "rockchip-pcie-ep", 626 .of_match_table = rockchip_pcie_ep_of_match, 627 }, 628 .probe = rockchip_pcie_ep_probe, 629 }; 630 631 builtin_platform_driver(rockchip_pcie_ep_driver); 632