1 // SPDX-License-Identifier: GPL-2.0+ 2 /* 3 * Rockchip AXI PCIe endpoint controller driver 4 * 5 * Copyright (c) 2018 Rockchip, Inc. 6 * 7 * Author: Shawn Lin <shawn.lin@rock-chips.com> 8 * Simon Xue <xxm@rock-chips.com> 9 */ 10 11 #include <linux/configfs.h> 12 #include <linux/delay.h> 13 #include <linux/kernel.h> 14 #include <linux/of.h> 15 #include <linux/pci-epc.h> 16 #include <linux/platform_device.h> 17 #include <linux/pci-epf.h> 18 #include <linux/sizes.h> 19 20 #include "pcie-rockchip.h" 21 22 /** 23 * struct rockchip_pcie_ep - private data for PCIe endpoint controller driver 24 * @rockchip: Rockchip PCIe controller 25 * @epc: PCI EPC device 26 * @max_regions: maximum number of regions supported by hardware 27 * @ob_region_map: bitmask of mapped outbound regions 28 * @ob_addr: base addresses in the AXI bus where the outbound regions start 29 * @irq_phys_addr: base address on the AXI bus where the MSI/legacy IRQ 30 * dedicated outbound regions is mapped. 31 * @irq_cpu_addr: base address in the CPU space where a write access triggers 32 * the sending of a memory write (MSI) / normal message (legacy 33 * IRQ) TLP through the PCIe bus. 34 * @irq_pci_addr: used to save the current mapping of the MSI/legacy IRQ 35 * dedicated outbound region. 36 * @irq_pci_fn: the latest PCI function that has updated the mapping of 37 * the MSI/legacy IRQ dedicated outbound region. 38 * @irq_pending: bitmask of asserted legacy IRQs. 39 */ 40 struct rockchip_pcie_ep { 41 struct rockchip_pcie rockchip; 42 struct pci_epc *epc; 43 u32 max_regions; 44 unsigned long ob_region_map; 45 phys_addr_t *ob_addr; 46 phys_addr_t irq_phys_addr; 47 void __iomem *irq_cpu_addr; 48 u64 irq_pci_addr; 49 u8 irq_pci_fn; 50 u8 irq_pending; 51 }; 52 53 static void rockchip_pcie_clear_ep_ob_atu(struct rockchip_pcie *rockchip, 54 u32 region) 55 { 56 rockchip_pcie_write(rockchip, 0, 57 ROCKCHIP_PCIE_AT_OB_REGION_PCI_ADDR0(region)); 58 rockchip_pcie_write(rockchip, 0, 59 ROCKCHIP_PCIE_AT_OB_REGION_PCI_ADDR1(region)); 60 rockchip_pcie_write(rockchip, 0, 61 ROCKCHIP_PCIE_AT_OB_REGION_DESC0(region)); 62 rockchip_pcie_write(rockchip, 0, 63 ROCKCHIP_PCIE_AT_OB_REGION_DESC1(region)); 64 } 65 66 static int rockchip_pcie_ep_ob_atu_num_bits(struct rockchip_pcie *rockchip, 67 u64 pci_addr, size_t size) 68 { 69 int num_pass_bits = fls64(pci_addr ^ (pci_addr + size - 1)); 70 71 return clamp(num_pass_bits, 72 ROCKCHIP_PCIE_AT_MIN_NUM_BITS, 73 ROCKCHIP_PCIE_AT_MAX_NUM_BITS); 74 } 75 76 static void rockchip_pcie_prog_ep_ob_atu(struct rockchip_pcie *rockchip, u8 fn, 77 u32 r, u64 cpu_addr, u64 pci_addr, 78 size_t size) 79 { 80 int num_pass_bits; 81 u32 addr0, addr1, desc0; 82 83 num_pass_bits = rockchip_pcie_ep_ob_atu_num_bits(rockchip, 84 pci_addr, size); 85 86 addr0 = ((num_pass_bits - 1) & PCIE_CORE_OB_REGION_ADDR0_NUM_BITS) | 87 (lower_32_bits(pci_addr) & PCIE_CORE_OB_REGION_ADDR0_LO_ADDR); 88 addr1 = upper_32_bits(pci_addr); 89 desc0 = ROCKCHIP_PCIE_AT_OB_REGION_DESC0_DEVFN(fn) | AXI_WRAPPER_MEM_WRITE; 90 91 /* PCI bus address region */ 92 rockchip_pcie_write(rockchip, addr0, 93 ROCKCHIP_PCIE_AT_OB_REGION_PCI_ADDR0(r)); 94 rockchip_pcie_write(rockchip, addr1, 95 ROCKCHIP_PCIE_AT_OB_REGION_PCI_ADDR1(r)); 96 rockchip_pcie_write(rockchip, desc0, 97 ROCKCHIP_PCIE_AT_OB_REGION_DESC0(r)); 98 rockchip_pcie_write(rockchip, 0, 99 ROCKCHIP_PCIE_AT_OB_REGION_DESC1(r)); 100 } 101 102 static int rockchip_pcie_ep_write_header(struct pci_epc *epc, u8 fn, u8 vfn, 103 struct pci_epf_header *hdr) 104 { 105 u32 reg; 106 struct rockchip_pcie_ep *ep = epc_get_drvdata(epc); 107 struct rockchip_pcie *rockchip = &ep->rockchip; 108 109 /* All functions share the same vendor ID with function 0 */ 110 if (fn == 0) { 111 rockchip_pcie_write(rockchip, 112 hdr->vendorid | hdr->subsys_vendor_id << 16, 113 PCIE_CORE_CONFIG_VENDOR); 114 } 115 116 reg = rockchip_pcie_read(rockchip, PCIE_EP_CONFIG_DID_VID); 117 reg = (reg & 0xFFFF) | (hdr->deviceid << 16); 118 rockchip_pcie_write(rockchip, reg, PCIE_EP_CONFIG_DID_VID); 119 120 rockchip_pcie_write(rockchip, 121 hdr->revid | 122 hdr->progif_code << 8 | 123 hdr->subclass_code << 16 | 124 hdr->baseclass_code << 24, 125 ROCKCHIP_PCIE_EP_FUNC_BASE(fn) + PCI_REVISION_ID); 126 rockchip_pcie_write(rockchip, hdr->cache_line_size, 127 ROCKCHIP_PCIE_EP_FUNC_BASE(fn) + 128 PCI_CACHE_LINE_SIZE); 129 rockchip_pcie_write(rockchip, hdr->subsys_id << 16, 130 ROCKCHIP_PCIE_EP_FUNC_BASE(fn) + 131 PCI_SUBSYSTEM_VENDOR_ID); 132 rockchip_pcie_write(rockchip, hdr->interrupt_pin << 8, 133 ROCKCHIP_PCIE_EP_FUNC_BASE(fn) + 134 PCI_INTERRUPT_LINE); 135 136 return 0; 137 } 138 139 static int rockchip_pcie_ep_set_bar(struct pci_epc *epc, u8 fn, u8 vfn, 140 struct pci_epf_bar *epf_bar) 141 { 142 struct rockchip_pcie_ep *ep = epc_get_drvdata(epc); 143 struct rockchip_pcie *rockchip = &ep->rockchip; 144 dma_addr_t bar_phys = epf_bar->phys_addr; 145 enum pci_barno bar = epf_bar->barno; 146 int flags = epf_bar->flags; 147 u32 addr0, addr1, reg, cfg, b, aperture, ctrl; 148 u64 sz; 149 150 /* BAR size is 2^(aperture + 7) */ 151 sz = max_t(size_t, epf_bar->size, MIN_EP_APERTURE); 152 153 /* 154 * roundup_pow_of_two() returns an unsigned long, which is not suited 155 * for 64bit values. 156 */ 157 sz = 1ULL << fls64(sz - 1); 158 aperture = ilog2(sz) - 7; /* 128B -> 0, 256B -> 1, 512B -> 2, ... */ 159 160 if ((flags & PCI_BASE_ADDRESS_SPACE) == PCI_BASE_ADDRESS_SPACE_IO) { 161 ctrl = ROCKCHIP_PCIE_CORE_BAR_CFG_CTRL_IO_32BITS; 162 } else { 163 bool is_prefetch = !!(flags & PCI_BASE_ADDRESS_MEM_PREFETCH); 164 bool is_64bits = sz > SZ_2G; 165 166 if (is_64bits && (bar & 1)) 167 return -EINVAL; 168 169 if (is_64bits && is_prefetch) 170 ctrl = 171 ROCKCHIP_PCIE_CORE_BAR_CFG_CTRL_PREFETCH_MEM_64BITS; 172 else if (is_prefetch) 173 ctrl = 174 ROCKCHIP_PCIE_CORE_BAR_CFG_CTRL_PREFETCH_MEM_32BITS; 175 else if (is_64bits) 176 ctrl = ROCKCHIP_PCIE_CORE_BAR_CFG_CTRL_MEM_64BITS; 177 else 178 ctrl = ROCKCHIP_PCIE_CORE_BAR_CFG_CTRL_MEM_32BITS; 179 } 180 181 if (bar < BAR_4) { 182 reg = ROCKCHIP_PCIE_CORE_EP_FUNC_BAR_CFG0(fn); 183 b = bar; 184 } else { 185 reg = ROCKCHIP_PCIE_CORE_EP_FUNC_BAR_CFG1(fn); 186 b = bar - BAR_4; 187 } 188 189 addr0 = lower_32_bits(bar_phys); 190 addr1 = upper_32_bits(bar_phys); 191 192 cfg = rockchip_pcie_read(rockchip, reg); 193 cfg &= ~(ROCKCHIP_PCIE_CORE_EP_FUNC_BAR_CFG_BAR_APERTURE_MASK(b) | 194 ROCKCHIP_PCIE_CORE_EP_FUNC_BAR_CFG_BAR_CTRL_MASK(b)); 195 cfg |= (ROCKCHIP_PCIE_CORE_EP_FUNC_BAR_CFG_BAR_APERTURE(b, aperture) | 196 ROCKCHIP_PCIE_CORE_EP_FUNC_BAR_CFG_BAR_CTRL(b, ctrl)); 197 198 rockchip_pcie_write(rockchip, cfg, reg); 199 rockchip_pcie_write(rockchip, addr0, 200 ROCKCHIP_PCIE_AT_IB_EP_FUNC_BAR_ADDR0(fn, bar)); 201 rockchip_pcie_write(rockchip, addr1, 202 ROCKCHIP_PCIE_AT_IB_EP_FUNC_BAR_ADDR1(fn, bar)); 203 204 return 0; 205 } 206 207 static void rockchip_pcie_ep_clear_bar(struct pci_epc *epc, u8 fn, u8 vfn, 208 struct pci_epf_bar *epf_bar) 209 { 210 struct rockchip_pcie_ep *ep = epc_get_drvdata(epc); 211 struct rockchip_pcie *rockchip = &ep->rockchip; 212 u32 reg, cfg, b, ctrl; 213 enum pci_barno bar = epf_bar->barno; 214 215 if (bar < BAR_4) { 216 reg = ROCKCHIP_PCIE_CORE_EP_FUNC_BAR_CFG0(fn); 217 b = bar; 218 } else { 219 reg = ROCKCHIP_PCIE_CORE_EP_FUNC_BAR_CFG1(fn); 220 b = bar - BAR_4; 221 } 222 223 ctrl = ROCKCHIP_PCIE_CORE_BAR_CFG_CTRL_DISABLED; 224 cfg = rockchip_pcie_read(rockchip, reg); 225 cfg &= ~(ROCKCHIP_PCIE_CORE_EP_FUNC_BAR_CFG_BAR_APERTURE_MASK(b) | 226 ROCKCHIP_PCIE_CORE_EP_FUNC_BAR_CFG_BAR_CTRL_MASK(b)); 227 cfg |= ROCKCHIP_PCIE_CORE_EP_FUNC_BAR_CFG_BAR_CTRL(b, ctrl); 228 229 rockchip_pcie_write(rockchip, cfg, reg); 230 rockchip_pcie_write(rockchip, 0x0, 231 ROCKCHIP_PCIE_AT_IB_EP_FUNC_BAR_ADDR0(fn, bar)); 232 rockchip_pcie_write(rockchip, 0x0, 233 ROCKCHIP_PCIE_AT_IB_EP_FUNC_BAR_ADDR1(fn, bar)); 234 } 235 236 static inline u32 rockchip_ob_region(phys_addr_t addr) 237 { 238 return (addr >> ilog2(SZ_1M)) & 0x1f; 239 } 240 241 static int rockchip_pcie_ep_map_addr(struct pci_epc *epc, u8 fn, u8 vfn, 242 phys_addr_t addr, u64 pci_addr, 243 size_t size) 244 { 245 struct rockchip_pcie_ep *ep = epc_get_drvdata(epc); 246 struct rockchip_pcie *pcie = &ep->rockchip; 247 u32 r = rockchip_ob_region(addr); 248 249 rockchip_pcie_prog_ep_ob_atu(pcie, fn, r, addr, pci_addr, size); 250 251 set_bit(r, &ep->ob_region_map); 252 ep->ob_addr[r] = addr; 253 254 return 0; 255 } 256 257 static void rockchip_pcie_ep_unmap_addr(struct pci_epc *epc, u8 fn, u8 vfn, 258 phys_addr_t addr) 259 { 260 struct rockchip_pcie_ep *ep = epc_get_drvdata(epc); 261 struct rockchip_pcie *rockchip = &ep->rockchip; 262 u32 r; 263 264 for (r = 0; r < ep->max_regions; r++) 265 if (ep->ob_addr[r] == addr) 266 break; 267 268 if (r == ep->max_regions) 269 return; 270 271 rockchip_pcie_clear_ep_ob_atu(rockchip, r); 272 273 ep->ob_addr[r] = 0; 274 clear_bit(r, &ep->ob_region_map); 275 } 276 277 static int rockchip_pcie_ep_set_msi(struct pci_epc *epc, u8 fn, u8 vfn, 278 u8 multi_msg_cap) 279 { 280 struct rockchip_pcie_ep *ep = epc_get_drvdata(epc); 281 struct rockchip_pcie *rockchip = &ep->rockchip; 282 u32 flags; 283 284 flags = rockchip_pcie_read(rockchip, 285 ROCKCHIP_PCIE_EP_FUNC_BASE(fn) + 286 ROCKCHIP_PCIE_EP_MSI_CTRL_REG); 287 flags &= ~ROCKCHIP_PCIE_EP_MSI_CTRL_MMC_MASK; 288 flags |= 289 (multi_msg_cap << ROCKCHIP_PCIE_EP_MSI_CTRL_MMC_OFFSET) | 290 (PCI_MSI_FLAGS_64BIT << ROCKCHIP_PCIE_EP_MSI_FLAGS_OFFSET); 291 flags &= ~ROCKCHIP_PCIE_EP_MSI_CTRL_MASK_MSI_CAP; 292 rockchip_pcie_write(rockchip, flags, 293 ROCKCHIP_PCIE_EP_FUNC_BASE(fn) + 294 ROCKCHIP_PCIE_EP_MSI_CTRL_REG); 295 return 0; 296 } 297 298 static int rockchip_pcie_ep_get_msi(struct pci_epc *epc, u8 fn, u8 vfn) 299 { 300 struct rockchip_pcie_ep *ep = epc_get_drvdata(epc); 301 struct rockchip_pcie *rockchip = &ep->rockchip; 302 u32 flags; 303 304 flags = rockchip_pcie_read(rockchip, 305 ROCKCHIP_PCIE_EP_FUNC_BASE(fn) + 306 ROCKCHIP_PCIE_EP_MSI_CTRL_REG); 307 if (!(flags & ROCKCHIP_PCIE_EP_MSI_CTRL_ME)) 308 return -EINVAL; 309 310 return ((flags & ROCKCHIP_PCIE_EP_MSI_CTRL_MME_MASK) >> 311 ROCKCHIP_PCIE_EP_MSI_CTRL_MME_OFFSET); 312 } 313 314 static void rockchip_pcie_ep_assert_intx(struct rockchip_pcie_ep *ep, u8 fn, 315 u8 intx, bool do_assert) 316 { 317 struct rockchip_pcie *rockchip = &ep->rockchip; 318 319 intx &= 3; 320 321 if (do_assert) { 322 ep->irq_pending |= BIT(intx); 323 rockchip_pcie_write(rockchip, 324 PCIE_CLIENT_INT_IN_ASSERT | 325 PCIE_CLIENT_INT_PEND_ST_PEND, 326 PCIE_CLIENT_LEGACY_INT_CTRL); 327 } else { 328 ep->irq_pending &= ~BIT(intx); 329 rockchip_pcie_write(rockchip, 330 PCIE_CLIENT_INT_IN_DEASSERT | 331 PCIE_CLIENT_INT_PEND_ST_NORMAL, 332 PCIE_CLIENT_LEGACY_INT_CTRL); 333 } 334 } 335 336 static int rockchip_pcie_ep_send_legacy_irq(struct rockchip_pcie_ep *ep, u8 fn, 337 u8 intx) 338 { 339 u16 cmd; 340 341 cmd = rockchip_pcie_read(&ep->rockchip, 342 ROCKCHIP_PCIE_EP_FUNC_BASE(fn) + 343 ROCKCHIP_PCIE_EP_CMD_STATUS); 344 345 if (cmd & PCI_COMMAND_INTX_DISABLE) 346 return -EINVAL; 347 348 /* 349 * Should add some delay between toggling INTx per TRM vaguely saying 350 * it depends on some cycles of the AHB bus clock to function it. So 351 * add sufficient 1ms here. 352 */ 353 rockchip_pcie_ep_assert_intx(ep, fn, intx, true); 354 mdelay(1); 355 rockchip_pcie_ep_assert_intx(ep, fn, intx, false); 356 return 0; 357 } 358 359 static int rockchip_pcie_ep_send_msi_irq(struct rockchip_pcie_ep *ep, u8 fn, 360 u8 interrupt_num) 361 { 362 struct rockchip_pcie *rockchip = &ep->rockchip; 363 u32 flags, mme, data, data_mask; 364 u8 msi_count; 365 u64 pci_addr; 366 u32 r; 367 368 /* Check MSI enable bit */ 369 flags = rockchip_pcie_read(&ep->rockchip, 370 ROCKCHIP_PCIE_EP_FUNC_BASE(fn) + 371 ROCKCHIP_PCIE_EP_MSI_CTRL_REG); 372 if (!(flags & ROCKCHIP_PCIE_EP_MSI_CTRL_ME)) 373 return -EINVAL; 374 375 /* Get MSI numbers from MME */ 376 mme = ((flags & ROCKCHIP_PCIE_EP_MSI_CTRL_MME_MASK) >> 377 ROCKCHIP_PCIE_EP_MSI_CTRL_MME_OFFSET); 378 msi_count = 1 << mme; 379 if (!interrupt_num || interrupt_num > msi_count) 380 return -EINVAL; 381 382 /* Set MSI private data */ 383 data_mask = msi_count - 1; 384 data = rockchip_pcie_read(rockchip, 385 ROCKCHIP_PCIE_EP_FUNC_BASE(fn) + 386 ROCKCHIP_PCIE_EP_MSI_CTRL_REG + 387 PCI_MSI_DATA_64); 388 data = (data & ~data_mask) | ((interrupt_num - 1) & data_mask); 389 390 /* Get MSI PCI address */ 391 pci_addr = rockchip_pcie_read(rockchip, 392 ROCKCHIP_PCIE_EP_FUNC_BASE(fn) + 393 ROCKCHIP_PCIE_EP_MSI_CTRL_REG + 394 PCI_MSI_ADDRESS_HI); 395 pci_addr <<= 32; 396 pci_addr |= rockchip_pcie_read(rockchip, 397 ROCKCHIP_PCIE_EP_FUNC_BASE(fn) + 398 ROCKCHIP_PCIE_EP_MSI_CTRL_REG + 399 PCI_MSI_ADDRESS_LO); 400 401 /* Set the outbound region if needed. */ 402 if (unlikely(ep->irq_pci_addr != (pci_addr & PCIE_ADDR_MASK) || 403 ep->irq_pci_fn != fn)) { 404 r = rockchip_ob_region(ep->irq_phys_addr); 405 rockchip_pcie_prog_ep_ob_atu(rockchip, fn, r, 406 ep->irq_phys_addr, 407 pci_addr & PCIE_ADDR_MASK, 408 ~PCIE_ADDR_MASK + 1); 409 ep->irq_pci_addr = (pci_addr & PCIE_ADDR_MASK); 410 ep->irq_pci_fn = fn; 411 } 412 413 writew(data, ep->irq_cpu_addr + (pci_addr & ~PCIE_ADDR_MASK)); 414 return 0; 415 } 416 417 static int rockchip_pcie_ep_raise_irq(struct pci_epc *epc, u8 fn, u8 vfn, 418 enum pci_epc_irq_type type, 419 u16 interrupt_num) 420 { 421 struct rockchip_pcie_ep *ep = epc_get_drvdata(epc); 422 423 switch (type) { 424 case PCI_EPC_IRQ_LEGACY: 425 return rockchip_pcie_ep_send_legacy_irq(ep, fn, 0); 426 case PCI_EPC_IRQ_MSI: 427 return rockchip_pcie_ep_send_msi_irq(ep, fn, interrupt_num); 428 default: 429 return -EINVAL; 430 } 431 } 432 433 static int rockchip_pcie_ep_start(struct pci_epc *epc) 434 { 435 struct rockchip_pcie_ep *ep = epc_get_drvdata(epc); 436 struct rockchip_pcie *rockchip = &ep->rockchip; 437 struct pci_epf *epf; 438 u32 cfg; 439 440 cfg = BIT(0); 441 list_for_each_entry(epf, &epc->pci_epf, list) 442 cfg |= BIT(epf->func_no); 443 444 rockchip_pcie_write(rockchip, cfg, PCIE_CORE_PHY_FUNC_CFG); 445 446 return 0; 447 } 448 449 static const struct pci_epc_features rockchip_pcie_epc_features = { 450 .linkup_notifier = false, 451 .msi_capable = true, 452 .msix_capable = false, 453 .align = 256, 454 }; 455 456 static const struct pci_epc_features* 457 rockchip_pcie_ep_get_features(struct pci_epc *epc, u8 func_no, u8 vfunc_no) 458 { 459 return &rockchip_pcie_epc_features; 460 } 461 462 static const struct pci_epc_ops rockchip_pcie_epc_ops = { 463 .write_header = rockchip_pcie_ep_write_header, 464 .set_bar = rockchip_pcie_ep_set_bar, 465 .clear_bar = rockchip_pcie_ep_clear_bar, 466 .map_addr = rockchip_pcie_ep_map_addr, 467 .unmap_addr = rockchip_pcie_ep_unmap_addr, 468 .set_msi = rockchip_pcie_ep_set_msi, 469 .get_msi = rockchip_pcie_ep_get_msi, 470 .raise_irq = rockchip_pcie_ep_raise_irq, 471 .start = rockchip_pcie_ep_start, 472 .get_features = rockchip_pcie_ep_get_features, 473 }; 474 475 static int rockchip_pcie_parse_ep_dt(struct rockchip_pcie *rockchip, 476 struct rockchip_pcie_ep *ep) 477 { 478 struct device *dev = rockchip->dev; 479 int err; 480 481 err = rockchip_pcie_parse_dt(rockchip); 482 if (err) 483 return err; 484 485 err = rockchip_pcie_get_phys(rockchip); 486 if (err) 487 return err; 488 489 err = of_property_read_u32(dev->of_node, 490 "rockchip,max-outbound-regions", 491 &ep->max_regions); 492 if (err < 0 || ep->max_regions > MAX_REGION_LIMIT) 493 ep->max_regions = MAX_REGION_LIMIT; 494 495 ep->ob_region_map = 0; 496 497 err = of_property_read_u8(dev->of_node, "max-functions", 498 &ep->epc->max_functions); 499 if (err < 0) 500 ep->epc->max_functions = 1; 501 502 return 0; 503 } 504 505 static const struct of_device_id rockchip_pcie_ep_of_match[] = { 506 { .compatible = "rockchip,rk3399-pcie-ep"}, 507 {}, 508 }; 509 510 static int rockchip_pcie_ep_probe(struct platform_device *pdev) 511 { 512 struct device *dev = &pdev->dev; 513 struct rockchip_pcie_ep *ep; 514 struct rockchip_pcie *rockchip; 515 struct pci_epc *epc; 516 size_t max_regions; 517 struct pci_epc_mem_window *windows = NULL; 518 int err, i; 519 u32 cfg_msi, cfg_msix_cp; 520 521 ep = devm_kzalloc(dev, sizeof(*ep), GFP_KERNEL); 522 if (!ep) 523 return -ENOMEM; 524 525 rockchip = &ep->rockchip; 526 rockchip->is_rc = false; 527 rockchip->dev = dev; 528 529 epc = devm_pci_epc_create(dev, &rockchip_pcie_epc_ops); 530 if (IS_ERR(epc)) { 531 dev_err(dev, "failed to create epc device\n"); 532 return PTR_ERR(epc); 533 } 534 535 ep->epc = epc; 536 epc_set_drvdata(epc, ep); 537 538 err = rockchip_pcie_parse_ep_dt(rockchip, ep); 539 if (err) 540 return err; 541 542 err = rockchip_pcie_enable_clocks(rockchip); 543 if (err) 544 return err; 545 546 err = rockchip_pcie_init_port(rockchip); 547 if (err) 548 goto err_disable_clocks; 549 550 /* Establish the link automatically */ 551 rockchip_pcie_write(rockchip, PCIE_CLIENT_LINK_TRAIN_ENABLE, 552 PCIE_CLIENT_CONFIG); 553 554 max_regions = ep->max_regions; 555 ep->ob_addr = devm_kcalloc(dev, max_regions, sizeof(*ep->ob_addr), 556 GFP_KERNEL); 557 558 if (!ep->ob_addr) { 559 err = -ENOMEM; 560 goto err_uninit_port; 561 } 562 563 /* Only enable function 0 by default */ 564 rockchip_pcie_write(rockchip, BIT(0), PCIE_CORE_PHY_FUNC_CFG); 565 566 windows = devm_kcalloc(dev, ep->max_regions, 567 sizeof(struct pci_epc_mem_window), GFP_KERNEL); 568 if (!windows) { 569 err = -ENOMEM; 570 goto err_uninit_port; 571 } 572 for (i = 0; i < ep->max_regions; i++) { 573 windows[i].phys_base = rockchip->mem_res->start + (SZ_1M * i); 574 windows[i].size = SZ_1M; 575 windows[i].page_size = SZ_1M; 576 } 577 err = pci_epc_multi_mem_init(epc, windows, ep->max_regions); 578 devm_kfree(dev, windows); 579 580 if (err < 0) { 581 dev_err(dev, "failed to initialize the memory space\n"); 582 goto err_uninit_port; 583 } 584 585 ep->irq_cpu_addr = pci_epc_mem_alloc_addr(epc, &ep->irq_phys_addr, 586 SZ_1M); 587 if (!ep->irq_cpu_addr) { 588 dev_err(dev, "failed to reserve memory space for MSI\n"); 589 err = -ENOMEM; 590 goto err_epc_mem_exit; 591 } 592 593 ep->irq_pci_addr = ROCKCHIP_PCIE_EP_DUMMY_IRQ_ADDR; 594 595 /* 596 * MSI-X is not supported but the controller still advertises the MSI-X 597 * capability by default, which can lead to the Root Complex side 598 * allocating MSI-X vectors which cannot be used. Avoid this by skipping 599 * the MSI-X capability entry in the PCIe capabilities linked-list: get 600 * the next pointer from the MSI-X entry and set that in the MSI 601 * capability entry (which is the previous entry). This way the MSI-X 602 * entry is skipped (left out of the linked-list) and not advertised. 603 */ 604 cfg_msi = rockchip_pcie_read(rockchip, PCIE_EP_CONFIG_BASE + 605 ROCKCHIP_PCIE_EP_MSI_CTRL_REG); 606 607 cfg_msi &= ~ROCKCHIP_PCIE_EP_MSI_CP1_MASK; 608 609 cfg_msix_cp = rockchip_pcie_read(rockchip, PCIE_EP_CONFIG_BASE + 610 ROCKCHIP_PCIE_EP_MSIX_CAP_REG) & 611 ROCKCHIP_PCIE_EP_MSIX_CAP_CP_MASK; 612 613 cfg_msi |= cfg_msix_cp; 614 615 rockchip_pcie_write(rockchip, cfg_msi, 616 PCIE_EP_CONFIG_BASE + ROCKCHIP_PCIE_EP_MSI_CTRL_REG); 617 618 rockchip_pcie_write(rockchip, PCIE_CLIENT_CONF_ENABLE, 619 PCIE_CLIENT_CONFIG); 620 621 return 0; 622 err_epc_mem_exit: 623 pci_epc_mem_exit(epc); 624 err_uninit_port: 625 rockchip_pcie_deinit_phys(rockchip); 626 err_disable_clocks: 627 rockchip_pcie_disable_clocks(rockchip); 628 return err; 629 } 630 631 static struct platform_driver rockchip_pcie_ep_driver = { 632 .driver = { 633 .name = "rockchip-pcie-ep", 634 .of_match_table = rockchip_pcie_ep_of_match, 635 }, 636 .probe = rockchip_pcie_ep_probe, 637 }; 638 639 builtin_platform_driver(rockchip_pcie_ep_driver); 640