1 /* 2 * Support PCI/PCIe on PowerNV platforms 3 * 4 * Copyright 2011 Benjamin Herrenschmidt, IBM Corp. 5 * 6 * This program is free software; you can redistribute it and/or 7 * modify it under the terms of the GNU General Public License 8 * as published by the Free Software Foundation; either version 9 * 2 of the License, or (at your option) any later version. 10 */ 11 12 #undef DEBUG 13 14 #include <linux/kernel.h> 15 #include <linux/pci.h> 16 #include <linux/delay.h> 17 #include <linux/string.h> 18 #include <linux/init.h> 19 #include <linux/bootmem.h> 20 #include <linux/irq.h> 21 #include <linux/io.h> 22 #include <linux/msi.h> 23 24 #include <asm/sections.h> 25 #include <asm/io.h> 26 #include <asm/prom.h> 27 #include <asm/pci-bridge.h> 28 #include <asm/machdep.h> 29 #include <asm/msi_bitmap.h> 30 #include <asm/ppc-pci.h> 31 #include <asm/opal.h> 32 #include <asm/iommu.h> 33 #include <asm/tce.h> 34 #include <asm/xics.h> 35 36 #include "powernv.h" 37 #include "pci.h" 38 39 #define define_pe_printk_level(func, kern_level) \ 40 static int func(const struct pnv_ioda_pe *pe, const char *fmt, ...) \ 41 { \ 42 struct va_format vaf; \ 43 va_list args; \ 44 char pfix[32]; \ 45 int r; \ 46 \ 47 va_start(args, fmt); \ 48 \ 49 vaf.fmt = fmt; \ 50 vaf.va = &args; \ 51 \ 52 if (pe->pdev) \ 53 strlcpy(pfix, dev_name(&pe->pdev->dev), \ 54 sizeof(pfix)); \ 55 else \ 56 sprintf(pfix, "%04x:%02x ", \ 57 pci_domain_nr(pe->pbus), \ 58 pe->pbus->number); \ 59 r = printk(kern_level "pci %s: [PE# %.3d] %pV", \ 60 pfix, pe->pe_number, &vaf); \ 61 \ 62 va_end(args); \ 63 \ 64 return r; \ 65 } \ 66 67 define_pe_printk_level(pe_err, KERN_ERR); 68 define_pe_printk_level(pe_warn, KERN_WARNING); 69 define_pe_printk_level(pe_info, KERN_INFO); 70 71 static int pnv_ioda_alloc_pe(struct pnv_phb *phb) 72 { 73 unsigned long pe; 74 75 do { 76 pe = find_next_zero_bit(phb->ioda.pe_alloc, 77 phb->ioda.total_pe, 0); 78 if (pe >= phb->ioda.total_pe) 79 return IODA_INVALID_PE; 80 } while(test_and_set_bit(pe, phb->ioda.pe_alloc)); 81 82 phb->ioda.pe_array[pe].phb = phb; 83 phb->ioda.pe_array[pe].pe_number = pe; 84 return pe; 85 } 86 87 static void pnv_ioda_free_pe(struct pnv_phb *phb, int pe) 88 { 89 WARN_ON(phb->ioda.pe_array[pe].pdev); 90 91 memset(&phb->ioda.pe_array[pe], 0, sizeof(struct pnv_ioda_pe)); 92 clear_bit(pe, phb->ioda.pe_alloc); 93 } 94 95 /* Currently those 2 are only used when MSIs are enabled, this will change 96 * but in the meantime, we need to protect them to avoid warnings 97 */ 98 #ifdef CONFIG_PCI_MSI 99 static struct pnv_ioda_pe *pnv_ioda_get_pe(struct pci_dev *dev) 100 { 101 struct pci_controller *hose = pci_bus_to_host(dev->bus); 102 struct pnv_phb *phb = hose->private_data; 103 struct pci_dn *pdn = pci_get_pdn(dev); 104 105 if (!pdn) 106 return NULL; 107 if (pdn->pe_number == IODA_INVALID_PE) 108 return NULL; 109 return &phb->ioda.pe_array[pdn->pe_number]; 110 } 111 #endif /* CONFIG_PCI_MSI */ 112 113 static int pnv_ioda_configure_pe(struct pnv_phb *phb, struct pnv_ioda_pe *pe) 114 { 115 struct pci_dev *parent; 116 uint8_t bcomp, dcomp, fcomp; 117 long rc, rid_end, rid; 118 119 /* Bus validation ? */ 120 if (pe->pbus) { 121 int count; 122 123 dcomp = OPAL_IGNORE_RID_DEVICE_NUMBER; 124 fcomp = OPAL_IGNORE_RID_FUNCTION_NUMBER; 125 parent = pe->pbus->self; 126 if (pe->flags & PNV_IODA_PE_BUS_ALL) 127 count = pe->pbus->busn_res.end - pe->pbus->busn_res.start + 1; 128 else 129 count = 1; 130 131 switch(count) { 132 case 1: bcomp = OpalPciBusAll; break; 133 case 2: bcomp = OpalPciBus7Bits; break; 134 case 4: bcomp = OpalPciBus6Bits; break; 135 case 8: bcomp = OpalPciBus5Bits; break; 136 case 16: bcomp = OpalPciBus4Bits; break; 137 case 32: bcomp = OpalPciBus3Bits; break; 138 default: 139 pr_err("%s: Number of subordinate busses %d" 140 " unsupported\n", 141 pci_name(pe->pbus->self), count); 142 /* Do an exact match only */ 143 bcomp = OpalPciBusAll; 144 } 145 rid_end = pe->rid + (count << 8); 146 } else { 147 parent = pe->pdev->bus->self; 148 bcomp = OpalPciBusAll; 149 dcomp = OPAL_COMPARE_RID_DEVICE_NUMBER; 150 fcomp = OPAL_COMPARE_RID_FUNCTION_NUMBER; 151 rid_end = pe->rid + 1; 152 } 153 154 /* Associate PE in PELT */ 155 rc = opal_pci_set_pe(phb->opal_id, pe->pe_number, pe->rid, 156 bcomp, dcomp, fcomp, OPAL_MAP_PE); 157 if (rc) { 158 pe_err(pe, "OPAL error %ld trying to setup PELT table\n", rc); 159 return -ENXIO; 160 } 161 opal_pci_eeh_freeze_clear(phb->opal_id, pe->pe_number, 162 OPAL_EEH_ACTION_CLEAR_FREEZE_ALL); 163 164 /* Add to all parents PELT-V */ 165 while (parent) { 166 struct pci_dn *pdn = pci_get_pdn(parent); 167 if (pdn && pdn->pe_number != IODA_INVALID_PE) { 168 rc = opal_pci_set_peltv(phb->opal_id, pdn->pe_number, 169 pe->pe_number, OPAL_ADD_PE_TO_DOMAIN); 170 /* XXX What to do in case of error ? */ 171 } 172 parent = parent->bus->self; 173 } 174 /* Setup reverse map */ 175 for (rid = pe->rid; rid < rid_end; rid++) 176 phb->ioda.pe_rmap[rid] = pe->pe_number; 177 178 /* Setup one MVTs on IODA1 */ 179 if (phb->type == PNV_PHB_IODA1) { 180 pe->mve_number = pe->pe_number; 181 rc = opal_pci_set_mve(phb->opal_id, pe->mve_number, 182 pe->pe_number); 183 if (rc) { 184 pe_err(pe, "OPAL error %ld setting up MVE %d\n", 185 rc, pe->mve_number); 186 pe->mve_number = -1; 187 } else { 188 rc = opal_pci_set_mve_enable(phb->opal_id, 189 pe->mve_number, OPAL_ENABLE_MVE); 190 if (rc) { 191 pe_err(pe, "OPAL error %ld enabling MVE %d\n", 192 rc, pe->mve_number); 193 pe->mve_number = -1; 194 } 195 } 196 } else if (phb->type == PNV_PHB_IODA2) 197 pe->mve_number = 0; 198 199 return 0; 200 } 201 202 static void pnv_ioda_link_pe_by_weight(struct pnv_phb *phb, 203 struct pnv_ioda_pe *pe) 204 { 205 struct pnv_ioda_pe *lpe; 206 207 list_for_each_entry(lpe, &phb->ioda.pe_dma_list, dma_link) { 208 if (lpe->dma_weight < pe->dma_weight) { 209 list_add_tail(&pe->dma_link, &lpe->dma_link); 210 return; 211 } 212 } 213 list_add_tail(&pe->dma_link, &phb->ioda.pe_dma_list); 214 } 215 216 static unsigned int pnv_ioda_dma_weight(struct pci_dev *dev) 217 { 218 /* This is quite simplistic. The "base" weight of a device 219 * is 10. 0 means no DMA is to be accounted for it. 220 */ 221 222 /* If it's a bridge, no DMA */ 223 if (dev->hdr_type != PCI_HEADER_TYPE_NORMAL) 224 return 0; 225 226 /* Reduce the weight of slow USB controllers */ 227 if (dev->class == PCI_CLASS_SERIAL_USB_UHCI || 228 dev->class == PCI_CLASS_SERIAL_USB_OHCI || 229 dev->class == PCI_CLASS_SERIAL_USB_EHCI) 230 return 3; 231 232 /* Increase the weight of RAID (includes Obsidian) */ 233 if ((dev->class >> 8) == PCI_CLASS_STORAGE_RAID) 234 return 15; 235 236 /* Default */ 237 return 10; 238 } 239 240 #if 0 241 static struct pnv_ioda_pe *pnv_ioda_setup_dev_PE(struct pci_dev *dev) 242 { 243 struct pci_controller *hose = pci_bus_to_host(dev->bus); 244 struct pnv_phb *phb = hose->private_data; 245 struct pci_dn *pdn = pci_get_pdn(dev); 246 struct pnv_ioda_pe *pe; 247 int pe_num; 248 249 if (!pdn) { 250 pr_err("%s: Device tree node not associated properly\n", 251 pci_name(dev)); 252 return NULL; 253 } 254 if (pdn->pe_number != IODA_INVALID_PE) 255 return NULL; 256 257 /* PE#0 has been pre-set */ 258 if (dev->bus->number == 0) 259 pe_num = 0; 260 else 261 pe_num = pnv_ioda_alloc_pe(phb); 262 if (pe_num == IODA_INVALID_PE) { 263 pr_warning("%s: Not enough PE# available, disabling device\n", 264 pci_name(dev)); 265 return NULL; 266 } 267 268 /* NOTE: We get only one ref to the pci_dev for the pdn, not for the 269 * pointer in the PE data structure, both should be destroyed at the 270 * same time. However, this needs to be looked at more closely again 271 * once we actually start removing things (Hotplug, SR-IOV, ...) 272 * 273 * At some point we want to remove the PDN completely anyways 274 */ 275 pe = &phb->ioda.pe_array[pe_num]; 276 pci_dev_get(dev); 277 pdn->pcidev = dev; 278 pdn->pe_number = pe_num; 279 pe->pdev = dev; 280 pe->pbus = NULL; 281 pe->tce32_seg = -1; 282 pe->mve_number = -1; 283 pe->rid = dev->bus->number << 8 | pdn->devfn; 284 285 pe_info(pe, "Associated device to PE\n"); 286 287 if (pnv_ioda_configure_pe(phb, pe)) { 288 /* XXX What do we do here ? */ 289 if (pe_num) 290 pnv_ioda_free_pe(phb, pe_num); 291 pdn->pe_number = IODA_INVALID_PE; 292 pe->pdev = NULL; 293 pci_dev_put(dev); 294 return NULL; 295 } 296 297 /* Assign a DMA weight to the device */ 298 pe->dma_weight = pnv_ioda_dma_weight(dev); 299 if (pe->dma_weight != 0) { 300 phb->ioda.dma_weight += pe->dma_weight; 301 phb->ioda.dma_pe_count++; 302 } 303 304 /* Link the PE */ 305 pnv_ioda_link_pe_by_weight(phb, pe); 306 307 return pe; 308 } 309 #endif /* Useful for SRIOV case */ 310 311 static void pnv_ioda_setup_same_PE(struct pci_bus *bus, struct pnv_ioda_pe *pe) 312 { 313 struct pci_dev *dev; 314 315 list_for_each_entry(dev, &bus->devices, bus_list) { 316 struct pci_dn *pdn = pci_get_pdn(dev); 317 318 if (pdn == NULL) { 319 pr_warn("%s: No device node associated with device !\n", 320 pci_name(dev)); 321 continue; 322 } 323 pci_dev_get(dev); 324 pdn->pcidev = dev; 325 pdn->pe_number = pe->pe_number; 326 pe->dma_weight += pnv_ioda_dma_weight(dev); 327 if ((pe->flags & PNV_IODA_PE_BUS_ALL) && dev->subordinate) 328 pnv_ioda_setup_same_PE(dev->subordinate, pe); 329 } 330 } 331 332 /* 333 * There're 2 types of PCI bus sensitive PEs: One that is compromised of 334 * single PCI bus. Another one that contains the primary PCI bus and its 335 * subordinate PCI devices and buses. The second type of PE is normally 336 * orgiriated by PCIe-to-PCI bridge or PLX switch downstream ports. 337 */ 338 static void pnv_ioda_setup_bus_PE(struct pci_bus *bus, int all) 339 { 340 struct pci_controller *hose = pci_bus_to_host(bus); 341 struct pnv_phb *phb = hose->private_data; 342 struct pnv_ioda_pe *pe; 343 int pe_num; 344 345 pe_num = pnv_ioda_alloc_pe(phb); 346 if (pe_num == IODA_INVALID_PE) { 347 pr_warning("%s: Not enough PE# available for PCI bus %04x:%02x\n", 348 __func__, pci_domain_nr(bus), bus->number); 349 return; 350 } 351 352 pe = &phb->ioda.pe_array[pe_num]; 353 pe->flags = (all ? PNV_IODA_PE_BUS_ALL : PNV_IODA_PE_BUS); 354 pe->pbus = bus; 355 pe->pdev = NULL; 356 pe->tce32_seg = -1; 357 pe->mve_number = -1; 358 pe->rid = bus->busn_res.start << 8; 359 pe->dma_weight = 0; 360 361 if (all) 362 pe_info(pe, "Secondary bus %d..%d associated with PE#%d\n", 363 bus->busn_res.start, bus->busn_res.end, pe_num); 364 else 365 pe_info(pe, "Secondary bus %d associated with PE#%d\n", 366 bus->busn_res.start, pe_num); 367 368 if (pnv_ioda_configure_pe(phb, pe)) { 369 /* XXX What do we do here ? */ 370 if (pe_num) 371 pnv_ioda_free_pe(phb, pe_num); 372 pe->pbus = NULL; 373 return; 374 } 375 376 /* Associate it with all child devices */ 377 pnv_ioda_setup_same_PE(bus, pe); 378 379 /* Put PE to the list */ 380 list_add_tail(&pe->list, &phb->ioda.pe_list); 381 382 /* Account for one DMA PE if at least one DMA capable device exist 383 * below the bridge 384 */ 385 if (pe->dma_weight != 0) { 386 phb->ioda.dma_weight += pe->dma_weight; 387 phb->ioda.dma_pe_count++; 388 } 389 390 /* Link the PE */ 391 pnv_ioda_link_pe_by_weight(phb, pe); 392 } 393 394 static void pnv_ioda_setup_PEs(struct pci_bus *bus) 395 { 396 struct pci_dev *dev; 397 398 pnv_ioda_setup_bus_PE(bus, 0); 399 400 list_for_each_entry(dev, &bus->devices, bus_list) { 401 if (dev->subordinate) { 402 if (pci_pcie_type(dev) == PCI_EXP_TYPE_PCI_BRIDGE) 403 pnv_ioda_setup_bus_PE(dev->subordinate, 1); 404 else 405 pnv_ioda_setup_PEs(dev->subordinate); 406 } 407 } 408 } 409 410 /* 411 * Configure PEs so that the downstream PCI buses and devices 412 * could have their associated PE#. Unfortunately, we didn't 413 * figure out the way to identify the PLX bridge yet. So we 414 * simply put the PCI bus and the subordinate behind the root 415 * port to PE# here. The game rule here is expected to be changed 416 * as soon as we can detected PLX bridge correctly. 417 */ 418 static void pnv_pci_ioda_setup_PEs(void) 419 { 420 struct pci_controller *hose, *tmp; 421 422 list_for_each_entry_safe(hose, tmp, &hose_list, list_node) { 423 pnv_ioda_setup_PEs(hose->bus); 424 } 425 } 426 427 static void pnv_pci_ioda_dma_dev_setup(struct pnv_phb *phb, struct pci_dev *pdev) 428 { 429 struct pci_dn *pdn = pci_get_pdn(pdev); 430 struct pnv_ioda_pe *pe; 431 432 /* 433 * The function can be called while the PE# 434 * hasn't been assigned. Do nothing for the 435 * case. 436 */ 437 if (!pdn || pdn->pe_number == IODA_INVALID_PE) 438 return; 439 440 pe = &phb->ioda.pe_array[pdn->pe_number]; 441 set_iommu_table_base(&pdev->dev, &pe->tce32_table); 442 } 443 444 static void pnv_pci_ioda1_tce_invalidate(struct iommu_table *tbl, 445 u64 *startp, u64 *endp) 446 { 447 u64 __iomem *invalidate = (u64 __iomem *)tbl->it_index; 448 unsigned long start, end, inc; 449 450 start = __pa(startp); 451 end = __pa(endp); 452 453 /* BML uses this case for p6/p7/galaxy2: Shift addr and put in node */ 454 if (tbl->it_busno) { 455 start <<= 12; 456 end <<= 12; 457 inc = 128 << 12; 458 start |= tbl->it_busno; 459 end |= tbl->it_busno; 460 } else if (tbl->it_type & TCE_PCI_SWINV_PAIR) { 461 /* p7ioc-style invalidation, 2 TCEs per write */ 462 start |= (1ull << 63); 463 end |= (1ull << 63); 464 inc = 16; 465 } else { 466 /* Default (older HW) */ 467 inc = 128; 468 } 469 470 end |= inc - 1; /* round up end to be different than start */ 471 472 mb(); /* Ensure above stores are visible */ 473 while (start <= end) { 474 __raw_writeq(start, invalidate); 475 start += inc; 476 } 477 478 /* 479 * The iommu layer will do another mb() for us on build() 480 * and we don't care on free() 481 */ 482 } 483 484 static void pnv_pci_ioda2_tce_invalidate(struct pnv_ioda_pe *pe, 485 struct iommu_table *tbl, 486 u64 *startp, u64 *endp) 487 { 488 unsigned long start, end, inc; 489 u64 __iomem *invalidate = (u64 __iomem *)tbl->it_index; 490 491 /* We'll invalidate DMA address in PE scope */ 492 start = 0x2ul << 60; 493 start |= (pe->pe_number & 0xFF); 494 end = start; 495 496 /* Figure out the start, end and step */ 497 inc = tbl->it_offset + (((u64)startp - tbl->it_base) / sizeof(u64)); 498 start |= (inc << 12); 499 inc = tbl->it_offset + (((u64)endp - tbl->it_base) / sizeof(u64)); 500 end |= (inc << 12); 501 inc = (0x1ul << 12); 502 mb(); 503 504 while (start <= end) { 505 __raw_writeq(start, invalidate); 506 start += inc; 507 } 508 } 509 510 void pnv_pci_ioda_tce_invalidate(struct iommu_table *tbl, 511 u64 *startp, u64 *endp) 512 { 513 struct pnv_ioda_pe *pe = container_of(tbl, struct pnv_ioda_pe, 514 tce32_table); 515 struct pnv_phb *phb = pe->phb; 516 517 if (phb->type == PNV_PHB_IODA1) 518 pnv_pci_ioda1_tce_invalidate(tbl, startp, endp); 519 else 520 pnv_pci_ioda2_tce_invalidate(pe, tbl, startp, endp); 521 } 522 523 static void pnv_pci_ioda_setup_dma_pe(struct pnv_phb *phb, 524 struct pnv_ioda_pe *pe, unsigned int base, 525 unsigned int segs) 526 { 527 528 struct page *tce_mem = NULL; 529 const __be64 *swinvp; 530 struct iommu_table *tbl; 531 unsigned int i; 532 int64_t rc; 533 void *addr; 534 535 /* 256M DMA window, 4K TCE pages, 8 bytes TCE */ 536 #define TCE32_TABLE_SIZE ((0x10000000 / 0x1000) * 8) 537 538 /* XXX FIXME: Handle 64-bit only DMA devices */ 539 /* XXX FIXME: Provide 64-bit DMA facilities & non-4K TCE tables etc.. */ 540 /* XXX FIXME: Allocate multi-level tables on PHB3 */ 541 542 /* We shouldn't already have a 32-bit DMA associated */ 543 if (WARN_ON(pe->tce32_seg >= 0)) 544 return; 545 546 /* Grab a 32-bit TCE table */ 547 pe->tce32_seg = base; 548 pe_info(pe, " Setting up 32-bit TCE table at %08x..%08x\n", 549 (base << 28), ((base + segs) << 28) - 1); 550 551 /* XXX Currently, we allocate one big contiguous table for the 552 * TCEs. We only really need one chunk per 256M of TCE space 553 * (ie per segment) but that's an optimization for later, it 554 * requires some added smarts with our get/put_tce implementation 555 */ 556 tce_mem = alloc_pages_node(phb->hose->node, GFP_KERNEL, 557 get_order(TCE32_TABLE_SIZE * segs)); 558 if (!tce_mem) { 559 pe_err(pe, " Failed to allocate a 32-bit TCE memory\n"); 560 goto fail; 561 } 562 addr = page_address(tce_mem); 563 memset(addr, 0, TCE32_TABLE_SIZE * segs); 564 565 /* Configure HW */ 566 for (i = 0; i < segs; i++) { 567 rc = opal_pci_map_pe_dma_window(phb->opal_id, 568 pe->pe_number, 569 base + i, 1, 570 __pa(addr) + TCE32_TABLE_SIZE * i, 571 TCE32_TABLE_SIZE, 0x1000); 572 if (rc) { 573 pe_err(pe, " Failed to configure 32-bit TCE table," 574 " err %ld\n", rc); 575 goto fail; 576 } 577 } 578 579 /* Setup linux iommu table */ 580 tbl = &pe->tce32_table; 581 pnv_pci_setup_iommu_table(tbl, addr, TCE32_TABLE_SIZE * segs, 582 base << 28); 583 584 /* OPAL variant of P7IOC SW invalidated TCEs */ 585 swinvp = of_get_property(phb->hose->dn, "ibm,opal-tce-kill", NULL); 586 if (swinvp) { 587 /* We need a couple more fields -- an address and a data 588 * to or. Since the bus is only printed out on table free 589 * errors, and on the first pass the data will be a relative 590 * bus number, print that out instead. 591 */ 592 tbl->it_busno = 0; 593 tbl->it_index = (unsigned long)ioremap(be64_to_cpup(swinvp), 8); 594 tbl->it_type = TCE_PCI_SWINV_CREATE | TCE_PCI_SWINV_FREE | 595 TCE_PCI_SWINV_PAIR; 596 } 597 iommu_init_table(tbl, phb->hose->node); 598 599 return; 600 fail: 601 /* XXX Failure: Try to fallback to 64-bit only ? */ 602 if (pe->tce32_seg >= 0) 603 pe->tce32_seg = -1; 604 if (tce_mem) 605 __free_pages(tce_mem, get_order(TCE32_TABLE_SIZE * segs)); 606 } 607 608 static void pnv_pci_ioda2_setup_dma_pe(struct pnv_phb *phb, 609 struct pnv_ioda_pe *pe) 610 { 611 struct page *tce_mem = NULL; 612 void *addr; 613 const __be64 *swinvp; 614 struct iommu_table *tbl; 615 unsigned int tce_table_size, end; 616 int64_t rc; 617 618 /* We shouldn't already have a 32-bit DMA associated */ 619 if (WARN_ON(pe->tce32_seg >= 0)) 620 return; 621 622 /* The PE will reserve all possible 32-bits space */ 623 pe->tce32_seg = 0; 624 end = (1 << ilog2(phb->ioda.m32_pci_base)); 625 tce_table_size = (end / 0x1000) * 8; 626 pe_info(pe, "Setting up 32-bit TCE table at 0..%08x\n", 627 end); 628 629 /* Allocate TCE table */ 630 tce_mem = alloc_pages_node(phb->hose->node, GFP_KERNEL, 631 get_order(tce_table_size)); 632 if (!tce_mem) { 633 pe_err(pe, "Failed to allocate a 32-bit TCE memory\n"); 634 goto fail; 635 } 636 addr = page_address(tce_mem); 637 memset(addr, 0, tce_table_size); 638 639 /* 640 * Map TCE table through TVT. The TVE index is the PE number 641 * shifted by 1 bit for 32-bits DMA space. 642 */ 643 rc = opal_pci_map_pe_dma_window(phb->opal_id, pe->pe_number, 644 pe->pe_number << 1, 1, __pa(addr), 645 tce_table_size, 0x1000); 646 if (rc) { 647 pe_err(pe, "Failed to configure 32-bit TCE table," 648 " err %ld\n", rc); 649 goto fail; 650 } 651 652 /* Setup linux iommu table */ 653 tbl = &pe->tce32_table; 654 pnv_pci_setup_iommu_table(tbl, addr, tce_table_size, 0); 655 656 /* OPAL variant of PHB3 invalidated TCEs */ 657 swinvp = of_get_property(phb->hose->dn, "ibm,opal-tce-kill", NULL); 658 if (swinvp) { 659 /* We need a couple more fields -- an address and a data 660 * to or. Since the bus is only printed out on table free 661 * errors, and on the first pass the data will be a relative 662 * bus number, print that out instead. 663 */ 664 tbl->it_busno = 0; 665 tbl->it_index = (unsigned long)ioremap(be64_to_cpup(swinvp), 8); 666 tbl->it_type = TCE_PCI_SWINV_CREATE | TCE_PCI_SWINV_FREE; 667 } 668 iommu_init_table(tbl, phb->hose->node); 669 670 return; 671 fail: 672 if (pe->tce32_seg >= 0) 673 pe->tce32_seg = -1; 674 if (tce_mem) 675 __free_pages(tce_mem, get_order(tce_table_size)); 676 } 677 678 static void pnv_ioda_setup_dma(struct pnv_phb *phb) 679 { 680 struct pci_controller *hose = phb->hose; 681 unsigned int residual, remaining, segs, tw, base; 682 struct pnv_ioda_pe *pe; 683 684 /* If we have more PE# than segments available, hand out one 685 * per PE until we run out and let the rest fail. If not, 686 * then we assign at least one segment per PE, plus more based 687 * on the amount of devices under that PE 688 */ 689 if (phb->ioda.dma_pe_count > phb->ioda.tce32_count) 690 residual = 0; 691 else 692 residual = phb->ioda.tce32_count - 693 phb->ioda.dma_pe_count; 694 695 pr_info("PCI: Domain %04x has %ld available 32-bit DMA segments\n", 696 hose->global_number, phb->ioda.tce32_count); 697 pr_info("PCI: %d PE# for a total weight of %d\n", 698 phb->ioda.dma_pe_count, phb->ioda.dma_weight); 699 700 /* Walk our PE list and configure their DMA segments, hand them 701 * out one base segment plus any residual segments based on 702 * weight 703 */ 704 remaining = phb->ioda.tce32_count; 705 tw = phb->ioda.dma_weight; 706 base = 0; 707 list_for_each_entry(pe, &phb->ioda.pe_dma_list, dma_link) { 708 if (!pe->dma_weight) 709 continue; 710 if (!remaining) { 711 pe_warn(pe, "No DMA32 resources available\n"); 712 continue; 713 } 714 segs = 1; 715 if (residual) { 716 segs += ((pe->dma_weight * residual) + (tw / 2)) / tw; 717 if (segs > remaining) 718 segs = remaining; 719 } 720 721 /* 722 * For IODA2 compliant PHB3, we needn't care about the weight. 723 * The all available 32-bits DMA space will be assigned to 724 * the specific PE. 725 */ 726 if (phb->type == PNV_PHB_IODA1) { 727 pe_info(pe, "DMA weight %d, assigned %d DMA32 segments\n", 728 pe->dma_weight, segs); 729 pnv_pci_ioda_setup_dma_pe(phb, pe, base, segs); 730 } else { 731 pe_info(pe, "Assign DMA32 space\n"); 732 segs = 0; 733 pnv_pci_ioda2_setup_dma_pe(phb, pe); 734 } 735 736 remaining -= segs; 737 base += segs; 738 } 739 } 740 741 #ifdef CONFIG_PCI_MSI 742 static void pnv_ioda2_msi_eoi(struct irq_data *d) 743 { 744 unsigned int hw_irq = (unsigned int)irqd_to_hwirq(d); 745 struct irq_chip *chip = irq_data_get_irq_chip(d); 746 struct pnv_phb *phb = container_of(chip, struct pnv_phb, 747 ioda.irq_chip); 748 int64_t rc; 749 750 rc = opal_pci_msi_eoi(phb->opal_id, hw_irq); 751 WARN_ON_ONCE(rc); 752 753 icp_native_eoi(d); 754 } 755 756 static int pnv_pci_ioda_msi_setup(struct pnv_phb *phb, struct pci_dev *dev, 757 unsigned int hwirq, unsigned int virq, 758 unsigned int is_64, struct msi_msg *msg) 759 { 760 struct pnv_ioda_pe *pe = pnv_ioda_get_pe(dev); 761 struct pci_dn *pdn = pci_get_pdn(dev); 762 struct irq_data *idata; 763 struct irq_chip *ichip; 764 unsigned int xive_num = hwirq - phb->msi_base; 765 uint64_t addr64; 766 uint32_t addr32, data; 767 int rc; 768 769 /* No PE assigned ? bail out ... no MSI for you ! */ 770 if (pe == NULL) 771 return -ENXIO; 772 773 /* Check if we have an MVE */ 774 if (pe->mve_number < 0) 775 return -ENXIO; 776 777 /* Force 32-bit MSI on some broken devices */ 778 if (pdn && pdn->force_32bit_msi) 779 is_64 = 0; 780 781 /* Assign XIVE to PE */ 782 rc = opal_pci_set_xive_pe(phb->opal_id, pe->pe_number, xive_num); 783 if (rc) { 784 pr_warn("%s: OPAL error %d setting XIVE %d PE\n", 785 pci_name(dev), rc, xive_num); 786 return -EIO; 787 } 788 789 if (is_64) { 790 rc = opal_get_msi_64(phb->opal_id, pe->mve_number, xive_num, 1, 791 &addr64, &data); 792 if (rc) { 793 pr_warn("%s: OPAL error %d getting 64-bit MSI data\n", 794 pci_name(dev), rc); 795 return -EIO; 796 } 797 msg->address_hi = addr64 >> 32; 798 msg->address_lo = addr64 & 0xfffffffful; 799 } else { 800 rc = opal_get_msi_32(phb->opal_id, pe->mve_number, xive_num, 1, 801 &addr32, &data); 802 if (rc) { 803 pr_warn("%s: OPAL error %d getting 32-bit MSI data\n", 804 pci_name(dev), rc); 805 return -EIO; 806 } 807 msg->address_hi = 0; 808 msg->address_lo = addr32; 809 } 810 msg->data = data; 811 812 /* 813 * Change the IRQ chip for the MSI interrupts on PHB3. 814 * The corresponding IRQ chip should be populated for 815 * the first time. 816 */ 817 if (phb->type == PNV_PHB_IODA2) { 818 if (!phb->ioda.irq_chip_init) { 819 idata = irq_get_irq_data(virq); 820 ichip = irq_data_get_irq_chip(idata); 821 phb->ioda.irq_chip_init = 1; 822 phb->ioda.irq_chip = *ichip; 823 phb->ioda.irq_chip.irq_eoi = pnv_ioda2_msi_eoi; 824 } 825 826 irq_set_chip(virq, &phb->ioda.irq_chip); 827 } 828 829 pr_devel("%s: %s-bit MSI on hwirq %x (xive #%d)," 830 " address=%x_%08x data=%x PE# %d\n", 831 pci_name(dev), is_64 ? "64" : "32", hwirq, xive_num, 832 msg->address_hi, msg->address_lo, data, pe->pe_number); 833 834 return 0; 835 } 836 837 static void pnv_pci_init_ioda_msis(struct pnv_phb *phb) 838 { 839 unsigned int count; 840 const __be32 *prop = of_get_property(phb->hose->dn, 841 "ibm,opal-msi-ranges", NULL); 842 if (!prop) { 843 /* BML Fallback */ 844 prop = of_get_property(phb->hose->dn, "msi-ranges", NULL); 845 } 846 if (!prop) 847 return; 848 849 phb->msi_base = be32_to_cpup(prop); 850 count = be32_to_cpup(prop + 1); 851 if (msi_bitmap_alloc(&phb->msi_bmp, count, phb->hose->dn)) { 852 pr_err("PCI %d: Failed to allocate MSI bitmap !\n", 853 phb->hose->global_number); 854 return; 855 } 856 857 phb->msi_setup = pnv_pci_ioda_msi_setup; 858 phb->msi32_support = 1; 859 pr_info(" Allocated bitmap for %d MSIs (base IRQ 0x%x)\n", 860 count, phb->msi_base); 861 } 862 #else 863 static void pnv_pci_init_ioda_msis(struct pnv_phb *phb) { } 864 #endif /* CONFIG_PCI_MSI */ 865 866 /* 867 * This function is supposed to be called on basis of PE from top 868 * to bottom style. So the the I/O or MMIO segment assigned to 869 * parent PE could be overrided by its child PEs if necessary. 870 */ 871 static void pnv_ioda_setup_pe_seg(struct pci_controller *hose, 872 struct pnv_ioda_pe *pe) 873 { 874 struct pnv_phb *phb = hose->private_data; 875 struct pci_bus_region region; 876 struct resource *res; 877 int i, index; 878 int rc; 879 880 /* 881 * NOTE: We only care PCI bus based PE for now. For PCI 882 * device based PE, for example SRIOV sensitive VF should 883 * be figured out later. 884 */ 885 BUG_ON(!(pe->flags & (PNV_IODA_PE_BUS | PNV_IODA_PE_BUS_ALL))); 886 887 pci_bus_for_each_resource(pe->pbus, res, i) { 888 if (!res || !res->flags || 889 res->start > res->end) 890 continue; 891 892 if (res->flags & IORESOURCE_IO) { 893 region.start = res->start - phb->ioda.io_pci_base; 894 region.end = res->end - phb->ioda.io_pci_base; 895 index = region.start / phb->ioda.io_segsize; 896 897 while (index < phb->ioda.total_pe && 898 region.start <= region.end) { 899 phb->ioda.io_segmap[index] = pe->pe_number; 900 rc = opal_pci_map_pe_mmio_window(phb->opal_id, 901 pe->pe_number, OPAL_IO_WINDOW_TYPE, 0, index); 902 if (rc != OPAL_SUCCESS) { 903 pr_err("%s: OPAL error %d when mapping IO " 904 "segment #%d to PE#%d\n", 905 __func__, rc, index, pe->pe_number); 906 break; 907 } 908 909 region.start += phb->ioda.io_segsize; 910 index++; 911 } 912 } else if (res->flags & IORESOURCE_MEM) { 913 /* WARNING: Assumes M32 is mem region 0 in PHB. We need to 914 * harden that algorithm when we start supporting M64 915 */ 916 region.start = res->start - 917 hose->mem_offset[0] - 918 phb->ioda.m32_pci_base; 919 region.end = res->end - 920 hose->mem_offset[0] - 921 phb->ioda.m32_pci_base; 922 index = region.start / phb->ioda.m32_segsize; 923 924 while (index < phb->ioda.total_pe && 925 region.start <= region.end) { 926 phb->ioda.m32_segmap[index] = pe->pe_number; 927 rc = opal_pci_map_pe_mmio_window(phb->opal_id, 928 pe->pe_number, OPAL_M32_WINDOW_TYPE, 0, index); 929 if (rc != OPAL_SUCCESS) { 930 pr_err("%s: OPAL error %d when mapping M32 " 931 "segment#%d to PE#%d", 932 __func__, rc, index, pe->pe_number); 933 break; 934 } 935 936 region.start += phb->ioda.m32_segsize; 937 index++; 938 } 939 } 940 } 941 } 942 943 static void pnv_pci_ioda_setup_seg(void) 944 { 945 struct pci_controller *tmp, *hose; 946 struct pnv_phb *phb; 947 struct pnv_ioda_pe *pe; 948 949 list_for_each_entry_safe(hose, tmp, &hose_list, list_node) { 950 phb = hose->private_data; 951 list_for_each_entry(pe, &phb->ioda.pe_list, list) { 952 pnv_ioda_setup_pe_seg(hose, pe); 953 } 954 } 955 } 956 957 static void pnv_pci_ioda_setup_DMA(void) 958 { 959 struct pci_controller *hose, *tmp; 960 struct pnv_phb *phb; 961 962 list_for_each_entry_safe(hose, tmp, &hose_list, list_node) { 963 pnv_ioda_setup_dma(hose->private_data); 964 965 /* Mark the PHB initialization done */ 966 phb = hose->private_data; 967 phb->initialized = 1; 968 } 969 } 970 971 static void pnv_pci_ioda_fixup(void) 972 { 973 pnv_pci_ioda_setup_PEs(); 974 pnv_pci_ioda_setup_seg(); 975 pnv_pci_ioda_setup_DMA(); 976 } 977 978 /* 979 * Returns the alignment for I/O or memory windows for P2P 980 * bridges. That actually depends on how PEs are segmented. 981 * For now, we return I/O or M32 segment size for PE sensitive 982 * P2P bridges. Otherwise, the default values (4KiB for I/O, 983 * 1MiB for memory) will be returned. 984 * 985 * The current PCI bus might be put into one PE, which was 986 * create against the parent PCI bridge. For that case, we 987 * needn't enlarge the alignment so that we can save some 988 * resources. 989 */ 990 static resource_size_t pnv_pci_window_alignment(struct pci_bus *bus, 991 unsigned long type) 992 { 993 struct pci_dev *bridge; 994 struct pci_controller *hose = pci_bus_to_host(bus); 995 struct pnv_phb *phb = hose->private_data; 996 int num_pci_bridges = 0; 997 998 bridge = bus->self; 999 while (bridge) { 1000 if (pci_pcie_type(bridge) == PCI_EXP_TYPE_PCI_BRIDGE) { 1001 num_pci_bridges++; 1002 if (num_pci_bridges >= 2) 1003 return 1; 1004 } 1005 1006 bridge = bridge->bus->self; 1007 } 1008 1009 /* We need support prefetchable memory window later */ 1010 if (type & IORESOURCE_MEM) 1011 return phb->ioda.m32_segsize; 1012 1013 return phb->ioda.io_segsize; 1014 } 1015 1016 /* Prevent enabling devices for which we couldn't properly 1017 * assign a PE 1018 */ 1019 static int pnv_pci_enable_device_hook(struct pci_dev *dev) 1020 { 1021 struct pci_controller *hose = pci_bus_to_host(dev->bus); 1022 struct pnv_phb *phb = hose->private_data; 1023 struct pci_dn *pdn; 1024 1025 /* The function is probably called while the PEs have 1026 * not be created yet. For example, resource reassignment 1027 * during PCI probe period. We just skip the check if 1028 * PEs isn't ready. 1029 */ 1030 if (!phb->initialized) 1031 return 0; 1032 1033 pdn = pci_get_pdn(dev); 1034 if (!pdn || pdn->pe_number == IODA_INVALID_PE) 1035 return -EINVAL; 1036 1037 return 0; 1038 } 1039 1040 static u32 pnv_ioda_bdfn_to_pe(struct pnv_phb *phb, struct pci_bus *bus, 1041 u32 devfn) 1042 { 1043 return phb->ioda.pe_rmap[(bus->number << 8) | devfn]; 1044 } 1045 1046 static void pnv_pci_ioda_shutdown(struct pnv_phb *phb) 1047 { 1048 opal_pci_reset(phb->opal_id, OPAL_PCI_IODA_TABLE_RESET, 1049 OPAL_ASSERT_RESET); 1050 } 1051 1052 void __init pnv_pci_init_ioda_phb(struct device_node *np, int ioda_type) 1053 { 1054 struct pci_controller *hose; 1055 static int primary = 1; 1056 struct pnv_phb *phb; 1057 unsigned long size, m32map_off, iomap_off, pemap_off; 1058 const u64 *prop64; 1059 const u32 *prop32; 1060 u64 phb_id; 1061 void *aux; 1062 long rc; 1063 1064 pr_info(" Initializing IODA%d OPAL PHB %s\n", ioda_type, np->full_name); 1065 1066 prop64 = of_get_property(np, "ibm,opal-phbid", NULL); 1067 if (!prop64) { 1068 pr_err(" Missing \"ibm,opal-phbid\" property !\n"); 1069 return; 1070 } 1071 phb_id = be64_to_cpup(prop64); 1072 pr_debug(" PHB-ID : 0x%016llx\n", phb_id); 1073 1074 phb = alloc_bootmem(sizeof(struct pnv_phb)); 1075 if (phb) { 1076 memset(phb, 0, sizeof(struct pnv_phb)); 1077 phb->hose = hose = pcibios_alloc_controller(np); 1078 } 1079 if (!phb || !phb->hose) { 1080 pr_err("PCI: Failed to allocate PCI controller for %s\n", 1081 np->full_name); 1082 return; 1083 } 1084 1085 spin_lock_init(&phb->lock); 1086 /* XXX Use device-tree */ 1087 hose->first_busno = 0; 1088 hose->last_busno = 0xff; 1089 hose->private_data = phb; 1090 phb->opal_id = phb_id; 1091 phb->type = ioda_type; 1092 1093 /* Detect specific models for error handling */ 1094 if (of_device_is_compatible(np, "ibm,p7ioc-pciex")) 1095 phb->model = PNV_PHB_MODEL_P7IOC; 1096 else if (of_device_is_compatible(np, "ibm,power8-pciex")) 1097 phb->model = PNV_PHB_MODEL_PHB3; 1098 else 1099 phb->model = PNV_PHB_MODEL_UNKNOWN; 1100 1101 /* Parse 32-bit and IO ranges (if any) */ 1102 pci_process_bridge_OF_ranges(phb->hose, np, primary); 1103 primary = 0; 1104 1105 /* Get registers */ 1106 phb->regs = of_iomap(np, 0); 1107 if (phb->regs == NULL) 1108 pr_err(" Failed to map registers !\n"); 1109 1110 /* Initialize more IODA stuff */ 1111 prop32 = of_get_property(np, "ibm,opal-num-pes", NULL); 1112 if (!prop32) 1113 phb->ioda.total_pe = 1; 1114 else 1115 phb->ioda.total_pe = *prop32; 1116 1117 phb->ioda.m32_size = resource_size(&hose->mem_resources[0]); 1118 /* FW Has already off top 64k of M32 space (MSI space) */ 1119 phb->ioda.m32_size += 0x10000; 1120 1121 phb->ioda.m32_segsize = phb->ioda.m32_size / phb->ioda.total_pe; 1122 phb->ioda.m32_pci_base = hose->mem_resources[0].start - hose->mem_offset[0]; 1123 phb->ioda.io_size = hose->pci_io_size; 1124 phb->ioda.io_segsize = phb->ioda.io_size / phb->ioda.total_pe; 1125 phb->ioda.io_pci_base = 0; /* XXX calculate this ? */ 1126 1127 /* Allocate aux data & arrays 1128 * 1129 * XXX TODO: Don't allocate io segmap on PHB3 1130 */ 1131 size = _ALIGN_UP(phb->ioda.total_pe / 8, sizeof(unsigned long)); 1132 m32map_off = size; 1133 size += phb->ioda.total_pe * sizeof(phb->ioda.m32_segmap[0]); 1134 iomap_off = size; 1135 size += phb->ioda.total_pe * sizeof(phb->ioda.io_segmap[0]); 1136 pemap_off = size; 1137 size += phb->ioda.total_pe * sizeof(struct pnv_ioda_pe); 1138 aux = alloc_bootmem(size); 1139 memset(aux, 0, size); 1140 phb->ioda.pe_alloc = aux; 1141 phb->ioda.m32_segmap = aux + m32map_off; 1142 phb->ioda.io_segmap = aux + iomap_off; 1143 phb->ioda.pe_array = aux + pemap_off; 1144 set_bit(0, phb->ioda.pe_alloc); 1145 1146 INIT_LIST_HEAD(&phb->ioda.pe_dma_list); 1147 INIT_LIST_HEAD(&phb->ioda.pe_list); 1148 1149 /* Calculate how many 32-bit TCE segments we have */ 1150 phb->ioda.tce32_count = phb->ioda.m32_pci_base >> 28; 1151 1152 /* Clear unusable m64 */ 1153 hose->mem_resources[1].flags = 0; 1154 hose->mem_resources[1].start = 0; 1155 hose->mem_resources[1].end = 0; 1156 hose->mem_resources[2].flags = 0; 1157 hose->mem_resources[2].start = 0; 1158 hose->mem_resources[2].end = 0; 1159 1160 #if 0 /* We should really do that ... */ 1161 rc = opal_pci_set_phb_mem_window(opal->phb_id, 1162 window_type, 1163 window_num, 1164 starting_real_address, 1165 starting_pci_address, 1166 segment_size); 1167 #endif 1168 1169 pr_info(" %d PE's M32: 0x%x [segment=0x%x] IO: 0x%x [segment=0x%x]\n", 1170 phb->ioda.total_pe, 1171 phb->ioda.m32_size, phb->ioda.m32_segsize, 1172 phb->ioda.io_size, phb->ioda.io_segsize); 1173 1174 phb->hose->ops = &pnv_pci_ops; 1175 1176 /* Setup RID -> PE mapping function */ 1177 phb->bdfn_to_pe = pnv_ioda_bdfn_to_pe; 1178 1179 /* Setup TCEs */ 1180 phb->dma_dev_setup = pnv_pci_ioda_dma_dev_setup; 1181 1182 /* Setup shutdown function for kexec */ 1183 phb->shutdown = pnv_pci_ioda_shutdown; 1184 1185 /* Setup MSI support */ 1186 pnv_pci_init_ioda_msis(phb); 1187 1188 /* 1189 * We pass the PCI probe flag PCI_REASSIGN_ALL_RSRC here 1190 * to let the PCI core do resource assignment. It's supposed 1191 * that the PCI core will do correct I/O and MMIO alignment 1192 * for the P2P bridge bars so that each PCI bus (excluding 1193 * the child P2P bridges) can form individual PE. 1194 */ 1195 ppc_md.pcibios_fixup = pnv_pci_ioda_fixup; 1196 ppc_md.pcibios_enable_device_hook = pnv_pci_enable_device_hook; 1197 ppc_md.pcibios_window_alignment = pnv_pci_window_alignment; 1198 pci_add_flags(PCI_REASSIGN_ALL_RSRC); 1199 1200 /* Reset IODA tables to a clean state */ 1201 rc = opal_pci_reset(phb_id, OPAL_PCI_IODA_TABLE_RESET, OPAL_ASSERT_RESET); 1202 if (rc) 1203 pr_warning(" OPAL Error %ld performing IODA table reset !\n", rc); 1204 1205 /* 1206 * On IODA1 map everything to PE#0, on IODA2 we assume the IODA reset 1207 * has cleared the RTT which has the same effect 1208 */ 1209 if (ioda_type == PNV_PHB_IODA1) 1210 opal_pci_set_pe(phb_id, 0, 0, 7, 1, 1 , OPAL_MAP_PE); 1211 } 1212 1213 void pnv_pci_init_ioda2_phb(struct device_node *np) 1214 { 1215 pnv_pci_init_ioda_phb(np, PNV_PHB_IODA2); 1216 } 1217 1218 void __init pnv_pci_init_ioda_hub(struct device_node *np) 1219 { 1220 struct device_node *phbn; 1221 const u64 *prop64; 1222 u64 hub_id; 1223 1224 pr_info("Probing IODA IO-Hub %s\n", np->full_name); 1225 1226 prop64 = of_get_property(np, "ibm,opal-hubid", NULL); 1227 if (!prop64) { 1228 pr_err(" Missing \"ibm,opal-hubid\" property !\n"); 1229 return; 1230 } 1231 hub_id = be64_to_cpup(prop64); 1232 pr_devel(" HUB-ID : 0x%016llx\n", hub_id); 1233 1234 /* Count child PHBs */ 1235 for_each_child_of_node(np, phbn) { 1236 /* Look for IODA1 PHBs */ 1237 if (of_device_is_compatible(phbn, "ibm,ioda-phb")) 1238 pnv_pci_init_ioda_phb(phbn, PNV_PHB_IODA1); 1239 } 1240 } 1241