1 /* 2 * Support PCI/PCIe on PowerNV platforms 3 * 4 * Currently supports only P5IOC2 5 * 6 * Copyright 2011 Benjamin Herrenschmidt, IBM Corp. 7 * 8 * This program is free software; you can redistribute it and/or 9 * modify it under the terms of the GNU General Public License 10 * as published by the Free Software Foundation; either version 11 * 2 of the License, or (at your option) any later version. 12 */ 13 14 #include <linux/kernel.h> 15 #include <linux/pci.h> 16 #include <linux/delay.h> 17 #include <linux/string.h> 18 #include <linux/init.h> 19 #include <linux/bootmem.h> 20 #include <linux/irq.h> 21 #include <linux/io.h> 22 #include <linux/msi.h> 23 #include <linux/iommu.h> 24 25 #include <asm/sections.h> 26 #include <asm/io.h> 27 #include <asm/prom.h> 28 #include <asm/pci-bridge.h> 29 #include <asm/machdep.h> 30 #include <asm/msi_bitmap.h> 31 #include <asm/ppc-pci.h> 32 #include <asm/opal.h> 33 #include <asm/iommu.h> 34 #include <asm/tce.h> 35 #include <asm/firmware.h> 36 #include <asm/eeh_event.h> 37 #include <asm/eeh.h> 38 39 #include "powernv.h" 40 #include "pci.h" 41 42 /* Delay in usec */ 43 #define PCI_RESET_DELAY_US 3000000 44 45 #define cfg_dbg(fmt...) do { } while(0) 46 //#define cfg_dbg(fmt...) printk(fmt) 47 48 #ifdef CONFIG_PCI_MSI 49 static int pnv_msi_check_device(struct pci_dev* pdev, int nvec, int type) 50 { 51 struct pci_controller *hose = pci_bus_to_host(pdev->bus); 52 struct pnv_phb *phb = hose->private_data; 53 struct pci_dn *pdn = pci_get_pdn(pdev); 54 55 if (pdn && pdn->force_32bit_msi && !phb->msi32_support) 56 return -ENODEV; 57 58 return (phb && phb->msi_bmp.bitmap) ? 0 : -ENODEV; 59 } 60 61 static int pnv_setup_msi_irqs(struct pci_dev *pdev, int nvec, int type) 62 { 63 struct pci_controller *hose = pci_bus_to_host(pdev->bus); 64 struct pnv_phb *phb = hose->private_data; 65 struct msi_desc *entry; 66 struct msi_msg msg; 67 int hwirq; 68 unsigned int virq; 69 int rc; 70 71 if (WARN_ON(!phb)) 72 return -ENODEV; 73 74 list_for_each_entry(entry, &pdev->msi_list, list) { 75 if (!entry->msi_attrib.is_64 && !phb->msi32_support) { 76 pr_warn("%s: Supports only 64-bit MSIs\n", 77 pci_name(pdev)); 78 return -ENXIO; 79 } 80 hwirq = msi_bitmap_alloc_hwirqs(&phb->msi_bmp, 1); 81 if (hwirq < 0) { 82 pr_warn("%s: Failed to find a free MSI\n", 83 pci_name(pdev)); 84 return -ENOSPC; 85 } 86 virq = irq_create_mapping(NULL, phb->msi_base + hwirq); 87 if (virq == NO_IRQ) { 88 pr_warn("%s: Failed to map MSI to linux irq\n", 89 pci_name(pdev)); 90 msi_bitmap_free_hwirqs(&phb->msi_bmp, hwirq, 1); 91 return -ENOMEM; 92 } 93 rc = phb->msi_setup(phb, pdev, phb->msi_base + hwirq, 94 virq, entry->msi_attrib.is_64, &msg); 95 if (rc) { 96 pr_warn("%s: Failed to setup MSI\n", pci_name(pdev)); 97 irq_dispose_mapping(virq); 98 msi_bitmap_free_hwirqs(&phb->msi_bmp, hwirq, 1); 99 return rc; 100 } 101 irq_set_msi_desc(virq, entry); 102 write_msi_msg(virq, &msg); 103 } 104 return 0; 105 } 106 107 static void pnv_teardown_msi_irqs(struct pci_dev *pdev) 108 { 109 struct pci_controller *hose = pci_bus_to_host(pdev->bus); 110 struct pnv_phb *phb = hose->private_data; 111 struct msi_desc *entry; 112 113 if (WARN_ON(!phb)) 114 return; 115 116 list_for_each_entry(entry, &pdev->msi_list, list) { 117 if (entry->irq == NO_IRQ) 118 continue; 119 irq_set_msi_desc(entry->irq, NULL); 120 msi_bitmap_free_hwirqs(&phb->msi_bmp, 121 virq_to_hw(entry->irq) - phb->msi_base, 1); 122 irq_dispose_mapping(entry->irq); 123 } 124 } 125 #endif /* CONFIG_PCI_MSI */ 126 127 static void pnv_pci_dump_p7ioc_diag_data(struct pci_controller *hose, 128 struct OpalIoPhbErrorCommon *common) 129 { 130 struct OpalIoP7IOCPhbErrorData *data; 131 int i; 132 133 data = (struct OpalIoP7IOCPhbErrorData *)common; 134 pr_info("P7IOC PHB#%d Diag-data (Version: %d)\n\n", 135 hose->global_number, common->version); 136 137 if (data->brdgCtl) 138 pr_info(" brdgCtl: %08x\n", 139 data->brdgCtl); 140 if (data->portStatusReg || data->rootCmplxStatus || 141 data->busAgentStatus) 142 pr_info(" UtlSts: %08x %08x %08x\n", 143 data->portStatusReg, data->rootCmplxStatus, 144 data->busAgentStatus); 145 if (data->deviceStatus || data->slotStatus || 146 data->linkStatus || data->devCmdStatus || 147 data->devSecStatus) 148 pr_info(" RootSts: %08x %08x %08x %08x %08x\n", 149 data->deviceStatus, data->slotStatus, 150 data->linkStatus, data->devCmdStatus, 151 data->devSecStatus); 152 if (data->rootErrorStatus || data->uncorrErrorStatus || 153 data->corrErrorStatus) 154 pr_info(" RootErrSts: %08x %08x %08x\n", 155 data->rootErrorStatus, data->uncorrErrorStatus, 156 data->corrErrorStatus); 157 if (data->tlpHdr1 || data->tlpHdr2 || 158 data->tlpHdr3 || data->tlpHdr4) 159 pr_info(" RootErrLog: %08x %08x %08x %08x\n", 160 data->tlpHdr1, data->tlpHdr2, 161 data->tlpHdr3, data->tlpHdr4); 162 if (data->sourceId || data->errorClass || 163 data->correlator) 164 pr_info(" RootErrLog1: %08x %016llx %016llx\n", 165 data->sourceId, data->errorClass, 166 data->correlator); 167 if (data->p7iocPlssr || data->p7iocCsr) 168 pr_info(" PhbSts: %016llx %016llx\n", 169 data->p7iocPlssr, data->p7iocCsr); 170 if (data->lemFir || data->lemErrorMask || 171 data->lemWOF) 172 pr_info(" Lem: %016llx %016llx %016llx\n", 173 data->lemFir, data->lemErrorMask, 174 data->lemWOF); 175 if (data->phbErrorStatus || data->phbFirstErrorStatus || 176 data->phbErrorLog0 || data->phbErrorLog1) 177 pr_info(" PhbErr: %016llx %016llx %016llx %016llx\n", 178 data->phbErrorStatus, data->phbFirstErrorStatus, 179 data->phbErrorLog0, data->phbErrorLog1); 180 if (data->mmioErrorStatus || data->mmioFirstErrorStatus || 181 data->mmioErrorLog0 || data->mmioErrorLog1) 182 pr_info(" OutErr: %016llx %016llx %016llx %016llx\n", 183 data->mmioErrorStatus, data->mmioFirstErrorStatus, 184 data->mmioErrorLog0, data->mmioErrorLog1); 185 if (data->dma0ErrorStatus || data->dma0FirstErrorStatus || 186 data->dma0ErrorLog0 || data->dma0ErrorLog1) 187 pr_info(" InAErr: %016llx %016llx %016llx %016llx\n", 188 data->dma0ErrorStatus, data->dma0FirstErrorStatus, 189 data->dma0ErrorLog0, data->dma0ErrorLog1); 190 if (data->dma1ErrorStatus || data->dma1FirstErrorStatus || 191 data->dma1ErrorLog0 || data->dma1ErrorLog1) 192 pr_info(" InBErr: %016llx %016llx %016llx %016llx\n", 193 data->dma1ErrorStatus, data->dma1FirstErrorStatus, 194 data->dma1ErrorLog0, data->dma1ErrorLog1); 195 196 for (i = 0; i < OPAL_P7IOC_NUM_PEST_REGS; i++) { 197 if ((data->pestA[i] >> 63) == 0 && 198 (data->pestB[i] >> 63) == 0) 199 continue; 200 201 pr_info(" PE[%3d] A/B: %016llx %016llx\n", 202 i, data->pestA[i], data->pestB[i]); 203 } 204 } 205 206 static void pnv_pci_dump_phb3_diag_data(struct pci_controller *hose, 207 struct OpalIoPhbErrorCommon *common) 208 { 209 struct OpalIoPhb3ErrorData *data; 210 int i; 211 212 data = (struct OpalIoPhb3ErrorData*)common; 213 pr_info("PHB3 PHB#%d Diag-data (Version: %d)\n\n", 214 hose->global_number, common->version); 215 if (data->brdgCtl) 216 pr_info(" brdgCtl: %08x\n", 217 data->brdgCtl); 218 if (data->portStatusReg || data->rootCmplxStatus || 219 data->busAgentStatus) 220 pr_info(" UtlSts: %08x %08x %08x\n", 221 data->portStatusReg, data->rootCmplxStatus, 222 data->busAgentStatus); 223 if (data->deviceStatus || data->slotStatus || 224 data->linkStatus || data->devCmdStatus || 225 data->devSecStatus) 226 pr_info(" RootSts: %08x %08x %08x %08x %08x\n", 227 data->deviceStatus, data->slotStatus, 228 data->linkStatus, data->devCmdStatus, 229 data->devSecStatus); 230 if (data->rootErrorStatus || data->uncorrErrorStatus || 231 data->corrErrorStatus) 232 pr_info(" RootErrSts: %08x %08x %08x\n", 233 data->rootErrorStatus, data->uncorrErrorStatus, 234 data->corrErrorStatus); 235 if (data->tlpHdr1 || data->tlpHdr2 || 236 data->tlpHdr3 || data->tlpHdr4) 237 pr_info(" RootErrLog: %08x %08x %08x %08x\n", 238 data->tlpHdr1, data->tlpHdr2, 239 data->tlpHdr3, data->tlpHdr4); 240 if (data->sourceId || data->errorClass || 241 data->correlator) 242 pr_info(" RootErrLog1: %08x %016llx %016llx\n", 243 data->sourceId, data->errorClass, 244 data->correlator); 245 if (data->nFir || data->nFirMask || 246 data->nFirWOF) 247 pr_info(" nFir: %016llx %016llx %016llx\n", 248 data->nFir, data->nFirMask, 249 data->nFirWOF); 250 if (data->phbPlssr || data->phbCsr) 251 pr_info(" PhbSts: %016llx %016llx\n", 252 data->phbPlssr, data->phbCsr); 253 if (data->lemFir || data->lemErrorMask || 254 data->lemWOF) 255 pr_info(" Lem: %016llx %016llx %016llx\n", 256 data->lemFir, data->lemErrorMask, 257 data->lemWOF); 258 if (data->phbErrorStatus || data->phbFirstErrorStatus || 259 data->phbErrorLog0 || data->phbErrorLog1) 260 pr_info(" PhbErr: %016llx %016llx %016llx %016llx\n", 261 data->phbErrorStatus, data->phbFirstErrorStatus, 262 data->phbErrorLog0, data->phbErrorLog1); 263 if (data->mmioErrorStatus || data->mmioFirstErrorStatus || 264 data->mmioErrorLog0 || data->mmioErrorLog1) 265 pr_info(" OutErr: %016llx %016llx %016llx %016llx\n", 266 data->mmioErrorStatus, data->mmioFirstErrorStatus, 267 data->mmioErrorLog0, data->mmioErrorLog1); 268 if (data->dma0ErrorStatus || data->dma0FirstErrorStatus || 269 data->dma0ErrorLog0 || data->dma0ErrorLog1) 270 pr_info(" InAErr: %016llx %016llx %016llx %016llx\n", 271 data->dma0ErrorStatus, data->dma0FirstErrorStatus, 272 data->dma0ErrorLog0, data->dma0ErrorLog1); 273 if (data->dma1ErrorStatus || data->dma1FirstErrorStatus || 274 data->dma1ErrorLog0 || data->dma1ErrorLog1) 275 pr_info(" InBErr: %016llx %016llx %016llx %016llx\n", 276 data->dma1ErrorStatus, data->dma1FirstErrorStatus, 277 data->dma1ErrorLog0, data->dma1ErrorLog1); 278 279 for (i = 0; i < OPAL_PHB3_NUM_PEST_REGS; i++) { 280 if ((data->pestA[i] >> 63) == 0 && 281 (data->pestB[i] >> 63) == 0) 282 continue; 283 284 pr_info(" PE[%3d] A/B: %016llx %016llx\n", 285 i, data->pestA[i], data->pestB[i]); 286 } 287 } 288 289 void pnv_pci_dump_phb_diag_data(struct pci_controller *hose, 290 unsigned char *log_buff) 291 { 292 struct OpalIoPhbErrorCommon *common; 293 294 if (!hose || !log_buff) 295 return; 296 297 common = (struct OpalIoPhbErrorCommon *)log_buff; 298 switch (common->ioType) { 299 case OPAL_PHB_ERROR_DATA_TYPE_P7IOC: 300 pnv_pci_dump_p7ioc_diag_data(hose, common); 301 break; 302 case OPAL_PHB_ERROR_DATA_TYPE_PHB3: 303 pnv_pci_dump_phb3_diag_data(hose, common); 304 break; 305 default: 306 pr_warn("%s: Unrecognized ioType %d\n", 307 __func__, common->ioType); 308 } 309 } 310 311 static void pnv_pci_handle_eeh_config(struct pnv_phb *phb, u32 pe_no) 312 { 313 unsigned long flags, rc; 314 int has_diag; 315 316 spin_lock_irqsave(&phb->lock, flags); 317 318 rc = opal_pci_get_phb_diag_data2(phb->opal_id, phb->diag.blob, 319 PNV_PCI_DIAG_BUF_SIZE); 320 has_diag = (rc == OPAL_SUCCESS); 321 322 rc = opal_pci_eeh_freeze_clear(phb->opal_id, pe_no, 323 OPAL_EEH_ACTION_CLEAR_FREEZE_ALL); 324 if (rc) { 325 pr_warning("PCI %d: Failed to clear EEH freeze state" 326 " for PE#%d, err %ld\n", 327 phb->hose->global_number, pe_no, rc); 328 329 /* For now, let's only display the diag buffer when we fail to clear 330 * the EEH status. We'll do more sensible things later when we have 331 * proper EEH support. We need to make sure we don't pollute ourselves 332 * with the normal errors generated when probing empty slots 333 */ 334 if (has_diag) 335 pnv_pci_dump_phb_diag_data(phb->hose, phb->diag.blob); 336 else 337 pr_warning("PCI %d: No diag data available\n", 338 phb->hose->global_number); 339 } 340 341 spin_unlock_irqrestore(&phb->lock, flags); 342 } 343 344 static void pnv_pci_config_check_eeh(struct pnv_phb *phb, 345 struct device_node *dn) 346 { 347 s64 rc; 348 u8 fstate; 349 __be16 pcierr; 350 u32 pe_no; 351 352 /* 353 * Get the PE#. During the PCI probe stage, we might not 354 * setup that yet. So all ER errors should be mapped to 355 * reserved PE. 356 */ 357 pe_no = PCI_DN(dn)->pe_number; 358 if (pe_no == IODA_INVALID_PE) { 359 if (phb->type == PNV_PHB_P5IOC2) 360 pe_no = 0; 361 else 362 pe_no = phb->ioda.reserved_pe; 363 } 364 365 /* Read freeze status */ 366 rc = opal_pci_eeh_freeze_status(phb->opal_id, pe_no, &fstate, &pcierr, 367 NULL); 368 if (rc) { 369 pr_warning("%s: Can't read EEH status (PE#%d) for " 370 "%s, err %lld\n", 371 __func__, pe_no, dn->full_name, rc); 372 return; 373 } 374 cfg_dbg(" -> EEH check, bdfn=%04x PE#%d fstate=%x\n", 375 (PCI_DN(dn)->busno << 8) | (PCI_DN(dn)->devfn), 376 pe_no, fstate); 377 if (fstate != 0) 378 pnv_pci_handle_eeh_config(phb, pe_no); 379 } 380 381 int pnv_pci_cfg_read(struct device_node *dn, 382 int where, int size, u32 *val) 383 { 384 struct pci_dn *pdn = PCI_DN(dn); 385 struct pnv_phb *phb = pdn->phb->private_data; 386 u32 bdfn = (pdn->busno << 8) | pdn->devfn; 387 #ifdef CONFIG_EEH 388 struct eeh_pe *phb_pe = NULL; 389 #endif 390 s64 rc; 391 392 switch (size) { 393 case 1: { 394 u8 v8; 395 rc = opal_pci_config_read_byte(phb->opal_id, bdfn, where, &v8); 396 *val = (rc == OPAL_SUCCESS) ? v8 : 0xff; 397 break; 398 } 399 case 2: { 400 __be16 v16; 401 rc = opal_pci_config_read_half_word(phb->opal_id, bdfn, where, 402 &v16); 403 *val = (rc == OPAL_SUCCESS) ? be16_to_cpu(v16) : 0xffff; 404 break; 405 } 406 case 4: { 407 __be32 v32; 408 rc = opal_pci_config_read_word(phb->opal_id, bdfn, where, &v32); 409 *val = (rc == OPAL_SUCCESS) ? be32_to_cpu(v32) : 0xffffffff; 410 break; 411 } 412 default: 413 return PCIBIOS_FUNC_NOT_SUPPORTED; 414 } 415 cfg_dbg("%s: bus: %x devfn: %x +%x/%x -> %08x\n", 416 __func__, pdn->busno, pdn->devfn, where, size, *val); 417 418 /* 419 * Check if the specified PE has been put into frozen 420 * state. On the other hand, we needn't do that while 421 * the PHB has been put into frozen state because of 422 * PHB-fatal errors. 423 */ 424 #ifdef CONFIG_EEH 425 phb_pe = eeh_phb_pe_get(pdn->phb); 426 if (phb_pe && (phb_pe->state & EEH_PE_ISOLATED)) 427 return PCIBIOS_SUCCESSFUL; 428 429 if (phb->eeh_state & PNV_EEH_STATE_ENABLED) { 430 if (*val == EEH_IO_ERROR_VALUE(size) && 431 eeh_dev_check_failure(of_node_to_eeh_dev(dn))) 432 return PCIBIOS_DEVICE_NOT_FOUND; 433 } else { 434 pnv_pci_config_check_eeh(phb, dn); 435 } 436 #else 437 pnv_pci_config_check_eeh(phb, dn); 438 #endif 439 440 return PCIBIOS_SUCCESSFUL; 441 } 442 443 int pnv_pci_cfg_write(struct device_node *dn, 444 int where, int size, u32 val) 445 { 446 struct pci_dn *pdn = PCI_DN(dn); 447 struct pnv_phb *phb = pdn->phb->private_data; 448 u32 bdfn = (pdn->busno << 8) | pdn->devfn; 449 450 cfg_dbg("%s: bus: %x devfn: %x +%x/%x -> %08x\n", 451 pdn->busno, pdn->devfn, where, size, val); 452 switch (size) { 453 case 1: 454 opal_pci_config_write_byte(phb->opal_id, bdfn, where, val); 455 break; 456 case 2: 457 opal_pci_config_write_half_word(phb->opal_id, bdfn, where, val); 458 break; 459 case 4: 460 opal_pci_config_write_word(phb->opal_id, bdfn, where, val); 461 break; 462 default: 463 return PCIBIOS_FUNC_NOT_SUPPORTED; 464 } 465 466 /* Check if the PHB got frozen due to an error (no response) */ 467 #ifdef CONFIG_EEH 468 if (!(phb->eeh_state & PNV_EEH_STATE_ENABLED)) 469 pnv_pci_config_check_eeh(phb, dn); 470 #else 471 pnv_pci_config_check_eeh(phb, dn); 472 #endif 473 474 return PCIBIOS_SUCCESSFUL; 475 } 476 477 static int pnv_pci_read_config(struct pci_bus *bus, 478 unsigned int devfn, 479 int where, int size, u32 *val) 480 { 481 struct device_node *dn, *busdn = pci_bus_to_OF_node(bus); 482 struct pci_dn *pdn; 483 484 for (dn = busdn->child; dn; dn = dn->sibling) { 485 pdn = PCI_DN(dn); 486 if (pdn && pdn->devfn == devfn) 487 return pnv_pci_cfg_read(dn, where, size, val); 488 } 489 490 *val = 0xFFFFFFFF; 491 return PCIBIOS_DEVICE_NOT_FOUND; 492 493 } 494 495 static int pnv_pci_write_config(struct pci_bus *bus, 496 unsigned int devfn, 497 int where, int size, u32 val) 498 { 499 struct device_node *dn, *busdn = pci_bus_to_OF_node(bus); 500 struct pci_dn *pdn; 501 502 for (dn = busdn->child; dn; dn = dn->sibling) { 503 pdn = PCI_DN(dn); 504 if (pdn && pdn->devfn == devfn) 505 return pnv_pci_cfg_write(dn, where, size, val); 506 } 507 508 return PCIBIOS_DEVICE_NOT_FOUND; 509 } 510 511 struct pci_ops pnv_pci_ops = { 512 .read = pnv_pci_read_config, 513 .write = pnv_pci_write_config, 514 }; 515 516 static int pnv_tce_build(struct iommu_table *tbl, long index, long npages, 517 unsigned long uaddr, enum dma_data_direction direction, 518 struct dma_attrs *attrs, bool rm) 519 { 520 u64 proto_tce; 521 __be64 *tcep, *tces; 522 u64 rpn; 523 524 proto_tce = TCE_PCI_READ; // Read allowed 525 526 if (direction != DMA_TO_DEVICE) 527 proto_tce |= TCE_PCI_WRITE; 528 529 tces = tcep = ((__be64 *)tbl->it_base) + index - tbl->it_offset; 530 rpn = __pa(uaddr) >> TCE_SHIFT; 531 532 while (npages--) 533 *(tcep++) = cpu_to_be64(proto_tce | (rpn++ << TCE_RPN_SHIFT)); 534 535 /* Some implementations won't cache invalid TCEs and thus may not 536 * need that flush. We'll probably turn it_type into a bit mask 537 * of flags if that becomes the case 538 */ 539 if (tbl->it_type & TCE_PCI_SWINV_CREATE) 540 pnv_pci_ioda_tce_invalidate(tbl, tces, tcep - 1, rm); 541 542 return 0; 543 } 544 545 static int pnv_tce_build_vm(struct iommu_table *tbl, long index, long npages, 546 unsigned long uaddr, 547 enum dma_data_direction direction, 548 struct dma_attrs *attrs) 549 { 550 return pnv_tce_build(tbl, index, npages, uaddr, direction, attrs, 551 false); 552 } 553 554 static void pnv_tce_free(struct iommu_table *tbl, long index, long npages, 555 bool rm) 556 { 557 __be64 *tcep, *tces; 558 559 tces = tcep = ((__be64 *)tbl->it_base) + index - tbl->it_offset; 560 561 while (npages--) 562 *(tcep++) = cpu_to_be64(0); 563 564 if (tbl->it_type & TCE_PCI_SWINV_FREE) 565 pnv_pci_ioda_tce_invalidate(tbl, tces, tcep - 1, rm); 566 } 567 568 static void pnv_tce_free_vm(struct iommu_table *tbl, long index, long npages) 569 { 570 pnv_tce_free(tbl, index, npages, false); 571 } 572 573 static unsigned long pnv_tce_get(struct iommu_table *tbl, long index) 574 { 575 return ((u64 *)tbl->it_base)[index - tbl->it_offset]; 576 } 577 578 static int pnv_tce_build_rm(struct iommu_table *tbl, long index, long npages, 579 unsigned long uaddr, 580 enum dma_data_direction direction, 581 struct dma_attrs *attrs) 582 { 583 return pnv_tce_build(tbl, index, npages, uaddr, direction, attrs, true); 584 } 585 586 static void pnv_tce_free_rm(struct iommu_table *tbl, long index, long npages) 587 { 588 pnv_tce_free(tbl, index, npages, true); 589 } 590 591 void pnv_pci_setup_iommu_table(struct iommu_table *tbl, 592 void *tce_mem, u64 tce_size, 593 u64 dma_offset) 594 { 595 tbl->it_blocksize = 16; 596 tbl->it_base = (unsigned long)tce_mem; 597 tbl->it_page_shift = IOMMU_PAGE_SHIFT_4K; 598 tbl->it_offset = dma_offset >> tbl->it_page_shift; 599 tbl->it_index = 0; 600 tbl->it_size = tce_size >> 3; 601 tbl->it_busno = 0; 602 tbl->it_type = TCE_PCI; 603 } 604 605 static struct iommu_table *pnv_pci_setup_bml_iommu(struct pci_controller *hose) 606 { 607 struct iommu_table *tbl; 608 const __be64 *basep, *swinvp; 609 const __be32 *sizep; 610 611 basep = of_get_property(hose->dn, "linux,tce-base", NULL); 612 sizep = of_get_property(hose->dn, "linux,tce-size", NULL); 613 if (basep == NULL || sizep == NULL) { 614 pr_err("PCI: %s has missing tce entries !\n", 615 hose->dn->full_name); 616 return NULL; 617 } 618 tbl = kzalloc_node(sizeof(struct iommu_table), GFP_KERNEL, hose->node); 619 if (WARN_ON(!tbl)) 620 return NULL; 621 pnv_pci_setup_iommu_table(tbl, __va(be64_to_cpup(basep)), 622 be32_to_cpup(sizep), 0); 623 iommu_init_table(tbl, hose->node); 624 iommu_register_group(tbl, pci_domain_nr(hose->bus), 0); 625 626 /* Deal with SW invalidated TCEs when needed (BML way) */ 627 swinvp = of_get_property(hose->dn, "linux,tce-sw-invalidate-info", 628 NULL); 629 if (swinvp) { 630 tbl->it_busno = be64_to_cpu(swinvp[1]); 631 tbl->it_index = (unsigned long)ioremap(be64_to_cpup(swinvp), 8); 632 tbl->it_type = TCE_PCI_SWINV_CREATE | TCE_PCI_SWINV_FREE; 633 } 634 return tbl; 635 } 636 637 static void pnv_pci_dma_fallback_setup(struct pci_controller *hose, 638 struct pci_dev *pdev) 639 { 640 struct device_node *np = pci_bus_to_OF_node(hose->bus); 641 struct pci_dn *pdn; 642 643 if (np == NULL) 644 return; 645 pdn = PCI_DN(np); 646 if (!pdn->iommu_table) 647 pdn->iommu_table = pnv_pci_setup_bml_iommu(hose); 648 if (!pdn->iommu_table) 649 return; 650 set_iommu_table_base_and_group(&pdev->dev, pdn->iommu_table); 651 } 652 653 static void pnv_pci_dma_dev_setup(struct pci_dev *pdev) 654 { 655 struct pci_controller *hose = pci_bus_to_host(pdev->bus); 656 struct pnv_phb *phb = hose->private_data; 657 658 /* If we have no phb structure, try to setup a fallback based on 659 * the device-tree (RTAS PCI for example) 660 */ 661 if (phb && phb->dma_dev_setup) 662 phb->dma_dev_setup(phb, pdev); 663 else 664 pnv_pci_dma_fallback_setup(hose, pdev); 665 } 666 667 int pnv_pci_dma_set_mask(struct pci_dev *pdev, u64 dma_mask) 668 { 669 struct pci_controller *hose = pci_bus_to_host(pdev->bus); 670 struct pnv_phb *phb = hose->private_data; 671 672 if (phb && phb->dma_set_mask) 673 return phb->dma_set_mask(phb, pdev, dma_mask); 674 return __dma_set_mask(&pdev->dev, dma_mask); 675 } 676 677 void pnv_pci_shutdown(void) 678 { 679 struct pci_controller *hose; 680 681 list_for_each_entry(hose, &hose_list, list_node) { 682 struct pnv_phb *phb = hose->private_data; 683 684 if (phb && phb->shutdown) 685 phb->shutdown(phb); 686 } 687 } 688 689 /* Fixup wrong class code in p7ioc and p8 root complex */ 690 static void pnv_p7ioc_rc_quirk(struct pci_dev *dev) 691 { 692 dev->class = PCI_CLASS_BRIDGE_PCI << 8; 693 } 694 DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_IBM, 0x3b9, pnv_p7ioc_rc_quirk); 695 696 static int pnv_pci_probe_mode(struct pci_bus *bus) 697 { 698 struct pci_controller *hose = pci_bus_to_host(bus); 699 const __be64 *tstamp; 700 u64 now, target; 701 702 703 /* We hijack this as a way to ensure we have waited long 704 * enough since the reset was lifted on the PCI bus 705 */ 706 if (bus != hose->bus) 707 return PCI_PROBE_NORMAL; 708 tstamp = of_get_property(hose->dn, "reset-clear-timestamp", NULL); 709 if (!tstamp || !*tstamp) 710 return PCI_PROBE_NORMAL; 711 712 now = mftb() / tb_ticks_per_usec; 713 target = (be64_to_cpup(tstamp) / tb_ticks_per_usec) 714 + PCI_RESET_DELAY_US; 715 716 pr_devel("pci %04d: Reset target: 0x%llx now: 0x%llx\n", 717 hose->global_number, target, now); 718 719 if (now < target) 720 msleep((target - now + 999) / 1000); 721 722 return PCI_PROBE_NORMAL; 723 } 724 725 void __init pnv_pci_init(void) 726 { 727 struct device_node *np; 728 729 pci_add_flags(PCI_CAN_SKIP_ISA_ALIGN); 730 731 /* OPAL absent, try POPAL first then RTAS detection of PHBs */ 732 if (!firmware_has_feature(FW_FEATURE_OPAL)) { 733 #ifdef CONFIG_PPC_POWERNV_RTAS 734 init_pci_config_tokens(); 735 find_and_init_phbs(); 736 #endif /* CONFIG_PPC_POWERNV_RTAS */ 737 } 738 /* OPAL is here, do our normal stuff */ 739 else { 740 int found_ioda = 0; 741 742 /* Look for IODA IO-Hubs. We don't support mixing IODA 743 * and p5ioc2 due to the need to change some global 744 * probing flags 745 */ 746 for_each_compatible_node(np, NULL, "ibm,ioda-hub") { 747 pnv_pci_init_ioda_hub(np); 748 found_ioda = 1; 749 } 750 751 /* Look for p5ioc2 IO-Hubs */ 752 if (!found_ioda) 753 for_each_compatible_node(np, NULL, "ibm,p5ioc2") 754 pnv_pci_init_p5ioc2_hub(np); 755 756 /* Look for ioda2 built-in PHB3's */ 757 for_each_compatible_node(np, NULL, "ibm,ioda2-phb") 758 pnv_pci_init_ioda2_phb(np); 759 } 760 761 /* Setup the linkage between OF nodes and PHBs */ 762 pci_devs_phb_init(); 763 764 /* Configure IOMMU DMA hooks */ 765 ppc_md.pci_dma_dev_setup = pnv_pci_dma_dev_setup; 766 ppc_md.tce_build = pnv_tce_build_vm; 767 ppc_md.tce_free = pnv_tce_free_vm; 768 ppc_md.tce_build_rm = pnv_tce_build_rm; 769 ppc_md.tce_free_rm = pnv_tce_free_rm; 770 ppc_md.tce_get = pnv_tce_get; 771 ppc_md.pci_probe_mode = pnv_pci_probe_mode; 772 set_pci_dma_ops(&dma_iommu_ops); 773 774 /* Configure MSIs */ 775 #ifdef CONFIG_PCI_MSI 776 ppc_md.msi_check_device = pnv_msi_check_device; 777 ppc_md.setup_msi_irqs = pnv_setup_msi_irqs; 778 ppc_md.teardown_msi_irqs = pnv_teardown_msi_irqs; 779 #endif 780 } 781 782 static int tce_iommu_bus_notifier(struct notifier_block *nb, 783 unsigned long action, void *data) 784 { 785 struct device *dev = data; 786 787 switch (action) { 788 case BUS_NOTIFY_ADD_DEVICE: 789 return iommu_add_device(dev); 790 case BUS_NOTIFY_DEL_DEVICE: 791 if (dev->iommu_group) 792 iommu_del_device(dev); 793 return 0; 794 default: 795 return 0; 796 } 797 } 798 799 static struct notifier_block tce_iommu_bus_nb = { 800 .notifier_call = tce_iommu_bus_notifier, 801 }; 802 803 static int __init tce_iommu_bus_notifier_init(void) 804 { 805 bus_register_notifier(&pci_bus_type, &tce_iommu_bus_nb); 806 return 0; 807 } 808 809 subsys_initcall_sync(tce_iommu_bus_notifier_init); 810