1 /* 2 * Copyright IBM Corp. 2012 3 * 4 * Author(s): 5 * Jan Glauber <jang@linux.vnet.ibm.com> 6 * 7 * The System z PCI code is a rewrite from a prototype by 8 * the following people (Kudoz!): 9 * Alexander Schmidt 10 * Christoph Raisch 11 * Hannes Hering 12 * Hoang-Nam Nguyen 13 * Jan-Bernd Themann 14 * Stefan Roscher 15 * Thomas Klein 16 */ 17 18 #define COMPONENT "zPCI" 19 #define pr_fmt(fmt) COMPONENT ": " fmt 20 21 #include <linux/kernel.h> 22 #include <linux/slab.h> 23 #include <linux/err.h> 24 #include <linux/export.h> 25 #include <linux/delay.h> 26 #include <linux/irq.h> 27 #include <linux/kernel_stat.h> 28 #include <linux/seq_file.h> 29 #include <linux/pci.h> 30 #include <linux/msi.h> 31 32 #include <asm/isc.h> 33 #include <asm/airq.h> 34 #include <asm/facility.h> 35 #include <asm/pci_insn.h> 36 #include <asm/pci_clp.h> 37 #include <asm/pci_dma.h> 38 39 #define DEBUG /* enable pr_debug */ 40 41 #define SIC_IRQ_MODE_ALL 0 42 #define SIC_IRQ_MODE_SINGLE 1 43 44 #define ZPCI_NR_DMA_SPACES 1 45 #define ZPCI_MSI_VEC_BITS 6 46 #define ZPCI_NR_DEVICES CONFIG_PCI_NR_FUNCTIONS 47 48 /* list of all detected zpci devices */ 49 LIST_HEAD(zpci_list); 50 EXPORT_SYMBOL_GPL(zpci_list); 51 DEFINE_MUTEX(zpci_list_lock); 52 EXPORT_SYMBOL_GPL(zpci_list_lock); 53 54 static struct pci_hp_callback_ops *hotplug_ops; 55 56 static DECLARE_BITMAP(zpci_domain, ZPCI_NR_DEVICES); 57 static DEFINE_SPINLOCK(zpci_domain_lock); 58 59 struct callback { 60 irq_handler_t handler; 61 void *data; 62 }; 63 64 struct zdev_irq_map { 65 unsigned long aibv; /* AI bit vector */ 66 int msi_vecs; /* consecutive MSI-vectors used */ 67 int __unused; 68 struct callback cb[ZPCI_NR_MSI_VECS]; /* callback handler array */ 69 spinlock_t lock; /* protect callbacks against de-reg */ 70 }; 71 72 struct intr_bucket { 73 /* amap of adapters, one bit per dev, corresponds to one irq nr */ 74 unsigned long *alloc; 75 /* AI summary bit, global page for all devices */ 76 unsigned long *aisb; 77 /* pointer to aibv and callback data in zdev */ 78 struct zdev_irq_map *imap[ZPCI_NR_DEVICES]; 79 /* protects the whole bucket struct */ 80 spinlock_t lock; 81 }; 82 83 static struct intr_bucket *bucket; 84 85 /* Adapter local summary indicator */ 86 static u8 *zpci_irq_si; 87 88 static atomic_t irq_retries = ATOMIC_INIT(0); 89 90 /* I/O Map */ 91 static DEFINE_SPINLOCK(zpci_iomap_lock); 92 static DECLARE_BITMAP(zpci_iomap, ZPCI_IOMAP_MAX_ENTRIES); 93 struct zpci_iomap_entry *zpci_iomap_start; 94 EXPORT_SYMBOL_GPL(zpci_iomap_start); 95 96 /* highest irq summary bit */ 97 static int __read_mostly aisb_max; 98 99 static struct kmem_cache *zdev_irq_cache; 100 static struct kmem_cache *zdev_fmb_cache; 101 102 debug_info_t *pci_debug_msg_id; 103 debug_info_t *pci_debug_err_id; 104 105 static inline int irq_to_msi_nr(unsigned int irq) 106 { 107 return irq & ZPCI_MSI_MASK; 108 } 109 110 static inline int irq_to_dev_nr(unsigned int irq) 111 { 112 return irq >> ZPCI_MSI_VEC_BITS; 113 } 114 115 static inline struct zdev_irq_map *get_imap(unsigned int irq) 116 { 117 return bucket->imap[irq_to_dev_nr(irq)]; 118 } 119 120 struct zpci_dev *get_zdev(struct pci_dev *pdev) 121 { 122 return (struct zpci_dev *) pdev->sysdata; 123 } 124 125 struct zpci_dev *get_zdev_by_fid(u32 fid) 126 { 127 struct zpci_dev *tmp, *zdev = NULL; 128 129 mutex_lock(&zpci_list_lock); 130 list_for_each_entry(tmp, &zpci_list, entry) { 131 if (tmp->fid == fid) { 132 zdev = tmp; 133 break; 134 } 135 } 136 mutex_unlock(&zpci_list_lock); 137 return zdev; 138 } 139 140 bool zpci_fid_present(u32 fid) 141 { 142 return (get_zdev_by_fid(fid) != NULL) ? true : false; 143 } 144 145 static struct zpci_dev *get_zdev_by_bus(struct pci_bus *bus) 146 { 147 return (bus && bus->sysdata) ? (struct zpci_dev *) bus->sysdata : NULL; 148 } 149 150 int pci_domain_nr(struct pci_bus *bus) 151 { 152 return ((struct zpci_dev *) bus->sysdata)->domain; 153 } 154 EXPORT_SYMBOL_GPL(pci_domain_nr); 155 156 int pci_proc_domain(struct pci_bus *bus) 157 { 158 return pci_domain_nr(bus); 159 } 160 EXPORT_SYMBOL_GPL(pci_proc_domain); 161 162 /* Modify PCI: Register adapter interruptions */ 163 static int zpci_register_airq(struct zpci_dev *zdev, unsigned int aisb, 164 u64 aibv) 165 { 166 u64 req = ZPCI_CREATE_REQ(zdev->fh, 0, ZPCI_MOD_FC_REG_INT); 167 struct zpci_fib *fib; 168 int rc; 169 170 fib = (void *) get_zeroed_page(GFP_KERNEL); 171 if (!fib) 172 return -ENOMEM; 173 174 fib->isc = PCI_ISC; 175 fib->noi = zdev->irq_map->msi_vecs; 176 fib->sum = 1; /* enable summary notifications */ 177 fib->aibv = aibv; 178 fib->aibvo = 0; /* every function has its own page */ 179 fib->aisb = (u64) bucket->aisb + aisb / 8; 180 fib->aisbo = aisb & ZPCI_MSI_MASK; 181 182 rc = mpcifc_instr(req, fib); 183 pr_debug("%s mpcifc returned noi: %d\n", __func__, fib->noi); 184 185 free_page((unsigned long) fib); 186 return rc; 187 } 188 189 struct mod_pci_args { 190 u64 base; 191 u64 limit; 192 u64 iota; 193 u64 fmb_addr; 194 }; 195 196 static int mod_pci(struct zpci_dev *zdev, int fn, u8 dmaas, struct mod_pci_args *args) 197 { 198 u64 req = ZPCI_CREATE_REQ(zdev->fh, dmaas, fn); 199 struct zpci_fib *fib; 200 int rc; 201 202 /* The FIB must be available even if it's not used */ 203 fib = (void *) get_zeroed_page(GFP_KERNEL); 204 if (!fib) 205 return -ENOMEM; 206 207 fib->pba = args->base; 208 fib->pal = args->limit; 209 fib->iota = args->iota; 210 fib->fmb_addr = args->fmb_addr; 211 212 rc = mpcifc_instr(req, fib); 213 free_page((unsigned long) fib); 214 return rc; 215 } 216 217 /* Modify PCI: Register I/O address translation parameters */ 218 int zpci_register_ioat(struct zpci_dev *zdev, u8 dmaas, 219 u64 base, u64 limit, u64 iota) 220 { 221 struct mod_pci_args args = { base, limit, iota, 0 }; 222 223 WARN_ON_ONCE(iota & 0x3fff); 224 args.iota |= ZPCI_IOTA_RTTO_FLAG; 225 return mod_pci(zdev, ZPCI_MOD_FC_REG_IOAT, dmaas, &args); 226 } 227 228 /* Modify PCI: Unregister I/O address translation parameters */ 229 int zpci_unregister_ioat(struct zpci_dev *zdev, u8 dmaas) 230 { 231 struct mod_pci_args args = { 0, 0, 0, 0 }; 232 233 return mod_pci(zdev, ZPCI_MOD_FC_DEREG_IOAT, dmaas, &args); 234 } 235 236 /* Modify PCI: Unregister adapter interruptions */ 237 static int zpci_unregister_airq(struct zpci_dev *zdev) 238 { 239 struct mod_pci_args args = { 0, 0, 0, 0 }; 240 241 return mod_pci(zdev, ZPCI_MOD_FC_DEREG_INT, 0, &args); 242 } 243 244 /* Modify PCI: Set PCI function measurement parameters */ 245 int zpci_fmb_enable_device(struct zpci_dev *zdev) 246 { 247 struct mod_pci_args args = { 0, 0, 0, 0 }; 248 249 if (zdev->fmb) 250 return -EINVAL; 251 252 zdev->fmb = kmem_cache_alloc(zdev_fmb_cache, GFP_KERNEL); 253 if (!zdev->fmb) 254 return -ENOMEM; 255 memset(zdev->fmb, 0, sizeof(*zdev->fmb)); 256 WARN_ON((u64) zdev->fmb & 0xf); 257 258 args.fmb_addr = virt_to_phys(zdev->fmb); 259 return mod_pci(zdev, ZPCI_MOD_FC_SET_MEASURE, 0, &args); 260 } 261 262 /* Modify PCI: Disable PCI function measurement */ 263 int zpci_fmb_disable_device(struct zpci_dev *zdev) 264 { 265 struct mod_pci_args args = { 0, 0, 0, 0 }; 266 int rc; 267 268 if (!zdev->fmb) 269 return -EINVAL; 270 271 /* Function measurement is disabled if fmb address is zero */ 272 rc = mod_pci(zdev, ZPCI_MOD_FC_SET_MEASURE, 0, &args); 273 274 kmem_cache_free(zdev_fmb_cache, zdev->fmb); 275 zdev->fmb = NULL; 276 return rc; 277 } 278 279 #define ZPCI_PCIAS_CFGSPC 15 280 281 static int zpci_cfg_load(struct zpci_dev *zdev, int offset, u32 *val, u8 len) 282 { 283 u64 req = ZPCI_CREATE_REQ(zdev->fh, ZPCI_PCIAS_CFGSPC, len); 284 u64 data; 285 int rc; 286 287 rc = pcilg_instr(&data, req, offset); 288 data = data << ((8 - len) * 8); 289 data = le64_to_cpu(data); 290 if (!rc) 291 *val = (u32) data; 292 else 293 *val = 0xffffffff; 294 return rc; 295 } 296 297 static int zpci_cfg_store(struct zpci_dev *zdev, int offset, u32 val, u8 len) 298 { 299 u64 req = ZPCI_CREATE_REQ(zdev->fh, ZPCI_PCIAS_CFGSPC, len); 300 u64 data = val; 301 int rc; 302 303 data = cpu_to_le64(data); 304 data = data >> ((8 - len) * 8); 305 rc = pcistg_instr(data, req, offset); 306 return rc; 307 } 308 309 void synchronize_irq(unsigned int irq) 310 { 311 /* 312 * Not needed, the handler is protected by a lock and IRQs that occur 313 * after the handler is deleted are just NOPs. 314 */ 315 } 316 EXPORT_SYMBOL_GPL(synchronize_irq); 317 318 void enable_irq(unsigned int irq) 319 { 320 struct msi_desc *msi = irq_get_msi_desc(irq); 321 322 zpci_msi_set_mask_bits(msi, 1, 0); 323 } 324 EXPORT_SYMBOL_GPL(enable_irq); 325 326 void disable_irq(unsigned int irq) 327 { 328 struct msi_desc *msi = irq_get_msi_desc(irq); 329 330 zpci_msi_set_mask_bits(msi, 1, 1); 331 } 332 EXPORT_SYMBOL_GPL(disable_irq); 333 334 void disable_irq_nosync(unsigned int irq) 335 { 336 disable_irq(irq); 337 } 338 EXPORT_SYMBOL_GPL(disable_irq_nosync); 339 340 unsigned long probe_irq_on(void) 341 { 342 return 0; 343 } 344 EXPORT_SYMBOL_GPL(probe_irq_on); 345 346 int probe_irq_off(unsigned long val) 347 { 348 return 0; 349 } 350 EXPORT_SYMBOL_GPL(probe_irq_off); 351 352 unsigned int probe_irq_mask(unsigned long val) 353 { 354 return val; 355 } 356 EXPORT_SYMBOL_GPL(probe_irq_mask); 357 358 void pcibios_fixup_bus(struct pci_bus *bus) 359 { 360 } 361 362 resource_size_t pcibios_align_resource(void *data, const struct resource *res, 363 resource_size_t size, 364 resource_size_t align) 365 { 366 return 0; 367 } 368 369 /* combine single writes by using store-block insn */ 370 void __iowrite64_copy(void __iomem *to, const void *from, size_t count) 371 { 372 zpci_memcpy_toio(to, from, count); 373 } 374 375 /* Create a virtual mapping cookie for a PCI BAR */ 376 void __iomem *pci_iomap(struct pci_dev *pdev, int bar, unsigned long max) 377 { 378 struct zpci_dev *zdev = get_zdev(pdev); 379 u64 addr; 380 int idx; 381 382 if ((bar & 7) != bar) 383 return NULL; 384 385 idx = zdev->bars[bar].map_idx; 386 spin_lock(&zpci_iomap_lock); 387 zpci_iomap_start[idx].fh = zdev->fh; 388 zpci_iomap_start[idx].bar = bar; 389 spin_unlock(&zpci_iomap_lock); 390 391 addr = ZPCI_IOMAP_ADDR_BASE | ((u64) idx << 48); 392 return (void __iomem *) addr; 393 } 394 EXPORT_SYMBOL_GPL(pci_iomap); 395 396 void pci_iounmap(struct pci_dev *pdev, void __iomem *addr) 397 { 398 unsigned int idx; 399 400 idx = (((__force u64) addr) & ~ZPCI_IOMAP_ADDR_BASE) >> 48; 401 spin_lock(&zpci_iomap_lock); 402 zpci_iomap_start[idx].fh = 0; 403 zpci_iomap_start[idx].bar = 0; 404 spin_unlock(&zpci_iomap_lock); 405 } 406 EXPORT_SYMBOL_GPL(pci_iounmap); 407 408 static int pci_read(struct pci_bus *bus, unsigned int devfn, int where, 409 int size, u32 *val) 410 { 411 struct zpci_dev *zdev = get_zdev_by_bus(bus); 412 413 if (!zdev || devfn != ZPCI_DEVFN) 414 return 0; 415 return zpci_cfg_load(zdev, where, val, size); 416 } 417 418 static int pci_write(struct pci_bus *bus, unsigned int devfn, int where, 419 int size, u32 val) 420 { 421 struct zpci_dev *zdev = get_zdev_by_bus(bus); 422 423 if (!zdev || devfn != ZPCI_DEVFN) 424 return 0; 425 return zpci_cfg_store(zdev, where, val, size); 426 } 427 428 static struct pci_ops pci_root_ops = { 429 .read = pci_read, 430 .write = pci_write, 431 }; 432 433 /* store the last handled bit to implement fair scheduling of devices */ 434 static DEFINE_PER_CPU(unsigned long, next_sbit); 435 436 static void zpci_irq_handler(void *dont, void *need) 437 { 438 unsigned long sbit, mbit, last = 0, start = __get_cpu_var(next_sbit); 439 int rescan = 0, max = aisb_max; 440 struct zdev_irq_map *imap; 441 442 inc_irq_stat(IRQIO_PCI); 443 sbit = start; 444 445 scan: 446 /* find summary_bit */ 447 for_each_set_bit_left_cont(sbit, bucket->aisb, max) { 448 clear_bit(63 - (sbit & 63), bucket->aisb + (sbit >> 6)); 449 last = sbit; 450 451 /* find vector bit */ 452 imap = bucket->imap[sbit]; 453 for_each_set_bit_left(mbit, &imap->aibv, imap->msi_vecs) { 454 inc_irq_stat(IRQIO_MSI); 455 clear_bit(63 - mbit, &imap->aibv); 456 457 spin_lock(&imap->lock); 458 if (imap->cb[mbit].handler) 459 imap->cb[mbit].handler(mbit, 460 imap->cb[mbit].data); 461 spin_unlock(&imap->lock); 462 } 463 } 464 465 if (rescan) 466 goto out; 467 468 /* scan the skipped bits */ 469 if (start > 0) { 470 sbit = 0; 471 max = start; 472 start = 0; 473 goto scan; 474 } 475 476 /* enable interrupts again */ 477 sic_instr(SIC_IRQ_MODE_SINGLE, NULL, PCI_ISC); 478 479 /* check again to not lose initiative */ 480 rmb(); 481 max = aisb_max; 482 sbit = find_first_bit_left(bucket->aisb, max); 483 if (sbit != max) { 484 atomic_inc(&irq_retries); 485 rescan++; 486 goto scan; 487 } 488 out: 489 /* store next device bit to scan */ 490 __get_cpu_var(next_sbit) = (++last >= aisb_max) ? 0 : last; 491 } 492 493 /* msi_vecs - number of requested interrupts, 0 place function to error state */ 494 static int zpci_setup_msi(struct pci_dev *pdev, int msi_vecs) 495 { 496 struct zpci_dev *zdev = get_zdev(pdev); 497 unsigned int aisb, msi_nr; 498 struct msi_desc *msi; 499 int rc; 500 501 /* store the number of used MSI vectors */ 502 zdev->irq_map->msi_vecs = min(msi_vecs, ZPCI_NR_MSI_VECS); 503 504 spin_lock(&bucket->lock); 505 aisb = find_first_zero_bit(bucket->alloc, PAGE_SIZE); 506 /* alloc map exhausted? */ 507 if (aisb == PAGE_SIZE) { 508 spin_unlock(&bucket->lock); 509 return -EIO; 510 } 511 set_bit(aisb, bucket->alloc); 512 spin_unlock(&bucket->lock); 513 514 zdev->aisb = aisb; 515 if (aisb + 1 > aisb_max) 516 aisb_max = aisb + 1; 517 518 /* wire up IRQ shortcut pointer */ 519 bucket->imap[zdev->aisb] = zdev->irq_map; 520 pr_debug("%s: imap[%u] linked to %p\n", __func__, zdev->aisb, zdev->irq_map); 521 522 /* TODO: irq number 0 wont be found if we return less than requested MSIs. 523 * ignore it for now and fix in common code. 524 */ 525 msi_nr = aisb << ZPCI_MSI_VEC_BITS; 526 527 list_for_each_entry(msi, &pdev->msi_list, list) { 528 rc = zpci_setup_msi_irq(zdev, msi, msi_nr, 529 aisb << ZPCI_MSI_VEC_BITS); 530 if (rc) 531 return rc; 532 msi_nr++; 533 } 534 535 rc = zpci_register_airq(zdev, aisb, (u64) &zdev->irq_map->aibv); 536 if (rc) { 537 clear_bit(aisb, bucket->alloc); 538 dev_err(&pdev->dev, "register MSI failed with: %d\n", rc); 539 return rc; 540 } 541 return (zdev->irq_map->msi_vecs == msi_vecs) ? 542 0 : zdev->irq_map->msi_vecs; 543 } 544 545 static void zpci_teardown_msi(struct pci_dev *pdev) 546 { 547 struct zpci_dev *zdev = get_zdev(pdev); 548 struct msi_desc *msi; 549 int aisb, rc; 550 551 rc = zpci_unregister_airq(zdev); 552 if (rc) { 553 dev_err(&pdev->dev, "deregister MSI failed with: %d\n", rc); 554 return; 555 } 556 557 msi = list_first_entry(&pdev->msi_list, struct msi_desc, list); 558 aisb = irq_to_dev_nr(msi->irq); 559 560 list_for_each_entry(msi, &pdev->msi_list, list) 561 zpci_teardown_msi_irq(zdev, msi); 562 563 clear_bit(aisb, bucket->alloc); 564 if (aisb + 1 == aisb_max) 565 aisb_max--; 566 } 567 568 int arch_setup_msi_irqs(struct pci_dev *pdev, int nvec, int type) 569 { 570 pr_debug("%s: requesting %d MSI-X interrupts...", __func__, nvec); 571 if (type != PCI_CAP_ID_MSIX && type != PCI_CAP_ID_MSI) 572 return -EINVAL; 573 return zpci_setup_msi(pdev, nvec); 574 } 575 576 void arch_teardown_msi_irqs(struct pci_dev *pdev) 577 { 578 pr_info("%s: on pdev: %p\n", __func__, pdev); 579 zpci_teardown_msi(pdev); 580 } 581 582 static void zpci_map_resources(struct zpci_dev *zdev) 583 { 584 struct pci_dev *pdev = zdev->pdev; 585 resource_size_t len; 586 int i; 587 588 for (i = 0; i < PCI_BAR_COUNT; i++) { 589 len = pci_resource_len(pdev, i); 590 if (!len) 591 continue; 592 pdev->resource[i].start = (resource_size_t) pci_iomap(pdev, i, 0); 593 pdev->resource[i].end = pdev->resource[i].start + len - 1; 594 pr_debug("BAR%i: -> start: %Lx end: %Lx\n", 595 i, pdev->resource[i].start, pdev->resource[i].end); 596 } 597 }; 598 599 static void zpci_unmap_resources(struct pci_dev *pdev) 600 { 601 resource_size_t len; 602 int i; 603 604 for (i = 0; i < PCI_BAR_COUNT; i++) { 605 len = pci_resource_len(pdev, i); 606 if (!len) 607 continue; 608 pci_iounmap(pdev, (void *) pdev->resource[i].start); 609 } 610 }; 611 612 struct zpci_dev *zpci_alloc_device(void) 613 { 614 struct zpci_dev *zdev; 615 616 /* Alloc memory for our private pci device data */ 617 zdev = kzalloc(sizeof(*zdev), GFP_KERNEL); 618 if (!zdev) 619 return ERR_PTR(-ENOMEM); 620 621 /* Alloc aibv & callback space */ 622 zdev->irq_map = kmem_cache_zalloc(zdev_irq_cache, GFP_KERNEL); 623 if (!zdev->irq_map) 624 goto error; 625 WARN_ON((u64) zdev->irq_map & 0xff); 626 return zdev; 627 628 error: 629 kfree(zdev); 630 return ERR_PTR(-ENOMEM); 631 } 632 633 void zpci_free_device(struct zpci_dev *zdev) 634 { 635 kmem_cache_free(zdev_irq_cache, zdev->irq_map); 636 kfree(zdev); 637 } 638 639 /* Called on removal of pci_dev, leaves zpci and bus device */ 640 static void zpci_remove_device(struct pci_dev *pdev) 641 { 642 struct zpci_dev *zdev = get_zdev(pdev); 643 644 dev_info(&pdev->dev, "Removing device %u\n", zdev->domain); 645 zdev->state = ZPCI_FN_STATE_CONFIGURED; 646 zpci_dma_exit_device(zdev); 647 zpci_fmb_disable_device(zdev); 648 zpci_sysfs_remove_device(&pdev->dev); 649 zpci_unmap_resources(pdev); 650 list_del(&zdev->entry); /* can be called from init */ 651 zdev->pdev = NULL; 652 } 653 654 static void zpci_scan_devices(void) 655 { 656 struct zpci_dev *zdev; 657 658 mutex_lock(&zpci_list_lock); 659 list_for_each_entry(zdev, &zpci_list, entry) 660 if (zdev->state == ZPCI_FN_STATE_CONFIGURED) 661 zpci_scan_device(zdev); 662 mutex_unlock(&zpci_list_lock); 663 } 664 665 /* 666 * Too late for any s390 specific setup, since interrupts must be set up 667 * already which requires DMA setup too and the pci scan will access the 668 * config space, which only works if the function handle is enabled. 669 */ 670 int pcibios_enable_device(struct pci_dev *pdev, int mask) 671 { 672 struct resource *res; 673 u16 cmd; 674 int i; 675 676 pci_read_config_word(pdev, PCI_COMMAND, &cmd); 677 678 for (i = 0; i < PCI_BAR_COUNT; i++) { 679 res = &pdev->resource[i]; 680 681 if (res->flags & IORESOURCE_IO) 682 return -EINVAL; 683 684 if (res->flags & IORESOURCE_MEM) 685 cmd |= PCI_COMMAND_MEMORY; 686 } 687 pci_write_config_word(pdev, PCI_COMMAND, cmd); 688 return 0; 689 } 690 691 void pcibios_disable_device(struct pci_dev *pdev) 692 { 693 zpci_remove_device(pdev); 694 pdev->sysdata = NULL; 695 } 696 697 int pcibios_add_platform_entries(struct pci_dev *pdev) 698 { 699 return zpci_sysfs_add_device(&pdev->dev); 700 } 701 702 int zpci_request_irq(unsigned int irq, irq_handler_t handler, void *data) 703 { 704 int msi_nr = irq_to_msi_nr(irq); 705 struct zdev_irq_map *imap; 706 struct msi_desc *msi; 707 708 msi = irq_get_msi_desc(irq); 709 if (!msi) 710 return -EIO; 711 712 imap = get_imap(irq); 713 spin_lock_init(&imap->lock); 714 715 pr_debug("%s: register handler for IRQ:MSI %d:%d\n", __func__, irq >> 6, msi_nr); 716 imap->cb[msi_nr].handler = handler; 717 imap->cb[msi_nr].data = data; 718 719 /* 720 * The generic MSI code returns with the interrupt disabled on the 721 * card, using the MSI mask bits. Firmware doesn't appear to unmask 722 * at that level, so we do it here by hand. 723 */ 724 zpci_msi_set_mask_bits(msi, 1, 0); 725 return 0; 726 } 727 728 void zpci_free_irq(unsigned int irq) 729 { 730 struct zdev_irq_map *imap = get_imap(irq); 731 int msi_nr = irq_to_msi_nr(irq); 732 unsigned long flags; 733 734 pr_debug("%s: for irq: %d\n", __func__, irq); 735 736 spin_lock_irqsave(&imap->lock, flags); 737 imap->cb[msi_nr].handler = NULL; 738 imap->cb[msi_nr].data = NULL; 739 spin_unlock_irqrestore(&imap->lock, flags); 740 } 741 742 int request_irq(unsigned int irq, irq_handler_t handler, 743 unsigned long irqflags, const char *devname, void *dev_id) 744 { 745 pr_debug("%s: irq: %d handler: %p flags: %lx dev: %s\n", 746 __func__, irq, handler, irqflags, devname); 747 748 return zpci_request_irq(irq, handler, dev_id); 749 } 750 EXPORT_SYMBOL_GPL(request_irq); 751 752 void free_irq(unsigned int irq, void *dev_id) 753 { 754 zpci_free_irq(irq); 755 } 756 EXPORT_SYMBOL_GPL(free_irq); 757 758 static int __init zpci_irq_init(void) 759 { 760 int cpu, rc; 761 762 bucket = kzalloc(sizeof(*bucket), GFP_KERNEL); 763 if (!bucket) 764 return -ENOMEM; 765 766 bucket->aisb = (unsigned long *) get_zeroed_page(GFP_KERNEL); 767 if (!bucket->aisb) { 768 rc = -ENOMEM; 769 goto out_aisb; 770 } 771 772 bucket->alloc = (unsigned long *) get_zeroed_page(GFP_KERNEL); 773 if (!bucket->alloc) { 774 rc = -ENOMEM; 775 goto out_alloc; 776 } 777 778 isc_register(PCI_ISC); 779 zpci_irq_si = s390_register_adapter_interrupt(&zpci_irq_handler, NULL, PCI_ISC); 780 if (IS_ERR(zpci_irq_si)) { 781 rc = PTR_ERR(zpci_irq_si); 782 zpci_irq_si = NULL; 783 goto out_ai; 784 } 785 786 for_each_online_cpu(cpu) 787 per_cpu(next_sbit, cpu) = 0; 788 789 spin_lock_init(&bucket->lock); 790 /* set summary to 1 to be called every time for the ISC */ 791 *zpci_irq_si = 1; 792 sic_instr(SIC_IRQ_MODE_SINGLE, NULL, PCI_ISC); 793 return 0; 794 795 out_ai: 796 isc_unregister(PCI_ISC); 797 free_page((unsigned long) bucket->alloc); 798 out_alloc: 799 free_page((unsigned long) bucket->aisb); 800 out_aisb: 801 kfree(bucket); 802 return rc; 803 } 804 805 static void zpci_irq_exit(void) 806 { 807 free_page((unsigned long) bucket->alloc); 808 free_page((unsigned long) bucket->aisb); 809 s390_unregister_adapter_interrupt(zpci_irq_si, PCI_ISC); 810 isc_unregister(PCI_ISC); 811 kfree(bucket); 812 } 813 814 void zpci_debug_info(struct zpci_dev *zdev, struct seq_file *m) 815 { 816 if (!zdev) 817 return; 818 819 seq_printf(m, "global irq retries: %u\n", atomic_read(&irq_retries)); 820 seq_printf(m, "aibv[0]:%016lx aibv[1]:%016lx aisb:%016lx\n", 821 get_imap(0)->aibv, get_imap(1)->aibv, *bucket->aisb); 822 } 823 824 static struct resource *zpci_alloc_bus_resource(unsigned long start, unsigned long size, 825 unsigned long flags, int domain) 826 { 827 struct resource *r; 828 char *name; 829 int rc; 830 831 r = kzalloc(sizeof(*r), GFP_KERNEL); 832 if (!r) 833 return ERR_PTR(-ENOMEM); 834 r->start = start; 835 r->end = r->start + size - 1; 836 r->flags = flags; 837 r->parent = &iomem_resource; 838 name = kmalloc(18, GFP_KERNEL); 839 if (!name) { 840 kfree(r); 841 return ERR_PTR(-ENOMEM); 842 } 843 sprintf(name, "PCI Bus: %04x:%02x", domain, ZPCI_BUS_NR); 844 r->name = name; 845 846 rc = request_resource(&iomem_resource, r); 847 if (rc) 848 pr_debug("request resource %pR failed\n", r); 849 return r; 850 } 851 852 static int zpci_alloc_iomap(struct zpci_dev *zdev) 853 { 854 int entry; 855 856 spin_lock(&zpci_iomap_lock); 857 entry = find_first_zero_bit(zpci_iomap, ZPCI_IOMAP_MAX_ENTRIES); 858 if (entry == ZPCI_IOMAP_MAX_ENTRIES) { 859 spin_unlock(&zpci_iomap_lock); 860 return -ENOSPC; 861 } 862 set_bit(entry, zpci_iomap); 863 spin_unlock(&zpci_iomap_lock); 864 return entry; 865 } 866 867 static void zpci_free_iomap(struct zpci_dev *zdev, int entry) 868 { 869 spin_lock(&zpci_iomap_lock); 870 memset(&zpci_iomap_start[entry], 0, sizeof(struct zpci_iomap_entry)); 871 clear_bit(entry, zpci_iomap); 872 spin_unlock(&zpci_iomap_lock); 873 } 874 875 static int zpci_create_device_bus(struct zpci_dev *zdev) 876 { 877 struct resource *res; 878 LIST_HEAD(resources); 879 int i; 880 881 /* allocate mapping entry for each used bar */ 882 for (i = 0; i < PCI_BAR_COUNT; i++) { 883 unsigned long addr, size, flags; 884 int entry; 885 886 if (!zdev->bars[i].size) 887 continue; 888 entry = zpci_alloc_iomap(zdev); 889 if (entry < 0) 890 return entry; 891 zdev->bars[i].map_idx = entry; 892 893 /* only MMIO is supported */ 894 flags = IORESOURCE_MEM; 895 if (zdev->bars[i].val & 8) 896 flags |= IORESOURCE_PREFETCH; 897 if (zdev->bars[i].val & 4) 898 flags |= IORESOURCE_MEM_64; 899 900 addr = ZPCI_IOMAP_ADDR_BASE + ((u64) entry << 48); 901 902 size = 1UL << zdev->bars[i].size; 903 904 res = zpci_alloc_bus_resource(addr, size, flags, zdev->domain); 905 if (IS_ERR(res)) { 906 zpci_free_iomap(zdev, entry); 907 return PTR_ERR(res); 908 } 909 pci_add_resource(&resources, res); 910 } 911 912 zdev->bus = pci_create_root_bus(NULL, ZPCI_BUS_NR, &pci_root_ops, 913 zdev, &resources); 914 if (!zdev->bus) 915 return -EIO; 916 917 zdev->bus->max_bus_speed = zdev->max_bus_speed; 918 return 0; 919 } 920 921 static int zpci_alloc_domain(struct zpci_dev *zdev) 922 { 923 spin_lock(&zpci_domain_lock); 924 zdev->domain = find_first_zero_bit(zpci_domain, ZPCI_NR_DEVICES); 925 if (zdev->domain == ZPCI_NR_DEVICES) { 926 spin_unlock(&zpci_domain_lock); 927 return -ENOSPC; 928 } 929 set_bit(zdev->domain, zpci_domain); 930 spin_unlock(&zpci_domain_lock); 931 return 0; 932 } 933 934 static void zpci_free_domain(struct zpci_dev *zdev) 935 { 936 spin_lock(&zpci_domain_lock); 937 clear_bit(zdev->domain, zpci_domain); 938 spin_unlock(&zpci_domain_lock); 939 } 940 941 int zpci_enable_device(struct zpci_dev *zdev) 942 { 943 int rc; 944 945 rc = clp_enable_fh(zdev, ZPCI_NR_DMA_SPACES); 946 if (rc) 947 goto out; 948 pr_info("Enabled fh: 0x%x fid: 0x%x\n", zdev->fh, zdev->fid); 949 950 rc = zpci_dma_init_device(zdev); 951 if (rc) 952 goto out_dma; 953 return 0; 954 955 out_dma: 956 clp_disable_fh(zdev); 957 out: 958 return rc; 959 } 960 EXPORT_SYMBOL_GPL(zpci_enable_device); 961 962 int zpci_create_device(struct zpci_dev *zdev) 963 { 964 int rc; 965 966 rc = zpci_alloc_domain(zdev); 967 if (rc) 968 goto out; 969 970 rc = zpci_create_device_bus(zdev); 971 if (rc) 972 goto out_bus; 973 974 mutex_lock(&zpci_list_lock); 975 list_add_tail(&zdev->entry, &zpci_list); 976 if (hotplug_ops) 977 hotplug_ops->create_slot(zdev); 978 mutex_unlock(&zpci_list_lock); 979 980 if (zdev->state == ZPCI_FN_STATE_STANDBY) 981 return 0; 982 983 rc = zpci_enable_device(zdev); 984 if (rc) 985 goto out_start; 986 return 0; 987 988 out_start: 989 mutex_lock(&zpci_list_lock); 990 list_del(&zdev->entry); 991 if (hotplug_ops) 992 hotplug_ops->remove_slot(zdev); 993 mutex_unlock(&zpci_list_lock); 994 out_bus: 995 zpci_free_domain(zdev); 996 out: 997 return rc; 998 } 999 1000 void zpci_stop_device(struct zpci_dev *zdev) 1001 { 1002 zpci_dma_exit_device(zdev); 1003 /* 1004 * Note: SCLP disables fh via set-pci-fn so don't 1005 * do that here. 1006 */ 1007 } 1008 EXPORT_SYMBOL_GPL(zpci_stop_device); 1009 1010 int zpci_scan_device(struct zpci_dev *zdev) 1011 { 1012 zdev->pdev = pci_scan_single_device(zdev->bus, ZPCI_DEVFN); 1013 if (!zdev->pdev) { 1014 pr_err("pci_scan_single_device failed for fid: 0x%x\n", 1015 zdev->fid); 1016 goto out; 1017 } 1018 1019 zpci_debug_init_device(zdev); 1020 zpci_fmb_enable_device(zdev); 1021 zpci_map_resources(zdev); 1022 pci_bus_add_devices(zdev->bus); 1023 1024 /* now that pdev was added to the bus mark it as used */ 1025 zdev->state = ZPCI_FN_STATE_ONLINE; 1026 return 0; 1027 1028 out: 1029 zpci_dma_exit_device(zdev); 1030 clp_disable_fh(zdev); 1031 return -EIO; 1032 } 1033 EXPORT_SYMBOL_GPL(zpci_scan_device); 1034 1035 static inline int barsize(u8 size) 1036 { 1037 return (size) ? (1 << size) >> 10 : 0; 1038 } 1039 1040 static int zpci_mem_init(void) 1041 { 1042 zdev_irq_cache = kmem_cache_create("PCI_IRQ_cache", sizeof(struct zdev_irq_map), 1043 L1_CACHE_BYTES, SLAB_HWCACHE_ALIGN, NULL); 1044 if (!zdev_irq_cache) 1045 goto error_zdev; 1046 1047 zdev_fmb_cache = kmem_cache_create("PCI_FMB_cache", sizeof(struct zpci_fmb), 1048 16, 0, NULL); 1049 if (!zdev_fmb_cache) 1050 goto error_fmb; 1051 1052 /* TODO: use realloc */ 1053 zpci_iomap_start = kzalloc(ZPCI_IOMAP_MAX_ENTRIES * sizeof(*zpci_iomap_start), 1054 GFP_KERNEL); 1055 if (!zpci_iomap_start) 1056 goto error_iomap; 1057 return 0; 1058 1059 error_iomap: 1060 kmem_cache_destroy(zdev_fmb_cache); 1061 error_fmb: 1062 kmem_cache_destroy(zdev_irq_cache); 1063 error_zdev: 1064 return -ENOMEM; 1065 } 1066 1067 static void zpci_mem_exit(void) 1068 { 1069 kfree(zpci_iomap_start); 1070 kmem_cache_destroy(zdev_irq_cache); 1071 kmem_cache_destroy(zdev_fmb_cache); 1072 } 1073 1074 void zpci_register_hp_ops(struct pci_hp_callback_ops *ops) 1075 { 1076 mutex_lock(&zpci_list_lock); 1077 hotplug_ops = ops; 1078 mutex_unlock(&zpci_list_lock); 1079 } 1080 EXPORT_SYMBOL_GPL(zpci_register_hp_ops); 1081 1082 void zpci_deregister_hp_ops(void) 1083 { 1084 mutex_lock(&zpci_list_lock); 1085 hotplug_ops = NULL; 1086 mutex_unlock(&zpci_list_lock); 1087 } 1088 EXPORT_SYMBOL_GPL(zpci_deregister_hp_ops); 1089 1090 unsigned int s390_pci_probe = 1; 1091 EXPORT_SYMBOL_GPL(s390_pci_probe); 1092 1093 char * __init pcibios_setup(char *str) 1094 { 1095 if (!strcmp(str, "off")) { 1096 s390_pci_probe = 0; 1097 return NULL; 1098 } 1099 return str; 1100 } 1101 1102 static int __init pci_base_init(void) 1103 { 1104 int rc; 1105 1106 if (!s390_pci_probe) 1107 return 0; 1108 1109 if (!test_facility(2) || !test_facility(69) 1110 || !test_facility(71) || !test_facility(72)) 1111 return 0; 1112 1113 pr_info("Probing PCI hardware: PCI:%d SID:%d AEN:%d\n", 1114 test_facility(69), test_facility(70), 1115 test_facility(71)); 1116 1117 rc = zpci_debug_init(); 1118 if (rc) 1119 return rc; 1120 1121 rc = zpci_mem_init(); 1122 if (rc) 1123 goto out_mem; 1124 1125 rc = zpci_msihash_init(); 1126 if (rc) 1127 goto out_hash; 1128 1129 rc = zpci_irq_init(); 1130 if (rc) 1131 goto out_irq; 1132 1133 rc = zpci_dma_init(); 1134 if (rc) 1135 goto out_dma; 1136 1137 rc = clp_find_pci_devices(); 1138 if (rc) 1139 goto out_find; 1140 1141 zpci_scan_devices(); 1142 return 0; 1143 1144 out_find: 1145 zpci_dma_exit(); 1146 out_dma: 1147 zpci_irq_exit(); 1148 out_irq: 1149 zpci_msihash_exit(); 1150 out_hash: 1151 zpci_mem_exit(); 1152 out_mem: 1153 zpci_debug_exit(); 1154 return rc; 1155 } 1156 subsys_initcall(pci_base_init); 1157