1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright IBM Corp. 2012 4 * 5 * Author(s): 6 * Jan Glauber <jang@linux.vnet.ibm.com> 7 * 8 * The System z PCI code is a rewrite from a prototype by 9 * the following people (Kudoz!): 10 * Alexander Schmidt 11 * Christoph Raisch 12 * Hannes Hering 13 * Hoang-Nam Nguyen 14 * Jan-Bernd Themann 15 * Stefan Roscher 16 * Thomas Klein 17 */ 18 19 #define KMSG_COMPONENT "zpci" 20 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt 21 22 #include <linux/kernel.h> 23 #include <linux/slab.h> 24 #include <linux/err.h> 25 #include <linux/export.h> 26 #include <linux/delay.h> 27 #include <linux/seq_file.h> 28 #include <linux/jump_label.h> 29 #include <linux/pci.h> 30 31 #include <asm/isc.h> 32 #include <asm/airq.h> 33 #include <asm/facility.h> 34 #include <asm/pci_insn.h> 35 #include <asm/pci_clp.h> 36 #include <asm/pci_dma.h> 37 38 /* list of all detected zpci devices */ 39 static LIST_HEAD(zpci_list); 40 static DEFINE_SPINLOCK(zpci_list_lock); 41 42 static DECLARE_BITMAP(zpci_domain, ZPCI_NR_DEVICES); 43 static DEFINE_SPINLOCK(zpci_domain_lock); 44 45 #define ZPCI_IOMAP_ENTRIES \ 46 min(((unsigned long) ZPCI_NR_DEVICES * PCI_BAR_COUNT / 2), \ 47 ZPCI_IOMAP_MAX_ENTRIES) 48 49 static DEFINE_SPINLOCK(zpci_iomap_lock); 50 static unsigned long *zpci_iomap_bitmap; 51 struct zpci_iomap_entry *zpci_iomap_start; 52 EXPORT_SYMBOL_GPL(zpci_iomap_start); 53 54 DEFINE_STATIC_KEY_FALSE(have_mio); 55 56 static struct kmem_cache *zdev_fmb_cache; 57 58 struct zpci_dev *get_zdev_by_fid(u32 fid) 59 { 60 struct zpci_dev *tmp, *zdev = NULL; 61 62 spin_lock(&zpci_list_lock); 63 list_for_each_entry(tmp, &zpci_list, entry) { 64 if (tmp->fid == fid) { 65 zdev = tmp; 66 break; 67 } 68 } 69 spin_unlock(&zpci_list_lock); 70 return zdev; 71 } 72 73 void zpci_remove_reserved_devices(void) 74 { 75 struct zpci_dev *tmp, *zdev; 76 enum zpci_state state; 77 LIST_HEAD(remove); 78 79 spin_lock(&zpci_list_lock); 80 list_for_each_entry_safe(zdev, tmp, &zpci_list, entry) { 81 if (zdev->state == ZPCI_FN_STATE_STANDBY && 82 !clp_get_state(zdev->fid, &state) && 83 state == ZPCI_FN_STATE_RESERVED) 84 list_move_tail(&zdev->entry, &remove); 85 } 86 spin_unlock(&zpci_list_lock); 87 88 list_for_each_entry_safe(zdev, tmp, &remove, entry) 89 zpci_remove_device(zdev); 90 } 91 92 static struct zpci_dev *get_zdev_by_bus(struct pci_bus *bus) 93 { 94 return (bus && bus->sysdata) ? (struct zpci_dev *) bus->sysdata : NULL; 95 } 96 97 int pci_domain_nr(struct pci_bus *bus) 98 { 99 return ((struct zpci_dev *) bus->sysdata)->domain; 100 } 101 EXPORT_SYMBOL_GPL(pci_domain_nr); 102 103 int pci_proc_domain(struct pci_bus *bus) 104 { 105 return pci_domain_nr(bus); 106 } 107 EXPORT_SYMBOL_GPL(pci_proc_domain); 108 109 /* Modify PCI: Register I/O address translation parameters */ 110 int zpci_register_ioat(struct zpci_dev *zdev, u8 dmaas, 111 u64 base, u64 limit, u64 iota) 112 { 113 u64 req = ZPCI_CREATE_REQ(zdev->fh, dmaas, ZPCI_MOD_FC_REG_IOAT); 114 struct zpci_fib fib = {0}; 115 u8 status; 116 117 WARN_ON_ONCE(iota & 0x3fff); 118 fib.pba = base; 119 fib.pal = limit; 120 fib.iota = iota | ZPCI_IOTA_RTTO_FLAG; 121 return zpci_mod_fc(req, &fib, &status) ? -EIO : 0; 122 } 123 124 /* Modify PCI: Unregister I/O address translation parameters */ 125 int zpci_unregister_ioat(struct zpci_dev *zdev, u8 dmaas) 126 { 127 u64 req = ZPCI_CREATE_REQ(zdev->fh, dmaas, ZPCI_MOD_FC_DEREG_IOAT); 128 struct zpci_fib fib = {0}; 129 u8 cc, status; 130 131 cc = zpci_mod_fc(req, &fib, &status); 132 if (cc == 3) /* Function already gone. */ 133 cc = 0; 134 return cc ? -EIO : 0; 135 } 136 137 /* Modify PCI: Set PCI function measurement parameters */ 138 int zpci_fmb_enable_device(struct zpci_dev *zdev) 139 { 140 u64 req = ZPCI_CREATE_REQ(zdev->fh, 0, ZPCI_MOD_FC_SET_MEASURE); 141 struct zpci_fib fib = {0}; 142 u8 cc, status; 143 144 if (zdev->fmb || sizeof(*zdev->fmb) < zdev->fmb_length) 145 return -EINVAL; 146 147 zdev->fmb = kmem_cache_zalloc(zdev_fmb_cache, GFP_KERNEL); 148 if (!zdev->fmb) 149 return -ENOMEM; 150 WARN_ON((u64) zdev->fmb & 0xf); 151 152 /* reset software counters */ 153 atomic64_set(&zdev->allocated_pages, 0); 154 atomic64_set(&zdev->mapped_pages, 0); 155 atomic64_set(&zdev->unmapped_pages, 0); 156 157 fib.fmb_addr = virt_to_phys(zdev->fmb); 158 cc = zpci_mod_fc(req, &fib, &status); 159 if (cc) { 160 kmem_cache_free(zdev_fmb_cache, zdev->fmb); 161 zdev->fmb = NULL; 162 } 163 return cc ? -EIO : 0; 164 } 165 166 /* Modify PCI: Disable PCI function measurement */ 167 int zpci_fmb_disable_device(struct zpci_dev *zdev) 168 { 169 u64 req = ZPCI_CREATE_REQ(zdev->fh, 0, ZPCI_MOD_FC_SET_MEASURE); 170 struct zpci_fib fib = {0}; 171 u8 cc, status; 172 173 if (!zdev->fmb) 174 return -EINVAL; 175 176 /* Function measurement is disabled if fmb address is zero */ 177 cc = zpci_mod_fc(req, &fib, &status); 178 if (cc == 3) /* Function already gone. */ 179 cc = 0; 180 181 if (!cc) { 182 kmem_cache_free(zdev_fmb_cache, zdev->fmb); 183 zdev->fmb = NULL; 184 } 185 return cc ? -EIO : 0; 186 } 187 188 static int zpci_cfg_load(struct zpci_dev *zdev, int offset, u32 *val, u8 len) 189 { 190 u64 req = ZPCI_CREATE_REQ(zdev->fh, ZPCI_PCIAS_CFGSPC, len); 191 u64 data; 192 int rc; 193 194 rc = __zpci_load(&data, req, offset); 195 if (!rc) { 196 data = le64_to_cpu((__force __le64) data); 197 data >>= (8 - len) * 8; 198 *val = (u32) data; 199 } else 200 *val = 0xffffffff; 201 return rc; 202 } 203 204 static int zpci_cfg_store(struct zpci_dev *zdev, int offset, u32 val, u8 len) 205 { 206 u64 req = ZPCI_CREATE_REQ(zdev->fh, ZPCI_PCIAS_CFGSPC, len); 207 u64 data = val; 208 int rc; 209 210 data <<= (8 - len) * 8; 211 data = (__force u64) cpu_to_le64(data); 212 rc = __zpci_store(data, req, offset); 213 return rc; 214 } 215 216 resource_size_t pcibios_align_resource(void *data, const struct resource *res, 217 resource_size_t size, 218 resource_size_t align) 219 { 220 return 0; 221 } 222 223 /* combine single writes by using store-block insn */ 224 void __iowrite64_copy(void __iomem *to, const void *from, size_t count) 225 { 226 zpci_memcpy_toio(to, from, count); 227 } 228 229 void __iomem *ioremap(unsigned long ioaddr, unsigned long size) 230 { 231 struct vm_struct *area; 232 unsigned long offset; 233 234 if (!size) 235 return NULL; 236 237 if (!static_branch_unlikely(&have_mio)) 238 return (void __iomem *) ioaddr; 239 240 offset = ioaddr & ~PAGE_MASK; 241 ioaddr &= PAGE_MASK; 242 size = PAGE_ALIGN(size + offset); 243 area = get_vm_area(size, VM_IOREMAP); 244 if (!area) 245 return NULL; 246 247 if (ioremap_page_range((unsigned long) area->addr, 248 (unsigned long) area->addr + size, 249 ioaddr, PAGE_KERNEL)) { 250 vunmap(area->addr); 251 return NULL; 252 } 253 return (void __iomem *) ((unsigned long) area->addr + offset); 254 } 255 EXPORT_SYMBOL(ioremap); 256 257 void iounmap(volatile void __iomem *addr) 258 { 259 if (static_branch_likely(&have_mio)) 260 vunmap((__force void *) ((unsigned long) addr & PAGE_MASK)); 261 } 262 EXPORT_SYMBOL(iounmap); 263 264 /* Create a virtual mapping cookie for a PCI BAR */ 265 static void __iomem *pci_iomap_range_fh(struct pci_dev *pdev, int bar, 266 unsigned long offset, unsigned long max) 267 { 268 struct zpci_dev *zdev = to_zpci(pdev); 269 int idx; 270 271 idx = zdev->bars[bar].map_idx; 272 spin_lock(&zpci_iomap_lock); 273 /* Detect overrun */ 274 WARN_ON(!++zpci_iomap_start[idx].count); 275 zpci_iomap_start[idx].fh = zdev->fh; 276 zpci_iomap_start[idx].bar = bar; 277 spin_unlock(&zpci_iomap_lock); 278 279 return (void __iomem *) ZPCI_ADDR(idx) + offset; 280 } 281 282 static void __iomem *pci_iomap_range_mio(struct pci_dev *pdev, int bar, 283 unsigned long offset, 284 unsigned long max) 285 { 286 unsigned long barsize = pci_resource_len(pdev, bar); 287 struct zpci_dev *zdev = to_zpci(pdev); 288 void __iomem *iova; 289 290 iova = ioremap((unsigned long) zdev->bars[bar].mio_wt, barsize); 291 return iova ? iova + offset : iova; 292 } 293 294 void __iomem *pci_iomap_range(struct pci_dev *pdev, int bar, 295 unsigned long offset, unsigned long max) 296 { 297 if (!pci_resource_len(pdev, bar) || bar >= PCI_BAR_COUNT) 298 return NULL; 299 300 if (static_branch_likely(&have_mio)) 301 return pci_iomap_range_mio(pdev, bar, offset, max); 302 else 303 return pci_iomap_range_fh(pdev, bar, offset, max); 304 } 305 EXPORT_SYMBOL(pci_iomap_range); 306 307 void __iomem *pci_iomap(struct pci_dev *dev, int bar, unsigned long maxlen) 308 { 309 return pci_iomap_range(dev, bar, 0, maxlen); 310 } 311 EXPORT_SYMBOL(pci_iomap); 312 313 static void __iomem *pci_iomap_wc_range_mio(struct pci_dev *pdev, int bar, 314 unsigned long offset, unsigned long max) 315 { 316 unsigned long barsize = pci_resource_len(pdev, bar); 317 struct zpci_dev *zdev = to_zpci(pdev); 318 void __iomem *iova; 319 320 iova = ioremap((unsigned long) zdev->bars[bar].mio_wb, barsize); 321 return iova ? iova + offset : iova; 322 } 323 324 void __iomem *pci_iomap_wc_range(struct pci_dev *pdev, int bar, 325 unsigned long offset, unsigned long max) 326 { 327 if (!pci_resource_len(pdev, bar) || bar >= PCI_BAR_COUNT) 328 return NULL; 329 330 if (static_branch_likely(&have_mio)) 331 return pci_iomap_wc_range_mio(pdev, bar, offset, max); 332 else 333 return pci_iomap_range_fh(pdev, bar, offset, max); 334 } 335 EXPORT_SYMBOL(pci_iomap_wc_range); 336 337 void __iomem *pci_iomap_wc(struct pci_dev *dev, int bar, unsigned long maxlen) 338 { 339 return pci_iomap_wc_range(dev, bar, 0, maxlen); 340 } 341 EXPORT_SYMBOL(pci_iomap_wc); 342 343 static void pci_iounmap_fh(struct pci_dev *pdev, void __iomem *addr) 344 { 345 unsigned int idx = ZPCI_IDX(addr); 346 347 spin_lock(&zpci_iomap_lock); 348 /* Detect underrun */ 349 WARN_ON(!zpci_iomap_start[idx].count); 350 if (!--zpci_iomap_start[idx].count) { 351 zpci_iomap_start[idx].fh = 0; 352 zpci_iomap_start[idx].bar = 0; 353 } 354 spin_unlock(&zpci_iomap_lock); 355 } 356 357 static void pci_iounmap_mio(struct pci_dev *pdev, void __iomem *addr) 358 { 359 iounmap(addr); 360 } 361 362 void pci_iounmap(struct pci_dev *pdev, void __iomem *addr) 363 { 364 if (static_branch_likely(&have_mio)) 365 pci_iounmap_mio(pdev, addr); 366 else 367 pci_iounmap_fh(pdev, addr); 368 } 369 EXPORT_SYMBOL(pci_iounmap); 370 371 static int pci_read(struct pci_bus *bus, unsigned int devfn, int where, 372 int size, u32 *val) 373 { 374 struct zpci_dev *zdev = get_zdev_by_bus(bus); 375 int ret; 376 377 if (!zdev || devfn != ZPCI_DEVFN) 378 ret = -ENODEV; 379 else 380 ret = zpci_cfg_load(zdev, where, val, size); 381 382 return ret; 383 } 384 385 static int pci_write(struct pci_bus *bus, unsigned int devfn, int where, 386 int size, u32 val) 387 { 388 struct zpci_dev *zdev = get_zdev_by_bus(bus); 389 int ret; 390 391 if (!zdev || devfn != ZPCI_DEVFN) 392 ret = -ENODEV; 393 else 394 ret = zpci_cfg_store(zdev, where, val, size); 395 396 return ret; 397 } 398 399 static struct pci_ops pci_root_ops = { 400 .read = pci_read, 401 .write = pci_write, 402 }; 403 404 #ifdef CONFIG_PCI_IOV 405 static struct resource iov_res = { 406 .name = "PCI IOV res", 407 .start = 0, 408 .end = -1, 409 .flags = IORESOURCE_MEM, 410 }; 411 #endif 412 413 static void zpci_map_resources(struct pci_dev *pdev) 414 { 415 struct zpci_dev *zdev = to_zpci(pdev); 416 resource_size_t len; 417 int i; 418 419 for (i = 0; i < PCI_BAR_COUNT; i++) { 420 len = pci_resource_len(pdev, i); 421 if (!len) 422 continue; 423 424 if (static_branch_likely(&have_mio)) 425 pdev->resource[i].start = 426 (resource_size_t __force) zdev->bars[i].mio_wb; 427 else 428 pdev->resource[i].start = 429 (resource_size_t __force) pci_iomap(pdev, i, 0); 430 pdev->resource[i].end = pdev->resource[i].start + len - 1; 431 } 432 433 #ifdef CONFIG_PCI_IOV 434 i = PCI_IOV_RESOURCES; 435 436 for (; i < PCI_SRIOV_NUM_BARS + PCI_IOV_RESOURCES; i++) { 437 len = pci_resource_len(pdev, i); 438 if (!len) 439 continue; 440 pdev->resource[i].parent = &iov_res; 441 } 442 #endif 443 } 444 445 static void zpci_unmap_resources(struct pci_dev *pdev) 446 { 447 resource_size_t len; 448 int i; 449 450 if (static_branch_likely(&have_mio)) 451 return; 452 453 for (i = 0; i < PCI_BAR_COUNT; i++) { 454 len = pci_resource_len(pdev, i); 455 if (!len) 456 continue; 457 pci_iounmap(pdev, (void __iomem __force *) 458 pdev->resource[i].start); 459 } 460 } 461 462 static int zpci_alloc_iomap(struct zpci_dev *zdev) 463 { 464 unsigned long entry; 465 466 spin_lock(&zpci_iomap_lock); 467 entry = find_first_zero_bit(zpci_iomap_bitmap, ZPCI_IOMAP_ENTRIES); 468 if (entry == ZPCI_IOMAP_ENTRIES) { 469 spin_unlock(&zpci_iomap_lock); 470 return -ENOSPC; 471 } 472 set_bit(entry, zpci_iomap_bitmap); 473 spin_unlock(&zpci_iomap_lock); 474 return entry; 475 } 476 477 static void zpci_free_iomap(struct zpci_dev *zdev, int entry) 478 { 479 spin_lock(&zpci_iomap_lock); 480 memset(&zpci_iomap_start[entry], 0, sizeof(struct zpci_iomap_entry)); 481 clear_bit(entry, zpci_iomap_bitmap); 482 spin_unlock(&zpci_iomap_lock); 483 } 484 485 static struct resource *__alloc_res(struct zpci_dev *zdev, unsigned long start, 486 unsigned long size, unsigned long flags) 487 { 488 struct resource *r; 489 490 r = kzalloc(sizeof(*r), GFP_KERNEL); 491 if (!r) 492 return NULL; 493 494 r->start = start; 495 r->end = r->start + size - 1; 496 r->flags = flags; 497 r->name = zdev->res_name; 498 499 if (request_resource(&iomem_resource, r)) { 500 kfree(r); 501 return NULL; 502 } 503 return r; 504 } 505 506 static int zpci_setup_bus_resources(struct zpci_dev *zdev, 507 struct list_head *resources) 508 { 509 unsigned long addr, size, flags; 510 struct resource *res; 511 int i, entry; 512 513 snprintf(zdev->res_name, sizeof(zdev->res_name), 514 "PCI Bus %04x:%02x", zdev->domain, ZPCI_BUS_NR); 515 516 for (i = 0; i < PCI_BAR_COUNT; i++) { 517 if (!zdev->bars[i].size) 518 continue; 519 entry = zpci_alloc_iomap(zdev); 520 if (entry < 0) 521 return entry; 522 zdev->bars[i].map_idx = entry; 523 524 /* only MMIO is supported */ 525 flags = IORESOURCE_MEM; 526 if (zdev->bars[i].val & 8) 527 flags |= IORESOURCE_PREFETCH; 528 if (zdev->bars[i].val & 4) 529 flags |= IORESOURCE_MEM_64; 530 531 addr = ZPCI_ADDR(entry); 532 size = 1UL << zdev->bars[i].size; 533 534 res = __alloc_res(zdev, addr, size, flags); 535 if (!res) { 536 zpci_free_iomap(zdev, entry); 537 return -ENOMEM; 538 } 539 zdev->bars[i].res = res; 540 pci_add_resource(resources, res); 541 } 542 543 return 0; 544 } 545 546 static void zpci_cleanup_bus_resources(struct zpci_dev *zdev) 547 { 548 int i; 549 550 for (i = 0; i < PCI_BAR_COUNT; i++) { 551 if (!zdev->bars[i].size || !zdev->bars[i].res) 552 continue; 553 554 zpci_free_iomap(zdev, zdev->bars[i].map_idx); 555 release_resource(zdev->bars[i].res); 556 kfree(zdev->bars[i].res); 557 } 558 } 559 560 int pcibios_add_device(struct pci_dev *pdev) 561 { 562 struct resource *res; 563 int i; 564 565 if (pdev->is_physfn) 566 pdev->no_vf_scan = 1; 567 568 pdev->dev.groups = zpci_attr_groups; 569 pdev->dev.dma_ops = &s390_pci_dma_ops; 570 zpci_map_resources(pdev); 571 572 for (i = 0; i < PCI_BAR_COUNT; i++) { 573 res = &pdev->resource[i]; 574 if (res->parent || !res->flags) 575 continue; 576 pci_claim_resource(pdev, i); 577 } 578 579 return 0; 580 } 581 582 void pcibios_release_device(struct pci_dev *pdev) 583 { 584 zpci_unmap_resources(pdev); 585 } 586 587 int pcibios_enable_device(struct pci_dev *pdev, int mask) 588 { 589 struct zpci_dev *zdev = to_zpci(pdev); 590 591 zpci_debug_init_device(zdev, dev_name(&pdev->dev)); 592 zpci_fmb_enable_device(zdev); 593 594 return pci_enable_resources(pdev, mask); 595 } 596 597 void pcibios_disable_device(struct pci_dev *pdev) 598 { 599 struct zpci_dev *zdev = to_zpci(pdev); 600 601 zpci_fmb_disable_device(zdev); 602 zpci_debug_exit_device(zdev); 603 } 604 605 #ifdef CONFIG_HIBERNATE_CALLBACKS 606 static int zpci_restore(struct device *dev) 607 { 608 struct pci_dev *pdev = to_pci_dev(dev); 609 struct zpci_dev *zdev = to_zpci(pdev); 610 int ret = 0; 611 612 if (zdev->state != ZPCI_FN_STATE_ONLINE) 613 goto out; 614 615 ret = clp_enable_fh(zdev, ZPCI_NR_DMA_SPACES); 616 if (ret) 617 goto out; 618 619 zpci_map_resources(pdev); 620 zpci_register_ioat(zdev, 0, zdev->start_dma, zdev->end_dma, 621 (u64) zdev->dma_table); 622 623 out: 624 return ret; 625 } 626 627 static int zpci_freeze(struct device *dev) 628 { 629 struct pci_dev *pdev = to_pci_dev(dev); 630 struct zpci_dev *zdev = to_zpci(pdev); 631 632 if (zdev->state != ZPCI_FN_STATE_ONLINE) 633 return 0; 634 635 zpci_unregister_ioat(zdev, 0); 636 zpci_unmap_resources(pdev); 637 return clp_disable_fh(zdev); 638 } 639 640 struct dev_pm_ops pcibios_pm_ops = { 641 .thaw_noirq = zpci_restore, 642 .freeze_noirq = zpci_freeze, 643 .restore_noirq = zpci_restore, 644 .poweroff_noirq = zpci_freeze, 645 }; 646 #endif /* CONFIG_HIBERNATE_CALLBACKS */ 647 648 static int zpci_alloc_domain(struct zpci_dev *zdev) 649 { 650 if (zpci_unique_uid) { 651 zdev->domain = (u16) zdev->uid; 652 if (zdev->domain >= ZPCI_NR_DEVICES) 653 return 0; 654 655 spin_lock(&zpci_domain_lock); 656 if (test_bit(zdev->domain, zpci_domain)) { 657 spin_unlock(&zpci_domain_lock); 658 return -EEXIST; 659 } 660 set_bit(zdev->domain, zpci_domain); 661 spin_unlock(&zpci_domain_lock); 662 return 0; 663 } 664 665 spin_lock(&zpci_domain_lock); 666 zdev->domain = find_first_zero_bit(zpci_domain, ZPCI_NR_DEVICES); 667 if (zdev->domain == ZPCI_NR_DEVICES) { 668 spin_unlock(&zpci_domain_lock); 669 return -ENOSPC; 670 } 671 set_bit(zdev->domain, zpci_domain); 672 spin_unlock(&zpci_domain_lock); 673 return 0; 674 } 675 676 static void zpci_free_domain(struct zpci_dev *zdev) 677 { 678 if (zdev->domain >= ZPCI_NR_DEVICES) 679 return; 680 681 spin_lock(&zpci_domain_lock); 682 clear_bit(zdev->domain, zpci_domain); 683 spin_unlock(&zpci_domain_lock); 684 } 685 686 void pcibios_remove_bus(struct pci_bus *bus) 687 { 688 struct zpci_dev *zdev = get_zdev_by_bus(bus); 689 690 zpci_exit_slot(zdev); 691 zpci_cleanup_bus_resources(zdev); 692 zpci_destroy_iommu(zdev); 693 zpci_free_domain(zdev); 694 695 spin_lock(&zpci_list_lock); 696 list_del(&zdev->entry); 697 spin_unlock(&zpci_list_lock); 698 699 zpci_dbg(3, "rem fid:%x\n", zdev->fid); 700 kfree(zdev); 701 } 702 703 static int zpci_scan_bus(struct zpci_dev *zdev) 704 { 705 LIST_HEAD(resources); 706 int ret; 707 708 ret = zpci_setup_bus_resources(zdev, &resources); 709 if (ret) 710 goto error; 711 712 zdev->bus = pci_scan_root_bus(NULL, ZPCI_BUS_NR, &pci_root_ops, 713 zdev, &resources); 714 if (!zdev->bus) { 715 ret = -EIO; 716 goto error; 717 } 718 zdev->bus->max_bus_speed = zdev->max_bus_speed; 719 pci_bus_add_devices(zdev->bus); 720 return 0; 721 722 error: 723 zpci_cleanup_bus_resources(zdev); 724 pci_free_resource_list(&resources); 725 return ret; 726 } 727 728 int zpci_enable_device(struct zpci_dev *zdev) 729 { 730 int rc; 731 732 rc = clp_enable_fh(zdev, ZPCI_NR_DMA_SPACES); 733 if (rc) 734 goto out; 735 736 rc = zpci_dma_init_device(zdev); 737 if (rc) 738 goto out_dma; 739 740 zdev->state = ZPCI_FN_STATE_ONLINE; 741 return 0; 742 743 out_dma: 744 clp_disable_fh(zdev); 745 out: 746 return rc; 747 } 748 EXPORT_SYMBOL_GPL(zpci_enable_device); 749 750 int zpci_disable_device(struct zpci_dev *zdev) 751 { 752 zpci_dma_exit_device(zdev); 753 return clp_disable_fh(zdev); 754 } 755 EXPORT_SYMBOL_GPL(zpci_disable_device); 756 757 int zpci_create_device(struct zpci_dev *zdev) 758 { 759 int rc; 760 761 rc = zpci_alloc_domain(zdev); 762 if (rc) 763 goto out; 764 765 rc = zpci_init_iommu(zdev); 766 if (rc) 767 goto out_free; 768 769 mutex_init(&zdev->lock); 770 if (zdev->state == ZPCI_FN_STATE_CONFIGURED) { 771 rc = zpci_enable_device(zdev); 772 if (rc) 773 goto out_destroy_iommu; 774 } 775 rc = zpci_scan_bus(zdev); 776 if (rc) 777 goto out_disable; 778 779 spin_lock(&zpci_list_lock); 780 list_add_tail(&zdev->entry, &zpci_list); 781 spin_unlock(&zpci_list_lock); 782 783 zpci_init_slot(zdev); 784 785 return 0; 786 787 out_disable: 788 if (zdev->state == ZPCI_FN_STATE_ONLINE) 789 zpci_disable_device(zdev); 790 out_destroy_iommu: 791 zpci_destroy_iommu(zdev); 792 out_free: 793 zpci_free_domain(zdev); 794 out: 795 return rc; 796 } 797 798 void zpci_remove_device(struct zpci_dev *zdev) 799 { 800 if (!zdev->bus) 801 return; 802 803 pci_stop_root_bus(zdev->bus); 804 pci_remove_root_bus(zdev->bus); 805 } 806 807 int zpci_report_error(struct pci_dev *pdev, 808 struct zpci_report_error_header *report) 809 { 810 struct zpci_dev *zdev = to_zpci(pdev); 811 812 return sclp_pci_report(report, zdev->fh, zdev->fid); 813 } 814 EXPORT_SYMBOL(zpci_report_error); 815 816 static int zpci_mem_init(void) 817 { 818 BUILD_BUG_ON(!is_power_of_2(__alignof__(struct zpci_fmb)) || 819 __alignof__(struct zpci_fmb) < sizeof(struct zpci_fmb)); 820 821 zdev_fmb_cache = kmem_cache_create("PCI_FMB_cache", sizeof(struct zpci_fmb), 822 __alignof__(struct zpci_fmb), 0, NULL); 823 if (!zdev_fmb_cache) 824 goto error_fmb; 825 826 zpci_iomap_start = kcalloc(ZPCI_IOMAP_ENTRIES, 827 sizeof(*zpci_iomap_start), GFP_KERNEL); 828 if (!zpci_iomap_start) 829 goto error_iomap; 830 831 zpci_iomap_bitmap = kcalloc(BITS_TO_LONGS(ZPCI_IOMAP_ENTRIES), 832 sizeof(*zpci_iomap_bitmap), GFP_KERNEL); 833 if (!zpci_iomap_bitmap) 834 goto error_iomap_bitmap; 835 836 return 0; 837 error_iomap_bitmap: 838 kfree(zpci_iomap_start); 839 error_iomap: 840 kmem_cache_destroy(zdev_fmb_cache); 841 error_fmb: 842 return -ENOMEM; 843 } 844 845 static void zpci_mem_exit(void) 846 { 847 kfree(zpci_iomap_bitmap); 848 kfree(zpci_iomap_start); 849 kmem_cache_destroy(zdev_fmb_cache); 850 } 851 852 static unsigned int s390_pci_probe __initdata = 1; 853 static unsigned int s390_pci_no_mio __initdata; 854 unsigned int s390_pci_force_floating __initdata; 855 static unsigned int s390_pci_initialized; 856 857 char * __init pcibios_setup(char *str) 858 { 859 if (!strcmp(str, "off")) { 860 s390_pci_probe = 0; 861 return NULL; 862 } 863 if (!strcmp(str, "nomio")) { 864 s390_pci_no_mio = 1; 865 return NULL; 866 } 867 if (!strcmp(str, "force_floating")) { 868 s390_pci_force_floating = 1; 869 return NULL; 870 } 871 return str; 872 } 873 874 bool zpci_is_enabled(void) 875 { 876 return s390_pci_initialized; 877 } 878 879 static int __init pci_base_init(void) 880 { 881 int rc; 882 883 if (!s390_pci_probe) 884 return 0; 885 886 if (!test_facility(69) || !test_facility(71)) 887 return 0; 888 889 if (test_facility(153) && !s390_pci_no_mio) 890 static_branch_enable(&have_mio); 891 892 rc = zpci_debug_init(); 893 if (rc) 894 goto out; 895 896 rc = zpci_mem_init(); 897 if (rc) 898 goto out_mem; 899 900 rc = zpci_irq_init(); 901 if (rc) 902 goto out_irq; 903 904 rc = zpci_dma_init(); 905 if (rc) 906 goto out_dma; 907 908 rc = clp_scan_pci_devices(); 909 if (rc) 910 goto out_find; 911 912 s390_pci_initialized = 1; 913 return 0; 914 915 out_find: 916 zpci_dma_exit(); 917 out_dma: 918 zpci_irq_exit(); 919 out_irq: 920 zpci_mem_exit(); 921 out_mem: 922 zpci_debug_exit(); 923 out: 924 return rc; 925 } 926 subsys_initcall_sync(pci_base_init); 927 928 void zpci_rescan(void) 929 { 930 if (zpci_is_enabled()) 931 clp_rescan_pci_devices_simple(); 932 } 933