1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright IBM Corp. 2012 4 * 5 * Author(s): 6 * Jan Glauber <jang@linux.vnet.ibm.com> 7 * 8 * The System z PCI code is a rewrite from a prototype by 9 * the following people (Kudoz!): 10 * Alexander Schmidt 11 * Christoph Raisch 12 * Hannes Hering 13 * Hoang-Nam Nguyen 14 * Jan-Bernd Themann 15 * Stefan Roscher 16 * Thomas Klein 17 */ 18 19 #define KMSG_COMPONENT "zpci" 20 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt 21 22 #include <linux/kernel.h> 23 #include <linux/slab.h> 24 #include <linux/err.h> 25 #include <linux/export.h> 26 #include <linux/delay.h> 27 #include <linux/seq_file.h> 28 #include <linux/jump_label.h> 29 #include <linux/pci.h> 30 31 #include <asm/isc.h> 32 #include <asm/airq.h> 33 #include <asm/facility.h> 34 #include <asm/pci_insn.h> 35 #include <asm/pci_clp.h> 36 #include <asm/pci_dma.h> 37 38 /* list of all detected zpci devices */ 39 static LIST_HEAD(zpci_list); 40 static DEFINE_SPINLOCK(zpci_list_lock); 41 42 static DECLARE_BITMAP(zpci_domain, ZPCI_NR_DEVICES); 43 static DEFINE_SPINLOCK(zpci_domain_lock); 44 45 #define ZPCI_IOMAP_ENTRIES \ 46 min(((unsigned long) ZPCI_NR_DEVICES * PCI_BAR_COUNT / 2), \ 47 ZPCI_IOMAP_MAX_ENTRIES) 48 49 static DEFINE_SPINLOCK(zpci_iomap_lock); 50 static unsigned long *zpci_iomap_bitmap; 51 struct zpci_iomap_entry *zpci_iomap_start; 52 EXPORT_SYMBOL_GPL(zpci_iomap_start); 53 54 DEFINE_STATIC_KEY_FALSE(have_mio); 55 56 static struct kmem_cache *zdev_fmb_cache; 57 58 struct zpci_dev *get_zdev_by_fid(u32 fid) 59 { 60 struct zpci_dev *tmp, *zdev = NULL; 61 62 spin_lock(&zpci_list_lock); 63 list_for_each_entry(tmp, &zpci_list, entry) { 64 if (tmp->fid == fid) { 65 zdev = tmp; 66 break; 67 } 68 } 69 spin_unlock(&zpci_list_lock); 70 return zdev; 71 } 72 73 void zpci_remove_reserved_devices(void) 74 { 75 struct zpci_dev *tmp, *zdev; 76 enum zpci_state state; 77 LIST_HEAD(remove); 78 79 spin_lock(&zpci_list_lock); 80 list_for_each_entry_safe(zdev, tmp, &zpci_list, entry) { 81 if (zdev->state == ZPCI_FN_STATE_STANDBY && 82 !clp_get_state(zdev->fid, &state) && 83 state == ZPCI_FN_STATE_RESERVED) 84 list_move_tail(&zdev->entry, &remove); 85 } 86 spin_unlock(&zpci_list_lock); 87 88 list_for_each_entry_safe(zdev, tmp, &remove, entry) 89 zpci_remove_device(zdev); 90 } 91 92 static struct zpci_dev *get_zdev_by_bus(struct pci_bus *bus) 93 { 94 return (bus && bus->sysdata) ? (struct zpci_dev *) bus->sysdata : NULL; 95 } 96 97 int pci_domain_nr(struct pci_bus *bus) 98 { 99 return ((struct zpci_dev *) bus->sysdata)->domain; 100 } 101 EXPORT_SYMBOL_GPL(pci_domain_nr); 102 103 int pci_proc_domain(struct pci_bus *bus) 104 { 105 return pci_domain_nr(bus); 106 } 107 EXPORT_SYMBOL_GPL(pci_proc_domain); 108 109 /* Modify PCI: Register I/O address translation parameters */ 110 int zpci_register_ioat(struct zpci_dev *zdev, u8 dmaas, 111 u64 base, u64 limit, u64 iota) 112 { 113 u64 req = ZPCI_CREATE_REQ(zdev->fh, dmaas, ZPCI_MOD_FC_REG_IOAT); 114 struct zpci_fib fib = {0}; 115 u8 status; 116 117 WARN_ON_ONCE(iota & 0x3fff); 118 fib.pba = base; 119 fib.pal = limit; 120 fib.iota = iota | ZPCI_IOTA_RTTO_FLAG; 121 return zpci_mod_fc(req, &fib, &status) ? -EIO : 0; 122 } 123 124 /* Modify PCI: Unregister I/O address translation parameters */ 125 int zpci_unregister_ioat(struct zpci_dev *zdev, u8 dmaas) 126 { 127 u64 req = ZPCI_CREATE_REQ(zdev->fh, dmaas, ZPCI_MOD_FC_DEREG_IOAT); 128 struct zpci_fib fib = {0}; 129 u8 cc, status; 130 131 cc = zpci_mod_fc(req, &fib, &status); 132 if (cc == 3) /* Function already gone. */ 133 cc = 0; 134 return cc ? -EIO : 0; 135 } 136 137 /* Modify PCI: Set PCI function measurement parameters */ 138 int zpci_fmb_enable_device(struct zpci_dev *zdev) 139 { 140 u64 req = ZPCI_CREATE_REQ(zdev->fh, 0, ZPCI_MOD_FC_SET_MEASURE); 141 struct zpci_fib fib = {0}; 142 u8 cc, status; 143 144 if (zdev->fmb || sizeof(*zdev->fmb) < zdev->fmb_length) 145 return -EINVAL; 146 147 zdev->fmb = kmem_cache_zalloc(zdev_fmb_cache, GFP_KERNEL); 148 if (!zdev->fmb) 149 return -ENOMEM; 150 WARN_ON((u64) zdev->fmb & 0xf); 151 152 /* reset software counters */ 153 atomic64_set(&zdev->allocated_pages, 0); 154 atomic64_set(&zdev->mapped_pages, 0); 155 atomic64_set(&zdev->unmapped_pages, 0); 156 157 fib.fmb_addr = virt_to_phys(zdev->fmb); 158 cc = zpci_mod_fc(req, &fib, &status); 159 if (cc) { 160 kmem_cache_free(zdev_fmb_cache, zdev->fmb); 161 zdev->fmb = NULL; 162 } 163 return cc ? -EIO : 0; 164 } 165 166 /* Modify PCI: Disable PCI function measurement */ 167 int zpci_fmb_disable_device(struct zpci_dev *zdev) 168 { 169 u64 req = ZPCI_CREATE_REQ(zdev->fh, 0, ZPCI_MOD_FC_SET_MEASURE); 170 struct zpci_fib fib = {0}; 171 u8 cc, status; 172 173 if (!zdev->fmb) 174 return -EINVAL; 175 176 /* Function measurement is disabled if fmb address is zero */ 177 cc = zpci_mod_fc(req, &fib, &status); 178 if (cc == 3) /* Function already gone. */ 179 cc = 0; 180 181 if (!cc) { 182 kmem_cache_free(zdev_fmb_cache, zdev->fmb); 183 zdev->fmb = NULL; 184 } 185 return cc ? -EIO : 0; 186 } 187 188 static int zpci_cfg_load(struct zpci_dev *zdev, int offset, u32 *val, u8 len) 189 { 190 u64 req = ZPCI_CREATE_REQ(zdev->fh, ZPCI_PCIAS_CFGSPC, len); 191 u64 data; 192 int rc; 193 194 rc = __zpci_load(&data, req, offset); 195 if (!rc) { 196 data = le64_to_cpu((__force __le64) data); 197 data >>= (8 - len) * 8; 198 *val = (u32) data; 199 } else 200 *val = 0xffffffff; 201 return rc; 202 } 203 204 static int zpci_cfg_store(struct zpci_dev *zdev, int offset, u32 val, u8 len) 205 { 206 u64 req = ZPCI_CREATE_REQ(zdev->fh, ZPCI_PCIAS_CFGSPC, len); 207 u64 data = val; 208 int rc; 209 210 data <<= (8 - len) * 8; 211 data = (__force u64) cpu_to_le64(data); 212 rc = __zpci_store(data, req, offset); 213 return rc; 214 } 215 216 resource_size_t pcibios_align_resource(void *data, const struct resource *res, 217 resource_size_t size, 218 resource_size_t align) 219 { 220 return 0; 221 } 222 223 /* combine single writes by using store-block insn */ 224 void __iowrite64_copy(void __iomem *to, const void *from, size_t count) 225 { 226 zpci_memcpy_toio(to, from, count); 227 } 228 229 void __iomem *ioremap(unsigned long ioaddr, unsigned long size) 230 { 231 struct vm_struct *area; 232 unsigned long offset; 233 234 if (!size) 235 return NULL; 236 237 if (!static_branch_unlikely(&have_mio)) 238 return (void __iomem *) ioaddr; 239 240 offset = ioaddr & ~PAGE_MASK; 241 ioaddr &= PAGE_MASK; 242 size = PAGE_ALIGN(size + offset); 243 area = get_vm_area(size, VM_IOREMAP); 244 if (!area) 245 return NULL; 246 247 if (ioremap_page_range((unsigned long) area->addr, 248 (unsigned long) area->addr + size, 249 ioaddr, PAGE_KERNEL)) { 250 vunmap(area->addr); 251 return NULL; 252 } 253 return (void __iomem *) ((unsigned long) area->addr + offset); 254 } 255 EXPORT_SYMBOL(ioremap); 256 257 void iounmap(volatile void __iomem *addr) 258 { 259 if (static_branch_likely(&have_mio)) 260 vunmap((__force void *) ((unsigned long) addr & PAGE_MASK)); 261 } 262 EXPORT_SYMBOL(iounmap); 263 264 /* Create a virtual mapping cookie for a PCI BAR */ 265 static void __iomem *pci_iomap_range_fh(struct pci_dev *pdev, int bar, 266 unsigned long offset, unsigned long max) 267 { 268 struct zpci_dev *zdev = to_zpci(pdev); 269 int idx; 270 271 idx = zdev->bars[bar].map_idx; 272 spin_lock(&zpci_iomap_lock); 273 /* Detect overrun */ 274 WARN_ON(!++zpci_iomap_start[idx].count); 275 zpci_iomap_start[idx].fh = zdev->fh; 276 zpci_iomap_start[idx].bar = bar; 277 spin_unlock(&zpci_iomap_lock); 278 279 return (void __iomem *) ZPCI_ADDR(idx) + offset; 280 } 281 282 static void __iomem *pci_iomap_range_mio(struct pci_dev *pdev, int bar, 283 unsigned long offset, 284 unsigned long max) 285 { 286 unsigned long barsize = pci_resource_len(pdev, bar); 287 struct zpci_dev *zdev = to_zpci(pdev); 288 void __iomem *iova; 289 290 iova = ioremap((unsigned long) zdev->bars[bar].mio_wt, barsize); 291 return iova ? iova + offset : iova; 292 } 293 294 void __iomem *pci_iomap_range(struct pci_dev *pdev, int bar, 295 unsigned long offset, unsigned long max) 296 { 297 if (!pci_resource_len(pdev, bar) || bar >= PCI_BAR_COUNT) 298 return NULL; 299 300 if (static_branch_likely(&have_mio)) 301 return pci_iomap_range_mio(pdev, bar, offset, max); 302 else 303 return pci_iomap_range_fh(pdev, bar, offset, max); 304 } 305 EXPORT_SYMBOL(pci_iomap_range); 306 307 void __iomem *pci_iomap(struct pci_dev *dev, int bar, unsigned long maxlen) 308 { 309 return pci_iomap_range(dev, bar, 0, maxlen); 310 } 311 EXPORT_SYMBOL(pci_iomap); 312 313 static void __iomem *pci_iomap_wc_range_mio(struct pci_dev *pdev, int bar, 314 unsigned long offset, unsigned long max) 315 { 316 unsigned long barsize = pci_resource_len(pdev, bar); 317 struct zpci_dev *zdev = to_zpci(pdev); 318 void __iomem *iova; 319 320 iova = ioremap((unsigned long) zdev->bars[bar].mio_wb, barsize); 321 return iova ? iova + offset : iova; 322 } 323 324 void __iomem *pci_iomap_wc_range(struct pci_dev *pdev, int bar, 325 unsigned long offset, unsigned long max) 326 { 327 if (!pci_resource_len(pdev, bar) || bar >= PCI_BAR_COUNT) 328 return NULL; 329 330 if (static_branch_likely(&have_mio)) 331 return pci_iomap_wc_range_mio(pdev, bar, offset, max); 332 else 333 return pci_iomap_range_fh(pdev, bar, offset, max); 334 } 335 EXPORT_SYMBOL(pci_iomap_wc_range); 336 337 void __iomem *pci_iomap_wc(struct pci_dev *dev, int bar, unsigned long maxlen) 338 { 339 return pci_iomap_wc_range(dev, bar, 0, maxlen); 340 } 341 EXPORT_SYMBOL(pci_iomap_wc); 342 343 static void pci_iounmap_fh(struct pci_dev *pdev, void __iomem *addr) 344 { 345 unsigned int idx = ZPCI_IDX(addr); 346 347 spin_lock(&zpci_iomap_lock); 348 /* Detect underrun */ 349 WARN_ON(!zpci_iomap_start[idx].count); 350 if (!--zpci_iomap_start[idx].count) { 351 zpci_iomap_start[idx].fh = 0; 352 zpci_iomap_start[idx].bar = 0; 353 } 354 spin_unlock(&zpci_iomap_lock); 355 } 356 357 static void pci_iounmap_mio(struct pci_dev *pdev, void __iomem *addr) 358 { 359 iounmap(addr); 360 } 361 362 void pci_iounmap(struct pci_dev *pdev, void __iomem *addr) 363 { 364 if (static_branch_likely(&have_mio)) 365 pci_iounmap_mio(pdev, addr); 366 else 367 pci_iounmap_fh(pdev, addr); 368 } 369 EXPORT_SYMBOL(pci_iounmap); 370 371 static int pci_read(struct pci_bus *bus, unsigned int devfn, int where, 372 int size, u32 *val) 373 { 374 struct zpci_dev *zdev = get_zdev_by_bus(bus); 375 int ret; 376 377 if (!zdev || devfn != ZPCI_DEVFN) 378 ret = -ENODEV; 379 else 380 ret = zpci_cfg_load(zdev, where, val, size); 381 382 return ret; 383 } 384 385 static int pci_write(struct pci_bus *bus, unsigned int devfn, int where, 386 int size, u32 val) 387 { 388 struct zpci_dev *zdev = get_zdev_by_bus(bus); 389 int ret; 390 391 if (!zdev || devfn != ZPCI_DEVFN) 392 ret = -ENODEV; 393 else 394 ret = zpci_cfg_store(zdev, where, val, size); 395 396 return ret; 397 } 398 399 static struct pci_ops pci_root_ops = { 400 .read = pci_read, 401 .write = pci_write, 402 }; 403 404 #ifdef CONFIG_PCI_IOV 405 static struct resource iov_res = { 406 .name = "PCI IOV res", 407 .start = 0, 408 .end = -1, 409 .flags = IORESOURCE_MEM, 410 }; 411 #endif 412 413 static void zpci_map_resources(struct pci_dev *pdev) 414 { 415 struct zpci_dev *zdev = to_zpci(pdev); 416 resource_size_t len; 417 int i; 418 419 for (i = 0; i < PCI_BAR_COUNT; i++) { 420 len = pci_resource_len(pdev, i); 421 if (!len) 422 continue; 423 424 if (static_branch_likely(&have_mio)) 425 pdev->resource[i].start = 426 (resource_size_t __force) zdev->bars[i].mio_wb; 427 else 428 pdev->resource[i].start = 429 (resource_size_t __force) pci_iomap(pdev, i, 0); 430 pdev->resource[i].end = pdev->resource[i].start + len - 1; 431 } 432 433 #ifdef CONFIG_PCI_IOV 434 i = PCI_IOV_RESOURCES; 435 436 for (; i < PCI_SRIOV_NUM_BARS + PCI_IOV_RESOURCES; i++) { 437 len = pci_resource_len(pdev, i); 438 if (!len) 439 continue; 440 pdev->resource[i].parent = &iov_res; 441 } 442 #endif 443 } 444 445 static void zpci_unmap_resources(struct pci_dev *pdev) 446 { 447 resource_size_t len; 448 int i; 449 450 if (static_branch_likely(&have_mio)) 451 return; 452 453 for (i = 0; i < PCI_BAR_COUNT; i++) { 454 len = pci_resource_len(pdev, i); 455 if (!len) 456 continue; 457 pci_iounmap(pdev, (void __iomem __force *) 458 pdev->resource[i].start); 459 } 460 } 461 462 static int zpci_alloc_iomap(struct zpci_dev *zdev) 463 { 464 unsigned long entry; 465 466 spin_lock(&zpci_iomap_lock); 467 entry = find_first_zero_bit(zpci_iomap_bitmap, ZPCI_IOMAP_ENTRIES); 468 if (entry == ZPCI_IOMAP_ENTRIES) { 469 spin_unlock(&zpci_iomap_lock); 470 return -ENOSPC; 471 } 472 set_bit(entry, zpci_iomap_bitmap); 473 spin_unlock(&zpci_iomap_lock); 474 return entry; 475 } 476 477 static void zpci_free_iomap(struct zpci_dev *zdev, int entry) 478 { 479 spin_lock(&zpci_iomap_lock); 480 memset(&zpci_iomap_start[entry], 0, sizeof(struct zpci_iomap_entry)); 481 clear_bit(entry, zpci_iomap_bitmap); 482 spin_unlock(&zpci_iomap_lock); 483 } 484 485 static struct resource *__alloc_res(struct zpci_dev *zdev, unsigned long start, 486 unsigned long size, unsigned long flags) 487 { 488 struct resource *r; 489 490 r = kzalloc(sizeof(*r), GFP_KERNEL); 491 if (!r) 492 return NULL; 493 494 r->start = start; 495 r->end = r->start + size - 1; 496 r->flags = flags; 497 r->name = zdev->res_name; 498 499 if (request_resource(&iomem_resource, r)) { 500 kfree(r); 501 return NULL; 502 } 503 return r; 504 } 505 506 static int zpci_setup_bus_resources(struct zpci_dev *zdev, 507 struct list_head *resources) 508 { 509 unsigned long addr, size, flags; 510 struct resource *res; 511 int i, entry; 512 513 snprintf(zdev->res_name, sizeof(zdev->res_name), 514 "PCI Bus %04x:%02x", zdev->domain, ZPCI_BUS_NR); 515 516 for (i = 0; i < PCI_BAR_COUNT; i++) { 517 if (!zdev->bars[i].size) 518 continue; 519 entry = zpci_alloc_iomap(zdev); 520 if (entry < 0) 521 return entry; 522 zdev->bars[i].map_idx = entry; 523 524 /* only MMIO is supported */ 525 flags = IORESOURCE_MEM; 526 if (zdev->bars[i].val & 8) 527 flags |= IORESOURCE_PREFETCH; 528 if (zdev->bars[i].val & 4) 529 flags |= IORESOURCE_MEM_64; 530 531 if (static_branch_likely(&have_mio)) 532 addr = (unsigned long) zdev->bars[i].mio_wb; 533 else 534 addr = ZPCI_ADDR(entry); 535 size = 1UL << zdev->bars[i].size; 536 537 res = __alloc_res(zdev, addr, size, flags); 538 if (!res) { 539 zpci_free_iomap(zdev, entry); 540 return -ENOMEM; 541 } 542 zdev->bars[i].res = res; 543 pci_add_resource(resources, res); 544 } 545 546 return 0; 547 } 548 549 static void zpci_cleanup_bus_resources(struct zpci_dev *zdev) 550 { 551 int i; 552 553 for (i = 0; i < PCI_BAR_COUNT; i++) { 554 if (!zdev->bars[i].size || !zdev->bars[i].res) 555 continue; 556 557 zpci_free_iomap(zdev, zdev->bars[i].map_idx); 558 release_resource(zdev->bars[i].res); 559 kfree(zdev->bars[i].res); 560 } 561 } 562 563 int pcibios_add_device(struct pci_dev *pdev) 564 { 565 struct resource *res; 566 int i; 567 568 if (pdev->is_physfn) 569 pdev->no_vf_scan = 1; 570 571 pdev->dev.groups = zpci_attr_groups; 572 pdev->dev.dma_ops = &s390_pci_dma_ops; 573 zpci_map_resources(pdev); 574 575 for (i = 0; i < PCI_BAR_COUNT; i++) { 576 res = &pdev->resource[i]; 577 if (res->parent || !res->flags) 578 continue; 579 pci_claim_resource(pdev, i); 580 } 581 582 return 0; 583 } 584 585 void pcibios_release_device(struct pci_dev *pdev) 586 { 587 zpci_unmap_resources(pdev); 588 } 589 590 int pcibios_enable_device(struct pci_dev *pdev, int mask) 591 { 592 struct zpci_dev *zdev = to_zpci(pdev); 593 594 zpci_debug_init_device(zdev, dev_name(&pdev->dev)); 595 zpci_fmb_enable_device(zdev); 596 597 return pci_enable_resources(pdev, mask); 598 } 599 600 void pcibios_disable_device(struct pci_dev *pdev) 601 { 602 struct zpci_dev *zdev = to_zpci(pdev); 603 604 zpci_fmb_disable_device(zdev); 605 zpci_debug_exit_device(zdev); 606 } 607 608 #ifdef CONFIG_HIBERNATE_CALLBACKS 609 static int zpci_restore(struct device *dev) 610 { 611 struct pci_dev *pdev = to_pci_dev(dev); 612 struct zpci_dev *zdev = to_zpci(pdev); 613 int ret = 0; 614 615 if (zdev->state != ZPCI_FN_STATE_ONLINE) 616 goto out; 617 618 ret = clp_enable_fh(zdev, ZPCI_NR_DMA_SPACES); 619 if (ret) 620 goto out; 621 622 zpci_map_resources(pdev); 623 zpci_register_ioat(zdev, 0, zdev->start_dma, zdev->end_dma, 624 (u64) zdev->dma_table); 625 626 out: 627 return ret; 628 } 629 630 static int zpci_freeze(struct device *dev) 631 { 632 struct pci_dev *pdev = to_pci_dev(dev); 633 struct zpci_dev *zdev = to_zpci(pdev); 634 635 if (zdev->state != ZPCI_FN_STATE_ONLINE) 636 return 0; 637 638 zpci_unregister_ioat(zdev, 0); 639 zpci_unmap_resources(pdev); 640 return clp_disable_fh(zdev); 641 } 642 643 struct dev_pm_ops pcibios_pm_ops = { 644 .thaw_noirq = zpci_restore, 645 .freeze_noirq = zpci_freeze, 646 .restore_noirq = zpci_restore, 647 .poweroff_noirq = zpci_freeze, 648 }; 649 #endif /* CONFIG_HIBERNATE_CALLBACKS */ 650 651 static int zpci_alloc_domain(struct zpci_dev *zdev) 652 { 653 if (zpci_unique_uid) { 654 zdev->domain = (u16) zdev->uid; 655 if (zdev->domain >= ZPCI_NR_DEVICES) 656 return 0; 657 658 spin_lock(&zpci_domain_lock); 659 if (test_bit(zdev->domain, zpci_domain)) { 660 spin_unlock(&zpci_domain_lock); 661 return -EEXIST; 662 } 663 set_bit(zdev->domain, zpci_domain); 664 spin_unlock(&zpci_domain_lock); 665 return 0; 666 } 667 668 spin_lock(&zpci_domain_lock); 669 zdev->domain = find_first_zero_bit(zpci_domain, ZPCI_NR_DEVICES); 670 if (zdev->domain == ZPCI_NR_DEVICES) { 671 spin_unlock(&zpci_domain_lock); 672 return -ENOSPC; 673 } 674 set_bit(zdev->domain, zpci_domain); 675 spin_unlock(&zpci_domain_lock); 676 return 0; 677 } 678 679 static void zpci_free_domain(struct zpci_dev *zdev) 680 { 681 if (zdev->domain >= ZPCI_NR_DEVICES) 682 return; 683 684 spin_lock(&zpci_domain_lock); 685 clear_bit(zdev->domain, zpci_domain); 686 spin_unlock(&zpci_domain_lock); 687 } 688 689 void pcibios_remove_bus(struct pci_bus *bus) 690 { 691 struct zpci_dev *zdev = get_zdev_by_bus(bus); 692 693 zpci_exit_slot(zdev); 694 zpci_cleanup_bus_resources(zdev); 695 zpci_destroy_iommu(zdev); 696 zpci_free_domain(zdev); 697 698 spin_lock(&zpci_list_lock); 699 list_del(&zdev->entry); 700 spin_unlock(&zpci_list_lock); 701 702 zpci_dbg(3, "rem fid:%x\n", zdev->fid); 703 kfree(zdev); 704 } 705 706 static int zpci_scan_bus(struct zpci_dev *zdev) 707 { 708 LIST_HEAD(resources); 709 int ret; 710 711 ret = zpci_setup_bus_resources(zdev, &resources); 712 if (ret) 713 goto error; 714 715 zdev->bus = pci_scan_root_bus(NULL, ZPCI_BUS_NR, &pci_root_ops, 716 zdev, &resources); 717 if (!zdev->bus) { 718 ret = -EIO; 719 goto error; 720 } 721 zdev->bus->max_bus_speed = zdev->max_bus_speed; 722 pci_bus_add_devices(zdev->bus); 723 return 0; 724 725 error: 726 zpci_cleanup_bus_resources(zdev); 727 pci_free_resource_list(&resources); 728 return ret; 729 } 730 731 int zpci_enable_device(struct zpci_dev *zdev) 732 { 733 int rc; 734 735 rc = clp_enable_fh(zdev, ZPCI_NR_DMA_SPACES); 736 if (rc) 737 goto out; 738 739 rc = zpci_dma_init_device(zdev); 740 if (rc) 741 goto out_dma; 742 743 zdev->state = ZPCI_FN_STATE_ONLINE; 744 return 0; 745 746 out_dma: 747 clp_disable_fh(zdev); 748 out: 749 return rc; 750 } 751 EXPORT_SYMBOL_GPL(zpci_enable_device); 752 753 int zpci_disable_device(struct zpci_dev *zdev) 754 { 755 zpci_dma_exit_device(zdev); 756 return clp_disable_fh(zdev); 757 } 758 EXPORT_SYMBOL_GPL(zpci_disable_device); 759 760 int zpci_create_device(struct zpci_dev *zdev) 761 { 762 int rc; 763 764 rc = zpci_alloc_domain(zdev); 765 if (rc) 766 goto out; 767 768 rc = zpci_init_iommu(zdev); 769 if (rc) 770 goto out_free; 771 772 mutex_init(&zdev->lock); 773 if (zdev->state == ZPCI_FN_STATE_CONFIGURED) { 774 rc = zpci_enable_device(zdev); 775 if (rc) 776 goto out_destroy_iommu; 777 } 778 rc = zpci_scan_bus(zdev); 779 if (rc) 780 goto out_disable; 781 782 spin_lock(&zpci_list_lock); 783 list_add_tail(&zdev->entry, &zpci_list); 784 spin_unlock(&zpci_list_lock); 785 786 zpci_init_slot(zdev); 787 788 return 0; 789 790 out_disable: 791 if (zdev->state == ZPCI_FN_STATE_ONLINE) 792 zpci_disable_device(zdev); 793 out_destroy_iommu: 794 zpci_destroy_iommu(zdev); 795 out_free: 796 zpci_free_domain(zdev); 797 out: 798 return rc; 799 } 800 801 void zpci_remove_device(struct zpci_dev *zdev) 802 { 803 if (!zdev->bus) 804 return; 805 806 pci_stop_root_bus(zdev->bus); 807 pci_remove_root_bus(zdev->bus); 808 } 809 810 int zpci_report_error(struct pci_dev *pdev, 811 struct zpci_report_error_header *report) 812 { 813 struct zpci_dev *zdev = to_zpci(pdev); 814 815 return sclp_pci_report(report, zdev->fh, zdev->fid); 816 } 817 EXPORT_SYMBOL(zpci_report_error); 818 819 static int zpci_mem_init(void) 820 { 821 BUILD_BUG_ON(!is_power_of_2(__alignof__(struct zpci_fmb)) || 822 __alignof__(struct zpci_fmb) < sizeof(struct zpci_fmb)); 823 824 zdev_fmb_cache = kmem_cache_create("PCI_FMB_cache", sizeof(struct zpci_fmb), 825 __alignof__(struct zpci_fmb), 0, NULL); 826 if (!zdev_fmb_cache) 827 goto error_fmb; 828 829 zpci_iomap_start = kcalloc(ZPCI_IOMAP_ENTRIES, 830 sizeof(*zpci_iomap_start), GFP_KERNEL); 831 if (!zpci_iomap_start) 832 goto error_iomap; 833 834 zpci_iomap_bitmap = kcalloc(BITS_TO_LONGS(ZPCI_IOMAP_ENTRIES), 835 sizeof(*zpci_iomap_bitmap), GFP_KERNEL); 836 if (!zpci_iomap_bitmap) 837 goto error_iomap_bitmap; 838 839 return 0; 840 error_iomap_bitmap: 841 kfree(zpci_iomap_start); 842 error_iomap: 843 kmem_cache_destroy(zdev_fmb_cache); 844 error_fmb: 845 return -ENOMEM; 846 } 847 848 static void zpci_mem_exit(void) 849 { 850 kfree(zpci_iomap_bitmap); 851 kfree(zpci_iomap_start); 852 kmem_cache_destroy(zdev_fmb_cache); 853 } 854 855 static unsigned int s390_pci_probe __initdata = 1; 856 static unsigned int s390_pci_no_mio __initdata; 857 unsigned int s390_pci_force_floating __initdata; 858 static unsigned int s390_pci_initialized; 859 860 char * __init pcibios_setup(char *str) 861 { 862 if (!strcmp(str, "off")) { 863 s390_pci_probe = 0; 864 return NULL; 865 } 866 if (!strcmp(str, "nomio")) { 867 s390_pci_no_mio = 1; 868 return NULL; 869 } 870 if (!strcmp(str, "force_floating")) { 871 s390_pci_force_floating = 1; 872 return NULL; 873 } 874 return str; 875 } 876 877 bool zpci_is_enabled(void) 878 { 879 return s390_pci_initialized; 880 } 881 882 static int __init pci_base_init(void) 883 { 884 int rc; 885 886 if (!s390_pci_probe) 887 return 0; 888 889 if (!test_facility(69) || !test_facility(71)) 890 return 0; 891 892 if (test_facility(153) && !s390_pci_no_mio) 893 static_branch_enable(&have_mio); 894 895 rc = zpci_debug_init(); 896 if (rc) 897 goto out; 898 899 rc = zpci_mem_init(); 900 if (rc) 901 goto out_mem; 902 903 rc = zpci_irq_init(); 904 if (rc) 905 goto out_irq; 906 907 rc = zpci_dma_init(); 908 if (rc) 909 goto out_dma; 910 911 rc = clp_scan_pci_devices(); 912 if (rc) 913 goto out_find; 914 915 s390_pci_initialized = 1; 916 return 0; 917 918 out_find: 919 zpci_dma_exit(); 920 out_dma: 921 zpci_irq_exit(); 922 out_irq: 923 zpci_mem_exit(); 924 out_mem: 925 zpci_debug_exit(); 926 out: 927 return rc; 928 } 929 subsys_initcall_sync(pci_base_init); 930 931 void zpci_rescan(void) 932 { 933 if (zpci_is_enabled()) 934 clp_rescan_pci_devices_simple(); 935 } 936