xref: /openbmc/linux/arch/s390/pci/pci.c (revision ce746d43) 
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright IBM Corp. 2012
4  *
5  * Author(s):
6  *   Jan Glauber <jang@linux.vnet.ibm.com>
7  *
8  * The System z PCI code is a rewrite from a prototype by
9  * the following people (Kudoz!):
10  *   Alexander Schmidt
11  *   Christoph Raisch
12  *   Hannes Hering
13  *   Hoang-Nam Nguyen
14  *   Jan-Bernd Themann
15  *   Stefan Roscher
16  *   Thomas Klein
17  */
18 
19 #define KMSG_COMPONENT "zpci"
20 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
21 
22 #include <linux/kernel.h>
23 #include <linux/slab.h>
24 #include <linux/err.h>
25 #include <linux/export.h>
26 #include <linux/delay.h>
27 #include <linux/seq_file.h>
28 #include <linux/jump_label.h>
29 #include <linux/pci.h>
30 #include <linux/printk.h>
31 
32 #include <asm/isc.h>
33 #include <asm/airq.h>
34 #include <asm/facility.h>
35 #include <asm/pci_insn.h>
36 #include <asm/pci_clp.h>
37 #include <asm/pci_dma.h>
38 
39 #include "pci_bus.h"
40 
41 /* list of all detected zpci devices */
42 static LIST_HEAD(zpci_list);
43 static DEFINE_SPINLOCK(zpci_list_lock);
44 
45 static DECLARE_BITMAP(zpci_domain, ZPCI_DOMAIN_BITMAP_SIZE);
46 static DEFINE_SPINLOCK(zpci_domain_lock);
47 
48 #define ZPCI_IOMAP_ENTRIES						\
49 	min(((unsigned long) ZPCI_NR_DEVICES * PCI_STD_NUM_BARS / 2),	\
50 	    ZPCI_IOMAP_MAX_ENTRIES)
51 
52 unsigned int s390_pci_no_rid;
53 
54 static DEFINE_SPINLOCK(zpci_iomap_lock);
55 static unsigned long *zpci_iomap_bitmap;
56 struct zpci_iomap_entry *zpci_iomap_start;
57 EXPORT_SYMBOL_GPL(zpci_iomap_start);
58 
59 DEFINE_STATIC_KEY_FALSE(have_mio);
60 
61 static struct kmem_cache *zdev_fmb_cache;
62 
63 struct zpci_dev *get_zdev_by_fid(u32 fid)
64 {
65 	struct zpci_dev *tmp, *zdev = NULL;
66 
67 	spin_lock(&zpci_list_lock);
68 	list_for_each_entry(tmp, &zpci_list, entry) {
69 		if (tmp->fid == fid) {
70 			zdev = tmp;
71 			break;
72 		}
73 	}
74 	spin_unlock(&zpci_list_lock);
75 	return zdev;
76 }
77 
78 void zpci_remove_reserved_devices(void)
79 {
80 	struct zpci_dev *tmp, *zdev;
81 	enum zpci_state state;
82 	LIST_HEAD(remove);
83 
84 	spin_lock(&zpci_list_lock);
85 	list_for_each_entry_safe(zdev, tmp, &zpci_list, entry) {
86 		if (zdev->state == ZPCI_FN_STATE_STANDBY &&
87 		    !clp_get_state(zdev->fid, &state) &&
88 		    state == ZPCI_FN_STATE_RESERVED)
89 			list_move_tail(&zdev->entry, &remove);
90 	}
91 	spin_unlock(&zpci_list_lock);
92 
93 	list_for_each_entry_safe(zdev, tmp, &remove, entry)
94 		zpci_zdev_put(zdev);
95 }
96 
97 int pci_domain_nr(struct pci_bus *bus)
98 {
99 	return ((struct zpci_bus *) bus->sysdata)->domain_nr;
100 }
101 EXPORT_SYMBOL_GPL(pci_domain_nr);
102 
103 int pci_proc_domain(struct pci_bus *bus)
104 {
105 	return pci_domain_nr(bus);
106 }
107 EXPORT_SYMBOL_GPL(pci_proc_domain);
108 
109 /* Modify PCI: Register I/O address translation parameters */
110 int zpci_register_ioat(struct zpci_dev *zdev, u8 dmaas,
111 		       u64 base, u64 limit, u64 iota)
112 {
113 	u64 req = ZPCI_CREATE_REQ(zdev->fh, dmaas, ZPCI_MOD_FC_REG_IOAT);
114 	struct zpci_fib fib = {0};
115 	u8 status;
116 
117 	WARN_ON_ONCE(iota & 0x3fff);
118 	fib.pba = base;
119 	fib.pal = limit;
120 	fib.iota = iota | ZPCI_IOTA_RTTO_FLAG;
121 	return zpci_mod_fc(req, &fib, &status) ? -EIO : 0;
122 }
123 
124 /* Modify PCI: Unregister I/O address translation parameters */
125 int zpci_unregister_ioat(struct zpci_dev *zdev, u8 dmaas)
126 {
127 	u64 req = ZPCI_CREATE_REQ(zdev->fh, dmaas, ZPCI_MOD_FC_DEREG_IOAT);
128 	struct zpci_fib fib = {0};
129 	u8 cc, status;
130 
131 	cc = zpci_mod_fc(req, &fib, &status);
132 	if (cc == 3) /* Function already gone. */
133 		cc = 0;
134 	return cc ? -EIO : 0;
135 }
136 
137 /* Modify PCI: Set PCI function measurement parameters */
138 int zpci_fmb_enable_device(struct zpci_dev *zdev)
139 {
140 	u64 req = ZPCI_CREATE_REQ(zdev->fh, 0, ZPCI_MOD_FC_SET_MEASURE);
141 	struct zpci_fib fib = {0};
142 	u8 cc, status;
143 
144 	if (zdev->fmb || sizeof(*zdev->fmb) < zdev->fmb_length)
145 		return -EINVAL;
146 
147 	zdev->fmb = kmem_cache_zalloc(zdev_fmb_cache, GFP_KERNEL);
148 	if (!zdev->fmb)
149 		return -ENOMEM;
150 	WARN_ON((u64) zdev->fmb & 0xf);
151 
152 	/* reset software counters */
153 	atomic64_set(&zdev->allocated_pages, 0);
154 	atomic64_set(&zdev->mapped_pages, 0);
155 	atomic64_set(&zdev->unmapped_pages, 0);
156 
157 	fib.fmb_addr = virt_to_phys(zdev->fmb);
158 	cc = zpci_mod_fc(req, &fib, &status);
159 	if (cc) {
160 		kmem_cache_free(zdev_fmb_cache, zdev->fmb);
161 		zdev->fmb = NULL;
162 	}
163 	return cc ? -EIO : 0;
164 }
165 
166 /* Modify PCI: Disable PCI function measurement */
167 int zpci_fmb_disable_device(struct zpci_dev *zdev)
168 {
169 	u64 req = ZPCI_CREATE_REQ(zdev->fh, 0, ZPCI_MOD_FC_SET_MEASURE);
170 	struct zpci_fib fib = {0};
171 	u8 cc, status;
172 
173 	if (!zdev->fmb)
174 		return -EINVAL;
175 
176 	/* Function measurement is disabled if fmb address is zero */
177 	cc = zpci_mod_fc(req, &fib, &status);
178 	if (cc == 3) /* Function already gone. */
179 		cc = 0;
180 
181 	if (!cc) {
182 		kmem_cache_free(zdev_fmb_cache, zdev->fmb);
183 		zdev->fmb = NULL;
184 	}
185 	return cc ? -EIO : 0;
186 }
187 
188 static int zpci_cfg_load(struct zpci_dev *zdev, int offset, u32 *val, u8 len)
189 {
190 	u64 req = ZPCI_CREATE_REQ(zdev->fh, ZPCI_PCIAS_CFGSPC, len);
191 	u64 data;
192 	int rc;
193 
194 	rc = __zpci_load(&data, req, offset);
195 	if (!rc) {
196 		data = le64_to_cpu((__force __le64) data);
197 		data >>= (8 - len) * 8;
198 		*val = (u32) data;
199 	} else
200 		*val = 0xffffffff;
201 	return rc;
202 }
203 
204 static int zpci_cfg_store(struct zpci_dev *zdev, int offset, u32 val, u8 len)
205 {
206 	u64 req = ZPCI_CREATE_REQ(zdev->fh, ZPCI_PCIAS_CFGSPC, len);
207 	u64 data = val;
208 	int rc;
209 
210 	data <<= (8 - len) * 8;
211 	data = (__force u64) cpu_to_le64(data);
212 	rc = __zpci_store(data, req, offset);
213 	return rc;
214 }
215 
216 resource_size_t pcibios_align_resource(void *data, const struct resource *res,
217 				       resource_size_t size,
218 				       resource_size_t align)
219 {
220 	return 0;
221 }
222 
223 /* combine single writes by using store-block insn */
224 void __iowrite64_copy(void __iomem *to, const void *from, size_t count)
225 {
226        zpci_memcpy_toio(to, from, count);
227 }
228 
229 void __iomem *ioremap(phys_addr_t addr, size_t size)
230 {
231 	unsigned long offset, vaddr;
232 	struct vm_struct *area;
233 	phys_addr_t last_addr;
234 
235 	last_addr = addr + size - 1;
236 	if (!size || last_addr < addr)
237 		return NULL;
238 
239 	if (!static_branch_unlikely(&have_mio))
240 		return (void __iomem *) addr;
241 
242 	offset = addr & ~PAGE_MASK;
243 	addr &= PAGE_MASK;
244 	size = PAGE_ALIGN(size + offset);
245 	area = get_vm_area(size, VM_IOREMAP);
246 	if (!area)
247 		return NULL;
248 
249 	vaddr = (unsigned long) area->addr;
250 	if (ioremap_page_range(vaddr, vaddr + size, addr, PAGE_KERNEL)) {
251 		free_vm_area(area);
252 		return NULL;
253 	}
254 	return (void __iomem *) ((unsigned long) area->addr + offset);
255 }
256 EXPORT_SYMBOL(ioremap);
257 
258 void iounmap(volatile void __iomem *addr)
259 {
260 	if (static_branch_likely(&have_mio))
261 		vunmap((__force void *) ((unsigned long) addr & PAGE_MASK));
262 }
263 EXPORT_SYMBOL(iounmap);
264 
265 /* Create a virtual mapping cookie for a PCI BAR */
266 static void __iomem *pci_iomap_range_fh(struct pci_dev *pdev, int bar,
267 					unsigned long offset, unsigned long max)
268 {
269 	struct zpci_dev *zdev =	to_zpci(pdev);
270 	int idx;
271 
272 	idx = zdev->bars[bar].map_idx;
273 	spin_lock(&zpci_iomap_lock);
274 	/* Detect overrun */
275 	WARN_ON(!++zpci_iomap_start[idx].count);
276 	zpci_iomap_start[idx].fh = zdev->fh;
277 	zpci_iomap_start[idx].bar = bar;
278 	spin_unlock(&zpci_iomap_lock);
279 
280 	return (void __iomem *) ZPCI_ADDR(idx) + offset;
281 }
282 
283 static void __iomem *pci_iomap_range_mio(struct pci_dev *pdev, int bar,
284 					 unsigned long offset,
285 					 unsigned long max)
286 {
287 	unsigned long barsize = pci_resource_len(pdev, bar);
288 	struct zpci_dev *zdev = to_zpci(pdev);
289 	void __iomem *iova;
290 
291 	iova = ioremap((unsigned long) zdev->bars[bar].mio_wt, barsize);
292 	return iova ? iova + offset : iova;
293 }
294 
295 void __iomem *pci_iomap_range(struct pci_dev *pdev, int bar,
296 			      unsigned long offset, unsigned long max)
297 {
298 	if (bar >= PCI_STD_NUM_BARS || !pci_resource_len(pdev, bar))
299 		return NULL;
300 
301 	if (static_branch_likely(&have_mio))
302 		return pci_iomap_range_mio(pdev, bar, offset, max);
303 	else
304 		return pci_iomap_range_fh(pdev, bar, offset, max);
305 }
306 EXPORT_SYMBOL(pci_iomap_range);
307 
308 void __iomem *pci_iomap(struct pci_dev *dev, int bar, unsigned long maxlen)
309 {
310 	return pci_iomap_range(dev, bar, 0, maxlen);
311 }
312 EXPORT_SYMBOL(pci_iomap);
313 
314 static void __iomem *pci_iomap_wc_range_mio(struct pci_dev *pdev, int bar,
315 					    unsigned long offset, unsigned long max)
316 {
317 	unsigned long barsize = pci_resource_len(pdev, bar);
318 	struct zpci_dev *zdev = to_zpci(pdev);
319 	void __iomem *iova;
320 
321 	iova = ioremap((unsigned long) zdev->bars[bar].mio_wb, barsize);
322 	return iova ? iova + offset : iova;
323 }
324 
325 void __iomem *pci_iomap_wc_range(struct pci_dev *pdev, int bar,
326 				 unsigned long offset, unsigned long max)
327 {
328 	if (bar >= PCI_STD_NUM_BARS || !pci_resource_len(pdev, bar))
329 		return NULL;
330 
331 	if (static_branch_likely(&have_mio))
332 		return pci_iomap_wc_range_mio(pdev, bar, offset, max);
333 	else
334 		return pci_iomap_range_fh(pdev, bar, offset, max);
335 }
336 EXPORT_SYMBOL(pci_iomap_wc_range);
337 
338 void __iomem *pci_iomap_wc(struct pci_dev *dev, int bar, unsigned long maxlen)
339 {
340 	return pci_iomap_wc_range(dev, bar, 0, maxlen);
341 }
342 EXPORT_SYMBOL(pci_iomap_wc);
343 
344 static void pci_iounmap_fh(struct pci_dev *pdev, void __iomem *addr)
345 {
346 	unsigned int idx = ZPCI_IDX(addr);
347 
348 	spin_lock(&zpci_iomap_lock);
349 	/* Detect underrun */
350 	WARN_ON(!zpci_iomap_start[idx].count);
351 	if (!--zpci_iomap_start[idx].count) {
352 		zpci_iomap_start[idx].fh = 0;
353 		zpci_iomap_start[idx].bar = 0;
354 	}
355 	spin_unlock(&zpci_iomap_lock);
356 }
357 
358 static void pci_iounmap_mio(struct pci_dev *pdev, void __iomem *addr)
359 {
360 	iounmap(addr);
361 }
362 
363 void pci_iounmap(struct pci_dev *pdev, void __iomem *addr)
364 {
365 	if (static_branch_likely(&have_mio))
366 		pci_iounmap_mio(pdev, addr);
367 	else
368 		pci_iounmap_fh(pdev, addr);
369 }
370 EXPORT_SYMBOL(pci_iounmap);
371 
372 static int pci_read(struct pci_bus *bus, unsigned int devfn, int where,
373 		    int size, u32 *val)
374 {
375 	struct zpci_dev *zdev = get_zdev_by_bus(bus, devfn);
376 
377 	return (zdev) ? zpci_cfg_load(zdev, where, val, size) : -ENODEV;
378 }
379 
380 static int pci_write(struct pci_bus *bus, unsigned int devfn, int where,
381 		     int size, u32 val)
382 {
383 	struct zpci_dev *zdev = get_zdev_by_bus(bus, devfn);
384 
385 	return (zdev) ? zpci_cfg_store(zdev, where, val, size) : -ENODEV;
386 }
387 
388 static struct pci_ops pci_root_ops = {
389 	.read = pci_read,
390 	.write = pci_write,
391 };
392 
393 #ifdef CONFIG_PCI_IOV
394 static struct resource iov_res = {
395 	.name	= "PCI IOV res",
396 	.start	= 0,
397 	.end	= -1,
398 	.flags	= IORESOURCE_MEM,
399 };
400 #endif
401 
402 static void zpci_map_resources(struct pci_dev *pdev)
403 {
404 	struct zpci_dev *zdev = to_zpci(pdev);
405 	resource_size_t len;
406 	int i;
407 
408 	for (i = 0; i < PCI_STD_NUM_BARS; i++) {
409 		len = pci_resource_len(pdev, i);
410 		if (!len)
411 			continue;
412 
413 		if (zpci_use_mio(zdev))
414 			pdev->resource[i].start =
415 				(resource_size_t __force) zdev->bars[i].mio_wt;
416 		else
417 			pdev->resource[i].start = (resource_size_t __force)
418 				pci_iomap_range_fh(pdev, i, 0, 0);
419 		pdev->resource[i].end = pdev->resource[i].start + len - 1;
420 	}
421 
422 #ifdef CONFIG_PCI_IOV
423 	for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
424 		int bar = i + PCI_IOV_RESOURCES;
425 
426 		len = pci_resource_len(pdev, bar);
427 		if (!len)
428 			continue;
429 		pdev->resource[bar].parent = &iov_res;
430 	}
431 #endif
432 }
433 
434 static void zpci_unmap_resources(struct pci_dev *pdev)
435 {
436 	struct zpci_dev *zdev = to_zpci(pdev);
437 	resource_size_t len;
438 	int i;
439 
440 	if (zpci_use_mio(zdev))
441 		return;
442 
443 	for (i = 0; i < PCI_STD_NUM_BARS; i++) {
444 		len = pci_resource_len(pdev, i);
445 		if (!len)
446 			continue;
447 		pci_iounmap_fh(pdev, (void __iomem __force *)
448 			       pdev->resource[i].start);
449 	}
450 }
451 
452 static int zpci_alloc_iomap(struct zpci_dev *zdev)
453 {
454 	unsigned long entry;
455 
456 	spin_lock(&zpci_iomap_lock);
457 	entry = find_first_zero_bit(zpci_iomap_bitmap, ZPCI_IOMAP_ENTRIES);
458 	if (entry == ZPCI_IOMAP_ENTRIES) {
459 		spin_unlock(&zpci_iomap_lock);
460 		return -ENOSPC;
461 	}
462 	set_bit(entry, zpci_iomap_bitmap);
463 	spin_unlock(&zpci_iomap_lock);
464 	return entry;
465 }
466 
467 static void zpci_free_iomap(struct zpci_dev *zdev, int entry)
468 {
469 	spin_lock(&zpci_iomap_lock);
470 	memset(&zpci_iomap_start[entry], 0, sizeof(struct zpci_iomap_entry));
471 	clear_bit(entry, zpci_iomap_bitmap);
472 	spin_unlock(&zpci_iomap_lock);
473 }
474 
475 static struct resource *__alloc_res(struct zpci_dev *zdev, unsigned long start,
476 				    unsigned long size, unsigned long flags)
477 {
478 	struct resource *r;
479 
480 	r = kzalloc(sizeof(*r), GFP_KERNEL);
481 	if (!r)
482 		return NULL;
483 
484 	r->start = start;
485 	r->end = r->start + size - 1;
486 	r->flags = flags;
487 	r->name = zdev->res_name;
488 
489 	if (request_resource(&iomem_resource, r)) {
490 		kfree(r);
491 		return NULL;
492 	}
493 	return r;
494 }
495 
496 int zpci_setup_bus_resources(struct zpci_dev *zdev,
497 			     struct list_head *resources)
498 {
499 	unsigned long addr, size, flags;
500 	struct resource *res;
501 	int i, entry;
502 
503 	snprintf(zdev->res_name, sizeof(zdev->res_name),
504 		 "PCI Bus %04x:%02x", zdev->uid, ZPCI_BUS_NR);
505 
506 	for (i = 0; i < PCI_STD_NUM_BARS; i++) {
507 		if (!zdev->bars[i].size)
508 			continue;
509 		entry = zpci_alloc_iomap(zdev);
510 		if (entry < 0)
511 			return entry;
512 		zdev->bars[i].map_idx = entry;
513 
514 		/* only MMIO is supported */
515 		flags = IORESOURCE_MEM;
516 		if (zdev->bars[i].val & 8)
517 			flags |= IORESOURCE_PREFETCH;
518 		if (zdev->bars[i].val & 4)
519 			flags |= IORESOURCE_MEM_64;
520 
521 		if (zpci_use_mio(zdev))
522 			addr = (unsigned long) zdev->bars[i].mio_wt;
523 		else
524 			addr = ZPCI_ADDR(entry);
525 		size = 1UL << zdev->bars[i].size;
526 
527 		res = __alloc_res(zdev, addr, size, flags);
528 		if (!res) {
529 			zpci_free_iomap(zdev, entry);
530 			return -ENOMEM;
531 		}
532 		zdev->bars[i].res = res;
533 		pci_add_resource(resources, res);
534 	}
535 
536 	return 0;
537 }
538 
539 static void zpci_cleanup_bus_resources(struct zpci_dev *zdev)
540 {
541 	int i;
542 
543 	for (i = 0; i < PCI_STD_NUM_BARS; i++) {
544 		if (!zdev->bars[i].size || !zdev->bars[i].res)
545 			continue;
546 
547 		zpci_free_iomap(zdev, zdev->bars[i].map_idx);
548 		release_resource(zdev->bars[i].res);
549 		kfree(zdev->bars[i].res);
550 	}
551 }
552 
553 int pcibios_add_device(struct pci_dev *pdev)
554 {
555 	struct resource *res;
556 	int i;
557 
558 	if (pdev->is_physfn)
559 		pdev->no_vf_scan = 1;
560 
561 	pdev->dev.groups = zpci_attr_groups;
562 	pdev->dev.dma_ops = &s390_pci_dma_ops;
563 	zpci_map_resources(pdev);
564 
565 	for (i = 0; i < PCI_STD_NUM_BARS; i++) {
566 		res = &pdev->resource[i];
567 		if (res->parent || !res->flags)
568 			continue;
569 		pci_claim_resource(pdev, i);
570 	}
571 
572 	return 0;
573 }
574 
575 void pcibios_release_device(struct pci_dev *pdev)
576 {
577 	zpci_unmap_resources(pdev);
578 }
579 
580 int pcibios_enable_device(struct pci_dev *pdev, int mask)
581 {
582 	struct zpci_dev *zdev = to_zpci(pdev);
583 
584 	zpci_debug_init_device(zdev, dev_name(&pdev->dev));
585 	zpci_fmb_enable_device(zdev);
586 
587 	return pci_enable_resources(pdev, mask);
588 }
589 
590 void pcibios_disable_device(struct pci_dev *pdev)
591 {
592 	struct zpci_dev *zdev = to_zpci(pdev);
593 
594 	zpci_fmb_disable_device(zdev);
595 	zpci_debug_exit_device(zdev);
596 }
597 
598 static int __zpci_register_domain(int domain)
599 {
600 	spin_lock(&zpci_domain_lock);
601 	if (test_bit(domain, zpci_domain)) {
602 		spin_unlock(&zpci_domain_lock);
603 		pr_err("Domain %04x is already assigned\n", domain);
604 		return -EEXIST;
605 	}
606 	set_bit(domain, zpci_domain);
607 	spin_unlock(&zpci_domain_lock);
608 	return domain;
609 }
610 
611 static int __zpci_alloc_domain(void)
612 {
613 	int domain;
614 
615 	spin_lock(&zpci_domain_lock);
616 	/*
617 	 * We can always auto allocate domains below ZPCI_NR_DEVICES.
618 	 * There is either a free domain or we have reached the maximum in
619 	 * which case we would have bailed earlier.
620 	 */
621 	domain = find_first_zero_bit(zpci_domain, ZPCI_NR_DEVICES);
622 	set_bit(domain, zpci_domain);
623 	spin_unlock(&zpci_domain_lock);
624 	return domain;
625 }
626 
627 int zpci_alloc_domain(int domain)
628 {
629 	if (zpci_unique_uid) {
630 		if (domain)
631 			return __zpci_register_domain(domain);
632 		pr_warn("UID checking was active but no UID is provided: switching to automatic domain allocation\n");
633 		update_uid_checking(false);
634 	}
635 	return __zpci_alloc_domain();
636 }
637 
638 void zpci_free_domain(int domain)
639 {
640 	spin_lock(&zpci_domain_lock);
641 	clear_bit(domain, zpci_domain);
642 	spin_unlock(&zpci_domain_lock);
643 }
644 
645 
646 int zpci_enable_device(struct zpci_dev *zdev)
647 {
648 	int rc;
649 
650 	rc = clp_enable_fh(zdev, ZPCI_NR_DMA_SPACES);
651 	if (rc)
652 		goto out;
653 
654 	rc = zpci_dma_init_device(zdev);
655 	if (rc)
656 		goto out_dma;
657 
658 	zdev->state = ZPCI_FN_STATE_ONLINE;
659 	return 0;
660 
661 out_dma:
662 	clp_disable_fh(zdev);
663 out:
664 	return rc;
665 }
666 EXPORT_SYMBOL_GPL(zpci_enable_device);
667 
668 int zpci_disable_device(struct zpci_dev *zdev)
669 {
670 	zpci_dma_exit_device(zdev);
671 	return clp_disable_fh(zdev);
672 }
673 EXPORT_SYMBOL_GPL(zpci_disable_device);
674 
675 int zpci_create_device(struct zpci_dev *zdev)
676 {
677 	int rc;
678 
679 	kref_init(&zdev->kref);
680 
681 	spin_lock(&zpci_list_lock);
682 	list_add_tail(&zdev->entry, &zpci_list);
683 	spin_unlock(&zpci_list_lock);
684 
685 	rc = zpci_init_iommu(zdev);
686 	if (rc)
687 		goto out;
688 
689 	mutex_init(&zdev->lock);
690 	if (zdev->state == ZPCI_FN_STATE_CONFIGURED) {
691 		rc = zpci_enable_device(zdev);
692 		if (rc)
693 			goto out_destroy_iommu;
694 	}
695 
696 	rc = zpci_bus_device_register(zdev, &pci_root_ops);
697 	if (rc)
698 		goto out_disable;
699 
700 	return 0;
701 
702 out_disable:
703 	if (zdev->state == ZPCI_FN_STATE_ONLINE)
704 		zpci_disable_device(zdev);
705 
706 out_destroy_iommu:
707 	zpci_destroy_iommu(zdev);
708 out:
709 	spin_lock(&zpci_list_lock);
710 	list_del(&zdev->entry);
711 	spin_unlock(&zpci_list_lock);
712 	return rc;
713 }
714 
715 void zpci_release_device(struct kref *kref)
716 {
717 	struct zpci_dev *zdev = container_of(kref, struct zpci_dev, kref);
718 
719 	if (zdev->zbus->bus) {
720 		struct pci_dev *pdev;
721 
722 		pdev = pci_get_slot(zdev->zbus->bus, zdev->devfn);
723 		if (pdev)
724 			pci_stop_and_remove_bus_device_locked(pdev);
725 	}
726 
727 	switch (zdev->state) {
728 	case ZPCI_FN_STATE_ONLINE:
729 	case ZPCI_FN_STATE_CONFIGURED:
730 		zpci_disable_device(zdev);
731 		fallthrough;
732 	case ZPCI_FN_STATE_STANDBY:
733 		if (zdev->has_hp_slot)
734 			zpci_exit_slot(zdev);
735 		zpci_cleanup_bus_resources(zdev);
736 		zpci_bus_device_unregister(zdev);
737 		zpci_destroy_iommu(zdev);
738 		fallthrough;
739 	default:
740 		break;
741 	}
742 
743 	spin_lock(&zpci_list_lock);
744 	list_del(&zdev->entry);
745 	spin_unlock(&zpci_list_lock);
746 	zpci_dbg(3, "rem fid:%x\n", zdev->fid);
747 	kfree(zdev);
748 }
749 
750 int zpci_report_error(struct pci_dev *pdev,
751 		      struct zpci_report_error_header *report)
752 {
753 	struct zpci_dev *zdev = to_zpci(pdev);
754 
755 	return sclp_pci_report(report, zdev->fh, zdev->fid);
756 }
757 EXPORT_SYMBOL(zpci_report_error);
758 
759 static int zpci_mem_init(void)
760 {
761 	BUILD_BUG_ON(!is_power_of_2(__alignof__(struct zpci_fmb)) ||
762 		     __alignof__(struct zpci_fmb) < sizeof(struct zpci_fmb));
763 
764 	zdev_fmb_cache = kmem_cache_create("PCI_FMB_cache", sizeof(struct zpci_fmb),
765 					   __alignof__(struct zpci_fmb), 0, NULL);
766 	if (!zdev_fmb_cache)
767 		goto error_fmb;
768 
769 	zpci_iomap_start = kcalloc(ZPCI_IOMAP_ENTRIES,
770 				   sizeof(*zpci_iomap_start), GFP_KERNEL);
771 	if (!zpci_iomap_start)
772 		goto error_iomap;
773 
774 	zpci_iomap_bitmap = kcalloc(BITS_TO_LONGS(ZPCI_IOMAP_ENTRIES),
775 				    sizeof(*zpci_iomap_bitmap), GFP_KERNEL);
776 	if (!zpci_iomap_bitmap)
777 		goto error_iomap_bitmap;
778 
779 	return 0;
780 error_iomap_bitmap:
781 	kfree(zpci_iomap_start);
782 error_iomap:
783 	kmem_cache_destroy(zdev_fmb_cache);
784 error_fmb:
785 	return -ENOMEM;
786 }
787 
788 static void zpci_mem_exit(void)
789 {
790 	kfree(zpci_iomap_bitmap);
791 	kfree(zpci_iomap_start);
792 	kmem_cache_destroy(zdev_fmb_cache);
793 }
794 
795 static unsigned int s390_pci_probe __initdata = 1;
796 static unsigned int s390_pci_no_mio __initdata;
797 unsigned int s390_pci_force_floating __initdata;
798 static unsigned int s390_pci_initialized;
799 
800 char * __init pcibios_setup(char *str)
801 {
802 	if (!strcmp(str, "off")) {
803 		s390_pci_probe = 0;
804 		return NULL;
805 	}
806 	if (!strcmp(str, "nomio")) {
807 		s390_pci_no_mio = 1;
808 		return NULL;
809 	}
810 	if (!strcmp(str, "force_floating")) {
811 		s390_pci_force_floating = 1;
812 		return NULL;
813 	}
814 	if (!strcmp(str, "norid")) {
815 		s390_pci_no_rid = 1;
816 		return NULL;
817 	}
818 	return str;
819 }
820 
821 bool zpci_is_enabled(void)
822 {
823 	return s390_pci_initialized;
824 }
825 
826 static int __init pci_base_init(void)
827 {
828 	int rc;
829 
830 	if (!s390_pci_probe)
831 		return 0;
832 
833 	if (!test_facility(69) || !test_facility(71))
834 		return 0;
835 
836 	if (test_facility(153) && !s390_pci_no_mio) {
837 		static_branch_enable(&have_mio);
838 		ctl_set_bit(2, 5);
839 	}
840 
841 	rc = zpci_debug_init();
842 	if (rc)
843 		goto out;
844 
845 	rc = zpci_mem_init();
846 	if (rc)
847 		goto out_mem;
848 
849 	rc = zpci_irq_init();
850 	if (rc)
851 		goto out_irq;
852 
853 	rc = zpci_dma_init();
854 	if (rc)
855 		goto out_dma;
856 
857 	rc = clp_scan_pci_devices();
858 	if (rc)
859 		goto out_find;
860 
861 	s390_pci_initialized = 1;
862 	return 0;
863 
864 out_find:
865 	zpci_dma_exit();
866 out_dma:
867 	zpci_irq_exit();
868 out_irq:
869 	zpci_mem_exit();
870 out_mem:
871 	zpci_debug_exit();
872 out:
873 	return rc;
874 }
875 subsys_initcall_sync(pci_base_init);
876 
877 void zpci_rescan(void)
878 {
879 	if (zpci_is_enabled())
880 		clp_rescan_pci_devices_simple(NULL);
881 }
882