xref: /openbmc/linux/arch/s390/pci/pci_dma.c (revision 2a81ada3)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright IBM Corp. 2012
4  *
5  * Author(s):
6  *   Jan Glauber <jang@linux.vnet.ibm.com>
7  */
8 
9 #include <linux/kernel.h>
10 #include <linux/slab.h>
11 #include <linux/export.h>
12 #include <linux/iommu-helper.h>
13 #include <linux/dma-map-ops.h>
14 #include <linux/vmalloc.h>
15 #include <linux/pci.h>
16 #include <asm/pci_dma.h>
17 
18 static struct kmem_cache *dma_region_table_cache;
19 static struct kmem_cache *dma_page_table_cache;
20 static int s390_iommu_strict;
21 static u64 s390_iommu_aperture;
22 static u32 s390_iommu_aperture_factor = 1;
23 
24 static int zpci_refresh_global(struct zpci_dev *zdev)
25 {
26 	return zpci_refresh_trans((u64) zdev->fh << 32, zdev->start_dma,
27 				  zdev->iommu_pages * PAGE_SIZE);
28 }
29 
30 unsigned long *dma_alloc_cpu_table(void)
31 {
32 	unsigned long *table, *entry;
33 
34 	table = kmem_cache_alloc(dma_region_table_cache, GFP_ATOMIC);
35 	if (!table)
36 		return NULL;
37 
38 	for (entry = table; entry < table + ZPCI_TABLE_ENTRIES; entry++)
39 		*entry = ZPCI_TABLE_INVALID;
40 	return table;
41 }
42 
43 static void dma_free_cpu_table(void *table)
44 {
45 	kmem_cache_free(dma_region_table_cache, table);
46 }
47 
48 static unsigned long *dma_alloc_page_table(void)
49 {
50 	unsigned long *table, *entry;
51 
52 	table = kmem_cache_alloc(dma_page_table_cache, GFP_ATOMIC);
53 	if (!table)
54 		return NULL;
55 
56 	for (entry = table; entry < table + ZPCI_PT_ENTRIES; entry++)
57 		*entry = ZPCI_PTE_INVALID;
58 	return table;
59 }
60 
61 static void dma_free_page_table(void *table)
62 {
63 	kmem_cache_free(dma_page_table_cache, table);
64 }
65 
66 static unsigned long *dma_get_seg_table_origin(unsigned long *rtep)
67 {
68 	unsigned long old_rte, rte;
69 	unsigned long *sto;
70 
71 	rte = READ_ONCE(*rtep);
72 	if (reg_entry_isvalid(rte)) {
73 		sto = get_rt_sto(rte);
74 	} else {
75 		sto = dma_alloc_cpu_table();
76 		if (!sto)
77 			return NULL;
78 
79 		set_rt_sto(&rte, virt_to_phys(sto));
80 		validate_rt_entry(&rte);
81 		entry_clr_protected(&rte);
82 
83 		old_rte = cmpxchg(rtep, ZPCI_TABLE_INVALID, rte);
84 		if (old_rte != ZPCI_TABLE_INVALID) {
85 			/* Somone else was faster, use theirs */
86 			dma_free_cpu_table(sto);
87 			sto = get_rt_sto(old_rte);
88 		}
89 	}
90 	return sto;
91 }
92 
93 static unsigned long *dma_get_page_table_origin(unsigned long *step)
94 {
95 	unsigned long old_ste, ste;
96 	unsigned long *pto;
97 
98 	ste = READ_ONCE(*step);
99 	if (reg_entry_isvalid(ste)) {
100 		pto = get_st_pto(ste);
101 	} else {
102 		pto = dma_alloc_page_table();
103 		if (!pto)
104 			return NULL;
105 		set_st_pto(&ste, virt_to_phys(pto));
106 		validate_st_entry(&ste);
107 		entry_clr_protected(&ste);
108 
109 		old_ste = cmpxchg(step, ZPCI_TABLE_INVALID, ste);
110 		if (old_ste != ZPCI_TABLE_INVALID) {
111 			/* Somone else was faster, use theirs */
112 			dma_free_page_table(pto);
113 			pto = get_st_pto(old_ste);
114 		}
115 	}
116 	return pto;
117 }
118 
119 unsigned long *dma_walk_cpu_trans(unsigned long *rto, dma_addr_t dma_addr)
120 {
121 	unsigned long *sto, *pto;
122 	unsigned int rtx, sx, px;
123 
124 	rtx = calc_rtx(dma_addr);
125 	sto = dma_get_seg_table_origin(&rto[rtx]);
126 	if (!sto)
127 		return NULL;
128 
129 	sx = calc_sx(dma_addr);
130 	pto = dma_get_page_table_origin(&sto[sx]);
131 	if (!pto)
132 		return NULL;
133 
134 	px = calc_px(dma_addr);
135 	return &pto[px];
136 }
137 
138 void dma_update_cpu_trans(unsigned long *ptep, phys_addr_t page_addr, int flags)
139 {
140 	unsigned long pte;
141 
142 	pte = READ_ONCE(*ptep);
143 	if (flags & ZPCI_PTE_INVALID) {
144 		invalidate_pt_entry(&pte);
145 	} else {
146 		set_pt_pfaa(&pte, page_addr);
147 		validate_pt_entry(&pte);
148 	}
149 
150 	if (flags & ZPCI_TABLE_PROTECTED)
151 		entry_set_protected(&pte);
152 	else
153 		entry_clr_protected(&pte);
154 
155 	xchg(ptep, pte);
156 }
157 
158 static int __dma_update_trans(struct zpci_dev *zdev, phys_addr_t pa,
159 			      dma_addr_t dma_addr, size_t size, int flags)
160 {
161 	unsigned int nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT;
162 	phys_addr_t page_addr = (pa & PAGE_MASK);
163 	unsigned long *entry;
164 	int i, rc = 0;
165 
166 	if (!nr_pages)
167 		return -EINVAL;
168 
169 	if (!zdev->dma_table)
170 		return -EINVAL;
171 
172 	for (i = 0; i < nr_pages; i++) {
173 		entry = dma_walk_cpu_trans(zdev->dma_table, dma_addr);
174 		if (!entry) {
175 			rc = -ENOMEM;
176 			goto undo_cpu_trans;
177 		}
178 		dma_update_cpu_trans(entry, page_addr, flags);
179 		page_addr += PAGE_SIZE;
180 		dma_addr += PAGE_SIZE;
181 	}
182 
183 undo_cpu_trans:
184 	if (rc && ((flags & ZPCI_PTE_VALID_MASK) == ZPCI_PTE_VALID)) {
185 		flags = ZPCI_PTE_INVALID;
186 		while (i-- > 0) {
187 			page_addr -= PAGE_SIZE;
188 			dma_addr -= PAGE_SIZE;
189 			entry = dma_walk_cpu_trans(zdev->dma_table, dma_addr);
190 			if (!entry)
191 				break;
192 			dma_update_cpu_trans(entry, page_addr, flags);
193 		}
194 	}
195 	return rc;
196 }
197 
198 static int __dma_purge_tlb(struct zpci_dev *zdev, dma_addr_t dma_addr,
199 			   size_t size, int flags)
200 {
201 	unsigned long irqflags;
202 	int ret;
203 
204 	/*
205 	 * With zdev->tlb_refresh == 0, rpcit is not required to establish new
206 	 * translations when previously invalid translation-table entries are
207 	 * validated. With lazy unmap, rpcit is skipped for previously valid
208 	 * entries, but a global rpcit is then required before any address can
209 	 * be re-used, i.e. after each iommu bitmap wrap-around.
210 	 */
211 	if ((flags & ZPCI_PTE_VALID_MASK) == ZPCI_PTE_VALID) {
212 		if (!zdev->tlb_refresh)
213 			return 0;
214 	} else {
215 		if (!s390_iommu_strict)
216 			return 0;
217 	}
218 
219 	ret = zpci_refresh_trans((u64) zdev->fh << 32, dma_addr,
220 				 PAGE_ALIGN(size));
221 	if (ret == -ENOMEM && !s390_iommu_strict) {
222 		/* enable the hypervisor to free some resources */
223 		if (zpci_refresh_global(zdev))
224 			goto out;
225 
226 		spin_lock_irqsave(&zdev->iommu_bitmap_lock, irqflags);
227 		bitmap_andnot(zdev->iommu_bitmap, zdev->iommu_bitmap,
228 			      zdev->lazy_bitmap, zdev->iommu_pages);
229 		bitmap_zero(zdev->lazy_bitmap, zdev->iommu_pages);
230 		spin_unlock_irqrestore(&zdev->iommu_bitmap_lock, irqflags);
231 		ret = 0;
232 	}
233 out:
234 	return ret;
235 }
236 
237 static int dma_update_trans(struct zpci_dev *zdev, phys_addr_t pa,
238 			    dma_addr_t dma_addr, size_t size, int flags)
239 {
240 	int rc;
241 
242 	rc = __dma_update_trans(zdev, pa, dma_addr, size, flags);
243 	if (rc)
244 		return rc;
245 
246 	rc = __dma_purge_tlb(zdev, dma_addr, size, flags);
247 	if (rc && ((flags & ZPCI_PTE_VALID_MASK) == ZPCI_PTE_VALID))
248 		__dma_update_trans(zdev, pa, dma_addr, size, ZPCI_PTE_INVALID);
249 
250 	return rc;
251 }
252 
253 void dma_free_seg_table(unsigned long entry)
254 {
255 	unsigned long *sto = get_rt_sto(entry);
256 	int sx;
257 
258 	for (sx = 0; sx < ZPCI_TABLE_ENTRIES; sx++)
259 		if (reg_entry_isvalid(sto[sx]))
260 			dma_free_page_table(get_st_pto(sto[sx]));
261 
262 	dma_free_cpu_table(sto);
263 }
264 
265 void dma_cleanup_tables(unsigned long *table)
266 {
267 	int rtx;
268 
269 	if (!table)
270 		return;
271 
272 	for (rtx = 0; rtx < ZPCI_TABLE_ENTRIES; rtx++)
273 		if (reg_entry_isvalid(table[rtx]))
274 			dma_free_seg_table(table[rtx]);
275 
276 	dma_free_cpu_table(table);
277 }
278 
279 static unsigned long __dma_alloc_iommu(struct device *dev,
280 				       unsigned long start, int size)
281 {
282 	struct zpci_dev *zdev = to_zpci(to_pci_dev(dev));
283 
284 	return iommu_area_alloc(zdev->iommu_bitmap, zdev->iommu_pages,
285 				start, size, zdev->start_dma >> PAGE_SHIFT,
286 				dma_get_seg_boundary_nr_pages(dev, PAGE_SHIFT),
287 				0);
288 }
289 
290 static dma_addr_t dma_alloc_address(struct device *dev, int size)
291 {
292 	struct zpci_dev *zdev = to_zpci(to_pci_dev(dev));
293 	unsigned long offset, flags;
294 
295 	spin_lock_irqsave(&zdev->iommu_bitmap_lock, flags);
296 	offset = __dma_alloc_iommu(dev, zdev->next_bit, size);
297 	if (offset == -1) {
298 		if (!s390_iommu_strict) {
299 			/* global flush before DMA addresses are reused */
300 			if (zpci_refresh_global(zdev))
301 				goto out_error;
302 
303 			bitmap_andnot(zdev->iommu_bitmap, zdev->iommu_bitmap,
304 				      zdev->lazy_bitmap, zdev->iommu_pages);
305 			bitmap_zero(zdev->lazy_bitmap, zdev->iommu_pages);
306 		}
307 		/* wrap-around */
308 		offset = __dma_alloc_iommu(dev, 0, size);
309 		if (offset == -1)
310 			goto out_error;
311 	}
312 	zdev->next_bit = offset + size;
313 	spin_unlock_irqrestore(&zdev->iommu_bitmap_lock, flags);
314 
315 	return zdev->start_dma + offset * PAGE_SIZE;
316 
317 out_error:
318 	spin_unlock_irqrestore(&zdev->iommu_bitmap_lock, flags);
319 	return DMA_MAPPING_ERROR;
320 }
321 
322 static void dma_free_address(struct device *dev, dma_addr_t dma_addr, int size)
323 {
324 	struct zpci_dev *zdev = to_zpci(to_pci_dev(dev));
325 	unsigned long flags, offset;
326 
327 	offset = (dma_addr - zdev->start_dma) >> PAGE_SHIFT;
328 
329 	spin_lock_irqsave(&zdev->iommu_bitmap_lock, flags);
330 	if (!zdev->iommu_bitmap)
331 		goto out;
332 
333 	if (s390_iommu_strict)
334 		bitmap_clear(zdev->iommu_bitmap, offset, size);
335 	else
336 		bitmap_set(zdev->lazy_bitmap, offset, size);
337 
338 out:
339 	spin_unlock_irqrestore(&zdev->iommu_bitmap_lock, flags);
340 }
341 
342 static inline void zpci_err_dma(unsigned long rc, unsigned long addr)
343 {
344 	struct {
345 		unsigned long rc;
346 		unsigned long addr;
347 	} __packed data = {rc, addr};
348 
349 	zpci_err_hex(&data, sizeof(data));
350 }
351 
352 static dma_addr_t s390_dma_map_pages(struct device *dev, struct page *page,
353 				     unsigned long offset, size_t size,
354 				     enum dma_data_direction direction,
355 				     unsigned long attrs)
356 {
357 	struct zpci_dev *zdev = to_zpci(to_pci_dev(dev));
358 	unsigned long pa = page_to_phys(page) + offset;
359 	int flags = ZPCI_PTE_VALID;
360 	unsigned long nr_pages;
361 	dma_addr_t dma_addr;
362 	int ret;
363 
364 	/* This rounds up number of pages based on size and offset */
365 	nr_pages = iommu_num_pages(pa, size, PAGE_SIZE);
366 	dma_addr = dma_alloc_address(dev, nr_pages);
367 	if (dma_addr == DMA_MAPPING_ERROR) {
368 		ret = -ENOSPC;
369 		goto out_err;
370 	}
371 
372 	/* Use rounded up size */
373 	size = nr_pages * PAGE_SIZE;
374 
375 	if (direction == DMA_NONE || direction == DMA_TO_DEVICE)
376 		flags |= ZPCI_TABLE_PROTECTED;
377 
378 	ret = dma_update_trans(zdev, pa, dma_addr, size, flags);
379 	if (ret)
380 		goto out_free;
381 
382 	atomic64_add(nr_pages, &zdev->mapped_pages);
383 	return dma_addr + (offset & ~PAGE_MASK);
384 
385 out_free:
386 	dma_free_address(dev, dma_addr, nr_pages);
387 out_err:
388 	zpci_err("map error:\n");
389 	zpci_err_dma(ret, pa);
390 	return DMA_MAPPING_ERROR;
391 }
392 
393 static void s390_dma_unmap_pages(struct device *dev, dma_addr_t dma_addr,
394 				 size_t size, enum dma_data_direction direction,
395 				 unsigned long attrs)
396 {
397 	struct zpci_dev *zdev = to_zpci(to_pci_dev(dev));
398 	int npages, ret;
399 
400 	npages = iommu_num_pages(dma_addr, size, PAGE_SIZE);
401 	dma_addr = dma_addr & PAGE_MASK;
402 	ret = dma_update_trans(zdev, 0, dma_addr, npages * PAGE_SIZE,
403 			       ZPCI_PTE_INVALID);
404 	if (ret) {
405 		zpci_err("unmap error:\n");
406 		zpci_err_dma(ret, dma_addr);
407 		return;
408 	}
409 
410 	atomic64_add(npages, &zdev->unmapped_pages);
411 	dma_free_address(dev, dma_addr, npages);
412 }
413 
414 static void *s390_dma_alloc(struct device *dev, size_t size,
415 			    dma_addr_t *dma_handle, gfp_t flag,
416 			    unsigned long attrs)
417 {
418 	struct zpci_dev *zdev = to_zpci(to_pci_dev(dev));
419 	struct page *page;
420 	phys_addr_t pa;
421 	dma_addr_t map;
422 
423 	size = PAGE_ALIGN(size);
424 	page = alloc_pages(flag | __GFP_ZERO, get_order(size));
425 	if (!page)
426 		return NULL;
427 
428 	pa = page_to_phys(page);
429 	map = s390_dma_map_pages(dev, page, 0, size, DMA_BIDIRECTIONAL, 0);
430 	if (dma_mapping_error(dev, map)) {
431 		__free_pages(page, get_order(size));
432 		return NULL;
433 	}
434 
435 	atomic64_add(size / PAGE_SIZE, &zdev->allocated_pages);
436 	if (dma_handle)
437 		*dma_handle = map;
438 	return phys_to_virt(pa);
439 }
440 
441 static void s390_dma_free(struct device *dev, size_t size,
442 			  void *vaddr, dma_addr_t dma_handle,
443 			  unsigned long attrs)
444 {
445 	struct zpci_dev *zdev = to_zpci(to_pci_dev(dev));
446 
447 	size = PAGE_ALIGN(size);
448 	atomic64_sub(size / PAGE_SIZE, &zdev->allocated_pages);
449 	s390_dma_unmap_pages(dev, dma_handle, size, DMA_BIDIRECTIONAL, 0);
450 	free_pages((unsigned long)vaddr, get_order(size));
451 }
452 
453 /* Map a segment into a contiguous dma address area */
454 static int __s390_dma_map_sg(struct device *dev, struct scatterlist *sg,
455 			     size_t size, dma_addr_t *handle,
456 			     enum dma_data_direction dir)
457 {
458 	unsigned long nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT;
459 	struct zpci_dev *zdev = to_zpci(to_pci_dev(dev));
460 	dma_addr_t dma_addr_base, dma_addr;
461 	int flags = ZPCI_PTE_VALID;
462 	struct scatterlist *s;
463 	phys_addr_t pa = 0;
464 	int ret;
465 
466 	dma_addr_base = dma_alloc_address(dev, nr_pages);
467 	if (dma_addr_base == DMA_MAPPING_ERROR)
468 		return -ENOMEM;
469 
470 	dma_addr = dma_addr_base;
471 	if (dir == DMA_NONE || dir == DMA_TO_DEVICE)
472 		flags |= ZPCI_TABLE_PROTECTED;
473 
474 	for (s = sg; dma_addr < dma_addr_base + size; s = sg_next(s)) {
475 		pa = page_to_phys(sg_page(s));
476 		ret = __dma_update_trans(zdev, pa, dma_addr,
477 					 s->offset + s->length, flags);
478 		if (ret)
479 			goto unmap;
480 
481 		dma_addr += s->offset + s->length;
482 	}
483 	ret = __dma_purge_tlb(zdev, dma_addr_base, size, flags);
484 	if (ret)
485 		goto unmap;
486 
487 	*handle = dma_addr_base;
488 	atomic64_add(nr_pages, &zdev->mapped_pages);
489 
490 	return ret;
491 
492 unmap:
493 	dma_update_trans(zdev, 0, dma_addr_base, dma_addr - dma_addr_base,
494 			 ZPCI_PTE_INVALID);
495 	dma_free_address(dev, dma_addr_base, nr_pages);
496 	zpci_err("map error:\n");
497 	zpci_err_dma(ret, pa);
498 	return ret;
499 }
500 
501 static int s390_dma_map_sg(struct device *dev, struct scatterlist *sg,
502 			   int nr_elements, enum dma_data_direction dir,
503 			   unsigned long attrs)
504 {
505 	struct scatterlist *s = sg, *start = sg, *dma = sg;
506 	unsigned int max = dma_get_max_seg_size(dev);
507 	unsigned int size = s->offset + s->length;
508 	unsigned int offset = s->offset;
509 	int count = 0, i, ret;
510 
511 	for (i = 1; i < nr_elements; i++) {
512 		s = sg_next(s);
513 
514 		s->dma_length = 0;
515 
516 		if (s->offset || (size & ~PAGE_MASK) ||
517 		    size + s->length > max) {
518 			ret = __s390_dma_map_sg(dev, start, size,
519 						&dma->dma_address, dir);
520 			if (ret)
521 				goto unmap;
522 
523 			dma->dma_address += offset;
524 			dma->dma_length = size - offset;
525 
526 			size = offset = s->offset;
527 			start = s;
528 			dma = sg_next(dma);
529 			count++;
530 		}
531 		size += s->length;
532 	}
533 	ret = __s390_dma_map_sg(dev, start, size, &dma->dma_address, dir);
534 	if (ret)
535 		goto unmap;
536 
537 	dma->dma_address += offset;
538 	dma->dma_length = size - offset;
539 
540 	return count + 1;
541 unmap:
542 	for_each_sg(sg, s, count, i)
543 		s390_dma_unmap_pages(dev, sg_dma_address(s), sg_dma_len(s),
544 				     dir, attrs);
545 
546 	return ret;
547 }
548 
549 static void s390_dma_unmap_sg(struct device *dev, struct scatterlist *sg,
550 			      int nr_elements, enum dma_data_direction dir,
551 			      unsigned long attrs)
552 {
553 	struct scatterlist *s;
554 	int i;
555 
556 	for_each_sg(sg, s, nr_elements, i) {
557 		if (s->dma_length)
558 			s390_dma_unmap_pages(dev, s->dma_address, s->dma_length,
559 					     dir, attrs);
560 		s->dma_address = 0;
561 		s->dma_length = 0;
562 	}
563 }
564 
565 int zpci_dma_init_device(struct zpci_dev *zdev)
566 {
567 	u8 status;
568 	int rc;
569 
570 	/*
571 	 * At this point, if the device is part of an IOMMU domain, this would
572 	 * be a strong hint towards a bug in the IOMMU API (common) code and/or
573 	 * simultaneous access via IOMMU and DMA API. So let's issue a warning.
574 	 */
575 	WARN_ON(zdev->s390_domain);
576 
577 	spin_lock_init(&zdev->iommu_bitmap_lock);
578 
579 	zdev->dma_table = dma_alloc_cpu_table();
580 	if (!zdev->dma_table) {
581 		rc = -ENOMEM;
582 		goto out;
583 	}
584 
585 	/*
586 	 * Restrict the iommu bitmap size to the minimum of the following:
587 	 * - s390_iommu_aperture which defaults to high_memory
588 	 * - 3-level pagetable address limit minus start_dma offset
589 	 * - DMA address range allowed by the hardware (clp query pci fn)
590 	 *
591 	 * Also set zdev->end_dma to the actual end address of the usable
592 	 * range, instead of the theoretical maximum as reported by hardware.
593 	 *
594 	 * This limits the number of concurrently usable DMA mappings since
595 	 * for each DMA mapped memory address we need a DMA address including
596 	 * extra DMA addresses for multiple mappings of the same memory address.
597 	 */
598 	zdev->start_dma = PAGE_ALIGN(zdev->start_dma);
599 	zdev->iommu_size = min3(s390_iommu_aperture,
600 				ZPCI_TABLE_SIZE_RT - zdev->start_dma,
601 				zdev->end_dma - zdev->start_dma + 1);
602 	zdev->end_dma = zdev->start_dma + zdev->iommu_size - 1;
603 	zdev->iommu_pages = zdev->iommu_size >> PAGE_SHIFT;
604 	zdev->iommu_bitmap = vzalloc(zdev->iommu_pages / 8);
605 	if (!zdev->iommu_bitmap) {
606 		rc = -ENOMEM;
607 		goto free_dma_table;
608 	}
609 	if (!s390_iommu_strict) {
610 		zdev->lazy_bitmap = vzalloc(zdev->iommu_pages / 8);
611 		if (!zdev->lazy_bitmap) {
612 			rc = -ENOMEM;
613 			goto free_bitmap;
614 		}
615 
616 	}
617 	if (zpci_register_ioat(zdev, 0, zdev->start_dma, zdev->end_dma,
618 			       virt_to_phys(zdev->dma_table), &status)) {
619 		rc = -EIO;
620 		goto free_bitmap;
621 	}
622 
623 	return 0;
624 free_bitmap:
625 	vfree(zdev->iommu_bitmap);
626 	zdev->iommu_bitmap = NULL;
627 	vfree(zdev->lazy_bitmap);
628 	zdev->lazy_bitmap = NULL;
629 free_dma_table:
630 	dma_free_cpu_table(zdev->dma_table);
631 	zdev->dma_table = NULL;
632 out:
633 	return rc;
634 }
635 
636 int zpci_dma_exit_device(struct zpci_dev *zdev)
637 {
638 	int cc = 0;
639 
640 	/*
641 	 * At this point, if the device is part of an IOMMU domain, this would
642 	 * be a strong hint towards a bug in the IOMMU API (common) code and/or
643 	 * simultaneous access via IOMMU and DMA API. So let's issue a warning.
644 	 */
645 	WARN_ON(zdev->s390_domain);
646 	if (zdev_enabled(zdev))
647 		cc = zpci_unregister_ioat(zdev, 0);
648 	/*
649 	 * cc == 3 indicates the function is gone already. This can happen
650 	 * if the function was deconfigured/disabled suddenly and we have not
651 	 * received a new handle yet.
652 	 */
653 	if (cc && cc != 3)
654 		return -EIO;
655 
656 	dma_cleanup_tables(zdev->dma_table);
657 	zdev->dma_table = NULL;
658 	vfree(zdev->iommu_bitmap);
659 	zdev->iommu_bitmap = NULL;
660 	vfree(zdev->lazy_bitmap);
661 	zdev->lazy_bitmap = NULL;
662 	zdev->next_bit = 0;
663 	return 0;
664 }
665 
666 static int __init dma_alloc_cpu_table_caches(void)
667 {
668 	dma_region_table_cache = kmem_cache_create("PCI_DMA_region_tables",
669 					ZPCI_TABLE_SIZE, ZPCI_TABLE_ALIGN,
670 					0, NULL);
671 	if (!dma_region_table_cache)
672 		return -ENOMEM;
673 
674 	dma_page_table_cache = kmem_cache_create("PCI_DMA_page_tables",
675 					ZPCI_PT_SIZE, ZPCI_PT_ALIGN,
676 					0, NULL);
677 	if (!dma_page_table_cache) {
678 		kmem_cache_destroy(dma_region_table_cache);
679 		return -ENOMEM;
680 	}
681 	return 0;
682 }
683 
684 int __init zpci_dma_init(void)
685 {
686 	s390_iommu_aperture = (u64)virt_to_phys(high_memory);
687 	if (!s390_iommu_aperture_factor)
688 		s390_iommu_aperture = ULONG_MAX;
689 	else
690 		s390_iommu_aperture *= s390_iommu_aperture_factor;
691 
692 	return dma_alloc_cpu_table_caches();
693 }
694 
695 void zpci_dma_exit(void)
696 {
697 	kmem_cache_destroy(dma_page_table_cache);
698 	kmem_cache_destroy(dma_region_table_cache);
699 }
700 
701 const struct dma_map_ops s390_pci_dma_ops = {
702 	.alloc		= s390_dma_alloc,
703 	.free		= s390_dma_free,
704 	.map_sg		= s390_dma_map_sg,
705 	.unmap_sg	= s390_dma_unmap_sg,
706 	.map_page	= s390_dma_map_pages,
707 	.unmap_page	= s390_dma_unmap_pages,
708 	.mmap		= dma_common_mmap,
709 	.get_sgtable	= dma_common_get_sgtable,
710 	.alloc_pages	= dma_common_alloc_pages,
711 	.free_pages	= dma_common_free_pages,
712 	/* dma_supported is unconditionally true without a callback */
713 };
714 EXPORT_SYMBOL_GPL(s390_pci_dma_ops);
715 
716 static int __init s390_iommu_setup(char *str)
717 {
718 	if (!strcmp(str, "strict"))
719 		s390_iommu_strict = 1;
720 	return 1;
721 }
722 
723 __setup("s390_iommu=", s390_iommu_setup);
724 
725 static int __init s390_iommu_aperture_setup(char *str)
726 {
727 	if (kstrtou32(str, 10, &s390_iommu_aperture_factor))
728 		s390_iommu_aperture_factor = 1;
729 	return 1;
730 }
731 
732 __setup("s390_iommu_aperture=", s390_iommu_aperture_setup);
733