Lines Matching refs:zdev
24 static int zpci_refresh_global(struct zpci_dev *zdev) in zpci_refresh_global() argument
26 return zpci_refresh_trans((u64) zdev->fh << 32, zdev->start_dma, in zpci_refresh_global()
27 zdev->iommu_pages * PAGE_SIZE); in zpci_refresh_global()
159 static int __dma_update_trans(struct zpci_dev *zdev, phys_addr_t pa, in __dma_update_trans() argument
170 if (!zdev->dma_table) in __dma_update_trans()
174 entry = dma_walk_cpu_trans(zdev->dma_table, dma_addr, in __dma_update_trans()
191 entry = dma_walk_cpu_trans(zdev->dma_table, dma_addr, in __dma_update_trans()
201 static int __dma_purge_tlb(struct zpci_dev *zdev, dma_addr_t dma_addr, in __dma_purge_tlb() argument
215 if (!zdev->tlb_refresh) in __dma_purge_tlb()
222 ret = zpci_refresh_trans((u64) zdev->fh << 32, dma_addr, in __dma_purge_tlb()
226 if (zpci_refresh_global(zdev)) in __dma_purge_tlb()
229 spin_lock_irqsave(&zdev->iommu_bitmap_lock, irqflags); in __dma_purge_tlb()
230 bitmap_andnot(zdev->iommu_bitmap, zdev->iommu_bitmap, in __dma_purge_tlb()
231 zdev->lazy_bitmap, zdev->iommu_pages); in __dma_purge_tlb()
232 bitmap_zero(zdev->lazy_bitmap, zdev->iommu_pages); in __dma_purge_tlb()
233 spin_unlock_irqrestore(&zdev->iommu_bitmap_lock, irqflags); in __dma_purge_tlb()
240 static int dma_update_trans(struct zpci_dev *zdev, phys_addr_t pa, in dma_update_trans() argument
245 rc = __dma_update_trans(zdev, pa, dma_addr, size, flags); in dma_update_trans()
249 rc = __dma_purge_tlb(zdev, dma_addr, size, flags); in dma_update_trans()
251 __dma_update_trans(zdev, pa, dma_addr, size, ZPCI_PTE_INVALID); in dma_update_trans()
285 struct zpci_dev *zdev = to_zpci(to_pci_dev(dev)); in __dma_alloc_iommu() local
287 return iommu_area_alloc(zdev->iommu_bitmap, zdev->iommu_pages, in __dma_alloc_iommu()
288 start, size, zdev->start_dma >> PAGE_SHIFT, in __dma_alloc_iommu()
295 struct zpci_dev *zdev = to_zpci(to_pci_dev(dev)); in dma_alloc_address() local
298 spin_lock_irqsave(&zdev->iommu_bitmap_lock, flags); in dma_alloc_address()
299 offset = __dma_alloc_iommu(dev, zdev->next_bit, size); in dma_alloc_address()
303 if (zpci_refresh_global(zdev)) in dma_alloc_address()
306 bitmap_andnot(zdev->iommu_bitmap, zdev->iommu_bitmap, in dma_alloc_address()
307 zdev->lazy_bitmap, zdev->iommu_pages); in dma_alloc_address()
308 bitmap_zero(zdev->lazy_bitmap, zdev->iommu_pages); in dma_alloc_address()
315 zdev->next_bit = offset + size; in dma_alloc_address()
316 spin_unlock_irqrestore(&zdev->iommu_bitmap_lock, flags); in dma_alloc_address()
318 return zdev->start_dma + offset * PAGE_SIZE; in dma_alloc_address()
321 spin_unlock_irqrestore(&zdev->iommu_bitmap_lock, flags); in dma_alloc_address()
327 struct zpci_dev *zdev = to_zpci(to_pci_dev(dev)); in dma_free_address() local
330 offset = (dma_addr - zdev->start_dma) >> PAGE_SHIFT; in dma_free_address()
332 spin_lock_irqsave(&zdev->iommu_bitmap_lock, flags); in dma_free_address()
333 if (!zdev->iommu_bitmap) in dma_free_address()
337 bitmap_clear(zdev->iommu_bitmap, offset, size); in dma_free_address()
339 bitmap_set(zdev->lazy_bitmap, offset, size); in dma_free_address()
342 spin_unlock_irqrestore(&zdev->iommu_bitmap_lock, flags); in dma_free_address()
360 struct zpci_dev *zdev = to_zpci(to_pci_dev(dev)); in s390_dma_map_pages() local
381 ret = dma_update_trans(zdev, pa, dma_addr, size, flags); in s390_dma_map_pages()
385 atomic64_add(nr_pages, &zdev->mapped_pages); in s390_dma_map_pages()
400 struct zpci_dev *zdev = to_zpci(to_pci_dev(dev)); in s390_dma_unmap_pages() local
405 ret = dma_update_trans(zdev, 0, dma_addr, npages * PAGE_SIZE, in s390_dma_unmap_pages()
413 atomic64_add(npages, &zdev->unmapped_pages); in s390_dma_unmap_pages()
421 struct zpci_dev *zdev = to_zpci(to_pci_dev(dev)); in s390_dma_alloc() local
438 atomic64_add(size / PAGE_SIZE, &zdev->allocated_pages); in s390_dma_alloc()
448 struct zpci_dev *zdev = to_zpci(to_pci_dev(dev)); in s390_dma_free() local
451 atomic64_sub(size / PAGE_SIZE, &zdev->allocated_pages); in s390_dma_free()
462 struct zpci_dev *zdev = to_zpci(to_pci_dev(dev)); in __s390_dma_map_sg() local
479 ret = __dma_update_trans(zdev, pa, dma_addr, in __s390_dma_map_sg()
486 ret = __dma_purge_tlb(zdev, dma_addr_base, size, flags); in __s390_dma_map_sg()
491 atomic64_add(nr_pages, &zdev->mapped_pages); in __s390_dma_map_sg()
496 dma_update_trans(zdev, 0, dma_addr_base, dma_addr - dma_addr_base, in __s390_dma_map_sg()
579 int zpci_dma_init_device(struct zpci_dev *zdev) in zpci_dma_init_device() argument
589 WARN_ON(zdev->s390_domain); in zpci_dma_init_device()
591 spin_lock_init(&zdev->iommu_bitmap_lock); in zpci_dma_init_device()
593 zdev->dma_table = dma_alloc_cpu_table(GFP_KERNEL); in zpci_dma_init_device()
594 if (!zdev->dma_table) { in zpci_dma_init_device()
612 zdev->start_dma = PAGE_ALIGN(zdev->start_dma); in zpci_dma_init_device()
613 zdev->iommu_size = min3(s390_iommu_aperture, in zpci_dma_init_device()
614 ZPCI_TABLE_SIZE_RT - zdev->start_dma, in zpci_dma_init_device()
615 zdev->end_dma - zdev->start_dma + 1); in zpci_dma_init_device()
616 zdev->end_dma = zdev->start_dma + zdev->iommu_size - 1; in zpci_dma_init_device()
617 zdev->iommu_pages = zdev->iommu_size >> PAGE_SHIFT; in zpci_dma_init_device()
618 zdev->iommu_bitmap = bitmap_vzalloc(zdev->iommu_pages, GFP_KERNEL); in zpci_dma_init_device()
619 if (!zdev->iommu_bitmap) { in zpci_dma_init_device()
624 zdev->lazy_bitmap = bitmap_vzalloc(zdev->iommu_pages, GFP_KERNEL); in zpci_dma_init_device()
625 if (!zdev->lazy_bitmap) { in zpci_dma_init_device()
631 if (zpci_register_ioat(zdev, 0, zdev->start_dma, zdev->end_dma, in zpci_dma_init_device()
632 virt_to_phys(zdev->dma_table), &status)) { in zpci_dma_init_device()
639 vfree(zdev->iommu_bitmap); in zpci_dma_init_device()
640 zdev->iommu_bitmap = NULL; in zpci_dma_init_device()
641 vfree(zdev->lazy_bitmap); in zpci_dma_init_device()
642 zdev->lazy_bitmap = NULL; in zpci_dma_init_device()
644 dma_free_cpu_table(zdev->dma_table); in zpci_dma_init_device()
645 zdev->dma_table = NULL; in zpci_dma_init_device()
650 int zpci_dma_exit_device(struct zpci_dev *zdev) in zpci_dma_exit_device() argument
659 WARN_ON(zdev->s390_domain); in zpci_dma_exit_device()
660 if (zdev_enabled(zdev)) in zpci_dma_exit_device()
661 cc = zpci_unregister_ioat(zdev, 0); in zpci_dma_exit_device()
670 dma_cleanup_tables(zdev->dma_table); in zpci_dma_exit_device()
671 zdev->dma_table = NULL; in zpci_dma_exit_device()
672 vfree(zdev->iommu_bitmap); in zpci_dma_exit_device()
673 zdev->iommu_bitmap = NULL; in zpci_dma_exit_device()
674 vfree(zdev->lazy_bitmap); in zpci_dma_exit_device()
675 zdev->lazy_bitmap = NULL; in zpci_dma_exit_device()
676 zdev->next_bit = 0; in zpci_dma_exit_device()