20 static int vduse_iotlb_add_range(struct vduse_iova_domain *domain,  in vduse_iotlb_add_range()  argument
35 	ret = vhost_iotlb_add_range_ctx(domain->iotlb, start, last,  in vduse_iotlb_add_range()
45 static void vduse_iotlb_del_range(struct vduse_iova_domain *domain,  in vduse_iotlb_del_range()  argument
51 	while ((map = vhost_iotlb_itree_first(domain->iotlb, start, last))) {  in vduse_iotlb_del_range()
55 		vhost_iotlb_map_free(domain->iotlb, map);  in vduse_iotlb_del_range()
59 int vduse_domain_set_map(struct vduse_iova_domain *domain,  in vduse_domain_set_map()  argument
67 	spin_lock(&domain->iotlb_lock);  in vduse_domain_set_map()
68 	vduse_iotlb_del_range(domain, start, last);  in vduse_domain_set_map()
73 		ret = vduse_iotlb_add_range(domain, map->start, map->last,  in vduse_domain_set_map()
80 	spin_unlock(&domain->iotlb_lock);  in vduse_domain_set_map()
84 	vduse_iotlb_del_range(domain, start, last);  in vduse_domain_set_map()
85 	spin_unlock(&domain->iotlb_lock);  in vduse_domain_set_map()
89 void vduse_domain_clear_map(struct vduse_iova_domain *domain,  in vduse_domain_clear_map()  argument
95 	spin_lock(&domain->iotlb_lock);  in vduse_domain_clear_map()
98 		vduse_iotlb_del_range(domain, map->start, map->last);  in vduse_domain_clear_map()
100 	spin_unlock(&domain->iotlb_lock);  in vduse_domain_clear_map()
103 static int vduse_domain_map_bounce_page(struct vduse_iova_domain *domain,  in vduse_domain_map_bounce_page()  argument
110 		map = &domain->bounce_maps[iova >> PAGE_SHIFT];  in vduse_domain_map_bounce_page()
123 static void vduse_domain_unmap_bounce_page(struct vduse_iova_domain *domain,  in vduse_domain_unmap_bounce_page()  argument
130 		map = &domain->bounce_maps[iova >> PAGE_SHIFT];  in vduse_domain_unmap_bounce_page()
160 static void vduse_domain_bounce(struct vduse_iova_domain *domain,  in vduse_domain_bounce()  argument
169 	if (iova >= domain->bounce_size)  in vduse_domain_bounce()
173 		map = &domain->bounce_maps[iova >> PAGE_SHIFT];  in vduse_domain_bounce()
190 vduse_domain_get_coherent_page(struct vduse_iova_domain *domain, u64 iova)  in vduse_domain_get_coherent_page()  argument
197 	spin_lock(&domain->iotlb_lock);  in vduse_domain_get_coherent_page()
198 	map = vhost_iotlb_itree_first(domain->iotlb, start, last);  in vduse_domain_get_coherent_page()
205 	spin_unlock(&domain->iotlb_lock);  in vduse_domain_get_coherent_page()
211 vduse_domain_get_bounce_page(struct vduse_iova_domain *domain, u64 iova)  in vduse_domain_get_bounce_page()  argument
216 	read_lock(&domain->bounce_lock);  in vduse_domain_get_bounce_page()
217 	map = &domain->bounce_maps[iova >> PAGE_SHIFT];  in vduse_domain_get_bounce_page()
218 	if (domain->user_bounce_pages || !map->bounce_page)  in vduse_domain_get_bounce_page()
224 	read_unlock(&domain->bounce_lock);  in vduse_domain_get_bounce_page()
230 vduse_domain_free_kernel_bounce_pages(struct vduse_iova_domain *domain)  in vduse_domain_free_kernel_bounce_pages()  argument
235 	bounce_pfns = domain->bounce_size >> PAGE_SHIFT;  in vduse_domain_free_kernel_bounce_pages()
238 		map = &domain->bounce_maps[pfn];  in vduse_domain_free_kernel_bounce_pages()
250 int vduse_domain_add_user_bounce_pages(struct vduse_iova_domain *domain,  in vduse_domain_add_user_bounce_pages()  argument
257 	if (count != (domain->bounce_size >> PAGE_SHIFT))  in vduse_domain_add_user_bounce_pages()
260 	write_lock(&domain->bounce_lock);  in vduse_domain_add_user_bounce_pages()
262 	if (domain->user_bounce_pages)  in vduse_domain_add_user_bounce_pages()
266 		map = &domain->bounce_maps[i];  in vduse_domain_add_user_bounce_pages()
278 	domain->user_bounce_pages = true;  in vduse_domain_add_user_bounce_pages()
281 	write_unlock(&domain->bounce_lock);  in vduse_domain_add_user_bounce_pages()
286 void vduse_domain_remove_user_bounce_pages(struct vduse_iova_domain *domain)  in vduse_domain_remove_user_bounce_pages()  argument
291 	write_lock(&domain->bounce_lock);  in vduse_domain_remove_user_bounce_pages()
292 	if (!domain->user_bounce_pages)  in vduse_domain_remove_user_bounce_pages()
295 	count = domain->bounce_size >> PAGE_SHIFT;  in vduse_domain_remove_user_bounce_pages()
299 		map = &domain->bounce_maps[i];  in vduse_domain_remove_user_bounce_pages()
312 	domain->user_bounce_pages = false;  in vduse_domain_remove_user_bounce_pages()
314 	write_unlock(&domain->bounce_lock);  in vduse_domain_remove_user_bounce_pages()
317 void vduse_domain_reset_bounce_map(struct vduse_iova_domain *domain)  in vduse_domain_reset_bounce_map()  argument
319 	if (!domain->bounce_map)  in vduse_domain_reset_bounce_map()
322 	spin_lock(&domain->iotlb_lock);  in vduse_domain_reset_bounce_map()
323 	if (!domain->bounce_map)  in vduse_domain_reset_bounce_map()
326 	vduse_iotlb_del_range(domain, 0, domain->bounce_size - 1);  in vduse_domain_reset_bounce_map()
327 	domain->bounce_map = 0;  in vduse_domain_reset_bounce_map()
329 	spin_unlock(&domain->iotlb_lock);  in vduse_domain_reset_bounce_map()
332 static int vduse_domain_init_bounce_map(struct vduse_iova_domain *domain)  in vduse_domain_init_bounce_map()  argument
336 	if (domain->bounce_map)  in vduse_domain_init_bounce_map()
339 	spin_lock(&domain->iotlb_lock);  in vduse_domain_init_bounce_map()
340 	if (domain->bounce_map)  in vduse_domain_init_bounce_map()
343 	ret = vduse_iotlb_add_range(domain, 0, domain->bounce_size - 1,  in vduse_domain_init_bounce_map()
344 				    0, VHOST_MAP_RW, domain->file, 0);  in vduse_domain_init_bounce_map()
348 	domain->bounce_map = 1;  in vduse_domain_init_bounce_map()
350 	spin_unlock(&domain->iotlb_lock);  in vduse_domain_init_bounce_map()
376 dma_addr_t vduse_domain_map_page(struct vduse_iova_domain *domain,  in vduse_domain_map_page()  argument
381 	struct iova_domain *iovad = &domain->stream_iovad;  in vduse_domain_map_page()
382 	unsigned long limit = domain->bounce_size - 1;  in vduse_domain_map_page()
389 	if (vduse_domain_init_bounce_map(domain))  in vduse_domain_map_page()
392 	read_lock(&domain->bounce_lock);  in vduse_domain_map_page()
393 	if (vduse_domain_map_bounce_page(domain, (u64)iova, (u64)size, pa))  in vduse_domain_map_page()
397 		vduse_domain_bounce(domain, iova, size, DMA_TO_DEVICE);  in vduse_domain_map_page()
399 	read_unlock(&domain->bounce_lock);  in vduse_domain_map_page()
403 	read_unlock(&domain->bounce_lock);  in vduse_domain_map_page()
409 void vduse_domain_unmap_page(struct vduse_iova_domain *domain,  in vduse_domain_unmap_page()  argument
413 	struct iova_domain *iovad = &domain->stream_iovad;  in vduse_domain_unmap_page()
415 	read_lock(&domain->bounce_lock);  in vduse_domain_unmap_page()
417 		vduse_domain_bounce(domain, dma_addr, size, DMA_FROM_DEVICE);  in vduse_domain_unmap_page()
419 	vduse_domain_unmap_bounce_page(domain, (u64)dma_addr, (u64)size);  in vduse_domain_unmap_page()
420 	read_unlock(&domain->bounce_lock);  in vduse_domain_unmap_page()
424 void *vduse_domain_alloc_coherent(struct vduse_iova_domain *domain,  in vduse_domain_alloc_coherent()  argument
428 	struct iova_domain *iovad = &domain->consistent_iovad;  in vduse_domain_alloc_coherent()
429 	unsigned long limit = domain->iova_limit;  in vduse_domain_alloc_coherent()
436 	spin_lock(&domain->iotlb_lock);  in vduse_domain_alloc_coherent()
437 	if (vduse_iotlb_add_range(domain, (u64)iova, (u64)iova + size - 1,  in vduse_domain_alloc_coherent()
439 				  domain->file, (u64)iova)) {  in vduse_domain_alloc_coherent()
440 		spin_unlock(&domain->iotlb_lock);  in vduse_domain_alloc_coherent()
443 	spin_unlock(&domain->iotlb_lock);  in vduse_domain_alloc_coherent()
458 void vduse_domain_free_coherent(struct vduse_iova_domain *domain, size_t size,  in vduse_domain_free_coherent()  argument
462 	struct iova_domain *iovad = &domain->consistent_iovad;  in vduse_domain_free_coherent()
467 	spin_lock(&domain->iotlb_lock);  in vduse_domain_free_coherent()
468 	map = vhost_iotlb_itree_first(domain->iotlb, (u64)dma_addr,  in vduse_domain_free_coherent()
471 		spin_unlock(&domain->iotlb_lock);  in vduse_domain_free_coherent()
478 	vhost_iotlb_map_free(domain->iotlb, map);  in vduse_domain_free_coherent()
479 	spin_unlock(&domain->iotlb_lock);  in vduse_domain_free_coherent()
487 	struct vduse_iova_domain *domain = vmf->vma->vm_private_data;  in vduse_domain_mmap_fault()  local
491 	if (!domain)  in vduse_domain_mmap_fault()
494 	if (iova < domain->bounce_size)  in vduse_domain_mmap_fault()
495 		page = vduse_domain_get_bounce_page(domain, iova);  in vduse_domain_mmap_fault()
497 		page = vduse_domain_get_coherent_page(domain, iova);  in vduse_domain_mmap_fault()
513 	struct vduse_iova_domain *domain = file->private_data;  in vduse_domain_mmap()  local
516 	vma->vm_private_data = domain;  in vduse_domain_mmap()
524 	struct vduse_iova_domain *domain = file->private_data;  in vduse_domain_release()  local
526 	spin_lock(&domain->iotlb_lock);  in vduse_domain_release()
527 	vduse_iotlb_del_range(domain, 0, ULLONG_MAX);  in vduse_domain_release()
528 	vduse_domain_remove_user_bounce_pages(domain);  in vduse_domain_release()
529 	vduse_domain_free_kernel_bounce_pages(domain);  in vduse_domain_release()
530 	spin_unlock(&domain->iotlb_lock);  in vduse_domain_release()
531 	put_iova_domain(&domain->stream_iovad);  in vduse_domain_release()
532 	put_iova_domain(&domain->consistent_iovad);  in vduse_domain_release()
533 	vhost_iotlb_free(domain->iotlb);  in vduse_domain_release()
534 	vfree(domain->bounce_maps);  in vduse_domain_release()
535 	kfree(domain);  in vduse_domain_release()
546 void vduse_domain_destroy(struct vduse_iova_domain *domain)  in vduse_domain_destroy()  argument
548 	fput(domain->file);  in vduse_domain_destroy()
554 	struct vduse_iova_domain *domain;  in vduse_domain_create()  local
564 	domain = kzalloc(sizeof(*domain), GFP_KERNEL);  in vduse_domain_create()
565 	if (!domain)  in vduse_domain_create()
568 	domain->iotlb = vhost_iotlb_alloc(0, 0);  in vduse_domain_create()
569 	if (!domain->iotlb)  in vduse_domain_create()
572 	domain->iova_limit = iova_limit;  in vduse_domain_create()
573 	domain->bounce_size = PAGE_ALIGN(bounce_size);  in vduse_domain_create()
574 	domain->bounce_maps = vzalloc(bounce_pfns *  in vduse_domain_create()
576 	if (!domain->bounce_maps)  in vduse_domain_create()
580 		map = &domain->bounce_maps[pfn];  in vduse_domain_create()
584 				domain, O_RDWR);  in vduse_domain_create()
588 	domain->file = file;  in vduse_domain_create()
589 	rwlock_init(&domain->bounce_lock);  in vduse_domain_create()
590 	spin_lock_init(&domain->iotlb_lock);  in vduse_domain_create()
591 	init_iova_domain(&domain->stream_iovad,  in vduse_domain_create()
593 	ret = iova_domain_init_rcaches(&domain->stream_iovad);  in vduse_domain_create()
596 	init_iova_domain(&domain->consistent_iovad,  in vduse_domain_create()
598 	ret = iova_domain_init_rcaches(&domain->consistent_iovad);  in vduse_domain_create()
602 	return domain;  in vduse_domain_create()
604 	put_iova_domain(&domain->stream_iovad);  in vduse_domain_create()
608 	vfree(domain->bounce_maps);  in vduse_domain_create()
610 	vhost_iotlb_free(domain->iotlb);  in vduse_domain_create()
612 	kfree(domain);  in vduse_domain_create()