30 	size_t		size;  member
40 	dma_free_attrs(dev, this->size, this->vaddr, this->dma_handle,  in dmam_release()
49 		WARN_ON(this->size != match->size ||  in dmam_match()
65 void dmam_free_coherent(struct device *dev, size_t size, void *vaddr,  in dmam_free_coherent()  argument
68 	struct dma_devres match_data = { size, vaddr, dma_handle };  in dmam_free_coherent()
70 	dma_free_coherent(dev, size, vaddr, dma_handle);  in dmam_free_coherent()
89 void *dmam_alloc_attrs(struct device *dev, size_t size, dma_addr_t *dma_handle,  in dmam_alloc_attrs()  argument
99 	vaddr = dma_alloc_attrs(dev, size, dma_handle, gfp, attrs);  in dmam_alloc_attrs()
107 	dr->size = size;  in dmam_alloc_attrs()
148 		size_t offset, size_t size, enum dma_data_direction dir,  in dma_map_page_attrs()  argument
160 	    arch_dma_map_page_direct(dev, page_to_phys(page) + offset + size))  in dma_map_page_attrs()
161 		addr = dma_direct_map_page(dev, page, offset, size, dir, attrs);  in dma_map_page_attrs()
163 		addr = ops->map_page(dev, page, offset, size, dir, attrs);  in dma_map_page_attrs()
164 	kmsan_handle_dma(page, offset, size, dir);  in dma_map_page_attrs()
165 	debug_dma_map_page(dev, page, offset, size, dir, addr, attrs);  in dma_map_page_attrs()
171 void dma_unmap_page_attrs(struct device *dev, dma_addr_t addr, size_t size,  in dma_unmap_page_attrs()  argument
178 	    arch_dma_unmap_page_direct(dev, addr + size))  in dma_unmap_page_attrs()
179 		dma_direct_unmap_page(dev, addr, size, dir, attrs);  in dma_unmap_page_attrs()
181 		ops->unmap_page(dev, addr, size, dir, attrs);  in dma_unmap_page_attrs()
182 	debug_dma_unmap_page(dev, addr, size, dir);  in dma_unmap_page_attrs()
300 		size_t size, enum dma_data_direction dir, unsigned long attrs)  in dma_map_resource()  argument
311 		addr = dma_direct_map_resource(dev, phys_addr, size, dir, attrs);  in dma_map_resource()
313 		addr = ops->map_resource(dev, phys_addr, size, dir, attrs);  in dma_map_resource()
315 	debug_dma_map_resource(dev, phys_addr, size, dir, addr, attrs);  in dma_map_resource()
320 void dma_unmap_resource(struct device *dev, dma_addr_t addr, size_t size,  in dma_unmap_resource()  argument
327 		ops->unmap_resource(dev, addr, size, dir, attrs);  in dma_unmap_resource()
328 	debug_dma_unmap_resource(dev, addr, size, dir);  in dma_unmap_resource()
332 void dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr, size_t size,  in dma_sync_single_for_cpu()  argument
339 		dma_direct_sync_single_for_cpu(dev, addr, size, dir);  in dma_sync_single_for_cpu()
341 		ops->sync_single_for_cpu(dev, addr, size, dir);  in dma_sync_single_for_cpu()
342 	debug_dma_sync_single_for_cpu(dev, addr, size, dir);  in dma_sync_single_for_cpu()
347 		size_t size, enum dma_data_direction dir)  in dma_sync_single_for_device()  argument
353 		dma_direct_sync_single_for_device(dev, addr, size, dir);  in dma_sync_single_for_device()
355 		ops->sync_single_for_device(dev, addr, size, dir);  in dma_sync_single_for_device()
356 	debug_dma_sync_single_for_device(dev, addr, size, dir);  in dma_sync_single_for_device()
400 		void *cpu_addr, dma_addr_t dma_addr, size_t size,  in dma_get_sgtable_attrs()  argument
407 				size, attrs);  in dma_get_sgtable_attrs()
410 	return ops->get_sgtable(dev, sgt, cpu_addr, dma_addr, size, attrs);  in dma_get_sgtable_attrs()
462 		void *cpu_addr, dma_addr_t dma_addr, size_t size,  in dma_mmap_attrs()  argument
468 		return dma_direct_mmap(dev, vma, cpu_addr, dma_addr, size,  in dma_mmap_attrs()
472 	return ops->mmap(dev, vma, cpu_addr, dma_addr, size, attrs);  in dma_mmap_attrs()
497 void *dma_alloc_attrs(struct device *dev, size_t size, dma_addr_t *dma_handle,  in dma_alloc_attrs()  argument
513 	if (dma_alloc_from_dev_coherent(dev, size, dma_handle, &cpu_addr))  in dma_alloc_attrs()
520 		cpu_addr = dma_direct_alloc(dev, size, dma_handle, flag, attrs);  in dma_alloc_attrs()
522 		cpu_addr = ops->alloc(dev, size, dma_handle, flag, attrs);  in dma_alloc_attrs()
526 	debug_dma_alloc_coherent(dev, size, *dma_handle, cpu_addr, attrs);  in dma_alloc_attrs()
531 void dma_free_attrs(struct device *dev, size_t size, void *cpu_addr,  in dma_free_attrs()  argument
536 	if (dma_release_from_dev_coherent(dev, get_order(size), cpu_addr))  in dma_free_attrs()
550 	debug_dma_free_coherent(dev, size, cpu_addr, dma_handle);  in dma_free_attrs()
552 		dma_direct_free(dev, size, cpu_addr, dma_handle, attrs);  in dma_free_attrs()
554 		ops->free(dev, size, cpu_addr, dma_handle, attrs);  in dma_free_attrs()
558 static struct page *__dma_alloc_pages(struct device *dev, size_t size,  in __dma_alloc_pages()  argument
570 	size = PAGE_ALIGN(size);  in __dma_alloc_pages()
572 		return dma_direct_alloc_pages(dev, size, dma_handle, dir, gfp);  in __dma_alloc_pages()
575 	return ops->alloc_pages(dev, size, dma_handle, dir, gfp);  in __dma_alloc_pages()
578 struct page *dma_alloc_pages(struct device *dev, size_t size,  in dma_alloc_pages()  argument
581 	struct page *page = __dma_alloc_pages(dev, size, dma_handle, dir, gfp);  in dma_alloc_pages()
584 		debug_dma_map_page(dev, page, 0, size, dir, *dma_handle, 0);  in dma_alloc_pages()
589 static void __dma_free_pages(struct device *dev, size_t size, struct page *page,  in __dma_free_pages()  argument
594 	size = PAGE_ALIGN(size);  in __dma_free_pages()
596 		dma_direct_free_pages(dev, size, page, dma_handle, dir);  in __dma_free_pages()
598 		ops->free_pages(dev, size, page, dma_handle, dir);  in __dma_free_pages()
601 void dma_free_pages(struct device *dev, size_t size, struct page *page,  in dma_free_pages()  argument
604 	debug_dma_unmap_page(dev, dma_handle, size, dir);  in dma_free_pages()
605 	__dma_free_pages(dev, size, page, dma_handle, dir);  in dma_free_pages()
610 		size_t size, struct page *page)  in dma_mmap_pages()  argument
612 	unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT;  in dma_mmap_pages()
622 static struct sg_table *alloc_single_sgt(struct device *dev, size_t size,  in alloc_single_sgt()  argument
633 	page = __dma_alloc_pages(dev, size, &sgt->sgl->dma_address, dir, gfp);  in alloc_single_sgt()
636 	sg_set_page(sgt->sgl, page, PAGE_ALIGN(size), 0);  in alloc_single_sgt()
646 struct sg_table *dma_alloc_noncontiguous(struct device *dev, size_t size,  in dma_alloc_noncontiguous()  argument
658 		sgt = ops->alloc_noncontiguous(dev, size, dir, gfp, attrs);  in dma_alloc_noncontiguous()
660 		sgt = alloc_single_sgt(dev, size, dir, gfp);  in dma_alloc_noncontiguous()
670 static void free_single_sgt(struct device *dev, size_t size,  in free_single_sgt()  argument
673 	__dma_free_pages(dev, size, sg_page(sgt->sgl), sgt->sgl->dma_address,  in free_single_sgt()
679 void dma_free_noncontiguous(struct device *dev, size_t size,  in dma_free_noncontiguous()  argument
686 		ops->free_noncontiguous(dev, size, sgt, dir);  in dma_free_noncontiguous()
688 		free_single_sgt(dev, size, sgt, dir);  in dma_free_noncontiguous()
692 void *dma_vmap_noncontiguous(struct device *dev, size_t size,  in dma_vmap_noncontiguous()  argument
696 	unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT;  in dma_vmap_noncontiguous()
714 		size_t size, struct sg_table *sgt)  in dma_mmap_noncontiguous()  argument
719 		unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT;  in dma_mmap_noncontiguous()
726 	return dma_mmap_pages(dev, vma, size, sg_page(sgt->sgl));  in dma_mmap_noncontiguous()
799 	size_t size = SIZE_MAX;  in dma_max_mapping_size()  local
802 		size = dma_direct_max_mapping_size(dev);  in dma_max_mapping_size()
804 		size = ops->max_mapping_size(dev);  in dma_max_mapping_size()
806 	return size;  in dma_max_mapping_size()
813 	size_t size = SIZE_MAX;  in dma_opt_mapping_size()  local
816 		size = ops->opt_mapping_size();  in dma_opt_mapping_size()
818 	return min(dma_max_mapping_size(dev), size);  in dma_opt_mapping_size()