193  * index is the number of PAGE_SIZE units from the start of the area's
194  * iopt_pages. If the iova is sub page-size then the area has an iova that
197 static unsigned long iopt_area_index_to_iova(struct iopt_area *area,  in iopt_area_index_to_iova()  argument
201 		WARN_ON(index < iopt_area_index(area) ||  in iopt_area_index_to_iova()
202 			index > iopt_area_last_index(area));  in iopt_area_index_to_iova()
203 	index -= iopt_area_index(area);  in iopt_area_index_to_iova()
205 		return iopt_area_iova(area);  in iopt_area_index_to_iova()
206 	return iopt_area_iova(area) - area->page_offset + index * PAGE_SIZE;  in iopt_area_index_to_iova()
209 static unsigned long iopt_area_index_to_iova_last(struct iopt_area *area,  in iopt_area_index_to_iova_last()  argument
213 		WARN_ON(index < iopt_area_index(area) ||  in iopt_area_index_to_iova_last()
214 			index > iopt_area_last_index(area));  in iopt_area_index_to_iova_last()
215 	if (index == iopt_area_last_index(area))  in iopt_area_index_to_iova_last()
216 		return iopt_area_last_iova(area);  in iopt_area_index_to_iova_last()
217 	return iopt_area_iova(area) - area->page_offset +  in iopt_area_index_to_iova_last()
218 	       (index - iopt_area_index(area) + 1) * PAGE_SIZE - 1;  in iopt_area_index_to_iova_last()
237 static void iopt_area_unmap_domain_range(struct iopt_area *area,  in iopt_area_unmap_domain_range()  argument
242 	unsigned long start_iova = iopt_area_index_to_iova(area, start_index);  in iopt_area_unmap_domain_range()
245 			   iopt_area_index_to_iova_last(area, last_index) -  in iopt_area_unmap_domain_range()
377 			      struct iopt_area *area, unsigned long start_index,  in batch_from_domain()  argument
384 	iova = iopt_area_index_to_iova(area, start_index);  in batch_from_domain()
385 	if (start_index == iopt_area_index(area))  in batch_from_domain()
386 		page_offset = area->page_offset;  in batch_from_domain()
403 					   struct iopt_area *area,  in raw_pages_from_domain()  argument
412 	iova = iopt_area_index_to_iova(area, start_index);  in raw_pages_from_domain()
413 	if (start_index == iopt_area_index(area))  in raw_pages_from_domain()
414 		page_offset = area->page_offset;  in raw_pages_from_domain()
428 				       struct iopt_area *area,  in batch_from_domain_continue()  argument
435 	batch_from_domain(batch, domain, area, start_index, last_index);  in batch_from_domain_continue()
441  * mode permits splitting a mapped area up, and then one of the splits is
476 			   struct iopt_area *area, unsigned long start_index)  in batch_to_domain()  argument
478 	bool disable_large_pages = area->iopt->disable_large_pages;  in batch_to_domain()
479 	unsigned long last_iova = iopt_area_last_iova(area);  in batch_to_domain()
488 	if (start_index == iopt_area_index(area))  in batch_to_domain()
489 		page_offset = area->page_offset;  in batch_to_domain()
491 		iopt_area_index_to_iova(area, start_index);  in batch_to_domain()
500 				next_iova - iova, area->iommu_prot);  in batch_to_domain()
504 				       next_iova - iova, area->iommu_prot,  in batch_to_domain()
930  * - The iommu_domain under an area
980 	struct iopt_area *area;  in pfn_reader_fill_span()  local
997 		 * and we'll find another area.  in pfn_reader_fill_span()
999 		area = iopt_pages_find_domain_area(pfns->pages, start_index);  in pfn_reader_fill_span()
1000 		if (WARN_ON(!area))  in pfn_reader_fill_span()
1005 			&pfns->batch, area->storage_domain, area, start_index,  in pfn_reader_fill_span()
1006 			min(iopt_area_last_index(area), span->last_used));  in pfn_reader_fill_span()
1197 iopt_area_unpin_domain(struct pfn_batch *batch, struct iopt_area *area,  in iopt_area_unpin_domain()  argument
1215 			batch_from_domain(batch, domain, area, start,  in iopt_area_unpin_domain()
1236 			batch_from_domain_continue(batch, domain, area,  in iopt_area_unpin_domain()
1242 				area, domain, *unmapped_end_index,  in iopt_area_unpin_domain()
1257 static void __iopt_area_unfill_domain(struct iopt_area *area,  in __iopt_area_unfill_domain()  argument
1263 	unsigned long start_index = iopt_area_index(area);  in __iopt_area_unfill_domain()
1293 		iopt_area_unpin_domain(&batch, area, pages, domain,  in __iopt_area_unfill_domain()
1302 		iopt_area_unmap_domain_range(area, domain, unmapped_end_index,  in __iopt_area_unfill_domain()
1309 static void iopt_area_unfill_partial_domain(struct iopt_area *area,  in iopt_area_unfill_partial_domain()  argument
1314 	if (end_index != iopt_area_index(area))  in iopt_area_unfill_partial_domain()
1315 		__iopt_area_unfill_domain(area, pages, domain, end_index - 1);  in iopt_area_unfill_partial_domain()
1320  * @area: The IOVA range to unmap
1326 void iopt_area_unmap_domain(struct iopt_area *area, struct iommu_domain *domain)  in iopt_area_unmap_domain()  argument
1328 	iommu_unmap_nofail(domain, iopt_area_iova(area),  in iopt_area_unmap_domain()
1329 			   iopt_area_length(area));  in iopt_area_unmap_domain()
1334  * @area: IOVA area to use
1335  * @pages: page supplier for the area (area->pages is NULL)
1342 void iopt_area_unfill_domain(struct iopt_area *area, struct iopt_pages *pages,  in iopt_area_unfill_domain()  argument
1345 	__iopt_area_unfill_domain(area, pages, domain,  in iopt_area_unfill_domain()
1346 				  iopt_area_last_index(area));  in iopt_area_unfill_domain()
1350  * iopt_area_fill_domain() - Map PFNs from the area into a domain
1351  * @area: IOVA area to use
1354  * Read the pfns from the area's underlying iopt_pages and map them into the
1357 int iopt_area_fill_domain(struct iopt_area *area, struct iommu_domain *domain)  in iopt_area_fill_domain()  argument
1363 	lockdep_assert_held(&area->pages->mutex);  in iopt_area_fill_domain()
1365 	rc = pfn_reader_first(&pfns, area->pages, iopt_area_index(area),  in iopt_area_fill_domain()
1366 			      iopt_area_last_index(area));  in iopt_area_fill_domain()
1372 		rc = batch_to_domain(&pfns.batch, domain, area,  in iopt_area_fill_domain()
1390 	iopt_area_unfill_partial_domain(area, area->pages, domain,  in iopt_area_fill_domain()
1398  * iopt_area_fill_domains() - Install PFNs into the area's domains
1399  * @area: The area to act on
1400  * @pages: The pages associated with the area (area->pages is NULL)
1402  * Called during area creation. The area is freshly created and not inserted in
1404  * area's io_pagetable and the area is installed in the domains_itree.
1408 int iopt_area_fill_domains(struct iopt_area *area, struct iopt_pages *pages)  in iopt_area_fill_domains()  argument
1418 	lockdep_assert_held(&area->iopt->domains_rwsem);  in iopt_area_fill_domains()
1420 	if (xa_empty(&area->iopt->domains))  in iopt_area_fill_domains()
1424 	rc = pfn_reader_first(&pfns, pages, iopt_area_index(area),  in iopt_area_fill_domains()
1425 			      iopt_area_last_index(area));  in iopt_area_fill_domains()
1432 		xa_for_each(&area->iopt->domains, index, domain) {  in iopt_area_fill_domains()
1433 			rc = batch_to_domain(&pfns.batch, domain, area,  in iopt_area_fill_domains()
1448 	area->storage_domain = xa_load(&area->iopt->domains, 0);  in iopt_area_fill_domains()
1449 	interval_tree_insert(&area->pages_node, &pages->domains_itree);  in iopt_area_fill_domains()
1454 	xa_for_each(&area->iopt->domains, unmap_index, domain) {  in iopt_area_fill_domains()
1463 		 * The area is not yet part of the domains_itree so we have to  in iopt_area_fill_domains()
1467 		if (unmap_index != area->iopt->next_domain_id - 1) {  in iopt_area_fill_domains()
1468 			if (end_index != iopt_area_index(area))  in iopt_area_fill_domains()
1470 					area, domain, iopt_area_index(area),  in iopt_area_fill_domains()
1473 			iopt_area_unfill_partial_domain(area, pages, domain,  in iopt_area_fill_domains()
1485  * iopt_area_unfill_domains() - unmap PFNs from the area's domains
1486  * @area: The area to act on
1487  * @pages: The pages associated with the area (area->pages is NULL)
1489  * Called during area destruction. This unmaps the iova's covered by all the
1490  * area's domains and releases the PFNs.
1492 void iopt_area_unfill_domains(struct iopt_area *area, struct iopt_pages *pages)  in iopt_area_unfill_domains()  argument
1494 	struct io_pagetable *iopt = area->iopt;  in iopt_area_unfill_domains()
1501 	if (!area->storage_domain)  in iopt_area_unfill_domains()
1505 		if (domain != area->storage_domain)  in iopt_area_unfill_domains()
1507 				area, domain, iopt_area_index(area),  in iopt_area_unfill_domains()
1508 				iopt_area_last_index(area));  in iopt_area_unfill_domains()
1511 		WARN_ON(RB_EMPTY_NODE(&area->pages_node.rb));  in iopt_area_unfill_domains()
1512 	interval_tree_remove(&area->pages_node, &pages->domains_itree);  in iopt_area_unfill_domains()
1513 	iopt_area_unfill_domain(area, pages, area->storage_domain);  in iopt_area_unfill_domains()
1514 	area->storage_domain = NULL;  in iopt_area_unfill_domains()
1617 		struct iopt_area *area;  in iopt_pages_fill_from_domain()  local
1619 		area = iopt_pages_find_domain_area(pages, start_index);  in iopt_pages_fill_from_domain()
1620 		if (WARN_ON(!area))  in iopt_pages_fill_from_domain()
1623 		domain_last = min(iopt_area_last_index(area), last_index);  in iopt_pages_fill_from_domain()
1624 		out_pages = raw_pages_from_domain(area->storage_domain, area,  in iopt_pages_fill_from_domain()
1904  * @area: The source of PFNs
1915 int iopt_area_add_access(struct iopt_area *area, unsigned long start_index,  in iopt_area_add_access()  argument
1919 	struct iopt_pages *pages = area->pages;  in iopt_area_add_access()
1929 		area->num_accesses++;  in iopt_area_add_access()
1950 	area->num_accesses++;  in iopt_area_add_access()
1964  * @area: The source of PFNs
1971 void iopt_area_remove_access(struct iopt_area *area, unsigned long start_index,  in iopt_area_remove_access()  argument
1974 	struct iopt_pages *pages = area->pages;  in iopt_area_remove_access()
1982 	WARN_ON(area->num_accesses == 0 || access->users == 0);  in iopt_area_remove_access()
1983 	area->num_accesses--;  in iopt_area_remove_access()