58 static int page_array_alloc(struct page_array *pa, unsigned int len)  in page_array_alloc()  argument
60 	if (pa->pa_nr || pa->pa_iova)  in page_array_alloc()
66 	pa->pa_nr = len;  in page_array_alloc()
68 	pa->pa_iova = kcalloc(len, sizeof(*pa->pa_iova), GFP_KERNEL);  in page_array_alloc()
69 	if (!pa->pa_iova)  in page_array_alloc()
72 	pa->pa_page = kcalloc(len, sizeof(*pa->pa_page), GFP_KERNEL);  in page_array_alloc()
73 	if (!pa->pa_page) {  in page_array_alloc()
74 		kfree(pa->pa_iova);  in page_array_alloc()
91 static void page_array_unpin(struct page_array *pa,  in page_array_unpin()  argument
97 		dma_addr_t *first = &pa->pa_iova[unpinned];  in page_array_unpin()
112 	pa->pa_nr = 0;  in page_array_unpin()
131 static int page_array_pin(struct page_array *pa, struct vfio_device *vdev, bool unaligned)  in page_array_pin()  argument
136 	while (pinned < pa->pa_nr) {  in page_array_pin()
137 		dma_addr_t *first = &pa->pa_iova[pinned];  in page_array_pin()
140 		if (pinned + npage < pa->pa_nr &&  in page_array_pin()
149 				     &pa->pa_page[pinned]);  in page_array_pin()
164 	page_array_unpin(pa, vdev, pinned, unaligned);  in page_array_pin()
169 static void page_array_unpin_free(struct page_array *pa, struct vfio_device *vdev, bool unaligned)  in page_array_unpin_free()  argument
171 	page_array_unpin(pa, vdev, pa->pa_nr, unaligned);  in page_array_unpin_free()
172 	kfree(pa->pa_page);  in page_array_unpin_free()
173 	kfree(pa->pa_iova);  in page_array_unpin_free()
176 static bool page_array_iova_pinned(struct page_array *pa, u64 iova, u64 length)  in page_array_iova_pinned()  argument
183 	for (i = 0; i < pa->pa_nr; i++) {  in page_array_iova_pinned()
184 		pfn = pa->pa_iova[i] >> PAGE_SHIFT;  in page_array_iova_pinned()
192 static inline void page_array_idal_create_words(struct page_array *pa,  in page_array_idal_create_words()  argument
205 	for (i = 0; i < pa->pa_nr; i++) {  in page_array_idal_create_words()
206 		idaws[i] = page_to_phys(pa->pa_page[i]);  in page_array_idal_create_words()
209 		idaws[i] += pa->pa_iova[i] & (PAGE_SIZE - 1);  in page_array_idal_create_words()
602 			      struct page_array *pa,  in ccwchain_fetch_ccw()  argument
631 	ret = page_array_alloc(pa, idaw_nr);  in ccwchain_fetch_ccw()
642 			pa->pa_iova[i] = idaws[i];  in ccwchain_fetch_ccw()
644 			pa->pa_iova[i] = idaws_f1[i];  in ccwchain_fetch_ccw()
648 		ret = page_array_pin(pa, vdev, idal_is_2k(cp));  in ccwchain_fetch_ccw()
652 		pa->pa_nr = 0;  in ccwchain_fetch_ccw()
659 	page_array_idal_create_words(pa, idaws);  in ccwchain_fetch_ccw()
664 	page_array_unpin_free(pa, vdev, idal_is_2k(cp));  in ccwchain_fetch_ccw()
679 			      struct page_array *pa,  in ccwchain_fetch_one()  argument
686 	return ccwchain_fetch_ccw(ccw, pa, cp);  in ccwchain_fetch_one()
810 	struct page_array *pa;  in cp_prefetch()  local
821 			pa = &chain->ch_pa[idx];  in cp_prefetch()
823 			ret = ccwchain_fetch_one(ccw, pa, cp);  in cp_prefetch()