16 static u32 find_phys_blocks(struct tid_user_buf *tidbuf, unsigned int npages);
133 			    struct tid_user_buf *tidbuf,  in unpin_rcv_pages()  argument
149 		pages = &tidbuf->pages[idx];  in unpin_rcv_pages()
159 static int pin_rcv_pages(struct hfi1_filedata *fd, struct tid_user_buf *tidbuf)  in pin_rcv_pages()  argument
162 	unsigned int npages = tidbuf->npages;  in pin_rcv_pages()
163 	unsigned long vaddr = tidbuf->vaddr;  in pin_rcv_pages()
192 	tidbuf->pages = pages;  in pin_rcv_pages()
255 	struct tid_user_buf *tidbuf;  in hfi1_user_exp_rcv_setup()  local
263 	tidbuf = kzalloc(sizeof(*tidbuf), GFP_KERNEL);  in hfi1_user_exp_rcv_setup()
264 	if (!tidbuf)  in hfi1_user_exp_rcv_setup()
267 	mutex_init(&tidbuf->cover_mutex);  in hfi1_user_exp_rcv_setup()
268 	tidbuf->vaddr = tinfo->vaddr;  in hfi1_user_exp_rcv_setup()
269 	tidbuf->length = tinfo->length;  in hfi1_user_exp_rcv_setup()
270 	tidbuf->npages = num_user_pages(tidbuf->vaddr, tidbuf->length);  in hfi1_user_exp_rcv_setup()
271 	tidbuf->psets = kcalloc(uctxt->expected_count, sizeof(*tidbuf->psets),  in hfi1_user_exp_rcv_setup()
273 	if (!tidbuf->psets) {  in hfi1_user_exp_rcv_setup()
280 			&tidbuf->notifier, current->mm,  in hfi1_user_exp_rcv_setup()
281 			tidbuf->vaddr, tidbuf->npages * PAGE_SIZE,  in hfi1_user_exp_rcv_setup()
285 		mmu_seq = mmu_interval_read_begin(&tidbuf->notifier);  in hfi1_user_exp_rcv_setup()
288 	pinned = pin_rcv_pages(fd, tidbuf);  in hfi1_user_exp_rcv_setup()
295 	tidbuf->n_psets = find_phys_blocks(tidbuf, pinned);  in hfi1_user_exp_rcv_setup()
299 	if (fd->tid_used + tidbuf->n_psets > fd->tid_limit)  in hfi1_user_exp_rcv_setup()
302 		pageset_count = tidbuf->n_psets;  in hfi1_user_exp_rcv_setup()
333 		ret = program_rcvarray(fd, tidbuf, grp,  in hfi1_user_exp_rcv_setup()
378 			ret = program_rcvarray(fd, tidbuf, grp,  in hfi1_user_exp_rcv_setup()
425 	unpin_rcv_pages(fd, tidbuf, NULL, mapped_pages, pinned - mapped_pages,  in hfi1_user_exp_rcv_setup()
432 		mutex_lock(&tidbuf->cover_mutex);  in hfi1_user_exp_rcv_setup()
433 		fail = mmu_interval_read_retry(&tidbuf->notifier, mmu_seq);  in hfi1_user_exp_rcv_setup()
434 		mutex_unlock(&tidbuf->cover_mutex);  in hfi1_user_exp_rcv_setup()
452 		mmu_interval_notifier_remove(&tidbuf->notifier);  in hfi1_user_exp_rcv_setup()
453 	kfree(tidbuf->pages);  in hfi1_user_exp_rcv_setup()
454 	kfree(tidbuf->psets);  in hfi1_user_exp_rcv_setup()
455 	kfree(tidbuf);  in hfi1_user_exp_rcv_setup()
472 		mmu_interval_notifier_remove(&tidbuf->notifier);  in hfi1_user_exp_rcv_setup()
474 		unpin_rcv_pages(fd, tidbuf, NULL, 0, pinned, false);  in hfi1_user_exp_rcv_setup()
476 	kfree(tidbuf->pages);  in hfi1_user_exp_rcv_setup()
477 	kfree(tidbuf->psets);  in hfi1_user_exp_rcv_setup()
478 	kfree(tidbuf);  in hfi1_user_exp_rcv_setup()
565 static u32 find_phys_blocks(struct tid_user_buf *tidbuf, unsigned int npages)  in find_phys_blocks()  argument
569 	struct page **pages = tidbuf->pages;  in find_phys_blocks()
570 	struct tid_pageset *list = tidbuf->psets;  in find_phys_blocks()
946 	struct tid_user_buf *tidbuf =  in tid_cover_invalidate()  local
951 		mutex_lock(&tidbuf->cover_mutex);  in tid_cover_invalidate()
953 		mutex_unlock(&tidbuf->cover_mutex);  in tid_cover_invalidate()