xref: /openbmc/linux/lib/test_hmm.c (revision 11a163f2)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * This is a module to test the HMM (Heterogeneous Memory Management)
4  * mirror and zone device private memory migration APIs of the kernel.
5  * Userspace programs can register with the driver to mirror their own address
6  * space and can use the device to read/write any valid virtual address.
7  */
8 #include <linux/init.h>
9 #include <linux/fs.h>
10 #include <linux/mm.h>
11 #include <linux/module.h>
12 #include <linux/kernel.h>
13 #include <linux/cdev.h>
14 #include <linux/device.h>
15 #include <linux/mutex.h>
16 #include <linux/rwsem.h>
17 #include <linux/sched.h>
18 #include <linux/slab.h>
19 #include <linux/highmem.h>
20 #include <linux/delay.h>
21 #include <linux/pagemap.h>
22 #include <linux/hmm.h>
23 #include <linux/vmalloc.h>
24 #include <linux/swap.h>
25 #include <linux/swapops.h>
26 #include <linux/sched/mm.h>
27 #include <linux/platform_device.h>
28 
29 #include "test_hmm_uapi.h"
30 
31 #define DMIRROR_NDEVICES		2
32 #define DMIRROR_RANGE_FAULT_TIMEOUT	1000
33 #define DEVMEM_CHUNK_SIZE		(256 * 1024 * 1024U)
34 #define DEVMEM_CHUNKS_RESERVE		16
35 
36 static const struct dev_pagemap_ops dmirror_devmem_ops;
37 static const struct mmu_interval_notifier_ops dmirror_min_ops;
38 static dev_t dmirror_dev;
39 
40 struct dmirror_device;
41 
42 struct dmirror_bounce {
43 	void			*ptr;
44 	unsigned long		size;
45 	unsigned long		addr;
46 	unsigned long		cpages;
47 };
48 
49 #define DPT_XA_TAG_WRITE 3UL
50 
51 /*
52  * Data structure to track address ranges and register for mmu interval
53  * notifier updates.
54  */
55 struct dmirror_interval {
56 	struct mmu_interval_notifier	notifier;
57 	struct dmirror			*dmirror;
58 };
59 
60 /*
61  * Data attached to the open device file.
62  * Note that it might be shared after a fork().
63  */
64 struct dmirror {
65 	struct dmirror_device		*mdevice;
66 	struct xarray			pt;
67 	struct mmu_interval_notifier	notifier;
68 	struct mutex			mutex;
69 };
70 
71 /*
72  * ZONE_DEVICE pages for migration and simulating device memory.
73  */
74 struct dmirror_chunk {
75 	struct dev_pagemap	pagemap;
76 	struct dmirror_device	*mdevice;
77 };
78 
79 /*
80  * Per device data.
81  */
82 struct dmirror_device {
83 	struct cdev		cdevice;
84 	struct hmm_devmem	*devmem;
85 
86 	unsigned int		devmem_capacity;
87 	unsigned int		devmem_count;
88 	struct dmirror_chunk	**devmem_chunks;
89 	struct mutex		devmem_lock;	/* protects the above */
90 
91 	unsigned long		calloc;
92 	unsigned long		cfree;
93 	struct page		*free_pages;
94 	spinlock_t		lock;		/* protects the above */
95 };
96 
97 static struct dmirror_device dmirror_devices[DMIRROR_NDEVICES];
98 
99 static int dmirror_bounce_init(struct dmirror_bounce *bounce,
100 			       unsigned long addr,
101 			       unsigned long size)
102 {
103 	bounce->addr = addr;
104 	bounce->size = size;
105 	bounce->cpages = 0;
106 	bounce->ptr = vmalloc(size);
107 	if (!bounce->ptr)
108 		return -ENOMEM;
109 	return 0;
110 }
111 
112 static void dmirror_bounce_fini(struct dmirror_bounce *bounce)
113 {
114 	vfree(bounce->ptr);
115 }
116 
117 static int dmirror_fops_open(struct inode *inode, struct file *filp)
118 {
119 	struct cdev *cdev = inode->i_cdev;
120 	struct dmirror *dmirror;
121 	int ret;
122 
123 	/* Mirror this process address space */
124 	dmirror = kzalloc(sizeof(*dmirror), GFP_KERNEL);
125 	if (dmirror == NULL)
126 		return -ENOMEM;
127 
128 	dmirror->mdevice = container_of(cdev, struct dmirror_device, cdevice);
129 	mutex_init(&dmirror->mutex);
130 	xa_init(&dmirror->pt);
131 
132 	ret = mmu_interval_notifier_insert(&dmirror->notifier, current->mm,
133 				0, ULONG_MAX & PAGE_MASK, &dmirror_min_ops);
134 	if (ret) {
135 		kfree(dmirror);
136 		return ret;
137 	}
138 
139 	filp->private_data = dmirror;
140 	return 0;
141 }
142 
143 static int dmirror_fops_release(struct inode *inode, struct file *filp)
144 {
145 	struct dmirror *dmirror = filp->private_data;
146 
147 	mmu_interval_notifier_remove(&dmirror->notifier);
148 	xa_destroy(&dmirror->pt);
149 	kfree(dmirror);
150 	return 0;
151 }
152 
153 static struct dmirror_device *dmirror_page_to_device(struct page *page)
154 
155 {
156 	return container_of(page->pgmap, struct dmirror_chunk,
157 			    pagemap)->mdevice;
158 }
159 
160 static int dmirror_do_fault(struct dmirror *dmirror, struct hmm_range *range)
161 {
162 	unsigned long *pfns = range->hmm_pfns;
163 	unsigned long pfn;
164 
165 	for (pfn = (range->start >> PAGE_SHIFT);
166 	     pfn < (range->end >> PAGE_SHIFT);
167 	     pfn++, pfns++) {
168 		struct page *page;
169 		void *entry;
170 
171 		/*
172 		 * Since we asked for hmm_range_fault() to populate pages,
173 		 * it shouldn't return an error entry on success.
174 		 */
175 		WARN_ON(*pfns & HMM_PFN_ERROR);
176 		WARN_ON(!(*pfns & HMM_PFN_VALID));
177 
178 		page = hmm_pfn_to_page(*pfns);
179 		WARN_ON(!page);
180 
181 		entry = page;
182 		if (*pfns & HMM_PFN_WRITE)
183 			entry = xa_tag_pointer(entry, DPT_XA_TAG_WRITE);
184 		else if (WARN_ON(range->default_flags & HMM_PFN_WRITE))
185 			return -EFAULT;
186 		entry = xa_store(&dmirror->pt, pfn, entry, GFP_ATOMIC);
187 		if (xa_is_err(entry))
188 			return xa_err(entry);
189 	}
190 
191 	return 0;
192 }
193 
194 static void dmirror_do_update(struct dmirror *dmirror, unsigned long start,
195 			      unsigned long end)
196 {
197 	unsigned long pfn;
198 	void *entry;
199 
200 	/*
201 	 * The XArray doesn't hold references to pages since it relies on
202 	 * the mmu notifier to clear page pointers when they become stale.
203 	 * Therefore, it is OK to just clear the entry.
204 	 */
205 	xa_for_each_range(&dmirror->pt, pfn, entry, start >> PAGE_SHIFT,
206 			  end >> PAGE_SHIFT)
207 		xa_erase(&dmirror->pt, pfn);
208 }
209 
210 static bool dmirror_interval_invalidate(struct mmu_interval_notifier *mni,
211 				const struct mmu_notifier_range *range,
212 				unsigned long cur_seq)
213 {
214 	struct dmirror *dmirror = container_of(mni, struct dmirror, notifier);
215 
216 	/*
217 	 * Ignore invalidation callbacks for device private pages since
218 	 * the invalidation is handled as part of the migration process.
219 	 */
220 	if (range->event == MMU_NOTIFY_MIGRATE &&
221 	    range->migrate_pgmap_owner == dmirror->mdevice)
222 		return true;
223 
224 	if (mmu_notifier_range_blockable(range))
225 		mutex_lock(&dmirror->mutex);
226 	else if (!mutex_trylock(&dmirror->mutex))
227 		return false;
228 
229 	mmu_interval_set_seq(mni, cur_seq);
230 	dmirror_do_update(dmirror, range->start, range->end);
231 
232 	mutex_unlock(&dmirror->mutex);
233 	return true;
234 }
235 
236 static const struct mmu_interval_notifier_ops dmirror_min_ops = {
237 	.invalidate = dmirror_interval_invalidate,
238 };
239 
240 static int dmirror_range_fault(struct dmirror *dmirror,
241 				struct hmm_range *range)
242 {
243 	struct mm_struct *mm = dmirror->notifier.mm;
244 	unsigned long timeout =
245 		jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT);
246 	int ret;
247 
248 	while (true) {
249 		if (time_after(jiffies, timeout)) {
250 			ret = -EBUSY;
251 			goto out;
252 		}
253 
254 		range->notifier_seq = mmu_interval_read_begin(range->notifier);
255 		mmap_read_lock(mm);
256 		ret = hmm_range_fault(range);
257 		mmap_read_unlock(mm);
258 		if (ret) {
259 			if (ret == -EBUSY)
260 				continue;
261 			goto out;
262 		}
263 
264 		mutex_lock(&dmirror->mutex);
265 		if (mmu_interval_read_retry(range->notifier,
266 					    range->notifier_seq)) {
267 			mutex_unlock(&dmirror->mutex);
268 			continue;
269 		}
270 		break;
271 	}
272 
273 	ret = dmirror_do_fault(dmirror, range);
274 
275 	mutex_unlock(&dmirror->mutex);
276 out:
277 	return ret;
278 }
279 
280 static int dmirror_fault(struct dmirror *dmirror, unsigned long start,
281 			 unsigned long end, bool write)
282 {
283 	struct mm_struct *mm = dmirror->notifier.mm;
284 	unsigned long addr;
285 	unsigned long pfns[64];
286 	struct hmm_range range = {
287 		.notifier = &dmirror->notifier,
288 		.hmm_pfns = pfns,
289 		.pfn_flags_mask = 0,
290 		.default_flags =
291 			HMM_PFN_REQ_FAULT | (write ? HMM_PFN_REQ_WRITE : 0),
292 		.dev_private_owner = dmirror->mdevice,
293 	};
294 	int ret = 0;
295 
296 	/* Since the mm is for the mirrored process, get a reference first. */
297 	if (!mmget_not_zero(mm))
298 		return 0;
299 
300 	for (addr = start; addr < end; addr = range.end) {
301 		range.start = addr;
302 		range.end = min(addr + (ARRAY_SIZE(pfns) << PAGE_SHIFT), end);
303 
304 		ret = dmirror_range_fault(dmirror, &range);
305 		if (ret)
306 			break;
307 	}
308 
309 	mmput(mm);
310 	return ret;
311 }
312 
313 static int dmirror_do_read(struct dmirror *dmirror, unsigned long start,
314 			   unsigned long end, struct dmirror_bounce *bounce)
315 {
316 	unsigned long pfn;
317 	void *ptr;
318 
319 	ptr = bounce->ptr + ((start - bounce->addr) & PAGE_MASK);
320 
321 	for (pfn = start >> PAGE_SHIFT; pfn < (end >> PAGE_SHIFT); pfn++) {
322 		void *entry;
323 		struct page *page;
324 		void *tmp;
325 
326 		entry = xa_load(&dmirror->pt, pfn);
327 		page = xa_untag_pointer(entry);
328 		if (!page)
329 			return -ENOENT;
330 
331 		tmp = kmap(page);
332 		memcpy(ptr, tmp, PAGE_SIZE);
333 		kunmap(page);
334 
335 		ptr += PAGE_SIZE;
336 		bounce->cpages++;
337 	}
338 
339 	return 0;
340 }
341 
342 static int dmirror_read(struct dmirror *dmirror, struct hmm_dmirror_cmd *cmd)
343 {
344 	struct dmirror_bounce bounce;
345 	unsigned long start, end;
346 	unsigned long size = cmd->npages << PAGE_SHIFT;
347 	int ret;
348 
349 	start = cmd->addr;
350 	end = start + size;
351 	if (end < start)
352 		return -EINVAL;
353 
354 	ret = dmirror_bounce_init(&bounce, start, size);
355 	if (ret)
356 		return ret;
357 
358 	while (1) {
359 		mutex_lock(&dmirror->mutex);
360 		ret = dmirror_do_read(dmirror, start, end, &bounce);
361 		mutex_unlock(&dmirror->mutex);
362 		if (ret != -ENOENT)
363 			break;
364 
365 		start = cmd->addr + (bounce.cpages << PAGE_SHIFT);
366 		ret = dmirror_fault(dmirror, start, end, false);
367 		if (ret)
368 			break;
369 		cmd->faults++;
370 	}
371 
372 	if (ret == 0) {
373 		if (copy_to_user(u64_to_user_ptr(cmd->ptr), bounce.ptr,
374 				 bounce.size))
375 			ret = -EFAULT;
376 	}
377 	cmd->cpages = bounce.cpages;
378 	dmirror_bounce_fini(&bounce);
379 	return ret;
380 }
381 
382 static int dmirror_do_write(struct dmirror *dmirror, unsigned long start,
383 			    unsigned long end, struct dmirror_bounce *bounce)
384 {
385 	unsigned long pfn;
386 	void *ptr;
387 
388 	ptr = bounce->ptr + ((start - bounce->addr) & PAGE_MASK);
389 
390 	for (pfn = start >> PAGE_SHIFT; pfn < (end >> PAGE_SHIFT); pfn++) {
391 		void *entry;
392 		struct page *page;
393 		void *tmp;
394 
395 		entry = xa_load(&dmirror->pt, pfn);
396 		page = xa_untag_pointer(entry);
397 		if (!page || xa_pointer_tag(entry) != DPT_XA_TAG_WRITE)
398 			return -ENOENT;
399 
400 		tmp = kmap(page);
401 		memcpy(tmp, ptr, PAGE_SIZE);
402 		kunmap(page);
403 
404 		ptr += PAGE_SIZE;
405 		bounce->cpages++;
406 	}
407 
408 	return 0;
409 }
410 
411 static int dmirror_write(struct dmirror *dmirror, struct hmm_dmirror_cmd *cmd)
412 {
413 	struct dmirror_bounce bounce;
414 	unsigned long start, end;
415 	unsigned long size = cmd->npages << PAGE_SHIFT;
416 	int ret;
417 
418 	start = cmd->addr;
419 	end = start + size;
420 	if (end < start)
421 		return -EINVAL;
422 
423 	ret = dmirror_bounce_init(&bounce, start, size);
424 	if (ret)
425 		return ret;
426 	if (copy_from_user(bounce.ptr, u64_to_user_ptr(cmd->ptr),
427 			   bounce.size)) {
428 		ret = -EFAULT;
429 		goto fini;
430 	}
431 
432 	while (1) {
433 		mutex_lock(&dmirror->mutex);
434 		ret = dmirror_do_write(dmirror, start, end, &bounce);
435 		mutex_unlock(&dmirror->mutex);
436 		if (ret != -ENOENT)
437 			break;
438 
439 		start = cmd->addr + (bounce.cpages << PAGE_SHIFT);
440 		ret = dmirror_fault(dmirror, start, end, true);
441 		if (ret)
442 			break;
443 		cmd->faults++;
444 	}
445 
446 fini:
447 	cmd->cpages = bounce.cpages;
448 	dmirror_bounce_fini(&bounce);
449 	return ret;
450 }
451 
452 static bool dmirror_allocate_chunk(struct dmirror_device *mdevice,
453 				   struct page **ppage)
454 {
455 	struct dmirror_chunk *devmem;
456 	struct resource *res;
457 	unsigned long pfn;
458 	unsigned long pfn_first;
459 	unsigned long pfn_last;
460 	void *ptr;
461 
462 	devmem = kzalloc(sizeof(*devmem), GFP_KERNEL);
463 	if (!devmem)
464 		return false;
465 
466 	res = request_free_mem_region(&iomem_resource, DEVMEM_CHUNK_SIZE,
467 				      "hmm_dmirror");
468 	if (IS_ERR(res))
469 		goto err_devmem;
470 
471 	devmem->pagemap.type = MEMORY_DEVICE_PRIVATE;
472 	devmem->pagemap.range.start = res->start;
473 	devmem->pagemap.range.end = res->end;
474 	devmem->pagemap.nr_range = 1;
475 	devmem->pagemap.ops = &dmirror_devmem_ops;
476 	devmem->pagemap.owner = mdevice;
477 
478 	mutex_lock(&mdevice->devmem_lock);
479 
480 	if (mdevice->devmem_count == mdevice->devmem_capacity) {
481 		struct dmirror_chunk **new_chunks;
482 		unsigned int new_capacity;
483 
484 		new_capacity = mdevice->devmem_capacity +
485 				DEVMEM_CHUNKS_RESERVE;
486 		new_chunks = krealloc(mdevice->devmem_chunks,
487 				sizeof(new_chunks[0]) * new_capacity,
488 				GFP_KERNEL);
489 		if (!new_chunks)
490 			goto err_release;
491 		mdevice->devmem_capacity = new_capacity;
492 		mdevice->devmem_chunks = new_chunks;
493 	}
494 
495 	ptr = memremap_pages(&devmem->pagemap, numa_node_id());
496 	if (IS_ERR(ptr))
497 		goto err_release;
498 
499 	devmem->mdevice = mdevice;
500 	pfn_first = devmem->pagemap.range.start >> PAGE_SHIFT;
501 	pfn_last = pfn_first + (range_len(&devmem->pagemap.range) >> PAGE_SHIFT);
502 	mdevice->devmem_chunks[mdevice->devmem_count++] = devmem;
503 
504 	mutex_unlock(&mdevice->devmem_lock);
505 
506 	pr_info("added new %u MB chunk (total %u chunks, %u MB) PFNs [0x%lx 0x%lx)\n",
507 		DEVMEM_CHUNK_SIZE / (1024 * 1024),
508 		mdevice->devmem_count,
509 		mdevice->devmem_count * (DEVMEM_CHUNK_SIZE / (1024 * 1024)),
510 		pfn_first, pfn_last);
511 
512 	spin_lock(&mdevice->lock);
513 	for (pfn = pfn_first; pfn < pfn_last; pfn++) {
514 		struct page *page = pfn_to_page(pfn);
515 
516 		page->zone_device_data = mdevice->free_pages;
517 		mdevice->free_pages = page;
518 	}
519 	if (ppage) {
520 		*ppage = mdevice->free_pages;
521 		mdevice->free_pages = (*ppage)->zone_device_data;
522 		mdevice->calloc++;
523 	}
524 	spin_unlock(&mdevice->lock);
525 
526 	return true;
527 
528 err_release:
529 	mutex_unlock(&mdevice->devmem_lock);
530 	release_mem_region(devmem->pagemap.range.start, range_len(&devmem->pagemap.range));
531 err_devmem:
532 	kfree(devmem);
533 
534 	return false;
535 }
536 
537 static struct page *dmirror_devmem_alloc_page(struct dmirror_device *mdevice)
538 {
539 	struct page *dpage = NULL;
540 	struct page *rpage;
541 
542 	/*
543 	 * This is a fake device so we alloc real system memory to store
544 	 * our device memory.
545 	 */
546 	rpage = alloc_page(GFP_HIGHUSER);
547 	if (!rpage)
548 		return NULL;
549 
550 	spin_lock(&mdevice->lock);
551 
552 	if (mdevice->free_pages) {
553 		dpage = mdevice->free_pages;
554 		mdevice->free_pages = dpage->zone_device_data;
555 		mdevice->calloc++;
556 		spin_unlock(&mdevice->lock);
557 	} else {
558 		spin_unlock(&mdevice->lock);
559 		if (!dmirror_allocate_chunk(mdevice, &dpage))
560 			goto error;
561 	}
562 
563 	dpage->zone_device_data = rpage;
564 	get_page(dpage);
565 	lock_page(dpage);
566 	return dpage;
567 
568 error:
569 	__free_page(rpage);
570 	return NULL;
571 }
572 
573 static void dmirror_migrate_alloc_and_copy(struct migrate_vma *args,
574 					   struct dmirror *dmirror)
575 {
576 	struct dmirror_device *mdevice = dmirror->mdevice;
577 	const unsigned long *src = args->src;
578 	unsigned long *dst = args->dst;
579 	unsigned long addr;
580 
581 	for (addr = args->start; addr < args->end; addr += PAGE_SIZE,
582 						   src++, dst++) {
583 		struct page *spage;
584 		struct page *dpage;
585 		struct page *rpage;
586 
587 		if (!(*src & MIGRATE_PFN_MIGRATE))
588 			continue;
589 
590 		/*
591 		 * Note that spage might be NULL which is OK since it is an
592 		 * unallocated pte_none() or read-only zero page.
593 		 */
594 		spage = migrate_pfn_to_page(*src);
595 
596 		dpage = dmirror_devmem_alloc_page(mdevice);
597 		if (!dpage)
598 			continue;
599 
600 		rpage = dpage->zone_device_data;
601 		if (spage)
602 			copy_highpage(rpage, spage);
603 		else
604 			clear_highpage(rpage);
605 
606 		/*
607 		 * Normally, a device would use the page->zone_device_data to
608 		 * point to the mirror but here we use it to hold the page for
609 		 * the simulated device memory and that page holds the pointer
610 		 * to the mirror.
611 		 */
612 		rpage->zone_device_data = dmirror;
613 
614 		*dst = migrate_pfn(page_to_pfn(dpage)) |
615 			    MIGRATE_PFN_LOCKED;
616 		if ((*src & MIGRATE_PFN_WRITE) ||
617 		    (!spage && args->vma->vm_flags & VM_WRITE))
618 			*dst |= MIGRATE_PFN_WRITE;
619 	}
620 }
621 
622 static int dmirror_migrate_finalize_and_map(struct migrate_vma *args,
623 					    struct dmirror *dmirror)
624 {
625 	unsigned long start = args->start;
626 	unsigned long end = args->end;
627 	const unsigned long *src = args->src;
628 	const unsigned long *dst = args->dst;
629 	unsigned long pfn;
630 
631 	/* Map the migrated pages into the device's page tables. */
632 	mutex_lock(&dmirror->mutex);
633 
634 	for (pfn = start >> PAGE_SHIFT; pfn < (end >> PAGE_SHIFT); pfn++,
635 								src++, dst++) {
636 		struct page *dpage;
637 		void *entry;
638 
639 		if (!(*src & MIGRATE_PFN_MIGRATE))
640 			continue;
641 
642 		dpage = migrate_pfn_to_page(*dst);
643 		if (!dpage)
644 			continue;
645 
646 		/*
647 		 * Store the page that holds the data so the page table
648 		 * doesn't have to deal with ZONE_DEVICE private pages.
649 		 */
650 		entry = dpage->zone_device_data;
651 		if (*dst & MIGRATE_PFN_WRITE)
652 			entry = xa_tag_pointer(entry, DPT_XA_TAG_WRITE);
653 		entry = xa_store(&dmirror->pt, pfn, entry, GFP_ATOMIC);
654 		if (xa_is_err(entry)) {
655 			mutex_unlock(&dmirror->mutex);
656 			return xa_err(entry);
657 		}
658 	}
659 
660 	mutex_unlock(&dmirror->mutex);
661 	return 0;
662 }
663 
664 static int dmirror_migrate(struct dmirror *dmirror,
665 			   struct hmm_dmirror_cmd *cmd)
666 {
667 	unsigned long start, end, addr;
668 	unsigned long size = cmd->npages << PAGE_SHIFT;
669 	struct mm_struct *mm = dmirror->notifier.mm;
670 	struct vm_area_struct *vma;
671 	unsigned long src_pfns[64];
672 	unsigned long dst_pfns[64];
673 	struct dmirror_bounce bounce;
674 	struct migrate_vma args;
675 	unsigned long next;
676 	int ret;
677 
678 	start = cmd->addr;
679 	end = start + size;
680 	if (end < start)
681 		return -EINVAL;
682 
683 	/* Since the mm is for the mirrored process, get a reference first. */
684 	if (!mmget_not_zero(mm))
685 		return -EINVAL;
686 
687 	mmap_read_lock(mm);
688 	for (addr = start; addr < end; addr = next) {
689 		vma = find_vma(mm, addr);
690 		if (!vma || addr < vma->vm_start ||
691 		    !(vma->vm_flags & VM_READ)) {
692 			ret = -EINVAL;
693 			goto out;
694 		}
695 		next = min(end, addr + (ARRAY_SIZE(src_pfns) << PAGE_SHIFT));
696 		if (next > vma->vm_end)
697 			next = vma->vm_end;
698 
699 		args.vma = vma;
700 		args.src = src_pfns;
701 		args.dst = dst_pfns;
702 		args.start = addr;
703 		args.end = next;
704 		args.pgmap_owner = dmirror->mdevice;
705 		args.flags = MIGRATE_VMA_SELECT_SYSTEM;
706 		ret = migrate_vma_setup(&args);
707 		if (ret)
708 			goto out;
709 
710 		dmirror_migrate_alloc_and_copy(&args, dmirror);
711 		migrate_vma_pages(&args);
712 		dmirror_migrate_finalize_and_map(&args, dmirror);
713 		migrate_vma_finalize(&args);
714 	}
715 	mmap_read_unlock(mm);
716 	mmput(mm);
717 
718 	/* Return the migrated data for verification. */
719 	ret = dmirror_bounce_init(&bounce, start, size);
720 	if (ret)
721 		return ret;
722 	mutex_lock(&dmirror->mutex);
723 	ret = dmirror_do_read(dmirror, start, end, &bounce);
724 	mutex_unlock(&dmirror->mutex);
725 	if (ret == 0) {
726 		if (copy_to_user(u64_to_user_ptr(cmd->ptr), bounce.ptr,
727 				 bounce.size))
728 			ret = -EFAULT;
729 	}
730 	cmd->cpages = bounce.cpages;
731 	dmirror_bounce_fini(&bounce);
732 	return ret;
733 
734 out:
735 	mmap_read_unlock(mm);
736 	mmput(mm);
737 	return ret;
738 }
739 
740 static void dmirror_mkentry(struct dmirror *dmirror, struct hmm_range *range,
741 			    unsigned char *perm, unsigned long entry)
742 {
743 	struct page *page;
744 
745 	if (entry & HMM_PFN_ERROR) {
746 		*perm = HMM_DMIRROR_PROT_ERROR;
747 		return;
748 	}
749 	if (!(entry & HMM_PFN_VALID)) {
750 		*perm = HMM_DMIRROR_PROT_NONE;
751 		return;
752 	}
753 
754 	page = hmm_pfn_to_page(entry);
755 	if (is_device_private_page(page)) {
756 		/* Is the page migrated to this device or some other? */
757 		if (dmirror->mdevice == dmirror_page_to_device(page))
758 			*perm = HMM_DMIRROR_PROT_DEV_PRIVATE_LOCAL;
759 		else
760 			*perm = HMM_DMIRROR_PROT_DEV_PRIVATE_REMOTE;
761 	} else if (is_zero_pfn(page_to_pfn(page)))
762 		*perm = HMM_DMIRROR_PROT_ZERO;
763 	else
764 		*perm = HMM_DMIRROR_PROT_NONE;
765 	if (entry & HMM_PFN_WRITE)
766 		*perm |= HMM_DMIRROR_PROT_WRITE;
767 	else
768 		*perm |= HMM_DMIRROR_PROT_READ;
769 	if (hmm_pfn_to_map_order(entry) + PAGE_SHIFT == PMD_SHIFT)
770 		*perm |= HMM_DMIRROR_PROT_PMD;
771 	else if (hmm_pfn_to_map_order(entry) + PAGE_SHIFT == PUD_SHIFT)
772 		*perm |= HMM_DMIRROR_PROT_PUD;
773 }
774 
775 static bool dmirror_snapshot_invalidate(struct mmu_interval_notifier *mni,
776 				const struct mmu_notifier_range *range,
777 				unsigned long cur_seq)
778 {
779 	struct dmirror_interval *dmi =
780 		container_of(mni, struct dmirror_interval, notifier);
781 	struct dmirror *dmirror = dmi->dmirror;
782 
783 	if (mmu_notifier_range_blockable(range))
784 		mutex_lock(&dmirror->mutex);
785 	else if (!mutex_trylock(&dmirror->mutex))
786 		return false;
787 
788 	/*
789 	 * Snapshots only need to set the sequence number since any
790 	 * invalidation in the interval invalidates the whole snapshot.
791 	 */
792 	mmu_interval_set_seq(mni, cur_seq);
793 
794 	mutex_unlock(&dmirror->mutex);
795 	return true;
796 }
797 
798 static const struct mmu_interval_notifier_ops dmirror_mrn_ops = {
799 	.invalidate = dmirror_snapshot_invalidate,
800 };
801 
802 static int dmirror_range_snapshot(struct dmirror *dmirror,
803 				  struct hmm_range *range,
804 				  unsigned char *perm)
805 {
806 	struct mm_struct *mm = dmirror->notifier.mm;
807 	struct dmirror_interval notifier;
808 	unsigned long timeout =
809 		jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT);
810 	unsigned long i;
811 	unsigned long n;
812 	int ret = 0;
813 
814 	notifier.dmirror = dmirror;
815 	range->notifier = &notifier.notifier;
816 
817 	ret = mmu_interval_notifier_insert(range->notifier, mm,
818 			range->start, range->end - range->start,
819 			&dmirror_mrn_ops);
820 	if (ret)
821 		return ret;
822 
823 	while (true) {
824 		if (time_after(jiffies, timeout)) {
825 			ret = -EBUSY;
826 			goto out;
827 		}
828 
829 		range->notifier_seq = mmu_interval_read_begin(range->notifier);
830 
831 		mmap_read_lock(mm);
832 		ret = hmm_range_fault(range);
833 		mmap_read_unlock(mm);
834 		if (ret) {
835 			if (ret == -EBUSY)
836 				continue;
837 			goto out;
838 		}
839 
840 		mutex_lock(&dmirror->mutex);
841 		if (mmu_interval_read_retry(range->notifier,
842 					    range->notifier_seq)) {
843 			mutex_unlock(&dmirror->mutex);
844 			continue;
845 		}
846 		break;
847 	}
848 
849 	n = (range->end - range->start) >> PAGE_SHIFT;
850 	for (i = 0; i < n; i++)
851 		dmirror_mkentry(dmirror, range, perm + i, range->hmm_pfns[i]);
852 
853 	mutex_unlock(&dmirror->mutex);
854 out:
855 	mmu_interval_notifier_remove(range->notifier);
856 	return ret;
857 }
858 
859 static int dmirror_snapshot(struct dmirror *dmirror,
860 			    struct hmm_dmirror_cmd *cmd)
861 {
862 	struct mm_struct *mm = dmirror->notifier.mm;
863 	unsigned long start, end;
864 	unsigned long size = cmd->npages << PAGE_SHIFT;
865 	unsigned long addr;
866 	unsigned long next;
867 	unsigned long pfns[64];
868 	unsigned char perm[64];
869 	char __user *uptr;
870 	struct hmm_range range = {
871 		.hmm_pfns = pfns,
872 		.dev_private_owner = dmirror->mdevice,
873 	};
874 	int ret = 0;
875 
876 	start = cmd->addr;
877 	end = start + size;
878 	if (end < start)
879 		return -EINVAL;
880 
881 	/* Since the mm is for the mirrored process, get a reference first. */
882 	if (!mmget_not_zero(mm))
883 		return -EINVAL;
884 
885 	/*
886 	 * Register a temporary notifier to detect invalidations even if it
887 	 * overlaps with other mmu_interval_notifiers.
888 	 */
889 	uptr = u64_to_user_ptr(cmd->ptr);
890 	for (addr = start; addr < end; addr = next) {
891 		unsigned long n;
892 
893 		next = min(addr + (ARRAY_SIZE(pfns) << PAGE_SHIFT), end);
894 		range.start = addr;
895 		range.end = next;
896 
897 		ret = dmirror_range_snapshot(dmirror, &range, perm);
898 		if (ret)
899 			break;
900 
901 		n = (range.end - range.start) >> PAGE_SHIFT;
902 		if (copy_to_user(uptr, perm, n)) {
903 			ret = -EFAULT;
904 			break;
905 		}
906 
907 		cmd->cpages += n;
908 		uptr += n;
909 	}
910 	mmput(mm);
911 
912 	return ret;
913 }
914 
915 static long dmirror_fops_unlocked_ioctl(struct file *filp,
916 					unsigned int command,
917 					unsigned long arg)
918 {
919 	void __user *uarg = (void __user *)arg;
920 	struct hmm_dmirror_cmd cmd;
921 	struct dmirror *dmirror;
922 	int ret;
923 
924 	dmirror = filp->private_data;
925 	if (!dmirror)
926 		return -EINVAL;
927 
928 	if (copy_from_user(&cmd, uarg, sizeof(cmd)))
929 		return -EFAULT;
930 
931 	if (cmd.addr & ~PAGE_MASK)
932 		return -EINVAL;
933 	if (cmd.addr >= (cmd.addr + (cmd.npages << PAGE_SHIFT)))
934 		return -EINVAL;
935 
936 	cmd.cpages = 0;
937 	cmd.faults = 0;
938 
939 	switch (command) {
940 	case HMM_DMIRROR_READ:
941 		ret = dmirror_read(dmirror, &cmd);
942 		break;
943 
944 	case HMM_DMIRROR_WRITE:
945 		ret = dmirror_write(dmirror, &cmd);
946 		break;
947 
948 	case HMM_DMIRROR_MIGRATE:
949 		ret = dmirror_migrate(dmirror, &cmd);
950 		break;
951 
952 	case HMM_DMIRROR_SNAPSHOT:
953 		ret = dmirror_snapshot(dmirror, &cmd);
954 		break;
955 
956 	default:
957 		return -EINVAL;
958 	}
959 	if (ret)
960 		return ret;
961 
962 	if (copy_to_user(uarg, &cmd, sizeof(cmd)))
963 		return -EFAULT;
964 
965 	return 0;
966 }
967 
968 static const struct file_operations dmirror_fops = {
969 	.open		= dmirror_fops_open,
970 	.release	= dmirror_fops_release,
971 	.unlocked_ioctl = dmirror_fops_unlocked_ioctl,
972 	.llseek		= default_llseek,
973 	.owner		= THIS_MODULE,
974 };
975 
976 static void dmirror_devmem_free(struct page *page)
977 {
978 	struct page *rpage = page->zone_device_data;
979 	struct dmirror_device *mdevice;
980 
981 	if (rpage)
982 		__free_page(rpage);
983 
984 	mdevice = dmirror_page_to_device(page);
985 
986 	spin_lock(&mdevice->lock);
987 	mdevice->cfree++;
988 	page->zone_device_data = mdevice->free_pages;
989 	mdevice->free_pages = page;
990 	spin_unlock(&mdevice->lock);
991 }
992 
993 static vm_fault_t dmirror_devmem_fault_alloc_and_copy(struct migrate_vma *args,
994 						      struct dmirror *dmirror)
995 {
996 	const unsigned long *src = args->src;
997 	unsigned long *dst = args->dst;
998 	unsigned long start = args->start;
999 	unsigned long end = args->end;
1000 	unsigned long addr;
1001 
1002 	for (addr = start; addr < end; addr += PAGE_SIZE,
1003 				       src++, dst++) {
1004 		struct page *dpage, *spage;
1005 
1006 		spage = migrate_pfn_to_page(*src);
1007 		if (!spage || !(*src & MIGRATE_PFN_MIGRATE))
1008 			continue;
1009 		spage = spage->zone_device_data;
1010 
1011 		dpage = alloc_page_vma(GFP_HIGHUSER_MOVABLE, args->vma, addr);
1012 		if (!dpage)
1013 			continue;
1014 
1015 		lock_page(dpage);
1016 		xa_erase(&dmirror->pt, addr >> PAGE_SHIFT);
1017 		copy_highpage(dpage, spage);
1018 		*dst = migrate_pfn(page_to_pfn(dpage)) | MIGRATE_PFN_LOCKED;
1019 		if (*src & MIGRATE_PFN_WRITE)
1020 			*dst |= MIGRATE_PFN_WRITE;
1021 	}
1022 	return 0;
1023 }
1024 
1025 static vm_fault_t dmirror_devmem_fault(struct vm_fault *vmf)
1026 {
1027 	struct migrate_vma args;
1028 	unsigned long src_pfns;
1029 	unsigned long dst_pfns;
1030 	struct page *rpage;
1031 	struct dmirror *dmirror;
1032 	vm_fault_t ret;
1033 
1034 	/*
1035 	 * Normally, a device would use the page->zone_device_data to point to
1036 	 * the mirror but here we use it to hold the page for the simulated
1037 	 * device memory and that page holds the pointer to the mirror.
1038 	 */
1039 	rpage = vmf->page->zone_device_data;
1040 	dmirror = rpage->zone_device_data;
1041 
1042 	/* FIXME demonstrate how we can adjust migrate range */
1043 	args.vma = vmf->vma;
1044 	args.start = vmf->address;
1045 	args.end = args.start + PAGE_SIZE;
1046 	args.src = &src_pfns;
1047 	args.dst = &dst_pfns;
1048 	args.pgmap_owner = dmirror->mdevice;
1049 	args.flags = MIGRATE_VMA_SELECT_DEVICE_PRIVATE;
1050 
1051 	if (migrate_vma_setup(&args))
1052 		return VM_FAULT_SIGBUS;
1053 
1054 	ret = dmirror_devmem_fault_alloc_and_copy(&args, dmirror);
1055 	if (ret)
1056 		return ret;
1057 	migrate_vma_pages(&args);
1058 	/*
1059 	 * No device finalize step is needed since
1060 	 * dmirror_devmem_fault_alloc_and_copy() will have already
1061 	 * invalidated the device page table.
1062 	 */
1063 	migrate_vma_finalize(&args);
1064 	return 0;
1065 }
1066 
1067 static const struct dev_pagemap_ops dmirror_devmem_ops = {
1068 	.page_free	= dmirror_devmem_free,
1069 	.migrate_to_ram	= dmirror_devmem_fault,
1070 };
1071 
1072 static int dmirror_device_init(struct dmirror_device *mdevice, int id)
1073 {
1074 	dev_t dev;
1075 	int ret;
1076 
1077 	dev = MKDEV(MAJOR(dmirror_dev), id);
1078 	mutex_init(&mdevice->devmem_lock);
1079 	spin_lock_init(&mdevice->lock);
1080 
1081 	cdev_init(&mdevice->cdevice, &dmirror_fops);
1082 	mdevice->cdevice.owner = THIS_MODULE;
1083 	ret = cdev_add(&mdevice->cdevice, dev, 1);
1084 	if (ret)
1085 		return ret;
1086 
1087 	/* Build a list of free ZONE_DEVICE private struct pages */
1088 	dmirror_allocate_chunk(mdevice, NULL);
1089 
1090 	return 0;
1091 }
1092 
1093 static void dmirror_device_remove(struct dmirror_device *mdevice)
1094 {
1095 	unsigned int i;
1096 
1097 	if (mdevice->devmem_chunks) {
1098 		for (i = 0; i < mdevice->devmem_count; i++) {
1099 			struct dmirror_chunk *devmem =
1100 				mdevice->devmem_chunks[i];
1101 
1102 			memunmap_pages(&devmem->pagemap);
1103 			release_mem_region(devmem->pagemap.range.start,
1104 					   range_len(&devmem->pagemap.range));
1105 			kfree(devmem);
1106 		}
1107 		kfree(mdevice->devmem_chunks);
1108 	}
1109 
1110 	cdev_del(&mdevice->cdevice);
1111 }
1112 
1113 static int __init hmm_dmirror_init(void)
1114 {
1115 	int ret;
1116 	int id;
1117 
1118 	ret = alloc_chrdev_region(&dmirror_dev, 0, DMIRROR_NDEVICES,
1119 				  "HMM_DMIRROR");
1120 	if (ret)
1121 		goto err_unreg;
1122 
1123 	for (id = 0; id < DMIRROR_NDEVICES; id++) {
1124 		ret = dmirror_device_init(dmirror_devices + id, id);
1125 		if (ret)
1126 			goto err_chrdev;
1127 	}
1128 
1129 	pr_info("HMM test module loaded. This is only for testing HMM.\n");
1130 	return 0;
1131 
1132 err_chrdev:
1133 	while (--id >= 0)
1134 		dmirror_device_remove(dmirror_devices + id);
1135 	unregister_chrdev_region(dmirror_dev, DMIRROR_NDEVICES);
1136 err_unreg:
1137 	return ret;
1138 }
1139 
1140 static void __exit hmm_dmirror_exit(void)
1141 {
1142 	int id;
1143 
1144 	for (id = 0; id < DMIRROR_NDEVICES; id++)
1145 		dmirror_device_remove(dmirror_devices + id);
1146 	unregister_chrdev_region(dmirror_dev, DMIRROR_NDEVICES);
1147 }
1148 
1149 module_init(hmm_dmirror_init);
1150 module_exit(hmm_dmirror_exit);
1151 MODULE_LICENSE("GPL");
1152