1 // SPDX-License-Identifier: GPL-2.0-only
5  * Copyright (C) 2015-2023  Red Hat, Inc.
8 #include "uffd-common.h"
15 int uffd = -1, uffd_flags, finished, *pipefd, test_type;
29 	mem_fd = memfd_create("uffd-test", memfd_flags);  in uffd_mem_fd_create()
51 			   MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);  in anon_allocate_area()
54 		return -errno;  in anon_allocate_area()
88 		return -errno;  in hugetlb_allocate_area()
95 			return -errno;  in hugetlb_allocate_area()
106 	close(mem_fd);  in hugetlb_allocate_area()
145 		return -errno;  in shmem_allocate_area()
155 		return -errno;  in shmem_allocate_area()
165 	close(mem_fd);  in shmem_allocate_area()
242 		return -1;  in userfaultfd_open()
249 		return -1;  in userfaultfd_open()
272 			if (close(pipefd[i]))  in uffd_test_ctx_clear()
273 				err("close pipefd");  in uffd_test_ctx_clear()
284 	if (uffd != -1) {  in uffd_test_ctx_clear()
285 		if (close(uffd))  in uffd_test_ctx_clear()
286 			err("close uffd");  in uffd_test_ctx_clear()
287 		uffd = -1;  in uffd_test_ctx_clear()
304 	ret = uffd_test_ops->allocate_area((void **)&area_src, true);  in uffd_test_ctx_init()
305 	ret |= uffd_test_ops->allocate_area((void **)&area_dst, false);  in uffd_test_ctx_init()
330 		 * zero, so leave a placeholder below always non-zero  in uffd_test_ctx_init()
355 	uffd_test_ops->release_pages(area_dst);  in uffd_test_ctx_init()
372 	prms.range.start = start;  in wp_range()
373 	prms.range.len = len;  in wp_range()
374 	/* Undo write-protect, do wakeup after that */  in wp_range()
386 	req.range.start = start;  in continue_range()
387 	req.range.len = len;  in continue_range()
399 	 * error (-EEXIST) on purpose, to verify doing so doesn't cause a BUG.  in continue_range()
403 	if (ret >= 0 || req.mapped != -EEXIST)  in continue_range()
429 	if (msg->event != UFFD_EVENT_PAGEFAULT)  in uffd_handle_page_fault()
430 		err("unexpected msg event %u", msg->event);  in uffd_handle_page_fault()
432 	if (msg->arg.pagefault.flags & UFFD_PAGEFAULT_FLAG_WP) {  in uffd_handle_page_fault()
434 		wp_range(uffd, msg->arg.pagefault.address, page_size, false);  in uffd_handle_page_fault()
435 		args->wp_faults++;  in uffd_handle_page_fault()
436 	} else if (msg->arg.pagefault.flags & UFFD_PAGEFAULT_FLAG_MINOR) {  in uffd_handle_page_fault()
443 		 * To prove we can modify the original range for testing  in uffd_handle_page_fault()
444 		 * purposes, we're going to bit flip this range before  in uffd_handle_page_fault()
448 		 * area_dst (non-UFFD-registered) and area_dst_alias  in uffd_handle_page_fault()
449 		 * (UFFD-registered).  in uffd_handle_page_fault()
453 				   ((char *)msg->arg.pagefault.address -  in uffd_handle_page_fault()
457 		continue_range(uffd, msg->arg.pagefault.address, page_size,  in uffd_handle_page_fault()
458 			       args->apply_wp);  in uffd_handle_page_fault()
459 		args->minor_faults++;  in uffd_handle_page_fault()
475 		 * logically a pthread-compatible lib can implement the  in uffd_handle_page_fault()
482 		if (msg->arg.pagefault.flags & UFFD_PAGEFAULT_FLAG_WRITE)  in uffd_handle_page_fault()
485 		offset = (char *)(unsigned long)msg->arg.pagefault.address - area_dst;  in uffd_handle_page_fault()
486 		offset &= ~(page_size-1);  in uffd_handle_page_fault()
488 		if (copy_page(uffd, offset, args->apply_wp))  in uffd_handle_page_fault()
489 			args->missing_faults++;  in uffd_handle_page_fault()
496 	unsigned long cpu = args->cpu;  in uffd_poll_thread()
503 	if (!args->handle_fault)  in uffd_poll_thread()
504 		args->handle_fault = uffd_handle_page_fault;  in uffd_poll_thread()
514 		ret = poll(pollfd, 2, -1);  in uffd_poll_thread()
536 			args->handle_fault(&msg, args);  in uffd_poll_thread()
539 			close(uffd);  in uffd_poll_thread()
544 			uffd_reg.range.start = msg.arg.remove.start;  in uffd_poll_thread()
545 			uffd_reg.range.len = msg.arg.remove.end -  in uffd_poll_thread()
547 			if (ioctl(uffd, UFFDIO_UNREGISTER, &uffd_reg.range))  in uffd_poll_thread()
563 	uffd_test_ops->alias_mapping(&uffdio_copy->dst,  in retry_copy_page()
564 				     uffdio_copy->len,  in retry_copy_page()
568 		if (uffdio_copy->copy != -EEXIST)  in retry_copy_page()
570 			    (int64_t)uffdio_copy->copy);  in retry_copy_page()
573 		    (int64_t)uffdio_copy->copy);  in retry_copy_page()
605 		if (uffdio_copy.copy != -EEXIST)  in __copy_page()
634 	close(fd);  in uffd_open_dev()
644 	return -1;  in uffd_open_sys()
668 		/* Maybe the kernel is older than user-only mode? */  in uffd_get_features()
675 		close(fd);  in uffd_get_features()
676 		return -errno;  in uffd_get_features()
680 	close(fd);  in uffd_get_features()