1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Userfaultfd tests common header
4  *
5  * Copyright (C) 2015-2023  Red Hat, Inc.
6  */
7 #ifndef __UFFD_COMMON_H__
8 #define __UFFD_COMMON_H__
9 
10 #define _GNU_SOURCE
11 #include <stdio.h>
12 #include <errno.h>
13 #include <unistd.h>
14 #include <stdlib.h>
15 #include <sys/types.h>
16 #include <sys/stat.h>
17 #include <fcntl.h>
18 #include <time.h>
19 #include <signal.h>
20 #include <poll.h>
21 #include <string.h>
22 #include <linux/mman.h>
23 #include <sys/mman.h>
24 #include <sys/syscall.h>
25 #include <sys/ioctl.h>
26 #include <sys/wait.h>
27 #include <pthread.h>
28 #include <linux/userfaultfd.h>
29 #include <setjmp.h>
30 #include <stdbool.h>
31 #include <assert.h>
32 #include <inttypes.h>
33 #include <stdint.h>
34 #include <sys/random.h>
35 #include <stdatomic.h>
36 
37 #include "../kselftest.h"
38 #include "vm_util.h"
39 
40 #define UFFD_FLAGS	(O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY)
41 
42 #define _err(fmt, ...)						\
43 	do {							\
44 		int ret = errno;				\
45 		fprintf(stderr, "ERROR: " fmt, ##__VA_ARGS__);	\
46 		fprintf(stderr, " (errno=%d, @%s:%d)\n",	\
47 			ret, __FILE__, __LINE__);		\
48 	} while (0)
49 
50 #define errexit(exitcode, fmt, ...)		\
51 	do {					\
52 		_err(fmt, ##__VA_ARGS__);	\
53 		exit(exitcode);			\
54 	} while (0)
55 
56 #define err(fmt, ...) errexit(1, fmt, ##__VA_ARGS__)
57 
58 /* pthread_mutex_t starts at page offset 0 */
59 #define area_mutex(___area, ___nr)					\
60 	((pthread_mutex_t *) ((___area) + (___nr)*page_size))
61 /*
62  * count is placed in the page after pthread_mutex_t naturally aligned
63  * to avoid non alignment faults on non-x86 archs.
64  */
65 #define area_count(___area, ___nr)					\
66 	((volatile unsigned long long *) ((unsigned long)		\
67 				 ((___area) + (___nr)*page_size +	\
68 				  sizeof(pthread_mutex_t) +		\
69 				  sizeof(unsigned long long) - 1) &	\
70 				 ~(unsigned long)(sizeof(unsigned long long) \
71 						  -  1)))
72 
73 /* Userfaultfd test statistics */
74 struct uffd_args {
75 	int cpu;
76 	/* Whether apply wr-protects when installing pages */
77 	bool apply_wp;
78 	unsigned long missing_faults;
79 	unsigned long wp_faults;
80 	unsigned long minor_faults;
81 
82 	/* A custom fault handler; defaults to uffd_handle_page_fault. */
83 	void (*handle_fault)(struct uffd_msg *msg, struct uffd_args *args);
84 };
85 
86 struct uffd_test_ops {
87 	int (*allocate_area)(void **alloc_area, bool is_src);
88 	void (*release_pages)(char *rel_area);
89 	void (*alias_mapping)(__u64 *start, size_t len, unsigned long offset);
90 	void (*check_pmd_mapping)(void *p, int expect_nr_hpages);
91 };
92 typedef struct uffd_test_ops uffd_test_ops_t;
93 
94 extern unsigned long nr_cpus, nr_pages, nr_pages_per_cpu, page_size;
95 extern char *area_src, *area_src_alias, *area_dst, *area_dst_alias, *area_remap;
96 extern int uffd, uffd_flags, finished, *pipefd, test_type;
97 extern bool map_shared;
98 extern bool test_uffdio_wp;
99 extern unsigned long long *count_verify;
100 extern volatile bool test_uffdio_copy_eexist;
101 extern atomic_bool ready_for_fork;
102 
103 extern uffd_test_ops_t anon_uffd_test_ops;
104 extern uffd_test_ops_t shmem_uffd_test_ops;
105 extern uffd_test_ops_t hugetlb_uffd_test_ops;
106 extern uffd_test_ops_t *uffd_test_ops;
107 
108 void uffd_stats_report(struct uffd_args *args, int n_cpus);
109 int uffd_test_ctx_init(uint64_t features, const char **errmsg);
110 int userfaultfd_open(uint64_t *features);
111 int uffd_read_msg(int ufd, struct uffd_msg *msg);
112 void wp_range(int ufd, __u64 start, __u64 len, bool wp);
113 void uffd_handle_page_fault(struct uffd_msg *msg, struct uffd_args *args);
114 int __copy_page(int ufd, unsigned long offset, bool retry, bool wp);
115 int copy_page(int ufd, unsigned long offset, bool wp);
116 void *uffd_poll_thread(void *arg);
117 
118 int uffd_open_dev(unsigned int flags);
119 int uffd_open_sys(unsigned int flags);
120 int uffd_open(unsigned int flags);
121 int uffd_get_features(uint64_t *features);
122 
123 #define TEST_ANON	1
124 #define TEST_HUGETLB	2
125 #define TEST_SHMEM	3
126 
127 #endif
128