1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * A test of splitting PMD THPs and PTE-mapped THPs from a specified virtual
4  * address range in a process via <debugfs>/split_huge_pages interface.
5  */
6 
7 #define _GNU_SOURCE
8 #include <stdio.h>
9 #include <stdlib.h>
10 #include <stdarg.h>
11 #include <unistd.h>
12 #include <inttypes.h>
13 #include <string.h>
14 #include <fcntl.h>
15 #include <sys/mman.h>
16 #include <sys/mount.h>
17 #include <malloc.h>
18 #include <stdbool.h>
19 #include "vm_util.h"
20 
21 uint64_t pagesize;
22 unsigned int pageshift;
23 uint64_t pmd_pagesize;
24 
25 #define SPLIT_DEBUGFS "/sys/kernel/debug/split_huge_pages"
26 #define INPUT_MAX 80
27 
28 #define PID_FMT "%d,0x%lx,0x%lx"
29 #define PATH_FMT "%s,0x%lx,0x%lx"
30 
31 #define PFN_MASK     ((1UL<<55)-1)
32 #define KPF_THP      (1UL<<22)
33 
34 int is_backed_by_thp(char *vaddr, int pagemap_file, int kpageflags_file)
35 {
36 	uint64_t paddr;
37 	uint64_t page_flags;
38 
39 	if (pagemap_file) {
40 		pread(pagemap_file, &paddr, sizeof(paddr),
41 			((long)vaddr >> pageshift) * sizeof(paddr));
42 
43 		if (kpageflags_file) {
44 			pread(kpageflags_file, &page_flags, sizeof(page_flags),
45 				(paddr & PFN_MASK) * sizeof(page_flags));
46 
47 			return !!(page_flags & KPF_THP);
48 		}
49 	}
50 	return 0;
51 }
52 
53 static int write_file(const char *path, const char *buf, size_t buflen)
54 {
55 	int fd;
56 	ssize_t numwritten;
57 
58 	fd = open(path, O_WRONLY);
59 	if (fd == -1)
60 		return 0;
61 
62 	numwritten = write(fd, buf, buflen - 1);
63 	close(fd);
64 	if (numwritten < 1)
65 		return 0;
66 
67 	return (unsigned int) numwritten;
68 }
69 
70 static void write_debugfs(const char *fmt, ...)
71 {
72 	char input[INPUT_MAX];
73 	int ret;
74 	va_list argp;
75 
76 	va_start(argp, fmt);
77 	ret = vsnprintf(input, INPUT_MAX, fmt, argp);
78 	va_end(argp);
79 
80 	if (ret >= INPUT_MAX) {
81 		printf("%s: Debugfs input is too long\n", __func__);
82 		exit(EXIT_FAILURE);
83 	}
84 
85 	if (!write_file(SPLIT_DEBUGFS, input, ret + 1)) {
86 		perror(SPLIT_DEBUGFS);
87 		exit(EXIT_FAILURE);
88 	}
89 }
90 
91 void split_pmd_thp(void)
92 {
93 	char *one_page;
94 	size_t len = 4 * pmd_pagesize;
95 	size_t i;
96 
97 	one_page = memalign(pmd_pagesize, len);
98 
99 	if (!one_page) {
100 		printf("Fail to allocate memory\n");
101 		exit(EXIT_FAILURE);
102 	}
103 
104 	madvise(one_page, len, MADV_HUGEPAGE);
105 
106 	for (i = 0; i < len; i++)
107 		one_page[i] = (char)i;
108 
109 	if (!check_huge_anon(one_page, 1, pmd_pagesize)) {
110 		printf("No THP is allocated\n");
111 		exit(EXIT_FAILURE);
112 	}
113 
114 	/* split all THPs */
115 	write_debugfs(PID_FMT, getpid(), (uint64_t)one_page,
116 		(uint64_t)one_page + len);
117 
118 	for (i = 0; i < len; i++)
119 		if (one_page[i] != (char)i) {
120 			printf("%ld byte corrupted\n", i);
121 			exit(EXIT_FAILURE);
122 		}
123 
124 
125 	if (check_huge_anon(one_page, 0, pmd_pagesize)) {
126 		printf("Still AnonHugePages not split\n");
127 		exit(EXIT_FAILURE);
128 	}
129 
130 	printf("Split huge pages successful\n");
131 	free(one_page);
132 }
133 
134 void split_pte_mapped_thp(void)
135 {
136 	char *one_page, *pte_mapped, *pte_mapped2;
137 	size_t len = 4 * pmd_pagesize;
138 	uint64_t thp_size;
139 	size_t i;
140 	const char *pagemap_template = "/proc/%d/pagemap";
141 	const char *kpageflags_proc = "/proc/kpageflags";
142 	char pagemap_proc[255];
143 	int pagemap_fd;
144 	int kpageflags_fd;
145 
146 	if (snprintf(pagemap_proc, 255, pagemap_template, getpid()) < 0) {
147 		perror("get pagemap proc error");
148 		exit(EXIT_FAILURE);
149 	}
150 	pagemap_fd = open(pagemap_proc, O_RDONLY);
151 
152 	if (pagemap_fd == -1) {
153 		perror("read pagemap:");
154 		exit(EXIT_FAILURE);
155 	}
156 
157 	kpageflags_fd = open(kpageflags_proc, O_RDONLY);
158 
159 	if (kpageflags_fd == -1) {
160 		perror("read kpageflags:");
161 		exit(EXIT_FAILURE);
162 	}
163 
164 	one_page = mmap((void *)(1UL << 30), len, PROT_READ | PROT_WRITE,
165 			MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
166 
167 	madvise(one_page, len, MADV_HUGEPAGE);
168 
169 	for (i = 0; i < len; i++)
170 		one_page[i] = (char)i;
171 
172 	if (!check_huge_anon(one_page, 1, pmd_pagesize)) {
173 		printf("No THP is allocated\n");
174 		exit(EXIT_FAILURE);
175 	}
176 
177 	/* remap the first pagesize of first THP */
178 	pte_mapped = mremap(one_page, pagesize, pagesize, MREMAP_MAYMOVE);
179 
180 	/* remap the Nth pagesize of Nth THP */
181 	for (i = 1; i < 4; i++) {
182 		pte_mapped2 = mremap(one_page + pmd_pagesize * i + pagesize * i,
183 				     pagesize, pagesize,
184 				     MREMAP_MAYMOVE|MREMAP_FIXED,
185 				     pte_mapped + pagesize * i);
186 		if (pte_mapped2 == (char *)-1) {
187 			perror("mremap failed");
188 			exit(EXIT_FAILURE);
189 		}
190 	}
191 
192 	/* smap does not show THPs after mremap, use kpageflags instead */
193 	thp_size = 0;
194 	for (i = 0; i < pagesize * 4; i++)
195 		if (i % pagesize == 0 &&
196 		    is_backed_by_thp(&pte_mapped[i], pagemap_fd, kpageflags_fd))
197 			thp_size++;
198 
199 	if (thp_size != 4) {
200 		printf("Some THPs are missing during mremap\n");
201 		exit(EXIT_FAILURE);
202 	}
203 
204 	/* split all remapped THPs */
205 	write_debugfs(PID_FMT, getpid(), (uint64_t)pte_mapped,
206 		      (uint64_t)pte_mapped + pagesize * 4);
207 
208 	/* smap does not show THPs after mremap, use kpageflags instead */
209 	thp_size = 0;
210 	for (i = 0; i < pagesize * 4; i++) {
211 		if (pte_mapped[i] != (char)i) {
212 			printf("%ld byte corrupted\n", i);
213 			exit(EXIT_FAILURE);
214 		}
215 		if (i % pagesize == 0 &&
216 		    is_backed_by_thp(&pte_mapped[i], pagemap_fd, kpageflags_fd))
217 			thp_size++;
218 	}
219 
220 	if (thp_size) {
221 		printf("Still %ld THPs not split\n", thp_size);
222 		exit(EXIT_FAILURE);
223 	}
224 
225 	printf("Split PTE-mapped huge pages successful\n");
226 	munmap(one_page, len);
227 	close(pagemap_fd);
228 	close(kpageflags_fd);
229 }
230 
231 void split_file_backed_thp(void)
232 {
233 	int status;
234 	int fd;
235 	ssize_t num_written;
236 	char tmpfs_template[] = "/tmp/thp_split_XXXXXX";
237 	const char *tmpfs_loc = mkdtemp(tmpfs_template);
238 	char testfile[INPUT_MAX];
239 	uint64_t pgoff_start = 0, pgoff_end = 1024;
240 
241 	printf("Please enable pr_debug in split_huge_pages_in_file() if you need more info.\n");
242 
243 	status = mount("tmpfs", tmpfs_loc, "tmpfs", 0, "huge=always,size=4m");
244 
245 	if (status) {
246 		printf("Unable to create a tmpfs for testing\n");
247 		exit(EXIT_FAILURE);
248 	}
249 
250 	status = snprintf(testfile, INPUT_MAX, "%s/thp_file", tmpfs_loc);
251 	if (status >= INPUT_MAX) {
252 		printf("Fail to create file-backed THP split testing file\n");
253 		goto cleanup;
254 	}
255 
256 	fd = open(testfile, O_CREAT|O_WRONLY);
257 	if (fd == -1) {
258 		perror("Cannot open testing file\n");
259 		goto cleanup;
260 	}
261 
262 	/* write something to the file, so a file-backed THP can be allocated */
263 	num_written = write(fd, tmpfs_loc, strlen(tmpfs_loc) + 1);
264 	close(fd);
265 
266 	if (num_written < 1) {
267 		printf("Fail to write data to testing file\n");
268 		goto cleanup;
269 	}
270 
271 	/* split the file-backed THP */
272 	write_debugfs(PATH_FMT, testfile, pgoff_start, pgoff_end);
273 
274 	status = unlink(testfile);
275 	if (status)
276 		perror("Cannot remove testing file\n");
277 
278 cleanup:
279 	status = umount(tmpfs_loc);
280 	if (status) {
281 		printf("Unable to umount %s\n", tmpfs_loc);
282 		exit(EXIT_FAILURE);
283 	}
284 	status = rmdir(tmpfs_loc);
285 	if (status) {
286 		perror("cannot remove tmp dir");
287 		exit(EXIT_FAILURE);
288 	}
289 
290 	printf("file-backed THP split test done, please check dmesg for more information\n");
291 }
292 
293 int main(int argc, char **argv)
294 {
295 	if (geteuid() != 0) {
296 		printf("Please run the benchmark as root\n");
297 		exit(EXIT_FAILURE);
298 	}
299 
300 	pagesize = getpagesize();
301 	pageshift = ffs(pagesize) - 1;
302 	pmd_pagesize = read_pmd_pagesize();
303 
304 	split_pmd_thp();
305 	split_pte_mapped_thp();
306 	split_file_backed_thp();
307 
308 	return 0;
309 }
310