1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * A test of splitting PMD THPs and PTE-mapped THPs from a specified virtual
4 * address range in a process via <debugfs>/split_huge_pages interface.
5 */
6
7 #define _GNU_SOURCE
8 #include <stdio.h>
9 #include <stdlib.h>
10 #include <stdarg.h>
11 #include <unistd.h>
12 #include <inttypes.h>
13 #include <string.h>
14 #include <fcntl.h>
15 #include <sys/mman.h>
16 #include <sys/mount.h>
17 #include <malloc.h>
18 #include <stdbool.h>
19 #include "vm_util.h"
20
21 uint64_t pagesize;
22 unsigned int pageshift;
23 uint64_t pmd_pagesize;
24
25 #define SPLIT_DEBUGFS "/sys/kernel/debug/split_huge_pages"
26 #define INPUT_MAX 80
27
28 #define PID_FMT "%d,0x%lx,0x%lx"
29 #define PATH_FMT "%s,0x%lx,0x%lx"
30
31 #define PFN_MASK ((1UL<<55)-1)
32 #define KPF_THP (1UL<<22)
33
is_backed_by_thp(char * vaddr,int pagemap_file,int kpageflags_file)34 int is_backed_by_thp(char *vaddr, int pagemap_file, int kpageflags_file)
35 {
36 uint64_t paddr;
37 uint64_t page_flags;
38
39 if (pagemap_file) {
40 pread(pagemap_file, &paddr, sizeof(paddr),
41 ((long)vaddr >> pageshift) * sizeof(paddr));
42
43 if (kpageflags_file) {
44 pread(kpageflags_file, &page_flags, sizeof(page_flags),
45 (paddr & PFN_MASK) * sizeof(page_flags));
46
47 return !!(page_flags & KPF_THP);
48 }
49 }
50 return 0;
51 }
52
write_file(const char * path,const char * buf,size_t buflen)53 static int write_file(const char *path, const char *buf, size_t buflen)
54 {
55 int fd;
56 ssize_t numwritten;
57
58 fd = open(path, O_WRONLY);
59 if (fd == -1)
60 return 0;
61
62 numwritten = write(fd, buf, buflen - 1);
63 close(fd);
64 if (numwritten < 1)
65 return 0;
66
67 return (unsigned int) numwritten;
68 }
69
write_debugfs(const char * fmt,...)70 static void write_debugfs(const char *fmt, ...)
71 {
72 char input[INPUT_MAX];
73 int ret;
74 va_list argp;
75
76 va_start(argp, fmt);
77 ret = vsnprintf(input, INPUT_MAX, fmt, argp);
78 va_end(argp);
79
80 if (ret >= INPUT_MAX) {
81 printf("%s: Debugfs input is too long\n", __func__);
82 exit(EXIT_FAILURE);
83 }
84
85 if (!write_file(SPLIT_DEBUGFS, input, ret + 1)) {
86 perror(SPLIT_DEBUGFS);
87 exit(EXIT_FAILURE);
88 }
89 }
90
split_pmd_thp(void)91 void split_pmd_thp(void)
92 {
93 char *one_page;
94 size_t len = 4 * pmd_pagesize;
95 size_t i;
96
97 one_page = memalign(pmd_pagesize, len);
98
99 if (!one_page) {
100 printf("Fail to allocate memory\n");
101 exit(EXIT_FAILURE);
102 }
103
104 madvise(one_page, len, MADV_HUGEPAGE);
105
106 for (i = 0; i < len; i++)
107 one_page[i] = (char)i;
108
109 if (!check_huge_anon(one_page, 4, pmd_pagesize)) {
110 printf("No THP is allocated\n");
111 exit(EXIT_FAILURE);
112 }
113
114 /* split all THPs */
115 write_debugfs(PID_FMT, getpid(), (uint64_t)one_page,
116 (uint64_t)one_page + len);
117
118 for (i = 0; i < len; i++)
119 if (one_page[i] != (char)i) {
120 printf("%ld byte corrupted\n", i);
121 exit(EXIT_FAILURE);
122 }
123
124
125 if (!check_huge_anon(one_page, 0, pmd_pagesize)) {
126 printf("Still AnonHugePages not split\n");
127 exit(EXIT_FAILURE);
128 }
129
130 printf("Split huge pages successful\n");
131 free(one_page);
132 }
133
split_pte_mapped_thp(void)134 void split_pte_mapped_thp(void)
135 {
136 char *one_page, *pte_mapped, *pte_mapped2;
137 size_t len = 4 * pmd_pagesize;
138 uint64_t thp_size;
139 size_t i;
140 const char *pagemap_template = "/proc/%d/pagemap";
141 const char *kpageflags_proc = "/proc/kpageflags";
142 char pagemap_proc[255];
143 int pagemap_fd;
144 int kpageflags_fd;
145
146 if (snprintf(pagemap_proc, 255, pagemap_template, getpid()) < 0) {
147 perror("get pagemap proc error");
148 exit(EXIT_FAILURE);
149 }
150 pagemap_fd = open(pagemap_proc, O_RDONLY);
151
152 if (pagemap_fd == -1) {
153 perror("read pagemap:");
154 exit(EXIT_FAILURE);
155 }
156
157 kpageflags_fd = open(kpageflags_proc, O_RDONLY);
158
159 if (kpageflags_fd == -1) {
160 perror("read kpageflags:");
161 exit(EXIT_FAILURE);
162 }
163
164 one_page = mmap((void *)(1UL << 30), len, PROT_READ | PROT_WRITE,
165 MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
166
167 madvise(one_page, len, MADV_HUGEPAGE);
168
169 for (i = 0; i < len; i++)
170 one_page[i] = (char)i;
171
172 if (!check_huge_anon(one_page, 4, pmd_pagesize)) {
173 printf("No THP is allocated\n");
174 exit(EXIT_FAILURE);
175 }
176
177 /* remap the first pagesize of first THP */
178 pte_mapped = mremap(one_page, pagesize, pagesize, MREMAP_MAYMOVE);
179
180 /* remap the Nth pagesize of Nth THP */
181 for (i = 1; i < 4; i++) {
182 pte_mapped2 = mremap(one_page + pmd_pagesize * i + pagesize * i,
183 pagesize, pagesize,
184 MREMAP_MAYMOVE|MREMAP_FIXED,
185 pte_mapped + pagesize * i);
186 if (pte_mapped2 == (char *)-1) {
187 perror("mremap failed");
188 exit(EXIT_FAILURE);
189 }
190 }
191
192 /* smap does not show THPs after mremap, use kpageflags instead */
193 thp_size = 0;
194 for (i = 0; i < pagesize * 4; i++)
195 if (i % pagesize == 0 &&
196 is_backed_by_thp(&pte_mapped[i], pagemap_fd, kpageflags_fd))
197 thp_size++;
198
199 if (thp_size != 4) {
200 printf("Some THPs are missing during mremap\n");
201 exit(EXIT_FAILURE);
202 }
203
204 /* split all remapped THPs */
205 write_debugfs(PID_FMT, getpid(), (uint64_t)pte_mapped,
206 (uint64_t)pte_mapped + pagesize * 4);
207
208 /* smap does not show THPs after mremap, use kpageflags instead */
209 thp_size = 0;
210 for (i = 0; i < pagesize * 4; i++) {
211 if (pte_mapped[i] != (char)i) {
212 printf("%ld byte corrupted\n", i);
213 exit(EXIT_FAILURE);
214 }
215 if (i % pagesize == 0 &&
216 is_backed_by_thp(&pte_mapped[i], pagemap_fd, kpageflags_fd))
217 thp_size++;
218 }
219
220 if (thp_size) {
221 printf("Still %ld THPs not split\n", thp_size);
222 exit(EXIT_FAILURE);
223 }
224
225 printf("Split PTE-mapped huge pages successful\n");
226 munmap(one_page, len);
227 close(pagemap_fd);
228 close(kpageflags_fd);
229 }
230
split_file_backed_thp(void)231 void split_file_backed_thp(void)
232 {
233 int status;
234 int fd;
235 ssize_t num_written;
236 char tmpfs_template[] = "/tmp/thp_split_XXXXXX";
237 const char *tmpfs_loc = mkdtemp(tmpfs_template);
238 char testfile[INPUT_MAX];
239 uint64_t pgoff_start = 0, pgoff_end = 1024;
240
241 printf("Please enable pr_debug in split_huge_pages_in_file() if you need more info.\n");
242
243 status = mount("tmpfs", tmpfs_loc, "tmpfs", 0, "huge=always,size=4m");
244
245 if (status) {
246 printf("Unable to create a tmpfs for testing\n");
247 exit(EXIT_FAILURE);
248 }
249
250 status = snprintf(testfile, INPUT_MAX, "%s/thp_file", tmpfs_loc);
251 if (status >= INPUT_MAX) {
252 printf("Fail to create file-backed THP split testing file\n");
253 goto cleanup;
254 }
255
256 fd = open(testfile, O_CREAT|O_WRONLY, 0664);
257 if (fd == -1) {
258 perror("Cannot open testing file\n");
259 goto cleanup;
260 }
261
262 /* write something to the file, so a file-backed THP can be allocated */
263 num_written = write(fd, tmpfs_loc, strlen(tmpfs_loc) + 1);
264 close(fd);
265
266 if (num_written < 1) {
267 printf("Fail to write data to testing file\n");
268 goto cleanup;
269 }
270
271 /* split the file-backed THP */
272 write_debugfs(PATH_FMT, testfile, pgoff_start, pgoff_end);
273
274 status = unlink(testfile);
275 if (status)
276 perror("Cannot remove testing file\n");
277
278 cleanup:
279 status = umount(tmpfs_loc);
280 if (status) {
281 printf("Unable to umount %s\n", tmpfs_loc);
282 exit(EXIT_FAILURE);
283 }
284 status = rmdir(tmpfs_loc);
285 if (status) {
286 perror("cannot remove tmp dir");
287 exit(EXIT_FAILURE);
288 }
289
290 printf("file-backed THP split test done, please check dmesg for more information\n");
291 }
292
main(int argc,char ** argv)293 int main(int argc, char **argv)
294 {
295 if (geteuid() != 0) {
296 printf("Please run the benchmark as root\n");
297 exit(EXIT_FAILURE);
298 }
299
300 pagesize = getpagesize();
301 pageshift = ffs(pagesize) - 1;
302 pmd_pagesize = read_pmd_pagesize();
303 if (!pmd_pagesize) {
304 printf("Reading PMD pagesize failed\n");
305 exit(EXIT_FAILURE);
306 }
307
308 split_pmd_thp();
309 split_pte_mapped_thp();
310 split_file_backed_thp();
311
312 return 0;
313 }
314