1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * A test of splitting PMD THPs and PTE-mapped THPs from a specified virtual 4 * address range in a process via <debugfs>/split_huge_pages interface. 5 */ 6 7 #define _GNU_SOURCE 8 #include <stdio.h> 9 #include <stdlib.h> 10 #include <stdarg.h> 11 #include <unistd.h> 12 #include <inttypes.h> 13 #include <string.h> 14 #include <fcntl.h> 15 #include <sys/mman.h> 16 #include <sys/mount.h> 17 #include <malloc.h> 18 #include <stdbool.h> 19 #include "vm_util.h" 20 21 uint64_t pagesize; 22 unsigned int pageshift; 23 uint64_t pmd_pagesize; 24 25 #define SPLIT_DEBUGFS "/sys/kernel/debug/split_huge_pages" 26 #define INPUT_MAX 80 27 28 #define PID_FMT "%d,0x%lx,0x%lx" 29 #define PATH_FMT "%s,0x%lx,0x%lx" 30 31 #define PFN_MASK ((1UL<<55)-1) 32 #define KPF_THP (1UL<<22) 33 34 int is_backed_by_thp(char *vaddr, int pagemap_file, int kpageflags_file) 35 { 36 uint64_t paddr; 37 uint64_t page_flags; 38 39 if (pagemap_file) { 40 pread(pagemap_file, &paddr, sizeof(paddr), 41 ((long)vaddr >> pageshift) * sizeof(paddr)); 42 43 if (kpageflags_file) { 44 pread(kpageflags_file, &page_flags, sizeof(page_flags), 45 (paddr & PFN_MASK) * sizeof(page_flags)); 46 47 return !!(page_flags & KPF_THP); 48 } 49 } 50 return 0; 51 } 52 53 static int write_file(const char *path, const char *buf, size_t buflen) 54 { 55 int fd; 56 ssize_t numwritten; 57 58 fd = open(path, O_WRONLY); 59 if (fd == -1) 60 return 0; 61 62 numwritten = write(fd, buf, buflen - 1); 63 close(fd); 64 if (numwritten < 1) 65 return 0; 66 67 return (unsigned int) numwritten; 68 } 69 70 static void write_debugfs(const char *fmt, ...) 71 { 72 char input[INPUT_MAX]; 73 int ret; 74 va_list argp; 75 76 va_start(argp, fmt); 77 ret = vsnprintf(input, INPUT_MAX, fmt, argp); 78 va_end(argp); 79 80 if (ret >= INPUT_MAX) { 81 printf("%s: Debugfs input is too long\n", __func__); 82 exit(EXIT_FAILURE); 83 } 84 85 if (!write_file(SPLIT_DEBUGFS, input, ret + 1)) { 86 perror(SPLIT_DEBUGFS); 87 exit(EXIT_FAILURE); 88 } 89 } 90 91 void split_pmd_thp(void) 92 { 93 char *one_page; 94 size_t len = 4 * pmd_pagesize; 95 size_t i; 96 97 one_page = memalign(pmd_pagesize, len); 98 99 if (!one_page) { 100 printf("Fail to allocate memory\n"); 101 exit(EXIT_FAILURE); 102 } 103 104 madvise(one_page, len, MADV_HUGEPAGE); 105 106 for (i = 0; i < len; i++) 107 one_page[i] = (char)i; 108 109 if (!check_huge_anon(one_page, 4, pmd_pagesize)) { 110 printf("No THP is allocated\n"); 111 exit(EXIT_FAILURE); 112 } 113 114 /* split all THPs */ 115 write_debugfs(PID_FMT, getpid(), (uint64_t)one_page, 116 (uint64_t)one_page + len); 117 118 for (i = 0; i < len; i++) 119 if (one_page[i] != (char)i) { 120 printf("%ld byte corrupted\n", i); 121 exit(EXIT_FAILURE); 122 } 123 124 125 if (!check_huge_anon(one_page, 0, pmd_pagesize)) { 126 printf("Still AnonHugePages not split\n"); 127 exit(EXIT_FAILURE); 128 } 129 130 printf("Split huge pages successful\n"); 131 free(one_page); 132 } 133 134 void split_pte_mapped_thp(void) 135 { 136 char *one_page, *pte_mapped, *pte_mapped2; 137 size_t len = 4 * pmd_pagesize; 138 uint64_t thp_size; 139 size_t i; 140 const char *pagemap_template = "/proc/%d/pagemap"; 141 const char *kpageflags_proc = "/proc/kpageflags"; 142 char pagemap_proc[255]; 143 int pagemap_fd; 144 int kpageflags_fd; 145 146 if (snprintf(pagemap_proc, 255, pagemap_template, getpid()) < 0) { 147 perror("get pagemap proc error"); 148 exit(EXIT_FAILURE); 149 } 150 pagemap_fd = open(pagemap_proc, O_RDONLY); 151 152 if (pagemap_fd == -1) { 153 perror("read pagemap:"); 154 exit(EXIT_FAILURE); 155 } 156 157 kpageflags_fd = open(kpageflags_proc, O_RDONLY); 158 159 if (kpageflags_fd == -1) { 160 perror("read kpageflags:"); 161 exit(EXIT_FAILURE); 162 } 163 164 one_page = mmap((void *)(1UL << 30), len, PROT_READ | PROT_WRITE, 165 MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); 166 167 madvise(one_page, len, MADV_HUGEPAGE); 168 169 for (i = 0; i < len; i++) 170 one_page[i] = (char)i; 171 172 if (!check_huge_anon(one_page, 4, pmd_pagesize)) { 173 printf("No THP is allocated\n"); 174 exit(EXIT_FAILURE); 175 } 176 177 /* remap the first pagesize of first THP */ 178 pte_mapped = mremap(one_page, pagesize, pagesize, MREMAP_MAYMOVE); 179 180 /* remap the Nth pagesize of Nth THP */ 181 for (i = 1; i < 4; i++) { 182 pte_mapped2 = mremap(one_page + pmd_pagesize * i + pagesize * i, 183 pagesize, pagesize, 184 MREMAP_MAYMOVE|MREMAP_FIXED, 185 pte_mapped + pagesize * i); 186 if (pte_mapped2 == (char *)-1) { 187 perror("mremap failed"); 188 exit(EXIT_FAILURE); 189 } 190 } 191 192 /* smap does not show THPs after mremap, use kpageflags instead */ 193 thp_size = 0; 194 for (i = 0; i < pagesize * 4; i++) 195 if (i % pagesize == 0 && 196 is_backed_by_thp(&pte_mapped[i], pagemap_fd, kpageflags_fd)) 197 thp_size++; 198 199 if (thp_size != 4) { 200 printf("Some THPs are missing during mremap\n"); 201 exit(EXIT_FAILURE); 202 } 203 204 /* split all remapped THPs */ 205 write_debugfs(PID_FMT, getpid(), (uint64_t)pte_mapped, 206 (uint64_t)pte_mapped + pagesize * 4); 207 208 /* smap does not show THPs after mremap, use kpageflags instead */ 209 thp_size = 0; 210 for (i = 0; i < pagesize * 4; i++) { 211 if (pte_mapped[i] != (char)i) { 212 printf("%ld byte corrupted\n", i); 213 exit(EXIT_FAILURE); 214 } 215 if (i % pagesize == 0 && 216 is_backed_by_thp(&pte_mapped[i], pagemap_fd, kpageflags_fd)) 217 thp_size++; 218 } 219 220 if (thp_size) { 221 printf("Still %ld THPs not split\n", thp_size); 222 exit(EXIT_FAILURE); 223 } 224 225 printf("Split PTE-mapped huge pages successful\n"); 226 munmap(one_page, len); 227 close(pagemap_fd); 228 close(kpageflags_fd); 229 } 230 231 void split_file_backed_thp(void) 232 { 233 int status; 234 int fd; 235 ssize_t num_written; 236 char tmpfs_template[] = "/tmp/thp_split_XXXXXX"; 237 const char *tmpfs_loc = mkdtemp(tmpfs_template); 238 char testfile[INPUT_MAX]; 239 uint64_t pgoff_start = 0, pgoff_end = 1024; 240 241 printf("Please enable pr_debug in split_huge_pages_in_file() if you need more info.\n"); 242 243 status = mount("tmpfs", tmpfs_loc, "tmpfs", 0, "huge=always,size=4m"); 244 245 if (status) { 246 printf("Unable to create a tmpfs for testing\n"); 247 exit(EXIT_FAILURE); 248 } 249 250 status = snprintf(testfile, INPUT_MAX, "%s/thp_file", tmpfs_loc); 251 if (status >= INPUT_MAX) { 252 printf("Fail to create file-backed THP split testing file\n"); 253 goto cleanup; 254 } 255 256 fd = open(testfile, O_CREAT|O_WRONLY); 257 if (fd == -1) { 258 perror("Cannot open testing file\n"); 259 goto cleanup; 260 } 261 262 /* write something to the file, so a file-backed THP can be allocated */ 263 num_written = write(fd, tmpfs_loc, strlen(tmpfs_loc) + 1); 264 close(fd); 265 266 if (num_written < 1) { 267 printf("Fail to write data to testing file\n"); 268 goto cleanup; 269 } 270 271 /* split the file-backed THP */ 272 write_debugfs(PATH_FMT, testfile, pgoff_start, pgoff_end); 273 274 status = unlink(testfile); 275 if (status) 276 perror("Cannot remove testing file\n"); 277 278 cleanup: 279 status = umount(tmpfs_loc); 280 if (status) { 281 printf("Unable to umount %s\n", tmpfs_loc); 282 exit(EXIT_FAILURE); 283 } 284 status = rmdir(tmpfs_loc); 285 if (status) { 286 perror("cannot remove tmp dir"); 287 exit(EXIT_FAILURE); 288 } 289 290 printf("file-backed THP split test done, please check dmesg for more information\n"); 291 } 292 293 int main(int argc, char **argv) 294 { 295 if (geteuid() != 0) { 296 printf("Please run the benchmark as root\n"); 297 exit(EXIT_FAILURE); 298 } 299 300 pagesize = getpagesize(); 301 pageshift = ffs(pagesize) - 1; 302 pmd_pagesize = read_pmd_pagesize(); 303 if (!pmd_pagesize) { 304 printf("Reading PMD pagesize failed\n"); 305 exit(EXIT_FAILURE); 306 } 307 308 split_pmd_thp(); 309 split_pte_mapped_thp(); 310 split_file_backed_thp(); 311 312 return 0; 313 } 314