1 // SPDX-License-Identifier: GPL-2.0 2 #include <string.h> 3 #include <fcntl.h> 4 #include <dirent.h> 5 #include <sys/ioctl.h> 6 #include <linux/userfaultfd.h> 7 #include <sys/syscall.h> 8 #include <unistd.h> 9 #include "../kselftest.h" 10 #include "vm_util.h" 11 12 #define PMD_SIZE_FILE_PATH "/sys/kernel/mm/transparent_hugepage/hpage_pmd_size" 13 #define SMAP_FILE_PATH "/proc/self/smaps" 14 #define MAX_LINE_LENGTH 500 15 16 unsigned int __page_size; 17 unsigned int __page_shift; 18 19 uint64_t pagemap_get_entry(int fd, char *start) 20 { 21 const unsigned long pfn = (unsigned long)start / getpagesize(); 22 uint64_t entry; 23 int ret; 24 25 ret = pread(fd, &entry, sizeof(entry), pfn * sizeof(entry)); 26 if (ret != sizeof(entry)) 27 ksft_exit_fail_msg("reading pagemap failed\n"); 28 return entry; 29 } 30 31 bool pagemap_is_softdirty(int fd, char *start) 32 { 33 return pagemap_get_entry(fd, start) & PM_SOFT_DIRTY; 34 } 35 36 bool pagemap_is_swapped(int fd, char *start) 37 { 38 return pagemap_get_entry(fd, start) & PM_SWAP; 39 } 40 41 bool pagemap_is_populated(int fd, char *start) 42 { 43 return pagemap_get_entry(fd, start) & (PM_PRESENT | PM_SWAP); 44 } 45 46 unsigned long pagemap_get_pfn(int fd, char *start) 47 { 48 uint64_t entry = pagemap_get_entry(fd, start); 49 50 /* If present (63th bit), PFN is at bit 0 -- 54. */ 51 if (entry & PM_PRESENT) 52 return entry & 0x007fffffffffffffull; 53 return -1ul; 54 } 55 56 void clear_softdirty(void) 57 { 58 int ret; 59 const char *ctrl = "4"; 60 int fd = open("/proc/self/clear_refs", O_WRONLY); 61 62 if (fd < 0) 63 ksft_exit_fail_msg("opening clear_refs failed\n"); 64 ret = write(fd, ctrl, strlen(ctrl)); 65 close(fd); 66 if (ret != strlen(ctrl)) 67 ksft_exit_fail_msg("writing clear_refs failed\n"); 68 } 69 70 bool check_for_pattern(FILE *fp, const char *pattern, char *buf, size_t len) 71 { 72 while (fgets(buf, len, fp)) { 73 if (!strncmp(buf, pattern, strlen(pattern))) 74 return true; 75 } 76 return false; 77 } 78 79 uint64_t read_pmd_pagesize(void) 80 { 81 int fd; 82 char buf[20]; 83 ssize_t num_read; 84 85 fd = open(PMD_SIZE_FILE_PATH, O_RDONLY); 86 if (fd == -1) 87 return 0; 88 89 num_read = read(fd, buf, 19); 90 if (num_read < 1) { 91 close(fd); 92 return 0; 93 } 94 buf[num_read] = '\0'; 95 close(fd); 96 97 return strtoul(buf, NULL, 10); 98 } 99 100 bool __check_huge(void *addr, char *pattern, int nr_hpages, 101 uint64_t hpage_size) 102 { 103 uint64_t thp = -1; 104 int ret; 105 FILE *fp; 106 char buffer[MAX_LINE_LENGTH]; 107 char addr_pattern[MAX_LINE_LENGTH]; 108 109 ret = snprintf(addr_pattern, MAX_LINE_LENGTH, "%08lx-", 110 (unsigned long) addr); 111 if (ret >= MAX_LINE_LENGTH) 112 ksft_exit_fail_msg("%s: Pattern is too long\n", __func__); 113 114 fp = fopen(SMAP_FILE_PATH, "r"); 115 if (!fp) 116 ksft_exit_fail_msg("%s: Failed to open file %s\n", __func__, SMAP_FILE_PATH); 117 118 if (!check_for_pattern(fp, addr_pattern, buffer, sizeof(buffer))) 119 goto err_out; 120 121 /* 122 * Fetch the pattern in the same block and check the number of 123 * hugepages. 124 */ 125 if (!check_for_pattern(fp, pattern, buffer, sizeof(buffer))) 126 goto err_out; 127 128 snprintf(addr_pattern, MAX_LINE_LENGTH, "%s%%9ld kB", pattern); 129 130 if (sscanf(buffer, addr_pattern, &thp) != 1) 131 ksft_exit_fail_msg("Reading smap error\n"); 132 133 err_out: 134 fclose(fp); 135 return thp == (nr_hpages * (hpage_size >> 10)); 136 } 137 138 bool check_huge_anon(void *addr, int nr_hpages, uint64_t hpage_size) 139 { 140 return __check_huge(addr, "AnonHugePages: ", nr_hpages, hpage_size); 141 } 142 143 bool check_huge_file(void *addr, int nr_hpages, uint64_t hpage_size) 144 { 145 return __check_huge(addr, "FilePmdMapped:", nr_hpages, hpage_size); 146 } 147 148 bool check_huge_shmem(void *addr, int nr_hpages, uint64_t hpage_size) 149 { 150 return __check_huge(addr, "ShmemPmdMapped:", nr_hpages, hpage_size); 151 } 152 153 int64_t allocate_transhuge(void *ptr, int pagemap_fd) 154 { 155 uint64_t ent[2]; 156 157 /* drop pmd */ 158 if (mmap(ptr, HPAGE_SIZE, PROT_READ | PROT_WRITE, 159 MAP_FIXED | MAP_ANONYMOUS | 160 MAP_NORESERVE | MAP_PRIVATE, -1, 0) != ptr) 161 errx(2, "mmap transhuge"); 162 163 if (madvise(ptr, HPAGE_SIZE, MADV_HUGEPAGE)) 164 err(2, "MADV_HUGEPAGE"); 165 166 /* allocate transparent huge page */ 167 *(volatile void **)ptr = ptr; 168 169 if (pread(pagemap_fd, ent, sizeof(ent), 170 (uintptr_t)ptr >> (pshift() - 3)) != sizeof(ent)) 171 err(2, "read pagemap"); 172 173 if (PAGEMAP_PRESENT(ent[0]) && PAGEMAP_PRESENT(ent[1]) && 174 PAGEMAP_PFN(ent[0]) + 1 == PAGEMAP_PFN(ent[1]) && 175 !(PAGEMAP_PFN(ent[0]) & ((1 << (HPAGE_SHIFT - pshift())) - 1))) 176 return PAGEMAP_PFN(ent[0]); 177 178 return -1; 179 } 180 181 unsigned long default_huge_page_size(void) 182 { 183 unsigned long hps = 0; 184 char *line = NULL; 185 size_t linelen = 0; 186 FILE *f = fopen("/proc/meminfo", "r"); 187 188 if (!f) 189 return 0; 190 while (getline(&line, &linelen, f) > 0) { 191 if (sscanf(line, "Hugepagesize: %lu kB", &hps) == 1) { 192 hps <<= 10; 193 break; 194 } 195 } 196 197 free(line); 198 fclose(f); 199 return hps; 200 } 201 202 int detect_hugetlb_page_sizes(size_t sizes[], int max) 203 { 204 DIR *dir = opendir("/sys/kernel/mm/hugepages/"); 205 int count = 0; 206 207 if (!dir) 208 return 0; 209 210 while (count < max) { 211 struct dirent *entry = readdir(dir); 212 size_t kb; 213 214 if (!entry) 215 break; 216 if (entry->d_type != DT_DIR) 217 continue; 218 if (sscanf(entry->d_name, "hugepages-%zukB", &kb) != 1) 219 continue; 220 sizes[count++] = kb * 1024; 221 ksft_print_msg("[INFO] detected hugetlb page size: %zu KiB\n", 222 kb); 223 } 224 closedir(dir); 225 return count; 226 } 227 228 /* If `ioctls' non-NULL, the allowed ioctls will be returned into the var */ 229 int uffd_register_with_ioctls(int uffd, void *addr, uint64_t len, 230 bool miss, bool wp, bool minor, uint64_t *ioctls) 231 { 232 struct uffdio_register uffdio_register = { 0 }; 233 uint64_t mode = 0; 234 int ret = 0; 235 236 if (miss) 237 mode |= UFFDIO_REGISTER_MODE_MISSING; 238 if (wp) 239 mode |= UFFDIO_REGISTER_MODE_WP; 240 if (minor) 241 mode |= UFFDIO_REGISTER_MODE_MINOR; 242 243 uffdio_register.range.start = (unsigned long)addr; 244 uffdio_register.range.len = len; 245 uffdio_register.mode = mode; 246 247 if (ioctl(uffd, UFFDIO_REGISTER, &uffdio_register) == -1) 248 ret = -errno; 249 else if (ioctls) 250 *ioctls = uffdio_register.ioctls; 251 252 return ret; 253 } 254 255 int uffd_register(int uffd, void *addr, uint64_t len, 256 bool miss, bool wp, bool minor) 257 { 258 return uffd_register_with_ioctls(uffd, addr, len, 259 miss, wp, minor, NULL); 260 } 261 262 int uffd_unregister(int uffd, void *addr, uint64_t len) 263 { 264 struct uffdio_range range = { .start = (uintptr_t)addr, .len = len }; 265 int ret = 0; 266 267 if (ioctl(uffd, UFFDIO_UNREGISTER, &range) == -1) 268 ret = -errno; 269 270 return ret; 271 } 272