1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * KSM functional tests 4 * 5 * Copyright 2022, Red Hat, Inc. 6 * 7 * Author(s): David Hildenbrand <david@redhat.com> 8 */ 9 #define _GNU_SOURCE 10 #include <stdlib.h> 11 #include <string.h> 12 #include <stdbool.h> 13 #include <stdint.h> 14 #include <unistd.h> 15 #include <errno.h> 16 #include <fcntl.h> 17 #include <sys/mman.h> 18 #include <sys/prctl.h> 19 #include <sys/syscall.h> 20 #include <sys/ioctl.h> 21 #include <sys/wait.h> 22 #include <linux/userfaultfd.h> 23 24 #include "../kselftest.h" 25 #include "vm_util.h" 26 27 #define KiB 1024u 28 #define MiB (1024 * KiB) 29 30 static int ksm_fd; 31 static int ksm_full_scans_fd; 32 static int pagemap_fd; 33 static size_t pagesize; 34 35 static bool range_maps_duplicates(char *addr, unsigned long size) 36 { 37 unsigned long offs_a, offs_b, pfn_a, pfn_b; 38 39 /* 40 * There is no easy way to check if there are KSM pages mapped into 41 * this range. We only check that the range does not map the same PFN 42 * twice by comparing each pair of mapped pages. 43 */ 44 for (offs_a = 0; offs_a < size; offs_a += pagesize) { 45 pfn_a = pagemap_get_pfn(pagemap_fd, addr + offs_a); 46 /* Page not present or PFN not exposed by the kernel. */ 47 if (pfn_a == -1ul || !pfn_a) 48 continue; 49 50 for (offs_b = offs_a + pagesize; offs_b < size; 51 offs_b += pagesize) { 52 pfn_b = pagemap_get_pfn(pagemap_fd, addr + offs_b); 53 if (pfn_b == -1ul || !pfn_b) 54 continue; 55 if (pfn_a == pfn_b) 56 return true; 57 } 58 } 59 return false; 60 } 61 62 static long ksm_get_full_scans(void) 63 { 64 char buf[10]; 65 ssize_t ret; 66 67 ret = pread(ksm_full_scans_fd, buf, sizeof(buf) - 1, 0); 68 if (ret <= 0) 69 return -errno; 70 buf[ret] = 0; 71 72 return strtol(buf, NULL, 10); 73 } 74 75 static int ksm_merge(void) 76 { 77 long start_scans, end_scans; 78 79 /* Wait for two full scans such that any possible merging happened. */ 80 start_scans = ksm_get_full_scans(); 81 if (start_scans < 0) 82 return start_scans; 83 if (write(ksm_fd, "1", 1) != 1) 84 return -errno; 85 do { 86 end_scans = ksm_get_full_scans(); 87 if (end_scans < 0) 88 return end_scans; 89 } while (end_scans < start_scans + 2); 90 91 return 0; 92 } 93 94 static char *mmap_and_merge_range(char val, unsigned long size, bool use_prctl) 95 { 96 char *map; 97 int ret; 98 99 map = mmap(NULL, size, PROT_READ|PROT_WRITE, 100 MAP_PRIVATE|MAP_ANON, -1, 0); 101 if (map == MAP_FAILED) { 102 ksft_test_result_fail("mmap() failed\n"); 103 return MAP_FAILED; 104 } 105 106 /* Don't use THP. Ignore if THP are not around on a kernel. */ 107 if (madvise(map, size, MADV_NOHUGEPAGE) && errno != EINVAL) { 108 ksft_test_result_fail("MADV_NOHUGEPAGE failed\n"); 109 goto unmap; 110 } 111 112 /* Make sure each page contains the same values to merge them. */ 113 memset(map, val, size); 114 115 if (use_prctl) { 116 ret = prctl(PR_SET_MEMORY_MERGE, 1, 0, 0, 0); 117 if (ret < 0 && errno == EINVAL) { 118 ksft_test_result_skip("PR_SET_MEMORY_MERGE not supported\n"); 119 goto unmap; 120 } else if (ret) { 121 ksft_test_result_fail("PR_SET_MEMORY_MERGE=1 failed\n"); 122 goto unmap; 123 } 124 } else if (madvise(map, size, MADV_MERGEABLE)) { 125 ksft_test_result_fail("MADV_MERGEABLE failed\n"); 126 goto unmap; 127 } 128 129 /* Run KSM to trigger merging and wait. */ 130 if (ksm_merge()) { 131 ksft_test_result_fail("Running KSM failed\n"); 132 goto unmap; 133 } 134 return map; 135 unmap: 136 munmap(map, size); 137 return MAP_FAILED; 138 } 139 140 static void test_unmerge(void) 141 { 142 const unsigned int size = 2 * MiB; 143 char *map; 144 145 ksft_print_msg("[RUN] %s\n", __func__); 146 147 map = mmap_and_merge_range(0xcf, size, false); 148 if (map == MAP_FAILED) 149 return; 150 151 if (madvise(map, size, MADV_UNMERGEABLE)) { 152 ksft_test_result_fail("MADV_UNMERGEABLE failed\n"); 153 goto unmap; 154 } 155 156 ksft_test_result(!range_maps_duplicates(map, size), 157 "Pages were unmerged\n"); 158 unmap: 159 munmap(map, size); 160 } 161 162 static void test_unmerge_discarded(void) 163 { 164 const unsigned int size = 2 * MiB; 165 char *map; 166 167 ksft_print_msg("[RUN] %s\n", __func__); 168 169 map = mmap_and_merge_range(0xcf, size, false); 170 if (map == MAP_FAILED) 171 return; 172 173 /* Discard half of all mapped pages so we have pte_none() entries. */ 174 if (madvise(map, size / 2, MADV_DONTNEED)) { 175 ksft_test_result_fail("MADV_DONTNEED failed\n"); 176 goto unmap; 177 } 178 179 if (madvise(map, size, MADV_UNMERGEABLE)) { 180 ksft_test_result_fail("MADV_UNMERGEABLE failed\n"); 181 goto unmap; 182 } 183 184 ksft_test_result(!range_maps_duplicates(map, size), 185 "Pages were unmerged\n"); 186 unmap: 187 munmap(map, size); 188 } 189 190 #ifdef __NR_userfaultfd 191 static void test_unmerge_uffd_wp(void) 192 { 193 struct uffdio_writeprotect uffd_writeprotect; 194 const unsigned int size = 2 * MiB; 195 struct uffdio_api uffdio_api; 196 char *map; 197 int uffd; 198 199 ksft_print_msg("[RUN] %s\n", __func__); 200 201 map = mmap_and_merge_range(0xcf, size, false); 202 if (map == MAP_FAILED) 203 return; 204 205 /* See if UFFD is around. */ 206 uffd = syscall(__NR_userfaultfd, O_CLOEXEC | O_NONBLOCK); 207 if (uffd < 0) { 208 ksft_test_result_skip("__NR_userfaultfd failed\n"); 209 goto unmap; 210 } 211 212 /* See if UFFD-WP is around. */ 213 uffdio_api.api = UFFD_API; 214 uffdio_api.features = UFFD_FEATURE_PAGEFAULT_FLAG_WP; 215 if (ioctl(uffd, UFFDIO_API, &uffdio_api) < 0) { 216 ksft_test_result_fail("UFFDIO_API failed\n"); 217 goto close_uffd; 218 } 219 if (!(uffdio_api.features & UFFD_FEATURE_PAGEFAULT_FLAG_WP)) { 220 ksft_test_result_skip("UFFD_FEATURE_PAGEFAULT_FLAG_WP not available\n"); 221 goto close_uffd; 222 } 223 224 /* Register UFFD-WP, no need for an actual handler. */ 225 if (uffd_register(uffd, map, size, false, true, false)) { 226 ksft_test_result_fail("UFFDIO_REGISTER_MODE_WP failed\n"); 227 goto close_uffd; 228 } 229 230 /* Write-protect the range using UFFD-WP. */ 231 uffd_writeprotect.range.start = (unsigned long) map; 232 uffd_writeprotect.range.len = size; 233 uffd_writeprotect.mode = UFFDIO_WRITEPROTECT_MODE_WP; 234 if (ioctl(uffd, UFFDIO_WRITEPROTECT, &uffd_writeprotect)) { 235 ksft_test_result_fail("UFFDIO_WRITEPROTECT failed\n"); 236 goto close_uffd; 237 } 238 239 if (madvise(map, size, MADV_UNMERGEABLE)) { 240 ksft_test_result_fail("MADV_UNMERGEABLE failed\n"); 241 goto close_uffd; 242 } 243 244 ksft_test_result(!range_maps_duplicates(map, size), 245 "Pages were unmerged\n"); 246 close_uffd: 247 close(uffd); 248 unmap: 249 munmap(map, size); 250 } 251 #endif 252 253 /* Verify that KSM can be enabled / queried with prctl. */ 254 static void test_prctl(void) 255 { 256 int ret; 257 258 ksft_print_msg("[RUN] %s\n", __func__); 259 260 ret = prctl(PR_SET_MEMORY_MERGE, 1, 0, 0, 0); 261 if (ret < 0 && errno == EINVAL) { 262 ksft_test_result_skip("PR_SET_MEMORY_MERGE not supported\n"); 263 return; 264 } else if (ret) { 265 ksft_test_result_fail("PR_SET_MEMORY_MERGE=1 failed\n"); 266 return; 267 } 268 269 ret = prctl(PR_GET_MEMORY_MERGE, 0, 0, 0, 0); 270 if (ret < 0) { 271 ksft_test_result_fail("PR_GET_MEMORY_MERGE failed\n"); 272 return; 273 } else if (ret != 1) { 274 ksft_test_result_fail("PR_SET_MEMORY_MERGE=1 not effective\n"); 275 return; 276 } 277 278 ret = prctl(PR_SET_MEMORY_MERGE, 0, 0, 0, 0); 279 if (ret) { 280 ksft_test_result_fail("PR_SET_MEMORY_MERGE=0 failed\n"); 281 return; 282 } 283 284 ret = prctl(PR_GET_MEMORY_MERGE, 0, 0, 0, 0); 285 if (ret < 0) { 286 ksft_test_result_fail("PR_GET_MEMORY_MERGE failed\n"); 287 return; 288 } else if (ret != 0) { 289 ksft_test_result_fail("PR_SET_MEMORY_MERGE=0 not effective\n"); 290 return; 291 } 292 293 ksft_test_result_pass("Setting/clearing PR_SET_MEMORY_MERGE works\n"); 294 } 295 296 /* Verify that prctl ksm flag is inherited. */ 297 static void test_prctl_fork(void) 298 { 299 int ret, status; 300 pid_t child_pid; 301 302 ksft_print_msg("[RUN] %s\n", __func__); 303 304 ret = prctl(PR_SET_MEMORY_MERGE, 1, 0, 0, 0); 305 if (ret < 0 && errno == EINVAL) { 306 ksft_test_result_skip("PR_SET_MEMORY_MERGE not supported\n"); 307 return; 308 } else if (ret) { 309 ksft_test_result_fail("PR_SET_MEMORY_MERGE=1 failed\n"); 310 return; 311 } 312 313 child_pid = fork(); 314 if (!child_pid) { 315 exit(prctl(PR_GET_MEMORY_MERGE, 0, 0, 0, 0)); 316 } else if (child_pid < 0) { 317 ksft_test_result_fail("fork() failed\n"); 318 return; 319 } 320 321 if (waitpid(child_pid, &status, 0) < 0) { 322 ksft_test_result_fail("waitpid() failed\n"); 323 return; 324 } else if (WEXITSTATUS(status) != 1) { 325 ksft_test_result_fail("unexpected PR_GET_MEMORY_MERGE result in child\n"); 326 return; 327 } 328 329 if (prctl(PR_SET_MEMORY_MERGE, 0, 0, 0, 0)) { 330 ksft_test_result_fail("PR_SET_MEMORY_MERGE=0 failed\n"); 331 return; 332 } 333 334 ksft_test_result_pass("PR_SET_MEMORY_MERGE value is inherited\n"); 335 } 336 337 static void test_prctl_unmerge(void) 338 { 339 const unsigned int size = 2 * MiB; 340 char *map; 341 342 ksft_print_msg("[RUN] %s\n", __func__); 343 344 map = mmap_and_merge_range(0xcf, size, true); 345 if (map == MAP_FAILED) 346 return; 347 348 if (prctl(PR_SET_MEMORY_MERGE, 0, 0, 0, 0)) { 349 ksft_test_result_fail("PR_SET_MEMORY_MERGE=0 failed\n"); 350 goto unmap; 351 } 352 353 ksft_test_result(!range_maps_duplicates(map, size), 354 "Pages were unmerged\n"); 355 unmap: 356 munmap(map, size); 357 } 358 359 int main(int argc, char **argv) 360 { 361 unsigned int tests = 5; 362 int err; 363 364 #ifdef __NR_userfaultfd 365 tests++; 366 #endif 367 368 ksft_print_header(); 369 ksft_set_plan(tests); 370 371 pagesize = getpagesize(); 372 373 ksm_fd = open("/sys/kernel/mm/ksm/run", O_RDWR); 374 if (ksm_fd < 0) 375 ksft_exit_skip("open(\"/sys/kernel/mm/ksm/run\") failed\n"); 376 ksm_full_scans_fd = open("/sys/kernel/mm/ksm/full_scans", O_RDONLY); 377 if (ksm_full_scans_fd < 0) 378 ksft_exit_skip("open(\"/sys/kernel/mm/ksm/full_scans\") failed\n"); 379 pagemap_fd = open("/proc/self/pagemap", O_RDONLY); 380 if (pagemap_fd < 0) 381 ksft_exit_skip("open(\"/proc/self/pagemap\") failed\n"); 382 383 test_unmerge(); 384 test_unmerge_discarded(); 385 #ifdef __NR_userfaultfd 386 test_unmerge_uffd_wp(); 387 #endif 388 389 test_prctl(); 390 test_prctl_fork(); 391 test_prctl_unmerge(); 392 393 err = ksft_get_fail_cnt(); 394 if (err) 395 ksft_exit_fail_msg("%d out of %d tests failed\n", 396 err, ksft_test_num()); 397 return ksft_exit_pass(); 398 } 399