1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * KSM functional tests 4 * 5 * Copyright 2022, Red Hat, Inc. 6 * 7 * Author(s): David Hildenbrand <david@redhat.com> 8 */ 9 #define _GNU_SOURCE 10 #include <stdlib.h> 11 #include <string.h> 12 #include <stdbool.h> 13 #include <stdint.h> 14 #include <unistd.h> 15 #include <errno.h> 16 #include <fcntl.h> 17 #include <sys/mman.h> 18 #include <sys/prctl.h> 19 #include <sys/syscall.h> 20 #include <sys/ioctl.h> 21 #include <sys/wait.h> 22 #include <linux/userfaultfd.h> 23 24 #include "../kselftest.h" 25 #include "vm_util.h" 26 27 #define KiB 1024u 28 #define MiB (1024 * KiB) 29 30 static int ksm_fd; 31 static int ksm_full_scans_fd; 32 static int proc_self_ksm_stat_fd; 33 static int ksm_use_zero_pages_fd; 34 static int pagemap_fd; 35 static size_t pagesize; 36 37 static bool range_maps_duplicates(char *addr, unsigned long size) 38 { 39 unsigned long offs_a, offs_b, pfn_a, pfn_b; 40 41 /* 42 * There is no easy way to check if there are KSM pages mapped into 43 * this range. We only check that the range does not map the same PFN 44 * twice by comparing each pair of mapped pages. 45 */ 46 for (offs_a = 0; offs_a < size; offs_a += pagesize) { 47 pfn_a = pagemap_get_pfn(pagemap_fd, addr + offs_a); 48 /* Page not present or PFN not exposed by the kernel. */ 49 if (pfn_a == -1ul || !pfn_a) 50 continue; 51 52 for (offs_b = offs_a + pagesize; offs_b < size; 53 offs_b += pagesize) { 54 pfn_b = pagemap_get_pfn(pagemap_fd, addr + offs_b); 55 if (pfn_b == -1ul || !pfn_b) 56 continue; 57 if (pfn_a == pfn_b) 58 return true; 59 } 60 } 61 return false; 62 } 63 64 static long get_my_ksm_zero_pages(void) 65 { 66 char buf[200]; 67 char *substr_ksm_zero; 68 size_t value_pos; 69 ssize_t read_size; 70 unsigned long my_ksm_zero_pages; 71 72 if (!proc_self_ksm_stat_fd) 73 return 0; 74 75 read_size = pread(proc_self_ksm_stat_fd, buf, sizeof(buf) - 1, 0); 76 if (read_size < 0) 77 return -errno; 78 79 buf[read_size] = 0; 80 81 substr_ksm_zero = strstr(buf, "ksm_zero_pages"); 82 if (!substr_ksm_zero) 83 return 0; 84 85 value_pos = strcspn(substr_ksm_zero, "0123456789"); 86 my_ksm_zero_pages = strtol(substr_ksm_zero + value_pos, NULL, 10); 87 88 return my_ksm_zero_pages; 89 } 90 91 static long ksm_get_full_scans(void) 92 { 93 char buf[10]; 94 ssize_t ret; 95 96 ret = pread(ksm_full_scans_fd, buf, sizeof(buf) - 1, 0); 97 if (ret <= 0) 98 return -errno; 99 buf[ret] = 0; 100 101 return strtol(buf, NULL, 10); 102 } 103 104 static int ksm_merge(void) 105 { 106 long start_scans, end_scans; 107 108 /* Wait for two full scans such that any possible merging happened. */ 109 start_scans = ksm_get_full_scans(); 110 if (start_scans < 0) 111 return start_scans; 112 if (write(ksm_fd, "1", 1) != 1) 113 return -errno; 114 do { 115 end_scans = ksm_get_full_scans(); 116 if (end_scans < 0) 117 return end_scans; 118 } while (end_scans < start_scans + 2); 119 120 return 0; 121 } 122 123 static char *mmap_and_merge_range(char val, unsigned long size, bool use_prctl) 124 { 125 char *map; 126 int ret; 127 128 map = mmap(NULL, size, PROT_READ|PROT_WRITE, 129 MAP_PRIVATE|MAP_ANON, -1, 0); 130 if (map == MAP_FAILED) { 131 ksft_test_result_fail("mmap() failed\n"); 132 return MAP_FAILED; 133 } 134 135 /* Don't use THP. Ignore if THP are not around on a kernel. */ 136 if (madvise(map, size, MADV_NOHUGEPAGE) && errno != EINVAL) { 137 ksft_test_result_fail("MADV_NOHUGEPAGE failed\n"); 138 goto unmap; 139 } 140 141 /* Make sure each page contains the same values to merge them. */ 142 memset(map, val, size); 143 144 if (use_prctl) { 145 ret = prctl(PR_SET_MEMORY_MERGE, 1, 0, 0, 0); 146 if (ret < 0 && errno == EINVAL) { 147 ksft_test_result_skip("PR_SET_MEMORY_MERGE not supported\n"); 148 goto unmap; 149 } else if (ret) { 150 ksft_test_result_fail("PR_SET_MEMORY_MERGE=1 failed\n"); 151 goto unmap; 152 } 153 } else if (madvise(map, size, MADV_MERGEABLE)) { 154 ksft_test_result_fail("MADV_MERGEABLE failed\n"); 155 goto unmap; 156 } 157 158 /* Run KSM to trigger merging and wait. */ 159 if (ksm_merge()) { 160 ksft_test_result_fail("Running KSM failed\n"); 161 goto unmap; 162 } 163 return map; 164 unmap: 165 munmap(map, size); 166 return MAP_FAILED; 167 } 168 169 static void test_unmerge(void) 170 { 171 const unsigned int size = 2 * MiB; 172 char *map; 173 174 ksft_print_msg("[RUN] %s\n", __func__); 175 176 map = mmap_and_merge_range(0xcf, size, false); 177 if (map == MAP_FAILED) 178 return; 179 180 if (madvise(map, size, MADV_UNMERGEABLE)) { 181 ksft_test_result_fail("MADV_UNMERGEABLE failed\n"); 182 goto unmap; 183 } 184 185 ksft_test_result(!range_maps_duplicates(map, size), 186 "Pages were unmerged\n"); 187 unmap: 188 munmap(map, size); 189 } 190 191 static void test_unmerge_zero_pages(void) 192 { 193 const unsigned int size = 2 * MiB; 194 char *map; 195 unsigned int offs; 196 unsigned long pages_expected; 197 198 ksft_print_msg("[RUN] %s\n", __func__); 199 200 if (proc_self_ksm_stat_fd < 0) { 201 ksft_test_result_skip("open(\"/proc/self/ksm_stat\") failed\n"); 202 return; 203 } 204 if (ksm_use_zero_pages_fd < 0) { 205 ksft_test_result_skip("open \"/sys/kernel/mm/ksm/use_zero_pages\" failed\n"); 206 return; 207 } 208 if (write(ksm_use_zero_pages_fd, "1", 1) != 1) { 209 ksft_test_result_skip("write \"/sys/kernel/mm/ksm/use_zero_pages\" failed\n"); 210 return; 211 } 212 213 /* Let KSM deduplicate zero pages. */ 214 map = mmap_and_merge_range(0x00, size, false); 215 if (map == MAP_FAILED) 216 return; 217 218 /* Check if ksm_zero_pages is updated correctly after KSM merging */ 219 pages_expected = size / pagesize; 220 if (pages_expected != get_my_ksm_zero_pages()) { 221 ksft_test_result_fail("'ksm_zero_pages' updated after merging\n"); 222 goto unmap; 223 } 224 225 /* Try to unmerge half of the region */ 226 if (madvise(map, size / 2, MADV_UNMERGEABLE)) { 227 ksft_test_result_fail("MADV_UNMERGEABLE failed\n"); 228 goto unmap; 229 } 230 231 /* Check if ksm_zero_pages is updated correctly after unmerging */ 232 pages_expected /= 2; 233 if (pages_expected != get_my_ksm_zero_pages()) { 234 ksft_test_result_fail("'ksm_zero_pages' updated after unmerging\n"); 235 goto unmap; 236 } 237 238 /* Trigger unmerging of the other half by writing to the pages. */ 239 for (offs = size / 2; offs < size; offs += pagesize) 240 *((unsigned int *)&map[offs]) = offs; 241 242 /* Now we should have no zeropages remaining. */ 243 if (get_my_ksm_zero_pages()) { 244 ksft_test_result_fail("'ksm_zero_pages' updated after write fault\n"); 245 goto unmap; 246 } 247 248 /* Check if ksm zero pages are really unmerged */ 249 ksft_test_result(!range_maps_duplicates(map, size), 250 "KSM zero pages were unmerged\n"); 251 unmap: 252 munmap(map, size); 253 } 254 255 static void test_unmerge_discarded(void) 256 { 257 const unsigned int size = 2 * MiB; 258 char *map; 259 260 ksft_print_msg("[RUN] %s\n", __func__); 261 262 map = mmap_and_merge_range(0xcf, size, false); 263 if (map == MAP_FAILED) 264 return; 265 266 /* Discard half of all mapped pages so we have pte_none() entries. */ 267 if (madvise(map, size / 2, MADV_DONTNEED)) { 268 ksft_test_result_fail("MADV_DONTNEED failed\n"); 269 goto unmap; 270 } 271 272 if (madvise(map, size, MADV_UNMERGEABLE)) { 273 ksft_test_result_fail("MADV_UNMERGEABLE failed\n"); 274 goto unmap; 275 } 276 277 ksft_test_result(!range_maps_duplicates(map, size), 278 "Pages were unmerged\n"); 279 unmap: 280 munmap(map, size); 281 } 282 283 #ifdef __NR_userfaultfd 284 static void test_unmerge_uffd_wp(void) 285 { 286 struct uffdio_writeprotect uffd_writeprotect; 287 const unsigned int size = 2 * MiB; 288 struct uffdio_api uffdio_api; 289 char *map; 290 int uffd; 291 292 ksft_print_msg("[RUN] %s\n", __func__); 293 294 map = mmap_and_merge_range(0xcf, size, false); 295 if (map == MAP_FAILED) 296 return; 297 298 /* See if UFFD is around. */ 299 uffd = syscall(__NR_userfaultfd, O_CLOEXEC | O_NONBLOCK); 300 if (uffd < 0) { 301 ksft_test_result_skip("__NR_userfaultfd failed\n"); 302 goto unmap; 303 } 304 305 /* See if UFFD-WP is around. */ 306 uffdio_api.api = UFFD_API; 307 uffdio_api.features = UFFD_FEATURE_PAGEFAULT_FLAG_WP; 308 if (ioctl(uffd, UFFDIO_API, &uffdio_api) < 0) { 309 ksft_test_result_fail("UFFDIO_API failed\n"); 310 goto close_uffd; 311 } 312 if (!(uffdio_api.features & UFFD_FEATURE_PAGEFAULT_FLAG_WP)) { 313 ksft_test_result_skip("UFFD_FEATURE_PAGEFAULT_FLAG_WP not available\n"); 314 goto close_uffd; 315 } 316 317 /* Register UFFD-WP, no need for an actual handler. */ 318 if (uffd_register(uffd, map, size, false, true, false)) { 319 ksft_test_result_fail("UFFDIO_REGISTER_MODE_WP failed\n"); 320 goto close_uffd; 321 } 322 323 /* Write-protect the range using UFFD-WP. */ 324 uffd_writeprotect.range.start = (unsigned long) map; 325 uffd_writeprotect.range.len = size; 326 uffd_writeprotect.mode = UFFDIO_WRITEPROTECT_MODE_WP; 327 if (ioctl(uffd, UFFDIO_WRITEPROTECT, &uffd_writeprotect)) { 328 ksft_test_result_fail("UFFDIO_WRITEPROTECT failed\n"); 329 goto close_uffd; 330 } 331 332 if (madvise(map, size, MADV_UNMERGEABLE)) { 333 ksft_test_result_fail("MADV_UNMERGEABLE failed\n"); 334 goto close_uffd; 335 } 336 337 ksft_test_result(!range_maps_duplicates(map, size), 338 "Pages were unmerged\n"); 339 close_uffd: 340 close(uffd); 341 unmap: 342 munmap(map, size); 343 } 344 #endif 345 346 /* Verify that KSM can be enabled / queried with prctl. */ 347 static void test_prctl(void) 348 { 349 int ret; 350 351 ksft_print_msg("[RUN] %s\n", __func__); 352 353 ret = prctl(PR_SET_MEMORY_MERGE, 1, 0, 0, 0); 354 if (ret < 0 && errno == EINVAL) { 355 ksft_test_result_skip("PR_SET_MEMORY_MERGE not supported\n"); 356 return; 357 } else if (ret) { 358 ksft_test_result_fail("PR_SET_MEMORY_MERGE=1 failed\n"); 359 return; 360 } 361 362 ret = prctl(PR_GET_MEMORY_MERGE, 0, 0, 0, 0); 363 if (ret < 0) { 364 ksft_test_result_fail("PR_GET_MEMORY_MERGE failed\n"); 365 return; 366 } else if (ret != 1) { 367 ksft_test_result_fail("PR_SET_MEMORY_MERGE=1 not effective\n"); 368 return; 369 } 370 371 ret = prctl(PR_SET_MEMORY_MERGE, 0, 0, 0, 0); 372 if (ret) { 373 ksft_test_result_fail("PR_SET_MEMORY_MERGE=0 failed\n"); 374 return; 375 } 376 377 ret = prctl(PR_GET_MEMORY_MERGE, 0, 0, 0, 0); 378 if (ret < 0) { 379 ksft_test_result_fail("PR_GET_MEMORY_MERGE failed\n"); 380 return; 381 } else if (ret != 0) { 382 ksft_test_result_fail("PR_SET_MEMORY_MERGE=0 not effective\n"); 383 return; 384 } 385 386 ksft_test_result_pass("Setting/clearing PR_SET_MEMORY_MERGE works\n"); 387 } 388 389 /* Verify that prctl ksm flag is inherited. */ 390 static void test_prctl_fork(void) 391 { 392 int ret, status; 393 pid_t child_pid; 394 395 ksft_print_msg("[RUN] %s\n", __func__); 396 397 ret = prctl(PR_SET_MEMORY_MERGE, 1, 0, 0, 0); 398 if (ret < 0 && errno == EINVAL) { 399 ksft_test_result_skip("PR_SET_MEMORY_MERGE not supported\n"); 400 return; 401 } else if (ret) { 402 ksft_test_result_fail("PR_SET_MEMORY_MERGE=1 failed\n"); 403 return; 404 } 405 406 child_pid = fork(); 407 if (!child_pid) { 408 exit(prctl(PR_GET_MEMORY_MERGE, 0, 0, 0, 0)); 409 } else if (child_pid < 0) { 410 ksft_test_result_fail("fork() failed\n"); 411 return; 412 } 413 414 if (waitpid(child_pid, &status, 0) < 0) { 415 ksft_test_result_fail("waitpid() failed\n"); 416 return; 417 } else if (WEXITSTATUS(status) != 1) { 418 ksft_test_result_fail("unexpected PR_GET_MEMORY_MERGE result in child\n"); 419 return; 420 } 421 422 if (prctl(PR_SET_MEMORY_MERGE, 0, 0, 0, 0)) { 423 ksft_test_result_fail("PR_SET_MEMORY_MERGE=0 failed\n"); 424 return; 425 } 426 427 ksft_test_result_pass("PR_SET_MEMORY_MERGE value is inherited\n"); 428 } 429 430 static void test_prctl_unmerge(void) 431 { 432 const unsigned int size = 2 * MiB; 433 char *map; 434 435 ksft_print_msg("[RUN] %s\n", __func__); 436 437 map = mmap_and_merge_range(0xcf, size, true); 438 if (map == MAP_FAILED) 439 return; 440 441 if (prctl(PR_SET_MEMORY_MERGE, 0, 0, 0, 0)) { 442 ksft_test_result_fail("PR_SET_MEMORY_MERGE=0 failed\n"); 443 goto unmap; 444 } 445 446 ksft_test_result(!range_maps_duplicates(map, size), 447 "Pages were unmerged\n"); 448 unmap: 449 munmap(map, size); 450 } 451 452 int main(int argc, char **argv) 453 { 454 unsigned int tests = 6; 455 int err; 456 457 #ifdef __NR_userfaultfd 458 tests++; 459 #endif 460 461 ksft_print_header(); 462 ksft_set_plan(tests); 463 464 pagesize = getpagesize(); 465 466 ksm_fd = open("/sys/kernel/mm/ksm/run", O_RDWR); 467 if (ksm_fd < 0) 468 ksft_exit_skip("open(\"/sys/kernel/mm/ksm/run\") failed\n"); 469 ksm_full_scans_fd = open("/sys/kernel/mm/ksm/full_scans", O_RDONLY); 470 if (ksm_full_scans_fd < 0) 471 ksft_exit_skip("open(\"/sys/kernel/mm/ksm/full_scans\") failed\n"); 472 pagemap_fd = open("/proc/self/pagemap", O_RDONLY); 473 if (pagemap_fd < 0) 474 ksft_exit_skip("open(\"/proc/self/pagemap\") failed\n"); 475 proc_self_ksm_stat_fd = open("/proc/self/ksm_stat", O_RDONLY); 476 ksm_use_zero_pages_fd = open("/sys/kernel/mm/ksm/use_zero_pages", O_RDWR); 477 478 test_unmerge(); 479 test_unmerge_zero_pages(); 480 test_unmerge_discarded(); 481 #ifdef __NR_userfaultfd 482 test_unmerge_uffd_wp(); 483 #endif 484 485 test_prctl(); 486 test_prctl_fork(); 487 test_prctl_unmerge(); 488 489 err = ksft_get_fail_cnt(); 490 if (err) 491 ksft_exit_fail_msg("%d out of %d tests failed\n", 492 err, ksft_test_num()); 493 return ksft_exit_pass(); 494 } 495