1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * HMM stands for Heterogeneous Memory Management, it is a helper layer inside 4 * the linux kernel to help device drivers mirror a process address space in 5 * the device. This allows the device to use the same address space which 6 * makes communication and data exchange a lot easier. 7 * 8 * This framework's sole purpose is to exercise various code paths inside 9 * the kernel to make sure that HMM performs as expected and to flush out any 10 * bugs. 11 */ 12 13 #include "../kselftest_harness.h" 14 15 #include <errno.h> 16 #include <fcntl.h> 17 #include <stdio.h> 18 #include <stdlib.h> 19 #include <stdint.h> 20 #include <unistd.h> 21 #include <strings.h> 22 #include <time.h> 23 #include <pthread.h> 24 #include <sys/types.h> 25 #include <sys/stat.h> 26 #include <sys/mman.h> 27 #include <sys/ioctl.h> 28 29 30 /* 31 * This is a private UAPI to the kernel test module so it isn't exported 32 * in the usual include/uapi/... directory. 33 */ 34 #include <lib/test_hmm_uapi.h> 35 #include <mm/gup_test.h> 36 37 struct hmm_buffer { 38 void *ptr; 39 void *mirror; 40 unsigned long size; 41 int fd; 42 uint64_t cpages; 43 uint64_t faults; 44 }; 45 46 enum { 47 HMM_PRIVATE_DEVICE_ONE, 48 HMM_PRIVATE_DEVICE_TWO, 49 HMM_COHERENCE_DEVICE_ONE, 50 HMM_COHERENCE_DEVICE_TWO, 51 }; 52 53 #define TWOMEG (1 << 21) 54 #define HMM_BUFFER_SIZE (1024 << 12) 55 #define HMM_PATH_MAX 64 56 #define NTIMES 10 57 58 #define ALIGN(x, a) (((x) + (a - 1)) & (~((a) - 1))) 59 /* Just the flags we need, copied from mm.h: */ 60 61 #ifndef FOLL_WRITE 62 #define FOLL_WRITE 0x01 /* check pte is writable */ 63 #endif 64 65 #ifndef FOLL_LONGTERM 66 #define FOLL_LONGTERM 0x100 /* mapping lifetime is indefinite */ 67 #endif 68 FIXTURE(hmm) 69 { 70 int fd; 71 unsigned int page_size; 72 unsigned int page_shift; 73 }; 74 75 FIXTURE_VARIANT(hmm) 76 { 77 int device_number; 78 }; 79 80 FIXTURE_VARIANT_ADD(hmm, hmm_device_private) 81 { 82 .device_number = HMM_PRIVATE_DEVICE_ONE, 83 }; 84 85 FIXTURE_VARIANT_ADD(hmm, hmm_device_coherent) 86 { 87 .device_number = HMM_COHERENCE_DEVICE_ONE, 88 }; 89 90 FIXTURE(hmm2) 91 { 92 int fd0; 93 int fd1; 94 unsigned int page_size; 95 unsigned int page_shift; 96 }; 97 98 FIXTURE_VARIANT(hmm2) 99 { 100 int device_number0; 101 int device_number1; 102 }; 103 104 FIXTURE_VARIANT_ADD(hmm2, hmm2_device_private) 105 { 106 .device_number0 = HMM_PRIVATE_DEVICE_ONE, 107 .device_number1 = HMM_PRIVATE_DEVICE_TWO, 108 }; 109 110 FIXTURE_VARIANT_ADD(hmm2, hmm2_device_coherent) 111 { 112 .device_number0 = HMM_COHERENCE_DEVICE_ONE, 113 .device_number1 = HMM_COHERENCE_DEVICE_TWO, 114 }; 115 116 static int hmm_open(int unit) 117 { 118 char pathname[HMM_PATH_MAX]; 119 int fd; 120 121 snprintf(pathname, sizeof(pathname), "/dev/hmm_dmirror%d", unit); 122 fd = open(pathname, O_RDWR, 0); 123 if (fd < 0) 124 fprintf(stderr, "could not open hmm dmirror driver (%s)\n", 125 pathname); 126 return fd; 127 } 128 129 static bool hmm_is_coherent_type(int dev_num) 130 { 131 return (dev_num >= HMM_COHERENCE_DEVICE_ONE); 132 } 133 134 FIXTURE_SETUP(hmm) 135 { 136 self->page_size = sysconf(_SC_PAGE_SIZE); 137 self->page_shift = ffs(self->page_size) - 1; 138 139 self->fd = hmm_open(variant->device_number); 140 if (self->fd < 0 && hmm_is_coherent_type(variant->device_number)) 141 SKIP(exit(0), "DEVICE_COHERENT not available"); 142 ASSERT_GE(self->fd, 0); 143 } 144 145 FIXTURE_SETUP(hmm2) 146 { 147 self->page_size = sysconf(_SC_PAGE_SIZE); 148 self->page_shift = ffs(self->page_size) - 1; 149 150 self->fd0 = hmm_open(variant->device_number0); 151 if (self->fd0 < 0 && hmm_is_coherent_type(variant->device_number0)) 152 SKIP(exit(0), "DEVICE_COHERENT not available"); 153 ASSERT_GE(self->fd0, 0); 154 self->fd1 = hmm_open(variant->device_number1); 155 ASSERT_GE(self->fd1, 0); 156 } 157 158 FIXTURE_TEARDOWN(hmm) 159 { 160 int ret = close(self->fd); 161 162 ASSERT_EQ(ret, 0); 163 self->fd = -1; 164 } 165 166 FIXTURE_TEARDOWN(hmm2) 167 { 168 int ret = close(self->fd0); 169 170 ASSERT_EQ(ret, 0); 171 self->fd0 = -1; 172 173 ret = close(self->fd1); 174 ASSERT_EQ(ret, 0); 175 self->fd1 = -1; 176 } 177 178 static int hmm_dmirror_cmd(int fd, 179 unsigned long request, 180 struct hmm_buffer *buffer, 181 unsigned long npages) 182 { 183 struct hmm_dmirror_cmd cmd; 184 int ret; 185 186 /* Simulate a device reading system memory. */ 187 cmd.addr = (__u64)buffer->ptr; 188 cmd.ptr = (__u64)buffer->mirror; 189 cmd.npages = npages; 190 191 for (;;) { 192 ret = ioctl(fd, request, &cmd); 193 if (ret == 0) 194 break; 195 if (errno == EINTR) 196 continue; 197 return -errno; 198 } 199 buffer->cpages = cmd.cpages; 200 buffer->faults = cmd.faults; 201 202 return 0; 203 } 204 205 static void hmm_buffer_free(struct hmm_buffer *buffer) 206 { 207 if (buffer == NULL) 208 return; 209 210 if (buffer->ptr) 211 munmap(buffer->ptr, buffer->size); 212 free(buffer->mirror); 213 free(buffer); 214 } 215 216 /* 217 * Create a temporary file that will be deleted on close. 218 */ 219 static int hmm_create_file(unsigned long size) 220 { 221 char path[HMM_PATH_MAX]; 222 int fd; 223 224 strcpy(path, "/tmp"); 225 fd = open(path, O_TMPFILE | O_EXCL | O_RDWR, 0600); 226 if (fd >= 0) { 227 int r; 228 229 do { 230 r = ftruncate(fd, size); 231 } while (r == -1 && errno == EINTR); 232 if (!r) 233 return fd; 234 close(fd); 235 } 236 return -1; 237 } 238 239 /* 240 * Return a random unsigned number. 241 */ 242 static unsigned int hmm_random(void) 243 { 244 static int fd = -1; 245 unsigned int r; 246 247 if (fd < 0) { 248 fd = open("/dev/urandom", O_RDONLY); 249 if (fd < 0) { 250 fprintf(stderr, "%s:%d failed to open /dev/urandom\n", 251 __FILE__, __LINE__); 252 return ~0U; 253 } 254 } 255 read(fd, &r, sizeof(r)); 256 return r; 257 } 258 259 static void hmm_nanosleep(unsigned int n) 260 { 261 struct timespec t; 262 263 t.tv_sec = 0; 264 t.tv_nsec = n; 265 nanosleep(&t, NULL); 266 } 267 268 static int hmm_migrate_sys_to_dev(int fd, 269 struct hmm_buffer *buffer, 270 unsigned long npages) 271 { 272 return hmm_dmirror_cmd(fd, HMM_DMIRROR_MIGRATE_TO_DEV, buffer, npages); 273 } 274 275 static int hmm_migrate_dev_to_sys(int fd, 276 struct hmm_buffer *buffer, 277 unsigned long npages) 278 { 279 return hmm_dmirror_cmd(fd, HMM_DMIRROR_MIGRATE_TO_SYS, buffer, npages); 280 } 281 282 /* 283 * Simple NULL test of device open/close. 284 */ 285 TEST_F(hmm, open_close) 286 { 287 } 288 289 /* 290 * Read private anonymous memory. 291 */ 292 TEST_F(hmm, anon_read) 293 { 294 struct hmm_buffer *buffer; 295 unsigned long npages; 296 unsigned long size; 297 unsigned long i; 298 int *ptr; 299 int ret; 300 int val; 301 302 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; 303 ASSERT_NE(npages, 0); 304 size = npages << self->page_shift; 305 306 buffer = malloc(sizeof(*buffer)); 307 ASSERT_NE(buffer, NULL); 308 309 buffer->fd = -1; 310 buffer->size = size; 311 buffer->mirror = malloc(size); 312 ASSERT_NE(buffer->mirror, NULL); 313 314 buffer->ptr = mmap(NULL, size, 315 PROT_READ | PROT_WRITE, 316 MAP_PRIVATE | MAP_ANONYMOUS, 317 buffer->fd, 0); 318 ASSERT_NE(buffer->ptr, MAP_FAILED); 319 320 /* 321 * Initialize buffer in system memory but leave the first two pages 322 * zero (pte_none and pfn_zero). 323 */ 324 i = 2 * self->page_size / sizeof(*ptr); 325 for (ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 326 ptr[i] = i; 327 328 /* Set buffer permission to read-only. */ 329 ret = mprotect(buffer->ptr, size, PROT_READ); 330 ASSERT_EQ(ret, 0); 331 332 /* Populate the CPU page table with a special zero page. */ 333 val = *(int *)(buffer->ptr + self->page_size); 334 ASSERT_EQ(val, 0); 335 336 /* Simulate a device reading system memory. */ 337 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ, buffer, npages); 338 ASSERT_EQ(ret, 0); 339 ASSERT_EQ(buffer->cpages, npages); 340 ASSERT_EQ(buffer->faults, 1); 341 342 /* Check what the device read. */ 343 ptr = buffer->mirror; 344 for (i = 0; i < 2 * self->page_size / sizeof(*ptr); ++i) 345 ASSERT_EQ(ptr[i], 0); 346 for (; i < size / sizeof(*ptr); ++i) 347 ASSERT_EQ(ptr[i], i); 348 349 hmm_buffer_free(buffer); 350 } 351 352 /* 353 * Read private anonymous memory which has been protected with 354 * mprotect() PROT_NONE. 355 */ 356 TEST_F(hmm, anon_read_prot) 357 { 358 struct hmm_buffer *buffer; 359 unsigned long npages; 360 unsigned long size; 361 unsigned long i; 362 int *ptr; 363 int ret; 364 365 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; 366 ASSERT_NE(npages, 0); 367 size = npages << self->page_shift; 368 369 buffer = malloc(sizeof(*buffer)); 370 ASSERT_NE(buffer, NULL); 371 372 buffer->fd = -1; 373 buffer->size = size; 374 buffer->mirror = malloc(size); 375 ASSERT_NE(buffer->mirror, NULL); 376 377 buffer->ptr = mmap(NULL, size, 378 PROT_READ | PROT_WRITE, 379 MAP_PRIVATE | MAP_ANONYMOUS, 380 buffer->fd, 0); 381 ASSERT_NE(buffer->ptr, MAP_FAILED); 382 383 /* Initialize buffer in system memory. */ 384 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 385 ptr[i] = i; 386 387 /* Initialize mirror buffer so we can verify it isn't written. */ 388 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 389 ptr[i] = -i; 390 391 /* Protect buffer from reading. */ 392 ret = mprotect(buffer->ptr, size, PROT_NONE); 393 ASSERT_EQ(ret, 0); 394 395 /* Simulate a device reading system memory. */ 396 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ, buffer, npages); 397 ASSERT_EQ(ret, -EFAULT); 398 399 /* Allow CPU to read the buffer so we can check it. */ 400 ret = mprotect(buffer->ptr, size, PROT_READ); 401 ASSERT_EQ(ret, 0); 402 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 403 ASSERT_EQ(ptr[i], i); 404 405 /* Check what the device read. */ 406 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 407 ASSERT_EQ(ptr[i], -i); 408 409 hmm_buffer_free(buffer); 410 } 411 412 /* 413 * Write private anonymous memory. 414 */ 415 TEST_F(hmm, anon_write) 416 { 417 struct hmm_buffer *buffer; 418 unsigned long npages; 419 unsigned long size; 420 unsigned long i; 421 int *ptr; 422 int ret; 423 424 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; 425 ASSERT_NE(npages, 0); 426 size = npages << self->page_shift; 427 428 buffer = malloc(sizeof(*buffer)); 429 ASSERT_NE(buffer, NULL); 430 431 buffer->fd = -1; 432 buffer->size = size; 433 buffer->mirror = malloc(size); 434 ASSERT_NE(buffer->mirror, NULL); 435 436 buffer->ptr = mmap(NULL, size, 437 PROT_READ | PROT_WRITE, 438 MAP_PRIVATE | MAP_ANONYMOUS, 439 buffer->fd, 0); 440 ASSERT_NE(buffer->ptr, MAP_FAILED); 441 442 /* Initialize data that the device will write to buffer->ptr. */ 443 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 444 ptr[i] = i; 445 446 /* Simulate a device writing system memory. */ 447 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages); 448 ASSERT_EQ(ret, 0); 449 ASSERT_EQ(buffer->cpages, npages); 450 ASSERT_EQ(buffer->faults, 1); 451 452 /* Check what the device wrote. */ 453 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 454 ASSERT_EQ(ptr[i], i); 455 456 hmm_buffer_free(buffer); 457 } 458 459 /* 460 * Write private anonymous memory which has been protected with 461 * mprotect() PROT_READ. 462 */ 463 TEST_F(hmm, anon_write_prot) 464 { 465 struct hmm_buffer *buffer; 466 unsigned long npages; 467 unsigned long size; 468 unsigned long i; 469 int *ptr; 470 int ret; 471 472 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; 473 ASSERT_NE(npages, 0); 474 size = npages << self->page_shift; 475 476 buffer = malloc(sizeof(*buffer)); 477 ASSERT_NE(buffer, NULL); 478 479 buffer->fd = -1; 480 buffer->size = size; 481 buffer->mirror = malloc(size); 482 ASSERT_NE(buffer->mirror, NULL); 483 484 buffer->ptr = mmap(NULL, size, 485 PROT_READ, 486 MAP_PRIVATE | MAP_ANONYMOUS, 487 buffer->fd, 0); 488 ASSERT_NE(buffer->ptr, MAP_FAILED); 489 490 /* Simulate a device reading a zero page of memory. */ 491 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ, buffer, 1); 492 ASSERT_EQ(ret, 0); 493 ASSERT_EQ(buffer->cpages, 1); 494 ASSERT_EQ(buffer->faults, 1); 495 496 /* Initialize data that the device will write to buffer->ptr. */ 497 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 498 ptr[i] = i; 499 500 /* Simulate a device writing system memory. */ 501 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages); 502 ASSERT_EQ(ret, -EPERM); 503 504 /* Check what the device wrote. */ 505 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 506 ASSERT_EQ(ptr[i], 0); 507 508 /* Now allow writing and see that the zero page is replaced. */ 509 ret = mprotect(buffer->ptr, size, PROT_WRITE | PROT_READ); 510 ASSERT_EQ(ret, 0); 511 512 /* Simulate a device writing system memory. */ 513 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages); 514 ASSERT_EQ(ret, 0); 515 ASSERT_EQ(buffer->cpages, npages); 516 ASSERT_EQ(buffer->faults, 1); 517 518 /* Check what the device wrote. */ 519 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 520 ASSERT_EQ(ptr[i], i); 521 522 hmm_buffer_free(buffer); 523 } 524 525 /* 526 * Check that a device writing an anonymous private mapping 527 * will copy-on-write if a child process inherits the mapping. 528 */ 529 TEST_F(hmm, anon_write_child) 530 { 531 struct hmm_buffer *buffer; 532 unsigned long npages; 533 unsigned long size; 534 unsigned long i; 535 int *ptr; 536 pid_t pid; 537 int child_fd; 538 int ret; 539 540 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; 541 ASSERT_NE(npages, 0); 542 size = npages << self->page_shift; 543 544 buffer = malloc(sizeof(*buffer)); 545 ASSERT_NE(buffer, NULL); 546 547 buffer->fd = -1; 548 buffer->size = size; 549 buffer->mirror = malloc(size); 550 ASSERT_NE(buffer->mirror, NULL); 551 552 buffer->ptr = mmap(NULL, size, 553 PROT_READ | PROT_WRITE, 554 MAP_PRIVATE | MAP_ANONYMOUS, 555 buffer->fd, 0); 556 ASSERT_NE(buffer->ptr, MAP_FAILED); 557 558 /* Initialize buffer->ptr so we can tell if it is written. */ 559 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 560 ptr[i] = i; 561 562 /* Initialize data that the device will write to buffer->ptr. */ 563 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 564 ptr[i] = -i; 565 566 pid = fork(); 567 if (pid == -1) 568 ASSERT_EQ(pid, 0); 569 if (pid != 0) { 570 waitpid(pid, &ret, 0); 571 ASSERT_EQ(WIFEXITED(ret), 1); 572 573 /* Check that the parent's buffer did not change. */ 574 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 575 ASSERT_EQ(ptr[i], i); 576 return; 577 } 578 579 /* Check that we see the parent's values. */ 580 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 581 ASSERT_EQ(ptr[i], i); 582 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 583 ASSERT_EQ(ptr[i], -i); 584 585 /* The child process needs its own mirror to its own mm. */ 586 child_fd = hmm_open(0); 587 ASSERT_GE(child_fd, 0); 588 589 /* Simulate a device writing system memory. */ 590 ret = hmm_dmirror_cmd(child_fd, HMM_DMIRROR_WRITE, buffer, npages); 591 ASSERT_EQ(ret, 0); 592 ASSERT_EQ(buffer->cpages, npages); 593 ASSERT_EQ(buffer->faults, 1); 594 595 /* Check what the device wrote. */ 596 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 597 ASSERT_EQ(ptr[i], -i); 598 599 close(child_fd); 600 exit(0); 601 } 602 603 /* 604 * Check that a device writing an anonymous shared mapping 605 * will not copy-on-write if a child process inherits the mapping. 606 */ 607 TEST_F(hmm, anon_write_child_shared) 608 { 609 struct hmm_buffer *buffer; 610 unsigned long npages; 611 unsigned long size; 612 unsigned long i; 613 int *ptr; 614 pid_t pid; 615 int child_fd; 616 int ret; 617 618 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; 619 ASSERT_NE(npages, 0); 620 size = npages << self->page_shift; 621 622 buffer = malloc(sizeof(*buffer)); 623 ASSERT_NE(buffer, NULL); 624 625 buffer->fd = -1; 626 buffer->size = size; 627 buffer->mirror = malloc(size); 628 ASSERT_NE(buffer->mirror, NULL); 629 630 buffer->ptr = mmap(NULL, size, 631 PROT_READ | PROT_WRITE, 632 MAP_SHARED | MAP_ANONYMOUS, 633 buffer->fd, 0); 634 ASSERT_NE(buffer->ptr, MAP_FAILED); 635 636 /* Initialize buffer->ptr so we can tell if it is written. */ 637 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 638 ptr[i] = i; 639 640 /* Initialize data that the device will write to buffer->ptr. */ 641 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 642 ptr[i] = -i; 643 644 pid = fork(); 645 if (pid == -1) 646 ASSERT_EQ(pid, 0); 647 if (pid != 0) { 648 waitpid(pid, &ret, 0); 649 ASSERT_EQ(WIFEXITED(ret), 1); 650 651 /* Check that the parent's buffer did change. */ 652 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 653 ASSERT_EQ(ptr[i], -i); 654 return; 655 } 656 657 /* Check that we see the parent's values. */ 658 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 659 ASSERT_EQ(ptr[i], i); 660 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 661 ASSERT_EQ(ptr[i], -i); 662 663 /* The child process needs its own mirror to its own mm. */ 664 child_fd = hmm_open(0); 665 ASSERT_GE(child_fd, 0); 666 667 /* Simulate a device writing system memory. */ 668 ret = hmm_dmirror_cmd(child_fd, HMM_DMIRROR_WRITE, buffer, npages); 669 ASSERT_EQ(ret, 0); 670 ASSERT_EQ(buffer->cpages, npages); 671 ASSERT_EQ(buffer->faults, 1); 672 673 /* Check what the device wrote. */ 674 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 675 ASSERT_EQ(ptr[i], -i); 676 677 close(child_fd); 678 exit(0); 679 } 680 681 /* 682 * Write private anonymous huge page. 683 */ 684 TEST_F(hmm, anon_write_huge) 685 { 686 struct hmm_buffer *buffer; 687 unsigned long npages; 688 unsigned long size; 689 unsigned long i; 690 void *old_ptr; 691 void *map; 692 int *ptr; 693 int ret; 694 695 size = 2 * TWOMEG; 696 697 buffer = malloc(sizeof(*buffer)); 698 ASSERT_NE(buffer, NULL); 699 700 buffer->fd = -1; 701 buffer->size = size; 702 buffer->mirror = malloc(size); 703 ASSERT_NE(buffer->mirror, NULL); 704 705 buffer->ptr = mmap(NULL, size, 706 PROT_READ | PROT_WRITE, 707 MAP_PRIVATE | MAP_ANONYMOUS, 708 buffer->fd, 0); 709 ASSERT_NE(buffer->ptr, MAP_FAILED); 710 711 size = TWOMEG; 712 npages = size >> self->page_shift; 713 map = (void *)ALIGN((uintptr_t)buffer->ptr, size); 714 ret = madvise(map, size, MADV_HUGEPAGE); 715 ASSERT_EQ(ret, 0); 716 old_ptr = buffer->ptr; 717 buffer->ptr = map; 718 719 /* Initialize data that the device will write to buffer->ptr. */ 720 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 721 ptr[i] = i; 722 723 /* Simulate a device writing system memory. */ 724 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages); 725 ASSERT_EQ(ret, 0); 726 ASSERT_EQ(buffer->cpages, npages); 727 ASSERT_EQ(buffer->faults, 1); 728 729 /* Check what the device wrote. */ 730 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 731 ASSERT_EQ(ptr[i], i); 732 733 buffer->ptr = old_ptr; 734 hmm_buffer_free(buffer); 735 } 736 737 /* 738 * Read numeric data from raw and tagged kernel status files. Used to read 739 * /proc and /sys data (without a tag) and from /proc/meminfo (with a tag). 740 */ 741 static long file_read_ulong(char *file, const char *tag) 742 { 743 int fd; 744 char buf[2048]; 745 int len; 746 char *p, *q; 747 long val; 748 749 fd = open(file, O_RDONLY); 750 if (fd < 0) { 751 /* Error opening the file */ 752 return -1; 753 } 754 755 len = read(fd, buf, sizeof(buf)); 756 close(fd); 757 if (len < 0) { 758 /* Error in reading the file */ 759 return -1; 760 } 761 if (len == sizeof(buf)) { 762 /* Error file is too large */ 763 return -1; 764 } 765 buf[len] = '\0'; 766 767 /* Search for a tag if provided */ 768 if (tag) { 769 p = strstr(buf, tag); 770 if (!p) 771 return -1; /* looks like the line we want isn't there */ 772 p += strlen(tag); 773 } else 774 p = buf; 775 776 val = strtol(p, &q, 0); 777 if (*q != ' ') { 778 /* Error parsing the file */ 779 return -1; 780 } 781 782 return val; 783 } 784 785 /* 786 * Write huge TLBFS page. 787 */ 788 TEST_F(hmm, anon_write_hugetlbfs) 789 { 790 struct hmm_buffer *buffer; 791 unsigned long npages; 792 unsigned long size; 793 unsigned long default_hsize; 794 unsigned long i; 795 int *ptr; 796 int ret; 797 798 default_hsize = file_read_ulong("/proc/meminfo", "Hugepagesize:"); 799 if (default_hsize < 0 || default_hsize*1024 < default_hsize) 800 SKIP(return, "Huge page size could not be determined"); 801 default_hsize = default_hsize*1024; /* KB to B */ 802 803 size = ALIGN(TWOMEG, default_hsize); 804 npages = size >> self->page_shift; 805 806 buffer = malloc(sizeof(*buffer)); 807 ASSERT_NE(buffer, NULL); 808 809 buffer->ptr = mmap(NULL, size, 810 PROT_READ | PROT_WRITE, 811 MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, 812 -1, 0); 813 if (buffer->ptr == MAP_FAILED) { 814 free(buffer); 815 SKIP(return, "Huge page could not be allocated"); 816 } 817 818 buffer->fd = -1; 819 buffer->size = size; 820 buffer->mirror = malloc(size); 821 ASSERT_NE(buffer->mirror, NULL); 822 823 /* Initialize data that the device will write to buffer->ptr. */ 824 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 825 ptr[i] = i; 826 827 /* Simulate a device writing system memory. */ 828 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages); 829 ASSERT_EQ(ret, 0); 830 ASSERT_EQ(buffer->cpages, npages); 831 ASSERT_EQ(buffer->faults, 1); 832 833 /* Check what the device wrote. */ 834 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 835 ASSERT_EQ(ptr[i], i); 836 837 munmap(buffer->ptr, buffer->size); 838 buffer->ptr = NULL; 839 hmm_buffer_free(buffer); 840 } 841 842 /* 843 * Read mmap'ed file memory. 844 */ 845 TEST_F(hmm, file_read) 846 { 847 struct hmm_buffer *buffer; 848 unsigned long npages; 849 unsigned long size; 850 unsigned long i; 851 int *ptr; 852 int ret; 853 int fd; 854 ssize_t len; 855 856 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; 857 ASSERT_NE(npages, 0); 858 size = npages << self->page_shift; 859 860 fd = hmm_create_file(size); 861 ASSERT_GE(fd, 0); 862 863 buffer = malloc(sizeof(*buffer)); 864 ASSERT_NE(buffer, NULL); 865 866 buffer->fd = fd; 867 buffer->size = size; 868 buffer->mirror = malloc(size); 869 ASSERT_NE(buffer->mirror, NULL); 870 871 /* Write initial contents of the file. */ 872 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 873 ptr[i] = i; 874 len = pwrite(fd, buffer->mirror, size, 0); 875 ASSERT_EQ(len, size); 876 memset(buffer->mirror, 0, size); 877 878 buffer->ptr = mmap(NULL, size, 879 PROT_READ, 880 MAP_SHARED, 881 buffer->fd, 0); 882 ASSERT_NE(buffer->ptr, MAP_FAILED); 883 884 /* Simulate a device reading system memory. */ 885 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ, buffer, npages); 886 ASSERT_EQ(ret, 0); 887 ASSERT_EQ(buffer->cpages, npages); 888 ASSERT_EQ(buffer->faults, 1); 889 890 /* Check what the device read. */ 891 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 892 ASSERT_EQ(ptr[i], i); 893 894 hmm_buffer_free(buffer); 895 } 896 897 /* 898 * Write mmap'ed file memory. 899 */ 900 TEST_F(hmm, file_write) 901 { 902 struct hmm_buffer *buffer; 903 unsigned long npages; 904 unsigned long size; 905 unsigned long i; 906 int *ptr; 907 int ret; 908 int fd; 909 ssize_t len; 910 911 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; 912 ASSERT_NE(npages, 0); 913 size = npages << self->page_shift; 914 915 fd = hmm_create_file(size); 916 ASSERT_GE(fd, 0); 917 918 buffer = malloc(sizeof(*buffer)); 919 ASSERT_NE(buffer, NULL); 920 921 buffer->fd = fd; 922 buffer->size = size; 923 buffer->mirror = malloc(size); 924 ASSERT_NE(buffer->mirror, NULL); 925 926 buffer->ptr = mmap(NULL, size, 927 PROT_READ | PROT_WRITE, 928 MAP_SHARED, 929 buffer->fd, 0); 930 ASSERT_NE(buffer->ptr, MAP_FAILED); 931 932 /* Initialize data that the device will write to buffer->ptr. */ 933 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 934 ptr[i] = i; 935 936 /* Simulate a device writing system memory. */ 937 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages); 938 ASSERT_EQ(ret, 0); 939 ASSERT_EQ(buffer->cpages, npages); 940 ASSERT_EQ(buffer->faults, 1); 941 942 /* Check what the device wrote. */ 943 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 944 ASSERT_EQ(ptr[i], i); 945 946 /* Check that the device also wrote the file. */ 947 len = pread(fd, buffer->mirror, size, 0); 948 ASSERT_EQ(len, size); 949 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 950 ASSERT_EQ(ptr[i], i); 951 952 hmm_buffer_free(buffer); 953 } 954 955 /* 956 * Migrate anonymous memory to device private memory. 957 */ 958 TEST_F(hmm, migrate) 959 { 960 struct hmm_buffer *buffer; 961 unsigned long npages; 962 unsigned long size; 963 unsigned long i; 964 int *ptr; 965 int ret; 966 967 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; 968 ASSERT_NE(npages, 0); 969 size = npages << self->page_shift; 970 971 buffer = malloc(sizeof(*buffer)); 972 ASSERT_NE(buffer, NULL); 973 974 buffer->fd = -1; 975 buffer->size = size; 976 buffer->mirror = malloc(size); 977 ASSERT_NE(buffer->mirror, NULL); 978 979 buffer->ptr = mmap(NULL, size, 980 PROT_READ | PROT_WRITE, 981 MAP_PRIVATE | MAP_ANONYMOUS, 982 buffer->fd, 0); 983 ASSERT_NE(buffer->ptr, MAP_FAILED); 984 985 /* Initialize buffer in system memory. */ 986 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 987 ptr[i] = i; 988 989 /* Migrate memory to device. */ 990 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); 991 ASSERT_EQ(ret, 0); 992 ASSERT_EQ(buffer->cpages, npages); 993 994 /* Check what the device read. */ 995 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 996 ASSERT_EQ(ptr[i], i); 997 998 hmm_buffer_free(buffer); 999 } 1000 1001 /* 1002 * Migrate anonymous memory to device private memory and fault some of it back 1003 * to system memory, then try migrating the resulting mix of system and device 1004 * private memory to the device. 1005 */ 1006 TEST_F(hmm, migrate_fault) 1007 { 1008 struct hmm_buffer *buffer; 1009 unsigned long npages; 1010 unsigned long size; 1011 unsigned long i; 1012 int *ptr; 1013 int ret; 1014 1015 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; 1016 ASSERT_NE(npages, 0); 1017 size = npages << self->page_shift; 1018 1019 buffer = malloc(sizeof(*buffer)); 1020 ASSERT_NE(buffer, NULL); 1021 1022 buffer->fd = -1; 1023 buffer->size = size; 1024 buffer->mirror = malloc(size); 1025 ASSERT_NE(buffer->mirror, NULL); 1026 1027 buffer->ptr = mmap(NULL, size, 1028 PROT_READ | PROT_WRITE, 1029 MAP_PRIVATE | MAP_ANONYMOUS, 1030 buffer->fd, 0); 1031 ASSERT_NE(buffer->ptr, MAP_FAILED); 1032 1033 /* Initialize buffer in system memory. */ 1034 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 1035 ptr[i] = i; 1036 1037 /* Migrate memory to device. */ 1038 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); 1039 ASSERT_EQ(ret, 0); 1040 ASSERT_EQ(buffer->cpages, npages); 1041 1042 /* Check what the device read. */ 1043 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 1044 ASSERT_EQ(ptr[i], i); 1045 1046 /* Fault half the pages back to system memory and check them. */ 1047 for (i = 0, ptr = buffer->ptr; i < size / (2 * sizeof(*ptr)); ++i) 1048 ASSERT_EQ(ptr[i], i); 1049 1050 /* Migrate memory to the device again. */ 1051 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); 1052 ASSERT_EQ(ret, 0); 1053 ASSERT_EQ(buffer->cpages, npages); 1054 1055 /* Check what the device read. */ 1056 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 1057 ASSERT_EQ(ptr[i], i); 1058 1059 hmm_buffer_free(buffer); 1060 } 1061 1062 TEST_F(hmm, migrate_release) 1063 { 1064 struct hmm_buffer *buffer; 1065 unsigned long npages; 1066 unsigned long size; 1067 unsigned long i; 1068 int *ptr; 1069 int ret; 1070 1071 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; 1072 ASSERT_NE(npages, 0); 1073 size = npages << self->page_shift; 1074 1075 buffer = malloc(sizeof(*buffer)); 1076 ASSERT_NE(buffer, NULL); 1077 1078 buffer->fd = -1; 1079 buffer->size = size; 1080 buffer->mirror = malloc(size); 1081 ASSERT_NE(buffer->mirror, NULL); 1082 1083 buffer->ptr = mmap(NULL, size, PROT_READ | PROT_WRITE, 1084 MAP_PRIVATE | MAP_ANONYMOUS, buffer->fd, 0); 1085 ASSERT_NE(buffer->ptr, MAP_FAILED); 1086 1087 /* Initialize buffer in system memory. */ 1088 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 1089 ptr[i] = i; 1090 1091 /* Migrate memory to device. */ 1092 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); 1093 ASSERT_EQ(ret, 0); 1094 ASSERT_EQ(buffer->cpages, npages); 1095 1096 /* Check what the device read. */ 1097 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 1098 ASSERT_EQ(ptr[i], i); 1099 1100 /* Release device memory. */ 1101 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_RELEASE, buffer, npages); 1102 ASSERT_EQ(ret, 0); 1103 1104 /* Fault pages back to system memory and check them. */ 1105 for (i = 0, ptr = buffer->ptr; i < size / (2 * sizeof(*ptr)); ++i) 1106 ASSERT_EQ(ptr[i], i); 1107 1108 hmm_buffer_free(buffer); 1109 } 1110 1111 /* 1112 * Migrate anonymous shared memory to device private memory. 1113 */ 1114 TEST_F(hmm, migrate_shared) 1115 { 1116 struct hmm_buffer *buffer; 1117 unsigned long npages; 1118 unsigned long size; 1119 int ret; 1120 1121 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; 1122 ASSERT_NE(npages, 0); 1123 size = npages << self->page_shift; 1124 1125 buffer = malloc(sizeof(*buffer)); 1126 ASSERT_NE(buffer, NULL); 1127 1128 buffer->fd = -1; 1129 buffer->size = size; 1130 buffer->mirror = malloc(size); 1131 ASSERT_NE(buffer->mirror, NULL); 1132 1133 buffer->ptr = mmap(NULL, size, 1134 PROT_READ | PROT_WRITE, 1135 MAP_SHARED | MAP_ANONYMOUS, 1136 buffer->fd, 0); 1137 ASSERT_NE(buffer->ptr, MAP_FAILED); 1138 1139 /* Migrate memory to device. */ 1140 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); 1141 ASSERT_EQ(ret, -ENOENT); 1142 1143 hmm_buffer_free(buffer); 1144 } 1145 1146 /* 1147 * Try to migrate various memory types to device private memory. 1148 */ 1149 TEST_F(hmm2, migrate_mixed) 1150 { 1151 struct hmm_buffer *buffer; 1152 unsigned long npages; 1153 unsigned long size; 1154 int *ptr; 1155 unsigned char *p; 1156 int ret; 1157 int val; 1158 1159 npages = 6; 1160 size = npages << self->page_shift; 1161 1162 buffer = malloc(sizeof(*buffer)); 1163 ASSERT_NE(buffer, NULL); 1164 1165 buffer->fd = -1; 1166 buffer->size = size; 1167 buffer->mirror = malloc(size); 1168 ASSERT_NE(buffer->mirror, NULL); 1169 1170 /* Reserve a range of addresses. */ 1171 buffer->ptr = mmap(NULL, size, 1172 PROT_NONE, 1173 MAP_PRIVATE | MAP_ANONYMOUS, 1174 buffer->fd, 0); 1175 ASSERT_NE(buffer->ptr, MAP_FAILED); 1176 p = buffer->ptr; 1177 1178 /* Migrating a protected area should be an error. */ 1179 ret = hmm_migrate_sys_to_dev(self->fd1, buffer, npages); 1180 ASSERT_EQ(ret, -EINVAL); 1181 1182 /* Punch a hole after the first page address. */ 1183 ret = munmap(buffer->ptr + self->page_size, self->page_size); 1184 ASSERT_EQ(ret, 0); 1185 1186 /* We expect an error if the vma doesn't cover the range. */ 1187 ret = hmm_migrate_sys_to_dev(self->fd1, buffer, 3); 1188 ASSERT_EQ(ret, -EINVAL); 1189 1190 /* Page 2 will be a read-only zero page. */ 1191 ret = mprotect(buffer->ptr + 2 * self->page_size, self->page_size, 1192 PROT_READ); 1193 ASSERT_EQ(ret, 0); 1194 ptr = (int *)(buffer->ptr + 2 * self->page_size); 1195 val = *ptr + 3; 1196 ASSERT_EQ(val, 3); 1197 1198 /* Page 3 will be read-only. */ 1199 ret = mprotect(buffer->ptr + 3 * self->page_size, self->page_size, 1200 PROT_READ | PROT_WRITE); 1201 ASSERT_EQ(ret, 0); 1202 ptr = (int *)(buffer->ptr + 3 * self->page_size); 1203 *ptr = val; 1204 ret = mprotect(buffer->ptr + 3 * self->page_size, self->page_size, 1205 PROT_READ); 1206 ASSERT_EQ(ret, 0); 1207 1208 /* Page 4-5 will be read-write. */ 1209 ret = mprotect(buffer->ptr + 4 * self->page_size, 2 * self->page_size, 1210 PROT_READ | PROT_WRITE); 1211 ASSERT_EQ(ret, 0); 1212 ptr = (int *)(buffer->ptr + 4 * self->page_size); 1213 *ptr = val; 1214 ptr = (int *)(buffer->ptr + 5 * self->page_size); 1215 *ptr = val; 1216 1217 /* Now try to migrate pages 2-5 to device 1. */ 1218 buffer->ptr = p + 2 * self->page_size; 1219 ret = hmm_migrate_sys_to_dev(self->fd1, buffer, 4); 1220 ASSERT_EQ(ret, 0); 1221 ASSERT_EQ(buffer->cpages, 4); 1222 1223 /* Page 5 won't be migrated to device 0 because it's on device 1. */ 1224 buffer->ptr = p + 5 * self->page_size; 1225 ret = hmm_migrate_sys_to_dev(self->fd0, buffer, 1); 1226 ASSERT_EQ(ret, -ENOENT); 1227 buffer->ptr = p; 1228 1229 buffer->ptr = p; 1230 hmm_buffer_free(buffer); 1231 } 1232 1233 /* 1234 * Migrate anonymous memory to device memory and back to system memory 1235 * multiple times. In case of private zone configuration, this is done 1236 * through fault pages accessed by CPU. In case of coherent zone configuration, 1237 * the pages from the device should be explicitly migrated back to system memory. 1238 * The reason is Coherent device zone has coherent access by CPU, therefore 1239 * it will not generate any page fault. 1240 */ 1241 TEST_F(hmm, migrate_multiple) 1242 { 1243 struct hmm_buffer *buffer; 1244 unsigned long npages; 1245 unsigned long size; 1246 unsigned long i; 1247 unsigned long c; 1248 int *ptr; 1249 int ret; 1250 1251 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; 1252 ASSERT_NE(npages, 0); 1253 size = npages << self->page_shift; 1254 1255 for (c = 0; c < NTIMES; c++) { 1256 buffer = malloc(sizeof(*buffer)); 1257 ASSERT_NE(buffer, NULL); 1258 1259 buffer->fd = -1; 1260 buffer->size = size; 1261 buffer->mirror = malloc(size); 1262 ASSERT_NE(buffer->mirror, NULL); 1263 1264 buffer->ptr = mmap(NULL, size, 1265 PROT_READ | PROT_WRITE, 1266 MAP_PRIVATE | MAP_ANONYMOUS, 1267 buffer->fd, 0); 1268 ASSERT_NE(buffer->ptr, MAP_FAILED); 1269 1270 /* Initialize buffer in system memory. */ 1271 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 1272 ptr[i] = i; 1273 1274 /* Migrate memory to device. */ 1275 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); 1276 ASSERT_EQ(ret, 0); 1277 ASSERT_EQ(buffer->cpages, npages); 1278 1279 /* Check what the device read. */ 1280 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 1281 ASSERT_EQ(ptr[i], i); 1282 1283 /* Migrate back to system memory and check them. */ 1284 if (hmm_is_coherent_type(variant->device_number)) { 1285 ret = hmm_migrate_dev_to_sys(self->fd, buffer, npages); 1286 ASSERT_EQ(ret, 0); 1287 ASSERT_EQ(buffer->cpages, npages); 1288 } 1289 1290 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 1291 ASSERT_EQ(ptr[i], i); 1292 1293 hmm_buffer_free(buffer); 1294 } 1295 } 1296 1297 /* 1298 * Read anonymous memory multiple times. 1299 */ 1300 TEST_F(hmm, anon_read_multiple) 1301 { 1302 struct hmm_buffer *buffer; 1303 unsigned long npages; 1304 unsigned long size; 1305 unsigned long i; 1306 unsigned long c; 1307 int *ptr; 1308 int ret; 1309 1310 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; 1311 ASSERT_NE(npages, 0); 1312 size = npages << self->page_shift; 1313 1314 for (c = 0; c < NTIMES; c++) { 1315 buffer = malloc(sizeof(*buffer)); 1316 ASSERT_NE(buffer, NULL); 1317 1318 buffer->fd = -1; 1319 buffer->size = size; 1320 buffer->mirror = malloc(size); 1321 ASSERT_NE(buffer->mirror, NULL); 1322 1323 buffer->ptr = mmap(NULL, size, 1324 PROT_READ | PROT_WRITE, 1325 MAP_PRIVATE | MAP_ANONYMOUS, 1326 buffer->fd, 0); 1327 ASSERT_NE(buffer->ptr, MAP_FAILED); 1328 1329 /* Initialize buffer in system memory. */ 1330 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 1331 ptr[i] = i + c; 1332 1333 /* Simulate a device reading system memory. */ 1334 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ, buffer, 1335 npages); 1336 ASSERT_EQ(ret, 0); 1337 ASSERT_EQ(buffer->cpages, npages); 1338 ASSERT_EQ(buffer->faults, 1); 1339 1340 /* Check what the device read. */ 1341 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 1342 ASSERT_EQ(ptr[i], i + c); 1343 1344 hmm_buffer_free(buffer); 1345 } 1346 } 1347 1348 void *unmap_buffer(void *p) 1349 { 1350 struct hmm_buffer *buffer = p; 1351 1352 /* Delay for a bit and then unmap buffer while it is being read. */ 1353 hmm_nanosleep(hmm_random() % 32000); 1354 munmap(buffer->ptr + buffer->size / 2, buffer->size / 2); 1355 buffer->ptr = NULL; 1356 1357 return NULL; 1358 } 1359 1360 /* 1361 * Try reading anonymous memory while it is being unmapped. 1362 */ 1363 TEST_F(hmm, anon_teardown) 1364 { 1365 unsigned long npages; 1366 unsigned long size; 1367 unsigned long c; 1368 void *ret; 1369 1370 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; 1371 ASSERT_NE(npages, 0); 1372 size = npages << self->page_shift; 1373 1374 for (c = 0; c < NTIMES; ++c) { 1375 pthread_t thread; 1376 struct hmm_buffer *buffer; 1377 unsigned long i; 1378 int *ptr; 1379 int rc; 1380 1381 buffer = malloc(sizeof(*buffer)); 1382 ASSERT_NE(buffer, NULL); 1383 1384 buffer->fd = -1; 1385 buffer->size = size; 1386 buffer->mirror = malloc(size); 1387 ASSERT_NE(buffer->mirror, NULL); 1388 1389 buffer->ptr = mmap(NULL, size, 1390 PROT_READ | PROT_WRITE, 1391 MAP_PRIVATE | MAP_ANONYMOUS, 1392 buffer->fd, 0); 1393 ASSERT_NE(buffer->ptr, MAP_FAILED); 1394 1395 /* Initialize buffer in system memory. */ 1396 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 1397 ptr[i] = i + c; 1398 1399 rc = pthread_create(&thread, NULL, unmap_buffer, buffer); 1400 ASSERT_EQ(rc, 0); 1401 1402 /* Simulate a device reading system memory. */ 1403 rc = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ, buffer, 1404 npages); 1405 if (rc == 0) { 1406 ASSERT_EQ(buffer->cpages, npages); 1407 ASSERT_EQ(buffer->faults, 1); 1408 1409 /* Check what the device read. */ 1410 for (i = 0, ptr = buffer->mirror; 1411 i < size / sizeof(*ptr); 1412 ++i) 1413 ASSERT_EQ(ptr[i], i + c); 1414 } 1415 1416 pthread_join(thread, &ret); 1417 hmm_buffer_free(buffer); 1418 } 1419 } 1420 1421 /* 1422 * Test memory snapshot without faulting in pages accessed by the device. 1423 */ 1424 TEST_F(hmm, mixedmap) 1425 { 1426 struct hmm_buffer *buffer; 1427 unsigned long npages; 1428 unsigned long size; 1429 unsigned char *m; 1430 int ret; 1431 1432 npages = 1; 1433 size = npages << self->page_shift; 1434 1435 buffer = malloc(sizeof(*buffer)); 1436 ASSERT_NE(buffer, NULL); 1437 1438 buffer->fd = -1; 1439 buffer->size = size; 1440 buffer->mirror = malloc(npages); 1441 ASSERT_NE(buffer->mirror, NULL); 1442 1443 1444 /* Reserve a range of addresses. */ 1445 buffer->ptr = mmap(NULL, size, 1446 PROT_READ | PROT_WRITE, 1447 MAP_PRIVATE, 1448 self->fd, 0); 1449 ASSERT_NE(buffer->ptr, MAP_FAILED); 1450 1451 /* Simulate a device snapshotting CPU pagetables. */ 1452 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_SNAPSHOT, buffer, npages); 1453 ASSERT_EQ(ret, 0); 1454 ASSERT_EQ(buffer->cpages, npages); 1455 1456 /* Check what the device saw. */ 1457 m = buffer->mirror; 1458 ASSERT_EQ(m[0], HMM_DMIRROR_PROT_READ); 1459 1460 hmm_buffer_free(buffer); 1461 } 1462 1463 /* 1464 * Test memory snapshot without faulting in pages accessed by the device. 1465 */ 1466 TEST_F(hmm2, snapshot) 1467 { 1468 struct hmm_buffer *buffer; 1469 unsigned long npages; 1470 unsigned long size; 1471 int *ptr; 1472 unsigned char *p; 1473 unsigned char *m; 1474 int ret; 1475 int val; 1476 1477 npages = 7; 1478 size = npages << self->page_shift; 1479 1480 buffer = malloc(sizeof(*buffer)); 1481 ASSERT_NE(buffer, NULL); 1482 1483 buffer->fd = -1; 1484 buffer->size = size; 1485 buffer->mirror = malloc(npages); 1486 ASSERT_NE(buffer->mirror, NULL); 1487 1488 /* Reserve a range of addresses. */ 1489 buffer->ptr = mmap(NULL, size, 1490 PROT_NONE, 1491 MAP_PRIVATE | MAP_ANONYMOUS, 1492 buffer->fd, 0); 1493 ASSERT_NE(buffer->ptr, MAP_FAILED); 1494 p = buffer->ptr; 1495 1496 /* Punch a hole after the first page address. */ 1497 ret = munmap(buffer->ptr + self->page_size, self->page_size); 1498 ASSERT_EQ(ret, 0); 1499 1500 /* Page 2 will be read-only zero page. */ 1501 ret = mprotect(buffer->ptr + 2 * self->page_size, self->page_size, 1502 PROT_READ); 1503 ASSERT_EQ(ret, 0); 1504 ptr = (int *)(buffer->ptr + 2 * self->page_size); 1505 val = *ptr + 3; 1506 ASSERT_EQ(val, 3); 1507 1508 /* Page 3 will be read-only. */ 1509 ret = mprotect(buffer->ptr + 3 * self->page_size, self->page_size, 1510 PROT_READ | PROT_WRITE); 1511 ASSERT_EQ(ret, 0); 1512 ptr = (int *)(buffer->ptr + 3 * self->page_size); 1513 *ptr = val; 1514 ret = mprotect(buffer->ptr + 3 * self->page_size, self->page_size, 1515 PROT_READ); 1516 ASSERT_EQ(ret, 0); 1517 1518 /* Page 4-6 will be read-write. */ 1519 ret = mprotect(buffer->ptr + 4 * self->page_size, 3 * self->page_size, 1520 PROT_READ | PROT_WRITE); 1521 ASSERT_EQ(ret, 0); 1522 ptr = (int *)(buffer->ptr + 4 * self->page_size); 1523 *ptr = val; 1524 1525 /* Page 5 will be migrated to device 0. */ 1526 buffer->ptr = p + 5 * self->page_size; 1527 ret = hmm_migrate_sys_to_dev(self->fd0, buffer, 1); 1528 ASSERT_EQ(ret, 0); 1529 ASSERT_EQ(buffer->cpages, 1); 1530 1531 /* Page 6 will be migrated to device 1. */ 1532 buffer->ptr = p + 6 * self->page_size; 1533 ret = hmm_migrate_sys_to_dev(self->fd1, buffer, 1); 1534 ASSERT_EQ(ret, 0); 1535 ASSERT_EQ(buffer->cpages, 1); 1536 1537 /* Simulate a device snapshotting CPU pagetables. */ 1538 buffer->ptr = p; 1539 ret = hmm_dmirror_cmd(self->fd0, HMM_DMIRROR_SNAPSHOT, buffer, npages); 1540 ASSERT_EQ(ret, 0); 1541 ASSERT_EQ(buffer->cpages, npages); 1542 1543 /* Check what the device saw. */ 1544 m = buffer->mirror; 1545 ASSERT_EQ(m[0], HMM_DMIRROR_PROT_ERROR); 1546 ASSERT_EQ(m[1], HMM_DMIRROR_PROT_ERROR); 1547 ASSERT_EQ(m[2], HMM_DMIRROR_PROT_ZERO | HMM_DMIRROR_PROT_READ); 1548 ASSERT_EQ(m[3], HMM_DMIRROR_PROT_READ); 1549 ASSERT_EQ(m[4], HMM_DMIRROR_PROT_WRITE); 1550 if (!hmm_is_coherent_type(variant->device_number0)) { 1551 ASSERT_EQ(m[5], HMM_DMIRROR_PROT_DEV_PRIVATE_LOCAL | 1552 HMM_DMIRROR_PROT_WRITE); 1553 ASSERT_EQ(m[6], HMM_DMIRROR_PROT_NONE); 1554 } else { 1555 ASSERT_EQ(m[5], HMM_DMIRROR_PROT_DEV_COHERENT_LOCAL | 1556 HMM_DMIRROR_PROT_WRITE); 1557 ASSERT_EQ(m[6], HMM_DMIRROR_PROT_DEV_COHERENT_REMOTE | 1558 HMM_DMIRROR_PROT_WRITE); 1559 } 1560 1561 hmm_buffer_free(buffer); 1562 } 1563 1564 /* 1565 * Test the hmm_range_fault() HMM_PFN_PMD flag for large pages that 1566 * should be mapped by a large page table entry. 1567 */ 1568 TEST_F(hmm, compound) 1569 { 1570 struct hmm_buffer *buffer; 1571 unsigned long npages; 1572 unsigned long size; 1573 unsigned long default_hsize; 1574 int *ptr; 1575 unsigned char *m; 1576 int ret; 1577 unsigned long i; 1578 1579 /* Skip test if we can't allocate a hugetlbfs page. */ 1580 1581 default_hsize = file_read_ulong("/proc/meminfo", "Hugepagesize:"); 1582 if (default_hsize < 0 || default_hsize*1024 < default_hsize) 1583 SKIP(return, "Huge page size could not be determined"); 1584 default_hsize = default_hsize*1024; /* KB to B */ 1585 1586 size = ALIGN(TWOMEG, default_hsize); 1587 npages = size >> self->page_shift; 1588 1589 buffer = malloc(sizeof(*buffer)); 1590 ASSERT_NE(buffer, NULL); 1591 1592 buffer->ptr = mmap(NULL, size, 1593 PROT_READ | PROT_WRITE, 1594 MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, 1595 -1, 0); 1596 if (buffer->ptr == MAP_FAILED) { 1597 free(buffer); 1598 return; 1599 } 1600 1601 buffer->size = size; 1602 buffer->mirror = malloc(npages); 1603 ASSERT_NE(buffer->mirror, NULL); 1604 1605 /* Initialize the pages the device will snapshot in buffer->ptr. */ 1606 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 1607 ptr[i] = i; 1608 1609 /* Simulate a device snapshotting CPU pagetables. */ 1610 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_SNAPSHOT, buffer, npages); 1611 ASSERT_EQ(ret, 0); 1612 ASSERT_EQ(buffer->cpages, npages); 1613 1614 /* Check what the device saw. */ 1615 m = buffer->mirror; 1616 for (i = 0; i < npages; ++i) 1617 ASSERT_EQ(m[i], HMM_DMIRROR_PROT_WRITE | 1618 HMM_DMIRROR_PROT_PMD); 1619 1620 /* Make the region read-only. */ 1621 ret = mprotect(buffer->ptr, size, PROT_READ); 1622 ASSERT_EQ(ret, 0); 1623 1624 /* Simulate a device snapshotting CPU pagetables. */ 1625 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_SNAPSHOT, buffer, npages); 1626 ASSERT_EQ(ret, 0); 1627 ASSERT_EQ(buffer->cpages, npages); 1628 1629 /* Check what the device saw. */ 1630 m = buffer->mirror; 1631 for (i = 0; i < npages; ++i) 1632 ASSERT_EQ(m[i], HMM_DMIRROR_PROT_READ | 1633 HMM_DMIRROR_PROT_PMD); 1634 1635 munmap(buffer->ptr, buffer->size); 1636 buffer->ptr = NULL; 1637 hmm_buffer_free(buffer); 1638 } 1639 1640 /* 1641 * Test two devices reading the same memory (double mapped). 1642 */ 1643 TEST_F(hmm2, double_map) 1644 { 1645 struct hmm_buffer *buffer; 1646 unsigned long npages; 1647 unsigned long size; 1648 unsigned long i; 1649 int *ptr; 1650 int ret; 1651 1652 npages = 6; 1653 size = npages << self->page_shift; 1654 1655 buffer = malloc(sizeof(*buffer)); 1656 ASSERT_NE(buffer, NULL); 1657 1658 buffer->fd = -1; 1659 buffer->size = size; 1660 buffer->mirror = malloc(npages); 1661 ASSERT_NE(buffer->mirror, NULL); 1662 1663 /* Reserve a range of addresses. */ 1664 buffer->ptr = mmap(NULL, size, 1665 PROT_READ | PROT_WRITE, 1666 MAP_PRIVATE | MAP_ANONYMOUS, 1667 buffer->fd, 0); 1668 ASSERT_NE(buffer->ptr, MAP_FAILED); 1669 1670 /* Initialize buffer in system memory. */ 1671 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 1672 ptr[i] = i; 1673 1674 /* Make region read-only. */ 1675 ret = mprotect(buffer->ptr, size, PROT_READ); 1676 ASSERT_EQ(ret, 0); 1677 1678 /* Simulate device 0 reading system memory. */ 1679 ret = hmm_dmirror_cmd(self->fd0, HMM_DMIRROR_READ, buffer, npages); 1680 ASSERT_EQ(ret, 0); 1681 ASSERT_EQ(buffer->cpages, npages); 1682 ASSERT_EQ(buffer->faults, 1); 1683 1684 /* Check what the device read. */ 1685 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 1686 ASSERT_EQ(ptr[i], i); 1687 1688 /* Simulate device 1 reading system memory. */ 1689 ret = hmm_dmirror_cmd(self->fd1, HMM_DMIRROR_READ, buffer, npages); 1690 ASSERT_EQ(ret, 0); 1691 ASSERT_EQ(buffer->cpages, npages); 1692 ASSERT_EQ(buffer->faults, 1); 1693 1694 /* Check what the device read. */ 1695 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 1696 ASSERT_EQ(ptr[i], i); 1697 1698 /* Migrate pages to device 1 and try to read from device 0. */ 1699 ret = hmm_migrate_sys_to_dev(self->fd1, buffer, npages); 1700 ASSERT_EQ(ret, 0); 1701 ASSERT_EQ(buffer->cpages, npages); 1702 1703 ret = hmm_dmirror_cmd(self->fd0, HMM_DMIRROR_READ, buffer, npages); 1704 ASSERT_EQ(ret, 0); 1705 ASSERT_EQ(buffer->cpages, npages); 1706 ASSERT_EQ(buffer->faults, 1); 1707 1708 /* Check what device 0 read. */ 1709 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 1710 ASSERT_EQ(ptr[i], i); 1711 1712 hmm_buffer_free(buffer); 1713 } 1714 1715 /* 1716 * Basic check of exclusive faulting. 1717 */ 1718 TEST_F(hmm, exclusive) 1719 { 1720 struct hmm_buffer *buffer; 1721 unsigned long npages; 1722 unsigned long size; 1723 unsigned long i; 1724 int *ptr; 1725 int ret; 1726 1727 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; 1728 ASSERT_NE(npages, 0); 1729 size = npages << self->page_shift; 1730 1731 buffer = malloc(sizeof(*buffer)); 1732 ASSERT_NE(buffer, NULL); 1733 1734 buffer->fd = -1; 1735 buffer->size = size; 1736 buffer->mirror = malloc(size); 1737 ASSERT_NE(buffer->mirror, NULL); 1738 1739 buffer->ptr = mmap(NULL, size, 1740 PROT_READ | PROT_WRITE, 1741 MAP_PRIVATE | MAP_ANONYMOUS, 1742 buffer->fd, 0); 1743 ASSERT_NE(buffer->ptr, MAP_FAILED); 1744 1745 /* Initialize buffer in system memory. */ 1746 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 1747 ptr[i] = i; 1748 1749 /* Map memory exclusively for device access. */ 1750 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_EXCLUSIVE, buffer, npages); 1751 ASSERT_EQ(ret, 0); 1752 ASSERT_EQ(buffer->cpages, npages); 1753 1754 /* Check what the device read. */ 1755 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 1756 ASSERT_EQ(ptr[i], i); 1757 1758 /* Fault pages back to system memory and check them. */ 1759 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 1760 ASSERT_EQ(ptr[i]++, i); 1761 1762 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 1763 ASSERT_EQ(ptr[i], i+1); 1764 1765 /* Check atomic access revoked */ 1766 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_CHECK_EXCLUSIVE, buffer, npages); 1767 ASSERT_EQ(ret, 0); 1768 1769 hmm_buffer_free(buffer); 1770 } 1771 1772 TEST_F(hmm, exclusive_mprotect) 1773 { 1774 struct hmm_buffer *buffer; 1775 unsigned long npages; 1776 unsigned long size; 1777 unsigned long i; 1778 int *ptr; 1779 int ret; 1780 1781 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; 1782 ASSERT_NE(npages, 0); 1783 size = npages << self->page_shift; 1784 1785 buffer = malloc(sizeof(*buffer)); 1786 ASSERT_NE(buffer, NULL); 1787 1788 buffer->fd = -1; 1789 buffer->size = size; 1790 buffer->mirror = malloc(size); 1791 ASSERT_NE(buffer->mirror, NULL); 1792 1793 buffer->ptr = mmap(NULL, size, 1794 PROT_READ | PROT_WRITE, 1795 MAP_PRIVATE | MAP_ANONYMOUS, 1796 buffer->fd, 0); 1797 ASSERT_NE(buffer->ptr, MAP_FAILED); 1798 1799 /* Initialize buffer in system memory. */ 1800 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 1801 ptr[i] = i; 1802 1803 /* Map memory exclusively for device access. */ 1804 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_EXCLUSIVE, buffer, npages); 1805 ASSERT_EQ(ret, 0); 1806 ASSERT_EQ(buffer->cpages, npages); 1807 1808 /* Check what the device read. */ 1809 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 1810 ASSERT_EQ(ptr[i], i); 1811 1812 ret = mprotect(buffer->ptr, size, PROT_READ); 1813 ASSERT_EQ(ret, 0); 1814 1815 /* Simulate a device writing system memory. */ 1816 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages); 1817 ASSERT_EQ(ret, -EPERM); 1818 1819 hmm_buffer_free(buffer); 1820 } 1821 1822 /* 1823 * Check copy-on-write works. 1824 */ 1825 TEST_F(hmm, exclusive_cow) 1826 { 1827 struct hmm_buffer *buffer; 1828 unsigned long npages; 1829 unsigned long size; 1830 unsigned long i; 1831 int *ptr; 1832 int ret; 1833 1834 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; 1835 ASSERT_NE(npages, 0); 1836 size = npages << self->page_shift; 1837 1838 buffer = malloc(sizeof(*buffer)); 1839 ASSERT_NE(buffer, NULL); 1840 1841 buffer->fd = -1; 1842 buffer->size = size; 1843 buffer->mirror = malloc(size); 1844 ASSERT_NE(buffer->mirror, NULL); 1845 1846 buffer->ptr = mmap(NULL, size, 1847 PROT_READ | PROT_WRITE, 1848 MAP_PRIVATE | MAP_ANONYMOUS, 1849 buffer->fd, 0); 1850 ASSERT_NE(buffer->ptr, MAP_FAILED); 1851 1852 /* Initialize buffer in system memory. */ 1853 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 1854 ptr[i] = i; 1855 1856 /* Map memory exclusively for device access. */ 1857 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_EXCLUSIVE, buffer, npages); 1858 ASSERT_EQ(ret, 0); 1859 ASSERT_EQ(buffer->cpages, npages); 1860 1861 fork(); 1862 1863 /* Fault pages back to system memory and check them. */ 1864 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 1865 ASSERT_EQ(ptr[i]++, i); 1866 1867 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 1868 ASSERT_EQ(ptr[i], i+1); 1869 1870 hmm_buffer_free(buffer); 1871 } 1872 1873 static int gup_test_exec(int gup_fd, unsigned long addr, int cmd, 1874 int npages, int size, int flags) 1875 { 1876 struct gup_test gup = { 1877 .nr_pages_per_call = npages, 1878 .addr = addr, 1879 .gup_flags = FOLL_WRITE | flags, 1880 .size = size, 1881 }; 1882 1883 if (ioctl(gup_fd, cmd, &gup)) { 1884 perror("ioctl on error\n"); 1885 return errno; 1886 } 1887 1888 return 0; 1889 } 1890 1891 /* 1892 * Test get user device pages through gup_test. Setting PIN_LONGTERM flag. 1893 * This should trigger a migration back to system memory for both, private 1894 * and coherent type pages. 1895 * This test makes use of gup_test module. Make sure GUP_TEST_CONFIG is added 1896 * to your configuration before you run it. 1897 */ 1898 TEST_F(hmm, hmm_gup_test) 1899 { 1900 struct hmm_buffer *buffer; 1901 int gup_fd; 1902 unsigned long npages; 1903 unsigned long size; 1904 unsigned long i; 1905 int *ptr; 1906 int ret; 1907 unsigned char *m; 1908 1909 gup_fd = open("/sys/kernel/debug/gup_test", O_RDWR); 1910 if (gup_fd == -1) 1911 SKIP(return, "Skipping test, could not find gup_test driver"); 1912 1913 npages = 4; 1914 size = npages << self->page_shift; 1915 1916 buffer = malloc(sizeof(*buffer)); 1917 ASSERT_NE(buffer, NULL); 1918 1919 buffer->fd = -1; 1920 buffer->size = size; 1921 buffer->mirror = malloc(size); 1922 ASSERT_NE(buffer->mirror, NULL); 1923 1924 buffer->ptr = mmap(NULL, size, 1925 PROT_READ | PROT_WRITE, 1926 MAP_PRIVATE | MAP_ANONYMOUS, 1927 buffer->fd, 0); 1928 ASSERT_NE(buffer->ptr, MAP_FAILED); 1929 1930 /* Initialize buffer in system memory. */ 1931 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 1932 ptr[i] = i; 1933 1934 /* Migrate memory to device. */ 1935 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); 1936 ASSERT_EQ(ret, 0); 1937 ASSERT_EQ(buffer->cpages, npages); 1938 /* Check what the device read. */ 1939 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) 1940 ASSERT_EQ(ptr[i], i); 1941 1942 ASSERT_EQ(gup_test_exec(gup_fd, 1943 (unsigned long)buffer->ptr, 1944 GUP_BASIC_TEST, 1, self->page_size, 0), 0); 1945 ASSERT_EQ(gup_test_exec(gup_fd, 1946 (unsigned long)buffer->ptr + 1 * self->page_size, 1947 GUP_FAST_BENCHMARK, 1, self->page_size, 0), 0); 1948 ASSERT_EQ(gup_test_exec(gup_fd, 1949 (unsigned long)buffer->ptr + 2 * self->page_size, 1950 PIN_FAST_BENCHMARK, 1, self->page_size, FOLL_LONGTERM), 0); 1951 ASSERT_EQ(gup_test_exec(gup_fd, 1952 (unsigned long)buffer->ptr + 3 * self->page_size, 1953 PIN_LONGTERM_BENCHMARK, 1, self->page_size, 0), 0); 1954 1955 /* Take snapshot to CPU pagetables */ 1956 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_SNAPSHOT, buffer, npages); 1957 ASSERT_EQ(ret, 0); 1958 ASSERT_EQ(buffer->cpages, npages); 1959 m = buffer->mirror; 1960 if (hmm_is_coherent_type(variant->device_number)) { 1961 ASSERT_EQ(HMM_DMIRROR_PROT_DEV_COHERENT_LOCAL | HMM_DMIRROR_PROT_WRITE, m[0]); 1962 ASSERT_EQ(HMM_DMIRROR_PROT_DEV_COHERENT_LOCAL | HMM_DMIRROR_PROT_WRITE, m[1]); 1963 } else { 1964 ASSERT_EQ(HMM_DMIRROR_PROT_WRITE, m[0]); 1965 ASSERT_EQ(HMM_DMIRROR_PROT_WRITE, m[1]); 1966 } 1967 ASSERT_EQ(HMM_DMIRROR_PROT_WRITE, m[2]); 1968 ASSERT_EQ(HMM_DMIRROR_PROT_WRITE, m[3]); 1969 /* 1970 * Check again the content on the pages. Make sure there's no 1971 * corrupted data. 1972 */ 1973 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 1974 ASSERT_EQ(ptr[i], i); 1975 1976 close(gup_fd); 1977 hmm_buffer_free(buffer); 1978 } 1979 1980 /* 1981 * Test copy-on-write in device pages. 1982 * In case of writing to COW private page(s), a page fault will migrate pages 1983 * back to system memory first. Then, these pages will be duplicated. In case 1984 * of COW device coherent type, pages are duplicated directly from device 1985 * memory. 1986 */ 1987 TEST_F(hmm, hmm_cow_in_device) 1988 { 1989 struct hmm_buffer *buffer; 1990 unsigned long npages; 1991 unsigned long size; 1992 unsigned long i; 1993 int *ptr; 1994 int ret; 1995 unsigned char *m; 1996 pid_t pid; 1997 int status; 1998 1999 npages = 4; 2000 size = npages << self->page_shift; 2001 2002 buffer = malloc(sizeof(*buffer)); 2003 ASSERT_NE(buffer, NULL); 2004 2005 buffer->fd = -1; 2006 buffer->size = size; 2007 buffer->mirror = malloc(size); 2008 ASSERT_NE(buffer->mirror, NULL); 2009 2010 buffer->ptr = mmap(NULL, size, 2011 PROT_READ | PROT_WRITE, 2012 MAP_PRIVATE | MAP_ANONYMOUS, 2013 buffer->fd, 0); 2014 ASSERT_NE(buffer->ptr, MAP_FAILED); 2015 2016 /* Initialize buffer in system memory. */ 2017 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 2018 ptr[i] = i; 2019 2020 /* Migrate memory to device. */ 2021 2022 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); 2023 ASSERT_EQ(ret, 0); 2024 ASSERT_EQ(buffer->cpages, npages); 2025 2026 pid = fork(); 2027 if (pid == -1) 2028 ASSERT_EQ(pid, 0); 2029 if (!pid) { 2030 /* Child process waitd for SIGTERM from the parent. */ 2031 while (1) { 2032 } 2033 perror("Should not reach this\n"); 2034 exit(0); 2035 } 2036 /* Parent process writes to COW pages(s) and gets a 2037 * new copy in system. In case of device private pages, 2038 * this write causes a migration to system mem first. 2039 */ 2040 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) 2041 ptr[i] = i; 2042 2043 /* Terminate child and wait */ 2044 EXPECT_EQ(0, kill(pid, SIGTERM)); 2045 EXPECT_EQ(pid, waitpid(pid, &status, 0)); 2046 EXPECT_NE(0, WIFSIGNALED(status)); 2047 EXPECT_EQ(SIGTERM, WTERMSIG(status)); 2048 2049 /* Take snapshot to CPU pagetables */ 2050 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_SNAPSHOT, buffer, npages); 2051 ASSERT_EQ(ret, 0); 2052 ASSERT_EQ(buffer->cpages, npages); 2053 m = buffer->mirror; 2054 for (i = 0; i < npages; i++) 2055 ASSERT_EQ(HMM_DMIRROR_PROT_WRITE, m[i]); 2056 2057 hmm_buffer_free(buffer); 2058 } 2059 TEST_HARNESS_MAIN 2060