Lines Matching +full:data +full:- +full:mirror
1 // SPDX-License-Identifier: GPL-2.0
4 * the linux kernel to help device drivers mirror a process address space in
6 * makes communication and data exchange a lot easier.
39 void *mirror; member
58 #define ALIGN(x, a) (((x) + (a - 1)) & (~((a) - 1)))
136 self->page_size = sysconf(_SC_PAGE_SIZE); in FIXTURE_SETUP()
137 self->page_shift = ffs(self->page_size) - 1; in FIXTURE_SETUP()
139 self->fd = hmm_open(variant->device_number); in FIXTURE_SETUP()
140 if (self->fd < 0 && hmm_is_coherent_type(variant->device_number)) in FIXTURE_SETUP()
142 ASSERT_GE(self->fd, 0); in FIXTURE_SETUP()
147 self->page_size = sysconf(_SC_PAGE_SIZE); in FIXTURE_SETUP()
148 self->page_shift = ffs(self->page_size) - 1; in FIXTURE_SETUP()
150 self->fd0 = hmm_open(variant->device_number0); in FIXTURE_SETUP()
151 if (self->fd0 < 0 && hmm_is_coherent_type(variant->device_number0)) in FIXTURE_SETUP()
153 ASSERT_GE(self->fd0, 0); in FIXTURE_SETUP()
154 self->fd1 = hmm_open(variant->device_number1); in FIXTURE_SETUP()
155 ASSERT_GE(self->fd1, 0); in FIXTURE_SETUP()
160 int ret = close(self->fd); in FIXTURE_TEARDOWN()
163 self->fd = -1; in FIXTURE_TEARDOWN()
168 int ret = close(self->fd0); in FIXTURE_TEARDOWN()
171 self->fd0 = -1; in FIXTURE_TEARDOWN()
173 ret = close(self->fd1); in FIXTURE_TEARDOWN()
175 self->fd1 = -1; in FIXTURE_TEARDOWN()
187 cmd.addr = (__u64)buffer->ptr; in hmm_dmirror_cmd()
188 cmd.ptr = (__u64)buffer->mirror; in hmm_dmirror_cmd()
197 return -errno; in hmm_dmirror_cmd()
199 buffer->cpages = cmd.cpages; in hmm_dmirror_cmd()
200 buffer->faults = cmd.faults; in hmm_dmirror_cmd()
210 if (buffer->ptr) in hmm_buffer_free()
211 munmap(buffer->ptr, buffer->size); in hmm_buffer_free()
212 free(buffer->mirror); in hmm_buffer_free()
231 } while (r == -1 && errno == EINTR); in hmm_create_file()
236 return -1; in hmm_create_file()
244 static int fd = -1; in hmm_random()
302 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; in TEST_F()
304 size = npages << self->page_shift; in TEST_F()
309 buffer->fd = -1; in TEST_F()
310 buffer->size = size; in TEST_F()
311 buffer->mirror = malloc(size); in TEST_F()
312 ASSERT_NE(buffer->mirror, NULL); in TEST_F()
314 buffer->ptr = mmap(NULL, size, in TEST_F()
317 buffer->fd, 0); in TEST_F()
318 ASSERT_NE(buffer->ptr, MAP_FAILED); in TEST_F()
324 i = 2 * self->page_size / sizeof(*ptr); in TEST_F()
325 for (ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) in TEST_F()
328 /* Set buffer permission to read-only. */ in TEST_F()
329 ret = mprotect(buffer->ptr, size, PROT_READ); in TEST_F()
333 val = *(int *)(buffer->ptr + self->page_size); in TEST_F()
337 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ, buffer, npages); in TEST_F()
339 ASSERT_EQ(buffer->cpages, npages); in TEST_F()
340 ASSERT_EQ(buffer->faults, 1); in TEST_F()
343 ptr = buffer->mirror; in TEST_F()
344 for (i = 0; i < 2 * self->page_size / sizeof(*ptr); ++i) in TEST_F()
365 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; in TEST_F()
367 size = npages << self->page_shift; in TEST_F()
372 buffer->fd = -1; in TEST_F()
373 buffer->size = size; in TEST_F()
374 buffer->mirror = malloc(size); in TEST_F()
375 ASSERT_NE(buffer->mirror, NULL); in TEST_F()
377 buffer->ptr = mmap(NULL, size, in TEST_F()
380 buffer->fd, 0); in TEST_F()
381 ASSERT_NE(buffer->ptr, MAP_FAILED); in TEST_F()
384 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) in TEST_F()
387 /* Initialize mirror buffer so we can verify it isn't written. */ in TEST_F()
388 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) in TEST_F()
389 ptr[i] = -i; in TEST_F()
392 ret = mprotect(buffer->ptr, size, PROT_NONE); in TEST_F()
396 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ, buffer, npages); in TEST_F()
397 ASSERT_EQ(ret, -EFAULT); in TEST_F()
400 ret = mprotect(buffer->ptr, size, PROT_READ); in TEST_F()
402 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) in TEST_F()
406 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) in TEST_F()
407 ASSERT_EQ(ptr[i], -i); in TEST_F()
424 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; in TEST_F()
426 size = npages << self->page_shift; in TEST_F()
431 buffer->fd = -1; in TEST_F()
432 buffer->size = size; in TEST_F()
433 buffer->mirror = malloc(size); in TEST_F()
434 ASSERT_NE(buffer->mirror, NULL); in TEST_F()
436 buffer->ptr = mmap(NULL, size, in TEST_F()
439 buffer->fd, 0); in TEST_F()
440 ASSERT_NE(buffer->ptr, MAP_FAILED); in TEST_F()
442 /* Initialize data that the device will write to buffer->ptr. */ in TEST_F()
443 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) in TEST_F()
447 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages); in TEST_F()
449 ASSERT_EQ(buffer->cpages, npages); in TEST_F()
450 ASSERT_EQ(buffer->faults, 1); in TEST_F()
453 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) in TEST_F()
472 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; in TEST_F()
474 size = npages << self->page_shift; in TEST_F()
479 buffer->fd = -1; in TEST_F()
480 buffer->size = size; in TEST_F()
481 buffer->mirror = malloc(size); in TEST_F()
482 ASSERT_NE(buffer->mirror, NULL); in TEST_F()
484 buffer->ptr = mmap(NULL, size, in TEST_F()
487 buffer->fd, 0); in TEST_F()
488 ASSERT_NE(buffer->ptr, MAP_FAILED); in TEST_F()
491 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ, buffer, 1); in TEST_F()
493 ASSERT_EQ(buffer->cpages, 1); in TEST_F()
494 ASSERT_EQ(buffer->faults, 1); in TEST_F()
496 /* Initialize data that the device will write to buffer->ptr. */ in TEST_F()
497 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) in TEST_F()
501 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages); in TEST_F()
502 ASSERT_EQ(ret, -EPERM); in TEST_F()
505 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) in TEST_F()
509 ret = mprotect(buffer->ptr, size, PROT_WRITE | PROT_READ); in TEST_F()
513 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages); in TEST_F()
515 ASSERT_EQ(buffer->cpages, npages); in TEST_F()
516 ASSERT_EQ(buffer->faults, 1); in TEST_F()
519 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) in TEST_F()
527 * will copy-on-write if a child process inherits the mapping.
540 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; in TEST_F()
542 size = npages << self->page_shift; in TEST_F()
547 buffer->fd = -1; in TEST_F()
548 buffer->size = size; in TEST_F()
549 buffer->mirror = malloc(size); in TEST_F()
550 ASSERT_NE(buffer->mirror, NULL); in TEST_F()
552 buffer->ptr = mmap(NULL, size, in TEST_F()
555 buffer->fd, 0); in TEST_F()
556 ASSERT_NE(buffer->ptr, MAP_FAILED); in TEST_F()
558 /* Initialize buffer->ptr so we can tell if it is written. */ in TEST_F()
559 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) in TEST_F()
562 /* Initialize data that the device will write to buffer->ptr. */ in TEST_F()
563 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) in TEST_F()
564 ptr[i] = -i; in TEST_F()
567 if (pid == -1) in TEST_F()
574 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) in TEST_F()
580 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) in TEST_F()
582 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) in TEST_F()
583 ASSERT_EQ(ptr[i], -i); in TEST_F()
585 /* The child process needs its own mirror to its own mm. */ in TEST_F()
592 ASSERT_EQ(buffer->cpages, npages); in TEST_F()
593 ASSERT_EQ(buffer->faults, 1); in TEST_F()
596 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) in TEST_F()
597 ASSERT_EQ(ptr[i], -i); in TEST_F()
605 * will not copy-on-write if a child process inherits the mapping.
618 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; in TEST_F()
620 size = npages << self->page_shift; in TEST_F()
625 buffer->fd = -1; in TEST_F()
626 buffer->size = size; in TEST_F()
627 buffer->mirror = malloc(size); in TEST_F()
628 ASSERT_NE(buffer->mirror, NULL); in TEST_F()
630 buffer->ptr = mmap(NULL, size, in TEST_F()
633 buffer->fd, 0); in TEST_F()
634 ASSERT_NE(buffer->ptr, MAP_FAILED); in TEST_F()
636 /* Initialize buffer->ptr so we can tell if it is written. */ in TEST_F()
637 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) in TEST_F()
640 /* Initialize data that the device will write to buffer->ptr. */ in TEST_F()
641 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) in TEST_F()
642 ptr[i] = -i; in TEST_F()
645 if (pid == -1) in TEST_F()
652 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) in TEST_F()
653 ASSERT_EQ(ptr[i], -i); in TEST_F()
658 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) in TEST_F()
660 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) in TEST_F()
661 ASSERT_EQ(ptr[i], -i); in TEST_F()
663 /* The child process needs its own mirror to its own mm. */ in TEST_F()
670 ASSERT_EQ(buffer->cpages, npages); in TEST_F()
671 ASSERT_EQ(buffer->faults, 1); in TEST_F()
674 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) in TEST_F()
675 ASSERT_EQ(ptr[i], -i); in TEST_F()
700 buffer->fd = -1; in TEST_F()
701 buffer->size = size; in TEST_F()
702 buffer->mirror = malloc(size); in TEST_F()
703 ASSERT_NE(buffer->mirror, NULL); in TEST_F()
705 buffer->ptr = mmap(NULL, size, in TEST_F()
708 buffer->fd, 0); in TEST_F()
709 ASSERT_NE(buffer->ptr, MAP_FAILED); in TEST_F()
712 npages = size >> self->page_shift; in TEST_F()
713 map = (void *)ALIGN((uintptr_t)buffer->ptr, size); in TEST_F()
716 old_ptr = buffer->ptr; in TEST_F()
717 buffer->ptr = map; in TEST_F()
719 /* Initialize data that the device will write to buffer->ptr. */ in TEST_F()
720 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) in TEST_F()
724 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages); in TEST_F()
726 ASSERT_EQ(buffer->cpages, npages); in TEST_F()
727 ASSERT_EQ(buffer->faults, 1); in TEST_F()
730 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) in TEST_F()
733 buffer->ptr = old_ptr; in TEST_F()
738 * Read numeric data from raw and tagged kernel status files. Used to read
739 * /proc and /sys data (without a tag) and from /proc/meminfo (with a tag).
752 return -1; in file_read_ulong()
759 return -1; in file_read_ulong()
763 return -1; in file_read_ulong()
771 return -1; /* looks like the line we want isn't there */ in file_read_ulong()
779 return -1; in file_read_ulong()
804 npages = size >> self->page_shift; in TEST_F()
809 buffer->ptr = mmap(NULL, size, in TEST_F()
812 -1, 0); in TEST_F()
813 if (buffer->ptr == MAP_FAILED) { in TEST_F()
818 buffer->fd = -1; in TEST_F()
819 buffer->size = size; in TEST_F()
820 buffer->mirror = malloc(size); in TEST_F()
821 ASSERT_NE(buffer->mirror, NULL); in TEST_F()
823 /* Initialize data that the device will write to buffer->ptr. */ in TEST_F()
824 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) in TEST_F()
828 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages); in TEST_F()
830 ASSERT_EQ(buffer->cpages, npages); in TEST_F()
831 ASSERT_EQ(buffer->faults, 1); in TEST_F()
834 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) in TEST_F()
837 munmap(buffer->ptr, buffer->size); in TEST_F()
838 buffer->ptr = NULL; in TEST_F()
856 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; in TEST_F()
858 size = npages << self->page_shift; in TEST_F()
866 buffer->fd = fd; in TEST_F()
867 buffer->size = size; in TEST_F()
868 buffer->mirror = malloc(size); in TEST_F()
869 ASSERT_NE(buffer->mirror, NULL); in TEST_F()
872 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) in TEST_F()
874 len = pwrite(fd, buffer->mirror, size, 0); in TEST_F()
876 memset(buffer->mirror, 0, size); in TEST_F()
878 buffer->ptr = mmap(NULL, size, in TEST_F()
881 buffer->fd, 0); in TEST_F()
882 ASSERT_NE(buffer->ptr, MAP_FAILED); in TEST_F()
885 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ, buffer, npages); in TEST_F()
887 ASSERT_EQ(buffer->cpages, npages); in TEST_F()
888 ASSERT_EQ(buffer->faults, 1); in TEST_F()
891 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) in TEST_F()
911 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; in TEST_F()
913 size = npages << self->page_shift; in TEST_F()
921 buffer->fd = fd; in TEST_F()
922 buffer->size = size; in TEST_F()
923 buffer->mirror = malloc(size); in TEST_F()
924 ASSERT_NE(buffer->mirror, NULL); in TEST_F()
926 buffer->ptr = mmap(NULL, size, in TEST_F()
929 buffer->fd, 0); in TEST_F()
930 ASSERT_NE(buffer->ptr, MAP_FAILED); in TEST_F()
932 /* Initialize data that the device will write to buffer->ptr. */ in TEST_F()
933 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) in TEST_F()
937 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages); in TEST_F()
939 ASSERT_EQ(buffer->cpages, npages); in TEST_F()
940 ASSERT_EQ(buffer->faults, 1); in TEST_F()
943 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) in TEST_F()
947 len = pread(fd, buffer->mirror, size, 0); in TEST_F()
949 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) in TEST_F()
967 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; in TEST_F()
969 size = npages << self->page_shift; in TEST_F()
974 buffer->fd = -1; in TEST_F()
975 buffer->size = size; in TEST_F()
976 buffer->mirror = malloc(size); in TEST_F()
977 ASSERT_NE(buffer->mirror, NULL); in TEST_F()
979 buffer->ptr = mmap(NULL, size, in TEST_F()
982 buffer->fd, 0); in TEST_F()
983 ASSERT_NE(buffer->ptr, MAP_FAILED); in TEST_F()
986 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) in TEST_F()
990 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); in TEST_F()
992 ASSERT_EQ(buffer->cpages, npages); in TEST_F()
995 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) in TEST_F()
1015 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; in TEST_F()
1017 size = npages << self->page_shift; in TEST_F()
1022 buffer->fd = -1; in TEST_F()
1023 buffer->size = size; in TEST_F()
1024 buffer->mirror = malloc(size); in TEST_F()
1025 ASSERT_NE(buffer->mirror, NULL); in TEST_F()
1027 buffer->ptr = mmap(NULL, size, in TEST_F()
1030 buffer->fd, 0); in TEST_F()
1031 ASSERT_NE(buffer->ptr, MAP_FAILED); in TEST_F()
1034 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) in TEST_F()
1038 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); in TEST_F()
1040 ASSERT_EQ(buffer->cpages, npages); in TEST_F()
1043 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) in TEST_F()
1047 for (i = 0, ptr = buffer->ptr; i < size / (2 * sizeof(*ptr)); ++i) in TEST_F()
1051 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); in TEST_F()
1053 ASSERT_EQ(buffer->cpages, npages); in TEST_F()
1056 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) in TEST_F()
1071 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; in TEST_F()
1073 size = npages << self->page_shift; in TEST_F()
1078 buffer->fd = -1; in TEST_F()
1079 buffer->size = size; in TEST_F()
1080 buffer->mirror = malloc(size); in TEST_F()
1081 ASSERT_NE(buffer->mirror, NULL); in TEST_F()
1083 buffer->ptr = mmap(NULL, size, PROT_READ | PROT_WRITE, in TEST_F()
1084 MAP_PRIVATE | MAP_ANONYMOUS, buffer->fd, 0); in TEST_F()
1085 ASSERT_NE(buffer->ptr, MAP_FAILED); in TEST_F()
1088 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) in TEST_F()
1092 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); in TEST_F()
1094 ASSERT_EQ(buffer->cpages, npages); in TEST_F()
1097 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) in TEST_F()
1101 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_RELEASE, buffer, npages); in TEST_F()
1105 for (i = 0, ptr = buffer->ptr; i < size / (2 * sizeof(*ptr)); ++i) in TEST_F()
1121 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; in TEST_F()
1123 size = npages << self->page_shift; in TEST_F()
1128 buffer->fd = -1; in TEST_F()
1129 buffer->size = size; in TEST_F()
1130 buffer->mirror = malloc(size); in TEST_F()
1131 ASSERT_NE(buffer->mirror, NULL); in TEST_F()
1133 buffer->ptr = mmap(NULL, size, in TEST_F()
1136 buffer->fd, 0); in TEST_F()
1137 ASSERT_NE(buffer->ptr, MAP_FAILED); in TEST_F()
1140 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); in TEST_F()
1141 ASSERT_EQ(ret, -ENOENT); in TEST_F()
1160 size = npages << self->page_shift; in TEST_F()
1165 buffer->fd = -1; in TEST_F()
1166 buffer->size = size; in TEST_F()
1167 buffer->mirror = malloc(size); in TEST_F()
1168 ASSERT_NE(buffer->mirror, NULL); in TEST_F()
1171 buffer->ptr = mmap(NULL, size, in TEST_F()
1174 buffer->fd, 0); in TEST_F()
1175 ASSERT_NE(buffer->ptr, MAP_FAILED); in TEST_F()
1176 p = buffer->ptr; in TEST_F()
1179 ret = hmm_migrate_sys_to_dev(self->fd1, buffer, npages); in TEST_F()
1180 ASSERT_EQ(ret, -EINVAL); in TEST_F()
1183 ret = munmap(buffer->ptr + self->page_size, self->page_size); in TEST_F()
1187 ret = hmm_migrate_sys_to_dev(self->fd1, buffer, 3); in TEST_F()
1188 ASSERT_EQ(ret, -EINVAL); in TEST_F()
1190 /* Page 2 will be a read-only zero page. */ in TEST_F()
1191 ret = mprotect(buffer->ptr + 2 * self->page_size, self->page_size, in TEST_F()
1194 ptr = (int *)(buffer->ptr + 2 * self->page_size); in TEST_F()
1198 /* Page 3 will be read-only. */ in TEST_F()
1199 ret = mprotect(buffer->ptr + 3 * self->page_size, self->page_size, in TEST_F()
1202 ptr = (int *)(buffer->ptr + 3 * self->page_size); in TEST_F()
1204 ret = mprotect(buffer->ptr + 3 * self->page_size, self->page_size, in TEST_F()
1208 /* Page 4-5 will be read-write. */ in TEST_F()
1209 ret = mprotect(buffer->ptr + 4 * self->page_size, 2 * self->page_size, in TEST_F()
1212 ptr = (int *)(buffer->ptr + 4 * self->page_size); in TEST_F()
1214 ptr = (int *)(buffer->ptr + 5 * self->page_size); in TEST_F()
1217 /* Now try to migrate pages 2-5 to device 1. */ in TEST_F()
1218 buffer->ptr = p + 2 * self->page_size; in TEST_F()
1219 ret = hmm_migrate_sys_to_dev(self->fd1, buffer, 4); in TEST_F()
1221 ASSERT_EQ(buffer->cpages, 4); in TEST_F()
1224 buffer->ptr = p + 5 * self->page_size; in TEST_F()
1225 ret = hmm_migrate_sys_to_dev(self->fd0, buffer, 1); in TEST_F()
1226 ASSERT_EQ(ret, -ENOENT); in TEST_F()
1227 buffer->ptr = p; in TEST_F()
1229 buffer->ptr = p; in TEST_F()
1251 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; in TEST_F()
1253 size = npages << self->page_shift; in TEST_F()
1259 buffer->fd = -1; in TEST_F()
1260 buffer->size = size; in TEST_F()
1261 buffer->mirror = malloc(size); in TEST_F()
1262 ASSERT_NE(buffer->mirror, NULL); in TEST_F()
1264 buffer->ptr = mmap(NULL, size, in TEST_F()
1267 buffer->fd, 0); in TEST_F()
1268 ASSERT_NE(buffer->ptr, MAP_FAILED); in TEST_F()
1271 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) in TEST_F()
1275 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); in TEST_F()
1277 ASSERT_EQ(buffer->cpages, npages); in TEST_F()
1280 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) in TEST_F()
1284 if (hmm_is_coherent_type(variant->device_number)) { in TEST_F()
1285 ret = hmm_migrate_dev_to_sys(self->fd, buffer, npages); in TEST_F()
1287 ASSERT_EQ(buffer->cpages, npages); in TEST_F()
1290 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) in TEST_F()
1310 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; in TEST_F()
1312 size = npages << self->page_shift; in TEST_F()
1318 buffer->fd = -1; in TEST_F()
1319 buffer->size = size; in TEST_F()
1320 buffer->mirror = malloc(size); in TEST_F()
1321 ASSERT_NE(buffer->mirror, NULL); in TEST_F()
1323 buffer->ptr = mmap(NULL, size, in TEST_F()
1326 buffer->fd, 0); in TEST_F()
1327 ASSERT_NE(buffer->ptr, MAP_FAILED); in TEST_F()
1330 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) in TEST_F()
1334 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ, buffer, in TEST_F()
1337 ASSERT_EQ(buffer->cpages, npages); in TEST_F()
1338 ASSERT_EQ(buffer->faults, 1); in TEST_F()
1341 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) in TEST_F()
1354 munmap(buffer->ptr + buffer->size / 2, buffer->size / 2); in unmap_buffer()
1355 buffer->ptr = NULL; in unmap_buffer()
1370 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; in TEST_F()
1372 size = npages << self->page_shift; in TEST_F()
1384 buffer->fd = -1; in TEST_F()
1385 buffer->size = size; in TEST_F()
1386 buffer->mirror = malloc(size); in TEST_F()
1387 ASSERT_NE(buffer->mirror, NULL); in TEST_F()
1389 buffer->ptr = mmap(NULL, size, in TEST_F()
1392 buffer->fd, 0); in TEST_F()
1393 ASSERT_NE(buffer->ptr, MAP_FAILED); in TEST_F()
1396 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) in TEST_F()
1403 rc = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ, buffer, in TEST_F()
1406 ASSERT_EQ(buffer->cpages, npages); in TEST_F()
1407 ASSERT_EQ(buffer->faults, 1); in TEST_F()
1410 for (i = 0, ptr = buffer->mirror; in TEST_F()
1433 size = npages << self->page_shift; in TEST_F()
1438 buffer->fd = -1; in TEST_F()
1439 buffer->size = size; in TEST_F()
1440 buffer->mirror = malloc(npages); in TEST_F()
1441 ASSERT_NE(buffer->mirror, NULL); in TEST_F()
1445 buffer->ptr = mmap(NULL, size, in TEST_F()
1448 self->fd, 0); in TEST_F()
1449 ASSERT_NE(buffer->ptr, MAP_FAILED); in TEST_F()
1452 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_SNAPSHOT, buffer, npages); in TEST_F()
1454 ASSERT_EQ(buffer->cpages, npages); in TEST_F()
1457 m = buffer->mirror; in TEST_F()
1478 size = npages << self->page_shift; in TEST_F()
1483 buffer->fd = -1; in TEST_F()
1484 buffer->size = size; in TEST_F()
1485 buffer->mirror = malloc(npages); in TEST_F()
1486 ASSERT_NE(buffer->mirror, NULL); in TEST_F()
1489 buffer->ptr = mmap(NULL, size, in TEST_F()
1492 buffer->fd, 0); in TEST_F()
1493 ASSERT_NE(buffer->ptr, MAP_FAILED); in TEST_F()
1494 p = buffer->ptr; in TEST_F()
1497 ret = munmap(buffer->ptr + self->page_size, self->page_size); in TEST_F()
1500 /* Page 2 will be read-only zero page. */ in TEST_F()
1501 ret = mprotect(buffer->ptr + 2 * self->page_size, self->page_size, in TEST_F()
1504 ptr = (int *)(buffer->ptr + 2 * self->page_size); in TEST_F()
1508 /* Page 3 will be read-only. */ in TEST_F()
1509 ret = mprotect(buffer->ptr + 3 * self->page_size, self->page_size, in TEST_F()
1512 ptr = (int *)(buffer->ptr + 3 * self->page_size); in TEST_F()
1514 ret = mprotect(buffer->ptr + 3 * self->page_size, self->page_size, in TEST_F()
1518 /* Page 4-6 will be read-write. */ in TEST_F()
1519 ret = mprotect(buffer->ptr + 4 * self->page_size, 3 * self->page_size, in TEST_F()
1522 ptr = (int *)(buffer->ptr + 4 * self->page_size); in TEST_F()
1526 buffer->ptr = p + 5 * self->page_size; in TEST_F()
1527 ret = hmm_migrate_sys_to_dev(self->fd0, buffer, 1); in TEST_F()
1529 ASSERT_EQ(buffer->cpages, 1); in TEST_F()
1532 buffer->ptr = p + 6 * self->page_size; in TEST_F()
1533 ret = hmm_migrate_sys_to_dev(self->fd1, buffer, 1); in TEST_F()
1535 ASSERT_EQ(buffer->cpages, 1); in TEST_F()
1538 buffer->ptr = p; in TEST_F()
1539 ret = hmm_dmirror_cmd(self->fd0, HMM_DMIRROR_SNAPSHOT, buffer, npages); in TEST_F()
1541 ASSERT_EQ(buffer->cpages, npages); in TEST_F()
1544 m = buffer->mirror; in TEST_F()
1550 if (!hmm_is_coherent_type(variant->device_number0)) { in TEST_F()
1587 npages = size >> self->page_shift; in TEST_F()
1592 buffer->ptr = mmap(NULL, size, in TEST_F()
1595 -1, 0); in TEST_F()
1596 if (buffer->ptr == MAP_FAILED) { in TEST_F()
1601 buffer->size = size; in TEST_F()
1602 buffer->mirror = malloc(npages); in TEST_F()
1603 ASSERT_NE(buffer->mirror, NULL); in TEST_F()
1605 /* Initialize the pages the device will snapshot in buffer->ptr. */ in TEST_F()
1606 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) in TEST_F()
1610 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_SNAPSHOT, buffer, npages); in TEST_F()
1612 ASSERT_EQ(buffer->cpages, npages); in TEST_F()
1615 m = buffer->mirror; in TEST_F()
1620 /* Make the region read-only. */ in TEST_F()
1621 ret = mprotect(buffer->ptr, size, PROT_READ); in TEST_F()
1625 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_SNAPSHOT, buffer, npages); in TEST_F()
1627 ASSERT_EQ(buffer->cpages, npages); in TEST_F()
1630 m = buffer->mirror; in TEST_F()
1635 munmap(buffer->ptr, buffer->size); in TEST_F()
1636 buffer->ptr = NULL; in TEST_F()
1653 size = npages << self->page_shift; in TEST_F()
1658 buffer->fd = -1; in TEST_F()
1659 buffer->size = size; in TEST_F()
1660 buffer->mirror = malloc(size); in TEST_F()
1661 ASSERT_NE(buffer->mirror, NULL); in TEST_F()
1664 buffer->ptr = mmap(NULL, size, in TEST_F()
1667 buffer->fd, 0); in TEST_F()
1668 ASSERT_NE(buffer->ptr, MAP_FAILED); in TEST_F()
1671 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) in TEST_F()
1674 /* Make region read-only. */ in TEST_F()
1675 ret = mprotect(buffer->ptr, size, PROT_READ); in TEST_F()
1679 ret = hmm_dmirror_cmd(self->fd0, HMM_DMIRROR_READ, buffer, npages); in TEST_F()
1681 ASSERT_EQ(buffer->cpages, npages); in TEST_F()
1682 ASSERT_EQ(buffer->faults, 1); in TEST_F()
1685 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) in TEST_F()
1689 ret = hmm_dmirror_cmd(self->fd1, HMM_DMIRROR_READ, buffer, npages); in TEST_F()
1691 ASSERT_EQ(buffer->cpages, npages); in TEST_F()
1692 ASSERT_EQ(buffer->faults, 1); in TEST_F()
1695 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) in TEST_F()
1699 ret = hmm_migrate_sys_to_dev(self->fd1, buffer, npages); in TEST_F()
1701 ASSERT_EQ(buffer->cpages, npages); in TEST_F()
1703 ret = hmm_dmirror_cmd(self->fd0, HMM_DMIRROR_READ, buffer, npages); in TEST_F()
1705 ASSERT_EQ(buffer->cpages, npages); in TEST_F()
1706 ASSERT_EQ(buffer->faults, 1); in TEST_F()
1709 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) in TEST_F()
1727 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; in TEST_F()
1729 size = npages << self->page_shift; in TEST_F()
1734 buffer->fd = -1; in TEST_F()
1735 buffer->size = size; in TEST_F()
1736 buffer->mirror = malloc(size); in TEST_F()
1737 ASSERT_NE(buffer->mirror, NULL); in TEST_F()
1739 buffer->ptr = mmap(NULL, size, in TEST_F()
1742 buffer->fd, 0); in TEST_F()
1743 ASSERT_NE(buffer->ptr, MAP_FAILED); in TEST_F()
1746 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) in TEST_F()
1750 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_EXCLUSIVE, buffer, npages); in TEST_F()
1752 ASSERT_EQ(buffer->cpages, npages); in TEST_F()
1755 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) in TEST_F()
1759 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) in TEST_F()
1762 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) in TEST_F()
1766 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_CHECK_EXCLUSIVE, buffer, npages); in TEST_F()
1781 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; in TEST_F()
1783 size = npages << self->page_shift; in TEST_F()
1788 buffer->fd = -1; in TEST_F()
1789 buffer->size = size; in TEST_F()
1790 buffer->mirror = malloc(size); in TEST_F()
1791 ASSERT_NE(buffer->mirror, NULL); in TEST_F()
1793 buffer->ptr = mmap(NULL, size, in TEST_F()
1796 buffer->fd, 0); in TEST_F()
1797 ASSERT_NE(buffer->ptr, MAP_FAILED); in TEST_F()
1800 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) in TEST_F()
1804 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_EXCLUSIVE, buffer, npages); in TEST_F()
1806 ASSERT_EQ(buffer->cpages, npages); in TEST_F()
1809 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) in TEST_F()
1812 ret = mprotect(buffer->ptr, size, PROT_READ); in TEST_F()
1816 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages); in TEST_F()
1817 ASSERT_EQ(ret, -EPERM); in TEST_F()
1823 * Check copy-on-write works.
1834 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; in TEST_F()
1836 size = npages << self->page_shift; in TEST_F()
1841 buffer->fd = -1; in TEST_F()
1842 buffer->size = size; in TEST_F()
1843 buffer->mirror = malloc(size); in TEST_F()
1844 ASSERT_NE(buffer->mirror, NULL); in TEST_F()
1846 buffer->ptr = mmap(NULL, size, in TEST_F()
1849 buffer->fd, 0); in TEST_F()
1850 ASSERT_NE(buffer->ptr, MAP_FAILED); in TEST_F()
1853 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) in TEST_F()
1857 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_EXCLUSIVE, buffer, npages); in TEST_F()
1859 ASSERT_EQ(buffer->cpages, npages); in TEST_F()
1864 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) in TEST_F()
1867 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) in TEST_F()
1910 if (gup_fd == -1) in TEST_F()
1914 size = npages << self->page_shift; in TEST_F()
1919 buffer->fd = -1; in TEST_F()
1920 buffer->size = size; in TEST_F()
1921 buffer->mirror = malloc(size); in TEST_F()
1922 ASSERT_NE(buffer->mirror, NULL); in TEST_F()
1924 buffer->ptr = mmap(NULL, size, in TEST_F()
1927 buffer->fd, 0); in TEST_F()
1928 ASSERT_NE(buffer->ptr, MAP_FAILED); in TEST_F()
1931 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) in TEST_F()
1935 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); in TEST_F()
1937 ASSERT_EQ(buffer->cpages, npages); in TEST_F()
1939 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) in TEST_F()
1943 (unsigned long)buffer->ptr, in TEST_F()
1944 GUP_BASIC_TEST, 1, self->page_size, 0), 0); in TEST_F()
1946 (unsigned long)buffer->ptr + 1 * self->page_size, in TEST_F()
1947 GUP_FAST_BENCHMARK, 1, self->page_size, 0), 0); in TEST_F()
1949 (unsigned long)buffer->ptr + 2 * self->page_size, in TEST_F()
1950 PIN_FAST_BENCHMARK, 1, self->page_size, FOLL_LONGTERM), 0); in TEST_F()
1952 (unsigned long)buffer->ptr + 3 * self->page_size, in TEST_F()
1953 PIN_LONGTERM_BENCHMARK, 1, self->page_size, 0), 0); in TEST_F()
1956 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_SNAPSHOT, buffer, npages); in TEST_F()
1958 ASSERT_EQ(buffer->cpages, npages); in TEST_F()
1959 m = buffer->mirror; in TEST_F()
1960 if (hmm_is_coherent_type(variant->device_number)) { in TEST_F()
1971 * corrupted data. in TEST_F()
1973 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) in TEST_F()
1981 * Test copy-on-write in device pages.
2000 size = npages << self->page_shift; in TEST_F()
2005 buffer->fd = -1; in TEST_F()
2006 buffer->size = size; in TEST_F()
2007 buffer->mirror = malloc(size); in TEST_F()
2008 ASSERT_NE(buffer->mirror, NULL); in TEST_F()
2010 buffer->ptr = mmap(NULL, size, in TEST_F()
2013 buffer->fd, 0); in TEST_F()
2014 ASSERT_NE(buffer->ptr, MAP_FAILED); in TEST_F()
2017 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) in TEST_F()
2022 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); in TEST_F()
2024 ASSERT_EQ(buffer->cpages, npages); in TEST_F()
2027 if (pid == -1) in TEST_F()
2040 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) in TEST_F()
2050 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_SNAPSHOT, buffer, npages); in TEST_F()
2052 ASSERT_EQ(buffer->cpages, npages); in TEST_F()
2053 m = buffer->mirror; in TEST_F()