100 static int child_memcmp_fn(char *mem, size_t size,  in child_memcmp_fn()  argument
107 	memcpy(old, mem, size);  in child_memcmp_fn()
115 	return memcmp(old, mem, size);  in child_memcmp_fn()
118 static int child_vmsplice_memcmp_fn(char *mem, size_t size,  in child_vmsplice_memcmp_fn()  argument
122 		.iov_base = mem,  in child_vmsplice_memcmp_fn()
134 	memcpy(old, mem, size);  in child_vmsplice_memcmp_fn()
147 	if (munmap(mem, size) < 0)  in child_vmsplice_memcmp_fn()
165 typedef int (*child_fn)(char *mem, size_t size, struct comm_pipes *comm_pipes);
167 static void do_test_cow_in_parent(char *mem, size_t size, bool do_mprotect,  in do_test_cow_in_parent()  argument
185 		exit(fn(mem, size, &comm_pipes));  in do_test_cow_in_parent()
196 		ret = mprotect(mem, size, PROT_READ);  in do_test_cow_in_parent()
197 		ret |= mprotect(mem, size, PROT_READ|PROT_WRITE);  in do_test_cow_in_parent()
207 	memset(mem, 0xff, size);  in do_test_cow_in_parent()
221 static void test_cow_in_parent(char *mem, size_t size)  in test_cow_in_parent()  argument
223 	do_test_cow_in_parent(mem, size, false, child_memcmp_fn);  in test_cow_in_parent()
226 static void test_cow_in_parent_mprotect(char *mem, size_t size)  in test_cow_in_parent_mprotect()  argument
228 	do_test_cow_in_parent(mem, size, true, child_memcmp_fn);  in test_cow_in_parent_mprotect()
231 static void test_vmsplice_in_child(char *mem, size_t size)  in test_vmsplice_in_child()  argument
233 	do_test_cow_in_parent(mem, size, false, child_vmsplice_memcmp_fn);  in test_vmsplice_in_child()
236 static void test_vmsplice_in_child_mprotect(char *mem, size_t size)  in test_vmsplice_in_child_mprotect()  argument
238 	do_test_cow_in_parent(mem, size, true, child_vmsplice_memcmp_fn);  in test_vmsplice_in_child_mprotect()
241 static void do_test_vmsplice_in_parent(char *mem, size_t size,  in do_test_vmsplice_in_parent()  argument
245 		.iov_base = mem,  in do_test_vmsplice_in_parent()
257 	memcpy(old, mem, size);  in do_test_vmsplice_in_parent()
287 		memset(mem, 0xff, size);  in do_test_vmsplice_in_parent()
302 	if (munmap(mem, size) < 0) {  in do_test_vmsplice_in_parent()
336 static void test_vmsplice_before_fork(char *mem, size_t size)  in test_vmsplice_before_fork()  argument
338 	do_test_vmsplice_in_parent(mem, size, true);  in test_vmsplice_before_fork()
341 static void test_vmsplice_after_fork(char *mem, size_t size)  in test_vmsplice_after_fork()  argument
343 	do_test_vmsplice_in_parent(mem, size, false);  in test_vmsplice_after_fork()
347 static void do_test_iouring(char *mem, size_t size, bool use_fork)  in do_test_iouring()  argument
393 	iov.iov_base = mem;  in do_test_iouring()
427 		ret = mprotect(mem, size, PROT_READ);  in do_test_iouring()
429 		ret |= mprotect(mem, size, PROT_READ | PROT_WRITE);  in do_test_iouring()
440 	memset(mem, 0xff, size);  in do_test_iouring()
446 	io_uring_prep_write_fixed(sqe, fd, mem, size, 0, 0);  in do_test_iouring()
478 	ksft_test_result(!memcmp(mem, tmp, size),  in do_test_iouring()
498 static void test_iouring_ro(char *mem, size_t size)  in test_iouring_ro()  argument
500 	do_test_iouring(mem, size, false);  in test_iouring_ro()
503 static void test_iouring_fork(char *mem, size_t size)  in test_iouring_fork()  argument
505 	do_test_iouring(mem, size, true);  in test_iouring_fork()
517 static void do_test_ro_pin(char *mem, size_t size, enum ro_pin_test test,  in do_test_ro_pin()  argument
587 		ret = mprotect(mem, size, PROT_READ);  in do_test_ro_pin()
589 		ret |= mprotect(mem, size, PROT_READ | PROT_WRITE);  in do_test_ro_pin()
600 	args.addr = (__u64)(uintptr_t)mem;  in do_test_ro_pin()
613 	memset(mem, 0xff, size);  in do_test_ro_pin()
624 		ksft_test_result(!memcmp(mem, tmp, size),  in do_test_ro_pin()
647 static void test_ro_pin_on_shared(char *mem, size_t size)  in test_ro_pin_on_shared()  argument
649 	do_test_ro_pin(mem, size, RO_PIN_TEST_SHARED, false);  in test_ro_pin_on_shared()
652 static void test_ro_fast_pin_on_shared(char *mem, size_t size)  in test_ro_fast_pin_on_shared()  argument
654 	do_test_ro_pin(mem, size, RO_PIN_TEST_SHARED, true);  in test_ro_fast_pin_on_shared()
657 static void test_ro_pin_on_ro_previously_shared(char *mem, size_t size)  in test_ro_pin_on_ro_previously_shared()  argument
659 	do_test_ro_pin(mem, size, RO_PIN_TEST_PREVIOUSLY_SHARED, false);  in test_ro_pin_on_ro_previously_shared()
662 static void test_ro_fast_pin_on_ro_previously_shared(char *mem, size_t size)  in test_ro_fast_pin_on_ro_previously_shared()  argument
664 	do_test_ro_pin(mem, size, RO_PIN_TEST_PREVIOUSLY_SHARED, true);  in test_ro_fast_pin_on_ro_previously_shared()
667 static void test_ro_pin_on_ro_exclusive(char *mem, size_t size)  in test_ro_pin_on_ro_exclusive()  argument
669 	do_test_ro_pin(mem, size, RO_PIN_TEST_RO_EXCLUSIVE, false);  in test_ro_pin_on_ro_exclusive()
672 static void test_ro_fast_pin_on_ro_exclusive(char *mem, size_t size)  in test_ro_fast_pin_on_ro_exclusive()  argument
674 	do_test_ro_pin(mem, size, RO_PIN_TEST_RO_EXCLUSIVE, true);  in test_ro_fast_pin_on_ro_exclusive()
677 typedef void (*test_fn)(char *mem, size_t size);
681 	char *mem;  in do_run_with_base_page()  local
684 	mem = mmap(NULL, pagesize, PROT_READ | PROT_WRITE,  in do_run_with_base_page()
686 	if (mem == MAP_FAILED) {  in do_run_with_base_page()
691 	ret = madvise(mem, pagesize, MADV_NOHUGEPAGE);  in do_run_with_base_page()
699 	memset(mem, 0, pagesize);  in do_run_with_base_page()
702 		madvise(mem, pagesize, MADV_PAGEOUT);  in do_run_with_base_page()
703 		if (!pagemap_is_swapped(pagemap_fd, mem)) {  in do_run_with_base_page()
709 	fn(mem, pagesize);  in do_run_with_base_page()
711 	munmap(mem, pagesize);  in do_run_with_base_page()
739 	char *mem, *mmap_mem, *tmp, *mremap_mem = MAP_FAILED;  in do_run_with_thp()  local
753 	mem = (char *)(((uintptr_t)mmap_mem + thpsize) & ~(thpsize - 1));  in do_run_with_thp()
755 	ret = madvise(mem, thpsize, MADV_HUGEPAGE);  in do_run_with_thp()
765 	mem[0] = 0;  in do_run_with_thp()
766 	if (!pagemap_is_populated(pagemap_fd, mem + pagesize)) {  in do_run_with_thp()
770 	memset(mem, 0, thpsize);  in do_run_with_thp()
783 		ret = mprotect(mem + pagesize, pagesize, PROT_READ);  in do_run_with_thp()
788 		ret = mprotect(mem + pagesize, pagesize, PROT_READ | PROT_WRITE);  in do_run_with_thp()
800 		ret = madvise(mem + pagesize, thpsize - pagesize, MADV_DONTNEED);  in do_run_with_thp()
815 		if (mem == MAP_FAILED) {  in do_run_with_thp()
819 		tmp = mremap(mem + mremap_size, mremap_size, mremap_size,  in do_run_with_thp()
833 		ret = madvise(mem + pagesize, thpsize - pagesize, MADV_DONTFORK);  in do_run_with_thp()
847 		ret = madvise(mem + pagesize, thpsize - pagesize, MADV_DOFORK);  in do_run_with_thp()
861 		madvise(mem, size, MADV_PAGEOUT);  in do_run_with_thp()
862 		if (!range_is_swapped(mem, size)) {  in do_run_with_thp()
871 	fn(mem, size);  in do_run_with_thp()
929 	char *mem, *dummy;  in run_with_hugetlb()  local
936 	mem = mmap(NULL, hugetlbsize, PROT_READ | PROT_WRITE, flags, -1, 0);  in run_with_hugetlb()
937 	if (mem == MAP_FAILED) {  in run_with_hugetlb()
943 	memset(mem, 0, hugetlbsize);  in run_with_hugetlb()
956 	fn(mem, hugetlbsize);  in run_with_hugetlb()
958 	munmap(mem, hugetlbsize);  in run_with_hugetlb()
1135 static void do_test_anon_thp_collapse(char *mem, size_t size,  in do_test_anon_thp_collapse()  argument
1152 	ret = mprotect(mem + pagesize, pagesize, PROT_READ);  in do_test_anon_thp_collapse()
1157 	ret = mprotect(mem + pagesize, pagesize, PROT_READ | PROT_WRITE);  in do_test_anon_thp_collapse()
1166 		ret = madvise(mem, size, MADV_COLLAPSE);  in do_test_anon_thp_collapse()
1178 		ret = madvise(mem + size / 2, size / 2, MADV_DONTFORK);  in do_test_anon_thp_collapse()
1186 		ret = madvise(mem, size / 2, MADV_DONTFORK);  in do_test_anon_thp_collapse()
1204 			exit(child_memcmp_fn(mem, size, &comm_pipes));  in do_test_anon_thp_collapse()
1207 			exit(child_memcmp_fn(mem, size / 2, &comm_pipes));  in do_test_anon_thp_collapse()
1210 			exit(child_memcmp_fn(mem + size / 2, size / 2,  in do_test_anon_thp_collapse()
1230 		ret = madvise(mem, size, MADV_DOFORK);  in do_test_anon_thp_collapse()
1240 		ret = madvise(mem, size, MADV_COLLAPSE);  in do_test_anon_thp_collapse()
1254 	memset(mem, 0xff, size);  in do_test_anon_thp_collapse()
1268 static void test_anon_thp_collapse_unshared(char *mem, size_t size)  in test_anon_thp_collapse_unshared()  argument
1270 	do_test_anon_thp_collapse(mem, size, ANON_THP_COLLAPSE_UNSHARED);  in test_anon_thp_collapse_unshared()
1273 static void test_anon_thp_collapse_fully_shared(char *mem, size_t size)  in test_anon_thp_collapse_fully_shared()  argument
1275 	do_test_anon_thp_collapse(mem, size, ANON_THP_COLLAPSE_FULLY_SHARED);  in test_anon_thp_collapse_fully_shared()
1278 static void test_anon_thp_collapse_lower_shared(char *mem, size_t size)  in test_anon_thp_collapse_lower_shared()  argument
1280 	do_test_anon_thp_collapse(mem, size, ANON_THP_COLLAPSE_LOWER_SHARED);  in test_anon_thp_collapse_lower_shared()
1283 static void test_anon_thp_collapse_upper_shared(char *mem, size_t size)  in test_anon_thp_collapse_upper_shared()  argument
1285 	do_test_anon_thp_collapse(mem, size, ANON_THP_COLLAPSE_UPPER_SHARED);  in test_anon_thp_collapse_upper_shared()
1350 typedef void (*non_anon_test_fn)(char *mem, const char *smem, size_t size);
1352 static void test_cow(char *mem, const char *smem, size_t size)  in test_cow()  argument
1360 	memset(mem, 0xff, size);  in test_cow()
1368 static void test_ro_pin(char *mem, const char *smem, size_t size)  in test_ro_pin()  argument
1370 	do_test_ro_pin(mem, size, RO_PIN_TEST, false);  in test_ro_pin()
1373 static void test_ro_fast_pin(char *mem, const char *smem, size_t size)  in test_ro_fast_pin()  argument
1375 	do_test_ro_pin(mem, size, RO_PIN_TEST, true);  in test_ro_fast_pin()
1380 	char *mem, *smem, tmp;  in run_with_zeropage()  local
1384 	mem = mmap(NULL, pagesize, PROT_READ | PROT_WRITE,  in run_with_zeropage()
1386 	if (mem == MAP_FAILED) {  in run_with_zeropage()
1392 	if (mem == MAP_FAILED) {  in run_with_zeropage()
1398 	tmp = *mem + *smem;  in run_with_zeropage()
1401 	fn(mem, smem, pagesize);  in run_with_zeropage()
1403 	munmap(mem, pagesize);  in run_with_zeropage()
1410 	char *mem, *smem, *mmap_mem, *mmap_smem, tmp;  in run_with_huge_zeropage()  local
1437 	mem = (char *)(((uintptr_t)mmap_mem + thpsize) & ~(thpsize - 1));  in run_with_huge_zeropage()
1440 	ret = madvise(mem, thpsize, MADV_HUGEPAGE);  in run_with_huge_zeropage()
1452 	tmp = *mem + *smem;  in run_with_huge_zeropage()
1454 	if (!pagemap_is_populated(pagemap_fd, mem + pagesize) ||  in run_with_huge_zeropage()
1460 	fn(mem, smem, thpsize);  in run_with_huge_zeropage()
1469 	char *mem, *smem, tmp;  in run_with_memfd()  local
1487 	mem = mmap(NULL, pagesize, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0);  in run_with_memfd()
1488 	if (mem == MAP_FAILED) {  in run_with_memfd()
1493 	if (mem == MAP_FAILED) {  in run_with_memfd()
1499 	tmp = *mem + *smem;  in run_with_memfd()
1502 	fn(mem, smem, pagesize);  in run_with_memfd()
1504 	munmap(mem, pagesize);  in run_with_memfd()
1513 	char *mem, *smem, tmp;  in run_with_tmpfile()  local
1538 	mem = mmap(NULL, pagesize, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0);  in run_with_tmpfile()
1539 	if (mem == MAP_FAILED) {  in run_with_tmpfile()
1544 	if (mem == MAP_FAILED) {  in run_with_tmpfile()
1550 	tmp = *mem + *smem;  in run_with_tmpfile()
1553 	fn(mem, smem, pagesize);  in run_with_tmpfile()
1555 	munmap(mem, pagesize);  in run_with_tmpfile()
1566 	char *mem, *smem, tmp;  in run_with_memfd_hugetlb()  local
1587 	mem = mmap(NULL, hugetlbsize, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd,  in run_with_memfd_hugetlb()
1589 	if (mem == MAP_FAILED) {  in run_with_memfd_hugetlb()
1594 	if (mem == MAP_FAILED) {  in run_with_memfd_hugetlb()
1600 	tmp = *mem + *smem;  in run_with_memfd_hugetlb()
1603 	fn(mem, smem, hugetlbsize);  in run_with_memfd_hugetlb()
1605 	munmap(mem, hugetlbsize);  in run_with_memfd_hugetlb()
1606 	if (mem != MAP_FAILED)  in run_with_memfd_hugetlb()