Lines Matching refs:p
45 void (*cleanup_area)(void *p, unsigned long size);
46 void (*fault)(void *p, unsigned long start, unsigned long end);
56 void (*collapse)(const char *msg, char *p, int nr_hpages,
556 void *p; in alloc_mapping() local
558 p = mmap(BASE_ADDR, nr * hpage_pmd_size, PROT_READ | PROT_WRITE, in alloc_mapping()
560 if (p != BASE_ADDR) { in alloc_mapping()
565 return p; in alloc_mapping()
568 static void fill_memory(int *p, unsigned long start, unsigned long end) in fill_memory() argument
573 p[i * page_size / sizeof(*p)] = i + 0xdead0000; in fill_memory()
582 static int madvise_collapse_retry(void *p, unsigned long size) in madvise_collapse_retry() argument
588 ret = madvise(p, size, MADV_COLLAPSE); in madvise_collapse_retry()
602 void *p = ops->setup_area(1); in alloc_hpage() local
604 ops->fault(p, 0, hpage_pmd_size); in alloc_hpage()
614 if (madvise_collapse_retry(p, hpage_pmd_size)) { in alloc_hpage()
618 if (!ops->check_huge(p, 1)) { in alloc_hpage()
622 if (madvise(p, hpage_pmd_size, MADV_HUGEPAGE)) { in alloc_hpage()
627 return p; in alloc_hpage()
630 static void validate_memory(int *p, unsigned long start, unsigned long end) in validate_memory() argument
635 if (p[i * page_size / sizeof(*p)] != i + 0xdead0000) { in validate_memory()
637 i, p[i * page_size / sizeof(*p)]); in validate_memory()
648 static void anon_cleanup_area(void *p, unsigned long size) in anon_cleanup_area() argument
650 munmap(p, size); in anon_cleanup_area()
653 static void anon_fault(void *p, unsigned long start, unsigned long end) in anon_fault() argument
655 fill_memory(p, start, end); in anon_fault()
666 void *p; in file_setup_area() local
680 p = alloc_mapping(nr_hpages); in file_setup_area()
681 fill_memory(p, 0, size); in file_setup_area()
682 write(fd, p, size); in file_setup_area()
684 munmap(p, size); in file_setup_area()
693 p = mmap(BASE_ADDR, size, PROT_READ | PROT_EXEC, in file_setup_area()
695 if (p == MAP_FAILED || p != BASE_ADDR) { in file_setup_area()
703 return p; in file_setup_area()
706 static void file_cleanup_area(void *p, unsigned long size) in file_cleanup_area() argument
708 munmap(p, size); in file_cleanup_area()
713 static void file_fault(void *p, unsigned long start, unsigned long end) in file_fault() argument
715 if (madvise(((char *)p) + start, end - start, MADV_POPULATE_READ)) { in file_fault()
736 void *p; in shmem_setup_area() local
748 p = mmap(BASE_ADDR, size, PROT_READ | PROT_WRITE, MAP_SHARED, finfo.fd, in shmem_setup_area()
750 if (p != BASE_ADDR) { in shmem_setup_area()
754 return p; in shmem_setup_area()
757 static void shmem_cleanup_area(void *p, unsigned long size) in shmem_cleanup_area() argument
759 munmap(p, size); in shmem_cleanup_area()
792 static void __madvise_collapse(const char *msg, char *p, int nr_hpages, in __madvise_collapse() argument
809 madvise(p, nr_hpages * hpage_pmd_size, MADV_HUGEPAGE); in __madvise_collapse()
810 ret = madvise_collapse_retry(p, nr_hpages * hpage_pmd_size); in __madvise_collapse()
813 else if (!ops->check_huge(p, expect ? nr_hpages : 0)) in __madvise_collapse()
821 static void madvise_collapse(const char *msg, char *p, int nr_hpages, in madvise_collapse() argument
825 if (!ops->check_huge(p, 0)) { in madvise_collapse()
829 __madvise_collapse(msg, p, nr_hpages, ops, expect); in madvise_collapse()
833 static bool wait_for_scan(const char *msg, char *p, int nr_hpages, in wait_for_scan() argument
840 if (!ops->check_huge(p, 0)) { in wait_for_scan()
845 madvise(p, nr_hpages * hpage_pmd_size, MADV_HUGEPAGE); in wait_for_scan()
852 if (ops->check_huge(p, nr_hpages)) in wait_for_scan()
860 madvise(p, nr_hpages * hpage_pmd_size, MADV_NOHUGEPAGE); in wait_for_scan()
865 static void khugepaged_collapse(const char *msg, char *p, int nr_hpages, in khugepaged_collapse() argument
868 if (wait_for_scan(msg, p, nr_hpages, ops)) { in khugepaged_collapse()
882 ops->fault(p, 0, nr_hpages * hpage_pmd_size); in khugepaged_collapse()
884 if (ops->check_huge(p, expect ? nr_hpages : 0)) in khugepaged_collapse()
910 char *p; in alloc_at_fault() local
915 p = alloc_mapping(1); in alloc_at_fault()
916 *p = 1; in alloc_at_fault()
918 if (check_huge_anon(p, 1, hpage_pmd_size)) in alloc_at_fault()
925 madvise(p, page_size, MADV_DONTNEED); in alloc_at_fault()
927 if (check_huge_anon(p, 0, hpage_pmd_size)) in alloc_at_fault()
931 munmap(p, hpage_pmd_size); in alloc_at_fault()
936 void *p; in collapse_full() local
940 p = ops->setup_area(nr_hpages); in collapse_full()
941 ops->fault(p, 0, size); in collapse_full()
942 c->collapse("Collapse multiple fully populated PTE table", p, nr_hpages, in collapse_full()
944 validate_memory(p, 0, size); in collapse_full()
945 ops->cleanup_area(p, size); in collapse_full()
950 void *p; in collapse_empty() local
952 p = ops->setup_area(1); in collapse_empty()
953 c->collapse("Do not collapse empty PTE table", p, 1, ops, false); in collapse_empty()
954 ops->cleanup_area(p, hpage_pmd_size); in collapse_empty()
959 void *p; in collapse_single_pte_entry() local
961 p = ops->setup_area(1); in collapse_single_pte_entry()
962 ops->fault(p, 0, page_size); in collapse_single_pte_entry()
963 c->collapse("Collapse PTE table with single PTE entry present", p, in collapse_single_pte_entry()
965 ops->cleanup_area(p, hpage_pmd_size); in collapse_single_pte_entry()
972 void *p; in collapse_max_ptes_none() local
977 p = ops->setup_area(1); in collapse_max_ptes_none()
986 ops->fault(p, 0, (hpage_pmd_nr - max_ptes_none - 1) * page_size); in collapse_max_ptes_none()
987 c->collapse("Maybe collapse with max_ptes_none exceeded", p, 1, in collapse_max_ptes_none()
989 validate_memory(p, 0, (hpage_pmd_nr - max_ptes_none - 1) * page_size); in collapse_max_ptes_none()
992 ops->fault(p, 0, (hpage_pmd_nr - max_ptes_none) * page_size); in collapse_max_ptes_none()
993 c->collapse("Collapse with max_ptes_none PTEs empty", p, 1, ops, in collapse_max_ptes_none()
995 validate_memory(p, 0, in collapse_max_ptes_none()
999 ops->cleanup_area(p, hpage_pmd_size); in collapse_max_ptes_none()
1005 void *p; in collapse_swapin_single_pte() local
1007 p = ops->setup_area(1); in collapse_swapin_single_pte()
1008 ops->fault(p, 0, hpage_pmd_size); in collapse_swapin_single_pte()
1011 if (madvise(p, page_size, MADV_PAGEOUT)) { in collapse_swapin_single_pte()
1015 if (check_swap(p, page_size)) { in collapse_swapin_single_pte()
1022 c->collapse("Collapse with swapping in single PTE entry", p, 1, ops, in collapse_swapin_single_pte()
1024 validate_memory(p, 0, hpage_pmd_size); in collapse_swapin_single_pte()
1026 ops->cleanup_area(p, hpage_pmd_size); in collapse_swapin_single_pte()
1032 void *p; in collapse_max_ptes_swap() local
1034 p = ops->setup_area(1); in collapse_max_ptes_swap()
1035 ops->fault(p, 0, hpage_pmd_size); in collapse_max_ptes_swap()
1038 if (madvise(p, (max_ptes_swap + 1) * page_size, MADV_PAGEOUT)) { in collapse_max_ptes_swap()
1042 if (check_swap(p, (max_ptes_swap + 1) * page_size)) { in collapse_max_ptes_swap()
1049 c->collapse("Maybe collapse with max_ptes_swap exceeded", p, 1, ops, in collapse_max_ptes_swap()
1051 validate_memory(p, 0, hpage_pmd_size); in collapse_max_ptes_swap()
1054 ops->fault(p, 0, hpage_pmd_size); in collapse_max_ptes_swap()
1057 if (madvise(p, max_ptes_swap * page_size, MADV_PAGEOUT)) { in collapse_max_ptes_swap()
1061 if (check_swap(p, max_ptes_swap * page_size)) { in collapse_max_ptes_swap()
1068 c->collapse("Collapse with max_ptes_swap pages swapped out", p, in collapse_max_ptes_swap()
1070 validate_memory(p, 0, hpage_pmd_size); in collapse_max_ptes_swap()
1073 ops->cleanup_area(p, hpage_pmd_size); in collapse_max_ptes_swap()
1078 void *p; in collapse_single_pte_entry_compound() local
1080 p = alloc_hpage(ops); in collapse_single_pte_entry_compound()
1089 madvise(p, hpage_pmd_size, MADV_NOHUGEPAGE); in collapse_single_pte_entry_compound()
1091 madvise(p + page_size, hpage_pmd_size - page_size, MADV_DONTNEED); in collapse_single_pte_entry_compound()
1092 if (ops->check_huge(p, 0)) in collapse_single_pte_entry_compound()
1098 p, 1, ops, true); in collapse_single_pte_entry_compound()
1099 validate_memory(p, 0, page_size); in collapse_single_pte_entry_compound()
1101 ops->cleanup_area(p, hpage_pmd_size); in collapse_single_pte_entry_compound()
1106 void *p; in collapse_full_of_compound() local
1108 p = alloc_hpage(ops); in collapse_full_of_compound()
1110 madvise(p, page_size, MADV_NOHUGEPAGE); in collapse_full_of_compound()
1111 madvise(p, hpage_pmd_size, MADV_NOHUGEPAGE); in collapse_full_of_compound()
1112 if (ops->check_huge(p, 0)) in collapse_full_of_compound()
1117 c->collapse("Collapse PTE table full of compound pages", p, 1, ops, in collapse_full_of_compound()
1119 validate_memory(p, 0, hpage_pmd_size); in collapse_full_of_compound()
1120 ops->cleanup_area(p, hpage_pmd_size); in collapse_full_of_compound()
1125 void *p; in collapse_compound_extreme() local
1128 p = ops->setup_area(1); in collapse_compound_extreme()
1141 p = mremap(BASE_ADDR - i * page_size, in collapse_compound_extreme()
1146 if (p == MAP_FAILED) { in collapse_compound_extreme()
1151 p = mremap(BASE_ADDR + 2 * hpage_pmd_size, in collapse_compound_extreme()
1156 if (p == MAP_FAILED) { in collapse_compound_extreme()
1163 ops->fault(p, 0, hpage_pmd_size); in collapse_compound_extreme()
1164 if (!ops->check_huge(p, 1)) in collapse_compound_extreme()
1169 c->collapse("Collapse PTE table full of different compound pages", p, 1, in collapse_compound_extreme()
1172 validate_memory(p, 0, hpage_pmd_size); in collapse_compound_extreme()
1173 ops->cleanup_area(p, hpage_pmd_size); in collapse_compound_extreme()
1179 void *p; in collapse_fork() local
1181 p = ops->setup_area(1); in collapse_fork()
1184 ops->fault(p, 0, page_size); in collapse_fork()
1185 if (ops->check_huge(p, 0)) in collapse_fork()
1196 if (ops->check_huge(p, 0)) in collapse_fork()
1201 ops->fault(p, page_size, 2 * page_size); in collapse_fork()
1203 p, 1, ops, true); in collapse_fork()
1205 validate_memory(p, 0, page_size); in collapse_fork()
1206 ops->cleanup_area(p, hpage_pmd_size); in collapse_fork()
1214 if (ops->check_huge(p, 0)) in collapse_fork()
1218 validate_memory(p, 0, page_size); in collapse_fork()
1219 ops->cleanup_area(p, hpage_pmd_size); in collapse_fork()
1225 void *p; in collapse_fork_compound() local
1227 p = alloc_hpage(ops); in collapse_fork_compound()
1234 if (ops->check_huge(p, 1)) in collapse_fork_compound()
1240 madvise(p, page_size, MADV_NOHUGEPAGE); in collapse_fork_compound()
1241 madvise(p, hpage_pmd_size, MADV_NOHUGEPAGE); in collapse_fork_compound()
1242 if (ops->check_huge(p, 0)) in collapse_fork_compound()
1246 ops->fault(p, 0, page_size); in collapse_fork_compound()
1250 p, 1, ops, true); in collapse_fork_compound()
1254 validate_memory(p, 0, hpage_pmd_size); in collapse_fork_compound()
1255 ops->cleanup_area(p, hpage_pmd_size); in collapse_fork_compound()
1263 if (ops->check_huge(p, 1)) in collapse_fork_compound()
1267 validate_memory(p, 0, hpage_pmd_size); in collapse_fork_compound()
1268 ops->cleanup_area(p, hpage_pmd_size); in collapse_fork_compound()
1275 void *p; in collapse_max_ptes_shared() local
1277 p = alloc_hpage(ops); in collapse_max_ptes_shared()
1284 if (ops->check_huge(p, 1)) in collapse_max_ptes_shared()
1291 ops->fault(p, 0, (hpage_pmd_nr - max_ptes_shared - 1) * page_size); in collapse_max_ptes_shared()
1292 if (ops->check_huge(p, 0)) in collapse_max_ptes_shared()
1297 c->collapse("Maybe collapse with max_ptes_shared exceeded", p, in collapse_max_ptes_shared()
1303 ops->fault(p, 0, (hpage_pmd_nr - max_ptes_shared) * in collapse_max_ptes_shared()
1305 if (ops->check_huge(p, 0)) in collapse_max_ptes_shared()
1311 p, 1, ops, true); in collapse_max_ptes_shared()
1314 validate_memory(p, 0, hpage_pmd_size); in collapse_max_ptes_shared()
1315 ops->cleanup_area(p, hpage_pmd_size); in collapse_max_ptes_shared()
1323 if (ops->check_huge(p, 1)) in collapse_max_ptes_shared()
1327 validate_memory(p, 0, hpage_pmd_size); in collapse_max_ptes_shared()
1328 ops->cleanup_area(p, hpage_pmd_size); in collapse_max_ptes_shared()
1334 void *p; in madvise_collapse_existing_thps() local
1336 p = ops->setup_area(1); in madvise_collapse_existing_thps()
1337 ops->fault(p, 0, hpage_pmd_size); in madvise_collapse_existing_thps()
1338 c->collapse("Collapse fully populated PTE table...", p, 1, ops, true); in madvise_collapse_existing_thps()
1339 validate_memory(p, 0, hpage_pmd_size); in madvise_collapse_existing_thps()
1342 __madvise_collapse("Re-collapse PMD-mapped hugepage", p, 1, ops, true); in madvise_collapse_existing_thps()
1343 validate_memory(p, 0, hpage_pmd_size); in madvise_collapse_existing_thps()
1344 ops->cleanup_area(p, hpage_pmd_size); in madvise_collapse_existing_thps()
1354 void *p; in madvise_retracted_page_tables() local
1358 p = ops->setup_area(nr_hpages); in madvise_retracted_page_tables()
1359 ops->fault(p, 0, size); in madvise_retracted_page_tables()
1362 if (wait_for_scan("Collapse and leave PMD cleared", p, nr_hpages, in madvise_retracted_page_tables()
1368 c->collapse("Install huge PMD from page cache", p, nr_hpages, ops, in madvise_retracted_page_tables()
1370 validate_memory(p, 0, size); in madvise_retracted_page_tables()
1371 ops->cleanup_area(p, size); in madvise_retracted_page_tables()