29 static void __init kasan_populate_pte(pmd_t *pmd, unsigned long vaddr, unsigned long end)  in kasan_populate_pte()  argument
39 	ptep = pte_offset_kernel(pmd, vaddr);  in kasan_populate_pte()
47 	} while (ptep++, vaddr += PAGE_SIZE, vaddr != end);  in kasan_populate_pte()
50 static void __init kasan_populate_pmd(pud_t *pud, unsigned long vaddr, unsigned long end)  in kasan_populate_pmd()  argument
61 	pmdp = pmd_offset(pud, vaddr);  in kasan_populate_pmd()
64 		next = pmd_addr_end(vaddr, end);  in kasan_populate_pmd()
66 		if (pmd_none(pmdp_get(pmdp)) && IS_ALIGNED(vaddr, PMD_SIZE) &&  in kasan_populate_pmd()
67 		    (next - vaddr) >= PMD_SIZE) {  in kasan_populate_pmd()
76 		kasan_populate_pte(pmdp, vaddr, next);  in kasan_populate_pmd()
77 	} while (pmdp++, vaddr = next, vaddr != end);  in kasan_populate_pmd()
81 				      unsigned long vaddr, unsigned long end)  in kasan_populate_pud()  argument
92 	pudp = pud_offset(p4d, vaddr);  in kasan_populate_pud()
95 		next = pud_addr_end(vaddr, end);  in kasan_populate_pud()
97 		if (pud_none(pudp_get(pudp)) && IS_ALIGNED(vaddr, PUD_SIZE) &&  in kasan_populate_pud()
98 		    (next - vaddr) >= PUD_SIZE) {  in kasan_populate_pud()
107 		kasan_populate_pmd(pudp, vaddr, next);  in kasan_populate_pud()
108 	} while (pudp++, vaddr = next, vaddr != end);  in kasan_populate_pud()
112 				      unsigned long vaddr, unsigned long end)  in kasan_populate_p4d()  argument
123 	p4dp = p4d_offset(pgd, vaddr);  in kasan_populate_p4d()
126 		next = p4d_addr_end(vaddr, end);  in kasan_populate_p4d()
128 		if (p4d_none(p4dp_get(p4dp)) && IS_ALIGNED(vaddr, P4D_SIZE) &&  in kasan_populate_p4d()
129 		    (next - vaddr) >= P4D_SIZE) {  in kasan_populate_p4d()
138 		kasan_populate_pud(p4dp, vaddr, next);  in kasan_populate_p4d()
139 	} while (p4dp++, vaddr = next, vaddr != end);  in kasan_populate_p4d()
143 				      unsigned long vaddr, unsigned long end)  in kasan_populate_pgd()  argument
149 		next = pgd_addr_end(vaddr, end);  in kasan_populate_pgd()
151 		if (pgd_none(pgdp_get(pgdp)) && IS_ALIGNED(vaddr, PGDIR_SIZE) &&  in kasan_populate_pgd()
152 		    (next - vaddr) >= PGDIR_SIZE) {  in kasan_populate_pgd()
161 		kasan_populate_p4d(pgdp, vaddr, next);  in kasan_populate_pgd()
162 	} while (pgdp++, vaddr = next, vaddr != end);  in kasan_populate_pgd()
166 					 unsigned long vaddr, unsigned long end)  in kasan_early_clear_pud()  argument
175 		pudp = base_pud + pud_index(vaddr);  in kasan_early_clear_pud()
179 		next = pud_addr_end(vaddr, end);  in kasan_early_clear_pud()
181 		if (IS_ALIGNED(vaddr, PUD_SIZE) && (next - vaddr) >= PUD_SIZE) {  in kasan_early_clear_pud()
187 	} while (pudp++, vaddr = next, vaddr != end);  in kasan_early_clear_pud()
191 					 unsigned long vaddr, unsigned long end)  in kasan_early_clear_p4d()  argument
200 		p4dp = base_p4d + p4d_index(vaddr);  in kasan_early_clear_p4d()
204 		next = p4d_addr_end(vaddr, end);  in kasan_early_clear_p4d()
206 		if (pgtable_l4_enabled && IS_ALIGNED(vaddr, P4D_SIZE) &&  in kasan_early_clear_p4d()
207 		    (next - vaddr) >= P4D_SIZE) {  in kasan_early_clear_p4d()
212 		kasan_early_clear_pud(p4dp, vaddr, next);  in kasan_early_clear_p4d()
213 	} while (p4dp++, vaddr = next, vaddr != end);  in kasan_early_clear_p4d()
217 					 unsigned long vaddr, unsigned long end)  in kasan_early_clear_pgd()  argument
222 		next = pgd_addr_end(vaddr, end);  in kasan_early_clear_pgd()
224 		if (pgtable_l5_enabled && IS_ALIGNED(vaddr, PGDIR_SIZE) &&  in kasan_early_clear_pgd()
225 		    (next - vaddr) >= PGDIR_SIZE) {  in kasan_early_clear_pgd()
230 		kasan_early_clear_p4d(pgdp, vaddr, next);  in kasan_early_clear_pgd()
231 	} while (pgdp++, vaddr = next, vaddr != end);  in kasan_early_clear_pgd()
235 					    unsigned long vaddr,  in kasan_early_populate_pud()  argument
246 		pudp = base_pud + pud_index(vaddr);  in kasan_early_populate_pud()
250 		next = pud_addr_end(vaddr, end);  in kasan_early_populate_pud()
252 		if (pud_none(pudp_get(pudp)) && IS_ALIGNED(vaddr, PUD_SIZE) &&  in kasan_early_populate_pud()
253 		    (next - vaddr) >= PUD_SIZE) {  in kasan_early_populate_pud()
260 	} while (pudp++, vaddr = next, vaddr != end);  in kasan_early_populate_pud()
264 					    unsigned long vaddr,  in kasan_early_populate_p4d()  argument
284 		p4dp = base_p4d + p4d_index(vaddr);  in kasan_early_populate_p4d()
288 		next = p4d_addr_end(vaddr, end);  in kasan_early_populate_p4d()
290 		if (p4d_none(p4dp_get(p4dp)) && IS_ALIGNED(vaddr, P4D_SIZE) &&  in kasan_early_populate_p4d()
291 		    (next - vaddr) >= P4D_SIZE) {  in kasan_early_populate_p4d()
297 		kasan_early_populate_pud(p4dp, vaddr, next);  in kasan_early_populate_p4d()
298 	} while (p4dp++, vaddr = next, vaddr != end);  in kasan_early_populate_p4d()
302 					    unsigned long vaddr,  in kasan_early_populate_pgd()  argument
309 		next = pgd_addr_end(vaddr, end);  in kasan_early_populate_pgd()
311 		if (pgd_none(pgdp_get(pgdp)) && IS_ALIGNED(vaddr, PGDIR_SIZE) &&  in kasan_early_populate_pgd()
312 		    (next - vaddr) >= PGDIR_SIZE) {  in kasan_early_populate_pgd()
318 		kasan_early_populate_p4d(pgdp, vaddr, next);  in kasan_early_populate_pgd()
319 	} while (pgdp++, vaddr = next, vaddr != end);  in kasan_early_populate_pgd()
371 	unsigned long vaddr = (unsigned long)start & PAGE_MASK;  in kasan_populate()  local
374 	kasan_populate_pgd(pgd_offset_k(vaddr), vaddr, vend);  in kasan_populate()
378 					      unsigned long vaddr, unsigned long end)  in kasan_shallow_populate_pud()  argument
382 	pud_t *pud_k = pud_offset(p4d, vaddr);  in kasan_shallow_populate_pud()
385 		next = pud_addr_end(vaddr, end);  in kasan_shallow_populate_pud()
394 	} while (pud_k++, vaddr = next, vaddr != end);  in kasan_shallow_populate_pud()
398 					      unsigned long vaddr, unsigned long end)  in kasan_shallow_populate_p4d()  argument
402 	p4d_t *p4d_k = p4d_offset(pgd, vaddr);  in kasan_shallow_populate_p4d()
405 		next = p4d_addr_end(vaddr, end);  in kasan_shallow_populate_p4d()
413 		kasan_shallow_populate_pud(p4d_k, vaddr, end);  in kasan_shallow_populate_p4d()
414 	} while (p4d_k++, vaddr = next, vaddr != end);  in kasan_shallow_populate_p4d()
417 static void __init kasan_shallow_populate_pgd(unsigned long vaddr, unsigned long end)  in kasan_shallow_populate_pgd()  argument
421 	pgd_t *pgd_k = pgd_offset_k(vaddr);  in kasan_shallow_populate_pgd()
424 		next = pgd_addr_end(vaddr, end);  in kasan_shallow_populate_pgd()
432 		kasan_shallow_populate_p4d(pgd_k, vaddr, next);  in kasan_shallow_populate_pgd()
433 	} while (pgd_k++, vaddr = next, vaddr != end);  in kasan_shallow_populate_pgd()
438 	unsigned long vaddr = (unsigned long)start & PAGE_MASK;  in kasan_shallow_populate()  local
441 	kasan_shallow_populate_pgd(vaddr, vend);  in kasan_shallow_populate()