62 static pte_t *__init kasan_pte_offset(pmd_t *pmdp, unsigned long addr, int node,  in kasan_pte_offset()  argument
72 	return early ? pte_offset_kimg(pmdp, addr)  in kasan_pte_offset()
73 		     : pte_offset_kernel(pmdp, addr);  in kasan_pte_offset()
76 static pmd_t *__init kasan_pmd_offset(pud_t *pudp, unsigned long addr, int node,  in kasan_pmd_offset()  argument
86 	return early ? pmd_offset_kimg(pudp, addr) : pmd_offset(pudp, addr);  in kasan_pmd_offset()
89 static pud_t *__init kasan_pud_offset(p4d_t *p4dp, unsigned long addr, int node,  in kasan_pud_offset()  argument
99 	return early ? pud_offset_kimg(p4dp, addr) : pud_offset(p4dp, addr);  in kasan_pud_offset()
102 static void __init kasan_pte_populate(pmd_t *pmdp, unsigned long addr,  in kasan_pte_populate()  argument
106 	pte_t *ptep = kasan_pte_offset(pmdp, addr, node, early);  in kasan_pte_populate()
114 		next = addr + PAGE_SIZE;  in kasan_pte_populate()
116 	} while (ptep++, addr = next, addr != end && pte_none(READ_ONCE(*ptep)));  in kasan_pte_populate()
119 static void __init kasan_pmd_populate(pud_t *pudp, unsigned long addr,  in kasan_pmd_populate()  argument
123 	pmd_t *pmdp = kasan_pmd_offset(pudp, addr, node, early);  in kasan_pmd_populate()
126 		next = pmd_addr_end(addr, end);  in kasan_pmd_populate()
127 		kasan_pte_populate(pmdp, addr, next, node, early);  in kasan_pmd_populate()
128 	} while (pmdp++, addr = next, addr != end && pmd_none(READ_ONCE(*pmdp)));  in kasan_pmd_populate()
131 static void __init kasan_pud_populate(p4d_t *p4dp, unsigned long addr,  in kasan_pud_populate()  argument
135 	pud_t *pudp = kasan_pud_offset(p4dp, addr, node, early);  in kasan_pud_populate()
138 		next = pud_addr_end(addr, end);  in kasan_pud_populate()
139 		kasan_pmd_populate(pudp, addr, next, node, early);  in kasan_pud_populate()
140 	} while (pudp++, addr = next, addr != end && pud_none(READ_ONCE(*pudp)));  in kasan_pud_populate()
143 static void __init kasan_p4d_populate(pgd_t *pgdp, unsigned long addr,  in kasan_p4d_populate()  argument
147 	p4d_t *p4dp = p4d_offset(pgdp, addr);  in kasan_p4d_populate()
150 		next = p4d_addr_end(addr, end);  in kasan_p4d_populate()
151 		kasan_pud_populate(p4dp, addr, next, node, early);  in kasan_p4d_populate()
152 	} while (p4dp++, addr = next, addr != end);  in kasan_p4d_populate()
155 static void __init kasan_pgd_populate(unsigned long addr, unsigned long end,  in kasan_pgd_populate()  argument
161 	pgdp = pgd_offset_k(addr);  in kasan_pgd_populate()
163 		next = pgd_addr_end(addr, end);  in kasan_pgd_populate()
164 		kasan_p4d_populate(pgdp, addr, next, node, early);  in kasan_pgd_populate()
165 	} while (pgdp++, addr = next, addr != end);  in kasan_pgd_populate()