57 static inline bool kasan_pmd_table(pud_t pud)  in kasan_pmd_table()  argument
59 	return pud_page(pud) == virt_to_page(lm_alias(kasan_early_shadow_pmd));  in kasan_pmd_table()
62 static inline bool kasan_pmd_table(pud_t pud)  in kasan_pmd_table()  argument
109 static int __ref zero_pmd_populate(pud_t *pud, unsigned long addr,  in zero_pmd_populate()  argument
112 	pmd_t *pmd = pmd_offset(pud, addr);  in zero_pmd_populate()
149 	pud_t *pud = pud_offset(p4d, addr);  in zero_pud_populate()  local
157 			pud_populate(&init_mm, pud,  in zero_pud_populate()
159 			pmd = pmd_offset(pud, addr);  in zero_pud_populate()
165 		if (pud_none(*pud)) {  in zero_pud_populate()
169 				p = pmd_alloc(&init_mm, pud, addr);  in zero_pud_populate()
175 				pud_populate(&init_mm, pud, p);  in zero_pud_populate()
178 		zero_pmd_populate(pud, addr, next);  in zero_pud_populate()
179 	} while (pud++, addr = next, addr != end);  in zero_pud_populate()
197 			pud_t *pud;  in zero_p4d_populate()  local
202 			pud = pud_offset(p4d, addr);  in zero_p4d_populate()
203 			pud_populate(&init_mm, pud,  in zero_p4d_populate()
205 			pmd = pmd_offset(pud, addr);  in zero_p4d_populate()
249 			pud_t *pud;  in kasan_populate_early_shadow()  local
265 			pud = pud_offset(p4d, addr);  in kasan_populate_early_shadow()
266 			pud_populate(&init_mm, pud,  in kasan_populate_early_shadow()
268 			pmd = pmd_offset(pud, addr);  in kasan_populate_early_shadow()
307 static void kasan_free_pmd(pmd_t *pmd_start, pud_t *pud)  in kasan_free_pmd()  argument
318 	pmd_free(&init_mm, (pmd_t *)page_to_virt(pud_page(*pud)));  in kasan_free_pmd()
319 	pud_clear(pud);  in kasan_free_pmd()
324 	pud_t *pud;  in kasan_free_pud()  local
328 		pud = pud_start + i;  in kasan_free_pud()
329 		if (!pud_none(*pud))  in kasan_free_pud()
400 static void kasan_remove_pud_table(pud_t *pud, unsigned long addr,  in kasan_remove_pud_table()  argument
405 	for (; addr < end; addr = next, pud++) {  in kasan_remove_pud_table()
410 		if (!pud_present(*pud))  in kasan_remove_pud_table()
413 		if (kasan_pmd_table(*pud)) {  in kasan_remove_pud_table()
416 				pud_clear(pud);  in kasan_remove_pud_table()
420 		pmd = pmd_offset(pud, addr);  in kasan_remove_pud_table()
421 		pmd_base = pmd_offset(pud, 0);  in kasan_remove_pud_table()
423 		kasan_free_pmd(pmd_base, pud);  in kasan_remove_pud_table()
433 		pud_t *pud;  in kasan_remove_p4d_table()  local
447 		pud = pud_offset(p4d, addr);  in kasan_remove_p4d_table()
448 		kasan_remove_pud_table(pud, addr, next);  in kasan_remove_p4d_table()