70 static inline bool kasan_pte_table(pmd_t pmd)  in kasan_pte_table()  argument
72 	return pmd_page(pmd) == virt_to_page(lm_alias(kasan_early_shadow_pte));  in kasan_pte_table()
92 static void __ref zero_pte_populate(pmd_t *pmd, unsigned long addr,  in zero_pte_populate()  argument
95 	pte_t *pte = pte_offset_kernel(pmd, addr);  in zero_pte_populate()
105 		pte = pte_offset_kernel(pmd, addr);  in zero_pte_populate()
112 	pmd_t *pmd = pmd_offset(pud, addr);  in zero_pmd_populate()  local
119 			pmd_populate_kernel(&init_mm, pmd,  in zero_pmd_populate()
124 		if (pmd_none(*pmd)) {  in zero_pmd_populate()
134 			pmd_populate_kernel(&init_mm, pmd, p);  in zero_pmd_populate()
136 		zero_pte_populate(pmd, addr, next);  in zero_pmd_populate()
137 	} while (pmd++, addr = next, addr != end);  in zero_pmd_populate()
155 			pmd_t *pmd;  in zero_pud_populate()  local
159 			pmd = pmd_offset(pud, addr);  in zero_pud_populate()
160 			pmd_populate_kernel(&init_mm, pmd,  in zero_pud_populate()
198 			pmd_t *pmd;  in zero_p4d_populate()  local
205 			pmd = pmd_offset(pud, addr);  in zero_p4d_populate()
206 			pmd_populate_kernel(&init_mm, pmd,  in zero_p4d_populate()
250 			pmd_t *pmd;  in kasan_populate_early_shadow()  local
255 			 * [pud,pmd]_populate*() below needed only for  in kasan_populate_early_shadow()
268 			pmd = pmd_offset(pud, addr);  in kasan_populate_early_shadow()
269 			pmd_populate_kernel(&init_mm, pmd,  in kasan_populate_early_shadow()
292 static void kasan_free_pte(pte_t *pte_start, pmd_t *pmd)  in kasan_free_pte()  argument
303 	pte_free_kernel(&init_mm, (pte_t *)page_to_virt(pmd_page(*pmd)));  in kasan_free_pte()
304 	pmd_clear(pmd);  in kasan_free_pte()
309 	pmd_t *pmd;  in kasan_free_pmd()  local
313 		pmd = pmd_start + i;  in kasan_free_pmd()
314 		if (!pmd_none(*pmd))  in kasan_free_pmd()
374 static void kasan_remove_pmd_table(pmd_t *pmd, unsigned long addr,  in kasan_remove_pmd_table()  argument
379 	for (; addr < end; addr = next, pmd++) {  in kasan_remove_pmd_table()
384 		if (!pmd_present(*pmd))  in kasan_remove_pmd_table()
387 		if (kasan_pte_table(*pmd)) {  in kasan_remove_pmd_table()
390 				pmd_clear(pmd);  in kasan_remove_pmd_table()
394 		pte = pte_offset_kernel(pmd, addr);  in kasan_remove_pmd_table()
396 		kasan_free_pte(pte_offset_kernel(pmd, 0), pmd);  in kasan_remove_pmd_table()
406 		pmd_t *pmd, *pmd_base;  in kasan_remove_pud_table()  local
420 		pmd = pmd_offset(pud, addr);  in kasan_remove_pud_table()
422 		kasan_remove_pmd_table(pmd, addr, next);  in kasan_remove_pud_table()