Lines Matching +full:addr +full:- +full:mode
1 // SPDX-License-Identifier: GPL-2.0
35 static void pgtable_populate(unsigned long addr, unsigned long end, enum populate_mode mode);
48 static inline void kasan_populate(unsigned long start, unsigned long end, enum populate_mode mode) in kasan_populate() argument
52 pgtable_populate(start, end, mode); in kasan_populate()
75 * +- 0 -------------+ +- shadow start -+ in kasan_populate_shadow()
78 * +-end of ident map+ / +----------------+ in kasan_populate_shadow()
81 * +- vmalloc area -+ / | mapping | in kasan_populate_shadow()
83 * +- modules vaddr -+ / +----------------+ in kasan_populate_shadow()
85 * +- shadow start -+ +----------------+ in kasan_populate_shadow()
86 * | 1/8 addr space | | zero pg mapping| (untracked) in kasan_populate_shadow()
87 * +- shadow end ----+---------+- shadow end ---+ in kasan_populate_shadow()
90 * +- 0 -------------+ +- shadow start -+ in kasan_populate_shadow()
93 * +-end of ident map+ / +----------------+ in kasan_populate_shadow()
96 * +- vmalloc area -+ / +----------------+ in kasan_populate_shadow()
98 * +- modules vaddr -+ / +----------------+ in kasan_populate_shadow()
100 * +- shadow start -+ +----------------+ in kasan_populate_shadow()
101 * | 1/8 addr space | | zero pg mapping| (untracked) in kasan_populate_shadow()
102 * +- shadow end ----+---------+- shadow end ---+ in kasan_populate_shadow()
123 static bool kasan_pgd_populate_zero_shadow(pgd_t *pgd, unsigned long addr, in kasan_pgd_populate_zero_shadow() argument
124 unsigned long end, enum populate_mode mode) in kasan_pgd_populate_zero_shadow() argument
126 if (mode == POPULATE_KASAN_ZERO_SHADOW && in kasan_pgd_populate_zero_shadow()
127 IS_ALIGNED(addr, PGDIR_SIZE) && end - addr >= PGDIR_SIZE) { in kasan_pgd_populate_zero_shadow()
134 static bool kasan_p4d_populate_zero_shadow(p4d_t *p4d, unsigned long addr, in kasan_p4d_populate_zero_shadow() argument
135 unsigned long end, enum populate_mode mode) in kasan_p4d_populate_zero_shadow() argument
137 if (mode == POPULATE_KASAN_ZERO_SHADOW && in kasan_p4d_populate_zero_shadow()
138 IS_ALIGNED(addr, P4D_SIZE) && end - addr >= P4D_SIZE) { in kasan_p4d_populate_zero_shadow()
145 static bool kasan_pud_populate_zero_shadow(pud_t *pud, unsigned long addr, in kasan_pud_populate_zero_shadow() argument
146 unsigned long end, enum populate_mode mode) in kasan_pud_populate_zero_shadow() argument
148 if (mode == POPULATE_KASAN_ZERO_SHADOW && in kasan_pud_populate_zero_shadow()
149 IS_ALIGNED(addr, PUD_SIZE) && end - addr >= PUD_SIZE) { in kasan_pud_populate_zero_shadow()
156 static bool kasan_pmd_populate_zero_shadow(pmd_t *pmd, unsigned long addr, in kasan_pmd_populate_zero_shadow() argument
157 unsigned long end, enum populate_mode mode) in kasan_pmd_populate_zero_shadow() argument
159 if (mode == POPULATE_KASAN_ZERO_SHADOW && in kasan_pmd_populate_zero_shadow()
160 IS_ALIGNED(addr, PMD_SIZE) && end - addr >= PMD_SIZE) { in kasan_pmd_populate_zero_shadow()
167 static bool kasan_pte_populate_zero_shadow(pte_t *pte, enum populate_mode mode) in kasan_pte_populate_zero_shadow() argument
171 if (mode == POPULATE_KASAN_ZERO_SHADOW) { in kasan_pte_populate_zero_shadow()
181 static inline bool kasan_pgd_populate_zero_shadow(pgd_t *pgd, unsigned long addr, in kasan_pgd_populate_zero_shadow() argument
182 unsigned long end, enum populate_mode mode) in kasan_pgd_populate_zero_shadow() argument
187 static inline bool kasan_p4d_populate_zero_shadow(p4d_t *p4d, unsigned long addr, in kasan_p4d_populate_zero_shadow() argument
188 unsigned long end, enum populate_mode mode) in kasan_p4d_populate_zero_shadow() argument
193 static inline bool kasan_pud_populate_zero_shadow(pud_t *pud, unsigned long addr, in kasan_pud_populate_zero_shadow() argument
194 unsigned long end, enum populate_mode mode) in kasan_pud_populate_zero_shadow() argument
199 static inline bool kasan_pmd_populate_zero_shadow(pmd_t *pmd, unsigned long addr, in kasan_pmd_populate_zero_shadow() argument
200 unsigned long end, enum populate_mode mode) in kasan_pmd_populate_zero_shadow() argument
205 static bool kasan_pte_populate_zero_shadow(pte_t *pte, enum populate_mode mode) in kasan_pte_populate_zero_shadow() argument
251 static unsigned long _pa(unsigned long addr, unsigned long size, enum populate_mode mode) in _pa() argument
253 switch (mode) { in _pa()
255 return -1; in _pa()
257 return addr; in _pa()
259 return __abs_lowcore_pa(addr); in _pa()
262 addr = physmem_alloc_top_down(RR_VMEM, size, size); in _pa()
263 memset((void *)addr, 0, size); in _pa()
264 return addr; in _pa()
267 return -1; in _pa()
271 static bool can_large_pud(pud_t *pu_dir, unsigned long addr, unsigned long end) in can_large_pud() argument
274 IS_ALIGNED(addr, PUD_SIZE) && (end - addr) >= PUD_SIZE; in can_large_pud()
277 static bool can_large_pmd(pmd_t *pm_dir, unsigned long addr, unsigned long end) in can_large_pmd() argument
280 IS_ALIGNED(addr, PMD_SIZE) && (end - addr) >= PMD_SIZE; in can_large_pmd()
283 static void pgtable_pte_populate(pmd_t *pmd, unsigned long addr, unsigned long end, in pgtable_pte_populate() argument
284 enum populate_mode mode) in pgtable_pte_populate() argument
289 pte = pte_offset_kernel(pmd, addr); in pgtable_pte_populate()
290 for (; addr < end; addr += PAGE_SIZE, pte++) { in pgtable_pte_populate()
292 if (kasan_pte_populate_zero_shadow(pte, mode)) in pgtable_pte_populate()
294 entry = __pte(_pa(addr, PAGE_SIZE, mode)); in pgtable_pte_populate()
302 if (mode == POPULATE_DIRECT) in pgtable_pte_populate()
306 static void pgtable_pmd_populate(pud_t *pud, unsigned long addr, unsigned long end, in pgtable_pmd_populate() argument
307 enum populate_mode mode) in pgtable_pmd_populate() argument
313 pmd = pmd_offset(pud, addr); in pgtable_pmd_populate()
314 for (; addr < end; addr = next, pmd++) { in pgtable_pmd_populate()
315 next = pmd_addr_end(addr, end); in pgtable_pmd_populate()
317 if (kasan_pmd_populate_zero_shadow(pmd, addr, next, mode)) in pgtable_pmd_populate()
319 if (can_large_pmd(pmd, addr, next)) { in pgtable_pmd_populate()
320 entry = __pmd(_pa(addr, _SEGMENT_SIZE, mode)); in pgtable_pmd_populate()
333 pgtable_pte_populate(pmd, addr, next, mode); in pgtable_pmd_populate()
335 if (mode == POPULATE_DIRECT) in pgtable_pmd_populate()
339 static void pgtable_pud_populate(p4d_t *p4d, unsigned long addr, unsigned long end, in pgtable_pud_populate() argument
340 enum populate_mode mode) in pgtable_pud_populate() argument
346 pud = pud_offset(p4d, addr); in pgtable_pud_populate()
347 for (; addr < end; addr = next, pud++) { in pgtable_pud_populate()
348 next = pud_addr_end(addr, end); in pgtable_pud_populate()
350 if (kasan_pud_populate_zero_shadow(pud, addr, next, mode)) in pgtable_pud_populate()
352 if (can_large_pud(pud, addr, next)) { in pgtable_pud_populate()
353 entry = __pud(_pa(addr, _REGION3_SIZE, mode)); in pgtable_pud_populate()
366 pgtable_pmd_populate(pud, addr, next, mode); in pgtable_pud_populate()
368 if (mode == POPULATE_DIRECT) in pgtable_pud_populate()
372 static void pgtable_p4d_populate(pgd_t *pgd, unsigned long addr, unsigned long end, in pgtable_p4d_populate() argument
373 enum populate_mode mode) in pgtable_p4d_populate() argument
379 p4d = p4d_offset(pgd, addr); in pgtable_p4d_populate()
380 for (; addr < end; addr = next, p4d++) { in pgtable_p4d_populate()
381 next = p4d_addr_end(addr, end); in pgtable_p4d_populate()
383 if (kasan_p4d_populate_zero_shadow(p4d, addr, next, mode)) in pgtable_p4d_populate()
388 pgtable_pud_populate(p4d, addr, next, mode); in pgtable_p4d_populate()
392 static void pgtable_populate(unsigned long addr, unsigned long end, enum populate_mode mode) in pgtable_populate() argument
398 pgd = pgd_offset(&init_mm, addr); in pgtable_populate()
399 for (; addr < end; addr = next, pgd++) { in pgtable_populate()
400 next = pgd_addr_end(addr, end); in pgtable_populate()
402 if (kasan_pgd_populate_zero_shadow(pgd, addr, next, mode)) in pgtable_populate()
408 if (mode == POPULATE_KASAN_SHALLOW) in pgtable_populate()
411 pgtable_p4d_populate(pgd, addr, next, mode); in pgtable_populate()