1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2019 SiFive 4 */ 5 6 #include <linux/pagewalk.h> 7 #include <linux/pgtable.h> 8 #include <linux/vmalloc.h> 9 #include <asm/tlbflush.h> 10 #include <asm/bitops.h> 11 #include <asm/set_memory.h> 12 13 struct pageattr_masks { 14 pgprot_t set_mask; 15 pgprot_t clear_mask; 16 }; 17 18 static unsigned long set_pageattr_masks(unsigned long val, struct mm_walk *walk) 19 { 20 struct pageattr_masks *masks = walk->private; 21 unsigned long new_val = val; 22 23 new_val &= ~(pgprot_val(masks->clear_mask)); 24 new_val |= (pgprot_val(masks->set_mask)); 25 26 return new_val; 27 } 28 29 static int pageattr_p4d_entry(p4d_t *p4d, unsigned long addr, 30 unsigned long next, struct mm_walk *walk) 31 { 32 p4d_t val = READ_ONCE(*p4d); 33 34 if (p4d_leaf(val)) { 35 val = __p4d(set_pageattr_masks(p4d_val(val), walk)); 36 set_p4d(p4d, val); 37 } 38 39 return 0; 40 } 41 42 static int pageattr_pud_entry(pud_t *pud, unsigned long addr, 43 unsigned long next, struct mm_walk *walk) 44 { 45 pud_t val = READ_ONCE(*pud); 46 47 if (pud_leaf(val)) { 48 val = __pud(set_pageattr_masks(pud_val(val), walk)); 49 set_pud(pud, val); 50 } 51 52 return 0; 53 } 54 55 static int pageattr_pmd_entry(pmd_t *pmd, unsigned long addr, 56 unsigned long next, struct mm_walk *walk) 57 { 58 pmd_t val = READ_ONCE(*pmd); 59 60 if (pmd_leaf(val)) { 61 val = __pmd(set_pageattr_masks(pmd_val(val), walk)); 62 set_pmd(pmd, val); 63 } 64 65 return 0; 66 } 67 68 static int pageattr_pte_entry(pte_t *pte, unsigned long addr, 69 unsigned long next, struct mm_walk *walk) 70 { 71 pte_t val = READ_ONCE(*pte); 72 73 val = __pte(set_pageattr_masks(pte_val(val), walk)); 74 set_pte(pte, val); 75 76 return 0; 77 } 78 79 static int pageattr_pte_hole(unsigned long addr, unsigned long next, 80 int depth, struct mm_walk *walk) 81 { 82 /* Nothing to do here */ 83 return 0; 84 } 85 86 static const struct mm_walk_ops pageattr_ops = { 87 .p4d_entry = pageattr_p4d_entry, 88 .pud_entry = pageattr_pud_entry, 89 .pmd_entry = pageattr_pmd_entry, 90 .pte_entry = pageattr_pte_entry, 91 .pte_hole = pageattr_pte_hole, 92 .walk_lock = PGWALK_RDLOCK, 93 }; 94 95 #ifdef CONFIG_64BIT 96 static int __split_linear_mapping_pmd(pud_t *pudp, 97 unsigned long vaddr, unsigned long end) 98 { 99 pmd_t *pmdp; 100 unsigned long next; 101 102 pmdp = pmd_offset(pudp, vaddr); 103 104 do { 105 next = pmd_addr_end(vaddr, end); 106 107 if (next - vaddr >= PMD_SIZE && 108 vaddr <= (vaddr & PMD_MASK) && end >= next) 109 continue; 110 111 if (pmd_leaf(*pmdp)) { 112 struct page *pte_page; 113 unsigned long pfn = _pmd_pfn(*pmdp); 114 pgprot_t prot = __pgprot(pmd_val(*pmdp) & ~_PAGE_PFN_MASK); 115 pte_t *ptep_new; 116 int i; 117 118 pte_page = alloc_page(GFP_KERNEL); 119 if (!pte_page) 120 return -ENOMEM; 121 122 ptep_new = (pte_t *)page_address(pte_page); 123 for (i = 0; i < PTRS_PER_PTE; ++i, ++ptep_new) 124 set_pte(ptep_new, pfn_pte(pfn + i, prot)); 125 126 smp_wmb(); 127 128 set_pmd(pmdp, pfn_pmd(page_to_pfn(pte_page), PAGE_TABLE)); 129 } 130 } while (pmdp++, vaddr = next, vaddr != end); 131 132 return 0; 133 } 134 135 static int __split_linear_mapping_pud(p4d_t *p4dp, 136 unsigned long vaddr, unsigned long end) 137 { 138 pud_t *pudp; 139 unsigned long next; 140 int ret; 141 142 pudp = pud_offset(p4dp, vaddr); 143 144 do { 145 next = pud_addr_end(vaddr, end); 146 147 if (next - vaddr >= PUD_SIZE && 148 vaddr <= (vaddr & PUD_MASK) && end >= next) 149 continue; 150 151 if (pud_leaf(*pudp)) { 152 struct page *pmd_page; 153 unsigned long pfn = _pud_pfn(*pudp); 154 pgprot_t prot = __pgprot(pud_val(*pudp) & ~_PAGE_PFN_MASK); 155 pmd_t *pmdp_new; 156 int i; 157 158 pmd_page = alloc_page(GFP_KERNEL); 159 if (!pmd_page) 160 return -ENOMEM; 161 162 pmdp_new = (pmd_t *)page_address(pmd_page); 163 for (i = 0; i < PTRS_PER_PMD; ++i, ++pmdp_new) 164 set_pmd(pmdp_new, 165 pfn_pmd(pfn + ((i * PMD_SIZE) >> PAGE_SHIFT), prot)); 166 167 smp_wmb(); 168 169 set_pud(pudp, pfn_pud(page_to_pfn(pmd_page), PAGE_TABLE)); 170 } 171 172 ret = __split_linear_mapping_pmd(pudp, vaddr, next); 173 if (ret) 174 return ret; 175 } while (pudp++, vaddr = next, vaddr != end); 176 177 return 0; 178 } 179 180 static int __split_linear_mapping_p4d(pgd_t *pgdp, 181 unsigned long vaddr, unsigned long end) 182 { 183 p4d_t *p4dp; 184 unsigned long next; 185 int ret; 186 187 p4dp = p4d_offset(pgdp, vaddr); 188 189 do { 190 next = p4d_addr_end(vaddr, end); 191 192 /* 193 * If [vaddr; end] contains [vaddr & P4D_MASK; next], we don't 194 * need to split, we'll change the protections on the whole P4D. 195 */ 196 if (next - vaddr >= P4D_SIZE && 197 vaddr <= (vaddr & P4D_MASK) && end >= next) 198 continue; 199 200 if (p4d_leaf(*p4dp)) { 201 struct page *pud_page; 202 unsigned long pfn = _p4d_pfn(*p4dp); 203 pgprot_t prot = __pgprot(p4d_val(*p4dp) & ~_PAGE_PFN_MASK); 204 pud_t *pudp_new; 205 int i; 206 207 pud_page = alloc_page(GFP_KERNEL); 208 if (!pud_page) 209 return -ENOMEM; 210 211 /* 212 * Fill the pud level with leaf puds that have the same 213 * protections as the leaf p4d. 214 */ 215 pudp_new = (pud_t *)page_address(pud_page); 216 for (i = 0; i < PTRS_PER_PUD; ++i, ++pudp_new) 217 set_pud(pudp_new, 218 pfn_pud(pfn + ((i * PUD_SIZE) >> PAGE_SHIFT), prot)); 219 220 /* 221 * Make sure the pud filling is not reordered with the 222 * p4d store which could result in seeing a partially 223 * filled pud level. 224 */ 225 smp_wmb(); 226 227 set_p4d(p4dp, pfn_p4d(page_to_pfn(pud_page), PAGE_TABLE)); 228 } 229 230 ret = __split_linear_mapping_pud(p4dp, vaddr, next); 231 if (ret) 232 return ret; 233 } while (p4dp++, vaddr = next, vaddr != end); 234 235 return 0; 236 } 237 238 static int __split_linear_mapping_pgd(pgd_t *pgdp, 239 unsigned long vaddr, 240 unsigned long end) 241 { 242 unsigned long next; 243 int ret; 244 245 do { 246 next = pgd_addr_end(vaddr, end); 247 /* We never use PGD mappings for the linear mapping */ 248 ret = __split_linear_mapping_p4d(pgdp, vaddr, next); 249 if (ret) 250 return ret; 251 } while (pgdp++, vaddr = next, vaddr != end); 252 253 return 0; 254 } 255 256 static int split_linear_mapping(unsigned long start, unsigned long end) 257 { 258 return __split_linear_mapping_pgd(pgd_offset_k(start), start, end); 259 } 260 #endif /* CONFIG_64BIT */ 261 262 static int __set_memory(unsigned long addr, int numpages, pgprot_t set_mask, 263 pgprot_t clear_mask) 264 { 265 int ret; 266 unsigned long start = addr; 267 unsigned long end = start + PAGE_SIZE * numpages; 268 unsigned long __maybe_unused lm_start; 269 unsigned long __maybe_unused lm_end; 270 struct pageattr_masks masks = { 271 .set_mask = set_mask, 272 .clear_mask = clear_mask 273 }; 274 275 if (!numpages) 276 return 0; 277 278 mmap_write_lock(&init_mm); 279 280 #ifdef CONFIG_64BIT 281 /* 282 * We are about to change the permissions of a kernel mapping, we must 283 * apply the same changes to its linear mapping alias, which may imply 284 * splitting a huge mapping. 285 */ 286 287 if (is_vmalloc_or_module_addr((void *)start)) { 288 struct vm_struct *area = NULL; 289 int i, page_start; 290 291 area = find_vm_area((void *)start); 292 page_start = (start - (unsigned long)area->addr) >> PAGE_SHIFT; 293 294 for (i = page_start; i < page_start + numpages; ++i) { 295 lm_start = (unsigned long)page_address(area->pages[i]); 296 lm_end = lm_start + PAGE_SIZE; 297 298 ret = split_linear_mapping(lm_start, lm_end); 299 if (ret) 300 goto unlock; 301 302 ret = walk_page_range_novma(&init_mm, lm_start, lm_end, 303 &pageattr_ops, NULL, &masks); 304 if (ret) 305 goto unlock; 306 } 307 } else if (is_kernel_mapping(start) || is_linear_mapping(start)) { 308 if (is_kernel_mapping(start)) { 309 lm_start = (unsigned long)lm_alias(start); 310 lm_end = (unsigned long)lm_alias(end); 311 } else { 312 lm_start = start; 313 lm_end = end; 314 } 315 316 ret = split_linear_mapping(lm_start, lm_end); 317 if (ret) 318 goto unlock; 319 320 ret = walk_page_range_novma(&init_mm, lm_start, lm_end, 321 &pageattr_ops, NULL, &masks); 322 if (ret) 323 goto unlock; 324 } 325 326 ret = walk_page_range_novma(&init_mm, start, end, &pageattr_ops, NULL, 327 &masks); 328 329 unlock: 330 mmap_write_unlock(&init_mm); 331 332 /* 333 * We can't use flush_tlb_kernel_range() here as we may have split a 334 * hugepage that is larger than that, so let's flush everything. 335 */ 336 flush_tlb_all(); 337 #else 338 ret = walk_page_range_novma(&init_mm, start, end, &pageattr_ops, NULL, 339 &masks); 340 341 mmap_write_unlock(&init_mm); 342 343 flush_tlb_kernel_range(start, end); 344 #endif 345 346 return ret; 347 } 348 349 int set_memory_rw_nx(unsigned long addr, int numpages) 350 { 351 return __set_memory(addr, numpages, __pgprot(_PAGE_READ | _PAGE_WRITE), 352 __pgprot(_PAGE_EXEC)); 353 } 354 355 int set_memory_ro(unsigned long addr, int numpages) 356 { 357 return __set_memory(addr, numpages, __pgprot(_PAGE_READ), 358 __pgprot(_PAGE_WRITE)); 359 } 360 361 int set_memory_rw(unsigned long addr, int numpages) 362 { 363 return __set_memory(addr, numpages, __pgprot(_PAGE_READ | _PAGE_WRITE), 364 __pgprot(0)); 365 } 366 367 int set_memory_x(unsigned long addr, int numpages) 368 { 369 return __set_memory(addr, numpages, __pgprot(_PAGE_EXEC), __pgprot(0)); 370 } 371 372 int set_memory_nx(unsigned long addr, int numpages) 373 { 374 return __set_memory(addr, numpages, __pgprot(0), __pgprot(_PAGE_EXEC)); 375 } 376 377 int set_direct_map_invalid_noflush(struct page *page) 378 { 379 return __set_memory((unsigned long)page_address(page), 1, 380 __pgprot(0), __pgprot(_PAGE_PRESENT)); 381 } 382 383 int set_direct_map_default_noflush(struct page *page) 384 { 385 return __set_memory((unsigned long)page_address(page), 1, 386 PAGE_KERNEL, __pgprot(_PAGE_EXEC)); 387 } 388 389 #ifdef CONFIG_DEBUG_PAGEALLOC 390 void __kernel_map_pages(struct page *page, int numpages, int enable) 391 { 392 if (!debug_pagealloc_enabled()) 393 return; 394 395 if (enable) 396 __set_memory((unsigned long)page_address(page), numpages, 397 __pgprot(_PAGE_PRESENT), __pgprot(0)); 398 else 399 __set_memory((unsigned long)page_address(page), numpages, 400 __pgprot(0), __pgprot(_PAGE_PRESENT)); 401 } 402 #endif 403 404 bool kernel_page_present(struct page *page) 405 { 406 unsigned long addr = (unsigned long)page_address(page); 407 pgd_t *pgd; 408 pud_t *pud; 409 p4d_t *p4d; 410 pmd_t *pmd; 411 pte_t *pte; 412 413 pgd = pgd_offset_k(addr); 414 if (!pgd_present(*pgd)) 415 return false; 416 if (pgd_leaf(*pgd)) 417 return true; 418 419 p4d = p4d_offset(pgd, addr); 420 if (!p4d_present(*p4d)) 421 return false; 422 if (p4d_leaf(*p4d)) 423 return true; 424 425 pud = pud_offset(p4d, addr); 426 if (!pud_present(*pud)) 427 return false; 428 if (pud_leaf(*pud)) 429 return true; 430 431 pmd = pmd_offset(pud, addr); 432 if (!pmd_present(*pmd)) 433 return false; 434 if (pmd_leaf(*pmd)) 435 return true; 436 437 pte = pte_offset_kernel(pmd, addr); 438 return pte_present(*pte); 439 } 440