1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (c) 2014, The Linux Foundation. All rights reserved. 4 */ 5 #include <linux/kernel.h> 6 #include <linux/mm.h> 7 #include <linux/module.h> 8 #include <linux/sched.h> 9 #include <linux/vmalloc.h> 10 11 #include <asm/cacheflush.h> 12 #include <asm/set_memory.h> 13 #include <asm/tlbflush.h> 14 #include <asm/kfence.h> 15 16 struct page_change_data { 17 pgprot_t set_mask; 18 pgprot_t clear_mask; 19 }; 20 21 bool rodata_full __ro_after_init = IS_ENABLED(CONFIG_RODATA_FULL_DEFAULT_ENABLED); 22 23 bool can_set_direct_map(void) 24 { 25 /* 26 * rodata_full and DEBUG_PAGEALLOC require linear map to be 27 * mapped at page granularity, so that it is possible to 28 * protect/unprotect single pages. 29 * 30 * KFENCE pool requires page-granular mapping if initialized late. 31 */ 32 return rodata_full || debug_pagealloc_enabled() || 33 arm64_kfence_can_set_direct_map(); 34 } 35 36 static int change_page_range(pte_t *ptep, unsigned long addr, void *data) 37 { 38 struct page_change_data *cdata = data; 39 pte_t pte = READ_ONCE(*ptep); 40 41 pte = clear_pte_bit(pte, cdata->clear_mask); 42 pte = set_pte_bit(pte, cdata->set_mask); 43 44 set_pte(ptep, pte); 45 return 0; 46 } 47 48 /* 49 * This function assumes that the range is mapped with PAGE_SIZE pages. 50 */ 51 static int __change_memory_common(unsigned long start, unsigned long size, 52 pgprot_t set_mask, pgprot_t clear_mask) 53 { 54 struct page_change_data data; 55 int ret; 56 57 data.set_mask = set_mask; 58 data.clear_mask = clear_mask; 59 60 ret = apply_to_page_range(&init_mm, start, size, change_page_range, 61 &data); 62 63 flush_tlb_kernel_range(start, start + size); 64 return ret; 65 } 66 67 static int change_memory_common(unsigned long addr, int numpages, 68 pgprot_t set_mask, pgprot_t clear_mask) 69 { 70 unsigned long start = addr; 71 unsigned long size = PAGE_SIZE * numpages; 72 unsigned long end = start + size; 73 struct vm_struct *area; 74 int i; 75 76 if (!PAGE_ALIGNED(addr)) { 77 start &= PAGE_MASK; 78 end = start + size; 79 WARN_ON_ONCE(1); 80 } 81 82 /* 83 * Kernel VA mappings are always live, and splitting live section 84 * mappings into page mappings may cause TLB conflicts. This means 85 * we have to ensure that changing the permission bits of the range 86 * we are operating on does not result in such splitting. 87 * 88 * Let's restrict ourselves to mappings created by vmalloc (or vmap). 89 * Those are guaranteed to consist entirely of page mappings, and 90 * splitting is never needed. 91 * 92 * So check whether the [addr, addr + size) interval is entirely 93 * covered by precisely one VM area that has the VM_ALLOC flag set. 94 */ 95 area = find_vm_area((void *)addr); 96 if (!area || 97 end > (unsigned long)kasan_reset_tag(area->addr) + area->size || 98 !(area->flags & VM_ALLOC)) 99 return -EINVAL; 100 101 if (!numpages) 102 return 0; 103 104 /* 105 * If we are manipulating read-only permissions, apply the same 106 * change to the linear mapping of the pages that back this VM area. 107 */ 108 if (rodata_full && (pgprot_val(set_mask) == PTE_RDONLY || 109 pgprot_val(clear_mask) == PTE_RDONLY)) { 110 for (i = 0; i < area->nr_pages; i++) { 111 __change_memory_common((u64)page_address(area->pages[i]), 112 PAGE_SIZE, set_mask, clear_mask); 113 } 114 } 115 116 /* 117 * Get rid of potentially aliasing lazily unmapped vm areas that may 118 * have permissions set that deviate from the ones we are setting here. 119 */ 120 vm_unmap_aliases(); 121 122 return __change_memory_common(start, size, set_mask, clear_mask); 123 } 124 125 int set_memory_ro(unsigned long addr, int numpages) 126 { 127 return change_memory_common(addr, numpages, 128 __pgprot(PTE_RDONLY), 129 __pgprot(PTE_WRITE)); 130 } 131 132 int set_memory_rw(unsigned long addr, int numpages) 133 { 134 return change_memory_common(addr, numpages, 135 __pgprot(PTE_WRITE), 136 __pgprot(PTE_RDONLY)); 137 } 138 139 int set_memory_nx(unsigned long addr, int numpages) 140 { 141 return change_memory_common(addr, numpages, 142 __pgprot(PTE_PXN), 143 __pgprot(PTE_MAYBE_GP)); 144 } 145 146 int set_memory_x(unsigned long addr, int numpages) 147 { 148 return change_memory_common(addr, numpages, 149 __pgprot(PTE_MAYBE_GP), 150 __pgprot(PTE_PXN)); 151 } 152 153 int set_memory_valid(unsigned long addr, int numpages, int enable) 154 { 155 if (enable) 156 return __change_memory_common(addr, PAGE_SIZE * numpages, 157 __pgprot(PTE_VALID), 158 __pgprot(0)); 159 else 160 return __change_memory_common(addr, PAGE_SIZE * numpages, 161 __pgprot(0), 162 __pgprot(PTE_VALID)); 163 } 164 165 int set_direct_map_invalid_noflush(struct page *page) 166 { 167 struct page_change_data data = { 168 .set_mask = __pgprot(0), 169 .clear_mask = __pgprot(PTE_VALID), 170 }; 171 172 if (!can_set_direct_map()) 173 return 0; 174 175 return apply_to_page_range(&init_mm, 176 (unsigned long)page_address(page), 177 PAGE_SIZE, change_page_range, &data); 178 } 179 180 int set_direct_map_default_noflush(struct page *page) 181 { 182 struct page_change_data data = { 183 .set_mask = __pgprot(PTE_VALID | PTE_WRITE), 184 .clear_mask = __pgprot(PTE_RDONLY), 185 }; 186 187 if (!can_set_direct_map()) 188 return 0; 189 190 return apply_to_page_range(&init_mm, 191 (unsigned long)page_address(page), 192 PAGE_SIZE, change_page_range, &data); 193 } 194 195 #ifdef CONFIG_DEBUG_PAGEALLOC 196 void __kernel_map_pages(struct page *page, int numpages, int enable) 197 { 198 if (!can_set_direct_map()) 199 return; 200 201 set_memory_valid((unsigned long)page_address(page), numpages, enable); 202 } 203 #endif /* CONFIG_DEBUG_PAGEALLOC */ 204 205 /* 206 * This function is used to determine if a linear map page has been marked as 207 * not-valid. Walk the page table and check the PTE_VALID bit. 208 * 209 * Because this is only called on the kernel linear map, p?d_sect() implies 210 * p?d_present(). When debug_pagealloc is enabled, sections mappings are 211 * disabled. 212 */ 213 bool kernel_page_present(struct page *page) 214 { 215 pgd_t *pgdp; 216 p4d_t *p4dp; 217 pud_t *pudp, pud; 218 pmd_t *pmdp, pmd; 219 pte_t *ptep; 220 unsigned long addr = (unsigned long)page_address(page); 221 222 pgdp = pgd_offset_k(addr); 223 if (pgd_none(READ_ONCE(*pgdp))) 224 return false; 225 226 p4dp = p4d_offset(pgdp, addr); 227 if (p4d_none(READ_ONCE(*p4dp))) 228 return false; 229 230 pudp = pud_offset(p4dp, addr); 231 pud = READ_ONCE(*pudp); 232 if (pud_none(pud)) 233 return false; 234 if (pud_sect(pud)) 235 return true; 236 237 pmdp = pmd_offset(pudp, addr); 238 pmd = READ_ONCE(*pmdp); 239 if (pmd_none(pmd)) 240 return false; 241 if (pmd_sect(pmd)) 242 return true; 243 244 ptep = pte_offset_kernel(pmdp, addr); 245 return pte_valid(READ_ONCE(*ptep)); 246 } 247