1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (c) 2014, The Linux Foundation. All rights reserved. 4 */ 5 #include <linux/kernel.h> 6 #include <linux/mm.h> 7 #include <linux/module.h> 8 #include <linux/sched.h> 9 #include <linux/vmalloc.h> 10 11 #include <asm/cacheflush.h> 12 #include <asm/set_memory.h> 13 #include <asm/tlbflush.h> 14 15 struct page_change_data { 16 pgprot_t set_mask; 17 pgprot_t clear_mask; 18 }; 19 20 bool rodata_full __ro_after_init = IS_ENABLED(CONFIG_RODATA_FULL_DEFAULT_ENABLED); 21 22 bool can_set_direct_map(void) 23 { 24 /* 25 * rodata_full, DEBUG_PAGEALLOC and KFENCE require linear map to be 26 * mapped at page granularity, so that it is possible to 27 * protect/unprotect single pages. 28 */ 29 return (rodata_enabled && rodata_full) || debug_pagealloc_enabled() || 30 IS_ENABLED(CONFIG_KFENCE); 31 } 32 33 static int change_page_range(pte_t *ptep, unsigned long addr, void *data) 34 { 35 struct page_change_data *cdata = data; 36 pte_t pte = READ_ONCE(*ptep); 37 38 pte = clear_pte_bit(pte, cdata->clear_mask); 39 pte = set_pte_bit(pte, cdata->set_mask); 40 41 set_pte(ptep, pte); 42 return 0; 43 } 44 45 /* 46 * This function assumes that the range is mapped with PAGE_SIZE pages. 47 */ 48 static int __change_memory_common(unsigned long start, unsigned long size, 49 pgprot_t set_mask, pgprot_t clear_mask) 50 { 51 struct page_change_data data; 52 int ret; 53 54 data.set_mask = set_mask; 55 data.clear_mask = clear_mask; 56 57 ret = apply_to_page_range(&init_mm, start, size, change_page_range, 58 &data); 59 60 flush_tlb_kernel_range(start, start + size); 61 return ret; 62 } 63 64 static int change_memory_common(unsigned long addr, int numpages, 65 pgprot_t set_mask, pgprot_t clear_mask) 66 { 67 unsigned long start = addr; 68 unsigned long size = PAGE_SIZE * numpages; 69 unsigned long end = start + size; 70 struct vm_struct *area; 71 int i; 72 73 if (!PAGE_ALIGNED(addr)) { 74 start &= PAGE_MASK; 75 end = start + size; 76 WARN_ON_ONCE(1); 77 } 78 79 /* 80 * Kernel VA mappings are always live, and splitting live section 81 * mappings into page mappings may cause TLB conflicts. This means 82 * we have to ensure that changing the permission bits of the range 83 * we are operating on does not result in such splitting. 84 * 85 * Let's restrict ourselves to mappings created by vmalloc (or vmap). 86 * Those are guaranteed to consist entirely of page mappings, and 87 * splitting is never needed. 88 * 89 * So check whether the [addr, addr + size) interval is entirely 90 * covered by precisely one VM area that has the VM_ALLOC flag set. 91 */ 92 area = find_vm_area((void *)addr); 93 if (!area || 94 end > (unsigned long)kasan_reset_tag(area->addr) + area->size || 95 !(area->flags & VM_ALLOC)) 96 return -EINVAL; 97 98 if (!numpages) 99 return 0; 100 101 /* 102 * If we are manipulating read-only permissions, apply the same 103 * change to the linear mapping of the pages that back this VM area. 104 */ 105 if (rodata_enabled && 106 rodata_full && (pgprot_val(set_mask) == PTE_RDONLY || 107 pgprot_val(clear_mask) == PTE_RDONLY)) { 108 for (i = 0; i < area->nr_pages; i++) { 109 __change_memory_common((u64)page_address(area->pages[i]), 110 PAGE_SIZE, set_mask, clear_mask); 111 } 112 } 113 114 /* 115 * Get rid of potentially aliasing lazily unmapped vm areas that may 116 * have permissions set that deviate from the ones we are setting here. 117 */ 118 vm_unmap_aliases(); 119 120 return __change_memory_common(start, size, set_mask, clear_mask); 121 } 122 123 int set_memory_ro(unsigned long addr, int numpages) 124 { 125 return change_memory_common(addr, numpages, 126 __pgprot(PTE_RDONLY), 127 __pgprot(PTE_WRITE)); 128 } 129 130 int set_memory_rw(unsigned long addr, int numpages) 131 { 132 return change_memory_common(addr, numpages, 133 __pgprot(PTE_WRITE), 134 __pgprot(PTE_RDONLY)); 135 } 136 137 int set_memory_nx(unsigned long addr, int numpages) 138 { 139 return change_memory_common(addr, numpages, 140 __pgprot(PTE_PXN), 141 __pgprot(PTE_MAYBE_GP)); 142 } 143 144 int set_memory_x(unsigned long addr, int numpages) 145 { 146 return change_memory_common(addr, numpages, 147 __pgprot(PTE_MAYBE_GP), 148 __pgprot(PTE_PXN)); 149 } 150 151 int set_memory_valid(unsigned long addr, int numpages, int enable) 152 { 153 if (enable) 154 return __change_memory_common(addr, PAGE_SIZE * numpages, 155 __pgprot(PTE_VALID), 156 __pgprot(0)); 157 else 158 return __change_memory_common(addr, PAGE_SIZE * numpages, 159 __pgprot(0), 160 __pgprot(PTE_VALID)); 161 } 162 163 int set_direct_map_invalid_noflush(struct page *page) 164 { 165 struct page_change_data data = { 166 .set_mask = __pgprot(0), 167 .clear_mask = __pgprot(PTE_VALID), 168 }; 169 170 if (!can_set_direct_map()) 171 return 0; 172 173 return apply_to_page_range(&init_mm, 174 (unsigned long)page_address(page), 175 PAGE_SIZE, change_page_range, &data); 176 } 177 178 int set_direct_map_default_noflush(struct page *page) 179 { 180 struct page_change_data data = { 181 .set_mask = __pgprot(PTE_VALID | PTE_WRITE), 182 .clear_mask = __pgprot(PTE_RDONLY), 183 }; 184 185 if (!can_set_direct_map()) 186 return 0; 187 188 return apply_to_page_range(&init_mm, 189 (unsigned long)page_address(page), 190 PAGE_SIZE, change_page_range, &data); 191 } 192 193 #ifdef CONFIG_DEBUG_PAGEALLOC 194 void __kernel_map_pages(struct page *page, int numpages, int enable) 195 { 196 if (!can_set_direct_map()) 197 return; 198 199 set_memory_valid((unsigned long)page_address(page), numpages, enable); 200 } 201 #endif /* CONFIG_DEBUG_PAGEALLOC */ 202 203 /* 204 * This function is used to determine if a linear map page has been marked as 205 * not-valid. Walk the page table and check the PTE_VALID bit. 206 * 207 * Because this is only called on the kernel linear map, p?d_sect() implies 208 * p?d_present(). When debug_pagealloc is enabled, sections mappings are 209 * disabled. 210 */ 211 bool kernel_page_present(struct page *page) 212 { 213 pgd_t *pgdp; 214 p4d_t *p4dp; 215 pud_t *pudp, pud; 216 pmd_t *pmdp, pmd; 217 pte_t *ptep; 218 unsigned long addr = (unsigned long)page_address(page); 219 220 if (!can_set_direct_map()) 221 return true; 222 223 pgdp = pgd_offset_k(addr); 224 if (pgd_none(READ_ONCE(*pgdp))) 225 return false; 226 227 p4dp = p4d_offset(pgdp, addr); 228 if (p4d_none(READ_ONCE(*p4dp))) 229 return false; 230 231 pudp = pud_offset(p4dp, addr); 232 pud = READ_ONCE(*pudp); 233 if (pud_none(pud)) 234 return false; 235 if (pud_sect(pud)) 236 return true; 237 238 pmdp = pmd_offset(pudp, addr); 239 pmd = READ_ONCE(*pmdp); 240 if (pmd_none(pmd)) 241 return false; 242 if (pmd_sect(pmd)) 243 return true; 244 245 ptep = pte_offset_kernel(pmdp, addr); 246 return pte_valid(READ_ONCE(*ptep)); 247 } 248