1 // SPDX-License-Identifier: GPL-2.0+ 2 /* 3 * PowerPC Memory Protection Keys management 4 * 5 * Copyright 2017, Ram Pai, IBM Corporation. 6 */ 7 8 #include <asm/mman.h> 9 #include <asm/mmu_context.h> 10 #include <asm/mmu.h> 11 #include <asm/setup.h> 12 #include <linux/pkeys.h> 13 #include <linux/of_fdt.h> 14 15 int num_pkey; /* Max number of pkeys supported */ 16 /* 17 * Keys marked in the reservation list cannot be allocated by userspace 18 */ 19 u32 reserved_allocation_mask __ro_after_init; 20 21 /* Bits set for the initially allocated keys */ 22 static u32 initial_allocation_mask __ro_after_init; 23 24 /* 25 * Even if we allocate keys with sys_pkey_alloc(), we need to make sure 26 * other thread still find the access denied using the same keys. 27 */ 28 static u64 default_amr = ~0x0UL; 29 static u64 default_iamr = 0x5555555555555555UL; 30 u64 default_uamor __ro_after_init; 31 /* 32 * Key used to implement PROT_EXEC mmap. Denies READ/WRITE 33 * We pick key 2 because 0 is special key and 1 is reserved as per ISA. 34 */ 35 static int execute_only_key = 2; 36 static bool pkey_execute_disable_supported; 37 38 39 #define AMR_BITS_PER_PKEY 2 40 #define AMR_RD_BIT 0x1UL 41 #define AMR_WR_BIT 0x2UL 42 #define IAMR_EX_BIT 0x1UL 43 #define PKEY_REG_BITS (sizeof(u64) * 8) 44 #define pkeyshift(pkey) (PKEY_REG_BITS - ((pkey+1) * AMR_BITS_PER_PKEY)) 45 46 static int __init dt_scan_storage_keys(unsigned long node, 47 const char *uname, int depth, 48 void *data) 49 { 50 const char *type = of_get_flat_dt_prop(node, "device_type", NULL); 51 const __be32 *prop; 52 int *pkeys_total = (int *) data; 53 54 /* We are scanning "cpu" nodes only */ 55 if (type == NULL || strcmp(type, "cpu") != 0) 56 return 0; 57 58 prop = of_get_flat_dt_prop(node, "ibm,processor-storage-keys", NULL); 59 if (!prop) 60 return 0; 61 *pkeys_total = be32_to_cpu(prop[0]); 62 return 1; 63 } 64 65 static int scan_pkey_feature(void) 66 { 67 int ret; 68 int pkeys_total = 0; 69 70 /* 71 * Pkey is not supported with Radix translation. 72 */ 73 if (early_radix_enabled()) 74 return 0; 75 76 ret = of_scan_flat_dt(dt_scan_storage_keys, &pkeys_total); 77 if (ret == 0) { 78 /* 79 * Let's assume 32 pkeys on P8/P9 bare metal, if its not defined by device 80 * tree. We make this exception since some version of skiboot forgot to 81 * expose this property on power8/9. 82 */ 83 if (!firmware_has_feature(FW_FEATURE_LPAR)) { 84 unsigned long pvr = mfspr(SPRN_PVR); 85 86 if (PVR_VER(pvr) == PVR_POWER8 || PVR_VER(pvr) == PVR_POWER8E || 87 PVR_VER(pvr) == PVR_POWER8NVL || PVR_VER(pvr) == PVR_POWER9) 88 pkeys_total = 32; 89 } 90 } 91 92 /* 93 * Adjust the upper limit, based on the number of bits supported by 94 * arch-neutral code. 95 */ 96 pkeys_total = min_t(int, pkeys_total, 97 ((ARCH_VM_PKEY_FLAGS >> VM_PKEY_SHIFT) + 1)); 98 return pkeys_total; 99 } 100 101 void __init pkey_early_init_devtree(void) 102 { 103 int pkeys_total, i; 104 105 /* 106 * We define PKEY_DISABLE_EXECUTE in addition to the arch-neutral 107 * generic defines for PKEY_DISABLE_ACCESS and PKEY_DISABLE_WRITE. 108 * Ensure that the bits a distinct. 109 */ 110 BUILD_BUG_ON(PKEY_DISABLE_EXECUTE & 111 (PKEY_DISABLE_ACCESS | PKEY_DISABLE_WRITE)); 112 113 /* 114 * pkey_to_vmflag_bits() assumes that the pkey bits are contiguous 115 * in the vmaflag. Make sure that is really the case. 116 */ 117 BUILD_BUG_ON(__builtin_clzl(ARCH_VM_PKEY_FLAGS >> VM_PKEY_SHIFT) + 118 __builtin_popcountl(ARCH_VM_PKEY_FLAGS >> VM_PKEY_SHIFT) 119 != (sizeof(u64) * BITS_PER_BYTE)); 120 121 /* scan the device tree for pkey feature */ 122 pkeys_total = scan_pkey_feature(); 123 if (!pkeys_total) 124 goto out; 125 126 /* Allow all keys to be modified by default */ 127 default_uamor = ~0x0UL; 128 129 cur_cpu_spec->mmu_features |= MMU_FTR_PKEY; 130 131 /* 132 * The device tree cannot be relied to indicate support for 133 * execute_disable support. Instead we use a PVR check. 134 */ 135 if (pvr_version_is(PVR_POWER7) || pvr_version_is(PVR_POWER7p)) 136 pkey_execute_disable_supported = false; 137 else 138 pkey_execute_disable_supported = true; 139 140 #ifdef CONFIG_PPC_4K_PAGES 141 /* 142 * The OS can manage only 8 pkeys due to its inability to represent them 143 * in the Linux 4K PTE. Mark all other keys reserved. 144 */ 145 num_pkey = min(8, pkeys_total); 146 #else 147 num_pkey = pkeys_total; 148 #endif 149 150 if (unlikely(num_pkey <= execute_only_key) || !pkey_execute_disable_supported) { 151 /* 152 * Insufficient number of keys to support 153 * execute only key. Mark it unavailable. 154 */ 155 execute_only_key = -1; 156 } else { 157 /* 158 * Mark the execute_only_pkey as not available for 159 * user allocation via pkey_alloc. 160 */ 161 reserved_allocation_mask |= (0x1 << execute_only_key); 162 163 /* 164 * Deny READ/WRITE for execute_only_key. 165 * Allow execute in IAMR. 166 */ 167 default_amr |= (0x3ul << pkeyshift(execute_only_key)); 168 default_iamr &= ~(0x1ul << pkeyshift(execute_only_key)); 169 170 /* 171 * Clear the uamor bits for this key. 172 */ 173 default_uamor &= ~(0x3ul << pkeyshift(execute_only_key)); 174 } 175 176 /* 177 * Allow access for only key 0. And prevent any other modification. 178 */ 179 default_amr &= ~(0x3ul << pkeyshift(0)); 180 default_iamr &= ~(0x1ul << pkeyshift(0)); 181 default_uamor &= ~(0x3ul << pkeyshift(0)); 182 /* 183 * key 0 is special in that we want to consider it an allocated 184 * key which is preallocated. We don't allow changing AMR bits 185 * w.r.t key 0. But one can pkey_free(key0) 186 */ 187 initial_allocation_mask |= (0x1 << 0); 188 189 /* 190 * key 1 is recommended not to be used. PowerISA(3.0) page 1015, 191 * programming note. 192 */ 193 reserved_allocation_mask |= (0x1 << 1); 194 default_uamor &= ~(0x3ul << pkeyshift(1)); 195 196 /* 197 * Prevent the usage of OS reserved keys. Update UAMOR 198 * for those keys. Also mark the rest of the bits in the 199 * 32 bit mask as reserved. 200 */ 201 for (i = num_pkey; i < 32 ; i++) { 202 reserved_allocation_mask |= (0x1 << i); 203 default_uamor &= ~(0x3ul << pkeyshift(i)); 204 } 205 /* 206 * Prevent the allocation of reserved keys too. 207 */ 208 initial_allocation_mask |= reserved_allocation_mask; 209 210 pr_info("Enabling pkeys with max key count %d\n", num_pkey); 211 out: 212 /* 213 * Setup uamor on boot cpu 214 */ 215 mtspr(SPRN_UAMOR, default_uamor); 216 217 return; 218 } 219 220 void pkey_mm_init(struct mm_struct *mm) 221 { 222 if (!mmu_has_feature(MMU_FTR_PKEY)) 223 return; 224 mm_pkey_allocation_map(mm) = initial_allocation_mask; 225 mm->context.execute_only_pkey = execute_only_key; 226 } 227 228 static inline u64 read_amr(void) 229 { 230 return mfspr(SPRN_AMR); 231 } 232 233 static inline void write_amr(u64 value) 234 { 235 mtspr(SPRN_AMR, value); 236 } 237 238 static inline u64 read_iamr(void) 239 { 240 if (!likely(pkey_execute_disable_supported)) 241 return 0x0UL; 242 243 return mfspr(SPRN_IAMR); 244 } 245 246 static inline void write_iamr(u64 value) 247 { 248 if (!likely(pkey_execute_disable_supported)) 249 return; 250 251 mtspr(SPRN_IAMR, value); 252 } 253 254 static inline void init_amr(int pkey, u8 init_bits) 255 { 256 u64 new_amr_bits = (((u64)init_bits & 0x3UL) << pkeyshift(pkey)); 257 u64 old_amr = read_amr() & ~((u64)(0x3ul) << pkeyshift(pkey)); 258 259 write_amr(old_amr | new_amr_bits); 260 } 261 262 static inline void init_iamr(int pkey, u8 init_bits) 263 { 264 u64 new_iamr_bits = (((u64)init_bits & 0x1UL) << pkeyshift(pkey)); 265 u64 old_iamr = read_iamr() & ~((u64)(0x1ul) << pkeyshift(pkey)); 266 267 write_iamr(old_iamr | new_iamr_bits); 268 } 269 270 /* 271 * Set the access rights in AMR IAMR and UAMOR registers for @pkey to that 272 * specified in @init_val. 273 */ 274 int __arch_set_user_pkey_access(struct task_struct *tsk, int pkey, 275 unsigned long init_val) 276 { 277 u64 new_amr_bits = 0x0ul; 278 u64 new_iamr_bits = 0x0ul; 279 u64 pkey_bits, uamor_pkey_bits; 280 281 /* 282 * Check whether the key is disabled by UAMOR. 283 */ 284 pkey_bits = 0x3ul << pkeyshift(pkey); 285 uamor_pkey_bits = (default_uamor & pkey_bits); 286 287 /* 288 * Both the bits in UAMOR corresponding to the key should be set 289 */ 290 if (uamor_pkey_bits != pkey_bits) 291 return -EINVAL; 292 293 if (init_val & PKEY_DISABLE_EXECUTE) { 294 if (!pkey_execute_disable_supported) 295 return -EINVAL; 296 new_iamr_bits |= IAMR_EX_BIT; 297 } 298 init_iamr(pkey, new_iamr_bits); 299 300 /* Set the bits we need in AMR: */ 301 if (init_val & PKEY_DISABLE_ACCESS) 302 new_amr_bits |= AMR_RD_BIT | AMR_WR_BIT; 303 else if (init_val & PKEY_DISABLE_WRITE) 304 new_amr_bits |= AMR_WR_BIT; 305 306 init_amr(pkey, new_amr_bits); 307 return 0; 308 } 309 310 void thread_pkey_regs_save(struct thread_struct *thread) 311 { 312 if (!mmu_has_feature(MMU_FTR_PKEY)) 313 return; 314 315 /* 316 * TODO: Skip saving registers if @thread hasn't used any keys yet. 317 */ 318 thread->amr = read_amr(); 319 thread->iamr = read_iamr(); 320 } 321 322 void thread_pkey_regs_restore(struct thread_struct *new_thread, 323 struct thread_struct *old_thread) 324 { 325 if (!mmu_has_feature(MMU_FTR_PKEY)) 326 return; 327 328 if (old_thread->amr != new_thread->amr) 329 write_amr(new_thread->amr); 330 if (old_thread->iamr != new_thread->iamr) 331 write_iamr(new_thread->iamr); 332 } 333 334 void thread_pkey_regs_init(struct thread_struct *thread) 335 { 336 if (!mmu_has_feature(MMU_FTR_PKEY)) 337 return; 338 339 thread->amr = default_amr; 340 thread->iamr = default_iamr; 341 342 write_amr(default_amr); 343 write_iamr(default_iamr); 344 } 345 346 int execute_only_pkey(struct mm_struct *mm) 347 { 348 return mm->context.execute_only_pkey; 349 } 350 351 static inline bool vma_is_pkey_exec_only(struct vm_area_struct *vma) 352 { 353 /* Do this check first since the vm_flags should be hot */ 354 if ((vma->vm_flags & VM_ACCESS_FLAGS) != VM_EXEC) 355 return false; 356 357 return (vma_pkey(vma) == vma->vm_mm->context.execute_only_pkey); 358 } 359 360 /* 361 * This should only be called for *plain* mprotect calls. 362 */ 363 int __arch_override_mprotect_pkey(struct vm_area_struct *vma, int prot, 364 int pkey) 365 { 366 /* 367 * If the currently associated pkey is execute-only, but the requested 368 * protection is not execute-only, move it back to the default pkey. 369 */ 370 if (vma_is_pkey_exec_only(vma) && (prot != PROT_EXEC)) 371 return 0; 372 373 /* 374 * The requested protection is execute-only. Hence let's use an 375 * execute-only pkey. 376 */ 377 if (prot == PROT_EXEC) { 378 pkey = execute_only_pkey(vma->vm_mm); 379 if (pkey > 0) 380 return pkey; 381 } 382 383 /* Nothing to override. */ 384 return vma_pkey(vma); 385 } 386 387 static bool pkey_access_permitted(int pkey, bool write, bool execute) 388 { 389 int pkey_shift; 390 u64 amr; 391 392 pkey_shift = pkeyshift(pkey); 393 if (execute) 394 return !(read_iamr() & (IAMR_EX_BIT << pkey_shift)); 395 396 amr = read_amr(); 397 if (write) 398 return !(amr & (AMR_WR_BIT << pkey_shift)); 399 400 return !(amr & (AMR_RD_BIT << pkey_shift)); 401 } 402 403 bool arch_pte_access_permitted(u64 pte, bool write, bool execute) 404 { 405 if (!mmu_has_feature(MMU_FTR_PKEY)) 406 return true; 407 408 return pkey_access_permitted(pte_to_pkey_bits(pte), write, execute); 409 } 410 411 /* 412 * We only want to enforce protection keys on the current thread because we 413 * effectively have no access to AMR/IAMR for other threads or any way to tell 414 * which AMR/IAMR in a threaded process we could use. 415 * 416 * So do not enforce things if the VMA is not from the current mm, or if we are 417 * in a kernel thread. 418 */ 419 bool arch_vma_access_permitted(struct vm_area_struct *vma, bool write, 420 bool execute, bool foreign) 421 { 422 if (!mmu_has_feature(MMU_FTR_PKEY)) 423 return true; 424 /* 425 * Do not enforce our key-permissions on a foreign vma. 426 */ 427 if (foreign || vma_is_foreign(vma)) 428 return true; 429 430 return pkey_access_permitted(vma_pkey(vma), write, execute); 431 } 432 433 void arch_dup_pkeys(struct mm_struct *oldmm, struct mm_struct *mm) 434 { 435 if (!mmu_has_feature(MMU_FTR_PKEY)) 436 return; 437 438 /* Duplicate the oldmm pkey state in mm: */ 439 mm_pkey_allocation_map(mm) = mm_pkey_allocation_map(oldmm); 440 mm->context.execute_only_pkey = oldmm->context.execute_only_pkey; 441 } 442