1 // SPDX-License-Identifier: GPL-2.0+ 2 /* 3 * PowerPC Memory Protection Keys management 4 * 5 * Copyright 2017, Ram Pai, IBM Corporation. 6 */ 7 8 #include <asm/mman.h> 9 #include <asm/mmu_context.h> 10 #include <asm/mmu.h> 11 #include <asm/setup.h> 12 #include <asm/smp.h> 13 14 #include <linux/pkeys.h> 15 #include <linux/of_fdt.h> 16 17 18 int num_pkey; /* Max number of pkeys supported */ 19 /* 20 * Keys marked in the reservation list cannot be allocated by userspace 21 */ 22 u32 reserved_allocation_mask __ro_after_init; 23 24 /* Bits set for the initially allocated keys */ 25 static u32 initial_allocation_mask __ro_after_init; 26 27 /* 28 * Even if we allocate keys with sys_pkey_alloc(), we need to make sure 29 * other thread still find the access denied using the same keys. 30 */ 31 u64 default_amr __ro_after_init = ~0x0UL; 32 u64 default_iamr __ro_after_init = 0x5555555555555555UL; 33 u64 default_uamor __ro_after_init; 34 /* 35 * Key used to implement PROT_EXEC mmap. Denies READ/WRITE 36 * We pick key 2 because 0 is special key and 1 is reserved as per ISA. 37 */ 38 static int execute_only_key = 2; 39 static bool pkey_execute_disable_supported; 40 41 42 #define AMR_BITS_PER_PKEY 2 43 #define AMR_RD_BIT 0x1UL 44 #define AMR_WR_BIT 0x2UL 45 #define IAMR_EX_BIT 0x1UL 46 #define PKEY_REG_BITS (sizeof(u64) * 8) 47 #define pkeyshift(pkey) (PKEY_REG_BITS - ((pkey+1) * AMR_BITS_PER_PKEY)) 48 49 static int __init dt_scan_storage_keys(unsigned long node, 50 const char *uname, int depth, 51 void *data) 52 { 53 const char *type = of_get_flat_dt_prop(node, "device_type", NULL); 54 const __be32 *prop; 55 int *pkeys_total = (int *) data; 56 57 /* We are scanning "cpu" nodes only */ 58 if (type == NULL || strcmp(type, "cpu") != 0) 59 return 0; 60 61 prop = of_get_flat_dt_prop(node, "ibm,processor-storage-keys", NULL); 62 if (!prop) 63 return 0; 64 *pkeys_total = be32_to_cpu(prop[0]); 65 return 1; 66 } 67 68 static int scan_pkey_feature(void) 69 { 70 int ret; 71 int pkeys_total = 0; 72 73 /* 74 * Pkey is not supported with Radix translation. 75 */ 76 if (early_radix_enabled()) 77 return 0; 78 79 ret = of_scan_flat_dt(dt_scan_storage_keys, &pkeys_total); 80 if (ret == 0) { 81 /* 82 * Let's assume 32 pkeys on P8/P9 bare metal, if its not defined by device 83 * tree. We make this exception since some version of skiboot forgot to 84 * expose this property on power8/9. 85 */ 86 if (!firmware_has_feature(FW_FEATURE_LPAR)) { 87 unsigned long pvr = mfspr(SPRN_PVR); 88 89 if (PVR_VER(pvr) == PVR_POWER8 || PVR_VER(pvr) == PVR_POWER8E || 90 PVR_VER(pvr) == PVR_POWER8NVL || PVR_VER(pvr) == PVR_POWER9) 91 pkeys_total = 32; 92 } 93 } 94 95 #ifdef CONFIG_PPC_MEM_KEYS 96 /* 97 * Adjust the upper limit, based on the number of bits supported by 98 * arch-neutral code. 99 */ 100 pkeys_total = min_t(int, pkeys_total, 101 ((ARCH_VM_PKEY_FLAGS >> VM_PKEY_SHIFT) + 1)); 102 #endif 103 return pkeys_total; 104 } 105 106 void __init pkey_early_init_devtree(void) 107 { 108 int pkeys_total, i; 109 110 #ifdef CONFIG_PPC_MEM_KEYS 111 /* 112 * We define PKEY_DISABLE_EXECUTE in addition to the arch-neutral 113 * generic defines for PKEY_DISABLE_ACCESS and PKEY_DISABLE_WRITE. 114 * Ensure that the bits a distinct. 115 */ 116 BUILD_BUG_ON(PKEY_DISABLE_EXECUTE & 117 (PKEY_DISABLE_ACCESS | PKEY_DISABLE_WRITE)); 118 119 /* 120 * pkey_to_vmflag_bits() assumes that the pkey bits are contiguous 121 * in the vmaflag. Make sure that is really the case. 122 */ 123 BUILD_BUG_ON(__builtin_clzl(ARCH_VM_PKEY_FLAGS >> VM_PKEY_SHIFT) + 124 __builtin_popcountl(ARCH_VM_PKEY_FLAGS >> VM_PKEY_SHIFT) 125 != (sizeof(u64) * BITS_PER_BYTE)); 126 #endif 127 /* 128 * Only P7 and above supports SPRN_AMR update with MSR[PR] = 1 129 */ 130 if (!early_cpu_has_feature(CPU_FTR_ARCH_206)) 131 return; 132 133 /* scan the device tree for pkey feature */ 134 pkeys_total = scan_pkey_feature(); 135 if (!pkeys_total) 136 goto out; 137 138 /* Allow all keys to be modified by default */ 139 default_uamor = ~0x0UL; 140 141 cur_cpu_spec->mmu_features |= MMU_FTR_PKEY; 142 143 /* 144 * The device tree cannot be relied to indicate support for 145 * execute_disable support. Instead we use a PVR check. 146 */ 147 if (pvr_version_is(PVR_POWER7) || pvr_version_is(PVR_POWER7p)) 148 pkey_execute_disable_supported = false; 149 else 150 pkey_execute_disable_supported = true; 151 152 #ifdef CONFIG_PPC_4K_PAGES 153 /* 154 * The OS can manage only 8 pkeys due to its inability to represent them 155 * in the Linux 4K PTE. Mark all other keys reserved. 156 */ 157 num_pkey = min(8, pkeys_total); 158 #else 159 num_pkey = pkeys_total; 160 #endif 161 162 if (unlikely(num_pkey <= execute_only_key) || !pkey_execute_disable_supported) { 163 /* 164 * Insufficient number of keys to support 165 * execute only key. Mark it unavailable. 166 */ 167 execute_only_key = -1; 168 } else { 169 /* 170 * Mark the execute_only_pkey as not available for 171 * user allocation via pkey_alloc. 172 */ 173 reserved_allocation_mask |= (0x1 << execute_only_key); 174 175 /* 176 * Deny READ/WRITE for execute_only_key. 177 * Allow execute in IAMR. 178 */ 179 default_amr |= (0x3ul << pkeyshift(execute_only_key)); 180 default_iamr &= ~(0x1ul << pkeyshift(execute_only_key)); 181 182 /* 183 * Clear the uamor bits for this key. 184 */ 185 default_uamor &= ~(0x3ul << pkeyshift(execute_only_key)); 186 } 187 188 if (unlikely(num_pkey <= 3)) { 189 /* 190 * Insufficient number of keys to support 191 * KUAP/KUEP feature. 192 */ 193 disable_kuep = true; 194 disable_kuap = true; 195 WARN(1, "Disabling kernel user protection due to low (%d) max supported keys\n", num_pkey); 196 } else { 197 /* handle key which is used by kernel for KAUP */ 198 reserved_allocation_mask |= (0x1 << 3); 199 /* 200 * Mark access for kup_key in default amr so that 201 * we continue to operate with that AMR in 202 * copy_to/from_user(). 203 */ 204 default_amr &= ~(0x3ul << pkeyshift(3)); 205 default_iamr &= ~(0x1ul << pkeyshift(3)); 206 default_uamor &= ~(0x3ul << pkeyshift(3)); 207 } 208 209 /* 210 * Allow access for only key 0. And prevent any other modification. 211 */ 212 default_amr &= ~(0x3ul << pkeyshift(0)); 213 default_iamr &= ~(0x1ul << pkeyshift(0)); 214 default_uamor &= ~(0x3ul << pkeyshift(0)); 215 /* 216 * key 0 is special in that we want to consider it an allocated 217 * key which is preallocated. We don't allow changing AMR bits 218 * w.r.t key 0. But one can pkey_free(key0) 219 */ 220 initial_allocation_mask |= (0x1 << 0); 221 222 /* 223 * key 1 is recommended not to be used. PowerISA(3.0) page 1015, 224 * programming note. 225 */ 226 reserved_allocation_mask |= (0x1 << 1); 227 default_uamor &= ~(0x3ul << pkeyshift(1)); 228 229 /* 230 * Prevent the usage of OS reserved keys. Update UAMOR 231 * for those keys. Also mark the rest of the bits in the 232 * 32 bit mask as reserved. 233 */ 234 for (i = num_pkey; i < 32 ; i++) { 235 reserved_allocation_mask |= (0x1 << i); 236 default_uamor &= ~(0x3ul << pkeyshift(i)); 237 } 238 /* 239 * Prevent the allocation of reserved keys too. 240 */ 241 initial_allocation_mask |= reserved_allocation_mask; 242 243 pr_info("Enabling pkeys with max key count %d\n", num_pkey); 244 out: 245 /* 246 * Setup uamor on boot cpu 247 */ 248 mtspr(SPRN_UAMOR, default_uamor); 249 250 return; 251 } 252 253 #ifdef CONFIG_PPC_KUEP 254 void setup_kuep(bool disabled) 255 { 256 if (disabled) 257 return; 258 /* 259 * On hash if PKEY feature is not enabled, disable KUAP too. 260 */ 261 if (!early_radix_enabled() && !early_mmu_has_feature(MMU_FTR_PKEY)) 262 return; 263 264 if (smp_processor_id() == boot_cpuid) { 265 pr_info("Activating Kernel Userspace Execution Prevention\n"); 266 cur_cpu_spec->mmu_features |= MMU_FTR_BOOK3S_KUEP; 267 } 268 269 /* 270 * Radix always uses key0 of the IAMR to determine if an access is 271 * allowed. We set bit 0 (IBM bit 1) of key0, to prevent instruction 272 * fetch. 273 */ 274 mtspr(SPRN_IAMR, AMR_KUEP_BLOCKED); 275 isync(); 276 } 277 #endif 278 279 #ifdef CONFIG_PPC_KUAP 280 void setup_kuap(bool disabled) 281 { 282 if (disabled) 283 return; 284 /* 285 * On hash if PKEY feature is not enabled, disable KUAP too. 286 */ 287 if (!early_radix_enabled() && !early_mmu_has_feature(MMU_FTR_PKEY)) 288 return; 289 290 if (smp_processor_id() == boot_cpuid) { 291 pr_info("Activating Kernel Userspace Access Prevention\n"); 292 cur_cpu_spec->mmu_features |= MMU_FTR_BOOK3S_KUAP; 293 } 294 295 /* 296 * Set the default kernel AMR values on all cpus. 297 */ 298 mtspr(SPRN_AMR, AMR_KUAP_BLOCKED); 299 isync(); 300 } 301 #endif 302 303 static inline void update_current_thread_amr(u64 value) 304 { 305 current->thread.regs->amr = value; 306 } 307 308 static inline void update_current_thread_iamr(u64 value) 309 { 310 if (!likely(pkey_execute_disable_supported)) 311 return; 312 313 current->thread.regs->iamr = value; 314 } 315 316 #ifdef CONFIG_PPC_MEM_KEYS 317 void pkey_mm_init(struct mm_struct *mm) 318 { 319 if (!mmu_has_feature(MMU_FTR_PKEY)) 320 return; 321 mm_pkey_allocation_map(mm) = initial_allocation_mask; 322 mm->context.execute_only_pkey = execute_only_key; 323 } 324 325 static inline void init_amr(int pkey, u8 init_bits) 326 { 327 u64 new_amr_bits = (((u64)init_bits & 0x3UL) << pkeyshift(pkey)); 328 u64 old_amr = current_thread_amr() & ~((u64)(0x3ul) << pkeyshift(pkey)); 329 330 update_current_thread_amr(old_amr | new_amr_bits); 331 } 332 333 static inline void init_iamr(int pkey, u8 init_bits) 334 { 335 u64 new_iamr_bits = (((u64)init_bits & 0x1UL) << pkeyshift(pkey)); 336 u64 old_iamr = current_thread_iamr() & ~((u64)(0x1ul) << pkeyshift(pkey)); 337 338 update_current_thread_iamr(old_iamr | new_iamr_bits); 339 } 340 341 /* 342 * Set the access rights in AMR IAMR and UAMOR registers for @pkey to that 343 * specified in @init_val. 344 */ 345 int __arch_set_user_pkey_access(struct task_struct *tsk, int pkey, 346 unsigned long init_val) 347 { 348 u64 new_amr_bits = 0x0ul; 349 u64 new_iamr_bits = 0x0ul; 350 u64 pkey_bits, uamor_pkey_bits; 351 352 /* 353 * Check whether the key is disabled by UAMOR. 354 */ 355 pkey_bits = 0x3ul << pkeyshift(pkey); 356 uamor_pkey_bits = (default_uamor & pkey_bits); 357 358 /* 359 * Both the bits in UAMOR corresponding to the key should be set 360 */ 361 if (uamor_pkey_bits != pkey_bits) 362 return -EINVAL; 363 364 if (init_val & PKEY_DISABLE_EXECUTE) { 365 if (!pkey_execute_disable_supported) 366 return -EINVAL; 367 new_iamr_bits |= IAMR_EX_BIT; 368 } 369 init_iamr(pkey, new_iamr_bits); 370 371 /* Set the bits we need in AMR: */ 372 if (init_val & PKEY_DISABLE_ACCESS) 373 new_amr_bits |= AMR_RD_BIT | AMR_WR_BIT; 374 else if (init_val & PKEY_DISABLE_WRITE) 375 new_amr_bits |= AMR_WR_BIT; 376 377 init_amr(pkey, new_amr_bits); 378 return 0; 379 } 380 381 int execute_only_pkey(struct mm_struct *mm) 382 { 383 return mm->context.execute_only_pkey; 384 } 385 386 static inline bool vma_is_pkey_exec_only(struct vm_area_struct *vma) 387 { 388 /* Do this check first since the vm_flags should be hot */ 389 if ((vma->vm_flags & VM_ACCESS_FLAGS) != VM_EXEC) 390 return false; 391 392 return (vma_pkey(vma) == vma->vm_mm->context.execute_only_pkey); 393 } 394 395 /* 396 * This should only be called for *plain* mprotect calls. 397 */ 398 int __arch_override_mprotect_pkey(struct vm_area_struct *vma, int prot, 399 int pkey) 400 { 401 /* 402 * If the currently associated pkey is execute-only, but the requested 403 * protection is not execute-only, move it back to the default pkey. 404 */ 405 if (vma_is_pkey_exec_only(vma) && (prot != PROT_EXEC)) 406 return 0; 407 408 /* 409 * The requested protection is execute-only. Hence let's use an 410 * execute-only pkey. 411 */ 412 if (prot == PROT_EXEC) { 413 pkey = execute_only_pkey(vma->vm_mm); 414 if (pkey > 0) 415 return pkey; 416 } 417 418 /* Nothing to override. */ 419 return vma_pkey(vma); 420 } 421 422 static bool pkey_access_permitted(int pkey, bool write, bool execute) 423 { 424 int pkey_shift; 425 u64 amr; 426 427 pkey_shift = pkeyshift(pkey); 428 if (execute) 429 return !(current_thread_iamr() & (IAMR_EX_BIT << pkey_shift)); 430 431 amr = current_thread_amr(); 432 if (write) 433 return !(amr & (AMR_WR_BIT << pkey_shift)); 434 435 return !(amr & (AMR_RD_BIT << pkey_shift)); 436 } 437 438 bool arch_pte_access_permitted(u64 pte, bool write, bool execute) 439 { 440 if (!mmu_has_feature(MMU_FTR_PKEY)) 441 return true; 442 443 return pkey_access_permitted(pte_to_pkey_bits(pte), write, execute); 444 } 445 446 /* 447 * We only want to enforce protection keys on the current thread because we 448 * effectively have no access to AMR/IAMR for other threads or any way to tell 449 * which AMR/IAMR in a threaded process we could use. 450 * 451 * So do not enforce things if the VMA is not from the current mm, or if we are 452 * in a kernel thread. 453 */ 454 bool arch_vma_access_permitted(struct vm_area_struct *vma, bool write, 455 bool execute, bool foreign) 456 { 457 if (!mmu_has_feature(MMU_FTR_PKEY)) 458 return true; 459 /* 460 * Do not enforce our key-permissions on a foreign vma. 461 */ 462 if (foreign || vma_is_foreign(vma)) 463 return true; 464 465 return pkey_access_permitted(vma_pkey(vma), write, execute); 466 } 467 468 void arch_dup_pkeys(struct mm_struct *oldmm, struct mm_struct *mm) 469 { 470 if (!mmu_has_feature(MMU_FTR_PKEY)) 471 return; 472 473 /* Duplicate the oldmm pkey state in mm: */ 474 mm_pkey_allocation_map(mm) = mm_pkey_allocation_map(oldmm); 475 mm->context.execute_only_pkey = oldmm->context.execute_only_pkey; 476 } 477 478 #endif /* CONFIG_PPC_MEM_KEYS */ 479