1 // SPDX-License-Identifier: GPL-2.0+ 2 /* 3 * PowerPC Memory Protection Keys management 4 * 5 * Copyright 2017, Ram Pai, IBM Corporation. 6 */ 7 8 #include <asm/mman.h> 9 #include <asm/mmu_context.h> 10 #include <asm/mmu.h> 11 #include <asm/setup.h> 12 #include <asm/smp.h> 13 14 #include <linux/pkeys.h> 15 #include <linux/of_fdt.h> 16 17 18 int num_pkey; /* Max number of pkeys supported */ 19 /* 20 * Keys marked in the reservation list cannot be allocated by userspace 21 */ 22 u32 reserved_allocation_mask __ro_after_init; 23 24 /* Bits set for the initially allocated keys */ 25 static u32 initial_allocation_mask __ro_after_init; 26 27 /* 28 * Even if we allocate keys with sys_pkey_alloc(), we need to make sure 29 * other thread still find the access denied using the same keys. 30 */ 31 u64 default_amr __ro_after_init = ~0x0UL; 32 u64 default_iamr __ro_after_init = 0x5555555555555555UL; 33 u64 default_uamor __ro_after_init; 34 EXPORT_SYMBOL(default_amr); 35 /* 36 * Key used to implement PROT_EXEC mmap. Denies READ/WRITE 37 * We pick key 2 because 0 is special key and 1 is reserved as per ISA. 38 */ 39 static int execute_only_key = 2; 40 static bool pkey_execute_disable_supported; 41 42 43 #define AMR_BITS_PER_PKEY 2 44 #define AMR_RD_BIT 0x1UL 45 #define AMR_WR_BIT 0x2UL 46 #define IAMR_EX_BIT 0x1UL 47 #define PKEY_REG_BITS (sizeof(u64) * 8) 48 #define pkeyshift(pkey) (PKEY_REG_BITS - ((pkey+1) * AMR_BITS_PER_PKEY)) 49 50 static int __init dt_scan_storage_keys(unsigned long node, 51 const char *uname, int depth, 52 void *data) 53 { 54 const char *type = of_get_flat_dt_prop(node, "device_type", NULL); 55 const __be32 *prop; 56 int *pkeys_total = (int *) data; 57 58 /* We are scanning "cpu" nodes only */ 59 if (type == NULL || strcmp(type, "cpu") != 0) 60 return 0; 61 62 prop = of_get_flat_dt_prop(node, "ibm,processor-storage-keys", NULL); 63 if (!prop) 64 return 0; 65 *pkeys_total = be32_to_cpu(prop[0]); 66 return 1; 67 } 68 69 static int scan_pkey_feature(void) 70 { 71 int ret; 72 int pkeys_total = 0; 73 74 /* 75 * Pkey is not supported with Radix translation. 76 */ 77 if (early_radix_enabled()) 78 return 0; 79 80 ret = of_scan_flat_dt(dt_scan_storage_keys, &pkeys_total); 81 if (ret == 0) { 82 /* 83 * Let's assume 32 pkeys on P8/P9 bare metal, if its not defined by device 84 * tree. We make this exception since some version of skiboot forgot to 85 * expose this property on power8/9. 86 */ 87 if (!firmware_has_feature(FW_FEATURE_LPAR)) { 88 unsigned long pvr = mfspr(SPRN_PVR); 89 90 if (PVR_VER(pvr) == PVR_POWER8 || PVR_VER(pvr) == PVR_POWER8E || 91 PVR_VER(pvr) == PVR_POWER8NVL || PVR_VER(pvr) == PVR_POWER9) 92 pkeys_total = 32; 93 } 94 } 95 96 #ifdef CONFIG_PPC_MEM_KEYS 97 /* 98 * Adjust the upper limit, based on the number of bits supported by 99 * arch-neutral code. 100 */ 101 pkeys_total = min_t(int, pkeys_total, 102 ((ARCH_VM_PKEY_FLAGS >> VM_PKEY_SHIFT) + 1)); 103 #endif 104 return pkeys_total; 105 } 106 107 void __init pkey_early_init_devtree(void) 108 { 109 int pkeys_total, i; 110 111 #ifdef CONFIG_PPC_MEM_KEYS 112 /* 113 * We define PKEY_DISABLE_EXECUTE in addition to the arch-neutral 114 * generic defines for PKEY_DISABLE_ACCESS and PKEY_DISABLE_WRITE. 115 * Ensure that the bits a distinct. 116 */ 117 BUILD_BUG_ON(PKEY_DISABLE_EXECUTE & 118 (PKEY_DISABLE_ACCESS | PKEY_DISABLE_WRITE)); 119 120 /* 121 * pkey_to_vmflag_bits() assumes that the pkey bits are contiguous 122 * in the vmaflag. Make sure that is really the case. 123 */ 124 BUILD_BUG_ON(__builtin_clzl(ARCH_VM_PKEY_FLAGS >> VM_PKEY_SHIFT) + 125 __builtin_popcountl(ARCH_VM_PKEY_FLAGS >> VM_PKEY_SHIFT) 126 != (sizeof(u64) * BITS_PER_BYTE)); 127 #endif 128 /* 129 * Only P7 and above supports SPRN_AMR update with MSR[PR] = 1 130 */ 131 if (!early_cpu_has_feature(CPU_FTR_ARCH_206)) 132 return; 133 134 /* scan the device tree for pkey feature */ 135 pkeys_total = scan_pkey_feature(); 136 if (!pkeys_total) 137 goto out; 138 139 /* Allow all keys to be modified by default */ 140 default_uamor = ~0x0UL; 141 142 cur_cpu_spec->mmu_features |= MMU_FTR_PKEY; 143 144 /* 145 * The device tree cannot be relied to indicate support for 146 * execute_disable support. Instead we use a PVR check. 147 */ 148 if (pvr_version_is(PVR_POWER7) || pvr_version_is(PVR_POWER7p)) 149 pkey_execute_disable_supported = false; 150 else 151 pkey_execute_disable_supported = true; 152 153 #ifdef CONFIG_PPC_4K_PAGES 154 /* 155 * The OS can manage only 8 pkeys due to its inability to represent them 156 * in the Linux 4K PTE. Mark all other keys reserved. 157 */ 158 num_pkey = min(8, pkeys_total); 159 #else 160 num_pkey = pkeys_total; 161 #endif 162 163 if (unlikely(num_pkey <= execute_only_key) || !pkey_execute_disable_supported) { 164 /* 165 * Insufficient number of keys to support 166 * execute only key. Mark it unavailable. 167 */ 168 execute_only_key = -1; 169 } else { 170 /* 171 * Mark the execute_only_pkey as not available for 172 * user allocation via pkey_alloc. 173 */ 174 reserved_allocation_mask |= (0x1 << execute_only_key); 175 176 /* 177 * Deny READ/WRITE for execute_only_key. 178 * Allow execute in IAMR. 179 */ 180 default_amr |= (0x3ul << pkeyshift(execute_only_key)); 181 default_iamr &= ~(0x1ul << pkeyshift(execute_only_key)); 182 183 /* 184 * Clear the uamor bits for this key. 185 */ 186 default_uamor &= ~(0x3ul << pkeyshift(execute_only_key)); 187 } 188 189 if (unlikely(num_pkey <= 3)) { 190 /* 191 * Insufficient number of keys to support 192 * KUAP/KUEP feature. 193 */ 194 disable_kuep = true; 195 disable_kuap = true; 196 WARN(1, "Disabling kernel user protection due to low (%d) max supported keys\n", num_pkey); 197 } else { 198 /* handle key which is used by kernel for KAUP */ 199 reserved_allocation_mask |= (0x1 << 3); 200 /* 201 * Mark access for kup_key in default amr so that 202 * we continue to operate with that AMR in 203 * copy_to/from_user(). 204 */ 205 default_amr &= ~(0x3ul << pkeyshift(3)); 206 default_iamr &= ~(0x1ul << pkeyshift(3)); 207 default_uamor &= ~(0x3ul << pkeyshift(3)); 208 } 209 210 /* 211 * Allow access for only key 0. And prevent any other modification. 212 */ 213 default_amr &= ~(0x3ul << pkeyshift(0)); 214 default_iamr &= ~(0x1ul << pkeyshift(0)); 215 default_uamor &= ~(0x3ul << pkeyshift(0)); 216 /* 217 * key 0 is special in that we want to consider it an allocated 218 * key which is preallocated. We don't allow changing AMR bits 219 * w.r.t key 0. But one can pkey_free(key0) 220 */ 221 initial_allocation_mask |= (0x1 << 0); 222 223 /* 224 * key 1 is recommended not to be used. PowerISA(3.0) page 1015, 225 * programming note. 226 */ 227 reserved_allocation_mask |= (0x1 << 1); 228 default_uamor &= ~(0x3ul << pkeyshift(1)); 229 230 /* 231 * Prevent the usage of OS reserved keys. Update UAMOR 232 * for those keys. Also mark the rest of the bits in the 233 * 32 bit mask as reserved. 234 */ 235 for (i = num_pkey; i < 32 ; i++) { 236 reserved_allocation_mask |= (0x1 << i); 237 default_uamor &= ~(0x3ul << pkeyshift(i)); 238 } 239 /* 240 * Prevent the allocation of reserved keys too. 241 */ 242 initial_allocation_mask |= reserved_allocation_mask; 243 244 pr_info("Enabling pkeys with max key count %d\n", num_pkey); 245 out: 246 /* 247 * Setup uamor on boot cpu 248 */ 249 mtspr(SPRN_UAMOR, default_uamor); 250 251 return; 252 } 253 254 #ifdef CONFIG_PPC_KUEP 255 void setup_kuep(bool disabled) 256 { 257 if (disabled) 258 return; 259 /* 260 * On hash if PKEY feature is not enabled, disable KUAP too. 261 */ 262 if (!early_radix_enabled() && !early_mmu_has_feature(MMU_FTR_PKEY)) 263 return; 264 265 if (smp_processor_id() == boot_cpuid) { 266 pr_info("Activating Kernel Userspace Execution Prevention\n"); 267 cur_cpu_spec->mmu_features |= MMU_FTR_BOOK3S_KUEP; 268 } 269 270 /* 271 * Radix always uses key0 of the IAMR to determine if an access is 272 * allowed. We set bit 0 (IBM bit 1) of key0, to prevent instruction 273 * fetch. 274 */ 275 mtspr(SPRN_IAMR, AMR_KUEP_BLOCKED); 276 isync(); 277 } 278 #endif 279 280 #ifdef CONFIG_PPC_KUAP 281 void setup_kuap(bool disabled) 282 { 283 if (disabled) 284 return; 285 /* 286 * On hash if PKEY feature is not enabled, disable KUAP too. 287 */ 288 if (!early_radix_enabled() && !early_mmu_has_feature(MMU_FTR_PKEY)) 289 return; 290 291 if (smp_processor_id() == boot_cpuid) { 292 pr_info("Activating Kernel Userspace Access Prevention\n"); 293 cur_cpu_spec->mmu_features |= MMU_FTR_BOOK3S_KUAP; 294 } 295 296 /* 297 * Set the default kernel AMR values on all cpus. 298 */ 299 mtspr(SPRN_AMR, AMR_KUAP_BLOCKED); 300 isync(); 301 } 302 #endif 303 304 #ifdef CONFIG_PPC_MEM_KEYS 305 void pkey_mm_init(struct mm_struct *mm) 306 { 307 if (!mmu_has_feature(MMU_FTR_PKEY)) 308 return; 309 mm_pkey_allocation_map(mm) = initial_allocation_mask; 310 mm->context.execute_only_pkey = execute_only_key; 311 } 312 313 static inline void init_amr(int pkey, u8 init_bits) 314 { 315 u64 new_amr_bits = (((u64)init_bits & 0x3UL) << pkeyshift(pkey)); 316 u64 old_amr = current_thread_amr() & ~((u64)(0x3ul) << pkeyshift(pkey)); 317 318 current->thread.regs->amr = old_amr | new_amr_bits; 319 } 320 321 static inline void init_iamr(int pkey, u8 init_bits) 322 { 323 u64 new_iamr_bits = (((u64)init_bits & 0x1UL) << pkeyshift(pkey)); 324 u64 old_iamr = current_thread_iamr() & ~((u64)(0x1ul) << pkeyshift(pkey)); 325 326 if (!likely(pkey_execute_disable_supported)) 327 return; 328 329 current->thread.regs->iamr = old_iamr | new_iamr_bits; 330 } 331 332 /* 333 * Set the access rights in AMR IAMR and UAMOR registers for @pkey to that 334 * specified in @init_val. 335 */ 336 int __arch_set_user_pkey_access(struct task_struct *tsk, int pkey, 337 unsigned long init_val) 338 { 339 u64 new_amr_bits = 0x0ul; 340 u64 new_iamr_bits = 0x0ul; 341 u64 pkey_bits, uamor_pkey_bits; 342 343 /* 344 * Check whether the key is disabled by UAMOR. 345 */ 346 pkey_bits = 0x3ul << pkeyshift(pkey); 347 uamor_pkey_bits = (default_uamor & pkey_bits); 348 349 /* 350 * Both the bits in UAMOR corresponding to the key should be set 351 */ 352 if (uamor_pkey_bits != pkey_bits) 353 return -EINVAL; 354 355 if (init_val & PKEY_DISABLE_EXECUTE) { 356 if (!pkey_execute_disable_supported) 357 return -EINVAL; 358 new_iamr_bits |= IAMR_EX_BIT; 359 } 360 init_iamr(pkey, new_iamr_bits); 361 362 /* Set the bits we need in AMR: */ 363 if (init_val & PKEY_DISABLE_ACCESS) 364 new_amr_bits |= AMR_RD_BIT | AMR_WR_BIT; 365 else if (init_val & PKEY_DISABLE_WRITE) 366 new_amr_bits |= AMR_WR_BIT; 367 368 init_amr(pkey, new_amr_bits); 369 return 0; 370 } 371 372 int execute_only_pkey(struct mm_struct *mm) 373 { 374 return mm->context.execute_only_pkey; 375 } 376 377 static inline bool vma_is_pkey_exec_only(struct vm_area_struct *vma) 378 { 379 /* Do this check first since the vm_flags should be hot */ 380 if ((vma->vm_flags & VM_ACCESS_FLAGS) != VM_EXEC) 381 return false; 382 383 return (vma_pkey(vma) == vma->vm_mm->context.execute_only_pkey); 384 } 385 386 /* 387 * This should only be called for *plain* mprotect calls. 388 */ 389 int __arch_override_mprotect_pkey(struct vm_area_struct *vma, int prot, 390 int pkey) 391 { 392 /* 393 * If the currently associated pkey is execute-only, but the requested 394 * protection is not execute-only, move it back to the default pkey. 395 */ 396 if (vma_is_pkey_exec_only(vma) && (prot != PROT_EXEC)) 397 return 0; 398 399 /* 400 * The requested protection is execute-only. Hence let's use an 401 * execute-only pkey. 402 */ 403 if (prot == PROT_EXEC) { 404 pkey = execute_only_pkey(vma->vm_mm); 405 if (pkey > 0) 406 return pkey; 407 } 408 409 /* Nothing to override. */ 410 return vma_pkey(vma); 411 } 412 413 static bool pkey_access_permitted(int pkey, bool write, bool execute) 414 { 415 int pkey_shift; 416 u64 amr; 417 418 pkey_shift = pkeyshift(pkey); 419 if (execute) 420 return !(current_thread_iamr() & (IAMR_EX_BIT << pkey_shift)); 421 422 amr = current_thread_amr(); 423 if (write) 424 return !(amr & (AMR_WR_BIT << pkey_shift)); 425 426 return !(amr & (AMR_RD_BIT << pkey_shift)); 427 } 428 429 bool arch_pte_access_permitted(u64 pte, bool write, bool execute) 430 { 431 if (!mmu_has_feature(MMU_FTR_PKEY)) 432 return true; 433 434 return pkey_access_permitted(pte_to_pkey_bits(pte), write, execute); 435 } 436 437 /* 438 * We only want to enforce protection keys on the current thread because we 439 * effectively have no access to AMR/IAMR for other threads or any way to tell 440 * which AMR/IAMR in a threaded process we could use. 441 * 442 * So do not enforce things if the VMA is not from the current mm, or if we are 443 * in a kernel thread. 444 */ 445 bool arch_vma_access_permitted(struct vm_area_struct *vma, bool write, 446 bool execute, bool foreign) 447 { 448 if (!mmu_has_feature(MMU_FTR_PKEY)) 449 return true; 450 /* 451 * Do not enforce our key-permissions on a foreign vma. 452 */ 453 if (foreign || vma_is_foreign(vma)) 454 return true; 455 456 return pkey_access_permitted(vma_pkey(vma), write, execute); 457 } 458 459 void arch_dup_pkeys(struct mm_struct *oldmm, struct mm_struct *mm) 460 { 461 if (!mmu_has_feature(MMU_FTR_PKEY)) 462 return; 463 464 /* Duplicate the oldmm pkey state in mm: */ 465 mm_pkey_allocation_map(mm) = mm_pkey_allocation_map(oldmm); 466 mm->context.execute_only_pkey = oldmm->context.execute_only_pkey; 467 } 468 469 #endif /* CONFIG_PPC_MEM_KEYS */ 470