1 // SPDX-License-Identifier: GPL-2.0+ 2 /* 3 * PowerPC Memory Protection Keys management 4 * 5 * Copyright 2017, Ram Pai, IBM Corporation. 6 */ 7 8 #include <asm/mman.h> 9 #include <asm/mmu_context.h> 10 #include <asm/mmu.h> 11 #include <asm/setup.h> 12 #include <linux/pkeys.h> 13 #include <linux/of_device.h> 14 15 DEFINE_STATIC_KEY_TRUE(pkey_disabled); 16 int pkeys_total; /* Total pkeys as per device tree */ 17 u32 initial_allocation_mask; /* Bits set for the initially allocated keys */ 18 u32 reserved_allocation_mask; /* Bits set for reserved keys */ 19 static bool pkey_execute_disable_supported; 20 static bool pkeys_devtree_defined; /* property exported by device tree */ 21 static u64 pkey_amr_mask; /* Bits in AMR not to be touched */ 22 static u64 pkey_iamr_mask; /* Bits in AMR not to be touched */ 23 static u64 pkey_uamor_mask; /* Bits in UMOR not to be touched */ 24 static int execute_only_key = 2; 25 26 #define AMR_BITS_PER_PKEY 2 27 #define AMR_RD_BIT 0x1UL 28 #define AMR_WR_BIT 0x2UL 29 #define IAMR_EX_BIT 0x1UL 30 #define PKEY_REG_BITS (sizeof(u64)*8) 31 #define pkeyshift(pkey) (PKEY_REG_BITS - ((pkey+1) * AMR_BITS_PER_PKEY)) 32 33 static void scan_pkey_feature(void) 34 { 35 u32 vals[2]; 36 struct device_node *cpu; 37 38 cpu = of_find_node_by_type(NULL, "cpu"); 39 if (!cpu) 40 return; 41 42 if (of_property_read_u32_array(cpu, 43 "ibm,processor-storage-keys", vals, 2)) 44 return; 45 46 /* 47 * Since any pkey can be used for data or execute, we will just treat 48 * all keys as equal and track them as one entity. 49 */ 50 pkeys_total = vals[0]; 51 pkeys_devtree_defined = true; 52 } 53 54 static inline bool pkey_mmu_enabled(void) 55 { 56 if (firmware_has_feature(FW_FEATURE_LPAR)) 57 return pkeys_total; 58 else 59 return cpu_has_feature(CPU_FTR_PKEY); 60 } 61 62 static int pkey_initialize(void) 63 { 64 int os_reserved, i; 65 66 /* 67 * We define PKEY_DISABLE_EXECUTE in addition to the arch-neutral 68 * generic defines for PKEY_DISABLE_ACCESS and PKEY_DISABLE_WRITE. 69 * Ensure that the bits a distinct. 70 */ 71 BUILD_BUG_ON(PKEY_DISABLE_EXECUTE & 72 (PKEY_DISABLE_ACCESS | PKEY_DISABLE_WRITE)); 73 74 /* 75 * pkey_to_vmflag_bits() assumes that the pkey bits are contiguous 76 * in the vmaflag. Make sure that is really the case. 77 */ 78 BUILD_BUG_ON(__builtin_clzl(ARCH_VM_PKEY_FLAGS >> VM_PKEY_SHIFT) + 79 __builtin_popcountl(ARCH_VM_PKEY_FLAGS >> VM_PKEY_SHIFT) 80 != (sizeof(u64) * BITS_PER_BYTE)); 81 82 /* scan the device tree for pkey feature */ 83 scan_pkey_feature(); 84 85 /* 86 * Let's assume 32 pkeys on P8 bare metal, if its not defined by device 87 * tree. We make this exception since skiboot forgot to expose this 88 * property on power8. 89 */ 90 if (!pkeys_devtree_defined && !firmware_has_feature(FW_FEATURE_LPAR) && 91 cpu_has_feature(CPU_FTRS_POWER8)) 92 pkeys_total = 32; 93 94 /* 95 * Adjust the upper limit, based on the number of bits supported by 96 * arch-neutral code. 97 */ 98 pkeys_total = min_t(int, pkeys_total, 99 ((ARCH_VM_PKEY_FLAGS >> VM_PKEY_SHIFT)+1)); 100 101 if (!pkey_mmu_enabled() || radix_enabled() || !pkeys_total) 102 static_branch_enable(&pkey_disabled); 103 else 104 static_branch_disable(&pkey_disabled); 105 106 if (static_branch_likely(&pkey_disabled)) 107 return 0; 108 109 /* 110 * The device tree cannot be relied to indicate support for 111 * execute_disable support. Instead we use a PVR check. 112 */ 113 if (pvr_version_is(PVR_POWER7) || pvr_version_is(PVR_POWER7p)) 114 pkey_execute_disable_supported = false; 115 else 116 pkey_execute_disable_supported = true; 117 118 #ifdef CONFIG_PPC_4K_PAGES 119 /* 120 * The OS can manage only 8 pkeys due to its inability to represent them 121 * in the Linux 4K PTE. 122 */ 123 os_reserved = pkeys_total - 8; 124 #else 125 os_reserved = 0; 126 #endif 127 /* Bits are in LE format. */ 128 reserved_allocation_mask = (0x1 << 1) | (0x1 << execute_only_key); 129 130 /* register mask is in BE format */ 131 pkey_amr_mask = ~0x0ul; 132 pkey_amr_mask &= ~(0x3ul << pkeyshift(0)); 133 134 pkey_iamr_mask = ~0x0ul; 135 pkey_iamr_mask &= ~(0x3ul << pkeyshift(0)); 136 pkey_iamr_mask &= ~(0x3ul << pkeyshift(execute_only_key)); 137 138 pkey_uamor_mask = ~0x0ul; 139 pkey_uamor_mask &= ~(0x3ul << pkeyshift(0)); 140 pkey_uamor_mask &= ~(0x3ul << pkeyshift(execute_only_key)); 141 142 /* mark the rest of the keys as reserved and hence unavailable */ 143 for (i = (pkeys_total - os_reserved); i < pkeys_total; i++) { 144 reserved_allocation_mask |= (0x1 << i); 145 pkey_uamor_mask &= ~(0x3ul << pkeyshift(i)); 146 } 147 initial_allocation_mask = reserved_allocation_mask | (0x1 << 0); 148 149 if (unlikely((pkeys_total - os_reserved) <= execute_only_key)) { 150 /* 151 * Insufficient number of keys to support 152 * execute only key. Mark it unavailable. 153 * Any AMR, UAMOR, IAMR bit set for 154 * this key is irrelevant since this key 155 * can never be allocated. 156 */ 157 execute_only_key = -1; 158 } 159 160 return 0; 161 } 162 163 arch_initcall(pkey_initialize); 164 165 void pkey_mm_init(struct mm_struct *mm) 166 { 167 if (static_branch_likely(&pkey_disabled)) 168 return; 169 mm_pkey_allocation_map(mm) = initial_allocation_mask; 170 mm->context.execute_only_pkey = execute_only_key; 171 } 172 173 static inline u64 read_amr(void) 174 { 175 return mfspr(SPRN_AMR); 176 } 177 178 static inline void write_amr(u64 value) 179 { 180 mtspr(SPRN_AMR, value); 181 } 182 183 static inline u64 read_iamr(void) 184 { 185 if (!likely(pkey_execute_disable_supported)) 186 return 0x0UL; 187 188 return mfspr(SPRN_IAMR); 189 } 190 191 static inline void write_iamr(u64 value) 192 { 193 if (!likely(pkey_execute_disable_supported)) 194 return; 195 196 mtspr(SPRN_IAMR, value); 197 } 198 199 static inline u64 read_uamor(void) 200 { 201 return mfspr(SPRN_UAMOR); 202 } 203 204 static inline void write_uamor(u64 value) 205 { 206 mtspr(SPRN_UAMOR, value); 207 } 208 209 static bool is_pkey_enabled(int pkey) 210 { 211 u64 uamor = read_uamor(); 212 u64 pkey_bits = 0x3ul << pkeyshift(pkey); 213 u64 uamor_pkey_bits = (uamor & pkey_bits); 214 215 /* 216 * Both the bits in UAMOR corresponding to the key should be set or 217 * reset. 218 */ 219 WARN_ON(uamor_pkey_bits && (uamor_pkey_bits != pkey_bits)); 220 return !!(uamor_pkey_bits); 221 } 222 223 static inline void init_amr(int pkey, u8 init_bits) 224 { 225 u64 new_amr_bits = (((u64)init_bits & 0x3UL) << pkeyshift(pkey)); 226 u64 old_amr = read_amr() & ~((u64)(0x3ul) << pkeyshift(pkey)); 227 228 write_amr(old_amr | new_amr_bits); 229 } 230 231 static inline void init_iamr(int pkey, u8 init_bits) 232 { 233 u64 new_iamr_bits = (((u64)init_bits & 0x1UL) << pkeyshift(pkey)); 234 u64 old_iamr = read_iamr() & ~((u64)(0x1ul) << pkeyshift(pkey)); 235 236 write_iamr(old_iamr | new_iamr_bits); 237 } 238 239 /* 240 * Set the access rights in AMR IAMR and UAMOR registers for @pkey to that 241 * specified in @init_val. 242 */ 243 int __arch_set_user_pkey_access(struct task_struct *tsk, int pkey, 244 unsigned long init_val) 245 { 246 u64 new_amr_bits = 0x0ul; 247 u64 new_iamr_bits = 0x0ul; 248 249 if (!is_pkey_enabled(pkey)) 250 return -EINVAL; 251 252 if (init_val & PKEY_DISABLE_EXECUTE) { 253 if (!pkey_execute_disable_supported) 254 return -EINVAL; 255 new_iamr_bits |= IAMR_EX_BIT; 256 } 257 init_iamr(pkey, new_iamr_bits); 258 259 /* Set the bits we need in AMR: */ 260 if (init_val & PKEY_DISABLE_ACCESS) 261 new_amr_bits |= AMR_RD_BIT | AMR_WR_BIT; 262 else if (init_val & PKEY_DISABLE_WRITE) 263 new_amr_bits |= AMR_WR_BIT; 264 265 init_amr(pkey, new_amr_bits); 266 return 0; 267 } 268 269 void thread_pkey_regs_save(struct thread_struct *thread) 270 { 271 if (static_branch_likely(&pkey_disabled)) 272 return; 273 274 /* 275 * TODO: Skip saving registers if @thread hasn't used any keys yet. 276 */ 277 thread->amr = read_amr(); 278 thread->iamr = read_iamr(); 279 thread->uamor = read_uamor(); 280 } 281 282 void thread_pkey_regs_restore(struct thread_struct *new_thread, 283 struct thread_struct *old_thread) 284 { 285 if (static_branch_likely(&pkey_disabled)) 286 return; 287 288 if (old_thread->amr != new_thread->amr) 289 write_amr(new_thread->amr); 290 if (old_thread->iamr != new_thread->iamr) 291 write_iamr(new_thread->iamr); 292 if (old_thread->uamor != new_thread->uamor) 293 write_uamor(new_thread->uamor); 294 } 295 296 void thread_pkey_regs_init(struct thread_struct *thread) 297 { 298 if (static_branch_likely(&pkey_disabled)) 299 return; 300 301 thread->amr = pkey_amr_mask; 302 thread->iamr = pkey_iamr_mask; 303 thread->uamor = pkey_uamor_mask; 304 305 write_uamor(pkey_uamor_mask); 306 write_amr(pkey_amr_mask); 307 write_iamr(pkey_iamr_mask); 308 } 309 310 static inline bool pkey_allows_readwrite(int pkey) 311 { 312 int pkey_shift = pkeyshift(pkey); 313 314 if (!is_pkey_enabled(pkey)) 315 return true; 316 317 return !(read_amr() & ((AMR_RD_BIT|AMR_WR_BIT) << pkey_shift)); 318 } 319 320 int __execute_only_pkey(struct mm_struct *mm) 321 { 322 return mm->context.execute_only_pkey; 323 } 324 325 static inline bool vma_is_pkey_exec_only(struct vm_area_struct *vma) 326 { 327 /* Do this check first since the vm_flags should be hot */ 328 if ((vma->vm_flags & (VM_READ | VM_WRITE | VM_EXEC)) != VM_EXEC) 329 return false; 330 331 return (vma_pkey(vma) == vma->vm_mm->context.execute_only_pkey); 332 } 333 334 /* 335 * This should only be called for *plain* mprotect calls. 336 */ 337 int __arch_override_mprotect_pkey(struct vm_area_struct *vma, int prot, 338 int pkey) 339 { 340 /* 341 * If the currently associated pkey is execute-only, but the requested 342 * protection is not execute-only, move it back to the default pkey. 343 */ 344 if (vma_is_pkey_exec_only(vma) && (prot != PROT_EXEC)) 345 return 0; 346 347 /* 348 * The requested protection is execute-only. Hence let's use an 349 * execute-only pkey. 350 */ 351 if (prot == PROT_EXEC) { 352 pkey = execute_only_pkey(vma->vm_mm); 353 if (pkey > 0) 354 return pkey; 355 } 356 357 /* Nothing to override. */ 358 return vma_pkey(vma); 359 } 360 361 static bool pkey_access_permitted(int pkey, bool write, bool execute) 362 { 363 int pkey_shift; 364 u64 amr; 365 366 if (!is_pkey_enabled(pkey)) 367 return true; 368 369 pkey_shift = pkeyshift(pkey); 370 if (execute && !(read_iamr() & (IAMR_EX_BIT << pkey_shift))) 371 return true; 372 373 amr = read_amr(); /* Delay reading amr until absolutely needed */ 374 return ((!write && !(amr & (AMR_RD_BIT << pkey_shift))) || 375 (write && !(amr & (AMR_WR_BIT << pkey_shift)))); 376 } 377 378 bool arch_pte_access_permitted(u64 pte, bool write, bool execute) 379 { 380 if (static_branch_likely(&pkey_disabled)) 381 return true; 382 383 return pkey_access_permitted(pte_to_pkey_bits(pte), write, execute); 384 } 385 386 /* 387 * We only want to enforce protection keys on the current thread because we 388 * effectively have no access to AMR/IAMR for other threads or any way to tell 389 * which AMR/IAMR in a threaded process we could use. 390 * 391 * So do not enforce things if the VMA is not from the current mm, or if we are 392 * in a kernel thread. 393 */ 394 static inline bool vma_is_foreign(struct vm_area_struct *vma) 395 { 396 if (!current->mm) 397 return true; 398 399 /* if it is not our ->mm, it has to be foreign */ 400 if (current->mm != vma->vm_mm) 401 return true; 402 403 return false; 404 } 405 406 bool arch_vma_access_permitted(struct vm_area_struct *vma, bool write, 407 bool execute, bool foreign) 408 { 409 if (static_branch_likely(&pkey_disabled)) 410 return true; 411 /* 412 * Do not enforce our key-permissions on a foreign vma. 413 */ 414 if (foreign || vma_is_foreign(vma)) 415 return true; 416 417 return pkey_access_permitted(vma_pkey(vma), write, execute); 418 } 419 420 void arch_dup_pkeys(struct mm_struct *oldmm, struct mm_struct *mm) 421 { 422 if (static_branch_likely(&pkey_disabled)) 423 return; 424 425 /* Duplicate the oldmm pkey state in mm: */ 426 mm_pkey_allocation_map(mm) = mm_pkey_allocation_map(oldmm); 427 mm->context.execute_only_pkey = oldmm->context.execute_only_pkey; 428 } 429