1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _ASM_POWERPC_BOOK3S_64_KUP_H 3 #define _ASM_POWERPC_BOOK3S_64_KUP_H 4 5 #include <linux/const.h> 6 #include <asm/reg.h> 7 8 #define AMR_KUAP_BLOCK_READ UL(0x5455555555555555) 9 #define AMR_KUAP_BLOCK_WRITE UL(0xa8aaaaaaaaaaaaaa) 10 #define AMR_KUEP_BLOCKED UL(0x5455555555555555) 11 #define AMR_KUAP_BLOCKED (AMR_KUAP_BLOCK_READ | AMR_KUAP_BLOCK_WRITE) 12 13 #ifdef __ASSEMBLY__ 14 15 .macro kuap_user_restore gpr1, gpr2 16 #if defined(CONFIG_PPC_PKEY) 17 BEGIN_MMU_FTR_SECTION_NESTED(67) 18 b 100f // skip_restore_amr 19 END_MMU_FTR_SECTION_NESTED_IFCLR(MMU_FTR_PKEY, 67) 20 /* 21 * AMR and IAMR are going to be different when 22 * returning to userspace. 23 */ 24 ld \gpr1, STACK_REGS_AMR(r1) 25 26 /* 27 * If kuap feature is not enabled, do the mtspr 28 * only if AMR value is different. 29 */ 30 BEGIN_MMU_FTR_SECTION_NESTED(68) 31 mfspr \gpr2, SPRN_AMR 32 cmpd \gpr1, \gpr2 33 beq 99f 34 END_MMU_FTR_SECTION_NESTED_IFCLR(MMU_FTR_BOOK3S_KUAP, 68) 35 36 isync 37 mtspr SPRN_AMR, \gpr1 38 99: 39 /* 40 * Restore IAMR only when returning to userspace 41 */ 42 ld \gpr1, STACK_REGS_IAMR(r1) 43 44 /* 45 * If kuep feature is not enabled, do the mtspr 46 * only if IAMR value is different. 47 */ 48 BEGIN_MMU_FTR_SECTION_NESTED(69) 49 mfspr \gpr2, SPRN_IAMR 50 cmpd \gpr1, \gpr2 51 beq 100f 52 END_MMU_FTR_SECTION_NESTED_IFCLR(MMU_FTR_BOOK3S_KUEP, 69) 53 54 isync 55 mtspr SPRN_IAMR, \gpr1 56 57 100: //skip_restore_amr 58 /* No isync required, see kuap_user_restore() */ 59 #endif 60 .endm 61 62 .macro kuap_kernel_restore gpr1, gpr2 63 #if defined(CONFIG_PPC_PKEY) 64 65 BEGIN_MMU_FTR_SECTION_NESTED(67) 66 /* 67 * AMR is going to be mostly the same since we are 68 * returning to the kernel. Compare and do a mtspr. 69 */ 70 ld \gpr2, STACK_REGS_AMR(r1) 71 mfspr \gpr1, SPRN_AMR 72 cmpd \gpr1, \gpr2 73 beq 100f 74 isync 75 mtspr SPRN_AMR, \gpr2 76 /* 77 * No isync required, see kuap_restore_amr() 78 * No need to restore IAMR when returning to kernel space. 79 */ 80 100: 81 END_MMU_FTR_SECTION_NESTED_IFSET(MMU_FTR_BOOK3S_KUAP, 67) 82 #endif 83 .endm 84 85 #ifdef CONFIG_PPC_KUAP 86 .macro kuap_check_amr gpr1, gpr2 87 #ifdef CONFIG_PPC_KUAP_DEBUG 88 BEGIN_MMU_FTR_SECTION_NESTED(67) 89 mfspr \gpr1, SPRN_AMR 90 /* Prevent access to userspace using any key values */ 91 LOAD_REG_IMMEDIATE(\gpr2, AMR_KUAP_BLOCKED) 92 999: tdne \gpr1, \gpr2 93 EMIT_WARN_ENTRY 999b, __FILE__, __LINE__, (BUGFLAG_WARNING | BUGFLAG_ONCE) 94 END_MMU_FTR_SECTION_NESTED_IFSET(MMU_FTR_BOOK3S_KUAP, 67) 95 #endif 96 .endm 97 #endif 98 99 /* 100 * if (pkey) { 101 * 102 * save AMR -> stack; 103 * if (kuap) { 104 * if (AMR != BLOCKED) 105 * KUAP_BLOCKED -> AMR; 106 * } 107 * if (from_user) { 108 * save IAMR -> stack; 109 * if (kuep) { 110 * KUEP_BLOCKED ->IAMR 111 * } 112 * } 113 * return; 114 * } 115 * 116 * if (kuap) { 117 * if (from_kernel) { 118 * save AMR -> stack; 119 * if (AMR != BLOCKED) 120 * KUAP_BLOCKED -> AMR; 121 * } 122 * 123 * } 124 */ 125 .macro kuap_save_amr_and_lock gpr1, gpr2, use_cr, msr_pr_cr 126 #if defined(CONFIG_PPC_PKEY) 127 128 /* 129 * if both pkey and kuap is disabled, nothing to do 130 */ 131 BEGIN_MMU_FTR_SECTION_NESTED(68) 132 b 100f // skip_save_amr 133 END_MMU_FTR_SECTION_NESTED_IFCLR(MMU_FTR_PKEY | MMU_FTR_BOOK3S_KUAP, 68) 134 135 /* 136 * if pkey is disabled and we are entering from userspace 137 * don't do anything. 138 */ 139 BEGIN_MMU_FTR_SECTION_NESTED(67) 140 .ifnb \msr_pr_cr 141 /* 142 * Without pkey we are not changing AMR outside the kernel 143 * hence skip this completely. 144 */ 145 bne \msr_pr_cr, 100f // from userspace 146 .endif 147 END_MMU_FTR_SECTION_NESTED_IFCLR(MMU_FTR_PKEY, 67) 148 149 /* 150 * pkey is enabled or pkey is disabled but entering from kernel 151 */ 152 mfspr \gpr1, SPRN_AMR 153 std \gpr1, STACK_REGS_AMR(r1) 154 155 /* 156 * update kernel AMR with AMR_KUAP_BLOCKED only 157 * if KUAP feature is enabled 158 */ 159 BEGIN_MMU_FTR_SECTION_NESTED(69) 160 LOAD_REG_IMMEDIATE(\gpr2, AMR_KUAP_BLOCKED) 161 cmpd \use_cr, \gpr1, \gpr2 162 beq \use_cr, 102f 163 /* 164 * We don't isync here because we very recently entered via an interrupt 165 */ 166 mtspr SPRN_AMR, \gpr2 167 isync 168 102: 169 END_MMU_FTR_SECTION_NESTED_IFSET(MMU_FTR_BOOK3S_KUAP, 69) 170 171 /* 172 * if entering from kernel we don't need save IAMR 173 */ 174 .ifnb \msr_pr_cr 175 beq \msr_pr_cr, 100f // from kernel space 176 mfspr \gpr1, SPRN_IAMR 177 std \gpr1, STACK_REGS_IAMR(r1) 178 179 /* 180 * update kernel IAMR with AMR_KUEP_BLOCKED only 181 * if KUEP feature is enabled 182 */ 183 BEGIN_MMU_FTR_SECTION_NESTED(70) 184 LOAD_REG_IMMEDIATE(\gpr2, AMR_KUEP_BLOCKED) 185 mtspr SPRN_IAMR, \gpr2 186 isync 187 END_MMU_FTR_SECTION_NESTED_IFSET(MMU_FTR_BOOK3S_KUEP, 70) 188 .endif 189 190 100: // skip_save_amr 191 #endif 192 .endm 193 194 #else /* !__ASSEMBLY__ */ 195 196 #include <linux/jump_label.h> 197 #include <linux/sched.h> 198 199 DECLARE_STATIC_KEY_FALSE(uaccess_flush_key); 200 201 #ifdef CONFIG_PPC_PKEY 202 203 extern u64 __ro_after_init default_uamor; 204 extern u64 __ro_after_init default_amr; 205 extern u64 __ro_after_init default_iamr; 206 207 #include <asm/mmu.h> 208 #include <asm/ptrace.h> 209 210 /* usage of kthread_use_mm() should inherit the 211 * AMR value of the operating address space. But, the AMR value is 212 * thread-specific and we inherit the address space and not thread 213 * access restrictions. Because of this ignore AMR value when accessing 214 * userspace via kernel thread. 215 */ 216 static inline u64 current_thread_amr(void) 217 { 218 if (current->thread.regs) 219 return current->thread.regs->amr; 220 return default_amr; 221 } 222 223 static inline u64 current_thread_iamr(void) 224 { 225 if (current->thread.regs) 226 return current->thread.regs->iamr; 227 return default_iamr; 228 } 229 #endif /* CONFIG_PPC_PKEY */ 230 231 #ifdef CONFIG_PPC_KUAP 232 233 static __always_inline bool kuap_is_disabled(void) 234 { 235 return !mmu_has_feature(MMU_FTR_BOOK3S_KUAP); 236 } 237 238 static inline void kuap_user_restore(struct pt_regs *regs) 239 { 240 bool restore_amr = false, restore_iamr = false; 241 unsigned long amr, iamr; 242 243 if (!mmu_has_feature(MMU_FTR_PKEY)) 244 return; 245 246 if (!mmu_has_feature(MMU_FTR_BOOK3S_KUAP)) { 247 amr = mfspr(SPRN_AMR); 248 if (amr != regs->amr) 249 restore_amr = true; 250 } else { 251 restore_amr = true; 252 } 253 254 if (!mmu_has_feature(MMU_FTR_BOOK3S_KUEP)) { 255 iamr = mfspr(SPRN_IAMR); 256 if (iamr != regs->iamr) 257 restore_iamr = true; 258 } else { 259 restore_iamr = true; 260 } 261 262 263 if (restore_amr || restore_iamr) { 264 isync(); 265 if (restore_amr) 266 mtspr(SPRN_AMR, regs->amr); 267 if (restore_iamr) 268 mtspr(SPRN_IAMR, regs->iamr); 269 } 270 /* 271 * No isync required here because we are about to rfi 272 * back to previous context before any user accesses 273 * would be made, which is a CSI. 274 */ 275 } 276 277 static inline void __kuap_kernel_restore(struct pt_regs *regs, unsigned long amr) 278 { 279 if (likely(regs->amr == amr)) 280 return; 281 282 isync(); 283 mtspr(SPRN_AMR, regs->amr); 284 /* 285 * No isync required here because we are about to rfi 286 * back to previous context before any user accesses 287 * would be made, which is a CSI. 288 * 289 * No need to restore IAMR when returning to kernel space. 290 */ 291 } 292 293 static inline unsigned long __kuap_get_and_assert_locked(void) 294 { 295 unsigned long amr = mfspr(SPRN_AMR); 296 297 if (IS_ENABLED(CONFIG_PPC_KUAP_DEBUG)) /* kuap_check_amr() */ 298 WARN_ON_ONCE(amr != AMR_KUAP_BLOCKED); 299 return amr; 300 } 301 302 /* Do nothing, book3s/64 does that in ASM */ 303 static inline void __kuap_lock(void) 304 { 305 } 306 307 static inline void __kuap_save_and_lock(struct pt_regs *regs) 308 { 309 } 310 311 /* 312 * We support individually allowing read or write, but we don't support nesting 313 * because that would require an expensive read/modify write of the AMR. 314 */ 315 316 static inline unsigned long get_kuap(void) 317 { 318 /* 319 * We return AMR_KUAP_BLOCKED when we don't support KUAP because 320 * prevent_user_access_return needs to return AMR_KUAP_BLOCKED to 321 * cause restore_user_access to do a flush. 322 * 323 * This has no effect in terms of actually blocking things on hash, 324 * so it doesn't break anything. 325 */ 326 if (!mmu_has_feature(MMU_FTR_BOOK3S_KUAP)) 327 return AMR_KUAP_BLOCKED; 328 329 return mfspr(SPRN_AMR); 330 } 331 332 static __always_inline void set_kuap(unsigned long value) 333 { 334 if (!mmu_has_feature(MMU_FTR_BOOK3S_KUAP)) 335 return; 336 337 /* 338 * ISA v3.0B says we need a CSI (Context Synchronising Instruction) both 339 * before and after the move to AMR. See table 6 on page 1134. 340 */ 341 isync(); 342 mtspr(SPRN_AMR, value); 343 isync(); 344 } 345 346 static inline bool __bad_kuap_fault(struct pt_regs *regs, unsigned long address, bool is_write) 347 { 348 /* 349 * For radix this will be a storage protection fault (DSISR_PROTFAULT). 350 * For hash this will be a key fault (DSISR_KEYFAULT) 351 */ 352 /* 353 * We do have exception table entry, but accessing the 354 * userspace results in fault. This could be because we 355 * didn't unlock the AMR or access is denied by userspace 356 * using a key value that blocks access. We are only interested 357 * in catching the use case of accessing without unlocking 358 * the AMR. Hence check for BLOCK_WRITE/READ against AMR. 359 */ 360 if (is_write) { 361 return (regs->amr & AMR_KUAP_BLOCK_WRITE) == AMR_KUAP_BLOCK_WRITE; 362 } 363 return (regs->amr & AMR_KUAP_BLOCK_READ) == AMR_KUAP_BLOCK_READ; 364 } 365 366 static __always_inline void allow_user_access(void __user *to, const void __user *from, 367 unsigned long size, unsigned long dir) 368 { 369 unsigned long thread_amr = 0; 370 371 // This is written so we can resolve to a single case at build time 372 BUILD_BUG_ON(!__builtin_constant_p(dir)); 373 374 if (mmu_has_feature(MMU_FTR_PKEY)) 375 thread_amr = current_thread_amr(); 376 377 if (dir == KUAP_READ) 378 set_kuap(thread_amr | AMR_KUAP_BLOCK_WRITE); 379 else if (dir == KUAP_WRITE) 380 set_kuap(thread_amr | AMR_KUAP_BLOCK_READ); 381 else if (dir == KUAP_READ_WRITE) 382 set_kuap(thread_amr); 383 else 384 BUILD_BUG(); 385 } 386 387 #else /* CONFIG_PPC_KUAP */ 388 389 static inline unsigned long get_kuap(void) 390 { 391 return AMR_KUAP_BLOCKED; 392 } 393 394 static inline void set_kuap(unsigned long value) { } 395 396 static __always_inline void allow_user_access(void __user *to, const void __user *from, 397 unsigned long size, unsigned long dir) 398 { } 399 400 #endif /* !CONFIG_PPC_KUAP */ 401 402 static __always_inline void prevent_user_access(unsigned long dir) 403 { 404 set_kuap(AMR_KUAP_BLOCKED); 405 if (static_branch_unlikely(&uaccess_flush_key)) 406 do_uaccess_flush(); 407 } 408 409 static inline unsigned long prevent_user_access_return(void) 410 { 411 unsigned long flags = get_kuap(); 412 413 set_kuap(AMR_KUAP_BLOCKED); 414 if (static_branch_unlikely(&uaccess_flush_key)) 415 do_uaccess_flush(); 416 417 return flags; 418 } 419 420 static inline void restore_user_access(unsigned long flags) 421 { 422 set_kuap(flags); 423 if (static_branch_unlikely(&uaccess_flush_key) && flags == AMR_KUAP_BLOCKED) 424 do_uaccess_flush(); 425 } 426 #endif /* __ASSEMBLY__ */ 427 428 #endif /* _ASM_POWERPC_BOOK3S_64_KUP_H */ 429