1 // SPDX-License-Identifier: GPL-2.0+ 2 // 3 // Security related flags and so on. 4 // 5 // Copyright 2018, Michael Ellerman, IBM Corporation. 6 7 #include <linux/cpu.h> 8 #include <linux/kernel.h> 9 #include <linux/device.h> 10 #include <linux/memblock.h> 11 #include <linux/nospec.h> 12 #include <linux/prctl.h> 13 #include <linux/seq_buf.h> 14 15 #include <asm/asm-prototypes.h> 16 #include <asm/code-patching.h> 17 #include <asm/debugfs.h> 18 #include <asm/security_features.h> 19 #include <asm/setup.h> 20 #include <asm/inst.h> 21 22 #include "setup.h" 23 24 u64 powerpc_security_features __read_mostly = SEC_FTR_DEFAULT; 25 26 enum branch_cache_flush_type { 27 BRANCH_CACHE_FLUSH_NONE = 0x1, 28 BRANCH_CACHE_FLUSH_SW = 0x2, 29 BRANCH_CACHE_FLUSH_HW = 0x4, 30 }; 31 static enum branch_cache_flush_type count_cache_flush_type = BRANCH_CACHE_FLUSH_NONE; 32 static enum branch_cache_flush_type link_stack_flush_type = BRANCH_CACHE_FLUSH_NONE; 33 34 bool barrier_nospec_enabled; 35 static bool no_nospec; 36 static bool btb_flush_enabled; 37 #if defined(CONFIG_PPC_FSL_BOOK3E) || defined(CONFIG_PPC_BOOK3S_64) 38 static bool no_spectrev2; 39 #endif 40 41 static void enable_barrier_nospec(bool enable) 42 { 43 barrier_nospec_enabled = enable; 44 do_barrier_nospec_fixups(enable); 45 } 46 47 void setup_barrier_nospec(void) 48 { 49 bool enable; 50 51 /* 52 * It would make sense to check SEC_FTR_SPEC_BAR_ORI31 below as well. 53 * But there's a good reason not to. The two flags we check below are 54 * both are enabled by default in the kernel, so if the hcall is not 55 * functional they will be enabled. 56 * On a system where the host firmware has been updated (so the ori 57 * functions as a barrier), but on which the hypervisor (KVM/Qemu) has 58 * not been updated, we would like to enable the barrier. Dropping the 59 * check for SEC_FTR_SPEC_BAR_ORI31 achieves that. The only downside is 60 * we potentially enable the barrier on systems where the host firmware 61 * is not updated, but that's harmless as it's a no-op. 62 */ 63 enable = security_ftr_enabled(SEC_FTR_FAVOUR_SECURITY) && 64 security_ftr_enabled(SEC_FTR_BNDS_CHK_SPEC_BAR); 65 66 if (!no_nospec && !cpu_mitigations_off()) 67 enable_barrier_nospec(enable); 68 } 69 70 static int __init handle_nospectre_v1(char *p) 71 { 72 no_nospec = true; 73 74 return 0; 75 } 76 early_param("nospectre_v1", handle_nospectre_v1); 77 78 #ifdef CONFIG_DEBUG_FS 79 static int barrier_nospec_set(void *data, u64 val) 80 { 81 switch (val) { 82 case 0: 83 case 1: 84 break; 85 default: 86 return -EINVAL; 87 } 88 89 if (!!val == !!barrier_nospec_enabled) 90 return 0; 91 92 enable_barrier_nospec(!!val); 93 94 return 0; 95 } 96 97 static int barrier_nospec_get(void *data, u64 *val) 98 { 99 *val = barrier_nospec_enabled ? 1 : 0; 100 return 0; 101 } 102 103 DEFINE_DEBUGFS_ATTRIBUTE(fops_barrier_nospec, barrier_nospec_get, 104 barrier_nospec_set, "%llu\n"); 105 106 static __init int barrier_nospec_debugfs_init(void) 107 { 108 debugfs_create_file_unsafe("barrier_nospec", 0600, 109 powerpc_debugfs_root, NULL, 110 &fops_barrier_nospec); 111 return 0; 112 } 113 device_initcall(barrier_nospec_debugfs_init); 114 115 static __init int security_feature_debugfs_init(void) 116 { 117 debugfs_create_x64("security_features", 0400, powerpc_debugfs_root, 118 &powerpc_security_features); 119 return 0; 120 } 121 device_initcall(security_feature_debugfs_init); 122 #endif /* CONFIG_DEBUG_FS */ 123 124 #if defined(CONFIG_PPC_FSL_BOOK3E) || defined(CONFIG_PPC_BOOK3S_64) 125 static int __init handle_nospectre_v2(char *p) 126 { 127 no_spectrev2 = true; 128 129 return 0; 130 } 131 early_param("nospectre_v2", handle_nospectre_v2); 132 #endif /* CONFIG_PPC_FSL_BOOK3E || CONFIG_PPC_BOOK3S_64 */ 133 134 #ifdef CONFIG_PPC_FSL_BOOK3E 135 void setup_spectre_v2(void) 136 { 137 if (no_spectrev2 || cpu_mitigations_off()) 138 do_btb_flush_fixups(); 139 else 140 btb_flush_enabled = true; 141 } 142 #endif /* CONFIG_PPC_FSL_BOOK3E */ 143 144 #ifdef CONFIG_PPC_BOOK3S_64 145 ssize_t cpu_show_meltdown(struct device *dev, struct device_attribute *attr, char *buf) 146 { 147 bool thread_priv; 148 149 thread_priv = security_ftr_enabled(SEC_FTR_L1D_THREAD_PRIV); 150 151 if (rfi_flush) { 152 struct seq_buf s; 153 seq_buf_init(&s, buf, PAGE_SIZE - 1); 154 155 seq_buf_printf(&s, "Mitigation: RFI Flush"); 156 if (thread_priv) 157 seq_buf_printf(&s, ", L1D private per thread"); 158 159 seq_buf_printf(&s, "\n"); 160 161 return s.len; 162 } 163 164 if (thread_priv) 165 return sprintf(buf, "Vulnerable: L1D private per thread\n"); 166 167 if (!security_ftr_enabled(SEC_FTR_L1D_FLUSH_HV) && 168 !security_ftr_enabled(SEC_FTR_L1D_FLUSH_PR)) 169 return sprintf(buf, "Not affected\n"); 170 171 return sprintf(buf, "Vulnerable\n"); 172 } 173 174 ssize_t cpu_show_l1tf(struct device *dev, struct device_attribute *attr, char *buf) 175 { 176 return cpu_show_meltdown(dev, attr, buf); 177 } 178 #endif 179 180 ssize_t cpu_show_spectre_v1(struct device *dev, struct device_attribute *attr, char *buf) 181 { 182 struct seq_buf s; 183 184 seq_buf_init(&s, buf, PAGE_SIZE - 1); 185 186 if (security_ftr_enabled(SEC_FTR_BNDS_CHK_SPEC_BAR)) { 187 if (barrier_nospec_enabled) 188 seq_buf_printf(&s, "Mitigation: __user pointer sanitization"); 189 else 190 seq_buf_printf(&s, "Vulnerable"); 191 192 if (security_ftr_enabled(SEC_FTR_SPEC_BAR_ORI31)) 193 seq_buf_printf(&s, ", ori31 speculation barrier enabled"); 194 195 seq_buf_printf(&s, "\n"); 196 } else 197 seq_buf_printf(&s, "Not affected\n"); 198 199 return s.len; 200 } 201 202 ssize_t cpu_show_spectre_v2(struct device *dev, struct device_attribute *attr, char *buf) 203 { 204 struct seq_buf s; 205 bool bcs, ccd; 206 207 seq_buf_init(&s, buf, PAGE_SIZE - 1); 208 209 bcs = security_ftr_enabled(SEC_FTR_BCCTRL_SERIALISED); 210 ccd = security_ftr_enabled(SEC_FTR_COUNT_CACHE_DISABLED); 211 212 if (bcs || ccd) { 213 seq_buf_printf(&s, "Mitigation: "); 214 215 if (bcs) 216 seq_buf_printf(&s, "Indirect branch serialisation (kernel only)"); 217 218 if (bcs && ccd) 219 seq_buf_printf(&s, ", "); 220 221 if (ccd) 222 seq_buf_printf(&s, "Indirect branch cache disabled"); 223 224 } else if (count_cache_flush_type != BRANCH_CACHE_FLUSH_NONE) { 225 seq_buf_printf(&s, "Mitigation: Software count cache flush"); 226 227 if (count_cache_flush_type == BRANCH_CACHE_FLUSH_HW) 228 seq_buf_printf(&s, " (hardware accelerated)"); 229 230 } else if (btb_flush_enabled) { 231 seq_buf_printf(&s, "Mitigation: Branch predictor state flush"); 232 } else { 233 seq_buf_printf(&s, "Vulnerable"); 234 } 235 236 if (bcs || ccd || count_cache_flush_type != BRANCH_CACHE_FLUSH_NONE) { 237 if (link_stack_flush_type != BRANCH_CACHE_FLUSH_NONE) 238 seq_buf_printf(&s, ", Software link stack flush"); 239 if (link_stack_flush_type == BRANCH_CACHE_FLUSH_HW) 240 seq_buf_printf(&s, " (hardware accelerated)"); 241 } 242 243 seq_buf_printf(&s, "\n"); 244 245 return s.len; 246 } 247 248 #ifdef CONFIG_PPC_BOOK3S_64 249 /* 250 * Store-forwarding barrier support. 251 */ 252 253 static enum stf_barrier_type stf_enabled_flush_types; 254 static bool no_stf_barrier; 255 static bool stf_barrier; 256 257 static int __init handle_no_stf_barrier(char *p) 258 { 259 pr_info("stf-barrier: disabled on command line."); 260 no_stf_barrier = true; 261 return 0; 262 } 263 264 early_param("no_stf_barrier", handle_no_stf_barrier); 265 266 /* This is the generic flag used by other architectures */ 267 static int __init handle_ssbd(char *p) 268 { 269 if (!p || strncmp(p, "auto", 5) == 0 || strncmp(p, "on", 2) == 0 ) { 270 /* Until firmware tells us, we have the barrier with auto */ 271 return 0; 272 } else if (strncmp(p, "off", 3) == 0) { 273 handle_no_stf_barrier(NULL); 274 return 0; 275 } else 276 return 1; 277 278 return 0; 279 } 280 early_param("spec_store_bypass_disable", handle_ssbd); 281 282 /* This is the generic flag used by other architectures */ 283 static int __init handle_no_ssbd(char *p) 284 { 285 handle_no_stf_barrier(NULL); 286 return 0; 287 } 288 early_param("nospec_store_bypass_disable", handle_no_ssbd); 289 290 static void stf_barrier_enable(bool enable) 291 { 292 if (enable) 293 do_stf_barrier_fixups(stf_enabled_flush_types); 294 else 295 do_stf_barrier_fixups(STF_BARRIER_NONE); 296 297 stf_barrier = enable; 298 } 299 300 void setup_stf_barrier(void) 301 { 302 enum stf_barrier_type type; 303 bool enable; 304 305 /* Default to fallback in case fw-features are not available */ 306 if (cpu_has_feature(CPU_FTR_ARCH_300)) 307 type = STF_BARRIER_EIEIO; 308 else if (cpu_has_feature(CPU_FTR_ARCH_207S)) 309 type = STF_BARRIER_SYNC_ORI; 310 else if (cpu_has_feature(CPU_FTR_ARCH_206)) 311 type = STF_BARRIER_FALLBACK; 312 else 313 type = STF_BARRIER_NONE; 314 315 enable = security_ftr_enabled(SEC_FTR_FAVOUR_SECURITY) && 316 security_ftr_enabled(SEC_FTR_STF_BARRIER); 317 318 if (type == STF_BARRIER_FALLBACK) { 319 pr_info("stf-barrier: fallback barrier available\n"); 320 } else if (type == STF_BARRIER_SYNC_ORI) { 321 pr_info("stf-barrier: hwsync barrier available\n"); 322 } else if (type == STF_BARRIER_EIEIO) { 323 pr_info("stf-barrier: eieio barrier available\n"); 324 } 325 326 stf_enabled_flush_types = type; 327 328 if (!no_stf_barrier && !cpu_mitigations_off()) 329 stf_barrier_enable(enable); 330 } 331 332 ssize_t cpu_show_spec_store_bypass(struct device *dev, struct device_attribute *attr, char *buf) 333 { 334 if (stf_barrier && stf_enabled_flush_types != STF_BARRIER_NONE) { 335 const char *type; 336 switch (stf_enabled_flush_types) { 337 case STF_BARRIER_EIEIO: 338 type = "eieio"; 339 break; 340 case STF_BARRIER_SYNC_ORI: 341 type = "hwsync"; 342 break; 343 case STF_BARRIER_FALLBACK: 344 type = "fallback"; 345 break; 346 default: 347 type = "unknown"; 348 } 349 return sprintf(buf, "Mitigation: Kernel entry/exit barrier (%s)\n", type); 350 } 351 352 if (!security_ftr_enabled(SEC_FTR_L1D_FLUSH_HV) && 353 !security_ftr_enabled(SEC_FTR_L1D_FLUSH_PR)) 354 return sprintf(buf, "Not affected\n"); 355 356 return sprintf(buf, "Vulnerable\n"); 357 } 358 359 static int ssb_prctl_get(struct task_struct *task) 360 { 361 if (stf_enabled_flush_types == STF_BARRIER_NONE) 362 /* 363 * We don't have an explicit signal from firmware that we're 364 * vulnerable or not, we only have certain CPU revisions that 365 * are known to be vulnerable. 366 * 367 * We assume that if we're on another CPU, where the barrier is 368 * NONE, then we are not vulnerable. 369 */ 370 return PR_SPEC_NOT_AFFECTED; 371 else 372 /* 373 * If we do have a barrier type then we are vulnerable. The 374 * barrier is not a global or per-process mitigation, so the 375 * only value we can report here is PR_SPEC_ENABLE, which 376 * appears as "vulnerable" in /proc. 377 */ 378 return PR_SPEC_ENABLE; 379 380 return -EINVAL; 381 } 382 383 int arch_prctl_spec_ctrl_get(struct task_struct *task, unsigned long which) 384 { 385 switch (which) { 386 case PR_SPEC_STORE_BYPASS: 387 return ssb_prctl_get(task); 388 default: 389 return -ENODEV; 390 } 391 } 392 393 #ifdef CONFIG_DEBUG_FS 394 static int stf_barrier_set(void *data, u64 val) 395 { 396 bool enable; 397 398 if (val == 1) 399 enable = true; 400 else if (val == 0) 401 enable = false; 402 else 403 return -EINVAL; 404 405 /* Only do anything if we're changing state */ 406 if (enable != stf_barrier) 407 stf_barrier_enable(enable); 408 409 return 0; 410 } 411 412 static int stf_barrier_get(void *data, u64 *val) 413 { 414 *val = stf_barrier ? 1 : 0; 415 return 0; 416 } 417 418 DEFINE_DEBUGFS_ATTRIBUTE(fops_stf_barrier, stf_barrier_get, stf_barrier_set, 419 "%llu\n"); 420 421 static __init int stf_barrier_debugfs_init(void) 422 { 423 debugfs_create_file_unsafe("stf_barrier", 0600, powerpc_debugfs_root, 424 NULL, &fops_stf_barrier); 425 return 0; 426 } 427 device_initcall(stf_barrier_debugfs_init); 428 #endif /* CONFIG_DEBUG_FS */ 429 430 static void update_branch_cache_flush(void) 431 { 432 u32 *site, __maybe_unused *site2; 433 434 #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE 435 site = &patch__call_kvm_flush_link_stack; 436 site2 = &patch__call_kvm_flush_link_stack_p9; 437 // This controls the branch from guest_exit_cont to kvm_flush_link_stack 438 if (link_stack_flush_type == BRANCH_CACHE_FLUSH_NONE) { 439 patch_instruction_site(site, ppc_inst(PPC_RAW_NOP())); 440 patch_instruction_site(site2, ppc_inst(PPC_RAW_NOP())); 441 } else { 442 // Could use HW flush, but that could also flush count cache 443 patch_branch_site(site, (u64)&kvm_flush_link_stack, BRANCH_SET_LINK); 444 patch_branch_site(site2, (u64)&kvm_flush_link_stack, BRANCH_SET_LINK); 445 } 446 #endif 447 448 // Patch out the bcctr first, then nop the rest 449 site = &patch__call_flush_branch_caches3; 450 patch_instruction_site(site, ppc_inst(PPC_RAW_NOP())); 451 site = &patch__call_flush_branch_caches2; 452 patch_instruction_site(site, ppc_inst(PPC_RAW_NOP())); 453 site = &patch__call_flush_branch_caches1; 454 patch_instruction_site(site, ppc_inst(PPC_RAW_NOP())); 455 456 // This controls the branch from _switch to flush_branch_caches 457 if (count_cache_flush_type == BRANCH_CACHE_FLUSH_NONE && 458 link_stack_flush_type == BRANCH_CACHE_FLUSH_NONE) { 459 // Nothing to be done 460 461 } else if (count_cache_flush_type == BRANCH_CACHE_FLUSH_HW && 462 link_stack_flush_type == BRANCH_CACHE_FLUSH_HW) { 463 // Patch in the bcctr last 464 site = &patch__call_flush_branch_caches1; 465 patch_instruction_site(site, ppc_inst(0x39207fff)); // li r9,0x7fff 466 site = &patch__call_flush_branch_caches2; 467 patch_instruction_site(site, ppc_inst(0x7d2903a6)); // mtctr r9 468 site = &patch__call_flush_branch_caches3; 469 patch_instruction_site(site, ppc_inst(PPC_INST_BCCTR_FLUSH)); 470 471 } else { 472 patch_branch_site(site, (u64)&flush_branch_caches, BRANCH_SET_LINK); 473 474 // If we just need to flush the link stack, early return 475 if (count_cache_flush_type == BRANCH_CACHE_FLUSH_NONE) { 476 patch_instruction_site(&patch__flush_link_stack_return, 477 ppc_inst(PPC_RAW_BLR())); 478 479 // If we have flush instruction, early return 480 } else if (count_cache_flush_type == BRANCH_CACHE_FLUSH_HW) { 481 patch_instruction_site(&patch__flush_count_cache_return, 482 ppc_inst(PPC_RAW_BLR())); 483 } 484 } 485 } 486 487 static void toggle_branch_cache_flush(bool enable) 488 { 489 if (!enable || !security_ftr_enabled(SEC_FTR_FLUSH_COUNT_CACHE)) { 490 if (count_cache_flush_type != BRANCH_CACHE_FLUSH_NONE) 491 count_cache_flush_type = BRANCH_CACHE_FLUSH_NONE; 492 493 pr_info("count-cache-flush: flush disabled.\n"); 494 } else { 495 if (security_ftr_enabled(SEC_FTR_BCCTR_FLUSH_ASSIST)) { 496 count_cache_flush_type = BRANCH_CACHE_FLUSH_HW; 497 pr_info("count-cache-flush: hardware flush enabled.\n"); 498 } else { 499 count_cache_flush_type = BRANCH_CACHE_FLUSH_SW; 500 pr_info("count-cache-flush: software flush enabled.\n"); 501 } 502 } 503 504 if (!enable || !security_ftr_enabled(SEC_FTR_FLUSH_LINK_STACK)) { 505 if (link_stack_flush_type != BRANCH_CACHE_FLUSH_NONE) 506 link_stack_flush_type = BRANCH_CACHE_FLUSH_NONE; 507 508 pr_info("link-stack-flush: flush disabled.\n"); 509 } else { 510 if (security_ftr_enabled(SEC_FTR_BCCTR_LINK_FLUSH_ASSIST)) { 511 link_stack_flush_type = BRANCH_CACHE_FLUSH_HW; 512 pr_info("link-stack-flush: hardware flush enabled.\n"); 513 } else { 514 link_stack_flush_type = BRANCH_CACHE_FLUSH_SW; 515 pr_info("link-stack-flush: software flush enabled.\n"); 516 } 517 } 518 519 update_branch_cache_flush(); 520 } 521 522 void setup_count_cache_flush(void) 523 { 524 bool enable = true; 525 526 if (no_spectrev2 || cpu_mitigations_off()) { 527 if (security_ftr_enabled(SEC_FTR_BCCTRL_SERIALISED) || 528 security_ftr_enabled(SEC_FTR_COUNT_CACHE_DISABLED)) 529 pr_warn("Spectre v2 mitigations not fully under software control, can't disable\n"); 530 531 enable = false; 532 } 533 534 /* 535 * There's no firmware feature flag/hypervisor bit to tell us we need to 536 * flush the link stack on context switch. So we set it here if we see 537 * either of the Spectre v2 mitigations that aim to protect userspace. 538 */ 539 if (security_ftr_enabled(SEC_FTR_COUNT_CACHE_DISABLED) || 540 security_ftr_enabled(SEC_FTR_FLUSH_COUNT_CACHE)) 541 security_ftr_set(SEC_FTR_FLUSH_LINK_STACK); 542 543 toggle_branch_cache_flush(enable); 544 } 545 546 static enum l1d_flush_type enabled_flush_types; 547 static void *l1d_flush_fallback_area; 548 static bool no_rfi_flush; 549 static bool no_entry_flush; 550 static bool no_uaccess_flush; 551 bool rfi_flush; 552 static bool entry_flush; 553 static bool uaccess_flush; 554 DEFINE_STATIC_KEY_FALSE(uaccess_flush_key); 555 EXPORT_SYMBOL(uaccess_flush_key); 556 557 static int __init handle_no_rfi_flush(char *p) 558 { 559 pr_info("rfi-flush: disabled on command line."); 560 no_rfi_flush = true; 561 return 0; 562 } 563 early_param("no_rfi_flush", handle_no_rfi_flush); 564 565 static int __init handle_no_entry_flush(char *p) 566 { 567 pr_info("entry-flush: disabled on command line."); 568 no_entry_flush = true; 569 return 0; 570 } 571 early_param("no_entry_flush", handle_no_entry_flush); 572 573 static int __init handle_no_uaccess_flush(char *p) 574 { 575 pr_info("uaccess-flush: disabled on command line."); 576 no_uaccess_flush = true; 577 return 0; 578 } 579 early_param("no_uaccess_flush", handle_no_uaccess_flush); 580 581 /* 582 * The RFI flush is not KPTI, but because users will see doco that says to use 583 * nopti we hijack that option here to also disable the RFI flush. 584 */ 585 static int __init handle_no_pti(char *p) 586 { 587 pr_info("rfi-flush: disabling due to 'nopti' on command line.\n"); 588 handle_no_rfi_flush(NULL); 589 return 0; 590 } 591 early_param("nopti", handle_no_pti); 592 593 static void do_nothing(void *unused) 594 { 595 /* 596 * We don't need to do the flush explicitly, just enter+exit kernel is 597 * sufficient, the RFI exit handlers will do the right thing. 598 */ 599 } 600 601 void rfi_flush_enable(bool enable) 602 { 603 if (enable) { 604 do_rfi_flush_fixups(enabled_flush_types); 605 on_each_cpu(do_nothing, NULL, 1); 606 } else 607 do_rfi_flush_fixups(L1D_FLUSH_NONE); 608 609 rfi_flush = enable; 610 } 611 612 static void entry_flush_enable(bool enable) 613 { 614 if (enable) { 615 do_entry_flush_fixups(enabled_flush_types); 616 on_each_cpu(do_nothing, NULL, 1); 617 } else { 618 do_entry_flush_fixups(L1D_FLUSH_NONE); 619 } 620 621 entry_flush = enable; 622 } 623 624 static void uaccess_flush_enable(bool enable) 625 { 626 if (enable) { 627 do_uaccess_flush_fixups(enabled_flush_types); 628 static_branch_enable(&uaccess_flush_key); 629 on_each_cpu(do_nothing, NULL, 1); 630 } else { 631 static_branch_disable(&uaccess_flush_key); 632 do_uaccess_flush_fixups(L1D_FLUSH_NONE); 633 } 634 635 uaccess_flush = enable; 636 } 637 638 static void __ref init_fallback_flush(void) 639 { 640 u64 l1d_size, limit; 641 int cpu; 642 643 /* Only allocate the fallback flush area once (at boot time). */ 644 if (l1d_flush_fallback_area) 645 return; 646 647 l1d_size = ppc64_caches.l1d.size; 648 649 /* 650 * If there is no d-cache-size property in the device tree, l1d_size 651 * could be zero. That leads to the loop in the asm wrapping around to 652 * 2^64-1, and then walking off the end of the fallback area and 653 * eventually causing a page fault which is fatal. Just default to 654 * something vaguely sane. 655 */ 656 if (!l1d_size) 657 l1d_size = (64 * 1024); 658 659 limit = min(ppc64_bolted_size(), ppc64_rma_size); 660 661 /* 662 * Align to L1d size, and size it at 2x L1d size, to catch possible 663 * hardware prefetch runoff. We don't have a recipe for load patterns to 664 * reliably avoid the prefetcher. 665 */ 666 l1d_flush_fallback_area = memblock_alloc_try_nid(l1d_size * 2, 667 l1d_size, MEMBLOCK_LOW_LIMIT, 668 limit, NUMA_NO_NODE); 669 if (!l1d_flush_fallback_area) 670 panic("%s: Failed to allocate %llu bytes align=0x%llx max_addr=%pa\n", 671 __func__, l1d_size * 2, l1d_size, &limit); 672 673 674 for_each_possible_cpu(cpu) { 675 struct paca_struct *paca = paca_ptrs[cpu]; 676 paca->rfi_flush_fallback_area = l1d_flush_fallback_area; 677 paca->l1d_flush_size = l1d_size; 678 } 679 } 680 681 void setup_rfi_flush(enum l1d_flush_type types, bool enable) 682 { 683 if (types & L1D_FLUSH_FALLBACK) { 684 pr_info("rfi-flush: fallback displacement flush available\n"); 685 init_fallback_flush(); 686 } 687 688 if (types & L1D_FLUSH_ORI) 689 pr_info("rfi-flush: ori type flush available\n"); 690 691 if (types & L1D_FLUSH_MTTRIG) 692 pr_info("rfi-flush: mttrig type flush available\n"); 693 694 enabled_flush_types = types; 695 696 if (!cpu_mitigations_off() && !no_rfi_flush) 697 rfi_flush_enable(enable); 698 } 699 700 void setup_entry_flush(bool enable) 701 { 702 if (cpu_mitigations_off()) 703 return; 704 705 if (!no_entry_flush) 706 entry_flush_enable(enable); 707 } 708 709 void setup_uaccess_flush(bool enable) 710 { 711 if (cpu_mitigations_off()) 712 return; 713 714 if (!no_uaccess_flush) 715 uaccess_flush_enable(enable); 716 } 717 718 #ifdef CONFIG_DEBUG_FS 719 static int count_cache_flush_set(void *data, u64 val) 720 { 721 bool enable; 722 723 if (val == 1) 724 enable = true; 725 else if (val == 0) 726 enable = false; 727 else 728 return -EINVAL; 729 730 toggle_branch_cache_flush(enable); 731 732 return 0; 733 } 734 735 static int count_cache_flush_get(void *data, u64 *val) 736 { 737 if (count_cache_flush_type == BRANCH_CACHE_FLUSH_NONE) 738 *val = 0; 739 else 740 *val = 1; 741 742 return 0; 743 } 744 745 DEFINE_DEBUGFS_ATTRIBUTE(fops_count_cache_flush, count_cache_flush_get, 746 count_cache_flush_set, "%llu\n"); 747 748 static __init int count_cache_flush_debugfs_init(void) 749 { 750 debugfs_create_file_unsafe("count_cache_flush", 0600, 751 powerpc_debugfs_root, NULL, 752 &fops_count_cache_flush); 753 return 0; 754 } 755 device_initcall(count_cache_flush_debugfs_init); 756 757 static int rfi_flush_set(void *data, u64 val) 758 { 759 bool enable; 760 761 if (val == 1) 762 enable = true; 763 else if (val == 0) 764 enable = false; 765 else 766 return -EINVAL; 767 768 /* Only do anything if we're changing state */ 769 if (enable != rfi_flush) 770 rfi_flush_enable(enable); 771 772 return 0; 773 } 774 775 static int rfi_flush_get(void *data, u64 *val) 776 { 777 *val = rfi_flush ? 1 : 0; 778 return 0; 779 } 780 781 DEFINE_SIMPLE_ATTRIBUTE(fops_rfi_flush, rfi_flush_get, rfi_flush_set, "%llu\n"); 782 783 static int entry_flush_set(void *data, u64 val) 784 { 785 bool enable; 786 787 if (val == 1) 788 enable = true; 789 else if (val == 0) 790 enable = false; 791 else 792 return -EINVAL; 793 794 /* Only do anything if we're changing state */ 795 if (enable != entry_flush) 796 entry_flush_enable(enable); 797 798 return 0; 799 } 800 801 static int entry_flush_get(void *data, u64 *val) 802 { 803 *val = entry_flush ? 1 : 0; 804 return 0; 805 } 806 807 DEFINE_SIMPLE_ATTRIBUTE(fops_entry_flush, entry_flush_get, entry_flush_set, "%llu\n"); 808 809 static int uaccess_flush_set(void *data, u64 val) 810 { 811 bool enable; 812 813 if (val == 1) 814 enable = true; 815 else if (val == 0) 816 enable = false; 817 else 818 return -EINVAL; 819 820 /* Only do anything if we're changing state */ 821 if (enable != uaccess_flush) 822 uaccess_flush_enable(enable); 823 824 return 0; 825 } 826 827 static int uaccess_flush_get(void *data, u64 *val) 828 { 829 *val = uaccess_flush ? 1 : 0; 830 return 0; 831 } 832 833 DEFINE_SIMPLE_ATTRIBUTE(fops_uaccess_flush, uaccess_flush_get, uaccess_flush_set, "%llu\n"); 834 835 static __init int rfi_flush_debugfs_init(void) 836 { 837 debugfs_create_file("rfi_flush", 0600, powerpc_debugfs_root, NULL, &fops_rfi_flush); 838 debugfs_create_file("entry_flush", 0600, powerpc_debugfs_root, NULL, &fops_entry_flush); 839 debugfs_create_file("uaccess_flush", 0600, powerpc_debugfs_root, NULL, &fops_uaccess_flush); 840 return 0; 841 } 842 device_initcall(rfi_flush_debugfs_init); 843 #endif /* CONFIG_DEBUG_FS */ 844 #endif /* CONFIG_PPC_BOOK3S_64 */ 845