1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Intel Transactional Synchronization Extensions (TSX) control. 4 * 5 * Copyright (C) 2019-2021 Intel Corporation 6 * 7 * Author: 8 * Pawan Gupta <pawan.kumar.gupta@linux.intel.com> 9 */ 10 11 #include <linux/cpufeature.h> 12 13 #include <asm/cmdline.h> 14 15 #include "cpu.h" 16 17 #undef pr_fmt 18 #define pr_fmt(fmt) "tsx: " fmt 19 20 enum tsx_ctrl_states tsx_ctrl_state __ro_after_init = TSX_CTRL_NOT_SUPPORTED; 21 22 static void tsx_disable(void) 23 { 24 u64 tsx; 25 26 rdmsrl(MSR_IA32_TSX_CTRL, tsx); 27 28 /* Force all transactions to immediately abort */ 29 tsx |= TSX_CTRL_RTM_DISABLE; 30 31 /* 32 * Ensure TSX support is not enumerated in CPUID. 33 * This is visible to userspace and will ensure they 34 * do not waste resources trying TSX transactions that 35 * will always abort. 36 */ 37 tsx |= TSX_CTRL_CPUID_CLEAR; 38 39 wrmsrl(MSR_IA32_TSX_CTRL, tsx); 40 } 41 42 static void tsx_enable(void) 43 { 44 u64 tsx; 45 46 rdmsrl(MSR_IA32_TSX_CTRL, tsx); 47 48 /* Enable the RTM feature in the cpu */ 49 tsx &= ~TSX_CTRL_RTM_DISABLE; 50 51 /* 52 * Ensure TSX support is enumerated in CPUID. 53 * This is visible to userspace and will ensure they 54 * can enumerate and use the TSX feature. 55 */ 56 tsx &= ~TSX_CTRL_CPUID_CLEAR; 57 58 wrmsrl(MSR_IA32_TSX_CTRL, tsx); 59 } 60 61 static enum tsx_ctrl_states x86_get_tsx_auto_mode(void) 62 { 63 if (boot_cpu_has_bug(X86_BUG_TAA)) 64 return TSX_CTRL_DISABLE; 65 66 return TSX_CTRL_ENABLE; 67 } 68 69 /* 70 * Disabling TSX is not a trivial business. 71 * 72 * First of all, there's a CPUID bit: X86_FEATURE_RTM_ALWAYS_ABORT 73 * which says that TSX is practically disabled (all transactions are 74 * aborted by default). When that bit is set, the kernel unconditionally 75 * disables TSX. 76 * 77 * In order to do that, however, it needs to dance a bit: 78 * 79 * 1. The first method to disable it is through MSR_TSX_FORCE_ABORT and 80 * the MSR is present only when *two* CPUID bits are set: 81 * 82 * - X86_FEATURE_RTM_ALWAYS_ABORT 83 * - X86_FEATURE_TSX_FORCE_ABORT 84 * 85 * 2. The second method is for CPUs which do not have the above-mentioned 86 * MSR: those use a different MSR - MSR_IA32_TSX_CTRL and disable TSX 87 * through that one. Those CPUs can also have the initially mentioned 88 * CPUID bit X86_FEATURE_RTM_ALWAYS_ABORT set and for those the same strategy 89 * applies: TSX gets disabled unconditionally. 90 * 91 * When either of the two methods are present, the kernel disables TSX and 92 * clears the respective RTM and HLE feature flags. 93 * 94 * An additional twist in the whole thing presents late microcode loading 95 * which, when done, may cause for the X86_FEATURE_RTM_ALWAYS_ABORT CPUID 96 * bit to be set after the update. 97 * 98 * A subsequent hotplug operation on any logical CPU except the BSP will 99 * cause for the supported CPUID feature bits to get re-detected and, if 100 * RTM and HLE get cleared all of a sudden, but, userspace did consult 101 * them before the update, then funny explosions will happen. Long story 102 * short: the kernel doesn't modify CPUID feature bits after booting. 103 * 104 * That's why, this function's call in init_intel() doesn't clear the 105 * feature flags. 106 */ 107 static void tsx_clear_cpuid(void) 108 { 109 u64 msr; 110 111 /* 112 * MSR_TFA_TSX_CPUID_CLEAR bit is only present when both CPUID 113 * bits RTM_ALWAYS_ABORT and TSX_FORCE_ABORT are present. 114 */ 115 if (boot_cpu_has(X86_FEATURE_RTM_ALWAYS_ABORT) && 116 boot_cpu_has(X86_FEATURE_TSX_FORCE_ABORT)) { 117 rdmsrl(MSR_TSX_FORCE_ABORT, msr); 118 msr |= MSR_TFA_TSX_CPUID_CLEAR; 119 wrmsrl(MSR_TSX_FORCE_ABORT, msr); 120 } else if (cpu_feature_enabled(X86_FEATURE_MSR_TSX_CTRL)) { 121 rdmsrl(MSR_IA32_TSX_CTRL, msr); 122 msr |= TSX_CTRL_CPUID_CLEAR; 123 wrmsrl(MSR_IA32_TSX_CTRL, msr); 124 } 125 } 126 127 /* 128 * Disable TSX development mode 129 * 130 * When the microcode released in Feb 2022 is applied, TSX will be disabled by 131 * default on some processors. MSR 0x122 (TSX_CTRL) and MSR 0x123 132 * (IA32_MCU_OPT_CTRL) can be used to re-enable TSX for development, doing so is 133 * not recommended for production deployments. In particular, applying MD_CLEAR 134 * flows for mitigation of the Intel TSX Asynchronous Abort (TAA) transient 135 * execution attack may not be effective on these processors when Intel TSX is 136 * enabled with updated microcode. 137 */ 138 static void tsx_dev_mode_disable(void) 139 { 140 u64 mcu_opt_ctrl; 141 142 /* Check if RTM_ALLOW exists */ 143 if (!boot_cpu_has_bug(X86_BUG_TAA) || 144 !cpu_feature_enabled(X86_FEATURE_MSR_TSX_CTRL) || 145 !cpu_feature_enabled(X86_FEATURE_SRBDS_CTRL)) 146 return; 147 148 rdmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_opt_ctrl); 149 150 if (mcu_opt_ctrl & RTM_ALLOW) { 151 mcu_opt_ctrl &= ~RTM_ALLOW; 152 wrmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_opt_ctrl); 153 setup_force_cpu_cap(X86_FEATURE_RTM_ALWAYS_ABORT); 154 } 155 } 156 157 void __init tsx_init(void) 158 { 159 char arg[5] = {}; 160 int ret; 161 162 tsx_dev_mode_disable(); 163 164 /* 165 * Hardware will always abort a TSX transaction when the CPUID bit 166 * RTM_ALWAYS_ABORT is set. In this case, it is better not to enumerate 167 * CPUID.RTM and CPUID.HLE bits. Clear them here. 168 */ 169 if (boot_cpu_has(X86_FEATURE_RTM_ALWAYS_ABORT)) { 170 tsx_ctrl_state = TSX_CTRL_RTM_ALWAYS_ABORT; 171 tsx_clear_cpuid(); 172 setup_clear_cpu_cap(X86_FEATURE_RTM); 173 setup_clear_cpu_cap(X86_FEATURE_HLE); 174 return; 175 } 176 177 /* 178 * TSX is controlled via MSR_IA32_TSX_CTRL. However, support for this 179 * MSR is enumerated by ARCH_CAP_TSX_MSR bit in MSR_IA32_ARCH_CAPABILITIES. 180 * 181 * TSX control (aka MSR_IA32_TSX_CTRL) is only available after a 182 * microcode update on CPUs that have their MSR_IA32_ARCH_CAPABILITIES 183 * bit MDS_NO=1. CPUs with MDS_NO=0 are not planned to get 184 * MSR_IA32_TSX_CTRL support even after a microcode update. Thus, 185 * tsx= cmdline requests will do nothing on CPUs without 186 * MSR_IA32_TSX_CTRL support. 187 */ 188 if (x86_read_arch_cap_msr() & ARCH_CAP_TSX_CTRL_MSR) { 189 setup_force_cpu_cap(X86_FEATURE_MSR_TSX_CTRL); 190 } else { 191 tsx_ctrl_state = TSX_CTRL_NOT_SUPPORTED; 192 return; 193 } 194 195 ret = cmdline_find_option(boot_command_line, "tsx", arg, sizeof(arg)); 196 if (ret >= 0) { 197 if (!strcmp(arg, "on")) { 198 tsx_ctrl_state = TSX_CTRL_ENABLE; 199 } else if (!strcmp(arg, "off")) { 200 tsx_ctrl_state = TSX_CTRL_DISABLE; 201 } else if (!strcmp(arg, "auto")) { 202 tsx_ctrl_state = x86_get_tsx_auto_mode(); 203 } else { 204 tsx_ctrl_state = TSX_CTRL_DISABLE; 205 pr_err("invalid option, defaulting to off\n"); 206 } 207 } else { 208 /* tsx= not provided */ 209 if (IS_ENABLED(CONFIG_X86_INTEL_TSX_MODE_AUTO)) 210 tsx_ctrl_state = x86_get_tsx_auto_mode(); 211 else if (IS_ENABLED(CONFIG_X86_INTEL_TSX_MODE_OFF)) 212 tsx_ctrl_state = TSX_CTRL_DISABLE; 213 else 214 tsx_ctrl_state = TSX_CTRL_ENABLE; 215 } 216 217 if (tsx_ctrl_state == TSX_CTRL_DISABLE) { 218 tsx_disable(); 219 220 /* 221 * tsx_disable() will change the state of the RTM and HLE CPUID 222 * bits. Clear them here since they are now expected to be not 223 * set. 224 */ 225 setup_clear_cpu_cap(X86_FEATURE_RTM); 226 setup_clear_cpu_cap(X86_FEATURE_HLE); 227 } else if (tsx_ctrl_state == TSX_CTRL_ENABLE) { 228 229 /* 230 * HW defaults TSX to be enabled at bootup. 231 * We may still need the TSX enable support 232 * during init for special cases like 233 * kexec after TSX is disabled. 234 */ 235 tsx_enable(); 236 237 /* 238 * tsx_enable() will change the state of the RTM and HLE CPUID 239 * bits. Force them here since they are now expected to be set. 240 */ 241 setup_force_cpu_cap(X86_FEATURE_RTM); 242 setup_force_cpu_cap(X86_FEATURE_HLE); 243 } 244 } 245 246 void tsx_ap_init(void) 247 { 248 tsx_dev_mode_disable(); 249 250 if (tsx_ctrl_state == TSX_CTRL_ENABLE) 251 tsx_enable(); 252 else if (tsx_ctrl_state == TSX_CTRL_DISABLE) 253 tsx_disable(); 254 else if (tsx_ctrl_state == TSX_CTRL_RTM_ALWAYS_ABORT) 255 /* See comment over that function for more details. */ 256 tsx_clear_cpuid(); 257 } 258