1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _ASM_X86_CPUFEATURE_H 3 #define _ASM_X86_CPUFEATURE_H 4 5 #include <asm/processor.h> 6 7 #if defined(__KERNEL__) && !defined(__ASSEMBLY__) 8 9 #include <asm/asm.h> 10 #include <linux/bitops.h> 11 12 enum cpuid_leafs 13 { 14 CPUID_1_EDX = 0, 15 CPUID_8000_0001_EDX, 16 CPUID_8086_0001_EDX, 17 CPUID_LNX_1, 18 CPUID_1_ECX, 19 CPUID_C000_0001_EDX, 20 CPUID_8000_0001_ECX, 21 CPUID_LNX_2, 22 CPUID_LNX_3, 23 CPUID_7_0_EBX, 24 CPUID_D_1_EAX, 25 CPUID_LNX_4, 26 CPUID_7_1_EAX, 27 CPUID_8000_0008_EBX, 28 CPUID_6_EAX, 29 CPUID_8000_000A_EDX, 30 CPUID_7_ECX, 31 CPUID_8000_0007_EBX, 32 CPUID_7_EDX, 33 CPUID_8000_001F_EAX, 34 }; 35 36 #ifdef CONFIG_X86_FEATURE_NAMES 37 extern const char * const x86_cap_flags[NCAPINTS*32]; 38 extern const char * const x86_power_flags[32]; 39 #define X86_CAP_FMT "%s" 40 #define x86_cap_flag(flag) x86_cap_flags[flag] 41 #else 42 #define X86_CAP_FMT "%d:%d" 43 #define x86_cap_flag(flag) ((flag) >> 5), ((flag) & 31) 44 #endif 45 46 /* 47 * In order to save room, we index into this array by doing 48 * X86_BUG_<name> - NCAPINTS*32. 49 */ 50 extern const char * const x86_bug_flags[NBUGINTS*32]; 51 52 #define test_cpu_cap(c, bit) \ 53 test_bit(bit, (unsigned long *)((c)->x86_capability)) 54 55 /* 56 * There are 32 bits/features in each mask word. The high bits 57 * (selected with (bit>>5) give us the word number and the low 5 58 * bits give us the bit/feature number inside the word. 59 * (1UL<<((bit)&31) gives us a mask for the feature_bit so we can 60 * see if it is set in the mask word. 61 */ 62 #define CHECK_BIT_IN_MASK_WORD(maskname, word, bit) \ 63 (((bit)>>5)==(word) && (1UL<<((bit)&31) & maskname##word )) 64 65 /* 66 * {REQUIRED,DISABLED}_MASK_CHECK below may seem duplicated with the 67 * following BUILD_BUG_ON_ZERO() check but when NCAPINTS gets changed, all 68 * header macros which use NCAPINTS need to be changed. The duplicated macro 69 * use causes the compiler to issue errors for all headers so that all usage 70 * sites can be corrected. 71 */ 72 #define REQUIRED_MASK_BIT_SET(feature_bit) \ 73 ( CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 0, feature_bit) || \ 74 CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 1, feature_bit) || \ 75 CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 2, feature_bit) || \ 76 CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 3, feature_bit) || \ 77 CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 4, feature_bit) || \ 78 CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 5, feature_bit) || \ 79 CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 6, feature_bit) || \ 80 CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 7, feature_bit) || \ 81 CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 8, feature_bit) || \ 82 CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 9, feature_bit) || \ 83 CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 10, feature_bit) || \ 84 CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 11, feature_bit) || \ 85 CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 12, feature_bit) || \ 86 CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 13, feature_bit) || \ 87 CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 14, feature_bit) || \ 88 CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 15, feature_bit) || \ 89 CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 16, feature_bit) || \ 90 CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 17, feature_bit) || \ 91 CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 18, feature_bit) || \ 92 CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 19, feature_bit) || \ 93 REQUIRED_MASK_CHECK || \ 94 BUILD_BUG_ON_ZERO(NCAPINTS != 20)) 95 96 #define DISABLED_MASK_BIT_SET(feature_bit) \ 97 ( CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 0, feature_bit) || \ 98 CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 1, feature_bit) || \ 99 CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 2, feature_bit) || \ 100 CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 3, feature_bit) || \ 101 CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 4, feature_bit) || \ 102 CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 5, feature_bit) || \ 103 CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 6, feature_bit) || \ 104 CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 7, feature_bit) || \ 105 CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 8, feature_bit) || \ 106 CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 9, feature_bit) || \ 107 CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 10, feature_bit) || \ 108 CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 11, feature_bit) || \ 109 CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 12, feature_bit) || \ 110 CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 13, feature_bit) || \ 111 CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 14, feature_bit) || \ 112 CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 15, feature_bit) || \ 113 CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 16, feature_bit) || \ 114 CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 17, feature_bit) || \ 115 CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 18, feature_bit) || \ 116 CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 19, feature_bit) || \ 117 DISABLED_MASK_CHECK || \ 118 BUILD_BUG_ON_ZERO(NCAPINTS != 20)) 119 120 #define cpu_has(c, bit) \ 121 (__builtin_constant_p(bit) && REQUIRED_MASK_BIT_SET(bit) ? 1 : \ 122 test_cpu_cap(c, bit)) 123 124 #define this_cpu_has(bit) \ 125 (__builtin_constant_p(bit) && REQUIRED_MASK_BIT_SET(bit) ? 1 : \ 126 x86_this_cpu_test_bit(bit, \ 127 (unsigned long __percpu *)&cpu_info.x86_capability)) 128 129 /* 130 * This macro is for detection of features which need kernel 131 * infrastructure to be used. It may *not* directly test the CPU 132 * itself. Use the cpu_has() family if you want true runtime 133 * testing of CPU features, like in hypervisor code where you are 134 * supporting a possible guest feature where host support for it 135 * is not relevant. 136 */ 137 #define cpu_feature_enabled(bit) \ 138 (__builtin_constant_p(bit) && DISABLED_MASK_BIT_SET(bit) ? 0 : static_cpu_has(bit)) 139 140 #define boot_cpu_has(bit) cpu_has(&boot_cpu_data, bit) 141 142 #define set_cpu_cap(c, bit) set_bit(bit, (unsigned long *)((c)->x86_capability)) 143 144 extern void setup_clear_cpu_cap(unsigned int bit); 145 extern void clear_cpu_cap(struct cpuinfo_x86 *c, unsigned int bit); 146 147 #define setup_force_cpu_cap(bit) do { \ 148 set_cpu_cap(&boot_cpu_data, bit); \ 149 set_bit(bit, (unsigned long *)cpu_caps_set); \ 150 } while (0) 151 152 #define setup_force_cpu_bug(bit) setup_force_cpu_cap(bit) 153 154 #if defined(__clang__) && !defined(CONFIG_CC_HAS_ASM_GOTO) 155 156 /* 157 * Workaround for the sake of BPF compilation which utilizes kernel 158 * headers, but clang does not support ASM GOTO and fails the build. 159 */ 160 #ifndef __BPF_TRACING__ 161 #warning "Compiler lacks ASM_GOTO support. Add -D __BPF_TRACING__ to your compiler arguments" 162 #endif 163 164 #define static_cpu_has(bit) boot_cpu_has(bit) 165 166 #else 167 168 /* 169 * Static testing of CPU features. Used the same as boot_cpu_has(). It 170 * statically patches the target code for additional performance. Use 171 * static_cpu_has() only in fast paths, where every cycle counts. Which 172 * means that the boot_cpu_has() variant is already fast enough for the 173 * majority of cases and you should stick to using it as it is generally 174 * only two instructions: a RIP-relative MOV and a TEST. 175 */ 176 static __always_inline bool _static_cpu_has(u16 bit) 177 { 178 asm_volatile_goto("1: jmp 6f\n" 179 "2:\n" 180 ".skip -(((5f-4f) - (2b-1b)) > 0) * " 181 "((5f-4f) - (2b-1b)),0x90\n" 182 "3:\n" 183 ".section .altinstructions,\"a\"\n" 184 " .long 1b - .\n" /* src offset */ 185 " .long 4f - .\n" /* repl offset */ 186 " .word %P[always]\n" /* always replace */ 187 " .byte 3b - 1b\n" /* src len */ 188 " .byte 5f - 4f\n" /* repl len */ 189 " .byte 3b - 2b\n" /* pad len */ 190 ".previous\n" 191 ".section .altinstr_replacement,\"ax\"\n" 192 "4: jmp %l[t_no]\n" 193 "5:\n" 194 ".previous\n" 195 ".section .altinstructions,\"a\"\n" 196 " .long 1b - .\n" /* src offset */ 197 " .long 0\n" /* no replacement */ 198 " .word %P[feature]\n" /* feature bit */ 199 " .byte 3b - 1b\n" /* src len */ 200 " .byte 0\n" /* repl len */ 201 " .byte 0\n" /* pad len */ 202 ".previous\n" 203 ".section .altinstr_aux,\"ax\"\n" 204 "6:\n" 205 " testb %[bitnum],%[cap_byte]\n" 206 " jnz %l[t_yes]\n" 207 " jmp %l[t_no]\n" 208 ".previous\n" 209 : : [feature] "i" (bit), 210 [always] "i" (X86_FEATURE_ALWAYS), 211 [bitnum] "i" (1 << (bit & 7)), 212 [cap_byte] "m" (((const char *)boot_cpu_data.x86_capability)[bit >> 3]) 213 : : t_yes, t_no); 214 t_yes: 215 return true; 216 t_no: 217 return false; 218 } 219 220 #define static_cpu_has(bit) \ 221 ( \ 222 __builtin_constant_p(boot_cpu_has(bit)) ? \ 223 boot_cpu_has(bit) : \ 224 _static_cpu_has(bit) \ 225 ) 226 #endif 227 228 #define cpu_has_bug(c, bit) cpu_has(c, (bit)) 229 #define set_cpu_bug(c, bit) set_cpu_cap(c, (bit)) 230 #define clear_cpu_bug(c, bit) clear_cpu_cap(c, (bit)) 231 232 #define static_cpu_has_bug(bit) static_cpu_has((bit)) 233 #define boot_cpu_has_bug(bit) cpu_has_bug(&boot_cpu_data, (bit)) 234 #define boot_cpu_set_bug(bit) set_cpu_cap(&boot_cpu_data, (bit)) 235 236 #define MAX_CPU_FEATURES (NCAPINTS * 32) 237 #define cpu_have_feature boot_cpu_has 238 239 #define CPU_FEATURE_TYPEFMT "x86,ven%04Xfam%04Xmod%04X" 240 #define CPU_FEATURE_TYPEVAL boot_cpu_data.x86_vendor, boot_cpu_data.x86, \ 241 boot_cpu_data.x86_model 242 243 #endif /* defined(__KERNEL__) && !defined(__ASSEMBLY__) */ 244 #endif /* _ASM_X86_CPUFEATURE_H */ 245