1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copied from arch/arm64/kernel/cpufeature.c 4 * 5 * Copyright (C) 2015 ARM Ltd. 6 * Copyright (C) 2017 SiFive 7 */ 8 9 #include <linux/bitmap.h> 10 #include <linux/ctype.h> 11 #include <linux/libfdt.h> 12 #include <linux/module.h> 13 #include <linux/of.h> 14 #include <asm/alternative.h> 15 #include <asm/cacheflush.h> 16 #include <asm/errata_list.h> 17 #include <asm/hwcap.h> 18 #include <asm/patch.h> 19 #include <asm/pgtable.h> 20 #include <asm/processor.h> 21 #include <asm/smp.h> 22 #include <asm/switch_to.h> 23 24 #define NUM_ALPHA_EXTS ('z' - 'a' + 1) 25 26 unsigned long elf_hwcap __read_mostly; 27 28 /* Host ISA bitmap */ 29 static DECLARE_BITMAP(riscv_isa, RISCV_ISA_EXT_MAX) __read_mostly; 30 31 __ro_after_init DEFINE_STATIC_KEY_ARRAY_FALSE(riscv_isa_ext_keys, RISCV_ISA_EXT_KEY_MAX); 32 EXPORT_SYMBOL(riscv_isa_ext_keys); 33 34 /** 35 * riscv_isa_extension_base() - Get base extension word 36 * 37 * @isa_bitmap: ISA bitmap to use 38 * Return: base extension word as unsigned long value 39 * 40 * NOTE: If isa_bitmap is NULL then Host ISA bitmap will be used. 41 */ 42 unsigned long riscv_isa_extension_base(const unsigned long *isa_bitmap) 43 { 44 if (!isa_bitmap) 45 return riscv_isa[0]; 46 return isa_bitmap[0]; 47 } 48 EXPORT_SYMBOL_GPL(riscv_isa_extension_base); 49 50 /** 51 * __riscv_isa_extension_available() - Check whether given extension 52 * is available or not 53 * 54 * @isa_bitmap: ISA bitmap to use 55 * @bit: bit position of the desired extension 56 * Return: true or false 57 * 58 * NOTE: If isa_bitmap is NULL then Host ISA bitmap will be used. 59 */ 60 bool __riscv_isa_extension_available(const unsigned long *isa_bitmap, int bit) 61 { 62 const unsigned long *bmap = (isa_bitmap) ? isa_bitmap : riscv_isa; 63 64 if (bit >= RISCV_ISA_EXT_MAX) 65 return false; 66 67 return test_bit(bit, bmap) ? true : false; 68 } 69 EXPORT_SYMBOL_GPL(__riscv_isa_extension_available); 70 71 void __init riscv_fill_hwcap(void) 72 { 73 struct device_node *node; 74 const char *isa; 75 char print_str[NUM_ALPHA_EXTS + 1]; 76 int i, j, rc; 77 static unsigned long isa2hwcap[256] = {0}; 78 unsigned long hartid; 79 80 isa2hwcap['i'] = isa2hwcap['I'] = COMPAT_HWCAP_ISA_I; 81 isa2hwcap['m'] = isa2hwcap['M'] = COMPAT_HWCAP_ISA_M; 82 isa2hwcap['a'] = isa2hwcap['A'] = COMPAT_HWCAP_ISA_A; 83 isa2hwcap['f'] = isa2hwcap['F'] = COMPAT_HWCAP_ISA_F; 84 isa2hwcap['d'] = isa2hwcap['D'] = COMPAT_HWCAP_ISA_D; 85 isa2hwcap['c'] = isa2hwcap['C'] = COMPAT_HWCAP_ISA_C; 86 87 elf_hwcap = 0; 88 89 bitmap_zero(riscv_isa, RISCV_ISA_EXT_MAX); 90 91 for_each_of_cpu_node(node) { 92 unsigned long this_hwcap = 0; 93 DECLARE_BITMAP(this_isa, RISCV_ISA_EXT_MAX); 94 const char *temp; 95 96 rc = riscv_of_processor_hartid(node, &hartid); 97 if (rc < 0) 98 continue; 99 100 if (of_property_read_string(node, "riscv,isa", &isa)) { 101 pr_warn("Unable to find \"riscv,isa\" devicetree entry\n"); 102 continue; 103 } 104 105 temp = isa; 106 #if IS_ENABLED(CONFIG_32BIT) 107 if (!strncmp(isa, "rv32", 4)) 108 isa += 4; 109 #elif IS_ENABLED(CONFIG_64BIT) 110 if (!strncmp(isa, "rv64", 4)) 111 isa += 4; 112 #endif 113 /* The riscv,isa DT property must start with rv64 or rv32 */ 114 if (temp == isa) 115 continue; 116 bitmap_zero(this_isa, RISCV_ISA_EXT_MAX); 117 for (; *isa; ++isa) { 118 const char *ext = isa++; 119 const char *ext_end = isa; 120 bool ext_long = false, ext_err = false; 121 122 switch (*ext) { 123 case 's': 124 /** 125 * Workaround for invalid single-letter 's' & 'u'(QEMU). 126 * No need to set the bit in riscv_isa as 's' & 'u' are 127 * not valid ISA extensions. It works until multi-letter 128 * extension starting with "Su" appears. 129 */ 130 if (ext[-1] != '_' && ext[1] == 'u') { 131 ++isa; 132 ext_err = true; 133 break; 134 } 135 fallthrough; 136 case 'x': 137 case 'z': 138 ext_long = true; 139 /* Multi-letter extension must be delimited */ 140 for (; *isa && *isa != '_'; ++isa) 141 if (unlikely(!islower(*isa) 142 && !isdigit(*isa))) 143 ext_err = true; 144 /* Parse backwards */ 145 ext_end = isa; 146 if (unlikely(ext_err)) 147 break; 148 if (!isdigit(ext_end[-1])) 149 break; 150 /* Skip the minor version */ 151 while (isdigit(*--ext_end)) 152 ; 153 if (ext_end[0] != 'p' 154 || !isdigit(ext_end[-1])) { 155 /* Advance it to offset the pre-decrement */ 156 ++ext_end; 157 break; 158 } 159 /* Skip the major version */ 160 while (isdigit(*--ext_end)) 161 ; 162 ++ext_end; 163 break; 164 default: 165 if (unlikely(!islower(*ext))) { 166 ext_err = true; 167 break; 168 } 169 /* Find next extension */ 170 if (!isdigit(*isa)) 171 break; 172 /* Skip the minor version */ 173 while (isdigit(*++isa)) 174 ; 175 if (*isa != 'p') 176 break; 177 if (!isdigit(*++isa)) { 178 --isa; 179 break; 180 } 181 /* Skip the major version */ 182 while (isdigit(*++isa)) 183 ; 184 break; 185 } 186 if (*isa != '_') 187 --isa; 188 189 #define SET_ISA_EXT_MAP(name, bit) \ 190 do { \ 191 if ((ext_end - ext == sizeof(name) - 1) && \ 192 !memcmp(ext, name, sizeof(name) - 1)) \ 193 set_bit(bit, this_isa); \ 194 } while (false) \ 195 196 if (unlikely(ext_err)) 197 continue; 198 if (!ext_long) { 199 this_hwcap |= isa2hwcap[(unsigned char)(*ext)]; 200 set_bit(*ext - 'a', this_isa); 201 } else { 202 SET_ISA_EXT_MAP("sscofpmf", RISCV_ISA_EXT_SSCOFPMF); 203 SET_ISA_EXT_MAP("svpbmt", RISCV_ISA_EXT_SVPBMT); 204 SET_ISA_EXT_MAP("zicbom", RISCV_ISA_EXT_ZICBOM); 205 SET_ISA_EXT_MAP("zihintpause", RISCV_ISA_EXT_ZIHINTPAUSE); 206 SET_ISA_EXT_MAP("sstc", RISCV_ISA_EXT_SSTC); 207 } 208 #undef SET_ISA_EXT_MAP 209 } 210 211 /* 212 * All "okay" hart should have same isa. Set HWCAP based on 213 * common capabilities of every "okay" hart, in case they don't 214 * have. 215 */ 216 if (elf_hwcap) 217 elf_hwcap &= this_hwcap; 218 else 219 elf_hwcap = this_hwcap; 220 221 if (bitmap_empty(riscv_isa, RISCV_ISA_EXT_MAX)) 222 bitmap_copy(riscv_isa, this_isa, RISCV_ISA_EXT_MAX); 223 else 224 bitmap_and(riscv_isa, riscv_isa, this_isa, RISCV_ISA_EXT_MAX); 225 } 226 227 /* We don't support systems with F but without D, so mask those out 228 * here. */ 229 if ((elf_hwcap & COMPAT_HWCAP_ISA_F) && !(elf_hwcap & COMPAT_HWCAP_ISA_D)) { 230 pr_info("This kernel does not support systems with F but not D\n"); 231 elf_hwcap &= ~COMPAT_HWCAP_ISA_F; 232 } 233 234 memset(print_str, 0, sizeof(print_str)); 235 for (i = 0, j = 0; i < NUM_ALPHA_EXTS; i++) 236 if (riscv_isa[0] & BIT_MASK(i)) 237 print_str[j++] = (char)('a' + i); 238 pr_info("riscv: base ISA extensions %s\n", print_str); 239 240 memset(print_str, 0, sizeof(print_str)); 241 for (i = 0, j = 0; i < NUM_ALPHA_EXTS; i++) 242 if (elf_hwcap & BIT_MASK(i)) 243 print_str[j++] = (char)('a' + i); 244 pr_info("riscv: ELF capabilities %s\n", print_str); 245 246 for_each_set_bit(i, riscv_isa, RISCV_ISA_EXT_MAX) { 247 j = riscv_isa_ext2key(i); 248 if (j >= 0) 249 static_branch_enable(&riscv_isa_ext_keys[j]); 250 } 251 } 252 253 #ifdef CONFIG_RISCV_ALTERNATIVE 254 static bool __init_or_module cpufeature_probe_svpbmt(unsigned int stage) 255 { 256 #ifdef CONFIG_RISCV_ISA_SVPBMT 257 switch (stage) { 258 case RISCV_ALTERNATIVES_EARLY_BOOT: 259 return false; 260 default: 261 return riscv_isa_extension_available(NULL, SVPBMT); 262 } 263 #endif 264 265 return false; 266 } 267 268 static bool __init_or_module cpufeature_probe_zicbom(unsigned int stage) 269 { 270 #ifdef CONFIG_RISCV_ISA_ZICBOM 271 switch (stage) { 272 case RISCV_ALTERNATIVES_EARLY_BOOT: 273 return false; 274 default: 275 if (riscv_isa_extension_available(NULL, ZICBOM)) { 276 riscv_noncoherent_supported(); 277 return true; 278 } else { 279 return false; 280 } 281 } 282 #endif 283 284 return false; 285 } 286 287 /* 288 * Probe presence of individual extensions. 289 * 290 * This code may also be executed before kernel relocation, so we cannot use 291 * addresses generated by the address-of operator as they won't be valid in 292 * this context. 293 */ 294 static u32 __init_or_module cpufeature_probe(unsigned int stage) 295 { 296 u32 cpu_req_feature = 0; 297 298 if (cpufeature_probe_svpbmt(stage)) 299 cpu_req_feature |= (1U << CPUFEATURE_SVPBMT); 300 301 if (cpufeature_probe_zicbom(stage)) 302 cpu_req_feature |= (1U << CPUFEATURE_ZICBOM); 303 304 return cpu_req_feature; 305 } 306 307 void __init_or_module riscv_cpufeature_patch_func(struct alt_entry *begin, 308 struct alt_entry *end, 309 unsigned int stage) 310 { 311 u32 cpu_req_feature = cpufeature_probe(stage); 312 struct alt_entry *alt; 313 u32 tmp; 314 315 for (alt = begin; alt < end; alt++) { 316 if (alt->vendor_id != 0) 317 continue; 318 if (alt->errata_id >= CPUFEATURE_NUMBER) { 319 WARN(1, "This feature id:%d is not in kernel cpufeature list", 320 alt->errata_id); 321 continue; 322 } 323 324 tmp = (1U << alt->errata_id); 325 if (cpu_req_feature & tmp) 326 patch_text_nosync(alt->old_ptr, alt->alt_ptr, alt->alt_len); 327 } 328 } 329 #endif 330