1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copied from arch/arm64/kernel/cpufeature.c 4 * 5 * Copyright (C) 2015 ARM Ltd. 6 * Copyright (C) 2017 SiFive 7 */ 8 9 #include <linux/bitmap.h> 10 #include <linux/ctype.h> 11 #include <linux/libfdt.h> 12 #include <linux/log2.h> 13 #include <linux/module.h> 14 #include <linux/of.h> 15 #include <asm/alternative.h> 16 #include <asm/cacheflush.h> 17 #include <asm/errata_list.h> 18 #include <asm/hwcap.h> 19 #include <asm/patch.h> 20 #include <asm/pgtable.h> 21 #include <asm/processor.h> 22 #include <asm/smp.h> 23 #include <asm/switch_to.h> 24 25 #define NUM_ALPHA_EXTS ('z' - 'a' + 1) 26 27 unsigned long elf_hwcap __read_mostly; 28 29 /* Host ISA bitmap */ 30 static DECLARE_BITMAP(riscv_isa, RISCV_ISA_EXT_MAX) __read_mostly; 31 32 DEFINE_STATIC_KEY_ARRAY_FALSE(riscv_isa_ext_keys, RISCV_ISA_EXT_KEY_MAX); 33 EXPORT_SYMBOL(riscv_isa_ext_keys); 34 35 /** 36 * riscv_isa_extension_base() - Get base extension word 37 * 38 * @isa_bitmap: ISA bitmap to use 39 * Return: base extension word as unsigned long value 40 * 41 * NOTE: If isa_bitmap is NULL then Host ISA bitmap will be used. 42 */ 43 unsigned long riscv_isa_extension_base(const unsigned long *isa_bitmap) 44 { 45 if (!isa_bitmap) 46 return riscv_isa[0]; 47 return isa_bitmap[0]; 48 } 49 EXPORT_SYMBOL_GPL(riscv_isa_extension_base); 50 51 /** 52 * __riscv_isa_extension_available() - Check whether given extension 53 * is available or not 54 * 55 * @isa_bitmap: ISA bitmap to use 56 * @bit: bit position of the desired extension 57 * Return: true or false 58 * 59 * NOTE: If isa_bitmap is NULL then Host ISA bitmap will be used. 60 */ 61 bool __riscv_isa_extension_available(const unsigned long *isa_bitmap, int bit) 62 { 63 const unsigned long *bmap = (isa_bitmap) ? isa_bitmap : riscv_isa; 64 65 if (bit >= RISCV_ISA_EXT_MAX) 66 return false; 67 68 return test_bit(bit, bmap) ? true : false; 69 } 70 EXPORT_SYMBOL_GPL(__riscv_isa_extension_available); 71 72 static bool riscv_isa_extension_check(int id) 73 { 74 switch (id) { 75 case RISCV_ISA_EXT_ZICBOM: 76 if (!riscv_cbom_block_size) { 77 pr_err("Zicbom detected in ISA string, but no cbom-block-size found\n"); 78 return false; 79 } else if (!is_power_of_2(riscv_cbom_block_size)) { 80 pr_err("cbom-block-size present, but is not a power-of-2\n"); 81 return false; 82 } 83 return true; 84 } 85 86 return true; 87 } 88 89 void __init riscv_fill_hwcap(void) 90 { 91 struct device_node *node; 92 const char *isa; 93 char print_str[NUM_ALPHA_EXTS + 1]; 94 int i, j, rc; 95 unsigned long isa2hwcap[26] = {0}; 96 unsigned long hartid; 97 98 isa2hwcap['i' - 'a'] = COMPAT_HWCAP_ISA_I; 99 isa2hwcap['m' - 'a'] = COMPAT_HWCAP_ISA_M; 100 isa2hwcap['a' - 'a'] = COMPAT_HWCAP_ISA_A; 101 isa2hwcap['f' - 'a'] = COMPAT_HWCAP_ISA_F; 102 isa2hwcap['d' - 'a'] = COMPAT_HWCAP_ISA_D; 103 isa2hwcap['c' - 'a'] = COMPAT_HWCAP_ISA_C; 104 105 elf_hwcap = 0; 106 107 bitmap_zero(riscv_isa, RISCV_ISA_EXT_MAX); 108 109 for_each_of_cpu_node(node) { 110 unsigned long this_hwcap = 0; 111 DECLARE_BITMAP(this_isa, RISCV_ISA_EXT_MAX); 112 const char *temp; 113 114 rc = riscv_of_processor_hartid(node, &hartid); 115 if (rc < 0) 116 continue; 117 118 if (of_property_read_string(node, "riscv,isa", &isa)) { 119 pr_warn("Unable to find \"riscv,isa\" devicetree entry\n"); 120 continue; 121 } 122 123 temp = isa; 124 #if IS_ENABLED(CONFIG_32BIT) 125 if (!strncmp(isa, "rv32", 4)) 126 isa += 4; 127 #elif IS_ENABLED(CONFIG_64BIT) 128 if (!strncmp(isa, "rv64", 4)) 129 isa += 4; 130 #endif 131 /* The riscv,isa DT property must start with rv64 or rv32 */ 132 if (temp == isa) 133 continue; 134 bitmap_zero(this_isa, RISCV_ISA_EXT_MAX); 135 for (; *isa; ++isa) { 136 const char *ext = isa++; 137 const char *ext_end = isa; 138 bool ext_long = false, ext_err = false; 139 140 switch (*ext) { 141 case 's': 142 /** 143 * Workaround for invalid single-letter 's' & 'u'(QEMU). 144 * No need to set the bit in riscv_isa as 's' & 'u' are 145 * not valid ISA extensions. It works until multi-letter 146 * extension starting with "Su" appears. 147 */ 148 if (ext[-1] != '_' && ext[1] == 'u') { 149 ++isa; 150 ext_err = true; 151 break; 152 } 153 fallthrough; 154 case 'x': 155 case 'z': 156 ext_long = true; 157 /* Multi-letter extension must be delimited */ 158 for (; *isa && *isa != '_'; ++isa) 159 if (unlikely(!islower(*isa) 160 && !isdigit(*isa))) 161 ext_err = true; 162 /* Parse backwards */ 163 ext_end = isa; 164 if (unlikely(ext_err)) 165 break; 166 if (!isdigit(ext_end[-1])) 167 break; 168 /* Skip the minor version */ 169 while (isdigit(*--ext_end)) 170 ; 171 if (ext_end[0] != 'p' 172 || !isdigit(ext_end[-1])) { 173 /* Advance it to offset the pre-decrement */ 174 ++ext_end; 175 break; 176 } 177 /* Skip the major version */ 178 while (isdigit(*--ext_end)) 179 ; 180 ++ext_end; 181 break; 182 default: 183 if (unlikely(!islower(*ext))) { 184 ext_err = true; 185 break; 186 } 187 /* Find next extension */ 188 if (!isdigit(*isa)) 189 break; 190 /* Skip the minor version */ 191 while (isdigit(*++isa)) 192 ; 193 if (*isa != 'p') 194 break; 195 if (!isdigit(*++isa)) { 196 --isa; 197 break; 198 } 199 /* Skip the major version */ 200 while (isdigit(*++isa)) 201 ; 202 break; 203 } 204 if (*isa != '_') 205 --isa; 206 207 #define SET_ISA_EXT_MAP(name, bit) \ 208 do { \ 209 if ((ext_end - ext == sizeof(name) - 1) && \ 210 !memcmp(ext, name, sizeof(name) - 1) && \ 211 riscv_isa_extension_check(bit)) \ 212 set_bit(bit, this_isa); \ 213 } while (false) \ 214 215 if (unlikely(ext_err)) 216 continue; 217 if (!ext_long) { 218 int nr = *ext - 'a'; 219 220 if (riscv_isa_extension_check(nr)) { 221 this_hwcap |= isa2hwcap[nr]; 222 set_bit(nr, this_isa); 223 } 224 } else { 225 SET_ISA_EXT_MAP("sscofpmf", RISCV_ISA_EXT_SSCOFPMF); 226 SET_ISA_EXT_MAP("svpbmt", RISCV_ISA_EXT_SVPBMT); 227 SET_ISA_EXT_MAP("zicbom", RISCV_ISA_EXT_ZICBOM); 228 SET_ISA_EXT_MAP("zihintpause", RISCV_ISA_EXT_ZIHINTPAUSE); 229 SET_ISA_EXT_MAP("sstc", RISCV_ISA_EXT_SSTC); 230 SET_ISA_EXT_MAP("svinval", RISCV_ISA_EXT_SVINVAL); 231 } 232 #undef SET_ISA_EXT_MAP 233 } 234 235 /* 236 * All "okay" hart should have same isa. Set HWCAP based on 237 * common capabilities of every "okay" hart, in case they don't 238 * have. 239 */ 240 if (elf_hwcap) 241 elf_hwcap &= this_hwcap; 242 else 243 elf_hwcap = this_hwcap; 244 245 if (bitmap_empty(riscv_isa, RISCV_ISA_EXT_MAX)) 246 bitmap_copy(riscv_isa, this_isa, RISCV_ISA_EXT_MAX); 247 else 248 bitmap_and(riscv_isa, riscv_isa, this_isa, RISCV_ISA_EXT_MAX); 249 } 250 251 /* We don't support systems with F but without D, so mask those out 252 * here. */ 253 if ((elf_hwcap & COMPAT_HWCAP_ISA_F) && !(elf_hwcap & COMPAT_HWCAP_ISA_D)) { 254 pr_info("This kernel does not support systems with F but not D\n"); 255 elf_hwcap &= ~COMPAT_HWCAP_ISA_F; 256 } 257 258 memset(print_str, 0, sizeof(print_str)); 259 for (i = 0, j = 0; i < NUM_ALPHA_EXTS; i++) 260 if (riscv_isa[0] & BIT_MASK(i)) 261 print_str[j++] = (char)('a' + i); 262 pr_info("riscv: base ISA extensions %s\n", print_str); 263 264 memset(print_str, 0, sizeof(print_str)); 265 for (i = 0, j = 0; i < NUM_ALPHA_EXTS; i++) 266 if (elf_hwcap & BIT_MASK(i)) 267 print_str[j++] = (char)('a' + i); 268 pr_info("riscv: ELF capabilities %s\n", print_str); 269 270 for_each_set_bit(i, riscv_isa, RISCV_ISA_EXT_MAX) { 271 j = riscv_isa_ext2key(i); 272 if (j >= 0) 273 static_branch_enable(&riscv_isa_ext_keys[j]); 274 } 275 } 276 277 #ifdef CONFIG_RISCV_ALTERNATIVE 278 static bool __init_or_module cpufeature_probe_svpbmt(unsigned int stage) 279 { 280 if (!IS_ENABLED(CONFIG_RISCV_ISA_SVPBMT)) 281 return false; 282 283 if (stage == RISCV_ALTERNATIVES_EARLY_BOOT) 284 return false; 285 286 return riscv_isa_extension_available(NULL, SVPBMT); 287 } 288 289 static bool __init_or_module cpufeature_probe_zicbom(unsigned int stage) 290 { 291 if (!IS_ENABLED(CONFIG_RISCV_ISA_ZICBOM)) 292 return false; 293 294 if (stage == RISCV_ALTERNATIVES_EARLY_BOOT) 295 return false; 296 297 if (!riscv_isa_extension_available(NULL, ZICBOM)) 298 return false; 299 300 riscv_noncoherent_supported(); 301 return true; 302 } 303 304 /* 305 * Probe presence of individual extensions. 306 * 307 * This code may also be executed before kernel relocation, so we cannot use 308 * addresses generated by the address-of operator as they won't be valid in 309 * this context. 310 */ 311 static u32 __init_or_module cpufeature_probe(unsigned int stage) 312 { 313 u32 cpu_req_feature = 0; 314 315 if (cpufeature_probe_svpbmt(stage)) 316 cpu_req_feature |= BIT(CPUFEATURE_SVPBMT); 317 318 if (cpufeature_probe_zicbom(stage)) 319 cpu_req_feature |= BIT(CPUFEATURE_ZICBOM); 320 321 return cpu_req_feature; 322 } 323 324 void __init_or_module riscv_cpufeature_patch_func(struct alt_entry *begin, 325 struct alt_entry *end, 326 unsigned int stage) 327 { 328 u32 cpu_req_feature = cpufeature_probe(stage); 329 struct alt_entry *alt; 330 u32 tmp; 331 332 for (alt = begin; alt < end; alt++) { 333 if (alt->vendor_id != 0) 334 continue; 335 if (alt->errata_id >= CPUFEATURE_NUMBER) { 336 WARN(1, "This feature id:%d is not in kernel cpufeature list", 337 alt->errata_id); 338 continue; 339 } 340 341 tmp = (1U << alt->errata_id); 342 if (cpu_req_feature & tmp) 343 patch_text_nosync(alt->old_ptr, alt->alt_ptr, alt->alt_len); 344 } 345 } 346 #endif 347