1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copied from arch/arm64/kernel/cpufeature.c 4 * 5 * Copyright (C) 2015 ARM Ltd. 6 * Copyright (C) 2017 SiFive 7 */ 8 9 #include <linux/bitmap.h> 10 #include <linux/ctype.h> 11 #include <linux/libfdt.h> 12 #include <linux/module.h> 13 #include <linux/of.h> 14 #include <asm/alternative.h> 15 #include <asm/errata_list.h> 16 #include <asm/hwcap.h> 17 #include <asm/patch.h> 18 #include <asm/pgtable.h> 19 #include <asm/processor.h> 20 #include <asm/smp.h> 21 #include <asm/switch_to.h> 22 23 #define NUM_ALPHA_EXTS ('z' - 'a' + 1) 24 25 unsigned long elf_hwcap __read_mostly; 26 27 /* Host ISA bitmap */ 28 static DECLARE_BITMAP(riscv_isa, RISCV_ISA_EXT_MAX) __read_mostly; 29 30 #ifdef CONFIG_FPU 31 __ro_after_init DEFINE_STATIC_KEY_FALSE(cpu_hwcap_fpu); 32 #endif 33 34 /** 35 * riscv_isa_extension_base() - Get base extension word 36 * 37 * @isa_bitmap: ISA bitmap to use 38 * Return: base extension word as unsigned long value 39 * 40 * NOTE: If isa_bitmap is NULL then Host ISA bitmap will be used. 41 */ 42 unsigned long riscv_isa_extension_base(const unsigned long *isa_bitmap) 43 { 44 if (!isa_bitmap) 45 return riscv_isa[0]; 46 return isa_bitmap[0]; 47 } 48 EXPORT_SYMBOL_GPL(riscv_isa_extension_base); 49 50 /** 51 * __riscv_isa_extension_available() - Check whether given extension 52 * is available or not 53 * 54 * @isa_bitmap: ISA bitmap to use 55 * @bit: bit position of the desired extension 56 * Return: true or false 57 * 58 * NOTE: If isa_bitmap is NULL then Host ISA bitmap will be used. 59 */ 60 bool __riscv_isa_extension_available(const unsigned long *isa_bitmap, int bit) 61 { 62 const unsigned long *bmap = (isa_bitmap) ? isa_bitmap : riscv_isa; 63 64 if (bit >= RISCV_ISA_EXT_MAX) 65 return false; 66 67 return test_bit(bit, bmap) ? true : false; 68 } 69 EXPORT_SYMBOL_GPL(__riscv_isa_extension_available); 70 71 void __init riscv_fill_hwcap(void) 72 { 73 struct device_node *node; 74 const char *isa; 75 char print_str[NUM_ALPHA_EXTS + 1]; 76 int i, j; 77 static unsigned long isa2hwcap[256] = {0}; 78 79 isa2hwcap['i'] = isa2hwcap['I'] = COMPAT_HWCAP_ISA_I; 80 isa2hwcap['m'] = isa2hwcap['M'] = COMPAT_HWCAP_ISA_M; 81 isa2hwcap['a'] = isa2hwcap['A'] = COMPAT_HWCAP_ISA_A; 82 isa2hwcap['f'] = isa2hwcap['F'] = COMPAT_HWCAP_ISA_F; 83 isa2hwcap['d'] = isa2hwcap['D'] = COMPAT_HWCAP_ISA_D; 84 isa2hwcap['c'] = isa2hwcap['C'] = COMPAT_HWCAP_ISA_C; 85 86 elf_hwcap = 0; 87 88 bitmap_zero(riscv_isa, RISCV_ISA_EXT_MAX); 89 90 for_each_of_cpu_node(node) { 91 unsigned long this_hwcap = 0; 92 DECLARE_BITMAP(this_isa, RISCV_ISA_EXT_MAX); 93 const char *temp; 94 95 if (riscv_of_processor_hartid(node) < 0) 96 continue; 97 98 if (of_property_read_string(node, "riscv,isa", &isa)) { 99 pr_warn("Unable to find \"riscv,isa\" devicetree entry\n"); 100 continue; 101 } 102 103 temp = isa; 104 #if IS_ENABLED(CONFIG_32BIT) 105 if (!strncmp(isa, "rv32", 4)) 106 isa += 4; 107 #elif IS_ENABLED(CONFIG_64BIT) 108 if (!strncmp(isa, "rv64", 4)) 109 isa += 4; 110 #endif 111 /* The riscv,isa DT property must start with rv64 or rv32 */ 112 if (temp == isa) 113 continue; 114 bitmap_zero(this_isa, RISCV_ISA_EXT_MAX); 115 for (; *isa; ++isa) { 116 const char *ext = isa++; 117 const char *ext_end = isa; 118 bool ext_long = false, ext_err = false; 119 120 switch (*ext) { 121 case 's': 122 /** 123 * Workaround for invalid single-letter 's' & 'u'(QEMU). 124 * No need to set the bit in riscv_isa as 's' & 'u' are 125 * not valid ISA extensions. It works until multi-letter 126 * extension starting with "Su" appears. 127 */ 128 if (ext[-1] != '_' && ext[1] == 'u') { 129 ++isa; 130 ext_err = true; 131 break; 132 } 133 fallthrough; 134 case 'x': 135 case 'z': 136 ext_long = true; 137 /* Multi-letter extension must be delimited */ 138 for (; *isa && *isa != '_'; ++isa) 139 if (unlikely(!islower(*isa) 140 && !isdigit(*isa))) 141 ext_err = true; 142 /* Parse backwards */ 143 ext_end = isa; 144 if (unlikely(ext_err)) 145 break; 146 if (!isdigit(ext_end[-1])) 147 break; 148 /* Skip the minor version */ 149 while (isdigit(*--ext_end)) 150 ; 151 if (ext_end[0] != 'p' 152 || !isdigit(ext_end[-1])) { 153 /* Advance it to offset the pre-decrement */ 154 ++ext_end; 155 break; 156 } 157 /* Skip the major version */ 158 while (isdigit(*--ext_end)) 159 ; 160 ++ext_end; 161 break; 162 default: 163 if (unlikely(!islower(*ext))) { 164 ext_err = true; 165 break; 166 } 167 /* Find next extension */ 168 if (!isdigit(*isa)) 169 break; 170 /* Skip the minor version */ 171 while (isdigit(*++isa)) 172 ; 173 if (*isa != 'p') 174 break; 175 if (!isdigit(*++isa)) { 176 --isa; 177 break; 178 } 179 /* Skip the major version */ 180 while (isdigit(*++isa)) 181 ; 182 break; 183 } 184 if (*isa != '_') 185 --isa; 186 187 #define SET_ISA_EXT_MAP(name, bit) \ 188 do { \ 189 if ((ext_end - ext == sizeof(name) - 1) && \ 190 !memcmp(ext, name, sizeof(name) - 1)) \ 191 set_bit(bit, this_isa); \ 192 } while (false) \ 193 194 if (unlikely(ext_err)) 195 continue; 196 if (!ext_long) { 197 this_hwcap |= isa2hwcap[(unsigned char)(*ext)]; 198 set_bit(*ext - 'a', this_isa); 199 } else { 200 SET_ISA_EXT_MAP("sscofpmf", RISCV_ISA_EXT_SSCOFPMF); 201 SET_ISA_EXT_MAP("svpbmt", RISCV_ISA_EXT_SVPBMT); 202 } 203 #undef SET_ISA_EXT_MAP 204 } 205 206 /* 207 * All "okay" hart should have same isa. Set HWCAP based on 208 * common capabilities of every "okay" hart, in case they don't 209 * have. 210 */ 211 if (elf_hwcap) 212 elf_hwcap &= this_hwcap; 213 else 214 elf_hwcap = this_hwcap; 215 216 if (bitmap_empty(riscv_isa, RISCV_ISA_EXT_MAX)) 217 bitmap_copy(riscv_isa, this_isa, RISCV_ISA_EXT_MAX); 218 else 219 bitmap_and(riscv_isa, riscv_isa, this_isa, RISCV_ISA_EXT_MAX); 220 } 221 222 /* We don't support systems with F but without D, so mask those out 223 * here. */ 224 if ((elf_hwcap & COMPAT_HWCAP_ISA_F) && !(elf_hwcap & COMPAT_HWCAP_ISA_D)) { 225 pr_info("This kernel does not support systems with F but not D\n"); 226 elf_hwcap &= ~COMPAT_HWCAP_ISA_F; 227 } 228 229 memset(print_str, 0, sizeof(print_str)); 230 for (i = 0, j = 0; i < NUM_ALPHA_EXTS; i++) 231 if (riscv_isa[0] & BIT_MASK(i)) 232 print_str[j++] = (char)('a' + i); 233 pr_info("riscv: base ISA extensions %s\n", print_str); 234 235 memset(print_str, 0, sizeof(print_str)); 236 for (i = 0, j = 0; i < NUM_ALPHA_EXTS; i++) 237 if (elf_hwcap & BIT_MASK(i)) 238 print_str[j++] = (char)('a' + i); 239 pr_info("riscv: ELF capabilities %s\n", print_str); 240 241 #ifdef CONFIG_FPU 242 if (elf_hwcap & (COMPAT_HWCAP_ISA_F | COMPAT_HWCAP_ISA_D)) 243 static_branch_enable(&cpu_hwcap_fpu); 244 #endif 245 } 246 247 #ifdef CONFIG_RISCV_ALTERNATIVE 248 struct cpufeature_info { 249 char name[ERRATA_STRING_LENGTH_MAX]; 250 bool (*check_func)(unsigned int stage); 251 }; 252 253 static bool __init_or_module cpufeature_svpbmt_check_func(unsigned int stage) 254 { 255 #ifdef CONFIG_RISCV_ISA_SVPBMT 256 switch (stage) { 257 case RISCV_ALTERNATIVES_EARLY_BOOT: 258 return false; 259 default: 260 return riscv_isa_extension_available(NULL, SVPBMT); 261 } 262 #endif 263 264 return false; 265 } 266 267 static const struct cpufeature_info __initdata_or_module 268 cpufeature_list[CPUFEATURE_NUMBER] = { 269 { 270 .name = "svpbmt", 271 .check_func = cpufeature_svpbmt_check_func 272 }, 273 }; 274 275 static u32 __init_or_module cpufeature_probe(unsigned int stage) 276 { 277 const struct cpufeature_info *info; 278 u32 cpu_req_feature = 0; 279 int idx; 280 281 for (idx = 0; idx < CPUFEATURE_NUMBER; idx++) { 282 info = &cpufeature_list[idx]; 283 284 if (info->check_func(stage)) 285 cpu_req_feature |= (1U << idx); 286 } 287 288 return cpu_req_feature; 289 } 290 291 void __init_or_module riscv_cpufeature_patch_func(struct alt_entry *begin, 292 struct alt_entry *end, 293 unsigned int stage) 294 { 295 u32 cpu_req_feature = cpufeature_probe(stage); 296 struct alt_entry *alt; 297 u32 tmp; 298 299 for (alt = begin; alt < end; alt++) { 300 if (alt->vendor_id != 0) 301 continue; 302 if (alt->errata_id >= CPUFEATURE_NUMBER) { 303 WARN(1, "This feature id:%d is not in kernel cpufeature list", 304 alt->errata_id); 305 continue; 306 } 307 308 tmp = (1U << alt->errata_id); 309 if (cpu_req_feature & tmp) 310 patch_text_nosync(alt->old_ptr, alt->alt_ptr, alt->alt_len); 311 } 312 } 313 #endif 314