1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copied from arch/arm64/kernel/cpufeature.c 4 * 5 * Copyright (C) 2015 ARM Ltd. 6 * Copyright (C) 2017 SiFive 7 */ 8 9 #include <linux/bitmap.h> 10 #include <linux/ctype.h> 11 #include <linux/libfdt.h> 12 #include <linux/module.h> 13 #include <linux/of.h> 14 #include <asm/alternative.h> 15 #include <asm/errata_list.h> 16 #include <asm/hwcap.h> 17 #include <asm/patch.h> 18 #include <asm/pgtable.h> 19 #include <asm/processor.h> 20 #include <asm/smp.h> 21 #include <asm/switch_to.h> 22 23 #define NUM_ALPHA_EXTS ('z' - 'a' + 1) 24 25 unsigned long elf_hwcap __read_mostly; 26 27 /* Host ISA bitmap */ 28 static DECLARE_BITMAP(riscv_isa, RISCV_ISA_EXT_MAX) __read_mostly; 29 30 __ro_after_init DEFINE_STATIC_KEY_ARRAY_FALSE(riscv_isa_ext_keys, RISCV_ISA_EXT_KEY_MAX); 31 EXPORT_SYMBOL(riscv_isa_ext_keys); 32 33 /** 34 * riscv_isa_extension_base() - Get base extension word 35 * 36 * @isa_bitmap: ISA bitmap to use 37 * Return: base extension word as unsigned long value 38 * 39 * NOTE: If isa_bitmap is NULL then Host ISA bitmap will be used. 40 */ 41 unsigned long riscv_isa_extension_base(const unsigned long *isa_bitmap) 42 { 43 if (!isa_bitmap) 44 return riscv_isa[0]; 45 return isa_bitmap[0]; 46 } 47 EXPORT_SYMBOL_GPL(riscv_isa_extension_base); 48 49 /** 50 * __riscv_isa_extension_available() - Check whether given extension 51 * is available or not 52 * 53 * @isa_bitmap: ISA bitmap to use 54 * @bit: bit position of the desired extension 55 * Return: true or false 56 * 57 * NOTE: If isa_bitmap is NULL then Host ISA bitmap will be used. 58 */ 59 bool __riscv_isa_extension_available(const unsigned long *isa_bitmap, int bit) 60 { 61 const unsigned long *bmap = (isa_bitmap) ? isa_bitmap : riscv_isa; 62 63 if (bit >= RISCV_ISA_EXT_MAX) 64 return false; 65 66 return test_bit(bit, bmap) ? true : false; 67 } 68 EXPORT_SYMBOL_GPL(__riscv_isa_extension_available); 69 70 void __init riscv_fill_hwcap(void) 71 { 72 struct device_node *node; 73 const char *isa; 74 char print_str[NUM_ALPHA_EXTS + 1]; 75 int i, j, rc; 76 static unsigned long isa2hwcap[256] = {0}; 77 unsigned long hartid; 78 79 isa2hwcap['i'] = isa2hwcap['I'] = COMPAT_HWCAP_ISA_I; 80 isa2hwcap['m'] = isa2hwcap['M'] = COMPAT_HWCAP_ISA_M; 81 isa2hwcap['a'] = isa2hwcap['A'] = COMPAT_HWCAP_ISA_A; 82 isa2hwcap['f'] = isa2hwcap['F'] = COMPAT_HWCAP_ISA_F; 83 isa2hwcap['d'] = isa2hwcap['D'] = COMPAT_HWCAP_ISA_D; 84 isa2hwcap['c'] = isa2hwcap['C'] = COMPAT_HWCAP_ISA_C; 85 86 elf_hwcap = 0; 87 88 bitmap_zero(riscv_isa, RISCV_ISA_EXT_MAX); 89 90 for_each_of_cpu_node(node) { 91 unsigned long this_hwcap = 0; 92 DECLARE_BITMAP(this_isa, RISCV_ISA_EXT_MAX); 93 const char *temp; 94 95 rc = riscv_of_processor_hartid(node, &hartid); 96 if (rc < 0) 97 continue; 98 99 if (of_property_read_string(node, "riscv,isa", &isa)) { 100 pr_warn("Unable to find \"riscv,isa\" devicetree entry\n"); 101 continue; 102 } 103 104 temp = isa; 105 #if IS_ENABLED(CONFIG_32BIT) 106 if (!strncmp(isa, "rv32", 4)) 107 isa += 4; 108 #elif IS_ENABLED(CONFIG_64BIT) 109 if (!strncmp(isa, "rv64", 4)) 110 isa += 4; 111 #endif 112 /* The riscv,isa DT property must start with rv64 or rv32 */ 113 if (temp == isa) 114 continue; 115 bitmap_zero(this_isa, RISCV_ISA_EXT_MAX); 116 for (; *isa; ++isa) { 117 const char *ext = isa++; 118 const char *ext_end = isa; 119 bool ext_long = false, ext_err = false; 120 121 switch (*ext) { 122 case 's': 123 /** 124 * Workaround for invalid single-letter 's' & 'u'(QEMU). 125 * No need to set the bit in riscv_isa as 's' & 'u' are 126 * not valid ISA extensions. It works until multi-letter 127 * extension starting with "Su" appears. 128 */ 129 if (ext[-1] != '_' && ext[1] == 'u') { 130 ++isa; 131 ext_err = true; 132 break; 133 } 134 fallthrough; 135 case 'x': 136 case 'z': 137 ext_long = true; 138 /* Multi-letter extension must be delimited */ 139 for (; *isa && *isa != '_'; ++isa) 140 if (unlikely(!islower(*isa) 141 && !isdigit(*isa))) 142 ext_err = true; 143 /* Parse backwards */ 144 ext_end = isa; 145 if (unlikely(ext_err)) 146 break; 147 if (!isdigit(ext_end[-1])) 148 break; 149 /* Skip the minor version */ 150 while (isdigit(*--ext_end)) 151 ; 152 if (ext_end[0] != 'p' 153 || !isdigit(ext_end[-1])) { 154 /* Advance it to offset the pre-decrement */ 155 ++ext_end; 156 break; 157 } 158 /* Skip the major version */ 159 while (isdigit(*--ext_end)) 160 ; 161 ++ext_end; 162 break; 163 default: 164 if (unlikely(!islower(*ext))) { 165 ext_err = true; 166 break; 167 } 168 /* Find next extension */ 169 if (!isdigit(*isa)) 170 break; 171 /* Skip the minor version */ 172 while (isdigit(*++isa)) 173 ; 174 if (*isa != 'p') 175 break; 176 if (!isdigit(*++isa)) { 177 --isa; 178 break; 179 } 180 /* Skip the major version */ 181 while (isdigit(*++isa)) 182 ; 183 break; 184 } 185 if (*isa != '_') 186 --isa; 187 188 #define SET_ISA_EXT_MAP(name, bit) \ 189 do { \ 190 if ((ext_end - ext == sizeof(name) - 1) && \ 191 !memcmp(ext, name, sizeof(name) - 1)) \ 192 set_bit(bit, this_isa); \ 193 } while (false) \ 194 195 if (unlikely(ext_err)) 196 continue; 197 if (!ext_long) { 198 this_hwcap |= isa2hwcap[(unsigned char)(*ext)]; 199 set_bit(*ext - 'a', this_isa); 200 } else { 201 SET_ISA_EXT_MAP("sscofpmf", RISCV_ISA_EXT_SSCOFPMF); 202 SET_ISA_EXT_MAP("svpbmt", RISCV_ISA_EXT_SVPBMT); 203 } 204 #undef SET_ISA_EXT_MAP 205 } 206 207 /* 208 * All "okay" hart should have same isa. Set HWCAP based on 209 * common capabilities of every "okay" hart, in case they don't 210 * have. 211 */ 212 if (elf_hwcap) 213 elf_hwcap &= this_hwcap; 214 else 215 elf_hwcap = this_hwcap; 216 217 if (bitmap_empty(riscv_isa, RISCV_ISA_EXT_MAX)) 218 bitmap_copy(riscv_isa, this_isa, RISCV_ISA_EXT_MAX); 219 else 220 bitmap_and(riscv_isa, riscv_isa, this_isa, RISCV_ISA_EXT_MAX); 221 } 222 223 /* We don't support systems with F but without D, so mask those out 224 * here. */ 225 if ((elf_hwcap & COMPAT_HWCAP_ISA_F) && !(elf_hwcap & COMPAT_HWCAP_ISA_D)) { 226 pr_info("This kernel does not support systems with F but not D\n"); 227 elf_hwcap &= ~COMPAT_HWCAP_ISA_F; 228 } 229 230 memset(print_str, 0, sizeof(print_str)); 231 for (i = 0, j = 0; i < NUM_ALPHA_EXTS; i++) 232 if (riscv_isa[0] & BIT_MASK(i)) 233 print_str[j++] = (char)('a' + i); 234 pr_info("riscv: base ISA extensions %s\n", print_str); 235 236 memset(print_str, 0, sizeof(print_str)); 237 for (i = 0, j = 0; i < NUM_ALPHA_EXTS; i++) 238 if (elf_hwcap & BIT_MASK(i)) 239 print_str[j++] = (char)('a' + i); 240 pr_info("riscv: ELF capabilities %s\n", print_str); 241 242 for_each_set_bit(i, riscv_isa, RISCV_ISA_EXT_MAX) { 243 j = riscv_isa_ext2key(i); 244 if (j >= 0) 245 static_branch_enable(&riscv_isa_ext_keys[j]); 246 } 247 } 248 249 #ifdef CONFIG_RISCV_ALTERNATIVE 250 static bool __init_or_module cpufeature_probe_svpbmt(unsigned int stage) 251 { 252 #ifdef CONFIG_RISCV_ISA_SVPBMT 253 switch (stage) { 254 case RISCV_ALTERNATIVES_EARLY_BOOT: 255 return false; 256 default: 257 return riscv_isa_extension_available(NULL, SVPBMT); 258 } 259 #endif 260 261 return false; 262 } 263 264 /* 265 * Probe presence of individual extensions. 266 * 267 * This code may also be executed before kernel relocation, so we cannot use 268 * addresses generated by the address-of operator as they won't be valid in 269 * this context. 270 */ 271 static u32 __init_or_module cpufeature_probe(unsigned int stage) 272 { 273 u32 cpu_req_feature = 0; 274 275 if (cpufeature_probe_svpbmt(stage)) 276 cpu_req_feature |= (1U << CPUFEATURE_SVPBMT); 277 278 return cpu_req_feature; 279 } 280 281 void __init_or_module riscv_cpufeature_patch_func(struct alt_entry *begin, 282 struct alt_entry *end, 283 unsigned int stage) 284 { 285 u32 cpu_req_feature = cpufeature_probe(stage); 286 struct alt_entry *alt; 287 u32 tmp; 288 289 for (alt = begin; alt < end; alt++) { 290 if (alt->vendor_id != 0) 291 continue; 292 if (alt->errata_id >= CPUFEATURE_NUMBER) { 293 WARN(1, "This feature id:%d is not in kernel cpufeature list", 294 alt->errata_id); 295 continue; 296 } 297 298 tmp = (1U << alt->errata_id); 299 if (cpu_req_feature & tmp) 300 patch_text_nosync(alt->old_ptr, alt->alt_ptr, alt->alt_len); 301 } 302 } 303 #endif 304