1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copied from arch/arm64/kernel/cpufeature.c 4 * 5 * Copyright (C) 2015 ARM Ltd. 6 * Copyright (C) 2017 SiFive 7 */ 8 9 #include <linux/bitmap.h> 10 #include <linux/ctype.h> 11 #include <linux/log2.h> 12 #include <linux/memory.h> 13 #include <linux/module.h> 14 #include <linux/of.h> 15 #include <asm/alternative.h> 16 #include <asm/cacheflush.h> 17 #include <asm/cpufeature.h> 18 #include <asm/hwcap.h> 19 #include <asm/patch.h> 20 #include <asm/processor.h> 21 22 #define NUM_ALPHA_EXTS ('z' - 'a' + 1) 23 24 unsigned long elf_hwcap __read_mostly; 25 26 /* Host ISA bitmap */ 27 static DECLARE_BITMAP(riscv_isa, RISCV_ISA_EXT_MAX) __read_mostly; 28 29 /* Performance information */ 30 DEFINE_PER_CPU(long, misaligned_access_speed); 31 32 /** 33 * riscv_isa_extension_base() - Get base extension word 34 * 35 * @isa_bitmap: ISA bitmap to use 36 * Return: base extension word as unsigned long value 37 * 38 * NOTE: If isa_bitmap is NULL then Host ISA bitmap will be used. 39 */ 40 unsigned long riscv_isa_extension_base(const unsigned long *isa_bitmap) 41 { 42 if (!isa_bitmap) 43 return riscv_isa[0]; 44 return isa_bitmap[0]; 45 } 46 EXPORT_SYMBOL_GPL(riscv_isa_extension_base); 47 48 /** 49 * __riscv_isa_extension_available() - Check whether given extension 50 * is available or not 51 * 52 * @isa_bitmap: ISA bitmap to use 53 * @bit: bit position of the desired extension 54 * Return: true or false 55 * 56 * NOTE: If isa_bitmap is NULL then Host ISA bitmap will be used. 57 */ 58 bool __riscv_isa_extension_available(const unsigned long *isa_bitmap, int bit) 59 { 60 const unsigned long *bmap = (isa_bitmap) ? isa_bitmap : riscv_isa; 61 62 if (bit >= RISCV_ISA_EXT_MAX) 63 return false; 64 65 return test_bit(bit, bmap) ? true : false; 66 } 67 EXPORT_SYMBOL_GPL(__riscv_isa_extension_available); 68 69 static bool riscv_isa_extension_check(int id) 70 { 71 switch (id) { 72 case RISCV_ISA_EXT_ZICBOM: 73 if (!riscv_cbom_block_size) { 74 pr_err("Zicbom detected in ISA string, but no cbom-block-size found\n"); 75 return false; 76 } else if (!is_power_of_2(riscv_cbom_block_size)) { 77 pr_err("cbom-block-size present, but is not a power-of-2\n"); 78 return false; 79 } 80 return true; 81 case RISCV_ISA_EXT_ZICBOZ: 82 if (!riscv_cboz_block_size) { 83 pr_err("Zicboz detected in ISA string, but no cboz-block-size found\n"); 84 return false; 85 } else if (!is_power_of_2(riscv_cboz_block_size)) { 86 pr_err("cboz-block-size present, but is not a power-of-2\n"); 87 return false; 88 } 89 return true; 90 } 91 92 return true; 93 } 94 95 void __init riscv_fill_hwcap(void) 96 { 97 struct device_node *node; 98 const char *isa; 99 char print_str[NUM_ALPHA_EXTS + 1]; 100 int i, j, rc; 101 unsigned long isa2hwcap[26] = {0}; 102 unsigned long hartid; 103 104 isa2hwcap['i' - 'a'] = COMPAT_HWCAP_ISA_I; 105 isa2hwcap['m' - 'a'] = COMPAT_HWCAP_ISA_M; 106 isa2hwcap['a' - 'a'] = COMPAT_HWCAP_ISA_A; 107 isa2hwcap['f' - 'a'] = COMPAT_HWCAP_ISA_F; 108 isa2hwcap['d' - 'a'] = COMPAT_HWCAP_ISA_D; 109 isa2hwcap['c' - 'a'] = COMPAT_HWCAP_ISA_C; 110 111 elf_hwcap = 0; 112 113 bitmap_zero(riscv_isa, RISCV_ISA_EXT_MAX); 114 115 for_each_of_cpu_node(node) { 116 unsigned long this_hwcap = 0; 117 DECLARE_BITMAP(this_isa, RISCV_ISA_EXT_MAX); 118 const char *temp; 119 120 rc = riscv_of_processor_hartid(node, &hartid); 121 if (rc < 0) 122 continue; 123 124 if (of_property_read_string(node, "riscv,isa", &isa)) { 125 pr_warn("Unable to find \"riscv,isa\" devicetree entry\n"); 126 continue; 127 } 128 129 temp = isa; 130 #if IS_ENABLED(CONFIG_32BIT) 131 if (!strncmp(isa, "rv32", 4)) 132 isa += 4; 133 #elif IS_ENABLED(CONFIG_64BIT) 134 if (!strncmp(isa, "rv64", 4)) 135 isa += 4; 136 #endif 137 /* The riscv,isa DT property must start with rv64 or rv32 */ 138 if (temp == isa) 139 continue; 140 bitmap_zero(this_isa, RISCV_ISA_EXT_MAX); 141 for (; *isa; ++isa) { 142 const char *ext = isa++; 143 const char *ext_end = isa; 144 bool ext_long = false, ext_err = false; 145 146 switch (*ext) { 147 case 's': 148 /** 149 * Workaround for invalid single-letter 's' & 'u'(QEMU). 150 * No need to set the bit in riscv_isa as 's' & 'u' are 151 * not valid ISA extensions. It works until multi-letter 152 * extension starting with "Su" appears. 153 */ 154 if (ext[-1] != '_' && ext[1] == 'u') { 155 ++isa; 156 ext_err = true; 157 break; 158 } 159 fallthrough; 160 case 'x': 161 case 'z': 162 ext_long = true; 163 /* Multi-letter extension must be delimited */ 164 for (; *isa && *isa != '_'; ++isa) 165 if (unlikely(!islower(*isa) 166 && !isdigit(*isa))) 167 ext_err = true; 168 /* Parse backwards */ 169 ext_end = isa; 170 if (unlikely(ext_err)) 171 break; 172 if (!isdigit(ext_end[-1])) 173 break; 174 /* Skip the minor version */ 175 while (isdigit(*--ext_end)) 176 ; 177 if (ext_end[0] != 'p' 178 || !isdigit(ext_end[-1])) { 179 /* Advance it to offset the pre-decrement */ 180 ++ext_end; 181 break; 182 } 183 /* Skip the major version */ 184 while (isdigit(*--ext_end)) 185 ; 186 ++ext_end; 187 break; 188 default: 189 if (unlikely(!islower(*ext))) { 190 ext_err = true; 191 break; 192 } 193 /* Find next extension */ 194 if (!isdigit(*isa)) 195 break; 196 /* Skip the minor version */ 197 while (isdigit(*++isa)) 198 ; 199 if (*isa != 'p') 200 break; 201 if (!isdigit(*++isa)) { 202 --isa; 203 break; 204 } 205 /* Skip the major version */ 206 while (isdigit(*++isa)) 207 ; 208 break; 209 } 210 if (*isa != '_') 211 --isa; 212 213 #define SET_ISA_EXT_MAP(name, bit) \ 214 do { \ 215 if ((ext_end - ext == sizeof(name) - 1) && \ 216 !memcmp(ext, name, sizeof(name) - 1) && \ 217 riscv_isa_extension_check(bit)) \ 218 set_bit(bit, this_isa); \ 219 } while (false) \ 220 221 if (unlikely(ext_err)) 222 continue; 223 if (!ext_long) { 224 int nr = *ext - 'a'; 225 226 if (riscv_isa_extension_check(nr)) { 227 this_hwcap |= isa2hwcap[nr]; 228 set_bit(nr, this_isa); 229 } 230 } else { 231 /* sorted alphabetically */ 232 SET_ISA_EXT_MAP("smaia", RISCV_ISA_EXT_SMAIA); 233 SET_ISA_EXT_MAP("ssaia", RISCV_ISA_EXT_SSAIA); 234 SET_ISA_EXT_MAP("sscofpmf", RISCV_ISA_EXT_SSCOFPMF); 235 SET_ISA_EXT_MAP("sstc", RISCV_ISA_EXT_SSTC); 236 SET_ISA_EXT_MAP("svinval", RISCV_ISA_EXT_SVINVAL); 237 SET_ISA_EXT_MAP("svnapot", RISCV_ISA_EXT_SVNAPOT); 238 SET_ISA_EXT_MAP("svpbmt", RISCV_ISA_EXT_SVPBMT); 239 SET_ISA_EXT_MAP("zbb", RISCV_ISA_EXT_ZBB); 240 SET_ISA_EXT_MAP("zicbom", RISCV_ISA_EXT_ZICBOM); 241 SET_ISA_EXT_MAP("zicboz", RISCV_ISA_EXT_ZICBOZ); 242 SET_ISA_EXT_MAP("zihintpause", RISCV_ISA_EXT_ZIHINTPAUSE); 243 } 244 #undef SET_ISA_EXT_MAP 245 } 246 247 /* 248 * All "okay" hart should have same isa. Set HWCAP based on 249 * common capabilities of every "okay" hart, in case they don't 250 * have. 251 */ 252 if (elf_hwcap) 253 elf_hwcap &= this_hwcap; 254 else 255 elf_hwcap = this_hwcap; 256 257 if (bitmap_empty(riscv_isa, RISCV_ISA_EXT_MAX)) 258 bitmap_copy(riscv_isa, this_isa, RISCV_ISA_EXT_MAX); 259 else 260 bitmap_and(riscv_isa, riscv_isa, this_isa, RISCV_ISA_EXT_MAX); 261 } 262 263 /* We don't support systems with F but without D, so mask those out 264 * here. */ 265 if ((elf_hwcap & COMPAT_HWCAP_ISA_F) && !(elf_hwcap & COMPAT_HWCAP_ISA_D)) { 266 pr_info("This kernel does not support systems with F but not D\n"); 267 elf_hwcap &= ~COMPAT_HWCAP_ISA_F; 268 } 269 270 memset(print_str, 0, sizeof(print_str)); 271 for (i = 0, j = 0; i < NUM_ALPHA_EXTS; i++) 272 if (riscv_isa[0] & BIT_MASK(i)) 273 print_str[j++] = (char)('a' + i); 274 pr_info("riscv: base ISA extensions %s\n", print_str); 275 276 memset(print_str, 0, sizeof(print_str)); 277 for (i = 0, j = 0; i < NUM_ALPHA_EXTS; i++) 278 if (elf_hwcap & BIT_MASK(i)) 279 print_str[j++] = (char)('a' + i); 280 pr_info("riscv: ELF capabilities %s\n", print_str); 281 } 282 283 #ifdef CONFIG_RISCV_ALTERNATIVE 284 /* 285 * Alternative patch sites consider 48 bits when determining when to patch 286 * the old instruction sequence with the new. These bits are broken into a 287 * 16-bit vendor ID and a 32-bit patch ID. A non-zero vendor ID means the 288 * patch site is for an erratum, identified by the 32-bit patch ID. When 289 * the vendor ID is zero, the patch site is for a cpufeature. cpufeatures 290 * further break down patch ID into two 16-bit numbers. The lower 16 bits 291 * are the cpufeature ID and the upper 16 bits are used for a value specific 292 * to the cpufeature and patch site. If the upper 16 bits are zero, then it 293 * implies no specific value is specified. cpufeatures that want to control 294 * patching on a per-site basis will provide non-zero values and implement 295 * checks here. The checks return true when patching should be done, and 296 * false otherwise. 297 */ 298 static bool riscv_cpufeature_patch_check(u16 id, u16 value) 299 { 300 if (!value) 301 return true; 302 303 switch (id) { 304 case RISCV_ISA_EXT_ZICBOZ: 305 /* 306 * Zicboz alternative applications provide the maximum 307 * supported block size order, or zero when it doesn't 308 * matter. If the current block size exceeds the maximum, 309 * then the alternative cannot be applied. 310 */ 311 return riscv_cboz_block_size <= (1U << value); 312 } 313 314 return false; 315 } 316 317 void __init_or_module riscv_cpufeature_patch_func(struct alt_entry *begin, 318 struct alt_entry *end, 319 unsigned int stage) 320 { 321 struct alt_entry *alt; 322 void *oldptr, *altptr; 323 u16 id, value; 324 325 if (stage == RISCV_ALTERNATIVES_EARLY_BOOT) 326 return; 327 328 for (alt = begin; alt < end; alt++) { 329 if (alt->vendor_id != 0) 330 continue; 331 332 id = PATCH_ID_CPUFEATURE_ID(alt->patch_id); 333 334 if (id >= RISCV_ISA_EXT_MAX) { 335 WARN(1, "This extension id:%d is not in ISA extension list", id); 336 continue; 337 } 338 339 if (!__riscv_isa_extension_available(NULL, id)) 340 continue; 341 342 value = PATCH_ID_CPUFEATURE_VALUE(alt->patch_id); 343 if (!riscv_cpufeature_patch_check(id, value)) 344 continue; 345 346 oldptr = ALT_OLD_PTR(alt); 347 altptr = ALT_ALT_PTR(alt); 348 349 mutex_lock(&text_mutex); 350 patch_text_nosync(oldptr, altptr, alt->alt_len); 351 riscv_alternative_fix_offsets(oldptr, alt->alt_len, oldptr - altptr); 352 mutex_unlock(&text_mutex); 353 } 354 } 355 #endif 356