1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * SBI initialilization and all extension implementation. 4 * 5 * Copyright (c) 2020 Western Digital Corporation or its affiliates. 6 */ 7 8 #include <linux/bits.h> 9 #include <linux/init.h> 10 #include <linux/pm.h> 11 #include <linux/reboot.h> 12 #include <asm/sbi.h> 13 #include <asm/smp.h> 14 #include <asm/tlbflush.h> 15 16 /* default SBI version is 0.1 */ 17 unsigned long sbi_spec_version __ro_after_init = SBI_SPEC_VERSION_DEFAULT; 18 EXPORT_SYMBOL(sbi_spec_version); 19 20 static void (*__sbi_set_timer)(uint64_t stime) __ro_after_init; 21 static void (*__sbi_send_ipi)(unsigned int cpu) __ro_after_init; 22 static int (*__sbi_rfence)(int fid, const struct cpumask *cpu_mask, 23 unsigned long start, unsigned long size, 24 unsigned long arg4, unsigned long arg5) __ro_after_init; 25 26 struct sbiret sbi_ecall(int ext, int fid, unsigned long arg0, 27 unsigned long arg1, unsigned long arg2, 28 unsigned long arg3, unsigned long arg4, 29 unsigned long arg5) 30 { 31 struct sbiret ret; 32 33 register uintptr_t a0 asm ("a0") = (uintptr_t)(arg0); 34 register uintptr_t a1 asm ("a1") = (uintptr_t)(arg1); 35 register uintptr_t a2 asm ("a2") = (uintptr_t)(arg2); 36 register uintptr_t a3 asm ("a3") = (uintptr_t)(arg3); 37 register uintptr_t a4 asm ("a4") = (uintptr_t)(arg4); 38 register uintptr_t a5 asm ("a5") = (uintptr_t)(arg5); 39 register uintptr_t a6 asm ("a6") = (uintptr_t)(fid); 40 register uintptr_t a7 asm ("a7") = (uintptr_t)(ext); 41 asm volatile ("ecall" 42 : "+r" (a0), "+r" (a1) 43 : "r" (a2), "r" (a3), "r" (a4), "r" (a5), "r" (a6), "r" (a7) 44 : "memory"); 45 ret.error = a0; 46 ret.value = a1; 47 48 return ret; 49 } 50 EXPORT_SYMBOL(sbi_ecall); 51 52 int sbi_err_map_linux_errno(int err) 53 { 54 switch (err) { 55 case SBI_SUCCESS: 56 return 0; 57 case SBI_ERR_DENIED: 58 return -EPERM; 59 case SBI_ERR_INVALID_PARAM: 60 return -EINVAL; 61 case SBI_ERR_INVALID_ADDRESS: 62 return -EFAULT; 63 case SBI_ERR_NOT_SUPPORTED: 64 case SBI_ERR_FAILURE: 65 default: 66 return -ENOTSUPP; 67 }; 68 } 69 EXPORT_SYMBOL(sbi_err_map_linux_errno); 70 71 #ifdef CONFIG_RISCV_SBI_V01 72 static unsigned long __sbi_v01_cpumask_to_hartmask(const struct cpumask *cpu_mask) 73 { 74 unsigned long cpuid, hartid; 75 unsigned long hmask = 0; 76 77 /* 78 * There is no maximum hartid concept in RISC-V and NR_CPUS must not be 79 * associated with hartid. As SBI v0.1 is only kept for backward compatibility 80 * and will be removed in the future, there is no point in supporting hartid 81 * greater than BITS_PER_LONG (32 for RV32 and 64 for RV64). Ideally, SBI v0.2 82 * should be used for platforms with hartid greater than BITS_PER_LONG. 83 */ 84 for_each_cpu(cpuid, cpu_mask) { 85 hartid = cpuid_to_hartid_map(cpuid); 86 if (hartid >= BITS_PER_LONG) { 87 pr_warn("Unable to send any request to hartid > BITS_PER_LONG for SBI v0.1\n"); 88 break; 89 } 90 hmask |= BIT(hartid); 91 } 92 93 return hmask; 94 } 95 96 /** 97 * sbi_console_putchar() - Writes given character to the console device. 98 * @ch: The data to be written to the console. 99 * 100 * Return: None 101 */ 102 void sbi_console_putchar(int ch) 103 { 104 sbi_ecall(SBI_EXT_0_1_CONSOLE_PUTCHAR, 0, ch, 0, 0, 0, 0, 0); 105 } 106 EXPORT_SYMBOL(sbi_console_putchar); 107 108 /** 109 * sbi_console_getchar() - Reads a byte from console device. 110 * 111 * Returns the value read from console. 112 */ 113 int sbi_console_getchar(void) 114 { 115 struct sbiret ret; 116 117 ret = sbi_ecall(SBI_EXT_0_1_CONSOLE_GETCHAR, 0, 0, 0, 0, 0, 0, 0); 118 119 return ret.error; 120 } 121 EXPORT_SYMBOL(sbi_console_getchar); 122 123 /** 124 * sbi_shutdown() - Remove all the harts from executing supervisor code. 125 * 126 * Return: None 127 */ 128 void sbi_shutdown(void) 129 { 130 sbi_ecall(SBI_EXT_0_1_SHUTDOWN, 0, 0, 0, 0, 0, 0, 0); 131 } 132 EXPORT_SYMBOL(sbi_shutdown); 133 134 /** 135 * __sbi_set_timer_v01() - Program the timer for next timer event. 136 * @stime_value: The value after which next timer event should fire. 137 * 138 * Return: None 139 */ 140 static void __sbi_set_timer_v01(uint64_t stime_value) 141 { 142 #if __riscv_xlen == 32 143 sbi_ecall(SBI_EXT_0_1_SET_TIMER, 0, stime_value, 144 stime_value >> 32, 0, 0, 0, 0); 145 #else 146 sbi_ecall(SBI_EXT_0_1_SET_TIMER, 0, stime_value, 0, 0, 0, 0, 0); 147 #endif 148 } 149 150 static void __sbi_send_ipi_v01(unsigned int cpu) 151 { 152 unsigned long hart_mask = 153 __sbi_v01_cpumask_to_hartmask(cpumask_of(cpu)); 154 sbi_ecall(SBI_EXT_0_1_SEND_IPI, 0, (unsigned long)(&hart_mask), 155 0, 0, 0, 0, 0); 156 } 157 158 static int __sbi_rfence_v01(int fid, const struct cpumask *cpu_mask, 159 unsigned long start, unsigned long size, 160 unsigned long arg4, unsigned long arg5) 161 { 162 int result = 0; 163 unsigned long hart_mask; 164 165 if (!cpu_mask || cpumask_empty(cpu_mask)) 166 cpu_mask = cpu_online_mask; 167 hart_mask = __sbi_v01_cpumask_to_hartmask(cpu_mask); 168 169 /* v0.2 function IDs are equivalent to v0.1 extension IDs */ 170 switch (fid) { 171 case SBI_EXT_RFENCE_REMOTE_FENCE_I: 172 sbi_ecall(SBI_EXT_0_1_REMOTE_FENCE_I, 0, 173 (unsigned long)&hart_mask, 0, 0, 0, 0, 0); 174 break; 175 case SBI_EXT_RFENCE_REMOTE_SFENCE_VMA: 176 sbi_ecall(SBI_EXT_0_1_REMOTE_SFENCE_VMA, 0, 177 (unsigned long)&hart_mask, start, size, 178 0, 0, 0); 179 break; 180 case SBI_EXT_RFENCE_REMOTE_SFENCE_VMA_ASID: 181 sbi_ecall(SBI_EXT_0_1_REMOTE_SFENCE_VMA_ASID, 0, 182 (unsigned long)&hart_mask, start, size, 183 arg4, 0, 0); 184 break; 185 default: 186 pr_err("SBI call [%d]not supported in SBI v0.1\n", fid); 187 result = -EINVAL; 188 } 189 190 return result; 191 } 192 193 static void sbi_set_power_off(void) 194 { 195 pm_power_off = sbi_shutdown; 196 } 197 #else 198 static void __sbi_set_timer_v01(uint64_t stime_value) 199 { 200 pr_warn("Timer extension is not available in SBI v%lu.%lu\n", 201 sbi_major_version(), sbi_minor_version()); 202 } 203 204 static void __sbi_send_ipi_v01(unsigned int cpu) 205 { 206 pr_warn("IPI extension is not available in SBI v%lu.%lu\n", 207 sbi_major_version(), sbi_minor_version()); 208 } 209 210 static int __sbi_rfence_v01(int fid, const struct cpumask *cpu_mask, 211 unsigned long start, unsigned long size, 212 unsigned long arg4, unsigned long arg5) 213 { 214 pr_warn("remote fence extension is not available in SBI v%lu.%lu\n", 215 sbi_major_version(), sbi_minor_version()); 216 217 return 0; 218 } 219 220 static void sbi_set_power_off(void) {} 221 #endif /* CONFIG_RISCV_SBI_V01 */ 222 223 static void __sbi_set_timer_v02(uint64_t stime_value) 224 { 225 #if __riscv_xlen == 32 226 sbi_ecall(SBI_EXT_TIME, SBI_EXT_TIME_SET_TIMER, stime_value, 227 stime_value >> 32, 0, 0, 0, 0); 228 #else 229 sbi_ecall(SBI_EXT_TIME, SBI_EXT_TIME_SET_TIMER, stime_value, 0, 230 0, 0, 0, 0); 231 #endif 232 } 233 234 static void __sbi_send_ipi_v02(unsigned int cpu) 235 { 236 int result; 237 struct sbiret ret = {0}; 238 239 ret = sbi_ecall(SBI_EXT_IPI, SBI_EXT_IPI_SEND_IPI, 240 1UL, cpuid_to_hartid_map(cpu), 0, 0, 0, 0); 241 if (ret.error) { 242 result = sbi_err_map_linux_errno(ret.error); 243 pr_err("%s: hbase = [%lu] failed (error [%d])\n", 244 __func__, cpuid_to_hartid_map(cpu), result); 245 } 246 } 247 248 static int __sbi_rfence_v02_call(unsigned long fid, unsigned long hmask, 249 unsigned long hbase, unsigned long start, 250 unsigned long size, unsigned long arg4, 251 unsigned long arg5) 252 { 253 struct sbiret ret = {0}; 254 int ext = SBI_EXT_RFENCE; 255 int result = 0; 256 257 switch (fid) { 258 case SBI_EXT_RFENCE_REMOTE_FENCE_I: 259 ret = sbi_ecall(ext, fid, hmask, hbase, 0, 0, 0, 0); 260 break; 261 case SBI_EXT_RFENCE_REMOTE_SFENCE_VMA: 262 ret = sbi_ecall(ext, fid, hmask, hbase, start, 263 size, 0, 0); 264 break; 265 case SBI_EXT_RFENCE_REMOTE_SFENCE_VMA_ASID: 266 ret = sbi_ecall(ext, fid, hmask, hbase, start, 267 size, arg4, 0); 268 break; 269 270 case SBI_EXT_RFENCE_REMOTE_HFENCE_GVMA: 271 ret = sbi_ecall(ext, fid, hmask, hbase, start, 272 size, 0, 0); 273 break; 274 case SBI_EXT_RFENCE_REMOTE_HFENCE_GVMA_VMID: 275 ret = sbi_ecall(ext, fid, hmask, hbase, start, 276 size, arg4, 0); 277 break; 278 case SBI_EXT_RFENCE_REMOTE_HFENCE_VVMA: 279 ret = sbi_ecall(ext, fid, hmask, hbase, start, 280 size, 0, 0); 281 break; 282 case SBI_EXT_RFENCE_REMOTE_HFENCE_VVMA_ASID: 283 ret = sbi_ecall(ext, fid, hmask, hbase, start, 284 size, arg4, 0); 285 break; 286 default: 287 pr_err("unknown function ID [%lu] for SBI extension [%d]\n", 288 fid, ext); 289 result = -EINVAL; 290 } 291 292 if (ret.error) { 293 result = sbi_err_map_linux_errno(ret.error); 294 pr_err("%s: hbase = [%lu] hmask = [0x%lx] failed (error [%d])\n", 295 __func__, hbase, hmask, result); 296 } 297 298 return result; 299 } 300 301 static int __sbi_rfence_v02(int fid, const struct cpumask *cpu_mask, 302 unsigned long start, unsigned long size, 303 unsigned long arg4, unsigned long arg5) 304 { 305 unsigned long hartid, cpuid, hmask = 0, hbase = 0, htop = 0; 306 int result; 307 308 if (!cpu_mask || cpumask_empty(cpu_mask)) 309 cpu_mask = cpu_online_mask; 310 311 for_each_cpu(cpuid, cpu_mask) { 312 hartid = cpuid_to_hartid_map(cpuid); 313 if (hmask) { 314 if (hartid + BITS_PER_LONG <= htop || 315 hbase + BITS_PER_LONG <= hartid) { 316 result = __sbi_rfence_v02_call(fid, hmask, 317 hbase, start, size, arg4, arg5); 318 if (result) 319 return result; 320 hmask = 0; 321 } else if (hartid < hbase) { 322 /* shift the mask to fit lower hartid */ 323 hmask <<= hbase - hartid; 324 hbase = hartid; 325 } 326 } 327 if (!hmask) { 328 hbase = hartid; 329 htop = hartid; 330 } else if (hartid > htop) { 331 htop = hartid; 332 } 333 hmask |= BIT(hartid - hbase); 334 } 335 336 if (hmask) { 337 result = __sbi_rfence_v02_call(fid, hmask, hbase, 338 start, size, arg4, arg5); 339 if (result) 340 return result; 341 } 342 343 return 0; 344 } 345 346 /** 347 * sbi_set_timer() - Program the timer for next timer event. 348 * @stime_value: The value after which next timer event should fire. 349 * 350 * Return: None. 351 */ 352 void sbi_set_timer(uint64_t stime_value) 353 { 354 __sbi_set_timer(stime_value); 355 } 356 357 /** 358 * sbi_send_ipi() - Send an IPI to any hart. 359 * @cpu: Logical id of the target CPU. 360 */ 361 void sbi_send_ipi(unsigned int cpu) 362 { 363 __sbi_send_ipi(cpu); 364 } 365 EXPORT_SYMBOL(sbi_send_ipi); 366 367 /** 368 * sbi_remote_fence_i() - Execute FENCE.I instruction on given remote harts. 369 * @cpu_mask: A cpu mask containing all the target harts. 370 * 371 * Return: 0 on success, appropriate linux error code otherwise. 372 */ 373 int sbi_remote_fence_i(const struct cpumask *cpu_mask) 374 { 375 return __sbi_rfence(SBI_EXT_RFENCE_REMOTE_FENCE_I, 376 cpu_mask, 0, 0, 0, 0); 377 } 378 EXPORT_SYMBOL(sbi_remote_fence_i); 379 380 /** 381 * sbi_remote_sfence_vma_asid() - Execute SFENCE.VMA instructions on given 382 * remote harts for a virtual address range belonging to a specific ASID or not. 383 * 384 * @cpu_mask: A cpu mask containing all the target harts. 385 * @start: Start of the virtual address 386 * @size: Total size of the virtual address range. 387 * @asid: The value of address space identifier (ASID), or FLUSH_TLB_NO_ASID 388 * for flushing all address spaces. 389 * 390 * Return: 0 on success, appropriate linux error code otherwise. 391 */ 392 int sbi_remote_sfence_vma_asid(const struct cpumask *cpu_mask, 393 unsigned long start, 394 unsigned long size, 395 unsigned long asid) 396 { 397 if (asid == FLUSH_TLB_NO_ASID) 398 return __sbi_rfence(SBI_EXT_RFENCE_REMOTE_SFENCE_VMA, 399 cpu_mask, start, size, 0, 0); 400 else 401 return __sbi_rfence(SBI_EXT_RFENCE_REMOTE_SFENCE_VMA_ASID, 402 cpu_mask, start, size, asid, 0); 403 } 404 EXPORT_SYMBOL(sbi_remote_sfence_vma_asid); 405 406 /** 407 * sbi_remote_hfence_gvma() - Execute HFENCE.GVMA instructions on given remote 408 * harts for the specified guest physical address range. 409 * @cpu_mask: A cpu mask containing all the target harts. 410 * @start: Start of the guest physical address 411 * @size: Total size of the guest physical address range. 412 * 413 * Return: None 414 */ 415 int sbi_remote_hfence_gvma(const struct cpumask *cpu_mask, 416 unsigned long start, 417 unsigned long size) 418 { 419 return __sbi_rfence(SBI_EXT_RFENCE_REMOTE_HFENCE_GVMA, 420 cpu_mask, start, size, 0, 0); 421 } 422 EXPORT_SYMBOL_GPL(sbi_remote_hfence_gvma); 423 424 /** 425 * sbi_remote_hfence_gvma_vmid() - Execute HFENCE.GVMA instructions on given 426 * remote harts for a guest physical address range belonging to a specific VMID. 427 * 428 * @cpu_mask: A cpu mask containing all the target harts. 429 * @start: Start of the guest physical address 430 * @size: Total size of the guest physical address range. 431 * @vmid: The value of guest ID (VMID). 432 * 433 * Return: 0 if success, Error otherwise. 434 */ 435 int sbi_remote_hfence_gvma_vmid(const struct cpumask *cpu_mask, 436 unsigned long start, 437 unsigned long size, 438 unsigned long vmid) 439 { 440 return __sbi_rfence(SBI_EXT_RFENCE_REMOTE_HFENCE_GVMA_VMID, 441 cpu_mask, start, size, vmid, 0); 442 } 443 EXPORT_SYMBOL(sbi_remote_hfence_gvma_vmid); 444 445 /** 446 * sbi_remote_hfence_vvma() - Execute HFENCE.VVMA instructions on given remote 447 * harts for the current guest virtual address range. 448 * @cpu_mask: A cpu mask containing all the target harts. 449 * @start: Start of the current guest virtual address 450 * @size: Total size of the current guest virtual address range. 451 * 452 * Return: None 453 */ 454 int sbi_remote_hfence_vvma(const struct cpumask *cpu_mask, 455 unsigned long start, 456 unsigned long size) 457 { 458 return __sbi_rfence(SBI_EXT_RFENCE_REMOTE_HFENCE_VVMA, 459 cpu_mask, start, size, 0, 0); 460 } 461 EXPORT_SYMBOL(sbi_remote_hfence_vvma); 462 463 /** 464 * sbi_remote_hfence_vvma_asid() - Execute HFENCE.VVMA instructions on given 465 * remote harts for current guest virtual address range belonging to a specific 466 * ASID. 467 * 468 * @cpu_mask: A cpu mask containing all the target harts. 469 * @start: Start of the current guest virtual address 470 * @size: Total size of the current guest virtual address range. 471 * @asid: The value of address space identifier (ASID). 472 * 473 * Return: None 474 */ 475 int sbi_remote_hfence_vvma_asid(const struct cpumask *cpu_mask, 476 unsigned long start, 477 unsigned long size, 478 unsigned long asid) 479 { 480 return __sbi_rfence(SBI_EXT_RFENCE_REMOTE_HFENCE_VVMA_ASID, 481 cpu_mask, start, size, asid, 0); 482 } 483 EXPORT_SYMBOL(sbi_remote_hfence_vvma_asid); 484 485 static void sbi_srst_reset(unsigned long type, unsigned long reason) 486 { 487 sbi_ecall(SBI_EXT_SRST, SBI_EXT_SRST_RESET, type, reason, 488 0, 0, 0, 0); 489 pr_warn("%s: type=0x%lx reason=0x%lx failed\n", 490 __func__, type, reason); 491 } 492 493 static int sbi_srst_reboot(struct notifier_block *this, 494 unsigned long mode, void *cmd) 495 { 496 sbi_srst_reset((mode == REBOOT_WARM || mode == REBOOT_SOFT) ? 497 SBI_SRST_RESET_TYPE_WARM_REBOOT : 498 SBI_SRST_RESET_TYPE_COLD_REBOOT, 499 SBI_SRST_RESET_REASON_NONE); 500 return NOTIFY_DONE; 501 } 502 503 static struct notifier_block sbi_srst_reboot_nb; 504 505 static void sbi_srst_power_off(void) 506 { 507 sbi_srst_reset(SBI_SRST_RESET_TYPE_SHUTDOWN, 508 SBI_SRST_RESET_REASON_NONE); 509 } 510 511 /** 512 * sbi_probe_extension() - Check if an SBI extension ID is supported or not. 513 * @extid: The extension ID to be probed. 514 * 515 * Return: 1 or an extension specific nonzero value if yes, 0 otherwise. 516 */ 517 long sbi_probe_extension(int extid) 518 { 519 struct sbiret ret; 520 521 ret = sbi_ecall(SBI_EXT_BASE, SBI_EXT_BASE_PROBE_EXT, extid, 522 0, 0, 0, 0, 0); 523 if (!ret.error) 524 return ret.value; 525 526 return 0; 527 } 528 EXPORT_SYMBOL(sbi_probe_extension); 529 530 static long __sbi_base_ecall(int fid) 531 { 532 struct sbiret ret; 533 534 ret = sbi_ecall(SBI_EXT_BASE, fid, 0, 0, 0, 0, 0, 0); 535 if (!ret.error) 536 return ret.value; 537 else 538 return sbi_err_map_linux_errno(ret.error); 539 } 540 541 static inline long sbi_get_spec_version(void) 542 { 543 return __sbi_base_ecall(SBI_EXT_BASE_GET_SPEC_VERSION); 544 } 545 546 static inline long sbi_get_firmware_id(void) 547 { 548 return __sbi_base_ecall(SBI_EXT_BASE_GET_IMP_ID); 549 } 550 551 static inline long sbi_get_firmware_version(void) 552 { 553 return __sbi_base_ecall(SBI_EXT_BASE_GET_IMP_VERSION); 554 } 555 556 long sbi_get_mvendorid(void) 557 { 558 return __sbi_base_ecall(SBI_EXT_BASE_GET_MVENDORID); 559 } 560 EXPORT_SYMBOL_GPL(sbi_get_mvendorid); 561 562 long sbi_get_marchid(void) 563 { 564 return __sbi_base_ecall(SBI_EXT_BASE_GET_MARCHID); 565 } 566 EXPORT_SYMBOL_GPL(sbi_get_marchid); 567 568 long sbi_get_mimpid(void) 569 { 570 return __sbi_base_ecall(SBI_EXT_BASE_GET_MIMPID); 571 } 572 EXPORT_SYMBOL_GPL(sbi_get_mimpid); 573 574 void __init sbi_init(void) 575 { 576 int ret; 577 578 sbi_set_power_off(); 579 ret = sbi_get_spec_version(); 580 if (ret > 0) 581 sbi_spec_version = ret; 582 583 pr_info("SBI specification v%lu.%lu detected\n", 584 sbi_major_version(), sbi_minor_version()); 585 586 if (!sbi_spec_is_0_1()) { 587 pr_info("SBI implementation ID=0x%lx Version=0x%lx\n", 588 sbi_get_firmware_id(), sbi_get_firmware_version()); 589 if (sbi_probe_extension(SBI_EXT_TIME)) { 590 __sbi_set_timer = __sbi_set_timer_v02; 591 pr_info("SBI TIME extension detected\n"); 592 } else { 593 __sbi_set_timer = __sbi_set_timer_v01; 594 } 595 if (sbi_probe_extension(SBI_EXT_IPI)) { 596 __sbi_send_ipi = __sbi_send_ipi_v02; 597 pr_info("SBI IPI extension detected\n"); 598 } else { 599 __sbi_send_ipi = __sbi_send_ipi_v01; 600 } 601 if (sbi_probe_extension(SBI_EXT_RFENCE)) { 602 __sbi_rfence = __sbi_rfence_v02; 603 pr_info("SBI RFENCE extension detected\n"); 604 } else { 605 __sbi_rfence = __sbi_rfence_v01; 606 } 607 if ((sbi_spec_version >= sbi_mk_version(0, 3)) && 608 sbi_probe_extension(SBI_EXT_SRST)) { 609 pr_info("SBI SRST extension detected\n"); 610 pm_power_off = sbi_srst_power_off; 611 sbi_srst_reboot_nb.notifier_call = sbi_srst_reboot; 612 sbi_srst_reboot_nb.priority = 192; 613 register_restart_handler(&sbi_srst_reboot_nb); 614 } 615 } else { 616 __sbi_set_timer = __sbi_set_timer_v01; 617 __sbi_send_ipi = __sbi_send_ipi_v01; 618 __sbi_rfence = __sbi_rfence_v01; 619 } 620 } 621