1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * SBI initialilization and all extension implementation. 4 * 5 * Copyright (c) 2020 Western Digital Corporation or its affiliates. 6 */ 7 8 #include <linux/init.h> 9 #include <linux/pm.h> 10 #include <asm/sbi.h> 11 #include <asm/smp.h> 12 13 /* default SBI version is 0.1 */ 14 unsigned long sbi_spec_version __ro_after_init = SBI_SPEC_VERSION_DEFAULT; 15 EXPORT_SYMBOL(sbi_spec_version); 16 17 static void (*__sbi_set_timer)(uint64_t stime) __ro_after_init; 18 static int (*__sbi_send_ipi)(const unsigned long *hart_mask) __ro_after_init; 19 static int (*__sbi_rfence)(int fid, const unsigned long *hart_mask, 20 unsigned long start, unsigned long size, 21 unsigned long arg4, unsigned long arg5) __ro_after_init; 22 23 struct sbiret sbi_ecall(int ext, int fid, unsigned long arg0, 24 unsigned long arg1, unsigned long arg2, 25 unsigned long arg3, unsigned long arg4, 26 unsigned long arg5) 27 { 28 struct sbiret ret; 29 30 register uintptr_t a0 asm ("a0") = (uintptr_t)(arg0); 31 register uintptr_t a1 asm ("a1") = (uintptr_t)(arg1); 32 register uintptr_t a2 asm ("a2") = (uintptr_t)(arg2); 33 register uintptr_t a3 asm ("a3") = (uintptr_t)(arg3); 34 register uintptr_t a4 asm ("a4") = (uintptr_t)(arg4); 35 register uintptr_t a5 asm ("a5") = (uintptr_t)(arg5); 36 register uintptr_t a6 asm ("a6") = (uintptr_t)(fid); 37 register uintptr_t a7 asm ("a7") = (uintptr_t)(ext); 38 asm volatile ("ecall" 39 : "+r" (a0), "+r" (a1) 40 : "r" (a2), "r" (a3), "r" (a4), "r" (a5), "r" (a6), "r" (a7) 41 : "memory"); 42 ret.error = a0; 43 ret.value = a1; 44 45 return ret; 46 } 47 EXPORT_SYMBOL(sbi_ecall); 48 49 int sbi_err_map_linux_errno(int err) 50 { 51 switch (err) { 52 case SBI_SUCCESS: 53 return 0; 54 case SBI_ERR_DENIED: 55 return -EPERM; 56 case SBI_ERR_INVALID_PARAM: 57 return -EINVAL; 58 case SBI_ERR_INVALID_ADDRESS: 59 return -EFAULT; 60 case SBI_ERR_NOT_SUPPORTED: 61 case SBI_ERR_FAILURE: 62 default: 63 return -ENOTSUPP; 64 }; 65 } 66 EXPORT_SYMBOL(sbi_err_map_linux_errno); 67 68 #ifdef CONFIG_RISCV_SBI_V01 69 /** 70 * sbi_console_putchar() - Writes given character to the console device. 71 * @ch: The data to be written to the console. 72 * 73 * Return: None 74 */ 75 void sbi_console_putchar(int ch) 76 { 77 sbi_ecall(SBI_EXT_0_1_CONSOLE_PUTCHAR, 0, ch, 0, 0, 0, 0, 0); 78 } 79 EXPORT_SYMBOL(sbi_console_putchar); 80 81 /** 82 * sbi_console_getchar() - Reads a byte from console device. 83 * 84 * Returns the value read from console. 85 */ 86 int sbi_console_getchar(void) 87 { 88 struct sbiret ret; 89 90 ret = sbi_ecall(SBI_EXT_0_1_CONSOLE_GETCHAR, 0, 0, 0, 0, 0, 0, 0); 91 92 return ret.error; 93 } 94 EXPORT_SYMBOL(sbi_console_getchar); 95 96 /** 97 * sbi_shutdown() - Remove all the harts from executing supervisor code. 98 * 99 * Return: None 100 */ 101 void sbi_shutdown(void) 102 { 103 sbi_ecall(SBI_EXT_0_1_SHUTDOWN, 0, 0, 0, 0, 0, 0, 0); 104 } 105 EXPORT_SYMBOL(sbi_shutdown); 106 107 /** 108 * sbi_clear_ipi() - Clear any pending IPIs for the calling hart. 109 * 110 * Return: None 111 */ 112 void sbi_clear_ipi(void) 113 { 114 sbi_ecall(SBI_EXT_0_1_CLEAR_IPI, 0, 0, 0, 0, 0, 0, 0); 115 } 116 EXPORT_SYMBOL(sbi_clear_ipi); 117 118 /** 119 * __sbi_set_timer_v01() - Program the timer for next timer event. 120 * @stime_value: The value after which next timer event should fire. 121 * 122 * Return: None 123 */ 124 static void __sbi_set_timer_v01(uint64_t stime_value) 125 { 126 #if __riscv_xlen == 32 127 sbi_ecall(SBI_EXT_0_1_SET_TIMER, 0, stime_value, 128 stime_value >> 32, 0, 0, 0, 0); 129 #else 130 sbi_ecall(SBI_EXT_0_1_SET_TIMER, 0, stime_value, 0, 0, 0, 0, 0); 131 #endif 132 } 133 134 static int __sbi_send_ipi_v01(const unsigned long *hart_mask) 135 { 136 sbi_ecall(SBI_EXT_0_1_SEND_IPI, 0, (unsigned long)hart_mask, 137 0, 0, 0, 0, 0); 138 return 0; 139 } 140 141 static int __sbi_rfence_v01(int fid, const unsigned long *hart_mask, 142 unsigned long start, unsigned long size, 143 unsigned long arg4, unsigned long arg5) 144 { 145 int result = 0; 146 147 /* v0.2 function IDs are equivalent to v0.1 extension IDs */ 148 switch (fid) { 149 case SBI_EXT_RFENCE_REMOTE_FENCE_I: 150 sbi_ecall(SBI_EXT_0_1_REMOTE_FENCE_I, 0, 151 (unsigned long)hart_mask, 0, 0, 0, 0, 0); 152 break; 153 case SBI_EXT_RFENCE_REMOTE_SFENCE_VMA: 154 sbi_ecall(SBI_EXT_0_1_REMOTE_SFENCE_VMA, 0, 155 (unsigned long)hart_mask, start, size, 156 0, 0, 0); 157 break; 158 case SBI_EXT_RFENCE_REMOTE_SFENCE_VMA_ASID: 159 sbi_ecall(SBI_EXT_0_1_REMOTE_SFENCE_VMA_ASID, 0, 160 (unsigned long)hart_mask, start, size, 161 arg4, 0, 0); 162 break; 163 default: 164 pr_err("SBI call [%d]not supported in SBI v0.1\n", fid); 165 result = -EINVAL; 166 } 167 168 return result; 169 } 170 171 static void sbi_set_power_off(void) 172 { 173 pm_power_off = sbi_shutdown; 174 } 175 #else 176 static void __sbi_set_timer_v01(uint64_t stime_value) 177 { 178 pr_warn("Timer extension is not available in SBI v%lu.%lu\n", 179 sbi_major_version(), sbi_minor_version()); 180 } 181 182 static int __sbi_send_ipi_v01(const unsigned long *hart_mask) 183 { 184 pr_warn("IPI extension is not available in SBI v%lu.%lu\n", 185 sbi_major_version(), sbi_minor_version()); 186 187 return 0; 188 } 189 190 static int __sbi_rfence_v01(int fid, const unsigned long *hart_mask, 191 unsigned long start, unsigned long size, 192 unsigned long arg4, unsigned long arg5) 193 { 194 pr_warn("remote fence extension is not available in SBI v%lu.%lu\n", 195 sbi_major_version(), sbi_minor_version()); 196 197 return 0; 198 } 199 200 static void sbi_set_power_off(void) {} 201 #endif /* CONFIG_RISCV_SBI_V01 */ 202 203 static void __sbi_set_timer_v02(uint64_t stime_value) 204 { 205 #if __riscv_xlen == 32 206 sbi_ecall(SBI_EXT_TIME, SBI_EXT_TIME_SET_TIMER, stime_value, 207 stime_value >> 32, 0, 0, 0, 0); 208 #else 209 sbi_ecall(SBI_EXT_TIME, SBI_EXT_TIME_SET_TIMER, stime_value, 0, 210 0, 0, 0, 0); 211 #endif 212 } 213 214 static int __sbi_send_ipi_v02(const unsigned long *hart_mask) 215 { 216 unsigned long hartid, hmask_val, hbase; 217 struct cpumask tmask; 218 struct sbiret ret = {0}; 219 int result; 220 221 if (!hart_mask || !(*hart_mask)) { 222 riscv_cpuid_to_hartid_mask(cpu_online_mask, &tmask); 223 hart_mask = cpumask_bits(&tmask); 224 } 225 226 hmask_val = 0; 227 hbase = 0; 228 for_each_set_bit(hartid, hart_mask, NR_CPUS) { 229 if (hmask_val && ((hbase + BITS_PER_LONG) <= hartid)) { 230 ret = sbi_ecall(SBI_EXT_IPI, SBI_EXT_IPI_SEND_IPI, 231 hmask_val, hbase, 0, 0, 0, 0); 232 if (ret.error) 233 goto ecall_failed; 234 hmask_val = 0; 235 hbase = 0; 236 } 237 if (!hmask_val) 238 hbase = hartid; 239 hmask_val |= 1UL << (hartid - hbase); 240 } 241 242 if (hmask_val) { 243 ret = sbi_ecall(SBI_EXT_IPI, SBI_EXT_IPI_SEND_IPI, 244 hmask_val, hbase, 0, 0, 0, 0); 245 if (ret.error) 246 goto ecall_failed; 247 } 248 249 return 0; 250 251 ecall_failed: 252 result = sbi_err_map_linux_errno(ret.error); 253 pr_err("%s: hbase = [%lu] hmask = [0x%lx] failed (error [%d])\n", 254 __func__, hbase, hmask_val, result); 255 return result; 256 } 257 258 static int __sbi_rfence_v02_call(unsigned long fid, unsigned long hmask_val, 259 unsigned long hbase, unsigned long start, 260 unsigned long size, unsigned long arg4, 261 unsigned long arg5) 262 { 263 struct sbiret ret = {0}; 264 int ext = SBI_EXT_RFENCE; 265 int result = 0; 266 267 switch (fid) { 268 case SBI_EXT_RFENCE_REMOTE_FENCE_I: 269 ret = sbi_ecall(ext, fid, hmask_val, hbase, 0, 0, 0, 0); 270 break; 271 case SBI_EXT_RFENCE_REMOTE_SFENCE_VMA: 272 ret = sbi_ecall(ext, fid, hmask_val, hbase, start, 273 size, 0, 0); 274 break; 275 case SBI_EXT_RFENCE_REMOTE_SFENCE_VMA_ASID: 276 ret = sbi_ecall(ext, fid, hmask_val, hbase, start, 277 size, arg4, 0); 278 break; 279 280 case SBI_EXT_RFENCE_REMOTE_HFENCE_GVMA: 281 ret = sbi_ecall(ext, fid, hmask_val, hbase, start, 282 size, 0, 0); 283 break; 284 case SBI_EXT_RFENCE_REMOTE_HFENCE_GVMA_VMID: 285 ret = sbi_ecall(ext, fid, hmask_val, hbase, start, 286 size, arg4, 0); 287 break; 288 case SBI_EXT_RFENCE_REMOTE_HFENCE_VVMA: 289 ret = sbi_ecall(ext, fid, hmask_val, hbase, start, 290 size, 0, 0); 291 break; 292 case SBI_EXT_RFENCE_REMOTE_HFENCE_VVMA_ASID: 293 ret = sbi_ecall(ext, fid, hmask_val, hbase, start, 294 size, arg4, 0); 295 break; 296 default: 297 pr_err("unknown function ID [%lu] for SBI extension [%d]\n", 298 fid, ext); 299 result = -EINVAL; 300 } 301 302 if (ret.error) { 303 result = sbi_err_map_linux_errno(ret.error); 304 pr_err("%s: hbase = [%lu] hmask = [0x%lx] failed (error [%d])\n", 305 __func__, hbase, hmask_val, result); 306 } 307 308 return result; 309 } 310 311 static int __sbi_rfence_v02(int fid, const unsigned long *hart_mask, 312 unsigned long start, unsigned long size, 313 unsigned long arg4, unsigned long arg5) 314 { 315 unsigned long hmask_val, hartid, hbase; 316 struct cpumask tmask; 317 int result; 318 319 if (!hart_mask || !(*hart_mask)) { 320 riscv_cpuid_to_hartid_mask(cpu_online_mask, &tmask); 321 hart_mask = cpumask_bits(&tmask); 322 } 323 324 hmask_val = 0; 325 hbase = 0; 326 for_each_set_bit(hartid, hart_mask, NR_CPUS) { 327 if (hmask_val && ((hbase + BITS_PER_LONG) <= hartid)) { 328 result = __sbi_rfence_v02_call(fid, hmask_val, hbase, 329 start, size, arg4, arg5); 330 if (result) 331 return result; 332 hmask_val = 0; 333 hbase = 0; 334 } 335 if (!hmask_val) 336 hbase = hartid; 337 hmask_val |= 1UL << (hartid - hbase); 338 } 339 340 if (hmask_val) { 341 result = __sbi_rfence_v02_call(fid, hmask_val, hbase, 342 start, size, arg4, arg5); 343 if (result) 344 return result; 345 } 346 347 return 0; 348 } 349 350 /** 351 * sbi_set_timer() - Program the timer for next timer event. 352 * @stime_value: The value after which next timer event should fire. 353 * 354 * Return: None. 355 */ 356 void sbi_set_timer(uint64_t stime_value) 357 { 358 __sbi_set_timer(stime_value); 359 } 360 361 /** 362 * sbi_send_ipi() - Send an IPI to any hart. 363 * @hart_mask: A cpu mask containing all the target harts. 364 * 365 * Return: 0 on success, appropriate linux error code otherwise. 366 */ 367 int sbi_send_ipi(const unsigned long *hart_mask) 368 { 369 return __sbi_send_ipi(hart_mask); 370 } 371 EXPORT_SYMBOL(sbi_send_ipi); 372 373 /** 374 * sbi_remote_fence_i() - Execute FENCE.I instruction on given remote harts. 375 * @hart_mask: A cpu mask containing all the target harts. 376 * 377 * Return: 0 on success, appropriate linux error code otherwise. 378 */ 379 int sbi_remote_fence_i(const unsigned long *hart_mask) 380 { 381 return __sbi_rfence(SBI_EXT_RFENCE_REMOTE_FENCE_I, 382 hart_mask, 0, 0, 0, 0); 383 } 384 EXPORT_SYMBOL(sbi_remote_fence_i); 385 386 /** 387 * sbi_remote_sfence_vma() - Execute SFENCE.VMA instructions on given remote 388 * harts for the specified virtual address range. 389 * @hart_mask: A cpu mask containing all the target harts. 390 * @start: Start of the virtual address 391 * @size: Total size of the virtual address range. 392 * 393 * Return: 0 on success, appropriate linux error code otherwise. 394 */ 395 int sbi_remote_sfence_vma(const unsigned long *hart_mask, 396 unsigned long start, 397 unsigned long size) 398 { 399 return __sbi_rfence(SBI_EXT_RFENCE_REMOTE_SFENCE_VMA, 400 hart_mask, start, size, 0, 0); 401 } 402 EXPORT_SYMBOL(sbi_remote_sfence_vma); 403 404 /** 405 * sbi_remote_sfence_vma_asid() - Execute SFENCE.VMA instructions on given 406 * remote harts for a virtual address range belonging to a specific ASID. 407 * 408 * @hart_mask: A cpu mask containing all the target harts. 409 * @start: Start of the virtual address 410 * @size: Total size of the virtual address range. 411 * @asid: The value of address space identifier (ASID). 412 * 413 * Return: 0 on success, appropriate linux error code otherwise. 414 */ 415 int sbi_remote_sfence_vma_asid(const unsigned long *hart_mask, 416 unsigned long start, 417 unsigned long size, 418 unsigned long asid) 419 { 420 return __sbi_rfence(SBI_EXT_RFENCE_REMOTE_SFENCE_VMA_ASID, 421 hart_mask, start, size, asid, 0); 422 } 423 EXPORT_SYMBOL(sbi_remote_sfence_vma_asid); 424 425 /** 426 * sbi_remote_hfence_gvma() - Execute HFENCE.GVMA instructions on given remote 427 * harts for the specified guest physical address range. 428 * @hart_mask: A cpu mask containing all the target harts. 429 * @start: Start of the guest physical address 430 * @size: Total size of the guest physical address range. 431 * 432 * Return: None 433 */ 434 int sbi_remote_hfence_gvma(const unsigned long *hart_mask, 435 unsigned long start, 436 unsigned long size) 437 { 438 return __sbi_rfence(SBI_EXT_RFENCE_REMOTE_HFENCE_GVMA, 439 hart_mask, start, size, 0, 0); 440 } 441 EXPORT_SYMBOL_GPL(sbi_remote_hfence_gvma); 442 443 /** 444 * sbi_remote_hfence_gvma_vmid() - Execute HFENCE.GVMA instructions on given 445 * remote harts for a guest physical address range belonging to a specific VMID. 446 * 447 * @hart_mask: A cpu mask containing all the target harts. 448 * @start: Start of the guest physical address 449 * @size: Total size of the guest physical address range. 450 * @vmid: The value of guest ID (VMID). 451 * 452 * Return: 0 if success, Error otherwise. 453 */ 454 int sbi_remote_hfence_gvma_vmid(const unsigned long *hart_mask, 455 unsigned long start, 456 unsigned long size, 457 unsigned long vmid) 458 { 459 return __sbi_rfence(SBI_EXT_RFENCE_REMOTE_HFENCE_GVMA_VMID, 460 hart_mask, start, size, vmid, 0); 461 } 462 EXPORT_SYMBOL(sbi_remote_hfence_gvma_vmid); 463 464 /** 465 * sbi_remote_hfence_vvma() - Execute HFENCE.VVMA instructions on given remote 466 * harts for the current guest virtual address range. 467 * @hart_mask: A cpu mask containing all the target harts. 468 * @start: Start of the current guest virtual address 469 * @size: Total size of the current guest virtual address range. 470 * 471 * Return: None 472 */ 473 int sbi_remote_hfence_vvma(const unsigned long *hart_mask, 474 unsigned long start, 475 unsigned long size) 476 { 477 return __sbi_rfence(SBI_EXT_RFENCE_REMOTE_HFENCE_VVMA, 478 hart_mask, start, size, 0, 0); 479 } 480 EXPORT_SYMBOL(sbi_remote_hfence_vvma); 481 482 /** 483 * sbi_remote_hfence_vvma_asid() - Execute HFENCE.VVMA instructions on given 484 * remote harts for current guest virtual address range belonging to a specific 485 * ASID. 486 * 487 * @hart_mask: A cpu mask containing all the target harts. 488 * @start: Start of the current guest virtual address 489 * @size: Total size of the current guest virtual address range. 490 * @asid: The value of address space identifier (ASID). 491 * 492 * Return: None 493 */ 494 int sbi_remote_hfence_vvma_asid(const unsigned long *hart_mask, 495 unsigned long start, 496 unsigned long size, 497 unsigned long asid) 498 { 499 return __sbi_rfence(SBI_EXT_RFENCE_REMOTE_HFENCE_VVMA_ASID, 500 hart_mask, start, size, asid, 0); 501 } 502 EXPORT_SYMBOL(sbi_remote_hfence_vvma_asid); 503 504 /** 505 * sbi_probe_extension() - Check if an SBI extension ID is supported or not. 506 * @extid: The extension ID to be probed. 507 * 508 * Return: Extension specific nonzero value f yes, -ENOTSUPP otherwise. 509 */ 510 int sbi_probe_extension(int extid) 511 { 512 struct sbiret ret; 513 514 ret = sbi_ecall(SBI_EXT_BASE, SBI_EXT_BASE_PROBE_EXT, extid, 515 0, 0, 0, 0, 0); 516 if (!ret.error) 517 if (ret.value) 518 return ret.value; 519 520 return -ENOTSUPP; 521 } 522 EXPORT_SYMBOL(sbi_probe_extension); 523 524 static long __sbi_base_ecall(int fid) 525 { 526 struct sbiret ret; 527 528 ret = sbi_ecall(SBI_EXT_BASE, fid, 0, 0, 0, 0, 0, 0); 529 if (!ret.error) 530 return ret.value; 531 else 532 return sbi_err_map_linux_errno(ret.error); 533 } 534 535 static inline long sbi_get_spec_version(void) 536 { 537 return __sbi_base_ecall(SBI_EXT_BASE_GET_SPEC_VERSION); 538 } 539 540 static inline long sbi_get_firmware_id(void) 541 { 542 return __sbi_base_ecall(SBI_EXT_BASE_GET_IMP_ID); 543 } 544 545 static inline long sbi_get_firmware_version(void) 546 { 547 return __sbi_base_ecall(SBI_EXT_BASE_GET_IMP_VERSION); 548 } 549 550 long sbi_get_mvendorid(void) 551 { 552 return __sbi_base_ecall(SBI_EXT_BASE_GET_MVENDORID); 553 } 554 555 long sbi_get_marchid(void) 556 { 557 return __sbi_base_ecall(SBI_EXT_BASE_GET_MARCHID); 558 } 559 560 long sbi_get_mimpid(void) 561 { 562 return __sbi_base_ecall(SBI_EXT_BASE_GET_MIMPID); 563 } 564 565 static void sbi_send_cpumask_ipi(const struct cpumask *target) 566 { 567 struct cpumask hartid_mask; 568 569 riscv_cpuid_to_hartid_mask(target, &hartid_mask); 570 571 sbi_send_ipi(cpumask_bits(&hartid_mask)); 572 } 573 574 static const struct riscv_ipi_ops sbi_ipi_ops = { 575 .ipi_inject = sbi_send_cpumask_ipi 576 }; 577 578 void __init sbi_init(void) 579 { 580 int ret; 581 582 sbi_set_power_off(); 583 ret = sbi_get_spec_version(); 584 if (ret > 0) 585 sbi_spec_version = ret; 586 587 pr_info("SBI specification v%lu.%lu detected\n", 588 sbi_major_version(), sbi_minor_version()); 589 590 if (!sbi_spec_is_0_1()) { 591 pr_info("SBI implementation ID=0x%lx Version=0x%lx\n", 592 sbi_get_firmware_id(), sbi_get_firmware_version()); 593 if (sbi_probe_extension(SBI_EXT_TIME) > 0) { 594 __sbi_set_timer = __sbi_set_timer_v02; 595 pr_info("SBI TIME extension detected\n"); 596 } else { 597 __sbi_set_timer = __sbi_set_timer_v01; 598 } 599 if (sbi_probe_extension(SBI_EXT_IPI) > 0) { 600 __sbi_send_ipi = __sbi_send_ipi_v02; 601 pr_info("SBI IPI extension detected\n"); 602 } else { 603 __sbi_send_ipi = __sbi_send_ipi_v01; 604 } 605 if (sbi_probe_extension(SBI_EXT_RFENCE) > 0) { 606 __sbi_rfence = __sbi_rfence_v02; 607 pr_info("SBI RFENCE extension detected\n"); 608 } else { 609 __sbi_rfence = __sbi_rfence_v01; 610 } 611 } else { 612 __sbi_set_timer = __sbi_set_timer_v01; 613 __sbi_send_ipi = __sbi_send_ipi_v01; 614 __sbi_rfence = __sbi_rfence_v01; 615 } 616 617 riscv_set_ipi_ops(&sbi_ipi_ops); 618 } 619