1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * SBI initialilization and all extension implementation. 4 * 5 * Copyright (c) 2020 Western Digital Corporation or its affiliates. 6 */ 7 8 #include <linux/init.h> 9 #include <linux/pm.h> 10 #include <asm/sbi.h> 11 #include <asm/smp.h> 12 13 /* default SBI version is 0.1 */ 14 unsigned long sbi_spec_version = SBI_SPEC_VERSION_DEFAULT; 15 EXPORT_SYMBOL(sbi_spec_version); 16 17 static void (*__sbi_set_timer)(uint64_t stime); 18 static int (*__sbi_send_ipi)(const unsigned long *hart_mask); 19 static int (*__sbi_rfence)(int fid, const unsigned long *hart_mask, 20 unsigned long start, unsigned long size, 21 unsigned long arg4, unsigned long arg5); 22 23 struct sbiret sbi_ecall(int ext, int fid, unsigned long arg0, 24 unsigned long arg1, unsigned long arg2, 25 unsigned long arg3, unsigned long arg4, 26 unsigned long arg5) 27 { 28 struct sbiret ret; 29 30 register uintptr_t a0 asm ("a0") = (uintptr_t)(arg0); 31 register uintptr_t a1 asm ("a1") = (uintptr_t)(arg1); 32 register uintptr_t a2 asm ("a2") = (uintptr_t)(arg2); 33 register uintptr_t a3 asm ("a3") = (uintptr_t)(arg3); 34 register uintptr_t a4 asm ("a4") = (uintptr_t)(arg4); 35 register uintptr_t a5 asm ("a5") = (uintptr_t)(arg5); 36 register uintptr_t a6 asm ("a6") = (uintptr_t)(fid); 37 register uintptr_t a7 asm ("a7") = (uintptr_t)(ext); 38 asm volatile ("ecall" 39 : "+r" (a0), "+r" (a1) 40 : "r" (a2), "r" (a3), "r" (a4), "r" (a5), "r" (a6), "r" (a7) 41 : "memory"); 42 ret.error = a0; 43 ret.value = a1; 44 45 return ret; 46 } 47 EXPORT_SYMBOL(sbi_ecall); 48 49 int sbi_err_map_linux_errno(int err) 50 { 51 switch (err) { 52 case SBI_SUCCESS: 53 return 0; 54 case SBI_ERR_DENIED: 55 return -EPERM; 56 case SBI_ERR_INVALID_PARAM: 57 return -EINVAL; 58 case SBI_ERR_INVALID_ADDRESS: 59 return -EFAULT; 60 case SBI_ERR_NOT_SUPPORTED: 61 case SBI_ERR_FAILURE: 62 default: 63 return -ENOTSUPP; 64 }; 65 } 66 EXPORT_SYMBOL(sbi_err_map_linux_errno); 67 68 #ifdef CONFIG_RISCV_SBI_V01 69 /** 70 * sbi_console_putchar() - Writes given character to the console device. 71 * @ch: The data to be written to the console. 72 * 73 * Return: None 74 */ 75 void sbi_console_putchar(int ch) 76 { 77 sbi_ecall(SBI_EXT_0_1_CONSOLE_PUTCHAR, 0, ch, 0, 0, 0, 0, 0); 78 } 79 EXPORT_SYMBOL(sbi_console_putchar); 80 81 /** 82 * sbi_console_getchar() - Reads a byte from console device. 83 * 84 * Returns the value read from console. 85 */ 86 int sbi_console_getchar(void) 87 { 88 struct sbiret ret; 89 90 ret = sbi_ecall(SBI_EXT_0_1_CONSOLE_GETCHAR, 0, 0, 0, 0, 0, 0, 0); 91 92 return ret.error; 93 } 94 EXPORT_SYMBOL(sbi_console_getchar); 95 96 /** 97 * sbi_shutdown() - Remove all the harts from executing supervisor code. 98 * 99 * Return: None 100 */ 101 void sbi_shutdown(void) 102 { 103 sbi_ecall(SBI_EXT_0_1_SHUTDOWN, 0, 0, 0, 0, 0, 0, 0); 104 } 105 EXPORT_SYMBOL(sbi_set_timer); 106 107 /** 108 * sbi_clear_ipi() - Clear any pending IPIs for the calling hart. 109 * 110 * Return: None 111 */ 112 void sbi_clear_ipi(void) 113 { 114 sbi_ecall(SBI_EXT_0_1_CLEAR_IPI, 0, 0, 0, 0, 0, 0, 0); 115 } 116 EXPORT_SYMBOL(sbi_shutdown); 117 118 /** 119 * sbi_set_timer_v01() - Program the timer for next timer event. 120 * @stime_value: The value after which next timer event should fire. 121 * 122 * Return: None 123 */ 124 static void __sbi_set_timer_v01(uint64_t stime_value) 125 { 126 #if __riscv_xlen == 32 127 sbi_ecall(SBI_EXT_0_1_SET_TIMER, 0, stime_value, 128 stime_value >> 32, 0, 0, 0, 0); 129 #else 130 sbi_ecall(SBI_EXT_0_1_SET_TIMER, 0, stime_value, 0, 0, 0, 0, 0); 131 #endif 132 } 133 134 static int __sbi_send_ipi_v01(const unsigned long *hart_mask) 135 { 136 sbi_ecall(SBI_EXT_0_1_SEND_IPI, 0, (unsigned long)hart_mask, 137 0, 0, 0, 0, 0); 138 return 0; 139 } 140 141 static int __sbi_rfence_v01(int fid, const unsigned long *hart_mask, 142 unsigned long start, unsigned long size, 143 unsigned long arg4, unsigned long arg5) 144 { 145 int result = 0; 146 147 /* v0.2 function IDs are equivalent to v0.1 extension IDs */ 148 switch (fid) { 149 case SBI_EXT_RFENCE_REMOTE_FENCE_I: 150 sbi_ecall(SBI_EXT_0_1_REMOTE_FENCE_I, 0, 151 (unsigned long)hart_mask, 0, 0, 0, 0, 0); 152 break; 153 case SBI_EXT_RFENCE_REMOTE_SFENCE_VMA: 154 sbi_ecall(SBI_EXT_0_1_REMOTE_SFENCE_VMA, 0, 155 (unsigned long)hart_mask, start, size, 156 0, 0, 0); 157 break; 158 case SBI_EXT_RFENCE_REMOTE_SFENCE_VMA_ASID: 159 sbi_ecall(SBI_EXT_0_1_REMOTE_SFENCE_VMA_ASID, 0, 160 (unsigned long)hart_mask, start, size, 161 arg4, 0, 0); 162 break; 163 default: 164 pr_err("SBI call [%d]not supported in SBI v0.1\n", fid); 165 result = -EINVAL; 166 } 167 168 return result; 169 } 170 #else 171 static void __sbi_set_timer_v01(uint64_t stime_value) 172 { 173 pr_warn("Timer extension is not available in SBI v%lu.%lu\n", 174 sbi_major_version(), sbi_minor_version()); 175 } 176 177 static int __sbi_send_ipi_v01(const unsigned long *hart_mask) 178 { 179 pr_warn("IPI extension is not available in SBI v%lu.%lu\n", 180 sbi_major_version(), sbi_minor_version()); 181 182 return 0; 183 } 184 185 static int __sbi_rfence_v01(int fid, const unsigned long *hart_mask, 186 unsigned long start, unsigned long size, 187 unsigned long arg4, unsigned long arg5) 188 { 189 pr_warn("remote fence extension is not available in SBI v%lu.%lu\n", 190 sbi_major_version(), sbi_minor_version()); 191 192 return 0; 193 } 194 #endif /* CONFIG_RISCV_SBI_V01 */ 195 196 static void __sbi_set_timer_v02(uint64_t stime_value) 197 { 198 #if __riscv_xlen == 32 199 sbi_ecall(SBI_EXT_TIME, SBI_EXT_TIME_SET_TIMER, stime_value, 200 stime_value >> 32, 0, 0, 0, 0); 201 #else 202 sbi_ecall(SBI_EXT_TIME, SBI_EXT_TIME_SET_TIMER, stime_value, 0, 203 0, 0, 0, 0); 204 #endif 205 } 206 207 static int __sbi_send_ipi_v02(const unsigned long *hart_mask) 208 { 209 unsigned long hartid, hmask_val, hbase; 210 struct cpumask tmask; 211 struct sbiret ret = {0}; 212 int result; 213 214 if (!hart_mask || !(*hart_mask)) { 215 riscv_cpuid_to_hartid_mask(cpu_online_mask, &tmask); 216 hart_mask = cpumask_bits(&tmask); 217 } 218 219 hmask_val = 0; 220 hbase = 0; 221 for_each_set_bit(hartid, hart_mask, NR_CPUS) { 222 if (hmask_val && ((hbase + BITS_PER_LONG) <= hartid)) { 223 ret = sbi_ecall(SBI_EXT_IPI, SBI_EXT_IPI_SEND_IPI, 224 hmask_val, hbase, 0, 0, 0, 0); 225 if (ret.error) 226 goto ecall_failed; 227 hmask_val = 0; 228 hbase = 0; 229 } 230 if (!hmask_val) 231 hbase = hartid; 232 hmask_val |= 1UL << (hartid - hbase); 233 } 234 235 if (hmask_val) { 236 ret = sbi_ecall(SBI_EXT_IPI, SBI_EXT_IPI_SEND_IPI, 237 hmask_val, hbase, 0, 0, 0, 0); 238 if (ret.error) 239 goto ecall_failed; 240 } 241 242 return 0; 243 244 ecall_failed: 245 result = sbi_err_map_linux_errno(ret.error); 246 pr_err("%s: hbase = [%lu] hmask = [0x%lx] failed (error [%d])\n", 247 __func__, hbase, hmask_val, result); 248 return result; 249 } 250 251 static int __sbi_rfence_v02_call(unsigned long fid, unsigned long hmask_val, 252 unsigned long hbase, unsigned long start, 253 unsigned long size, unsigned long arg4, 254 unsigned long arg5) 255 { 256 struct sbiret ret = {0}; 257 int ext = SBI_EXT_RFENCE; 258 int result = 0; 259 260 switch (fid) { 261 case SBI_EXT_RFENCE_REMOTE_FENCE_I: 262 ret = sbi_ecall(ext, fid, hmask_val, hbase, 0, 0, 0, 0); 263 break; 264 case SBI_EXT_RFENCE_REMOTE_SFENCE_VMA: 265 ret = sbi_ecall(ext, fid, hmask_val, hbase, start, 266 size, 0, 0); 267 break; 268 case SBI_EXT_RFENCE_REMOTE_SFENCE_VMA_ASID: 269 ret = sbi_ecall(ext, fid, hmask_val, hbase, start, 270 size, arg4, 0); 271 break; 272 273 case SBI_EXT_RFENCE_REMOTE_HFENCE_GVMA: 274 ret = sbi_ecall(ext, fid, hmask_val, hbase, start, 275 size, 0, 0); 276 break; 277 case SBI_EXT_RFENCE_REMOTE_HFENCE_GVMA_VMID: 278 ret = sbi_ecall(ext, fid, hmask_val, hbase, start, 279 size, arg4, 0); 280 break; 281 case SBI_EXT_RFENCE_REMOTE_HFENCE_VVMA: 282 ret = sbi_ecall(ext, fid, hmask_val, hbase, start, 283 size, 0, 0); 284 break; 285 case SBI_EXT_RFENCE_REMOTE_HFENCE_VVMA_ASID: 286 ret = sbi_ecall(ext, fid, hmask_val, hbase, start, 287 size, arg4, 0); 288 break; 289 default: 290 pr_err("unknown function ID [%lu] for SBI extension [%d]\n", 291 fid, ext); 292 result = -EINVAL; 293 } 294 295 if (ret.error) { 296 result = sbi_err_map_linux_errno(ret.error); 297 pr_err("%s: hbase = [%lu] hmask = [0x%lx] failed (error [%d])\n", 298 __func__, hbase, hmask_val, result); 299 } 300 301 return result; 302 } 303 304 static int __sbi_rfence_v02(int fid, const unsigned long *hart_mask, 305 unsigned long start, unsigned long size, 306 unsigned long arg4, unsigned long arg5) 307 { 308 unsigned long hmask_val, hartid, hbase; 309 struct cpumask tmask; 310 int result; 311 312 if (!hart_mask || !(*hart_mask)) { 313 riscv_cpuid_to_hartid_mask(cpu_online_mask, &tmask); 314 hart_mask = cpumask_bits(&tmask); 315 } 316 317 hmask_val = 0; 318 hbase = 0; 319 for_each_set_bit(hartid, hart_mask, NR_CPUS) { 320 if (hmask_val && ((hbase + BITS_PER_LONG) <= hartid)) { 321 result = __sbi_rfence_v02_call(fid, hmask_val, hbase, 322 start, size, arg4, arg5); 323 if (result) 324 return result; 325 hmask_val = 0; 326 hbase = 0; 327 } 328 if (!hmask_val) 329 hbase = hartid; 330 hmask_val |= 1UL << (hartid - hbase); 331 } 332 333 if (hmask_val) { 334 result = __sbi_rfence_v02_call(fid, hmask_val, hbase, 335 start, size, arg4, arg5); 336 if (result) 337 return result; 338 } 339 340 return 0; 341 } 342 343 /** 344 * sbi_set_timer() - Program the timer for next timer event. 345 * @stime_value: The value after which next timer event should fire. 346 * 347 * Return: None 348 */ 349 void sbi_set_timer(uint64_t stime_value) 350 { 351 __sbi_set_timer(stime_value); 352 } 353 354 /** 355 * sbi_send_ipi() - Send an IPI to any hart. 356 * @hart_mask: A cpu mask containing all the target harts. 357 * 358 * Return: None 359 */ 360 void sbi_send_ipi(const unsigned long *hart_mask) 361 { 362 __sbi_send_ipi(hart_mask); 363 } 364 EXPORT_SYMBOL(sbi_send_ipi); 365 366 /** 367 * sbi_remote_fence_i() - Execute FENCE.I instruction on given remote harts. 368 * @hart_mask: A cpu mask containing all the target harts. 369 * 370 * Return: None 371 */ 372 void sbi_remote_fence_i(const unsigned long *hart_mask) 373 { 374 __sbi_rfence(SBI_EXT_RFENCE_REMOTE_FENCE_I, 375 hart_mask, 0, 0, 0, 0); 376 } 377 EXPORT_SYMBOL(sbi_remote_fence_i); 378 379 /** 380 * sbi_remote_sfence_vma() - Execute SFENCE.VMA instructions on given remote 381 * harts for the specified virtual address range. 382 * @hart_mask: A cpu mask containing all the target harts. 383 * @start: Start of the virtual address 384 * @size: Total size of the virtual address range. 385 * 386 * Return: None 387 */ 388 void sbi_remote_sfence_vma(const unsigned long *hart_mask, 389 unsigned long start, 390 unsigned long size) 391 { 392 __sbi_rfence(SBI_EXT_RFENCE_REMOTE_SFENCE_VMA, 393 hart_mask, start, size, 0, 0); 394 } 395 EXPORT_SYMBOL(sbi_remote_sfence_vma); 396 397 /** 398 * sbi_remote_sfence_vma_asid() - Execute SFENCE.VMA instructions on given 399 * remote harts for a virtual address range belonging to a specific ASID. 400 * 401 * @hart_mask: A cpu mask containing all the target harts. 402 * @start: Start of the virtual address 403 * @size: Total size of the virtual address range. 404 * @asid: The value of address space identifier (ASID). 405 * 406 * Return: None 407 */ 408 void sbi_remote_sfence_vma_asid(const unsigned long *hart_mask, 409 unsigned long start, 410 unsigned long size, 411 unsigned long asid) 412 { 413 __sbi_rfence(SBI_EXT_RFENCE_REMOTE_SFENCE_VMA_ASID, 414 hart_mask, start, size, asid, 0); 415 } 416 EXPORT_SYMBOL(sbi_remote_sfence_vma_asid); 417 418 /** 419 * sbi_remote_hfence_gvma() - Execute HFENCE.GVMA instructions on given remote 420 * harts for the specified guest physical address range. 421 * @hart_mask: A cpu mask containing all the target harts. 422 * @start: Start of the guest physical address 423 * @size: Total size of the guest physical address range. 424 * 425 * Return: None 426 */ 427 int sbi_remote_hfence_gvma(const unsigned long *hart_mask, 428 unsigned long start, 429 unsigned long size) 430 { 431 return __sbi_rfence(SBI_EXT_RFENCE_REMOTE_HFENCE_GVMA, 432 hart_mask, start, size, 0, 0); 433 } 434 EXPORT_SYMBOL_GPL(sbi_remote_hfence_gvma); 435 436 /** 437 * sbi_remote_hfence_gvma_vmid() - Execute HFENCE.GVMA instructions on given 438 * remote harts for a guest physical address range belonging to a specific VMID. 439 * 440 * @hart_mask: A cpu mask containing all the target harts. 441 * @start: Start of the guest physical address 442 * @size: Total size of the guest physical address range. 443 * @vmid: The value of guest ID (VMID). 444 * 445 * Return: 0 if success, Error otherwise. 446 */ 447 int sbi_remote_hfence_gvma_vmid(const unsigned long *hart_mask, 448 unsigned long start, 449 unsigned long size, 450 unsigned long vmid) 451 { 452 return __sbi_rfence(SBI_EXT_RFENCE_REMOTE_HFENCE_GVMA_VMID, 453 hart_mask, start, size, vmid, 0); 454 } 455 EXPORT_SYMBOL(sbi_remote_hfence_gvma_vmid); 456 457 /** 458 * sbi_remote_hfence_vvma() - Execute HFENCE.VVMA instructions on given remote 459 * harts for the current guest virtual address range. 460 * @hart_mask: A cpu mask containing all the target harts. 461 * @start: Start of the current guest virtual address 462 * @size: Total size of the current guest virtual address range. 463 * 464 * Return: None 465 */ 466 int sbi_remote_hfence_vvma(const unsigned long *hart_mask, 467 unsigned long start, 468 unsigned long size) 469 { 470 return __sbi_rfence(SBI_EXT_RFENCE_REMOTE_HFENCE_VVMA, 471 hart_mask, start, size, 0, 0); 472 } 473 EXPORT_SYMBOL(sbi_remote_hfence_vvma); 474 475 /** 476 * sbi_remote_hfence_vvma_asid() - Execute HFENCE.VVMA instructions on given 477 * remote harts for current guest virtual address range belonging to a specific 478 * ASID. 479 * 480 * @hart_mask: A cpu mask containing all the target harts. 481 * @start: Start of the current guest virtual address 482 * @size: Total size of the current guest virtual address range. 483 * @asid: The value of address space identifier (ASID). 484 * 485 * Return: None 486 */ 487 int sbi_remote_hfence_vvma_asid(const unsigned long *hart_mask, 488 unsigned long start, 489 unsigned long size, 490 unsigned long asid) 491 { 492 return __sbi_rfence(SBI_EXT_RFENCE_REMOTE_HFENCE_VVMA_ASID, 493 hart_mask, start, size, asid, 0); 494 } 495 EXPORT_SYMBOL(sbi_remote_hfence_vvma_asid); 496 497 /** 498 * sbi_probe_extension() - Check if an SBI extension ID is supported or not. 499 * @extid: The extension ID to be probed. 500 * 501 * Return: Extension specific nonzero value f yes, -ENOTSUPP otherwise. 502 */ 503 int sbi_probe_extension(int extid) 504 { 505 struct sbiret ret; 506 507 ret = sbi_ecall(SBI_EXT_BASE, SBI_EXT_BASE_PROBE_EXT, extid, 508 0, 0, 0, 0, 0); 509 if (!ret.error) 510 if (ret.value) 511 return ret.value; 512 513 return -ENOTSUPP; 514 } 515 EXPORT_SYMBOL(sbi_probe_extension); 516 517 static long __sbi_base_ecall(int fid) 518 { 519 struct sbiret ret; 520 521 ret = sbi_ecall(SBI_EXT_BASE, fid, 0, 0, 0, 0, 0, 0); 522 if (!ret.error) 523 return ret.value; 524 else 525 return sbi_err_map_linux_errno(ret.error); 526 } 527 528 static inline long sbi_get_spec_version(void) 529 { 530 return __sbi_base_ecall(SBI_EXT_BASE_GET_SPEC_VERSION); 531 } 532 533 static inline long sbi_get_firmware_id(void) 534 { 535 return __sbi_base_ecall(SBI_EXT_BASE_GET_IMP_ID); 536 } 537 538 static inline long sbi_get_firmware_version(void) 539 { 540 return __sbi_base_ecall(SBI_EXT_BASE_GET_IMP_VERSION); 541 } 542 543 static void sbi_power_off(void) 544 { 545 sbi_shutdown(); 546 } 547 548 int __init sbi_init(void) 549 { 550 int ret; 551 552 pm_power_off = sbi_power_off; 553 ret = sbi_get_spec_version(); 554 if (ret > 0) 555 sbi_spec_version = ret; 556 557 pr_info("SBI specification v%lu.%lu detected\n", 558 sbi_major_version(), sbi_minor_version()); 559 560 if (!sbi_spec_is_0_1()) { 561 pr_info("SBI implementation ID=0x%lx Version=0x%lx\n", 562 sbi_get_firmware_id(), sbi_get_firmware_version()); 563 if (sbi_probe_extension(SBI_EXT_TIME) > 0) { 564 __sbi_set_timer = __sbi_set_timer_v02; 565 pr_info("SBI v0.2 TIME extension detected\n"); 566 } else { 567 __sbi_set_timer = __sbi_set_timer_v01; 568 } 569 if (sbi_probe_extension(SBI_EXT_IPI) > 0) { 570 __sbi_send_ipi = __sbi_send_ipi_v02; 571 pr_info("SBI v0.2 IPI extension detected\n"); 572 } else { 573 __sbi_send_ipi = __sbi_send_ipi_v01; 574 } 575 if (sbi_probe_extension(SBI_EXT_RFENCE) > 0) { 576 __sbi_rfence = __sbi_rfence_v02; 577 pr_info("SBI v0.2 RFENCE extension detected\n"); 578 } else { 579 __sbi_rfence = __sbi_rfence_v01; 580 } 581 } else { 582 __sbi_set_timer = __sbi_set_timer_v01; 583 __sbi_send_ipi = __sbi_send_ipi_v01; 584 __sbi_rfence = __sbi_rfence_v01; 585 } 586 587 return 0; 588 } 589