1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * SBI initialilization and all extension implementation. 4 * 5 * Copyright (c) 2020 Western Digital Corporation or its affiliates. 6 */ 7 8 #include <linux/bits.h> 9 #include <linux/init.h> 10 #include <linux/pm.h> 11 #include <linux/reboot.h> 12 #include <asm/sbi.h> 13 #include <asm/smp.h> 14 15 /* default SBI version is 0.1 */ 16 unsigned long sbi_spec_version __ro_after_init = SBI_SPEC_VERSION_DEFAULT; 17 EXPORT_SYMBOL(sbi_spec_version); 18 19 static void (*__sbi_set_timer)(uint64_t stime) __ro_after_init; 20 static void (*__sbi_send_ipi)(unsigned int cpu) __ro_after_init; 21 static int (*__sbi_rfence)(int fid, const struct cpumask *cpu_mask, 22 unsigned long start, unsigned long size, 23 unsigned long arg4, unsigned long arg5) __ro_after_init; 24 25 struct sbiret sbi_ecall(int ext, int fid, unsigned long arg0, 26 unsigned long arg1, unsigned long arg2, 27 unsigned long arg3, unsigned long arg4, 28 unsigned long arg5) 29 { 30 struct sbiret ret; 31 32 register uintptr_t a0 asm ("a0") = (uintptr_t)(arg0); 33 register uintptr_t a1 asm ("a1") = (uintptr_t)(arg1); 34 register uintptr_t a2 asm ("a2") = (uintptr_t)(arg2); 35 register uintptr_t a3 asm ("a3") = (uintptr_t)(arg3); 36 register uintptr_t a4 asm ("a4") = (uintptr_t)(arg4); 37 register uintptr_t a5 asm ("a5") = (uintptr_t)(arg5); 38 register uintptr_t a6 asm ("a6") = (uintptr_t)(fid); 39 register uintptr_t a7 asm ("a7") = (uintptr_t)(ext); 40 asm volatile ("ecall" 41 : "+r" (a0), "+r" (a1) 42 : "r" (a2), "r" (a3), "r" (a4), "r" (a5), "r" (a6), "r" (a7) 43 : "memory"); 44 ret.error = a0; 45 ret.value = a1; 46 47 return ret; 48 } 49 EXPORT_SYMBOL(sbi_ecall); 50 51 int sbi_err_map_linux_errno(int err) 52 { 53 switch (err) { 54 case SBI_SUCCESS: 55 return 0; 56 case SBI_ERR_DENIED: 57 return -EPERM; 58 case SBI_ERR_INVALID_PARAM: 59 return -EINVAL; 60 case SBI_ERR_INVALID_ADDRESS: 61 return -EFAULT; 62 case SBI_ERR_NOT_SUPPORTED: 63 case SBI_ERR_FAILURE: 64 default: 65 return -ENOTSUPP; 66 }; 67 } 68 EXPORT_SYMBOL(sbi_err_map_linux_errno); 69 70 #ifdef CONFIG_RISCV_SBI_V01 71 static unsigned long __sbi_v01_cpumask_to_hartmask(const struct cpumask *cpu_mask) 72 { 73 unsigned long cpuid, hartid; 74 unsigned long hmask = 0; 75 76 /* 77 * There is no maximum hartid concept in RISC-V and NR_CPUS must not be 78 * associated with hartid. As SBI v0.1 is only kept for backward compatibility 79 * and will be removed in the future, there is no point in supporting hartid 80 * greater than BITS_PER_LONG (32 for RV32 and 64 for RV64). Ideally, SBI v0.2 81 * should be used for platforms with hartid greater than BITS_PER_LONG. 82 */ 83 for_each_cpu(cpuid, cpu_mask) { 84 hartid = cpuid_to_hartid_map(cpuid); 85 if (hartid >= BITS_PER_LONG) { 86 pr_warn("Unable to send any request to hartid > BITS_PER_LONG for SBI v0.1\n"); 87 break; 88 } 89 hmask |= BIT(hartid); 90 } 91 92 return hmask; 93 } 94 95 /** 96 * sbi_console_putchar() - Writes given character to the console device. 97 * @ch: The data to be written to the console. 98 * 99 * Return: None 100 */ 101 void sbi_console_putchar(int ch) 102 { 103 sbi_ecall(SBI_EXT_0_1_CONSOLE_PUTCHAR, 0, ch, 0, 0, 0, 0, 0); 104 } 105 EXPORT_SYMBOL(sbi_console_putchar); 106 107 /** 108 * sbi_console_getchar() - Reads a byte from console device. 109 * 110 * Returns the value read from console. 111 */ 112 int sbi_console_getchar(void) 113 { 114 struct sbiret ret; 115 116 ret = sbi_ecall(SBI_EXT_0_1_CONSOLE_GETCHAR, 0, 0, 0, 0, 0, 0, 0); 117 118 return ret.error; 119 } 120 EXPORT_SYMBOL(sbi_console_getchar); 121 122 /** 123 * sbi_shutdown() - Remove all the harts from executing supervisor code. 124 * 125 * Return: None 126 */ 127 void sbi_shutdown(void) 128 { 129 sbi_ecall(SBI_EXT_0_1_SHUTDOWN, 0, 0, 0, 0, 0, 0, 0); 130 } 131 EXPORT_SYMBOL(sbi_shutdown); 132 133 /** 134 * __sbi_set_timer_v01() - Program the timer for next timer event. 135 * @stime_value: The value after which next timer event should fire. 136 * 137 * Return: None 138 */ 139 static void __sbi_set_timer_v01(uint64_t stime_value) 140 { 141 #if __riscv_xlen == 32 142 sbi_ecall(SBI_EXT_0_1_SET_TIMER, 0, stime_value, 143 stime_value >> 32, 0, 0, 0, 0); 144 #else 145 sbi_ecall(SBI_EXT_0_1_SET_TIMER, 0, stime_value, 0, 0, 0, 0, 0); 146 #endif 147 } 148 149 static void __sbi_send_ipi_v01(unsigned int cpu) 150 { 151 unsigned long hart_mask = 152 __sbi_v01_cpumask_to_hartmask(cpumask_of(cpu)); 153 sbi_ecall(SBI_EXT_0_1_SEND_IPI, 0, (unsigned long)(&hart_mask), 154 0, 0, 0, 0, 0); 155 } 156 157 static int __sbi_rfence_v01(int fid, const struct cpumask *cpu_mask, 158 unsigned long start, unsigned long size, 159 unsigned long arg4, unsigned long arg5) 160 { 161 int result = 0; 162 unsigned long hart_mask; 163 164 if (!cpu_mask || cpumask_empty(cpu_mask)) 165 cpu_mask = cpu_online_mask; 166 hart_mask = __sbi_v01_cpumask_to_hartmask(cpu_mask); 167 168 /* v0.2 function IDs are equivalent to v0.1 extension IDs */ 169 switch (fid) { 170 case SBI_EXT_RFENCE_REMOTE_FENCE_I: 171 sbi_ecall(SBI_EXT_0_1_REMOTE_FENCE_I, 0, 172 (unsigned long)&hart_mask, 0, 0, 0, 0, 0); 173 break; 174 case SBI_EXT_RFENCE_REMOTE_SFENCE_VMA: 175 sbi_ecall(SBI_EXT_0_1_REMOTE_SFENCE_VMA, 0, 176 (unsigned long)&hart_mask, start, size, 177 0, 0, 0); 178 break; 179 case SBI_EXT_RFENCE_REMOTE_SFENCE_VMA_ASID: 180 sbi_ecall(SBI_EXT_0_1_REMOTE_SFENCE_VMA_ASID, 0, 181 (unsigned long)&hart_mask, start, size, 182 arg4, 0, 0); 183 break; 184 default: 185 pr_err("SBI call [%d]not supported in SBI v0.1\n", fid); 186 result = -EINVAL; 187 } 188 189 return result; 190 } 191 192 static void sbi_set_power_off(void) 193 { 194 pm_power_off = sbi_shutdown; 195 } 196 #else 197 static void __sbi_set_timer_v01(uint64_t stime_value) 198 { 199 pr_warn("Timer extension is not available in SBI v%lu.%lu\n", 200 sbi_major_version(), sbi_minor_version()); 201 } 202 203 static void __sbi_send_ipi_v01(unsigned int cpu) 204 { 205 pr_warn("IPI extension is not available in SBI v%lu.%lu\n", 206 sbi_major_version(), sbi_minor_version()); 207 } 208 209 static int __sbi_rfence_v01(int fid, const struct cpumask *cpu_mask, 210 unsigned long start, unsigned long size, 211 unsigned long arg4, unsigned long arg5) 212 { 213 pr_warn("remote fence extension is not available in SBI v%lu.%lu\n", 214 sbi_major_version(), sbi_minor_version()); 215 216 return 0; 217 } 218 219 static void sbi_set_power_off(void) {} 220 #endif /* CONFIG_RISCV_SBI_V01 */ 221 222 static void __sbi_set_timer_v02(uint64_t stime_value) 223 { 224 #if __riscv_xlen == 32 225 sbi_ecall(SBI_EXT_TIME, SBI_EXT_TIME_SET_TIMER, stime_value, 226 stime_value >> 32, 0, 0, 0, 0); 227 #else 228 sbi_ecall(SBI_EXT_TIME, SBI_EXT_TIME_SET_TIMER, stime_value, 0, 229 0, 0, 0, 0); 230 #endif 231 } 232 233 static void __sbi_send_ipi_v02(unsigned int cpu) 234 { 235 int result; 236 struct sbiret ret = {0}; 237 238 ret = sbi_ecall(SBI_EXT_IPI, SBI_EXT_IPI_SEND_IPI, 239 1UL, cpuid_to_hartid_map(cpu), 0, 0, 0, 0); 240 if (ret.error) { 241 result = sbi_err_map_linux_errno(ret.error); 242 pr_err("%s: hbase = [%lu] failed (error [%d])\n", 243 __func__, cpuid_to_hartid_map(cpu), result); 244 } 245 } 246 247 static int __sbi_rfence_v02_call(unsigned long fid, unsigned long hmask, 248 unsigned long hbase, unsigned long start, 249 unsigned long size, unsigned long arg4, 250 unsigned long arg5) 251 { 252 struct sbiret ret = {0}; 253 int ext = SBI_EXT_RFENCE; 254 int result = 0; 255 256 switch (fid) { 257 case SBI_EXT_RFENCE_REMOTE_FENCE_I: 258 ret = sbi_ecall(ext, fid, hmask, hbase, 0, 0, 0, 0); 259 break; 260 case SBI_EXT_RFENCE_REMOTE_SFENCE_VMA: 261 ret = sbi_ecall(ext, fid, hmask, hbase, start, 262 size, 0, 0); 263 break; 264 case SBI_EXT_RFENCE_REMOTE_SFENCE_VMA_ASID: 265 ret = sbi_ecall(ext, fid, hmask, hbase, start, 266 size, arg4, 0); 267 break; 268 269 case SBI_EXT_RFENCE_REMOTE_HFENCE_GVMA: 270 ret = sbi_ecall(ext, fid, hmask, hbase, start, 271 size, 0, 0); 272 break; 273 case SBI_EXT_RFENCE_REMOTE_HFENCE_GVMA_VMID: 274 ret = sbi_ecall(ext, fid, hmask, hbase, start, 275 size, arg4, 0); 276 break; 277 case SBI_EXT_RFENCE_REMOTE_HFENCE_VVMA: 278 ret = sbi_ecall(ext, fid, hmask, hbase, start, 279 size, 0, 0); 280 break; 281 case SBI_EXT_RFENCE_REMOTE_HFENCE_VVMA_ASID: 282 ret = sbi_ecall(ext, fid, hmask, hbase, start, 283 size, arg4, 0); 284 break; 285 default: 286 pr_err("unknown function ID [%lu] for SBI extension [%d]\n", 287 fid, ext); 288 result = -EINVAL; 289 } 290 291 if (ret.error) { 292 result = sbi_err_map_linux_errno(ret.error); 293 pr_err("%s: hbase = [%lu] hmask = [0x%lx] failed (error [%d])\n", 294 __func__, hbase, hmask, result); 295 } 296 297 return result; 298 } 299 300 static int __sbi_rfence_v02(int fid, const struct cpumask *cpu_mask, 301 unsigned long start, unsigned long size, 302 unsigned long arg4, unsigned long arg5) 303 { 304 unsigned long hartid, cpuid, hmask = 0, hbase = 0, htop = 0; 305 int result; 306 307 if (!cpu_mask || cpumask_empty(cpu_mask)) 308 cpu_mask = cpu_online_mask; 309 310 for_each_cpu(cpuid, cpu_mask) { 311 hartid = cpuid_to_hartid_map(cpuid); 312 if (hmask) { 313 if (hartid + BITS_PER_LONG <= htop || 314 hbase + BITS_PER_LONG <= hartid) { 315 result = __sbi_rfence_v02_call(fid, hmask, 316 hbase, start, size, arg4, arg5); 317 if (result) 318 return result; 319 hmask = 0; 320 } else if (hartid < hbase) { 321 /* shift the mask to fit lower hartid */ 322 hmask <<= hbase - hartid; 323 hbase = hartid; 324 } 325 } 326 if (!hmask) { 327 hbase = hartid; 328 htop = hartid; 329 } else if (hartid > htop) { 330 htop = hartid; 331 } 332 hmask |= BIT(hartid - hbase); 333 } 334 335 if (hmask) { 336 result = __sbi_rfence_v02_call(fid, hmask, hbase, 337 start, size, arg4, arg5); 338 if (result) 339 return result; 340 } 341 342 return 0; 343 } 344 345 /** 346 * sbi_set_timer() - Program the timer for next timer event. 347 * @stime_value: The value after which next timer event should fire. 348 * 349 * Return: None. 350 */ 351 void sbi_set_timer(uint64_t stime_value) 352 { 353 __sbi_set_timer(stime_value); 354 } 355 356 /** 357 * sbi_send_ipi() - Send an IPI to any hart. 358 * @cpu: Logical id of the target CPU. 359 */ 360 void sbi_send_ipi(unsigned int cpu) 361 { 362 __sbi_send_ipi(cpu); 363 } 364 EXPORT_SYMBOL(sbi_send_ipi); 365 366 /** 367 * sbi_remote_fence_i() - Execute FENCE.I instruction on given remote harts. 368 * @cpu_mask: A cpu mask containing all the target harts. 369 * 370 * Return: 0 on success, appropriate linux error code otherwise. 371 */ 372 int sbi_remote_fence_i(const struct cpumask *cpu_mask) 373 { 374 return __sbi_rfence(SBI_EXT_RFENCE_REMOTE_FENCE_I, 375 cpu_mask, 0, 0, 0, 0); 376 } 377 EXPORT_SYMBOL(sbi_remote_fence_i); 378 379 /** 380 * sbi_remote_sfence_vma() - Execute SFENCE.VMA instructions on given remote 381 * harts for the specified virtual address range. 382 * @cpu_mask: A cpu mask containing all the target harts. 383 * @start: Start of the virtual address 384 * @size: Total size of the virtual address range. 385 * 386 * Return: 0 on success, appropriate linux error code otherwise. 387 */ 388 int sbi_remote_sfence_vma(const struct cpumask *cpu_mask, 389 unsigned long start, 390 unsigned long size) 391 { 392 return __sbi_rfence(SBI_EXT_RFENCE_REMOTE_SFENCE_VMA, 393 cpu_mask, start, size, 0, 0); 394 } 395 EXPORT_SYMBOL(sbi_remote_sfence_vma); 396 397 /** 398 * sbi_remote_sfence_vma_asid() - Execute SFENCE.VMA instructions on given 399 * remote harts for a virtual address range belonging to a specific ASID. 400 * 401 * @cpu_mask: A cpu mask containing all the target harts. 402 * @start: Start of the virtual address 403 * @size: Total size of the virtual address range. 404 * @asid: The value of address space identifier (ASID). 405 * 406 * Return: 0 on success, appropriate linux error code otherwise. 407 */ 408 int sbi_remote_sfence_vma_asid(const struct cpumask *cpu_mask, 409 unsigned long start, 410 unsigned long size, 411 unsigned long asid) 412 { 413 return __sbi_rfence(SBI_EXT_RFENCE_REMOTE_SFENCE_VMA_ASID, 414 cpu_mask, start, size, asid, 0); 415 } 416 EXPORT_SYMBOL(sbi_remote_sfence_vma_asid); 417 418 /** 419 * sbi_remote_hfence_gvma() - Execute HFENCE.GVMA instructions on given remote 420 * harts for the specified guest physical address range. 421 * @cpu_mask: A cpu mask containing all the target harts. 422 * @start: Start of the guest physical address 423 * @size: Total size of the guest physical address range. 424 * 425 * Return: None 426 */ 427 int sbi_remote_hfence_gvma(const struct cpumask *cpu_mask, 428 unsigned long start, 429 unsigned long size) 430 { 431 return __sbi_rfence(SBI_EXT_RFENCE_REMOTE_HFENCE_GVMA, 432 cpu_mask, start, size, 0, 0); 433 } 434 EXPORT_SYMBOL_GPL(sbi_remote_hfence_gvma); 435 436 /** 437 * sbi_remote_hfence_gvma_vmid() - Execute HFENCE.GVMA instructions on given 438 * remote harts for a guest physical address range belonging to a specific VMID. 439 * 440 * @cpu_mask: A cpu mask containing all the target harts. 441 * @start: Start of the guest physical address 442 * @size: Total size of the guest physical address range. 443 * @vmid: The value of guest ID (VMID). 444 * 445 * Return: 0 if success, Error otherwise. 446 */ 447 int sbi_remote_hfence_gvma_vmid(const struct cpumask *cpu_mask, 448 unsigned long start, 449 unsigned long size, 450 unsigned long vmid) 451 { 452 return __sbi_rfence(SBI_EXT_RFENCE_REMOTE_HFENCE_GVMA_VMID, 453 cpu_mask, start, size, vmid, 0); 454 } 455 EXPORT_SYMBOL(sbi_remote_hfence_gvma_vmid); 456 457 /** 458 * sbi_remote_hfence_vvma() - Execute HFENCE.VVMA instructions on given remote 459 * harts for the current guest virtual address range. 460 * @cpu_mask: A cpu mask containing all the target harts. 461 * @start: Start of the current guest virtual address 462 * @size: Total size of the current guest virtual address range. 463 * 464 * Return: None 465 */ 466 int sbi_remote_hfence_vvma(const struct cpumask *cpu_mask, 467 unsigned long start, 468 unsigned long size) 469 { 470 return __sbi_rfence(SBI_EXT_RFENCE_REMOTE_HFENCE_VVMA, 471 cpu_mask, start, size, 0, 0); 472 } 473 EXPORT_SYMBOL(sbi_remote_hfence_vvma); 474 475 /** 476 * sbi_remote_hfence_vvma_asid() - Execute HFENCE.VVMA instructions on given 477 * remote harts for current guest virtual address range belonging to a specific 478 * ASID. 479 * 480 * @cpu_mask: A cpu mask containing all the target harts. 481 * @start: Start of the current guest virtual address 482 * @size: Total size of the current guest virtual address range. 483 * @asid: The value of address space identifier (ASID). 484 * 485 * Return: None 486 */ 487 int sbi_remote_hfence_vvma_asid(const struct cpumask *cpu_mask, 488 unsigned long start, 489 unsigned long size, 490 unsigned long asid) 491 { 492 return __sbi_rfence(SBI_EXT_RFENCE_REMOTE_HFENCE_VVMA_ASID, 493 cpu_mask, start, size, asid, 0); 494 } 495 EXPORT_SYMBOL(sbi_remote_hfence_vvma_asid); 496 497 static void sbi_srst_reset(unsigned long type, unsigned long reason) 498 { 499 sbi_ecall(SBI_EXT_SRST, SBI_EXT_SRST_RESET, type, reason, 500 0, 0, 0, 0); 501 pr_warn("%s: type=0x%lx reason=0x%lx failed\n", 502 __func__, type, reason); 503 } 504 505 static int sbi_srst_reboot(struct notifier_block *this, 506 unsigned long mode, void *cmd) 507 { 508 sbi_srst_reset((mode == REBOOT_WARM || mode == REBOOT_SOFT) ? 509 SBI_SRST_RESET_TYPE_WARM_REBOOT : 510 SBI_SRST_RESET_TYPE_COLD_REBOOT, 511 SBI_SRST_RESET_REASON_NONE); 512 return NOTIFY_DONE; 513 } 514 515 static struct notifier_block sbi_srst_reboot_nb; 516 517 static void sbi_srst_power_off(void) 518 { 519 sbi_srst_reset(SBI_SRST_RESET_TYPE_SHUTDOWN, 520 SBI_SRST_RESET_REASON_NONE); 521 } 522 523 /** 524 * sbi_probe_extension() - Check if an SBI extension ID is supported or not. 525 * @extid: The extension ID to be probed. 526 * 527 * Return: 1 or an extension specific nonzero value if yes, 0 otherwise. 528 */ 529 long sbi_probe_extension(int extid) 530 { 531 struct sbiret ret; 532 533 ret = sbi_ecall(SBI_EXT_BASE, SBI_EXT_BASE_PROBE_EXT, extid, 534 0, 0, 0, 0, 0); 535 if (!ret.error) 536 return ret.value; 537 538 return 0; 539 } 540 EXPORT_SYMBOL(sbi_probe_extension); 541 542 static long __sbi_base_ecall(int fid) 543 { 544 struct sbiret ret; 545 546 ret = sbi_ecall(SBI_EXT_BASE, fid, 0, 0, 0, 0, 0, 0); 547 if (!ret.error) 548 return ret.value; 549 else 550 return sbi_err_map_linux_errno(ret.error); 551 } 552 553 static inline long sbi_get_spec_version(void) 554 { 555 return __sbi_base_ecall(SBI_EXT_BASE_GET_SPEC_VERSION); 556 } 557 558 static inline long sbi_get_firmware_id(void) 559 { 560 return __sbi_base_ecall(SBI_EXT_BASE_GET_IMP_ID); 561 } 562 563 static inline long sbi_get_firmware_version(void) 564 { 565 return __sbi_base_ecall(SBI_EXT_BASE_GET_IMP_VERSION); 566 } 567 568 long sbi_get_mvendorid(void) 569 { 570 return __sbi_base_ecall(SBI_EXT_BASE_GET_MVENDORID); 571 } 572 EXPORT_SYMBOL_GPL(sbi_get_mvendorid); 573 574 long sbi_get_marchid(void) 575 { 576 return __sbi_base_ecall(SBI_EXT_BASE_GET_MARCHID); 577 } 578 EXPORT_SYMBOL_GPL(sbi_get_marchid); 579 580 long sbi_get_mimpid(void) 581 { 582 return __sbi_base_ecall(SBI_EXT_BASE_GET_MIMPID); 583 } 584 EXPORT_SYMBOL_GPL(sbi_get_mimpid); 585 586 void __init sbi_init(void) 587 { 588 int ret; 589 590 sbi_set_power_off(); 591 ret = sbi_get_spec_version(); 592 if (ret > 0) 593 sbi_spec_version = ret; 594 595 pr_info("SBI specification v%lu.%lu detected\n", 596 sbi_major_version(), sbi_minor_version()); 597 598 if (!sbi_spec_is_0_1()) { 599 pr_info("SBI implementation ID=0x%lx Version=0x%lx\n", 600 sbi_get_firmware_id(), sbi_get_firmware_version()); 601 if (sbi_probe_extension(SBI_EXT_TIME)) { 602 __sbi_set_timer = __sbi_set_timer_v02; 603 pr_info("SBI TIME extension detected\n"); 604 } else { 605 __sbi_set_timer = __sbi_set_timer_v01; 606 } 607 if (sbi_probe_extension(SBI_EXT_IPI)) { 608 __sbi_send_ipi = __sbi_send_ipi_v02; 609 pr_info("SBI IPI extension detected\n"); 610 } else { 611 __sbi_send_ipi = __sbi_send_ipi_v01; 612 } 613 if (sbi_probe_extension(SBI_EXT_RFENCE)) { 614 __sbi_rfence = __sbi_rfence_v02; 615 pr_info("SBI RFENCE extension detected\n"); 616 } else { 617 __sbi_rfence = __sbi_rfence_v01; 618 } 619 if ((sbi_spec_version >= sbi_mk_version(0, 3)) && 620 sbi_probe_extension(SBI_EXT_SRST)) { 621 pr_info("SBI SRST extension detected\n"); 622 pm_power_off = sbi_srst_power_off; 623 sbi_srst_reboot_nb.notifier_call = sbi_srst_reboot; 624 sbi_srst_reboot_nb.priority = 192; 625 register_restart_handler(&sbi_srst_reboot_nb); 626 } 627 } else { 628 __sbi_set_timer = __sbi_set_timer_v01; 629 __sbi_send_ipi = __sbi_send_ipi_v01; 630 __sbi_rfence = __sbi_rfence_v01; 631 } 632 } 633