1 /* 2 * RISC-V Control and Status Registers. 3 * 4 * Copyright (c) 2016-2017 Sagar Karandikar, sagark@eecs.berkeley.edu 5 * Copyright (c) 2017-2018 SiFive, Inc. 6 * 7 * This program is free software; you can redistribute it and/or modify it 8 * under the terms and conditions of the GNU General Public License, 9 * version 2 or later, as published by the Free Software Foundation. 10 * 11 * This program is distributed in the hope it will be useful, but WITHOUT 12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 14 * more details. 15 * 16 * You should have received a copy of the GNU General Public License along with 17 * this program. If not, see <http://www.gnu.org/licenses/>. 18 */ 19 20 #include "qemu/osdep.h" 21 #include "qemu/log.h" 22 #include "cpu.h" 23 #include "qemu/main-loop.h" 24 #include "exec/exec-all.h" 25 26 /* CSR function table */ 27 static riscv_csr_operations csr_ops[]; 28 29 /* CSR function table constants */ 30 enum { 31 CSR_TABLE_SIZE = 0x1000 32 }; 33 34 /* CSR function table public API */ 35 void riscv_get_csr_ops(int csrno, riscv_csr_operations *ops) 36 { 37 *ops = csr_ops[csrno & (CSR_TABLE_SIZE - 1)]; 38 } 39 40 void riscv_set_csr_ops(int csrno, riscv_csr_operations *ops) 41 { 42 csr_ops[csrno & (CSR_TABLE_SIZE - 1)] = *ops; 43 } 44 45 /* Predicates */ 46 static int fs(CPURISCVState *env, int csrno) 47 { 48 #if !defined(CONFIG_USER_ONLY) 49 /* loose check condition for fcsr in vector extension */ 50 if ((csrno == CSR_FCSR) && (env->misa & RVV)) { 51 return 0; 52 } 53 if (!env->debugger && !riscv_cpu_fp_enabled(env)) { 54 return -1; 55 } 56 #endif 57 return 0; 58 } 59 60 static int vs(CPURISCVState *env, int csrno) 61 { 62 if (env->misa & RVV) { 63 return 0; 64 } 65 return -1; 66 } 67 68 static int ctr(CPURISCVState *env, int csrno) 69 { 70 #if !defined(CONFIG_USER_ONLY) 71 CPUState *cs = env_cpu(env); 72 RISCVCPU *cpu = RISCV_CPU(cs); 73 74 if (!cpu->cfg.ext_counters) { 75 /* The Counters extensions is not enabled */ 76 return -1; 77 } 78 #endif 79 return 0; 80 } 81 82 #if !defined(CONFIG_USER_ONLY) 83 static int any(CPURISCVState *env, int csrno) 84 { 85 return 0; 86 } 87 88 static int smode(CPURISCVState *env, int csrno) 89 { 90 return -!riscv_has_ext(env, RVS); 91 } 92 93 static int hmode(CPURISCVState *env, int csrno) 94 { 95 if (riscv_has_ext(env, RVS) && 96 riscv_has_ext(env, RVH)) { 97 /* Hypervisor extension is supported */ 98 if ((env->priv == PRV_S && !riscv_cpu_virt_enabled(env)) || 99 env->priv == PRV_M) { 100 return 0; 101 } 102 } 103 104 return -1; 105 } 106 107 static int pmp(CPURISCVState *env, int csrno) 108 { 109 return -!riscv_feature(env, RISCV_FEATURE_PMP); 110 } 111 #endif 112 113 /* User Floating-Point CSRs */ 114 static int read_fflags(CPURISCVState *env, int csrno, target_ulong *val) 115 { 116 #if !defined(CONFIG_USER_ONLY) 117 if (!env->debugger && !riscv_cpu_fp_enabled(env)) { 118 return -1; 119 } 120 #endif 121 *val = riscv_cpu_get_fflags(env); 122 return 0; 123 } 124 125 static int write_fflags(CPURISCVState *env, int csrno, target_ulong val) 126 { 127 #if !defined(CONFIG_USER_ONLY) 128 if (!env->debugger && !riscv_cpu_fp_enabled(env)) { 129 return -1; 130 } 131 env->mstatus |= MSTATUS_FS; 132 #endif 133 riscv_cpu_set_fflags(env, val & (FSR_AEXC >> FSR_AEXC_SHIFT)); 134 return 0; 135 } 136 137 static int read_frm(CPURISCVState *env, int csrno, target_ulong *val) 138 { 139 #if !defined(CONFIG_USER_ONLY) 140 if (!env->debugger && !riscv_cpu_fp_enabled(env)) { 141 return -1; 142 } 143 #endif 144 *val = env->frm; 145 return 0; 146 } 147 148 static int write_frm(CPURISCVState *env, int csrno, target_ulong val) 149 { 150 #if !defined(CONFIG_USER_ONLY) 151 if (!env->debugger && !riscv_cpu_fp_enabled(env)) { 152 return -1; 153 } 154 env->mstatus |= MSTATUS_FS; 155 #endif 156 env->frm = val & (FSR_RD >> FSR_RD_SHIFT); 157 return 0; 158 } 159 160 static int read_fcsr(CPURISCVState *env, int csrno, target_ulong *val) 161 { 162 #if !defined(CONFIG_USER_ONLY) 163 if (!env->debugger && !riscv_cpu_fp_enabled(env)) { 164 return -1; 165 } 166 #endif 167 *val = (riscv_cpu_get_fflags(env) << FSR_AEXC_SHIFT) 168 | (env->frm << FSR_RD_SHIFT); 169 if (vs(env, csrno) >= 0) { 170 *val |= (env->vxrm << FSR_VXRM_SHIFT) 171 | (env->vxsat << FSR_VXSAT_SHIFT); 172 } 173 return 0; 174 } 175 176 static int write_fcsr(CPURISCVState *env, int csrno, target_ulong val) 177 { 178 #if !defined(CONFIG_USER_ONLY) 179 if (!env->debugger && !riscv_cpu_fp_enabled(env)) { 180 return -1; 181 } 182 env->mstatus |= MSTATUS_FS; 183 #endif 184 env->frm = (val & FSR_RD) >> FSR_RD_SHIFT; 185 if (vs(env, csrno) >= 0) { 186 env->vxrm = (val & FSR_VXRM) >> FSR_VXRM_SHIFT; 187 env->vxsat = (val & FSR_VXSAT) >> FSR_VXSAT_SHIFT; 188 } 189 riscv_cpu_set_fflags(env, (val & FSR_AEXC) >> FSR_AEXC_SHIFT); 190 return 0; 191 } 192 193 static int read_vtype(CPURISCVState *env, int csrno, target_ulong *val) 194 { 195 *val = env->vtype; 196 return 0; 197 } 198 199 static int read_vl(CPURISCVState *env, int csrno, target_ulong *val) 200 { 201 *val = env->vl; 202 return 0; 203 } 204 205 static int read_vxrm(CPURISCVState *env, int csrno, target_ulong *val) 206 { 207 *val = env->vxrm; 208 return 0; 209 } 210 211 static int write_vxrm(CPURISCVState *env, int csrno, target_ulong val) 212 { 213 env->vxrm = val; 214 return 0; 215 } 216 217 static int read_vxsat(CPURISCVState *env, int csrno, target_ulong *val) 218 { 219 *val = env->vxsat; 220 return 0; 221 } 222 223 static int write_vxsat(CPURISCVState *env, int csrno, target_ulong val) 224 { 225 env->vxsat = val; 226 return 0; 227 } 228 229 static int read_vstart(CPURISCVState *env, int csrno, target_ulong *val) 230 { 231 *val = env->vstart; 232 return 0; 233 } 234 235 static int write_vstart(CPURISCVState *env, int csrno, target_ulong val) 236 { 237 env->vstart = val; 238 return 0; 239 } 240 241 /* User Timers and Counters */ 242 static int read_instret(CPURISCVState *env, int csrno, target_ulong *val) 243 { 244 #if !defined(CONFIG_USER_ONLY) 245 if (use_icount) { 246 *val = cpu_get_icount(); 247 } else { 248 *val = cpu_get_host_ticks(); 249 } 250 #else 251 *val = cpu_get_host_ticks(); 252 #endif 253 return 0; 254 } 255 256 #if defined(TARGET_RISCV32) 257 static int read_instreth(CPURISCVState *env, int csrno, target_ulong *val) 258 { 259 #if !defined(CONFIG_USER_ONLY) 260 if (use_icount) { 261 *val = cpu_get_icount() >> 32; 262 } else { 263 *val = cpu_get_host_ticks() >> 32; 264 } 265 #else 266 *val = cpu_get_host_ticks() >> 32; 267 #endif 268 return 0; 269 } 270 #endif /* TARGET_RISCV32 */ 271 272 #if defined(CONFIG_USER_ONLY) 273 static int read_time(CPURISCVState *env, int csrno, target_ulong *val) 274 { 275 *val = cpu_get_host_ticks(); 276 return 0; 277 } 278 279 #if defined(TARGET_RISCV32) 280 static int read_timeh(CPURISCVState *env, int csrno, target_ulong *val) 281 { 282 *val = cpu_get_host_ticks() >> 32; 283 return 0; 284 } 285 #endif 286 287 #else /* CONFIG_USER_ONLY */ 288 289 static int read_time(CPURISCVState *env, int csrno, target_ulong *val) 290 { 291 uint64_t delta = riscv_cpu_virt_enabled(env) ? env->htimedelta : 0; 292 293 if (!env->rdtime_fn) { 294 return -1; 295 } 296 297 *val = env->rdtime_fn() + delta; 298 return 0; 299 } 300 301 #if defined(TARGET_RISCV32) 302 static int read_timeh(CPURISCVState *env, int csrno, target_ulong *val) 303 { 304 uint64_t delta = riscv_cpu_virt_enabled(env) ? env->htimedelta : 0; 305 306 if (!env->rdtime_fn) { 307 return -1; 308 } 309 310 *val = (env->rdtime_fn() + delta) >> 32; 311 return 0; 312 } 313 #endif 314 315 /* Machine constants */ 316 317 #define M_MODE_INTERRUPTS (MIP_MSIP | MIP_MTIP | MIP_MEIP) 318 #define S_MODE_INTERRUPTS (MIP_SSIP | MIP_STIP | MIP_SEIP) 319 #define VS_MODE_INTERRUPTS (MIP_VSSIP | MIP_VSTIP | MIP_VSEIP) 320 321 static const target_ulong delegable_ints = S_MODE_INTERRUPTS | 322 VS_MODE_INTERRUPTS; 323 static const target_ulong all_ints = M_MODE_INTERRUPTS | S_MODE_INTERRUPTS | 324 VS_MODE_INTERRUPTS; 325 static const target_ulong delegable_excps = 326 (1ULL << (RISCV_EXCP_INST_ADDR_MIS)) | 327 (1ULL << (RISCV_EXCP_INST_ACCESS_FAULT)) | 328 (1ULL << (RISCV_EXCP_ILLEGAL_INST)) | 329 (1ULL << (RISCV_EXCP_BREAKPOINT)) | 330 (1ULL << (RISCV_EXCP_LOAD_ADDR_MIS)) | 331 (1ULL << (RISCV_EXCP_LOAD_ACCESS_FAULT)) | 332 (1ULL << (RISCV_EXCP_STORE_AMO_ADDR_MIS)) | 333 (1ULL << (RISCV_EXCP_STORE_AMO_ACCESS_FAULT)) | 334 (1ULL << (RISCV_EXCP_U_ECALL)) | 335 (1ULL << (RISCV_EXCP_S_ECALL)) | 336 (1ULL << (RISCV_EXCP_VS_ECALL)) | 337 (1ULL << (RISCV_EXCP_M_ECALL)) | 338 (1ULL << (RISCV_EXCP_INST_PAGE_FAULT)) | 339 (1ULL << (RISCV_EXCP_LOAD_PAGE_FAULT)) | 340 (1ULL << (RISCV_EXCP_STORE_PAGE_FAULT)) | 341 (1ULL << (RISCV_EXCP_INST_GUEST_PAGE_FAULT)) | 342 (1ULL << (RISCV_EXCP_LOAD_GUEST_ACCESS_FAULT)) | 343 (1ULL << (RISCV_EXCP_STORE_GUEST_AMO_ACCESS_FAULT)); 344 static const target_ulong sstatus_v1_10_mask = SSTATUS_SIE | SSTATUS_SPIE | 345 SSTATUS_UIE | SSTATUS_UPIE | SSTATUS_SPP | SSTATUS_FS | SSTATUS_XS | 346 SSTATUS_SUM | SSTATUS_MXR | SSTATUS_SD; 347 static const target_ulong sip_writable_mask = SIP_SSIP | MIP_USIP | MIP_UEIP; 348 static const target_ulong hip_writable_mask = MIP_VSSIP | MIP_VSTIP | MIP_VSEIP; 349 static const target_ulong vsip_writable_mask = MIP_VSSIP; 350 351 #if defined(TARGET_RISCV32) 352 static const char valid_vm_1_10[16] = { 353 [VM_1_10_MBARE] = 1, 354 [VM_1_10_SV32] = 1 355 }; 356 #elif defined(TARGET_RISCV64) 357 static const char valid_vm_1_10[16] = { 358 [VM_1_10_MBARE] = 1, 359 [VM_1_10_SV39] = 1, 360 [VM_1_10_SV48] = 1, 361 [VM_1_10_SV57] = 1 362 }; 363 #endif /* CONFIG_USER_ONLY */ 364 365 /* Machine Information Registers */ 366 static int read_zero(CPURISCVState *env, int csrno, target_ulong *val) 367 { 368 return *val = 0; 369 } 370 371 static int read_mhartid(CPURISCVState *env, int csrno, target_ulong *val) 372 { 373 *val = env->mhartid; 374 return 0; 375 } 376 377 /* Machine Trap Setup */ 378 static int read_mstatus(CPURISCVState *env, int csrno, target_ulong *val) 379 { 380 *val = env->mstatus; 381 return 0; 382 } 383 384 static int validate_vm(CPURISCVState *env, target_ulong vm) 385 { 386 return valid_vm_1_10[vm & 0xf]; 387 } 388 389 static int write_mstatus(CPURISCVState *env, int csrno, target_ulong val) 390 { 391 target_ulong mstatus = env->mstatus; 392 target_ulong mask = 0; 393 int dirty; 394 395 /* flush tlb on mstatus fields that affect VM */ 396 if ((val ^ mstatus) & (MSTATUS_MXR | MSTATUS_MPP | MSTATUS_MPV | 397 MSTATUS_MPRV | MSTATUS_SUM)) { 398 tlb_flush(env_cpu(env)); 399 } 400 mask = MSTATUS_SIE | MSTATUS_SPIE | MSTATUS_MIE | MSTATUS_MPIE | 401 MSTATUS_SPP | MSTATUS_FS | MSTATUS_MPRV | MSTATUS_SUM | 402 MSTATUS_MPP | MSTATUS_MXR | MSTATUS_TVM | MSTATUS_TSR | 403 MSTATUS_TW; 404 #if defined(TARGET_RISCV64) 405 /* 406 * RV32: MPV and GVA are not in mstatus. The current plan is to 407 * add them to mstatush. For now, we just don't support it. 408 */ 409 mask |= MSTATUS_MPV | MSTATUS_GVA; 410 #endif 411 412 mstatus = (mstatus & ~mask) | (val & mask); 413 414 dirty = ((mstatus & MSTATUS_FS) == MSTATUS_FS) | 415 ((mstatus & MSTATUS_XS) == MSTATUS_XS); 416 mstatus = set_field(mstatus, MSTATUS_SD, dirty); 417 env->mstatus = mstatus; 418 419 return 0; 420 } 421 422 #ifdef TARGET_RISCV32 423 static int read_mstatush(CPURISCVState *env, int csrno, target_ulong *val) 424 { 425 *val = env->mstatush; 426 return 0; 427 } 428 429 static int write_mstatush(CPURISCVState *env, int csrno, target_ulong val) 430 { 431 if ((val ^ env->mstatush) & (MSTATUS_MPV)) { 432 tlb_flush(env_cpu(env)); 433 } 434 435 val &= MSTATUS_MPV | MSTATUS_GVA; 436 437 env->mstatush = val; 438 439 return 0; 440 } 441 #endif 442 443 static int read_misa(CPURISCVState *env, int csrno, target_ulong *val) 444 { 445 *val = env->misa; 446 return 0; 447 } 448 449 static int write_misa(CPURISCVState *env, int csrno, target_ulong val) 450 { 451 if (!riscv_feature(env, RISCV_FEATURE_MISA)) { 452 /* drop write to misa */ 453 return 0; 454 } 455 456 /* 'I' or 'E' must be present */ 457 if (!(val & (RVI | RVE))) { 458 /* It is not, drop write to misa */ 459 return 0; 460 } 461 462 /* 'E' excludes all other extensions */ 463 if (val & RVE) { 464 /* when we support 'E' we can do "val = RVE;" however 465 * for now we just drop writes if 'E' is present. 466 */ 467 return 0; 468 } 469 470 /* Mask extensions that are not supported by this hart */ 471 val &= env->misa_mask; 472 473 /* Mask extensions that are not supported by QEMU */ 474 val &= (RVI | RVE | RVM | RVA | RVF | RVD | RVC | RVS | RVU); 475 476 /* 'D' depends on 'F', so clear 'D' if 'F' is not present */ 477 if ((val & RVD) && !(val & RVF)) { 478 val &= ~RVD; 479 } 480 481 /* Suppress 'C' if next instruction is not aligned 482 * TODO: this should check next_pc 483 */ 484 if ((val & RVC) && (GETPC() & ~3) != 0) { 485 val &= ~RVC; 486 } 487 488 /* misa.MXL writes are not supported by QEMU */ 489 val = (env->misa & MISA_MXL) | (val & ~MISA_MXL); 490 491 /* flush translation cache */ 492 if (val != env->misa) { 493 tb_flush(env_cpu(env)); 494 } 495 496 env->misa = val; 497 498 return 0; 499 } 500 501 static int read_medeleg(CPURISCVState *env, int csrno, target_ulong *val) 502 { 503 *val = env->medeleg; 504 return 0; 505 } 506 507 static int write_medeleg(CPURISCVState *env, int csrno, target_ulong val) 508 { 509 env->medeleg = (env->medeleg & ~delegable_excps) | (val & delegable_excps); 510 return 0; 511 } 512 513 static int read_mideleg(CPURISCVState *env, int csrno, target_ulong *val) 514 { 515 *val = env->mideleg; 516 return 0; 517 } 518 519 static int write_mideleg(CPURISCVState *env, int csrno, target_ulong val) 520 { 521 env->mideleg = (env->mideleg & ~delegable_ints) | (val & delegable_ints); 522 if (riscv_has_ext(env, RVH)) { 523 env->mideleg |= VS_MODE_INTERRUPTS; 524 } 525 return 0; 526 } 527 528 static int read_mie(CPURISCVState *env, int csrno, target_ulong *val) 529 { 530 *val = env->mie; 531 return 0; 532 } 533 534 static int write_mie(CPURISCVState *env, int csrno, target_ulong val) 535 { 536 env->mie = (env->mie & ~all_ints) | (val & all_ints); 537 return 0; 538 } 539 540 static int read_mtvec(CPURISCVState *env, int csrno, target_ulong *val) 541 { 542 *val = env->mtvec; 543 return 0; 544 } 545 546 static int write_mtvec(CPURISCVState *env, int csrno, target_ulong val) 547 { 548 /* bits [1:0] encode mode; 0 = direct, 1 = vectored, 2 >= reserved */ 549 if ((val & 3) < 2) { 550 env->mtvec = val; 551 } else { 552 qemu_log_mask(LOG_UNIMP, "CSR_MTVEC: reserved mode not supported\n"); 553 } 554 return 0; 555 } 556 557 static int read_mcounteren(CPURISCVState *env, int csrno, target_ulong *val) 558 { 559 *val = env->mcounteren; 560 return 0; 561 } 562 563 static int write_mcounteren(CPURISCVState *env, int csrno, target_ulong val) 564 { 565 env->mcounteren = val; 566 return 0; 567 } 568 569 /* This regiser is replaced with CSR_MCOUNTINHIBIT in 1.11.0 */ 570 static int read_mscounteren(CPURISCVState *env, int csrno, target_ulong *val) 571 { 572 if (env->priv_ver < PRIV_VERSION_1_11_0) { 573 return -1; 574 } 575 *val = env->mcounteren; 576 return 0; 577 } 578 579 /* This regiser is replaced with CSR_MCOUNTINHIBIT in 1.11.0 */ 580 static int write_mscounteren(CPURISCVState *env, int csrno, target_ulong val) 581 { 582 if (env->priv_ver < PRIV_VERSION_1_11_0) { 583 return -1; 584 } 585 env->mcounteren = val; 586 return 0; 587 } 588 589 /* Machine Trap Handling */ 590 static int read_mscratch(CPURISCVState *env, int csrno, target_ulong *val) 591 { 592 *val = env->mscratch; 593 return 0; 594 } 595 596 static int write_mscratch(CPURISCVState *env, int csrno, target_ulong val) 597 { 598 env->mscratch = val; 599 return 0; 600 } 601 602 static int read_mepc(CPURISCVState *env, int csrno, target_ulong *val) 603 { 604 *val = env->mepc; 605 return 0; 606 } 607 608 static int write_mepc(CPURISCVState *env, int csrno, target_ulong val) 609 { 610 env->mepc = val; 611 return 0; 612 } 613 614 static int read_mcause(CPURISCVState *env, int csrno, target_ulong *val) 615 { 616 *val = env->mcause; 617 return 0; 618 } 619 620 static int write_mcause(CPURISCVState *env, int csrno, target_ulong val) 621 { 622 env->mcause = val; 623 return 0; 624 } 625 626 static int read_mbadaddr(CPURISCVState *env, int csrno, target_ulong *val) 627 { 628 *val = env->mbadaddr; 629 return 0; 630 } 631 632 static int write_mbadaddr(CPURISCVState *env, int csrno, target_ulong val) 633 { 634 env->mbadaddr = val; 635 return 0; 636 } 637 638 static int rmw_mip(CPURISCVState *env, int csrno, target_ulong *ret_value, 639 target_ulong new_value, target_ulong write_mask) 640 { 641 RISCVCPU *cpu = env_archcpu(env); 642 /* Allow software control of delegable interrupts not claimed by hardware */ 643 target_ulong mask = write_mask & delegable_ints & ~env->miclaim; 644 uint32_t old_mip; 645 646 if (mask) { 647 old_mip = riscv_cpu_update_mip(cpu, mask, (new_value & mask)); 648 } else { 649 old_mip = env->mip; 650 } 651 652 if (ret_value) { 653 *ret_value = old_mip; 654 } 655 656 return 0; 657 } 658 659 /* Supervisor Trap Setup */ 660 static int read_sstatus(CPURISCVState *env, int csrno, target_ulong *val) 661 { 662 target_ulong mask = (sstatus_v1_10_mask); 663 *val = env->mstatus & mask; 664 return 0; 665 } 666 667 static int write_sstatus(CPURISCVState *env, int csrno, target_ulong val) 668 { 669 target_ulong mask = (sstatus_v1_10_mask); 670 target_ulong newval = (env->mstatus & ~mask) | (val & mask); 671 return write_mstatus(env, CSR_MSTATUS, newval); 672 } 673 674 static int read_sie(CPURISCVState *env, int csrno, target_ulong *val) 675 { 676 if (riscv_cpu_virt_enabled(env)) { 677 /* Tell the guest the VS bits, shifted to the S bit locations */ 678 *val = (env->mie & env->mideleg & VS_MODE_INTERRUPTS) >> 1; 679 } else { 680 *val = env->mie & env->mideleg; 681 } 682 return 0; 683 } 684 685 static int write_sie(CPURISCVState *env, int csrno, target_ulong val) 686 { 687 target_ulong newval; 688 689 if (riscv_cpu_virt_enabled(env)) { 690 /* Shift the guests S bits to VS */ 691 newval = (env->mie & ~VS_MODE_INTERRUPTS) | 692 ((val << 1) & VS_MODE_INTERRUPTS); 693 } else { 694 newval = (env->mie & ~S_MODE_INTERRUPTS) | (val & S_MODE_INTERRUPTS); 695 } 696 697 return write_mie(env, CSR_MIE, newval); 698 } 699 700 static int read_stvec(CPURISCVState *env, int csrno, target_ulong *val) 701 { 702 *val = env->stvec; 703 return 0; 704 } 705 706 static int write_stvec(CPURISCVState *env, int csrno, target_ulong val) 707 { 708 /* bits [1:0] encode mode; 0 = direct, 1 = vectored, 2 >= reserved */ 709 if ((val & 3) < 2) { 710 env->stvec = val; 711 } else { 712 qemu_log_mask(LOG_UNIMP, "CSR_STVEC: reserved mode not supported\n"); 713 } 714 return 0; 715 } 716 717 static int read_scounteren(CPURISCVState *env, int csrno, target_ulong *val) 718 { 719 *val = env->scounteren; 720 return 0; 721 } 722 723 static int write_scounteren(CPURISCVState *env, int csrno, target_ulong val) 724 { 725 env->scounteren = val; 726 return 0; 727 } 728 729 /* Supervisor Trap Handling */ 730 static int read_sscratch(CPURISCVState *env, int csrno, target_ulong *val) 731 { 732 *val = env->sscratch; 733 return 0; 734 } 735 736 static int write_sscratch(CPURISCVState *env, int csrno, target_ulong val) 737 { 738 env->sscratch = val; 739 return 0; 740 } 741 742 static int read_sepc(CPURISCVState *env, int csrno, target_ulong *val) 743 { 744 *val = env->sepc; 745 return 0; 746 } 747 748 static int write_sepc(CPURISCVState *env, int csrno, target_ulong val) 749 { 750 env->sepc = val; 751 return 0; 752 } 753 754 static int read_scause(CPURISCVState *env, int csrno, target_ulong *val) 755 { 756 *val = env->scause; 757 return 0; 758 } 759 760 static int write_scause(CPURISCVState *env, int csrno, target_ulong val) 761 { 762 env->scause = val; 763 return 0; 764 } 765 766 static int read_sbadaddr(CPURISCVState *env, int csrno, target_ulong *val) 767 { 768 *val = env->sbadaddr; 769 return 0; 770 } 771 772 static int write_sbadaddr(CPURISCVState *env, int csrno, target_ulong val) 773 { 774 env->sbadaddr = val; 775 return 0; 776 } 777 778 static int rmw_sip(CPURISCVState *env, int csrno, target_ulong *ret_value, 779 target_ulong new_value, target_ulong write_mask) 780 { 781 int ret; 782 783 if (riscv_cpu_virt_enabled(env)) { 784 /* Shift the new values to line up with the VS bits */ 785 ret = rmw_mip(env, CSR_MSTATUS, ret_value, new_value << 1, 786 (write_mask & sip_writable_mask) << 1 & env->mideleg); 787 ret &= vsip_writable_mask; 788 ret >>= 1; 789 } else { 790 ret = rmw_mip(env, CSR_MSTATUS, ret_value, new_value, 791 write_mask & env->mideleg & sip_writable_mask); 792 } 793 794 *ret_value &= env->mideleg; 795 return ret; 796 } 797 798 /* Supervisor Protection and Translation */ 799 static int read_satp(CPURISCVState *env, int csrno, target_ulong *val) 800 { 801 if (!riscv_feature(env, RISCV_FEATURE_MMU)) { 802 *val = 0; 803 return 0; 804 } 805 806 if (env->priv == PRV_S && get_field(env->mstatus, MSTATUS_TVM)) { 807 return -1; 808 } else { 809 *val = env->satp; 810 } 811 812 return 0; 813 } 814 815 static int write_satp(CPURISCVState *env, int csrno, target_ulong val) 816 { 817 if (!riscv_feature(env, RISCV_FEATURE_MMU)) { 818 return 0; 819 } 820 if (validate_vm(env, get_field(val, SATP_MODE)) && 821 ((val ^ env->satp) & (SATP_MODE | SATP_ASID | SATP_PPN))) 822 { 823 if (env->priv == PRV_S && get_field(env->mstatus, MSTATUS_TVM)) { 824 return -1; 825 } else { 826 if((val ^ env->satp) & SATP_ASID) { 827 tlb_flush(env_cpu(env)); 828 } 829 env->satp = val; 830 } 831 } 832 return 0; 833 } 834 835 /* Hypervisor Extensions */ 836 static int read_hstatus(CPURISCVState *env, int csrno, target_ulong *val) 837 { 838 *val = env->hstatus; 839 #ifdef TARGET_RISCV64 840 /* We only support 64-bit VSXL */ 841 *val = set_field(*val, HSTATUS_VSXL, 2); 842 #endif 843 /* We only support little endian */ 844 *val = set_field(*val, HSTATUS_VSBE, 0); 845 return 0; 846 } 847 848 static int write_hstatus(CPURISCVState *env, int csrno, target_ulong val) 849 { 850 env->hstatus = val; 851 #ifdef TARGET_RISCV64 852 if (get_field(val, HSTATUS_VSXL) != 2) { 853 qemu_log_mask(LOG_UNIMP, "QEMU does not support mixed HSXLEN options."); 854 } 855 #endif 856 if (get_field(val, HSTATUS_VSBE) != 0) { 857 qemu_log_mask(LOG_UNIMP, "QEMU does not support big endian guests."); 858 } 859 return 0; 860 } 861 862 static int read_hedeleg(CPURISCVState *env, int csrno, target_ulong *val) 863 { 864 *val = env->hedeleg; 865 return 0; 866 } 867 868 static int write_hedeleg(CPURISCVState *env, int csrno, target_ulong val) 869 { 870 env->hedeleg = val; 871 return 0; 872 } 873 874 static int read_hideleg(CPURISCVState *env, int csrno, target_ulong *val) 875 { 876 *val = env->hideleg; 877 return 0; 878 } 879 880 static int write_hideleg(CPURISCVState *env, int csrno, target_ulong val) 881 { 882 env->hideleg = val; 883 return 0; 884 } 885 886 static int rmw_hip(CPURISCVState *env, int csrno, target_ulong *ret_value, 887 target_ulong new_value, target_ulong write_mask) 888 { 889 int ret = rmw_mip(env, 0, ret_value, new_value, 890 write_mask & hip_writable_mask); 891 892 return ret; 893 } 894 895 static int read_hie(CPURISCVState *env, int csrno, target_ulong *val) 896 { 897 *val = env->mie & VS_MODE_INTERRUPTS; 898 return 0; 899 } 900 901 static int write_hie(CPURISCVState *env, int csrno, target_ulong val) 902 { 903 target_ulong newval = (env->mie & ~VS_MODE_INTERRUPTS) | (val & VS_MODE_INTERRUPTS); 904 return write_mie(env, CSR_MIE, newval); 905 } 906 907 static int read_hcounteren(CPURISCVState *env, int csrno, target_ulong *val) 908 { 909 *val = env->hcounteren; 910 return 0; 911 } 912 913 static int write_hcounteren(CPURISCVState *env, int csrno, target_ulong val) 914 { 915 env->hcounteren = val; 916 return 0; 917 } 918 919 static int read_htval(CPURISCVState *env, int csrno, target_ulong *val) 920 { 921 *val = env->htval; 922 return 0; 923 } 924 925 static int write_htval(CPURISCVState *env, int csrno, target_ulong val) 926 { 927 env->htval = val; 928 return 0; 929 } 930 931 static int read_htinst(CPURISCVState *env, int csrno, target_ulong *val) 932 { 933 *val = env->htinst; 934 return 0; 935 } 936 937 static int write_htinst(CPURISCVState *env, int csrno, target_ulong val) 938 { 939 return 0; 940 } 941 942 static int read_hgatp(CPURISCVState *env, int csrno, target_ulong *val) 943 { 944 *val = env->hgatp; 945 return 0; 946 } 947 948 static int write_hgatp(CPURISCVState *env, int csrno, target_ulong val) 949 { 950 env->hgatp = val; 951 return 0; 952 } 953 954 static int read_htimedelta(CPURISCVState *env, int csrno, target_ulong *val) 955 { 956 if (!env->rdtime_fn) { 957 return -1; 958 } 959 960 #if defined(TARGET_RISCV32) 961 *val = env->htimedelta & 0xffffffff; 962 #else 963 *val = env->htimedelta; 964 #endif 965 return 0; 966 } 967 968 static int write_htimedelta(CPURISCVState *env, int csrno, target_ulong val) 969 { 970 if (!env->rdtime_fn) { 971 return -1; 972 } 973 974 #if defined(TARGET_RISCV32) 975 env->htimedelta = deposit64(env->htimedelta, 0, 32, (uint64_t)val); 976 #else 977 env->htimedelta = val; 978 #endif 979 return 0; 980 } 981 982 #if defined(TARGET_RISCV32) 983 static int read_htimedeltah(CPURISCVState *env, int csrno, target_ulong *val) 984 { 985 if (!env->rdtime_fn) { 986 return -1; 987 } 988 989 *val = env->htimedelta >> 32; 990 return 0; 991 } 992 993 static int write_htimedeltah(CPURISCVState *env, int csrno, target_ulong val) 994 { 995 if (!env->rdtime_fn) { 996 return -1; 997 } 998 999 env->htimedelta = deposit64(env->htimedelta, 32, 32, (uint64_t)val); 1000 return 0; 1001 } 1002 #endif 1003 1004 /* Virtual CSR Registers */ 1005 static int read_vsstatus(CPURISCVState *env, int csrno, target_ulong *val) 1006 { 1007 *val = env->vsstatus; 1008 return 0; 1009 } 1010 1011 static int write_vsstatus(CPURISCVState *env, int csrno, target_ulong val) 1012 { 1013 env->vsstatus = val; 1014 return 0; 1015 } 1016 1017 static int rmw_vsip(CPURISCVState *env, int csrno, target_ulong *ret_value, 1018 target_ulong new_value, target_ulong write_mask) 1019 { 1020 int ret = rmw_mip(env, 0, ret_value, new_value, 1021 write_mask & env->mideleg & vsip_writable_mask); 1022 return ret; 1023 } 1024 1025 static int read_vsie(CPURISCVState *env, int csrno, target_ulong *val) 1026 { 1027 *val = env->mie & env->mideleg & VS_MODE_INTERRUPTS; 1028 return 0; 1029 } 1030 1031 static int write_vsie(CPURISCVState *env, int csrno, target_ulong val) 1032 { 1033 target_ulong newval = (env->mie & ~env->mideleg) | (val & env->mideleg & MIP_VSSIP); 1034 return write_mie(env, CSR_MIE, newval); 1035 } 1036 1037 static int read_vstvec(CPURISCVState *env, int csrno, target_ulong *val) 1038 { 1039 *val = env->vstvec; 1040 return 0; 1041 } 1042 1043 static int write_vstvec(CPURISCVState *env, int csrno, target_ulong val) 1044 { 1045 env->vstvec = val; 1046 return 0; 1047 } 1048 1049 static int read_vsscratch(CPURISCVState *env, int csrno, target_ulong *val) 1050 { 1051 *val = env->vsscratch; 1052 return 0; 1053 } 1054 1055 static int write_vsscratch(CPURISCVState *env, int csrno, target_ulong val) 1056 { 1057 env->vsscratch = val; 1058 return 0; 1059 } 1060 1061 static int read_vsepc(CPURISCVState *env, int csrno, target_ulong *val) 1062 { 1063 *val = env->vsepc; 1064 return 0; 1065 } 1066 1067 static int write_vsepc(CPURISCVState *env, int csrno, target_ulong val) 1068 { 1069 env->vsepc = val; 1070 return 0; 1071 } 1072 1073 static int read_vscause(CPURISCVState *env, int csrno, target_ulong *val) 1074 { 1075 *val = env->vscause; 1076 return 0; 1077 } 1078 1079 static int write_vscause(CPURISCVState *env, int csrno, target_ulong val) 1080 { 1081 env->vscause = val; 1082 return 0; 1083 } 1084 1085 static int read_vstval(CPURISCVState *env, int csrno, target_ulong *val) 1086 { 1087 *val = env->vstval; 1088 return 0; 1089 } 1090 1091 static int write_vstval(CPURISCVState *env, int csrno, target_ulong val) 1092 { 1093 env->vstval = val; 1094 return 0; 1095 } 1096 1097 static int read_vsatp(CPURISCVState *env, int csrno, target_ulong *val) 1098 { 1099 *val = env->vsatp; 1100 return 0; 1101 } 1102 1103 static int write_vsatp(CPURISCVState *env, int csrno, target_ulong val) 1104 { 1105 env->vsatp = val; 1106 return 0; 1107 } 1108 1109 static int read_mtval2(CPURISCVState *env, int csrno, target_ulong *val) 1110 { 1111 *val = env->mtval2; 1112 return 0; 1113 } 1114 1115 static int write_mtval2(CPURISCVState *env, int csrno, target_ulong val) 1116 { 1117 env->mtval2 = val; 1118 return 0; 1119 } 1120 1121 static int read_mtinst(CPURISCVState *env, int csrno, target_ulong *val) 1122 { 1123 *val = env->mtinst; 1124 return 0; 1125 } 1126 1127 static int write_mtinst(CPURISCVState *env, int csrno, target_ulong val) 1128 { 1129 env->mtinst = val; 1130 return 0; 1131 } 1132 1133 /* Physical Memory Protection */ 1134 static int read_pmpcfg(CPURISCVState *env, int csrno, target_ulong *val) 1135 { 1136 *val = pmpcfg_csr_read(env, csrno - CSR_PMPCFG0); 1137 return 0; 1138 } 1139 1140 static int write_pmpcfg(CPURISCVState *env, int csrno, target_ulong val) 1141 { 1142 pmpcfg_csr_write(env, csrno - CSR_PMPCFG0, val); 1143 return 0; 1144 } 1145 1146 static int read_pmpaddr(CPURISCVState *env, int csrno, target_ulong *val) 1147 { 1148 *val = pmpaddr_csr_read(env, csrno - CSR_PMPADDR0); 1149 return 0; 1150 } 1151 1152 static int write_pmpaddr(CPURISCVState *env, int csrno, target_ulong val) 1153 { 1154 pmpaddr_csr_write(env, csrno - CSR_PMPADDR0, val); 1155 return 0; 1156 } 1157 1158 #endif 1159 1160 /* 1161 * riscv_csrrw - read and/or update control and status register 1162 * 1163 * csrr <-> riscv_csrrw(env, csrno, ret_value, 0, 0); 1164 * csrrw <-> riscv_csrrw(env, csrno, ret_value, value, -1); 1165 * csrrs <-> riscv_csrrw(env, csrno, ret_value, -1, value); 1166 * csrrc <-> riscv_csrrw(env, csrno, ret_value, 0, value); 1167 */ 1168 1169 int riscv_csrrw(CPURISCVState *env, int csrno, target_ulong *ret_value, 1170 target_ulong new_value, target_ulong write_mask) 1171 { 1172 int ret; 1173 target_ulong old_value; 1174 RISCVCPU *cpu = env_archcpu(env); 1175 1176 /* check privileges and return -1 if check fails */ 1177 #if !defined(CONFIG_USER_ONLY) 1178 int effective_priv = env->priv; 1179 int read_only = get_field(csrno, 0xC00) == 3; 1180 1181 if (riscv_has_ext(env, RVH) && 1182 env->priv == PRV_S && 1183 !riscv_cpu_virt_enabled(env)) { 1184 /* 1185 * We are in S mode without virtualisation, therefore we are in HS Mode. 1186 * Add 1 to the effective privledge level to allow us to access the 1187 * Hypervisor CSRs. 1188 */ 1189 effective_priv++; 1190 } 1191 1192 if ((write_mask && read_only) || 1193 (!env->debugger && (effective_priv < get_field(csrno, 0x300)))) { 1194 return -1; 1195 } 1196 #endif 1197 1198 /* ensure the CSR extension is enabled. */ 1199 if (!cpu->cfg.ext_icsr) { 1200 return -1; 1201 } 1202 1203 /* check predicate */ 1204 if (!csr_ops[csrno].predicate || csr_ops[csrno].predicate(env, csrno) < 0) { 1205 return -1; 1206 } 1207 1208 /* execute combined read/write operation if it exists */ 1209 if (csr_ops[csrno].op) { 1210 return csr_ops[csrno].op(env, csrno, ret_value, new_value, write_mask); 1211 } 1212 1213 /* if no accessor exists then return failure */ 1214 if (!csr_ops[csrno].read) { 1215 return -1; 1216 } 1217 1218 /* read old value */ 1219 ret = csr_ops[csrno].read(env, csrno, &old_value); 1220 if (ret < 0) { 1221 return ret; 1222 } 1223 1224 /* write value if writable and write mask set, otherwise drop writes */ 1225 if (write_mask) { 1226 new_value = (old_value & ~write_mask) | (new_value & write_mask); 1227 if (csr_ops[csrno].write) { 1228 ret = csr_ops[csrno].write(env, csrno, new_value); 1229 if (ret < 0) { 1230 return ret; 1231 } 1232 } 1233 } 1234 1235 /* return old value */ 1236 if (ret_value) { 1237 *ret_value = old_value; 1238 } 1239 1240 return 0; 1241 } 1242 1243 /* 1244 * Debugger support. If not in user mode, set env->debugger before the 1245 * riscv_csrrw call and clear it after the call. 1246 */ 1247 int riscv_csrrw_debug(CPURISCVState *env, int csrno, target_ulong *ret_value, 1248 target_ulong new_value, target_ulong write_mask) 1249 { 1250 int ret; 1251 #if !defined(CONFIG_USER_ONLY) 1252 env->debugger = true; 1253 #endif 1254 ret = riscv_csrrw(env, csrno, ret_value, new_value, write_mask); 1255 #if !defined(CONFIG_USER_ONLY) 1256 env->debugger = false; 1257 #endif 1258 return ret; 1259 } 1260 1261 /* Control and Status Register function table */ 1262 static riscv_csr_operations csr_ops[CSR_TABLE_SIZE] = { 1263 /* User Floating-Point CSRs */ 1264 [CSR_FFLAGS] = { fs, read_fflags, write_fflags }, 1265 [CSR_FRM] = { fs, read_frm, write_frm }, 1266 [CSR_FCSR] = { fs, read_fcsr, write_fcsr }, 1267 /* Vector CSRs */ 1268 [CSR_VSTART] = { vs, read_vstart, write_vstart }, 1269 [CSR_VXSAT] = { vs, read_vxsat, write_vxsat }, 1270 [CSR_VXRM] = { vs, read_vxrm, write_vxrm }, 1271 [CSR_VL] = { vs, read_vl }, 1272 [CSR_VTYPE] = { vs, read_vtype }, 1273 /* User Timers and Counters */ 1274 [CSR_CYCLE] = { ctr, read_instret }, 1275 [CSR_INSTRET] = { ctr, read_instret }, 1276 #if defined(TARGET_RISCV32) 1277 [CSR_CYCLEH] = { ctr, read_instreth }, 1278 [CSR_INSTRETH] = { ctr, read_instreth }, 1279 #endif 1280 1281 /* In privileged mode, the monitor will have to emulate TIME CSRs only if 1282 * rdtime callback is not provided by machine/platform emulation */ 1283 [CSR_TIME] = { ctr, read_time }, 1284 #if defined(TARGET_RISCV32) 1285 [CSR_TIMEH] = { ctr, read_timeh }, 1286 #endif 1287 1288 #if !defined(CONFIG_USER_ONLY) 1289 /* Machine Timers and Counters */ 1290 [CSR_MCYCLE] = { any, read_instret }, 1291 [CSR_MINSTRET] = { any, read_instret }, 1292 #if defined(TARGET_RISCV32) 1293 [CSR_MCYCLEH] = { any, read_instreth }, 1294 [CSR_MINSTRETH] = { any, read_instreth }, 1295 #endif 1296 1297 /* Machine Information Registers */ 1298 [CSR_MVENDORID] = { any, read_zero }, 1299 [CSR_MARCHID] = { any, read_zero }, 1300 [CSR_MIMPID] = { any, read_zero }, 1301 [CSR_MHARTID] = { any, read_mhartid }, 1302 1303 /* Machine Trap Setup */ 1304 [CSR_MSTATUS] = { any, read_mstatus, write_mstatus }, 1305 [CSR_MISA] = { any, read_misa, write_misa }, 1306 [CSR_MIDELEG] = { any, read_mideleg, write_mideleg }, 1307 [CSR_MEDELEG] = { any, read_medeleg, write_medeleg }, 1308 [CSR_MIE] = { any, read_mie, write_mie }, 1309 [CSR_MTVEC] = { any, read_mtvec, write_mtvec }, 1310 [CSR_MCOUNTEREN] = { any, read_mcounteren, write_mcounteren }, 1311 1312 #if defined(TARGET_RISCV32) 1313 [CSR_MSTATUSH] = { any, read_mstatush, write_mstatush }, 1314 #endif 1315 1316 [CSR_MSCOUNTEREN] = { any, read_mscounteren, write_mscounteren }, 1317 1318 /* Machine Trap Handling */ 1319 [CSR_MSCRATCH] = { any, read_mscratch, write_mscratch }, 1320 [CSR_MEPC] = { any, read_mepc, write_mepc }, 1321 [CSR_MCAUSE] = { any, read_mcause, write_mcause }, 1322 [CSR_MBADADDR] = { any, read_mbadaddr, write_mbadaddr }, 1323 [CSR_MIP] = { any, NULL, NULL, rmw_mip }, 1324 1325 /* Supervisor Trap Setup */ 1326 [CSR_SSTATUS] = { smode, read_sstatus, write_sstatus }, 1327 [CSR_SIE] = { smode, read_sie, write_sie }, 1328 [CSR_STVEC] = { smode, read_stvec, write_stvec }, 1329 [CSR_SCOUNTEREN] = { smode, read_scounteren, write_scounteren }, 1330 1331 /* Supervisor Trap Handling */ 1332 [CSR_SSCRATCH] = { smode, read_sscratch, write_sscratch }, 1333 [CSR_SEPC] = { smode, read_sepc, write_sepc }, 1334 [CSR_SCAUSE] = { smode, read_scause, write_scause }, 1335 [CSR_SBADADDR] = { smode, read_sbadaddr, write_sbadaddr }, 1336 [CSR_SIP] = { smode, NULL, NULL, rmw_sip }, 1337 1338 /* Supervisor Protection and Translation */ 1339 [CSR_SATP] = { smode, read_satp, write_satp }, 1340 1341 [CSR_HSTATUS] = { hmode, read_hstatus, write_hstatus }, 1342 [CSR_HEDELEG] = { hmode, read_hedeleg, write_hedeleg }, 1343 [CSR_HIDELEG] = { hmode, read_hideleg, write_hideleg }, 1344 [CSR_HIP] = { hmode, NULL, NULL, rmw_hip }, 1345 [CSR_HIE] = { hmode, read_hie, write_hie }, 1346 [CSR_HCOUNTEREN] = { hmode, read_hcounteren, write_hcounteren }, 1347 [CSR_HTVAL] = { hmode, read_htval, write_htval }, 1348 [CSR_HTINST] = { hmode, read_htinst, write_htinst }, 1349 [CSR_HGATP] = { hmode, read_hgatp, write_hgatp }, 1350 [CSR_HTIMEDELTA] = { hmode, read_htimedelta, write_htimedelta }, 1351 #if defined(TARGET_RISCV32) 1352 [CSR_HTIMEDELTAH] = { hmode, read_htimedeltah, write_htimedeltah}, 1353 #endif 1354 1355 [CSR_VSSTATUS] = { hmode, read_vsstatus, write_vsstatus }, 1356 [CSR_VSIP] = { hmode, NULL, NULL, rmw_vsip }, 1357 [CSR_VSIE] = { hmode, read_vsie, write_vsie }, 1358 [CSR_VSTVEC] = { hmode, read_vstvec, write_vstvec }, 1359 [CSR_VSSCRATCH] = { hmode, read_vsscratch, write_vsscratch }, 1360 [CSR_VSEPC] = { hmode, read_vsepc, write_vsepc }, 1361 [CSR_VSCAUSE] = { hmode, read_vscause, write_vscause }, 1362 [CSR_VSTVAL] = { hmode, read_vstval, write_vstval }, 1363 [CSR_VSATP] = { hmode, read_vsatp, write_vsatp }, 1364 1365 [CSR_MTVAL2] = { hmode, read_mtval2, write_mtval2 }, 1366 [CSR_MTINST] = { hmode, read_mtinst, write_mtinst }, 1367 1368 /* Physical Memory Protection */ 1369 [CSR_PMPCFG0 ... CSR_PMPCFG3] = { pmp, read_pmpcfg, write_pmpcfg }, 1370 [CSR_PMPADDR0 ... CSR_PMPADDR15] = { pmp, read_pmpaddr, write_pmpaddr }, 1371 1372 /* Performance Counters */ 1373 [CSR_HPMCOUNTER3 ... CSR_HPMCOUNTER31] = { ctr, read_zero }, 1374 [CSR_MHPMCOUNTER3 ... CSR_MHPMCOUNTER31] = { any, read_zero }, 1375 [CSR_MHPMEVENT3 ... CSR_MHPMEVENT31] = { any, read_zero }, 1376 #if defined(TARGET_RISCV32) 1377 [CSR_HPMCOUNTER3H ... CSR_HPMCOUNTER31H] = { ctr, read_zero }, 1378 [CSR_MHPMCOUNTER3H ... CSR_MHPMCOUNTER31H] = { any, read_zero }, 1379 #endif 1380 #endif /* !CONFIG_USER_ONLY */ 1381 }; 1382