xref: /openbmc/qemu/target/riscv/cpu_helper.c (revision 2b74dd91)
1 /*
2  * RISC-V CPU helpers for qemu.
3  *
4  * Copyright (c) 2016-2017 Sagar Karandikar, sagark@eecs.berkeley.edu
5  * Copyright (c) 2017-2018 SiFive, Inc.
6  *
7  * This program is free software; you can redistribute it and/or modify it
8  * under the terms and conditions of the GNU General Public License,
9  * version 2 or later, as published by the Free Software Foundation.
10  *
11  * This program is distributed in the hope it will be useful, but WITHOUT
12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
14  * more details.
15  *
16  * You should have received a copy of the GNU General Public License along with
17  * this program.  If not, see <http://www.gnu.org/licenses/>.
18  */
19 
20 #include "qemu/osdep.h"
21 #include "qemu/log.h"
22 #include "qemu/main-loop.h"
23 #include "cpu.h"
24 #include "internals.h"
25 #include "pmu.h"
26 #include "exec/exec-all.h"
27 #include "exec/page-protection.h"
28 #include "instmap.h"
29 #include "tcg/tcg-op.h"
30 #include "trace.h"
31 #include "semihosting/common-semi.h"
32 #include "sysemu/cpu-timers.h"
33 #include "cpu_bits.h"
34 #include "debug.h"
35 #include "tcg/oversized-guest.h"
36 
37 int riscv_env_mmu_index(CPURISCVState *env, bool ifetch)
38 {
39 #ifdef CONFIG_USER_ONLY
40     return 0;
41 #else
42     bool virt = env->virt_enabled;
43     int mode = env->priv;
44 
45     /* All priv -> mmu_idx mapping are here */
46     if (!ifetch) {
47         uint64_t status = env->mstatus;
48 
49         if (mode == PRV_M && get_field(status, MSTATUS_MPRV)) {
50             mode = get_field(env->mstatus, MSTATUS_MPP);
51             virt = get_field(env->mstatus, MSTATUS_MPV) &&
52                    (mode != PRV_M);
53             if (virt) {
54                 status = env->vsstatus;
55             }
56         }
57         if (mode == PRV_S && get_field(status, MSTATUS_SUM)) {
58             mode = MMUIdx_S_SUM;
59         }
60     }
61 
62     return mode | (virt ? MMU_2STAGE_BIT : 0);
63 #endif
64 }
65 
66 void cpu_get_tb_cpu_state(CPURISCVState *env, vaddr *pc,
67                           uint64_t *cs_base, uint32_t *pflags)
68 {
69     RISCVCPU *cpu = env_archcpu(env);
70     RISCVExtStatus fs, vs;
71     uint32_t flags = 0;
72 
73     *pc = env->xl == MXL_RV32 ? env->pc & UINT32_MAX : env->pc;
74     *cs_base = 0;
75 
76     if (cpu->cfg.ext_zve32x) {
77         /*
78          * If env->vl equals to VLMAX, we can use generic vector operation
79          * expanders (GVEC) to accerlate the vector operations.
80          * However, as LMUL could be a fractional number. The maximum
81          * vector size can be operated might be less than 8 bytes,
82          * which is not supported by GVEC. So we set vl_eq_vlmax flag to true
83          * only when maxsz >= 8 bytes.
84          */
85 
86         /* lmul encoded as in DisasContext::lmul */
87         int8_t lmul = sextract32(FIELD_EX64(env->vtype, VTYPE, VLMUL), 0, 3);
88         uint32_t vsew = FIELD_EX64(env->vtype, VTYPE, VSEW);
89         uint32_t vlmax = vext_get_vlmax(cpu->cfg.vlenb, vsew, lmul);
90         uint32_t maxsz = vlmax << vsew;
91         bool vl_eq_vlmax = (env->vstart == 0) && (vlmax == env->vl) &&
92                            (maxsz >= 8);
93         flags = FIELD_DP32(flags, TB_FLAGS, VILL, env->vill);
94         flags = FIELD_DP32(flags, TB_FLAGS, SEW, vsew);
95         flags = FIELD_DP32(flags, TB_FLAGS, LMUL,
96                            FIELD_EX64(env->vtype, VTYPE, VLMUL));
97         flags = FIELD_DP32(flags, TB_FLAGS, VL_EQ_VLMAX, vl_eq_vlmax);
98         flags = FIELD_DP32(flags, TB_FLAGS, VTA,
99                            FIELD_EX64(env->vtype, VTYPE, VTA));
100         flags = FIELD_DP32(flags, TB_FLAGS, VMA,
101                            FIELD_EX64(env->vtype, VTYPE, VMA));
102         flags = FIELD_DP32(flags, TB_FLAGS, VSTART_EQ_ZERO, env->vstart == 0);
103     } else {
104         flags = FIELD_DP32(flags, TB_FLAGS, VILL, 1);
105     }
106 
107 #ifdef CONFIG_USER_ONLY
108     fs = EXT_STATUS_DIRTY;
109     vs = EXT_STATUS_DIRTY;
110 #else
111     flags = FIELD_DP32(flags, TB_FLAGS, PRIV, env->priv);
112 
113     flags |= riscv_env_mmu_index(env, 0);
114     fs = get_field(env->mstatus, MSTATUS_FS);
115     vs = get_field(env->mstatus, MSTATUS_VS);
116 
117     if (env->virt_enabled) {
118         flags = FIELD_DP32(flags, TB_FLAGS, VIRT_ENABLED, 1);
119         /*
120          * Merge DISABLED and !DIRTY states using MIN.
121          * We will set both fields when dirtying.
122          */
123         fs = MIN(fs, get_field(env->mstatus_hs, MSTATUS_FS));
124         vs = MIN(vs, get_field(env->mstatus_hs, MSTATUS_VS));
125     }
126 
127     /* With Zfinx, floating point is enabled/disabled by Smstateen. */
128     if (!riscv_has_ext(env, RVF)) {
129         fs = (smstateen_acc_ok(env, 0, SMSTATEEN0_FCSR) == RISCV_EXCP_NONE)
130              ? EXT_STATUS_DIRTY : EXT_STATUS_DISABLED;
131     }
132 
133     if (cpu->cfg.debug && !icount_enabled()) {
134         flags = FIELD_DP32(flags, TB_FLAGS, ITRIGGER, env->itrigger_enabled);
135     }
136 #endif
137 
138     flags = FIELD_DP32(flags, TB_FLAGS, FS, fs);
139     flags = FIELD_DP32(flags, TB_FLAGS, VS, vs);
140     flags = FIELD_DP32(flags, TB_FLAGS, XL, env->xl);
141     flags = FIELD_DP32(flags, TB_FLAGS, AXL, cpu_address_xl(env));
142     if (env->cur_pmmask != 0) {
143         flags = FIELD_DP32(flags, TB_FLAGS, PM_MASK_ENABLED, 1);
144     }
145     if (env->cur_pmbase != 0) {
146         flags = FIELD_DP32(flags, TB_FLAGS, PM_BASE_ENABLED, 1);
147     }
148 
149     *pflags = flags;
150 }
151 
152 void riscv_cpu_update_mask(CPURISCVState *env)
153 {
154     target_ulong mask = 0, base = 0;
155     RISCVMXL xl = env->xl;
156     /*
157      * TODO: Current RVJ spec does not specify
158      * how the extension interacts with XLEN.
159      */
160 #ifndef CONFIG_USER_ONLY
161     int mode = cpu_address_mode(env);
162     xl = cpu_get_xl(env, mode);
163     if (riscv_has_ext(env, RVJ)) {
164         switch (mode) {
165         case PRV_M:
166             if (env->mmte & M_PM_ENABLE) {
167                 mask = env->mpmmask;
168                 base = env->mpmbase;
169             }
170             break;
171         case PRV_S:
172             if (env->mmte & S_PM_ENABLE) {
173                 mask = env->spmmask;
174                 base = env->spmbase;
175             }
176             break;
177         case PRV_U:
178             if (env->mmte & U_PM_ENABLE) {
179                 mask = env->upmmask;
180                 base = env->upmbase;
181             }
182             break;
183         default:
184             g_assert_not_reached();
185         }
186     }
187 #endif
188     if (xl == MXL_RV32) {
189         env->cur_pmmask = mask & UINT32_MAX;
190         env->cur_pmbase = base & UINT32_MAX;
191     } else {
192         env->cur_pmmask = mask;
193         env->cur_pmbase = base;
194     }
195 }
196 
197 #ifndef CONFIG_USER_ONLY
198 
199 /*
200  * The HS-mode is allowed to configure priority only for the
201  * following VS-mode local interrupts:
202  *
203  * 0  (Reserved interrupt, reads as zero)
204  * 1  Supervisor software interrupt
205  * 4  (Reserved interrupt, reads as zero)
206  * 5  Supervisor timer interrupt
207  * 8  (Reserved interrupt, reads as zero)
208  * 13 (Reserved interrupt)
209  * 14 "
210  * 15 "
211  * 16 "
212  * 17 "
213  * 18 "
214  * 19 "
215  * 20 "
216  * 21 "
217  * 22 "
218  * 23 "
219  */
220 
221 static const int hviprio_index2irq[] = {
222     0, 1, 4, 5, 8, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23 };
223 static const int hviprio_index2rdzero[] = {
224     1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
225 
226 int riscv_cpu_hviprio_index2irq(int index, int *out_irq, int *out_rdzero)
227 {
228     if (index < 0 || ARRAY_SIZE(hviprio_index2irq) <= index) {
229         return -EINVAL;
230     }
231 
232     if (out_irq) {
233         *out_irq = hviprio_index2irq[index];
234     }
235 
236     if (out_rdzero) {
237         *out_rdzero = hviprio_index2rdzero[index];
238     }
239 
240     return 0;
241 }
242 
243 /*
244  * Default priorities of local interrupts are defined in the
245  * RISC-V Advanced Interrupt Architecture specification.
246  *
247  * ----------------------------------------------------------------
248  *  Default  |
249  *  Priority | Major Interrupt Numbers
250  * ----------------------------------------------------------------
251  *  Highest  | 47, 23, 46, 45, 22, 44,
252  *           | 43, 21, 42, 41, 20, 40
253  *           |
254  *           | 11 (0b),  3 (03),  7 (07)
255  *           |  9 (09),  1 (01),  5 (05)
256  *           | 12 (0c)
257  *           | 10 (0a),  2 (02),  6 (06)
258  *           |
259  *           | 39, 19, 38, 37, 18, 36,
260  *  Lowest   | 35, 17, 34, 33, 16, 32
261  * ----------------------------------------------------------------
262  */
263 static const uint8_t default_iprio[64] = {
264     /* Custom interrupts 48 to 63 */
265     [63] = IPRIO_MMAXIPRIO,
266     [62] = IPRIO_MMAXIPRIO,
267     [61] = IPRIO_MMAXIPRIO,
268     [60] = IPRIO_MMAXIPRIO,
269     [59] = IPRIO_MMAXIPRIO,
270     [58] = IPRIO_MMAXIPRIO,
271     [57] = IPRIO_MMAXIPRIO,
272     [56] = IPRIO_MMAXIPRIO,
273     [55] = IPRIO_MMAXIPRIO,
274     [54] = IPRIO_MMAXIPRIO,
275     [53] = IPRIO_MMAXIPRIO,
276     [52] = IPRIO_MMAXIPRIO,
277     [51] = IPRIO_MMAXIPRIO,
278     [50] = IPRIO_MMAXIPRIO,
279     [49] = IPRIO_MMAXIPRIO,
280     [48] = IPRIO_MMAXIPRIO,
281 
282     /* Custom interrupts 24 to 31 */
283     [31] = IPRIO_MMAXIPRIO,
284     [30] = IPRIO_MMAXIPRIO,
285     [29] = IPRIO_MMAXIPRIO,
286     [28] = IPRIO_MMAXIPRIO,
287     [27] = IPRIO_MMAXIPRIO,
288     [26] = IPRIO_MMAXIPRIO,
289     [25] = IPRIO_MMAXIPRIO,
290     [24] = IPRIO_MMAXIPRIO,
291 
292     [47] = IPRIO_DEFAULT_UPPER,
293     [23] = IPRIO_DEFAULT_UPPER + 1,
294     [46] = IPRIO_DEFAULT_UPPER + 2,
295     [45] = IPRIO_DEFAULT_UPPER + 3,
296     [22] = IPRIO_DEFAULT_UPPER + 4,
297     [44] = IPRIO_DEFAULT_UPPER + 5,
298 
299     [43] = IPRIO_DEFAULT_UPPER + 6,
300     [21] = IPRIO_DEFAULT_UPPER + 7,
301     [42] = IPRIO_DEFAULT_UPPER + 8,
302     [41] = IPRIO_DEFAULT_UPPER + 9,
303     [20] = IPRIO_DEFAULT_UPPER + 10,
304     [40] = IPRIO_DEFAULT_UPPER + 11,
305 
306     [11] = IPRIO_DEFAULT_M,
307     [3]  = IPRIO_DEFAULT_M + 1,
308     [7]  = IPRIO_DEFAULT_M + 2,
309 
310     [9]  = IPRIO_DEFAULT_S,
311     [1]  = IPRIO_DEFAULT_S + 1,
312     [5]  = IPRIO_DEFAULT_S + 2,
313 
314     [12] = IPRIO_DEFAULT_SGEXT,
315 
316     [10] = IPRIO_DEFAULT_VS,
317     [2]  = IPRIO_DEFAULT_VS + 1,
318     [6]  = IPRIO_DEFAULT_VS + 2,
319 
320     [39] = IPRIO_DEFAULT_LOWER,
321     [19] = IPRIO_DEFAULT_LOWER + 1,
322     [38] = IPRIO_DEFAULT_LOWER + 2,
323     [37] = IPRIO_DEFAULT_LOWER + 3,
324     [18] = IPRIO_DEFAULT_LOWER + 4,
325     [36] = IPRIO_DEFAULT_LOWER + 5,
326 
327     [35] = IPRIO_DEFAULT_LOWER + 6,
328     [17] = IPRIO_DEFAULT_LOWER + 7,
329     [34] = IPRIO_DEFAULT_LOWER + 8,
330     [33] = IPRIO_DEFAULT_LOWER + 9,
331     [16] = IPRIO_DEFAULT_LOWER + 10,
332     [32] = IPRIO_DEFAULT_LOWER + 11,
333 };
334 
335 uint8_t riscv_cpu_default_priority(int irq)
336 {
337     if (irq < 0 || irq > 63) {
338         return IPRIO_MMAXIPRIO;
339     }
340 
341     return default_iprio[irq] ? default_iprio[irq] : IPRIO_MMAXIPRIO;
342 };
343 
344 static int riscv_cpu_pending_to_irq(CPURISCVState *env,
345                                     int extirq, unsigned int extirq_def_prio,
346                                     uint64_t pending, uint8_t *iprio)
347 {
348     int irq, best_irq = RISCV_EXCP_NONE;
349     unsigned int prio, best_prio = UINT_MAX;
350 
351     if (!pending) {
352         return RISCV_EXCP_NONE;
353     }
354 
355     irq = ctz64(pending);
356     if (!((extirq == IRQ_M_EXT) ? riscv_cpu_cfg(env)->ext_smaia :
357                                   riscv_cpu_cfg(env)->ext_ssaia)) {
358         return irq;
359     }
360 
361     pending = pending >> irq;
362     while (pending) {
363         prio = iprio[irq];
364         if (!prio) {
365             if (irq == extirq) {
366                 prio = extirq_def_prio;
367             } else {
368                 prio = (riscv_cpu_default_priority(irq) < extirq_def_prio) ?
369                        1 : IPRIO_MMAXIPRIO;
370             }
371         }
372         if ((pending & 0x1) && (prio <= best_prio)) {
373             best_irq = irq;
374             best_prio = prio;
375         }
376         irq++;
377         pending = pending >> 1;
378     }
379 
380     return best_irq;
381 }
382 
383 /*
384  * Doesn't report interrupts inserted using mvip from M-mode firmware or
385  * using hvip bits 13:63 from HS-mode. Those are returned in
386  * riscv_cpu_sirq_pending() and riscv_cpu_vsirq_pending().
387  */
388 uint64_t riscv_cpu_all_pending(CPURISCVState *env)
389 {
390     uint32_t gein = get_field(env->hstatus, HSTATUS_VGEIN);
391     uint64_t vsgein = (env->hgeip & (1ULL << gein)) ? MIP_VSEIP : 0;
392     uint64_t vstip = (env->vstime_irq) ? MIP_VSTIP : 0;
393 
394     return (env->mip | vsgein | vstip) & env->mie;
395 }
396 
397 int riscv_cpu_mirq_pending(CPURISCVState *env)
398 {
399     uint64_t irqs = riscv_cpu_all_pending(env) & ~env->mideleg &
400                     ~(MIP_SGEIP | MIP_VSSIP | MIP_VSTIP | MIP_VSEIP);
401 
402     return riscv_cpu_pending_to_irq(env, IRQ_M_EXT, IPRIO_DEFAULT_M,
403                                     irqs, env->miprio);
404 }
405 
406 int riscv_cpu_sirq_pending(CPURISCVState *env)
407 {
408     uint64_t irqs = riscv_cpu_all_pending(env) & env->mideleg &
409                     ~(MIP_VSSIP | MIP_VSTIP | MIP_VSEIP);
410     uint64_t irqs_f = env->mvip & env->mvien & ~env->mideleg & env->sie;
411 
412     return riscv_cpu_pending_to_irq(env, IRQ_S_EXT, IPRIO_DEFAULT_S,
413                                     irqs | irqs_f, env->siprio);
414 }
415 
416 int riscv_cpu_vsirq_pending(CPURISCVState *env)
417 {
418     uint64_t irqs = riscv_cpu_all_pending(env) & env->mideleg & env->hideleg;
419     uint64_t irqs_f_vs = env->hvip & env->hvien & ~env->hideleg & env->vsie;
420     uint64_t vsbits;
421 
422     /* Bring VS-level bits to correct position */
423     vsbits = irqs & VS_MODE_INTERRUPTS;
424     irqs &= ~VS_MODE_INTERRUPTS;
425     irqs |= vsbits >> 1;
426 
427     return riscv_cpu_pending_to_irq(env, IRQ_S_EXT, IPRIO_DEFAULT_S,
428                                     (irqs | irqs_f_vs), env->hviprio);
429 }
430 
431 static int riscv_cpu_local_irq_pending(CPURISCVState *env)
432 {
433     uint64_t irqs, pending, mie, hsie, vsie, irqs_f, irqs_f_vs;
434     uint64_t vsbits, irq_delegated;
435     int virq;
436 
437     /* Determine interrupt enable state of all privilege modes */
438     if (env->virt_enabled) {
439         mie = 1;
440         hsie = 1;
441         vsie = (env->priv < PRV_S) ||
442                (env->priv == PRV_S && get_field(env->mstatus, MSTATUS_SIE));
443     } else {
444         mie = (env->priv < PRV_M) ||
445               (env->priv == PRV_M && get_field(env->mstatus, MSTATUS_MIE));
446         hsie = (env->priv < PRV_S) ||
447                (env->priv == PRV_S && get_field(env->mstatus, MSTATUS_SIE));
448         vsie = 0;
449     }
450 
451     /* Determine all pending interrupts */
452     pending = riscv_cpu_all_pending(env);
453 
454     /* Check M-mode interrupts */
455     irqs = pending & ~env->mideleg & -mie;
456     if (irqs) {
457         return riscv_cpu_pending_to_irq(env, IRQ_M_EXT, IPRIO_DEFAULT_M,
458                                         irqs, env->miprio);
459     }
460 
461     /* Check for virtual S-mode interrupts. */
462     irqs_f = env->mvip & (env->mvien & ~env->mideleg) & env->sie;
463 
464     /* Check HS-mode interrupts */
465     irqs =  ((pending & env->mideleg & ~env->hideleg) | irqs_f) & -hsie;
466     if (irqs) {
467         return riscv_cpu_pending_to_irq(env, IRQ_S_EXT, IPRIO_DEFAULT_S,
468                                         irqs, env->siprio);
469     }
470 
471     /* Check for virtual VS-mode interrupts. */
472     irqs_f_vs = env->hvip & env->hvien & ~env->hideleg & env->vsie;
473 
474     /* Check VS-mode interrupts */
475     irq_delegated = pending & env->mideleg & env->hideleg;
476 
477     /* Bring VS-level bits to correct position */
478     vsbits = irq_delegated & VS_MODE_INTERRUPTS;
479     irq_delegated &= ~VS_MODE_INTERRUPTS;
480     irq_delegated |= vsbits >> 1;
481 
482     irqs = (irq_delegated | irqs_f_vs) & -vsie;
483     if (irqs) {
484         virq = riscv_cpu_pending_to_irq(env, IRQ_S_EXT, IPRIO_DEFAULT_S,
485                                         irqs, env->hviprio);
486         if (virq <= 0 || (virq > 12 && virq <= 63)) {
487             return virq;
488         } else {
489             return virq + 1;
490         }
491     }
492 
493     /* Indicate no pending interrupt */
494     return RISCV_EXCP_NONE;
495 }
496 
497 bool riscv_cpu_exec_interrupt(CPUState *cs, int interrupt_request)
498 {
499     if (interrupt_request & CPU_INTERRUPT_HARD) {
500         RISCVCPU *cpu = RISCV_CPU(cs);
501         CPURISCVState *env = &cpu->env;
502         int interruptno = riscv_cpu_local_irq_pending(env);
503         if (interruptno >= 0) {
504             cs->exception_index = RISCV_EXCP_INT_FLAG | interruptno;
505             riscv_cpu_do_interrupt(cs);
506             return true;
507         }
508     }
509     return false;
510 }
511 
512 /* Return true is floating point support is currently enabled */
513 bool riscv_cpu_fp_enabled(CPURISCVState *env)
514 {
515     if (env->mstatus & MSTATUS_FS) {
516         if (env->virt_enabled && !(env->mstatus_hs & MSTATUS_FS)) {
517             return false;
518         }
519         return true;
520     }
521 
522     return false;
523 }
524 
525 /* Return true is vector support is currently enabled */
526 bool riscv_cpu_vector_enabled(CPURISCVState *env)
527 {
528     if (env->mstatus & MSTATUS_VS) {
529         if (env->virt_enabled && !(env->mstatus_hs & MSTATUS_VS)) {
530             return false;
531         }
532         return true;
533     }
534 
535     return false;
536 }
537 
538 void riscv_cpu_swap_hypervisor_regs(CPURISCVState *env)
539 {
540     uint64_t mstatus_mask = MSTATUS_MXR | MSTATUS_SUM |
541                             MSTATUS_SPP | MSTATUS_SPIE | MSTATUS_SIE |
542                             MSTATUS64_UXL | MSTATUS_VS;
543 
544     if (riscv_has_ext(env, RVF)) {
545         mstatus_mask |= MSTATUS_FS;
546     }
547     bool current_virt = env->virt_enabled;
548 
549     g_assert(riscv_has_ext(env, RVH));
550 
551     if (current_virt) {
552         /* Current V=1 and we are about to change to V=0 */
553         env->vsstatus = env->mstatus & mstatus_mask;
554         env->mstatus &= ~mstatus_mask;
555         env->mstatus |= env->mstatus_hs;
556 
557         env->vstvec = env->stvec;
558         env->stvec = env->stvec_hs;
559 
560         env->vsscratch = env->sscratch;
561         env->sscratch = env->sscratch_hs;
562 
563         env->vsepc = env->sepc;
564         env->sepc = env->sepc_hs;
565 
566         env->vscause = env->scause;
567         env->scause = env->scause_hs;
568 
569         env->vstval = env->stval;
570         env->stval = env->stval_hs;
571 
572         env->vsatp = env->satp;
573         env->satp = env->satp_hs;
574     } else {
575         /* Current V=0 and we are about to change to V=1 */
576         env->mstatus_hs = env->mstatus & mstatus_mask;
577         env->mstatus &= ~mstatus_mask;
578         env->mstatus |= env->vsstatus;
579 
580         env->stvec_hs = env->stvec;
581         env->stvec = env->vstvec;
582 
583         env->sscratch_hs = env->sscratch;
584         env->sscratch = env->vsscratch;
585 
586         env->sepc_hs = env->sepc;
587         env->sepc = env->vsepc;
588 
589         env->scause_hs = env->scause;
590         env->scause = env->vscause;
591 
592         env->stval_hs = env->stval;
593         env->stval = env->vstval;
594 
595         env->satp_hs = env->satp;
596         env->satp = env->vsatp;
597     }
598 }
599 
600 target_ulong riscv_cpu_get_geilen(CPURISCVState *env)
601 {
602     if (!riscv_has_ext(env, RVH)) {
603         return 0;
604     }
605 
606     return env->geilen;
607 }
608 
609 void riscv_cpu_set_geilen(CPURISCVState *env, target_ulong geilen)
610 {
611     if (!riscv_has_ext(env, RVH)) {
612         return;
613     }
614 
615     if (geilen > (TARGET_LONG_BITS - 1)) {
616         return;
617     }
618 
619     env->geilen = geilen;
620 }
621 
622 int riscv_cpu_claim_interrupts(RISCVCPU *cpu, uint64_t interrupts)
623 {
624     CPURISCVState *env = &cpu->env;
625     if (env->miclaim & interrupts) {
626         return -1;
627     } else {
628         env->miclaim |= interrupts;
629         return 0;
630     }
631 }
632 
633 void riscv_cpu_interrupt(CPURISCVState *env)
634 {
635     uint64_t gein, vsgein = 0, vstip = 0, irqf = 0;
636     CPUState *cs = env_cpu(env);
637 
638     BQL_LOCK_GUARD();
639 
640     if (env->virt_enabled) {
641         gein = get_field(env->hstatus, HSTATUS_VGEIN);
642         vsgein = (env->hgeip & (1ULL << gein)) ? MIP_VSEIP : 0;
643         irqf = env->hvien & env->hvip & env->vsie;
644     } else {
645         irqf = env->mvien & env->mvip & env->sie;
646     }
647 
648     vstip = env->vstime_irq ? MIP_VSTIP : 0;
649 
650     if (env->mip | vsgein | vstip | irqf) {
651         cpu_interrupt(cs, CPU_INTERRUPT_HARD);
652     } else {
653         cpu_reset_interrupt(cs, CPU_INTERRUPT_HARD);
654     }
655 }
656 
657 uint64_t riscv_cpu_update_mip(CPURISCVState *env, uint64_t mask, uint64_t value)
658 {
659     uint64_t old = env->mip;
660 
661     /* No need to update mip for VSTIP */
662     mask = ((mask == MIP_VSTIP) && env->vstime_irq) ? 0 : mask;
663 
664     BQL_LOCK_GUARD();
665 
666     env->mip = (env->mip & ~mask) | (value & mask);
667 
668     riscv_cpu_interrupt(env);
669 
670     return old;
671 }
672 
673 void riscv_cpu_set_rdtime_fn(CPURISCVState *env, uint64_t (*fn)(void *),
674                              void *arg)
675 {
676     env->rdtime_fn = fn;
677     env->rdtime_fn_arg = arg;
678 }
679 
680 void riscv_cpu_set_aia_ireg_rmw_fn(CPURISCVState *env, uint32_t priv,
681                                    int (*rmw_fn)(void *arg,
682                                                  target_ulong reg,
683                                                  target_ulong *val,
684                                                  target_ulong new_val,
685                                                  target_ulong write_mask),
686                                    void *rmw_fn_arg)
687 {
688     if (priv <= PRV_M) {
689         env->aia_ireg_rmw_fn[priv] = rmw_fn;
690         env->aia_ireg_rmw_fn_arg[priv] = rmw_fn_arg;
691     }
692 }
693 
694 void riscv_cpu_set_mode(CPURISCVState *env, target_ulong newpriv, bool virt_en)
695 {
696     g_assert(newpriv <= PRV_M && newpriv != PRV_RESERVED);
697 
698     if (newpriv != env->priv || env->virt_enabled != virt_en) {
699         if (icount_enabled()) {
700             riscv_itrigger_update_priv(env);
701         }
702 
703         riscv_pmu_update_fixed_ctrs(env, newpriv, virt_en);
704     }
705 
706     /* tlb_flush is unnecessary as mode is contained in mmu_idx */
707     env->priv = newpriv;
708     env->xl = cpu_recompute_xl(env);
709     riscv_cpu_update_mask(env);
710 
711     /*
712      * Clear the load reservation - otherwise a reservation placed in one
713      * context/process can be used by another, resulting in an SC succeeding
714      * incorrectly. Version 2.2 of the ISA specification explicitly requires
715      * this behaviour, while later revisions say that the kernel "should" use
716      * an SC instruction to force the yielding of a load reservation on a
717      * preemptive context switch. As a result, do both.
718      */
719     env->load_res = -1;
720 
721     if (riscv_has_ext(env, RVH)) {
722         /* Flush the TLB on all virt mode changes. */
723         if (env->virt_enabled != virt_en) {
724             tlb_flush(env_cpu(env));
725         }
726 
727         env->virt_enabled = virt_en;
728         if (virt_en) {
729             /*
730              * The guest external interrupts from an interrupt controller are
731              * delivered only when the Guest/VM is running (i.e. V=1). This
732              * means any guest external interrupt which is triggered while the
733              * Guest/VM is not running (i.e. V=0) will be missed on QEMU
734              * resulting in guest with sluggish response to serial console
735              * input and other I/O events.
736              *
737              * To solve this, we check and inject interrupt after setting V=1.
738              */
739             riscv_cpu_update_mip(env, 0, 0);
740         }
741     }
742 }
743 
744 /*
745  * get_physical_address_pmp - check PMP permission for this physical address
746  *
747  * Match the PMP region and check permission for this physical address and it's
748  * TLB page. Returns 0 if the permission checking was successful
749  *
750  * @env: CPURISCVState
751  * @prot: The returned protection attributes
752  * @addr: The physical address to be checked permission
753  * @access_type: The type of MMU access
754  * @mode: Indicates current privilege level.
755  */
756 static int get_physical_address_pmp(CPURISCVState *env, int *prot, hwaddr addr,
757                                     int size, MMUAccessType access_type,
758                                     int mode)
759 {
760     pmp_priv_t pmp_priv;
761     bool pmp_has_privs;
762 
763     if (!riscv_cpu_cfg(env)->pmp) {
764         *prot = PAGE_READ | PAGE_WRITE | PAGE_EXEC;
765         return TRANSLATE_SUCCESS;
766     }
767 
768     pmp_has_privs = pmp_hart_has_privs(env, addr, size, 1 << access_type,
769                                        &pmp_priv, mode);
770     if (!pmp_has_privs) {
771         *prot = 0;
772         return TRANSLATE_PMP_FAIL;
773     }
774 
775     *prot = pmp_priv_to_page_prot(pmp_priv);
776 
777     return TRANSLATE_SUCCESS;
778 }
779 
780 /*
781  * get_physical_address - get the physical address for this virtual address
782  *
783  * Do a page table walk to obtain the physical address corresponding to a
784  * virtual address. Returns 0 if the translation was successful
785  *
786  * Adapted from Spike's mmu_t::translate and mmu_t::walk
787  *
788  * @env: CPURISCVState
789  * @physical: This will be set to the calculated physical address
790  * @prot: The returned protection attributes
791  * @addr: The virtual address or guest physical address to be translated
792  * @fault_pte_addr: If not NULL, this will be set to fault pte address
793  *                  when a error occurs on pte address translation.
794  *                  This will already be shifted to match htval.
795  * @access_type: The type of MMU access
796  * @mmu_idx: Indicates current privilege level
797  * @first_stage: Are we in first stage translation?
798  *               Second stage is used for hypervisor guest translation
799  * @two_stage: Are we going to perform two stage translation
800  * @is_debug: Is this access from a debugger or the monitor?
801  */
802 static int get_physical_address(CPURISCVState *env, hwaddr *physical,
803                                 int *ret_prot, vaddr addr,
804                                 target_ulong *fault_pte_addr,
805                                 int access_type, int mmu_idx,
806                                 bool first_stage, bool two_stage,
807                                 bool is_debug)
808 {
809     /*
810      * NOTE: the env->pc value visible here will not be
811      * correct, but the value visible to the exception handler
812      * (riscv_cpu_do_interrupt) is correct
813      */
814     MemTxResult res;
815     MemTxAttrs attrs = MEMTXATTRS_UNSPECIFIED;
816     int mode = mmuidx_priv(mmu_idx);
817     bool use_background = false;
818     hwaddr ppn;
819     int napot_bits = 0;
820     target_ulong napot_mask;
821 
822     /*
823      * Check if we should use the background registers for the two
824      * stage translation. We don't need to check if we actually need
825      * two stage translation as that happened before this function
826      * was called. Background registers will be used if the guest has
827      * forced a two stage translation to be on (in HS or M mode).
828      */
829     if (!env->virt_enabled && two_stage) {
830         use_background = true;
831     }
832 
833     if (mode == PRV_M || !riscv_cpu_cfg(env)->mmu) {
834         *physical = addr;
835         *ret_prot = PAGE_READ | PAGE_WRITE | PAGE_EXEC;
836         return TRANSLATE_SUCCESS;
837     }
838 
839     *ret_prot = 0;
840 
841     hwaddr base;
842     int levels, ptidxbits, ptesize, vm, widened;
843 
844     if (first_stage == true) {
845         if (use_background) {
846             if (riscv_cpu_mxl(env) == MXL_RV32) {
847                 base = (hwaddr)get_field(env->vsatp, SATP32_PPN) << PGSHIFT;
848                 vm = get_field(env->vsatp, SATP32_MODE);
849             } else {
850                 base = (hwaddr)get_field(env->vsatp, SATP64_PPN) << PGSHIFT;
851                 vm = get_field(env->vsatp, SATP64_MODE);
852             }
853         } else {
854             if (riscv_cpu_mxl(env) == MXL_RV32) {
855                 base = (hwaddr)get_field(env->satp, SATP32_PPN) << PGSHIFT;
856                 vm = get_field(env->satp, SATP32_MODE);
857             } else {
858                 base = (hwaddr)get_field(env->satp, SATP64_PPN) << PGSHIFT;
859                 vm = get_field(env->satp, SATP64_MODE);
860             }
861         }
862         widened = 0;
863     } else {
864         if (riscv_cpu_mxl(env) == MXL_RV32) {
865             base = (hwaddr)get_field(env->hgatp, SATP32_PPN) << PGSHIFT;
866             vm = get_field(env->hgatp, SATP32_MODE);
867         } else {
868             base = (hwaddr)get_field(env->hgatp, SATP64_PPN) << PGSHIFT;
869             vm = get_field(env->hgatp, SATP64_MODE);
870         }
871         widened = 2;
872     }
873 
874     switch (vm) {
875     case VM_1_10_SV32:
876       levels = 2; ptidxbits = 10; ptesize = 4; break;
877     case VM_1_10_SV39:
878       levels = 3; ptidxbits = 9; ptesize = 8; break;
879     case VM_1_10_SV48:
880       levels = 4; ptidxbits = 9; ptesize = 8; break;
881     case VM_1_10_SV57:
882       levels = 5; ptidxbits = 9; ptesize = 8; break;
883     case VM_1_10_MBARE:
884         *physical = addr;
885         *ret_prot = PAGE_READ | PAGE_WRITE | PAGE_EXEC;
886         return TRANSLATE_SUCCESS;
887     default:
888       g_assert_not_reached();
889     }
890 
891     CPUState *cs = env_cpu(env);
892     int va_bits = PGSHIFT + levels * ptidxbits + widened;
893 
894     if (first_stage == true) {
895         target_ulong mask, masked_msbs;
896 
897         if (TARGET_LONG_BITS > (va_bits - 1)) {
898             mask = (1L << (TARGET_LONG_BITS - (va_bits - 1))) - 1;
899         } else {
900             mask = 0;
901         }
902         masked_msbs = (addr >> (va_bits - 1)) & mask;
903 
904         if (masked_msbs != 0 && masked_msbs != mask) {
905             return TRANSLATE_FAIL;
906         }
907     } else {
908         if (vm != VM_1_10_SV32 && addr >> va_bits != 0) {
909             return TRANSLATE_FAIL;
910         }
911     }
912 
913     bool pbmte = env->menvcfg & MENVCFG_PBMTE;
914     bool svade = riscv_cpu_cfg(env)->ext_svade;
915     bool svadu = riscv_cpu_cfg(env)->ext_svadu;
916     bool adue = svadu ? env->menvcfg & MENVCFG_ADUE : !svade;
917 
918     if (first_stage && two_stage && env->virt_enabled) {
919         pbmte = pbmte && (env->henvcfg & HENVCFG_PBMTE);
920         adue = adue && (env->henvcfg & HENVCFG_ADUE);
921     }
922 
923     int ptshift = (levels - 1) * ptidxbits;
924     target_ulong pte;
925     hwaddr pte_addr;
926     int i;
927 
928 #if !TCG_OVERSIZED_GUEST
929 restart:
930 #endif
931     for (i = 0; i < levels; i++, ptshift -= ptidxbits) {
932         target_ulong idx;
933         if (i == 0) {
934             idx = (addr >> (PGSHIFT + ptshift)) &
935                            ((1 << (ptidxbits + widened)) - 1);
936         } else {
937             idx = (addr >> (PGSHIFT + ptshift)) &
938                            ((1 << ptidxbits) - 1);
939         }
940 
941         /* check that physical address of PTE is legal */
942 
943         if (two_stage && first_stage) {
944             int vbase_prot;
945             hwaddr vbase;
946 
947             /* Do the second stage translation on the base PTE address. */
948             int vbase_ret = get_physical_address(env, &vbase, &vbase_prot,
949                                                  base, NULL, MMU_DATA_LOAD,
950                                                  MMUIdx_U, false, true,
951                                                  is_debug);
952 
953             if (vbase_ret != TRANSLATE_SUCCESS) {
954                 if (fault_pte_addr) {
955                     *fault_pte_addr = (base + idx * ptesize) >> 2;
956                 }
957                 return TRANSLATE_G_STAGE_FAIL;
958             }
959 
960             pte_addr = vbase + idx * ptesize;
961         } else {
962             pte_addr = base + idx * ptesize;
963         }
964 
965         int pmp_prot;
966         int pmp_ret = get_physical_address_pmp(env, &pmp_prot, pte_addr,
967                                                sizeof(target_ulong),
968                                                MMU_DATA_LOAD, PRV_S);
969         if (pmp_ret != TRANSLATE_SUCCESS) {
970             return TRANSLATE_PMP_FAIL;
971         }
972 
973         if (riscv_cpu_mxl(env) == MXL_RV32) {
974             pte = address_space_ldl(cs->as, pte_addr, attrs, &res);
975         } else {
976             pte = address_space_ldq(cs->as, pte_addr, attrs, &res);
977         }
978 
979         if (res != MEMTX_OK) {
980             return TRANSLATE_FAIL;
981         }
982 
983         if (riscv_cpu_sxl(env) == MXL_RV32) {
984             ppn = pte >> PTE_PPN_SHIFT;
985         } else {
986             if (pte & PTE_RESERVED) {
987                 return TRANSLATE_FAIL;
988             }
989 
990             if (!pbmte && (pte & PTE_PBMT)) {
991                 return TRANSLATE_FAIL;
992             }
993 
994             if (!riscv_cpu_cfg(env)->ext_svnapot && (pte & PTE_N)) {
995                 return TRANSLATE_FAIL;
996             }
997 
998             ppn = (pte & (target_ulong)PTE_PPN_MASK) >> PTE_PPN_SHIFT;
999         }
1000 
1001         if (!(pte & PTE_V)) {
1002             /* Invalid PTE */
1003             return TRANSLATE_FAIL;
1004         }
1005         if (pte & (PTE_R | PTE_W | PTE_X)) {
1006             goto leaf;
1007         }
1008 
1009         /* Inner PTE, continue walking */
1010         if (pte & (PTE_D | PTE_A | PTE_U | PTE_ATTR)) {
1011             return TRANSLATE_FAIL;
1012         }
1013         base = ppn << PGSHIFT;
1014     }
1015 
1016     /* No leaf pte at any translation level. */
1017     return TRANSLATE_FAIL;
1018 
1019  leaf:
1020     if (ppn & ((1ULL << ptshift) - 1)) {
1021         /* Misaligned PPN */
1022         return TRANSLATE_FAIL;
1023     }
1024     if (!pbmte && (pte & PTE_PBMT)) {
1025         /* Reserved without Svpbmt. */
1026         return TRANSLATE_FAIL;
1027     }
1028 
1029     /* Check for reserved combinations of RWX flags. */
1030     switch (pte & (PTE_R | PTE_W | PTE_X)) {
1031     case PTE_W:
1032     case PTE_W | PTE_X:
1033         return TRANSLATE_FAIL;
1034     }
1035 
1036     int prot = 0;
1037     if (pte & PTE_R) {
1038         prot |= PAGE_READ;
1039     }
1040     if (pte & PTE_W) {
1041         prot |= PAGE_WRITE;
1042     }
1043     if (pte & PTE_X) {
1044         bool mxr = false;
1045 
1046         /*
1047          * Use mstatus for first stage or for the second stage without
1048          * virt_enabled (MPRV+MPV)
1049          */
1050         if (first_stage || !env->virt_enabled) {
1051             mxr = get_field(env->mstatus, MSTATUS_MXR);
1052         }
1053 
1054         /* MPRV+MPV case, check VSSTATUS */
1055         if (first_stage && two_stage && !env->virt_enabled) {
1056             mxr |= get_field(env->vsstatus, MSTATUS_MXR);
1057         }
1058 
1059         /*
1060          * Setting MXR at HS-level overrides both VS-stage and G-stage
1061          * execute-only permissions
1062          */
1063         if (env->virt_enabled) {
1064             mxr |= get_field(env->mstatus_hs, MSTATUS_MXR);
1065         }
1066 
1067         if (mxr) {
1068             prot |= PAGE_READ;
1069         }
1070         prot |= PAGE_EXEC;
1071     }
1072 
1073     if (pte & PTE_U) {
1074         if (mode != PRV_U) {
1075             if (!mmuidx_sum(mmu_idx)) {
1076                 return TRANSLATE_FAIL;
1077             }
1078             /* SUM allows only read+write, not execute. */
1079             prot &= PAGE_READ | PAGE_WRITE;
1080         }
1081     } else if (mode != PRV_S) {
1082         /* Supervisor PTE flags when not S mode */
1083         return TRANSLATE_FAIL;
1084     }
1085 
1086     if (!((prot >> access_type) & 1)) {
1087         /* Access check failed */
1088         return TRANSLATE_FAIL;
1089     }
1090 
1091     target_ulong updated_pte = pte;
1092 
1093     /*
1094      * If ADUE is enabled, set accessed and dirty bits.
1095      * Otherwise raise an exception if necessary.
1096      */
1097     if (adue) {
1098         updated_pte |= PTE_A | (access_type == MMU_DATA_STORE ? PTE_D : 0);
1099     } else if (!(pte & PTE_A) ||
1100                (access_type == MMU_DATA_STORE && !(pte & PTE_D))) {
1101         return TRANSLATE_FAIL;
1102     }
1103 
1104     /* Page table updates need to be atomic with MTTCG enabled */
1105     if (updated_pte != pte && !is_debug) {
1106         if (!adue) {
1107             return TRANSLATE_FAIL;
1108         }
1109 
1110         /*
1111          * - if accessed or dirty bits need updating, and the PTE is
1112          *   in RAM, then we do so atomically with a compare and swap.
1113          * - if the PTE is in IO space or ROM, then it can't be updated
1114          *   and we return TRANSLATE_FAIL.
1115          * - if the PTE changed by the time we went to update it, then
1116          *   it is no longer valid and we must re-walk the page table.
1117          */
1118         MemoryRegion *mr;
1119         hwaddr l = sizeof(target_ulong), addr1;
1120         mr = address_space_translate(cs->as, pte_addr, &addr1, &l,
1121                                      false, MEMTXATTRS_UNSPECIFIED);
1122         if (memory_region_is_ram(mr)) {
1123             target_ulong *pte_pa = qemu_map_ram_ptr(mr->ram_block, addr1);
1124 #if TCG_OVERSIZED_GUEST
1125             /*
1126              * MTTCG is not enabled on oversized TCG guests so
1127              * page table updates do not need to be atomic
1128              */
1129             *pte_pa = pte = updated_pte;
1130 #else
1131             target_ulong old_pte = qatomic_cmpxchg(pte_pa, pte, updated_pte);
1132             if (old_pte != pte) {
1133                 goto restart;
1134             }
1135             pte = updated_pte;
1136 #endif
1137         } else {
1138             /*
1139              * Misconfigured PTE in ROM (AD bits are not preset) or
1140              * PTE is in IO space and can't be updated atomically.
1141              */
1142             return TRANSLATE_FAIL;
1143         }
1144     }
1145 
1146     /* For superpage mappings, make a fake leaf PTE for the TLB's benefit. */
1147     target_ulong vpn = addr >> PGSHIFT;
1148 
1149     if (riscv_cpu_cfg(env)->ext_svnapot && (pte & PTE_N)) {
1150         napot_bits = ctzl(ppn) + 1;
1151         if ((i != (levels - 1)) || (napot_bits != 4)) {
1152             return TRANSLATE_FAIL;
1153         }
1154     }
1155 
1156     napot_mask = (1 << napot_bits) - 1;
1157     *physical = (((ppn & ~napot_mask) | (vpn & napot_mask) |
1158                   (vpn & (((target_ulong)1 << ptshift) - 1))
1159                  ) << PGSHIFT) | (addr & ~TARGET_PAGE_MASK);
1160 
1161     /*
1162      * Remove write permission unless this is a store, or the page is
1163      * already dirty, so that we TLB miss on later writes to update
1164      * the dirty bit.
1165      */
1166     if (access_type != MMU_DATA_STORE && !(pte & PTE_D)) {
1167         prot &= ~PAGE_WRITE;
1168     }
1169     *ret_prot = prot;
1170 
1171     return TRANSLATE_SUCCESS;
1172 }
1173 
1174 static void raise_mmu_exception(CPURISCVState *env, target_ulong address,
1175                                 MMUAccessType access_type, bool pmp_violation,
1176                                 bool first_stage, bool two_stage,
1177                                 bool two_stage_indirect)
1178 {
1179     CPUState *cs = env_cpu(env);
1180 
1181     switch (access_type) {
1182     case MMU_INST_FETCH:
1183         if (pmp_violation) {
1184             cs->exception_index = RISCV_EXCP_INST_ACCESS_FAULT;
1185         } else if (env->virt_enabled && !first_stage) {
1186             cs->exception_index = RISCV_EXCP_INST_GUEST_PAGE_FAULT;
1187         } else {
1188             cs->exception_index = RISCV_EXCP_INST_PAGE_FAULT;
1189         }
1190         break;
1191     case MMU_DATA_LOAD:
1192         if (pmp_violation) {
1193             cs->exception_index = RISCV_EXCP_LOAD_ACCESS_FAULT;
1194         } else if (two_stage && !first_stage) {
1195             cs->exception_index = RISCV_EXCP_LOAD_GUEST_ACCESS_FAULT;
1196         } else {
1197             cs->exception_index = RISCV_EXCP_LOAD_PAGE_FAULT;
1198         }
1199         break;
1200     case MMU_DATA_STORE:
1201         if (pmp_violation) {
1202             cs->exception_index = RISCV_EXCP_STORE_AMO_ACCESS_FAULT;
1203         } else if (two_stage && !first_stage) {
1204             cs->exception_index = RISCV_EXCP_STORE_GUEST_AMO_ACCESS_FAULT;
1205         } else {
1206             cs->exception_index = RISCV_EXCP_STORE_PAGE_FAULT;
1207         }
1208         break;
1209     default:
1210         g_assert_not_reached();
1211     }
1212     env->badaddr = address;
1213     env->two_stage_lookup = two_stage;
1214     env->two_stage_indirect_lookup = two_stage_indirect;
1215 }
1216 
1217 hwaddr riscv_cpu_get_phys_page_debug(CPUState *cs, vaddr addr)
1218 {
1219     RISCVCPU *cpu = RISCV_CPU(cs);
1220     CPURISCVState *env = &cpu->env;
1221     hwaddr phys_addr;
1222     int prot;
1223     int mmu_idx = riscv_env_mmu_index(&cpu->env, false);
1224 
1225     if (get_physical_address(env, &phys_addr, &prot, addr, NULL, 0, mmu_idx,
1226                              true, env->virt_enabled, true)) {
1227         return -1;
1228     }
1229 
1230     if (env->virt_enabled) {
1231         if (get_physical_address(env, &phys_addr, &prot, phys_addr, NULL,
1232                                  0, MMUIdx_U, false, true, true)) {
1233             return -1;
1234         }
1235     }
1236 
1237     return phys_addr & TARGET_PAGE_MASK;
1238 }
1239 
1240 void riscv_cpu_do_transaction_failed(CPUState *cs, hwaddr physaddr,
1241                                      vaddr addr, unsigned size,
1242                                      MMUAccessType access_type,
1243                                      int mmu_idx, MemTxAttrs attrs,
1244                                      MemTxResult response, uintptr_t retaddr)
1245 {
1246     RISCVCPU *cpu = RISCV_CPU(cs);
1247     CPURISCVState *env = &cpu->env;
1248 
1249     if (access_type == MMU_DATA_STORE) {
1250         cs->exception_index = RISCV_EXCP_STORE_AMO_ACCESS_FAULT;
1251     } else if (access_type == MMU_DATA_LOAD) {
1252         cs->exception_index = RISCV_EXCP_LOAD_ACCESS_FAULT;
1253     } else {
1254         cs->exception_index = RISCV_EXCP_INST_ACCESS_FAULT;
1255     }
1256 
1257     env->badaddr = addr;
1258     env->two_stage_lookup = mmuidx_2stage(mmu_idx);
1259     env->two_stage_indirect_lookup = false;
1260     cpu_loop_exit_restore(cs, retaddr);
1261 }
1262 
1263 void riscv_cpu_do_unaligned_access(CPUState *cs, vaddr addr,
1264                                    MMUAccessType access_type, int mmu_idx,
1265                                    uintptr_t retaddr)
1266 {
1267     RISCVCPU *cpu = RISCV_CPU(cs);
1268     CPURISCVState *env = &cpu->env;
1269     switch (access_type) {
1270     case MMU_INST_FETCH:
1271         cs->exception_index = RISCV_EXCP_INST_ADDR_MIS;
1272         break;
1273     case MMU_DATA_LOAD:
1274         cs->exception_index = RISCV_EXCP_LOAD_ADDR_MIS;
1275         break;
1276     case MMU_DATA_STORE:
1277         cs->exception_index = RISCV_EXCP_STORE_AMO_ADDR_MIS;
1278         break;
1279     default:
1280         g_assert_not_reached();
1281     }
1282     env->badaddr = addr;
1283     env->two_stage_lookup = mmuidx_2stage(mmu_idx);
1284     env->two_stage_indirect_lookup = false;
1285     cpu_loop_exit_restore(cs, retaddr);
1286 }
1287 
1288 
1289 static void pmu_tlb_fill_incr_ctr(RISCVCPU *cpu, MMUAccessType access_type)
1290 {
1291     enum riscv_pmu_event_idx pmu_event_type;
1292 
1293     switch (access_type) {
1294     case MMU_INST_FETCH:
1295         pmu_event_type = RISCV_PMU_EVENT_CACHE_ITLB_PREFETCH_MISS;
1296         break;
1297     case MMU_DATA_LOAD:
1298         pmu_event_type = RISCV_PMU_EVENT_CACHE_DTLB_READ_MISS;
1299         break;
1300     case MMU_DATA_STORE:
1301         pmu_event_type = RISCV_PMU_EVENT_CACHE_DTLB_WRITE_MISS;
1302         break;
1303     default:
1304         return;
1305     }
1306 
1307     riscv_pmu_incr_ctr(cpu, pmu_event_type);
1308 }
1309 
1310 bool riscv_cpu_tlb_fill(CPUState *cs, vaddr address, int size,
1311                         MMUAccessType access_type, int mmu_idx,
1312                         bool probe, uintptr_t retaddr)
1313 {
1314     RISCVCPU *cpu = RISCV_CPU(cs);
1315     CPURISCVState *env = &cpu->env;
1316     vaddr im_address;
1317     hwaddr pa = 0;
1318     int prot, prot2, prot_pmp;
1319     bool pmp_violation = false;
1320     bool first_stage_error = true;
1321     bool two_stage_lookup = mmuidx_2stage(mmu_idx);
1322     bool two_stage_indirect_error = false;
1323     int ret = TRANSLATE_FAIL;
1324     int mode = mmuidx_priv(mmu_idx);
1325     /* default TLB page size */
1326     hwaddr tlb_size = TARGET_PAGE_SIZE;
1327 
1328     env->guest_phys_fault_addr = 0;
1329 
1330     qemu_log_mask(CPU_LOG_MMU, "%s ad %" VADDR_PRIx " rw %d mmu_idx %d\n",
1331                   __func__, address, access_type, mmu_idx);
1332 
1333     pmu_tlb_fill_incr_ctr(cpu, access_type);
1334     if (two_stage_lookup) {
1335         /* Two stage lookup */
1336         ret = get_physical_address(env, &pa, &prot, address,
1337                                    &env->guest_phys_fault_addr, access_type,
1338                                    mmu_idx, true, true, false);
1339 
1340         /*
1341          * A G-stage exception may be triggered during two state lookup.
1342          * And the env->guest_phys_fault_addr has already been set in
1343          * get_physical_address().
1344          */
1345         if (ret == TRANSLATE_G_STAGE_FAIL) {
1346             first_stage_error = false;
1347             two_stage_indirect_error = true;
1348         }
1349 
1350         qemu_log_mask(CPU_LOG_MMU,
1351                       "%s 1st-stage address=%" VADDR_PRIx " ret %d physical "
1352                       HWADDR_FMT_plx " prot %d\n",
1353                       __func__, address, ret, pa, prot);
1354 
1355         if (ret == TRANSLATE_SUCCESS) {
1356             /* Second stage lookup */
1357             im_address = pa;
1358 
1359             ret = get_physical_address(env, &pa, &prot2, im_address, NULL,
1360                                        access_type, MMUIdx_U, false, true,
1361                                        false);
1362 
1363             qemu_log_mask(CPU_LOG_MMU,
1364                           "%s 2nd-stage address=%" VADDR_PRIx
1365                           " ret %d physical "
1366                           HWADDR_FMT_plx " prot %d\n",
1367                           __func__, im_address, ret, pa, prot2);
1368 
1369             prot &= prot2;
1370 
1371             if (ret == TRANSLATE_SUCCESS) {
1372                 ret = get_physical_address_pmp(env, &prot_pmp, pa,
1373                                                size, access_type, mode);
1374                 tlb_size = pmp_get_tlb_size(env, pa);
1375 
1376                 qemu_log_mask(CPU_LOG_MMU,
1377                               "%s PMP address=" HWADDR_FMT_plx " ret %d prot"
1378                               " %d tlb_size %" HWADDR_PRIu "\n",
1379                               __func__, pa, ret, prot_pmp, tlb_size);
1380 
1381                 prot &= prot_pmp;
1382             } else {
1383                 /*
1384                  * Guest physical address translation failed, this is a HS
1385                  * level exception
1386                  */
1387                 first_stage_error = false;
1388                 if (ret != TRANSLATE_PMP_FAIL) {
1389                     env->guest_phys_fault_addr = (im_address |
1390                                                   (address &
1391                                                    (TARGET_PAGE_SIZE - 1))) >> 2;
1392                 }
1393             }
1394         }
1395     } else {
1396         /* Single stage lookup */
1397         ret = get_physical_address(env, &pa, &prot, address, NULL,
1398                                    access_type, mmu_idx, true, false, false);
1399 
1400         qemu_log_mask(CPU_LOG_MMU,
1401                       "%s address=%" VADDR_PRIx " ret %d physical "
1402                       HWADDR_FMT_plx " prot %d\n",
1403                       __func__, address, ret, pa, prot);
1404 
1405         if (ret == TRANSLATE_SUCCESS) {
1406             ret = get_physical_address_pmp(env, &prot_pmp, pa,
1407                                            size, access_type, mode);
1408             tlb_size = pmp_get_tlb_size(env, pa);
1409 
1410             qemu_log_mask(CPU_LOG_MMU,
1411                           "%s PMP address=" HWADDR_FMT_plx " ret %d prot"
1412                           " %d tlb_size %" HWADDR_PRIu "\n",
1413                           __func__, pa, ret, prot_pmp, tlb_size);
1414 
1415             prot &= prot_pmp;
1416         }
1417     }
1418 
1419     if (ret == TRANSLATE_PMP_FAIL) {
1420         pmp_violation = true;
1421     }
1422 
1423     if (ret == TRANSLATE_SUCCESS) {
1424         tlb_set_page(cs, address & ~(tlb_size - 1), pa & ~(tlb_size - 1),
1425                      prot, mmu_idx, tlb_size);
1426         return true;
1427     } else if (probe) {
1428         return false;
1429     } else {
1430         raise_mmu_exception(env, address, access_type, pmp_violation,
1431                             first_stage_error, two_stage_lookup,
1432                             two_stage_indirect_error);
1433         cpu_loop_exit_restore(cs, retaddr);
1434     }
1435 
1436     return true;
1437 }
1438 
1439 static target_ulong riscv_transformed_insn(CPURISCVState *env,
1440                                            target_ulong insn,
1441                                            target_ulong taddr)
1442 {
1443     target_ulong xinsn = 0;
1444     target_ulong access_rs1 = 0, access_imm = 0, access_size = 0;
1445 
1446     /*
1447      * Only Quadrant 0 and Quadrant 2 of RVC instruction space need to
1448      * be uncompressed. The Quadrant 1 of RVC instruction space need
1449      * not be transformed because these instructions won't generate
1450      * any load/store trap.
1451      */
1452 
1453     if ((insn & 0x3) != 0x3) {
1454         /* Transform 16bit instruction into 32bit instruction */
1455         switch (GET_C_OP(insn)) {
1456         case OPC_RISC_C_OP_QUAD0: /* Quadrant 0 */
1457             switch (GET_C_FUNC(insn)) {
1458             case OPC_RISC_C_FUNC_FLD_LQ:
1459                 if (riscv_cpu_xlen(env) != 128) { /* C.FLD (RV32/64) */
1460                     xinsn = OPC_RISC_FLD;
1461                     xinsn = SET_RD(xinsn, GET_C_RS2S(insn));
1462                     access_rs1 = GET_C_RS1S(insn);
1463                     access_imm = GET_C_LD_IMM(insn);
1464                     access_size = 8;
1465                 }
1466                 break;
1467             case OPC_RISC_C_FUNC_LW: /* C.LW */
1468                 xinsn = OPC_RISC_LW;
1469                 xinsn = SET_RD(xinsn, GET_C_RS2S(insn));
1470                 access_rs1 = GET_C_RS1S(insn);
1471                 access_imm = GET_C_LW_IMM(insn);
1472                 access_size = 4;
1473                 break;
1474             case OPC_RISC_C_FUNC_FLW_LD:
1475                 if (riscv_cpu_xlen(env) == 32) { /* C.FLW (RV32) */
1476                     xinsn = OPC_RISC_FLW;
1477                     xinsn = SET_RD(xinsn, GET_C_RS2S(insn));
1478                     access_rs1 = GET_C_RS1S(insn);
1479                     access_imm = GET_C_LW_IMM(insn);
1480                     access_size = 4;
1481                 } else { /* C.LD (RV64/RV128) */
1482                     xinsn = OPC_RISC_LD;
1483                     xinsn = SET_RD(xinsn, GET_C_RS2S(insn));
1484                     access_rs1 = GET_C_RS1S(insn);
1485                     access_imm = GET_C_LD_IMM(insn);
1486                     access_size = 8;
1487                 }
1488                 break;
1489             case OPC_RISC_C_FUNC_FSD_SQ:
1490                 if (riscv_cpu_xlen(env) != 128) { /* C.FSD (RV32/64) */
1491                     xinsn = OPC_RISC_FSD;
1492                     xinsn = SET_RS2(xinsn, GET_C_RS2S(insn));
1493                     access_rs1 = GET_C_RS1S(insn);
1494                     access_imm = GET_C_SD_IMM(insn);
1495                     access_size = 8;
1496                 }
1497                 break;
1498             case OPC_RISC_C_FUNC_SW: /* C.SW */
1499                 xinsn = OPC_RISC_SW;
1500                 xinsn = SET_RS2(xinsn, GET_C_RS2S(insn));
1501                 access_rs1 = GET_C_RS1S(insn);
1502                 access_imm = GET_C_SW_IMM(insn);
1503                 access_size = 4;
1504                 break;
1505             case OPC_RISC_C_FUNC_FSW_SD:
1506                 if (riscv_cpu_xlen(env) == 32) { /* C.FSW (RV32) */
1507                     xinsn = OPC_RISC_FSW;
1508                     xinsn = SET_RS2(xinsn, GET_C_RS2S(insn));
1509                     access_rs1 = GET_C_RS1S(insn);
1510                     access_imm = GET_C_SW_IMM(insn);
1511                     access_size = 4;
1512                 } else { /* C.SD (RV64/RV128) */
1513                     xinsn = OPC_RISC_SD;
1514                     xinsn = SET_RS2(xinsn, GET_C_RS2S(insn));
1515                     access_rs1 = GET_C_RS1S(insn);
1516                     access_imm = GET_C_SD_IMM(insn);
1517                     access_size = 8;
1518                 }
1519                 break;
1520             default:
1521                 break;
1522             }
1523             break;
1524         case OPC_RISC_C_OP_QUAD2: /* Quadrant 2 */
1525             switch (GET_C_FUNC(insn)) {
1526             case OPC_RISC_C_FUNC_FLDSP_LQSP:
1527                 if (riscv_cpu_xlen(env) != 128) { /* C.FLDSP (RV32/64) */
1528                     xinsn = OPC_RISC_FLD;
1529                     xinsn = SET_RD(xinsn, GET_C_RD(insn));
1530                     access_rs1 = 2;
1531                     access_imm = GET_C_LDSP_IMM(insn);
1532                     access_size = 8;
1533                 }
1534                 break;
1535             case OPC_RISC_C_FUNC_LWSP: /* C.LWSP */
1536                 xinsn = OPC_RISC_LW;
1537                 xinsn = SET_RD(xinsn, GET_C_RD(insn));
1538                 access_rs1 = 2;
1539                 access_imm = GET_C_LWSP_IMM(insn);
1540                 access_size = 4;
1541                 break;
1542             case OPC_RISC_C_FUNC_FLWSP_LDSP:
1543                 if (riscv_cpu_xlen(env) == 32) { /* C.FLWSP (RV32) */
1544                     xinsn = OPC_RISC_FLW;
1545                     xinsn = SET_RD(xinsn, GET_C_RD(insn));
1546                     access_rs1 = 2;
1547                     access_imm = GET_C_LWSP_IMM(insn);
1548                     access_size = 4;
1549                 } else { /* C.LDSP (RV64/RV128) */
1550                     xinsn = OPC_RISC_LD;
1551                     xinsn = SET_RD(xinsn, GET_C_RD(insn));
1552                     access_rs1 = 2;
1553                     access_imm = GET_C_LDSP_IMM(insn);
1554                     access_size = 8;
1555                 }
1556                 break;
1557             case OPC_RISC_C_FUNC_FSDSP_SQSP:
1558                 if (riscv_cpu_xlen(env) != 128) { /* C.FSDSP (RV32/64) */
1559                     xinsn = OPC_RISC_FSD;
1560                     xinsn = SET_RS2(xinsn, GET_C_RS2(insn));
1561                     access_rs1 = 2;
1562                     access_imm = GET_C_SDSP_IMM(insn);
1563                     access_size = 8;
1564                 }
1565                 break;
1566             case OPC_RISC_C_FUNC_SWSP: /* C.SWSP */
1567                 xinsn = OPC_RISC_SW;
1568                 xinsn = SET_RS2(xinsn, GET_C_RS2(insn));
1569                 access_rs1 = 2;
1570                 access_imm = GET_C_SWSP_IMM(insn);
1571                 access_size = 4;
1572                 break;
1573             case 7:
1574                 if (riscv_cpu_xlen(env) == 32) { /* C.FSWSP (RV32) */
1575                     xinsn = OPC_RISC_FSW;
1576                     xinsn = SET_RS2(xinsn, GET_C_RS2(insn));
1577                     access_rs1 = 2;
1578                     access_imm = GET_C_SWSP_IMM(insn);
1579                     access_size = 4;
1580                 } else { /* C.SDSP (RV64/RV128) */
1581                     xinsn = OPC_RISC_SD;
1582                     xinsn = SET_RS2(xinsn, GET_C_RS2(insn));
1583                     access_rs1 = 2;
1584                     access_imm = GET_C_SDSP_IMM(insn);
1585                     access_size = 8;
1586                 }
1587                 break;
1588             default:
1589                 break;
1590             }
1591             break;
1592         default:
1593             break;
1594         }
1595 
1596         /*
1597          * Clear Bit1 of transformed instruction to indicate that
1598          * original insruction was a 16bit instruction
1599          */
1600         xinsn &= ~((target_ulong)0x2);
1601     } else {
1602         /* Transform 32bit (or wider) instructions */
1603         switch (MASK_OP_MAJOR(insn)) {
1604         case OPC_RISC_ATOMIC:
1605             xinsn = insn;
1606             access_rs1 = GET_RS1(insn);
1607             access_size = 1 << GET_FUNCT3(insn);
1608             break;
1609         case OPC_RISC_LOAD:
1610         case OPC_RISC_FP_LOAD:
1611             xinsn = SET_I_IMM(insn, 0);
1612             access_rs1 = GET_RS1(insn);
1613             access_imm = GET_IMM(insn);
1614             access_size = 1 << GET_FUNCT3(insn);
1615             break;
1616         case OPC_RISC_STORE:
1617         case OPC_RISC_FP_STORE:
1618             xinsn = SET_S_IMM(insn, 0);
1619             access_rs1 = GET_RS1(insn);
1620             access_imm = GET_STORE_IMM(insn);
1621             access_size = 1 << GET_FUNCT3(insn);
1622             break;
1623         case OPC_RISC_SYSTEM:
1624             if (MASK_OP_SYSTEM(insn) == OPC_RISC_HLVHSV) {
1625                 xinsn = insn;
1626                 access_rs1 = GET_RS1(insn);
1627                 access_size = 1 << ((GET_FUNCT7(insn) >> 1) & 0x3);
1628                 access_size = 1 << access_size;
1629             }
1630             break;
1631         default:
1632             break;
1633         }
1634     }
1635 
1636     if (access_size) {
1637         xinsn = SET_RS1(xinsn, (taddr - (env->gpr[access_rs1] + access_imm)) &
1638                                (access_size - 1));
1639     }
1640 
1641     return xinsn;
1642 }
1643 
1644 /*
1645  * Handle Traps
1646  *
1647  * Adapted from Spike's processor_t::take_trap.
1648  *
1649  */
1650 void riscv_cpu_do_interrupt(CPUState *cs)
1651 {
1652     RISCVCPU *cpu = RISCV_CPU(cs);
1653     CPURISCVState *env = &cpu->env;
1654     bool virt = env->virt_enabled;
1655     bool write_gva = false;
1656     uint64_t s;
1657 
1658     /*
1659      * cs->exception is 32-bits wide unlike mcause which is XLEN-bits wide
1660      * so we mask off the MSB and separate into trap type and cause.
1661      */
1662     bool async = !!(cs->exception_index & RISCV_EXCP_INT_FLAG);
1663     target_ulong cause = cs->exception_index & RISCV_EXCP_INT_MASK;
1664     uint64_t deleg = async ? env->mideleg : env->medeleg;
1665     bool s_injected = env->mvip & (1 << cause) & env->mvien &&
1666         !(env->mip & (1 << cause));
1667     bool vs_injected = env->hvip & (1 << cause) & env->hvien &&
1668         !(env->mip & (1 << cause));
1669     target_ulong tval = 0;
1670     target_ulong tinst = 0;
1671     target_ulong htval = 0;
1672     target_ulong mtval2 = 0;
1673 
1674     if (!async) {
1675         /* set tval to badaddr for traps with address information */
1676         switch (cause) {
1677 #ifdef CONFIG_TCG
1678         case RISCV_EXCP_SEMIHOST:
1679             do_common_semihosting(cs);
1680             env->pc += 4;
1681             return;
1682 #endif
1683         case RISCV_EXCP_LOAD_GUEST_ACCESS_FAULT:
1684         case RISCV_EXCP_STORE_GUEST_AMO_ACCESS_FAULT:
1685         case RISCV_EXCP_LOAD_ADDR_MIS:
1686         case RISCV_EXCP_STORE_AMO_ADDR_MIS:
1687         case RISCV_EXCP_LOAD_ACCESS_FAULT:
1688         case RISCV_EXCP_STORE_AMO_ACCESS_FAULT:
1689         case RISCV_EXCP_LOAD_PAGE_FAULT:
1690         case RISCV_EXCP_STORE_PAGE_FAULT:
1691             write_gva = env->two_stage_lookup;
1692             tval = env->badaddr;
1693             if (env->two_stage_indirect_lookup) {
1694                 /*
1695                  * special pseudoinstruction for G-stage fault taken while
1696                  * doing VS-stage page table walk.
1697                  */
1698                 tinst = (riscv_cpu_xlen(env) == 32) ? 0x00002000 : 0x00003000;
1699             } else {
1700                 /*
1701                  * The "Addr. Offset" field in transformed instruction is
1702                  * non-zero only for misaligned access.
1703                  */
1704                 tinst = riscv_transformed_insn(env, env->bins, tval);
1705             }
1706             break;
1707         case RISCV_EXCP_INST_GUEST_PAGE_FAULT:
1708         case RISCV_EXCP_INST_ADDR_MIS:
1709         case RISCV_EXCP_INST_ACCESS_FAULT:
1710         case RISCV_EXCP_INST_PAGE_FAULT:
1711             write_gva = env->two_stage_lookup;
1712             tval = env->badaddr;
1713             if (env->two_stage_indirect_lookup) {
1714                 /*
1715                  * special pseudoinstruction for G-stage fault taken while
1716                  * doing VS-stage page table walk.
1717                  */
1718                 tinst = (riscv_cpu_xlen(env) == 32) ? 0x00002000 : 0x00003000;
1719             }
1720             break;
1721         case RISCV_EXCP_ILLEGAL_INST:
1722         case RISCV_EXCP_VIRT_INSTRUCTION_FAULT:
1723             tval = env->bins;
1724             break;
1725         case RISCV_EXCP_BREAKPOINT:
1726             tval = env->badaddr;
1727             if (cs->watchpoint_hit) {
1728                 tval = cs->watchpoint_hit->hitaddr;
1729                 cs->watchpoint_hit = NULL;
1730             }
1731             break;
1732         default:
1733             break;
1734         }
1735         /* ecall is dispatched as one cause so translate based on mode */
1736         if (cause == RISCV_EXCP_U_ECALL) {
1737             assert(env->priv <= 3);
1738 
1739             if (env->priv == PRV_M) {
1740                 cause = RISCV_EXCP_M_ECALL;
1741             } else if (env->priv == PRV_S && env->virt_enabled) {
1742                 cause = RISCV_EXCP_VS_ECALL;
1743             } else if (env->priv == PRV_S && !env->virt_enabled) {
1744                 cause = RISCV_EXCP_S_ECALL;
1745             } else if (env->priv == PRV_U) {
1746                 cause = RISCV_EXCP_U_ECALL;
1747             }
1748         }
1749     }
1750 
1751     trace_riscv_trap(env->mhartid, async, cause, env->pc, tval,
1752                      riscv_cpu_get_trap_name(cause, async));
1753 
1754     qemu_log_mask(CPU_LOG_INT,
1755                   "%s: hart:"TARGET_FMT_ld", async:%d, cause:"TARGET_FMT_lx", "
1756                   "epc:0x"TARGET_FMT_lx", tval:0x"TARGET_FMT_lx", desc=%s\n",
1757                   __func__, env->mhartid, async, cause, env->pc, tval,
1758                   riscv_cpu_get_trap_name(cause, async));
1759 
1760     if (env->priv <= PRV_S && cause < 64 &&
1761         (((deleg >> cause) & 1) || s_injected || vs_injected)) {
1762         /* handle the trap in S-mode */
1763         if (riscv_has_ext(env, RVH)) {
1764             uint64_t hdeleg = async ? env->hideleg : env->hedeleg;
1765 
1766             if (env->virt_enabled &&
1767                 (((hdeleg >> cause) & 1) || vs_injected)) {
1768                 /* Trap to VS mode */
1769                 /*
1770                  * See if we need to adjust cause. Yes if its VS mode interrupt
1771                  * no if hypervisor has delegated one of hs mode's interrupt
1772                  */
1773                 if (async && (cause == IRQ_VS_TIMER || cause == IRQ_VS_SOFT ||
1774                               cause == IRQ_VS_EXT)) {
1775                     cause = cause - 1;
1776                 }
1777                 write_gva = false;
1778             } else if (env->virt_enabled) {
1779                 /* Trap into HS mode, from virt */
1780                 riscv_cpu_swap_hypervisor_regs(env);
1781                 env->hstatus = set_field(env->hstatus, HSTATUS_SPVP,
1782                                          env->priv);
1783                 env->hstatus = set_field(env->hstatus, HSTATUS_SPV, true);
1784 
1785                 htval = env->guest_phys_fault_addr;
1786 
1787                 virt = false;
1788             } else {
1789                 /* Trap into HS mode */
1790                 env->hstatus = set_field(env->hstatus, HSTATUS_SPV, false);
1791                 htval = env->guest_phys_fault_addr;
1792             }
1793             env->hstatus = set_field(env->hstatus, HSTATUS_GVA, write_gva);
1794         }
1795 
1796         s = env->mstatus;
1797         s = set_field(s, MSTATUS_SPIE, get_field(s, MSTATUS_SIE));
1798         s = set_field(s, MSTATUS_SPP, env->priv);
1799         s = set_field(s, MSTATUS_SIE, 0);
1800         env->mstatus = s;
1801         env->scause = cause | ((target_ulong)async << (TARGET_LONG_BITS - 1));
1802         env->sepc = env->pc;
1803         env->stval = tval;
1804         env->htval = htval;
1805         env->htinst = tinst;
1806         env->pc = (env->stvec >> 2 << 2) +
1807                   ((async && (env->stvec & 3) == 1) ? cause * 4 : 0);
1808         riscv_cpu_set_mode(env, PRV_S, virt);
1809     } else {
1810         /* handle the trap in M-mode */
1811         if (riscv_has_ext(env, RVH)) {
1812             if (env->virt_enabled) {
1813                 riscv_cpu_swap_hypervisor_regs(env);
1814             }
1815             env->mstatus = set_field(env->mstatus, MSTATUS_MPV,
1816                                      env->virt_enabled);
1817             if (env->virt_enabled && tval) {
1818                 env->mstatus = set_field(env->mstatus, MSTATUS_GVA, 1);
1819             }
1820 
1821             mtval2 = env->guest_phys_fault_addr;
1822 
1823             /* Trapping to M mode, virt is disabled */
1824             virt = false;
1825         }
1826 
1827         s = env->mstatus;
1828         s = set_field(s, MSTATUS_MPIE, get_field(s, MSTATUS_MIE));
1829         s = set_field(s, MSTATUS_MPP, env->priv);
1830         s = set_field(s, MSTATUS_MIE, 0);
1831         env->mstatus = s;
1832         env->mcause = cause | ~(((target_ulong)-1) >> async);
1833         env->mepc = env->pc;
1834         env->mtval = tval;
1835         env->mtval2 = mtval2;
1836         env->mtinst = tinst;
1837         env->pc = (env->mtvec >> 2 << 2) +
1838                   ((async && (env->mtvec & 3) == 1) ? cause * 4 : 0);
1839         riscv_cpu_set_mode(env, PRV_M, virt);
1840     }
1841 
1842     /*
1843      * NOTE: it is not necessary to yield load reservations here. It is only
1844      * necessary for an SC from "another hart" to cause a load reservation
1845      * to be yielded. Refer to the memory consistency model section of the
1846      * RISC-V ISA Specification.
1847      */
1848 
1849     env->two_stage_lookup = false;
1850     env->two_stage_indirect_lookup = false;
1851 }
1852 
1853 #endif /* !CONFIG_USER_ONLY */
1854