xref: /openbmc/qemu/target/riscv/cpu_helper.c (revision 5f87dddb)
1 /*
2  * RISC-V CPU helpers for qemu.
3  *
4  * Copyright (c) 2016-2017 Sagar Karandikar, sagark@eecs.berkeley.edu
5  * Copyright (c) 2017-2018 SiFive, Inc.
6  *
7  * This program is free software; you can redistribute it and/or modify it
8  * under the terms and conditions of the GNU General Public License,
9  * version 2 or later, as published by the Free Software Foundation.
10  *
11  * This program is distributed in the hope it will be useful, but WITHOUT
12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
14  * more details.
15  *
16  * You should have received a copy of the GNU General Public License along with
17  * this program.  If not, see <http://www.gnu.org/licenses/>.
18  */
19 
20 #include "qemu/osdep.h"
21 #include "qemu/log.h"
22 #include "qemu/main-loop.h"
23 #include "cpu.h"
24 #include "internals.h"
25 #include "pmu.h"
26 #include "exec/exec-all.h"
27 #include "instmap.h"
28 #include "tcg/tcg-op.h"
29 #include "trace.h"
30 #include "semihosting/common-semi.h"
31 #include "sysemu/cpu-timers.h"
32 #include "cpu_bits.h"
33 #include "debug.h"
34 #include "tcg/oversized-guest.h"
35 
36 int riscv_cpu_mmu_index(CPURISCVState *env, bool ifetch)
37 {
38 #ifdef CONFIG_USER_ONLY
39     return 0;
40 #else
41     bool virt = env->virt_enabled;
42     int mode = env->priv;
43 
44     /* All priv -> mmu_idx mapping are here */
45     if (!ifetch) {
46         uint64_t status = env->mstatus;
47 
48         if (mode == PRV_M && get_field(status, MSTATUS_MPRV)) {
49             mode = get_field(env->mstatus, MSTATUS_MPP);
50             virt = get_field(env->mstatus, MSTATUS_MPV) &&
51                    (mode != PRV_M);
52             if (virt) {
53                 status = env->vsstatus;
54             }
55         }
56         if (mode == PRV_S && get_field(status, MSTATUS_SUM)) {
57             mode = MMUIdx_S_SUM;
58         }
59     }
60 
61     return mode | (virt ? MMU_2STAGE_BIT : 0);
62 #endif
63 }
64 
65 void cpu_get_tb_cpu_state(CPURISCVState *env, vaddr *pc,
66                           uint64_t *cs_base, uint32_t *pflags)
67 {
68     CPUState *cs = env_cpu(env);
69     RISCVCPU *cpu = RISCV_CPU(cs);
70     RISCVExtStatus fs, vs;
71     uint32_t flags = 0;
72 
73     *pc = env->xl == MXL_RV32 ? env->pc & UINT32_MAX : env->pc;
74     *cs_base = 0;
75 
76     if (cpu->cfg.ext_zve32f) {
77         /*
78          * If env->vl equals to VLMAX, we can use generic vector operation
79          * expanders (GVEC) to accerlate the vector operations.
80          * However, as LMUL could be a fractional number. The maximum
81          * vector size can be operated might be less than 8 bytes,
82          * which is not supported by GVEC. So we set vl_eq_vlmax flag to true
83          * only when maxsz >= 8 bytes.
84          */
85         uint32_t vlmax = vext_get_vlmax(cpu, env->vtype);
86         uint32_t sew = FIELD_EX64(env->vtype, VTYPE, VSEW);
87         uint32_t maxsz = vlmax << sew;
88         bool vl_eq_vlmax = (env->vstart == 0) && (vlmax == env->vl) &&
89                            (maxsz >= 8);
90         flags = FIELD_DP32(flags, TB_FLAGS, VILL, env->vill);
91         flags = FIELD_DP32(flags, TB_FLAGS, SEW, sew);
92         flags = FIELD_DP32(flags, TB_FLAGS, LMUL,
93                            FIELD_EX64(env->vtype, VTYPE, VLMUL));
94         flags = FIELD_DP32(flags, TB_FLAGS, VL_EQ_VLMAX, vl_eq_vlmax);
95         flags = FIELD_DP32(flags, TB_FLAGS, VTA,
96                            FIELD_EX64(env->vtype, VTYPE, VTA));
97         flags = FIELD_DP32(flags, TB_FLAGS, VMA,
98                            FIELD_EX64(env->vtype, VTYPE, VMA));
99         flags = FIELD_DP32(flags, TB_FLAGS, VSTART_EQ_ZERO, env->vstart == 0);
100     } else {
101         flags = FIELD_DP32(flags, TB_FLAGS, VILL, 1);
102     }
103 
104 #ifdef CONFIG_USER_ONLY
105     fs = EXT_STATUS_DIRTY;
106     vs = EXT_STATUS_DIRTY;
107 #else
108     flags = FIELD_DP32(flags, TB_FLAGS, PRIV, env->priv);
109 
110     flags |= cpu_mmu_index(env, 0);
111     fs = get_field(env->mstatus, MSTATUS_FS);
112     vs = get_field(env->mstatus, MSTATUS_VS);
113 
114     if (env->virt_enabled) {
115         flags = FIELD_DP32(flags, TB_FLAGS, VIRT_ENABLED, 1);
116         /*
117          * Merge DISABLED and !DIRTY states using MIN.
118          * We will set both fields when dirtying.
119          */
120         fs = MIN(fs, get_field(env->mstatus_hs, MSTATUS_FS));
121         vs = MIN(vs, get_field(env->mstatus_hs, MSTATUS_VS));
122     }
123 
124     /* With Zfinx, floating point is enabled/disabled by Smstateen. */
125     if (!riscv_has_ext(env, RVF)) {
126         fs = (smstateen_acc_ok(env, 0, SMSTATEEN0_FCSR) == RISCV_EXCP_NONE)
127              ? EXT_STATUS_DIRTY : EXT_STATUS_DISABLED;
128     }
129 
130     if (cpu->cfg.debug && !icount_enabled()) {
131         flags = FIELD_DP32(flags, TB_FLAGS, ITRIGGER, env->itrigger_enabled);
132     }
133 #endif
134 
135     flags = FIELD_DP32(flags, TB_FLAGS, FS, fs);
136     flags = FIELD_DP32(flags, TB_FLAGS, VS, vs);
137     flags = FIELD_DP32(flags, TB_FLAGS, XL, env->xl);
138     flags = FIELD_DP32(flags, TB_FLAGS, AXL, cpu_address_xl(env));
139     if (env->cur_pmmask != 0) {
140         flags = FIELD_DP32(flags, TB_FLAGS, PM_MASK_ENABLED, 1);
141     }
142     if (env->cur_pmbase != 0) {
143         flags = FIELD_DP32(flags, TB_FLAGS, PM_BASE_ENABLED, 1);
144     }
145 
146     *pflags = flags;
147 }
148 
149 void riscv_cpu_update_mask(CPURISCVState *env)
150 {
151     target_ulong mask = 0, base = 0;
152     RISCVMXL xl = env->xl;
153     /*
154      * TODO: Current RVJ spec does not specify
155      * how the extension interacts with XLEN.
156      */
157 #ifndef CONFIG_USER_ONLY
158     int mode = cpu_address_mode(env);
159     xl = cpu_get_xl(env, mode);
160     if (riscv_has_ext(env, RVJ)) {
161         switch (mode) {
162         case PRV_M:
163             if (env->mmte & M_PM_ENABLE) {
164                 mask = env->mpmmask;
165                 base = env->mpmbase;
166             }
167             break;
168         case PRV_S:
169             if (env->mmte & S_PM_ENABLE) {
170                 mask = env->spmmask;
171                 base = env->spmbase;
172             }
173             break;
174         case PRV_U:
175             if (env->mmte & U_PM_ENABLE) {
176                 mask = env->upmmask;
177                 base = env->upmbase;
178             }
179             break;
180         default:
181             g_assert_not_reached();
182         }
183     }
184 #endif
185     if (xl == MXL_RV32) {
186         env->cur_pmmask = mask & UINT32_MAX;
187         env->cur_pmbase = base & UINT32_MAX;
188     } else {
189         env->cur_pmmask = mask;
190         env->cur_pmbase = base;
191     }
192 }
193 
194 #ifndef CONFIG_USER_ONLY
195 
196 /*
197  * The HS-mode is allowed to configure priority only for the
198  * following VS-mode local interrupts:
199  *
200  * 0  (Reserved interrupt, reads as zero)
201  * 1  Supervisor software interrupt
202  * 4  (Reserved interrupt, reads as zero)
203  * 5  Supervisor timer interrupt
204  * 8  (Reserved interrupt, reads as zero)
205  * 13 (Reserved interrupt)
206  * 14 "
207  * 15 "
208  * 16 "
209  * 17 "
210  * 18 "
211  * 19 "
212  * 20 "
213  * 21 "
214  * 22 "
215  * 23 "
216  */
217 
218 static const int hviprio_index2irq[] = {
219     0, 1, 4, 5, 8, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23 };
220 static const int hviprio_index2rdzero[] = {
221     1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
222 
223 int riscv_cpu_hviprio_index2irq(int index, int *out_irq, int *out_rdzero)
224 {
225     if (index < 0 || ARRAY_SIZE(hviprio_index2irq) <= index) {
226         return -EINVAL;
227     }
228 
229     if (out_irq) {
230         *out_irq = hviprio_index2irq[index];
231     }
232 
233     if (out_rdzero) {
234         *out_rdzero = hviprio_index2rdzero[index];
235     }
236 
237     return 0;
238 }
239 
240 /*
241  * Default priorities of local interrupts are defined in the
242  * RISC-V Advanced Interrupt Architecture specification.
243  *
244  * ----------------------------------------------------------------
245  *  Default  |
246  *  Priority | Major Interrupt Numbers
247  * ----------------------------------------------------------------
248  *  Highest  | 47, 23, 46, 45, 22, 44,
249  *           | 43, 21, 42, 41, 20, 40
250  *           |
251  *           | 11 (0b),  3 (03),  7 (07)
252  *           |  9 (09),  1 (01),  5 (05)
253  *           | 12 (0c)
254  *           | 10 (0a),  2 (02),  6 (06)
255  *           |
256  *           | 39, 19, 38, 37, 18, 36,
257  *  Lowest   | 35, 17, 34, 33, 16, 32
258  * ----------------------------------------------------------------
259  */
260 static const uint8_t default_iprio[64] = {
261     /* Custom interrupts 48 to 63 */
262     [63] = IPRIO_MMAXIPRIO,
263     [62] = IPRIO_MMAXIPRIO,
264     [61] = IPRIO_MMAXIPRIO,
265     [60] = IPRIO_MMAXIPRIO,
266     [59] = IPRIO_MMAXIPRIO,
267     [58] = IPRIO_MMAXIPRIO,
268     [57] = IPRIO_MMAXIPRIO,
269     [56] = IPRIO_MMAXIPRIO,
270     [55] = IPRIO_MMAXIPRIO,
271     [54] = IPRIO_MMAXIPRIO,
272     [53] = IPRIO_MMAXIPRIO,
273     [52] = IPRIO_MMAXIPRIO,
274     [51] = IPRIO_MMAXIPRIO,
275     [50] = IPRIO_MMAXIPRIO,
276     [49] = IPRIO_MMAXIPRIO,
277     [48] = IPRIO_MMAXIPRIO,
278 
279     /* Custom interrupts 24 to 31 */
280     [31] = IPRIO_MMAXIPRIO,
281     [30] = IPRIO_MMAXIPRIO,
282     [29] = IPRIO_MMAXIPRIO,
283     [28] = IPRIO_MMAXIPRIO,
284     [27] = IPRIO_MMAXIPRIO,
285     [26] = IPRIO_MMAXIPRIO,
286     [25] = IPRIO_MMAXIPRIO,
287     [24] = IPRIO_MMAXIPRIO,
288 
289     [47] = IPRIO_DEFAULT_UPPER,
290     [23] = IPRIO_DEFAULT_UPPER + 1,
291     [46] = IPRIO_DEFAULT_UPPER + 2,
292     [45] = IPRIO_DEFAULT_UPPER + 3,
293     [22] = IPRIO_DEFAULT_UPPER + 4,
294     [44] = IPRIO_DEFAULT_UPPER + 5,
295 
296     [43] = IPRIO_DEFAULT_UPPER + 6,
297     [21] = IPRIO_DEFAULT_UPPER + 7,
298     [42] = IPRIO_DEFAULT_UPPER + 8,
299     [41] = IPRIO_DEFAULT_UPPER + 9,
300     [20] = IPRIO_DEFAULT_UPPER + 10,
301     [40] = IPRIO_DEFAULT_UPPER + 11,
302 
303     [11] = IPRIO_DEFAULT_M,
304     [3]  = IPRIO_DEFAULT_M + 1,
305     [7]  = IPRIO_DEFAULT_M + 2,
306 
307     [9]  = IPRIO_DEFAULT_S,
308     [1]  = IPRIO_DEFAULT_S + 1,
309     [5]  = IPRIO_DEFAULT_S + 2,
310 
311     [12] = IPRIO_DEFAULT_SGEXT,
312 
313     [10] = IPRIO_DEFAULT_VS,
314     [2]  = IPRIO_DEFAULT_VS + 1,
315     [6]  = IPRIO_DEFAULT_VS + 2,
316 
317     [39] = IPRIO_DEFAULT_LOWER,
318     [19] = IPRIO_DEFAULT_LOWER + 1,
319     [38] = IPRIO_DEFAULT_LOWER + 2,
320     [37] = IPRIO_DEFAULT_LOWER + 3,
321     [18] = IPRIO_DEFAULT_LOWER + 4,
322     [36] = IPRIO_DEFAULT_LOWER + 5,
323 
324     [35] = IPRIO_DEFAULT_LOWER + 6,
325     [17] = IPRIO_DEFAULT_LOWER + 7,
326     [34] = IPRIO_DEFAULT_LOWER + 8,
327     [33] = IPRIO_DEFAULT_LOWER + 9,
328     [16] = IPRIO_DEFAULT_LOWER + 10,
329     [32] = IPRIO_DEFAULT_LOWER + 11,
330 };
331 
332 uint8_t riscv_cpu_default_priority(int irq)
333 {
334     if (irq < 0 || irq > 63) {
335         return IPRIO_MMAXIPRIO;
336     }
337 
338     return default_iprio[irq] ? default_iprio[irq] : IPRIO_MMAXIPRIO;
339 };
340 
341 static int riscv_cpu_pending_to_irq(CPURISCVState *env,
342                                     int extirq, unsigned int extirq_def_prio,
343                                     uint64_t pending, uint8_t *iprio)
344 {
345     int irq, best_irq = RISCV_EXCP_NONE;
346     unsigned int prio, best_prio = UINT_MAX;
347 
348     if (!pending) {
349         return RISCV_EXCP_NONE;
350     }
351 
352     irq = ctz64(pending);
353     if (!((extirq == IRQ_M_EXT) ? riscv_cpu_cfg(env)->ext_smaia :
354                                   riscv_cpu_cfg(env)->ext_ssaia)) {
355         return irq;
356     }
357 
358     pending = pending >> irq;
359     while (pending) {
360         prio = iprio[irq];
361         if (!prio) {
362             if (irq == extirq) {
363                 prio = extirq_def_prio;
364             } else {
365                 prio = (riscv_cpu_default_priority(irq) < extirq_def_prio) ?
366                        1 : IPRIO_MMAXIPRIO;
367             }
368         }
369         if ((pending & 0x1) && (prio <= best_prio)) {
370             best_irq = irq;
371             best_prio = prio;
372         }
373         irq++;
374         pending = pending >> 1;
375     }
376 
377     return best_irq;
378 }
379 
380 uint64_t riscv_cpu_all_pending(CPURISCVState *env)
381 {
382     uint32_t gein = get_field(env->hstatus, HSTATUS_VGEIN);
383     uint64_t vsgein = (env->hgeip & (1ULL << gein)) ? MIP_VSEIP : 0;
384     uint64_t vstip = (env->vstime_irq) ? MIP_VSTIP : 0;
385 
386     return (env->mip | vsgein | vstip) & env->mie;
387 }
388 
389 int riscv_cpu_mirq_pending(CPURISCVState *env)
390 {
391     uint64_t irqs = riscv_cpu_all_pending(env) & ~env->mideleg &
392                     ~(MIP_SGEIP | MIP_VSSIP | MIP_VSTIP | MIP_VSEIP);
393 
394     return riscv_cpu_pending_to_irq(env, IRQ_M_EXT, IPRIO_DEFAULT_M,
395                                     irqs, env->miprio);
396 }
397 
398 int riscv_cpu_sirq_pending(CPURISCVState *env)
399 {
400     uint64_t irqs = riscv_cpu_all_pending(env) & env->mideleg &
401                     ~(MIP_VSSIP | MIP_VSTIP | MIP_VSEIP);
402 
403     return riscv_cpu_pending_to_irq(env, IRQ_S_EXT, IPRIO_DEFAULT_S,
404                                     irqs, env->siprio);
405 }
406 
407 int riscv_cpu_vsirq_pending(CPURISCVState *env)
408 {
409     uint64_t irqs = riscv_cpu_all_pending(env) & env->mideleg &
410                     (MIP_VSSIP | MIP_VSTIP | MIP_VSEIP);
411 
412     return riscv_cpu_pending_to_irq(env, IRQ_S_EXT, IPRIO_DEFAULT_S,
413                                     irqs >> 1, env->hviprio);
414 }
415 
416 static int riscv_cpu_local_irq_pending(CPURISCVState *env)
417 {
418     int virq;
419     uint64_t irqs, pending, mie, hsie, vsie;
420 
421     /* Determine interrupt enable state of all privilege modes */
422     if (env->virt_enabled) {
423         mie = 1;
424         hsie = 1;
425         vsie = (env->priv < PRV_S) ||
426                (env->priv == PRV_S && get_field(env->mstatus, MSTATUS_SIE));
427     } else {
428         mie = (env->priv < PRV_M) ||
429               (env->priv == PRV_M && get_field(env->mstatus, MSTATUS_MIE));
430         hsie = (env->priv < PRV_S) ||
431                (env->priv == PRV_S && get_field(env->mstatus, MSTATUS_SIE));
432         vsie = 0;
433     }
434 
435     /* Determine all pending interrupts */
436     pending = riscv_cpu_all_pending(env);
437 
438     /* Check M-mode interrupts */
439     irqs = pending & ~env->mideleg & -mie;
440     if (irqs) {
441         return riscv_cpu_pending_to_irq(env, IRQ_M_EXT, IPRIO_DEFAULT_M,
442                                         irqs, env->miprio);
443     }
444 
445     /* Check HS-mode interrupts */
446     irqs = pending & env->mideleg & ~env->hideleg & -hsie;
447     if (irqs) {
448         return riscv_cpu_pending_to_irq(env, IRQ_S_EXT, IPRIO_DEFAULT_S,
449                                         irqs, env->siprio);
450     }
451 
452     /* Check VS-mode interrupts */
453     irqs = pending & env->mideleg & env->hideleg & -vsie;
454     if (irqs) {
455         virq = riscv_cpu_pending_to_irq(env, IRQ_S_EXT, IPRIO_DEFAULT_S,
456                                         irqs >> 1, env->hviprio);
457         return (virq <= 0) ? virq : virq + 1;
458     }
459 
460     /* Indicate no pending interrupt */
461     return RISCV_EXCP_NONE;
462 }
463 
464 bool riscv_cpu_exec_interrupt(CPUState *cs, int interrupt_request)
465 {
466     if (interrupt_request & CPU_INTERRUPT_HARD) {
467         RISCVCPU *cpu = RISCV_CPU(cs);
468         CPURISCVState *env = &cpu->env;
469         int interruptno = riscv_cpu_local_irq_pending(env);
470         if (interruptno >= 0) {
471             cs->exception_index = RISCV_EXCP_INT_FLAG | interruptno;
472             riscv_cpu_do_interrupt(cs);
473             return true;
474         }
475     }
476     return false;
477 }
478 
479 /* Return true is floating point support is currently enabled */
480 bool riscv_cpu_fp_enabled(CPURISCVState *env)
481 {
482     if (env->mstatus & MSTATUS_FS) {
483         if (env->virt_enabled && !(env->mstatus_hs & MSTATUS_FS)) {
484             return false;
485         }
486         return true;
487     }
488 
489     return false;
490 }
491 
492 /* Return true is vector support is currently enabled */
493 bool riscv_cpu_vector_enabled(CPURISCVState *env)
494 {
495     if (env->mstatus & MSTATUS_VS) {
496         if (env->virt_enabled && !(env->mstatus_hs & MSTATUS_VS)) {
497             return false;
498         }
499         return true;
500     }
501 
502     return false;
503 }
504 
505 void riscv_cpu_swap_hypervisor_regs(CPURISCVState *env)
506 {
507     uint64_t mstatus_mask = MSTATUS_MXR | MSTATUS_SUM |
508                             MSTATUS_SPP | MSTATUS_SPIE | MSTATUS_SIE |
509                             MSTATUS64_UXL | MSTATUS_VS;
510 
511     if (riscv_has_ext(env, RVF)) {
512         mstatus_mask |= MSTATUS_FS;
513     }
514     bool current_virt = env->virt_enabled;
515 
516     g_assert(riscv_has_ext(env, RVH));
517 
518     if (current_virt) {
519         /* Current V=1 and we are about to change to V=0 */
520         env->vsstatus = env->mstatus & mstatus_mask;
521         env->mstatus &= ~mstatus_mask;
522         env->mstatus |= env->mstatus_hs;
523 
524         env->vstvec = env->stvec;
525         env->stvec = env->stvec_hs;
526 
527         env->vsscratch = env->sscratch;
528         env->sscratch = env->sscratch_hs;
529 
530         env->vsepc = env->sepc;
531         env->sepc = env->sepc_hs;
532 
533         env->vscause = env->scause;
534         env->scause = env->scause_hs;
535 
536         env->vstval = env->stval;
537         env->stval = env->stval_hs;
538 
539         env->vsatp = env->satp;
540         env->satp = env->satp_hs;
541     } else {
542         /* Current V=0 and we are about to change to V=1 */
543         env->mstatus_hs = env->mstatus & mstatus_mask;
544         env->mstatus &= ~mstatus_mask;
545         env->mstatus |= env->vsstatus;
546 
547         env->stvec_hs = env->stvec;
548         env->stvec = env->vstvec;
549 
550         env->sscratch_hs = env->sscratch;
551         env->sscratch = env->vsscratch;
552 
553         env->sepc_hs = env->sepc;
554         env->sepc = env->vsepc;
555 
556         env->scause_hs = env->scause;
557         env->scause = env->vscause;
558 
559         env->stval_hs = env->stval;
560         env->stval = env->vstval;
561 
562         env->satp_hs = env->satp;
563         env->satp = env->vsatp;
564     }
565 }
566 
567 target_ulong riscv_cpu_get_geilen(CPURISCVState *env)
568 {
569     if (!riscv_has_ext(env, RVH)) {
570         return 0;
571     }
572 
573     return env->geilen;
574 }
575 
576 void riscv_cpu_set_geilen(CPURISCVState *env, target_ulong geilen)
577 {
578     if (!riscv_has_ext(env, RVH)) {
579         return;
580     }
581 
582     if (geilen > (TARGET_LONG_BITS - 1)) {
583         return;
584     }
585 
586     env->geilen = geilen;
587 }
588 
589 /* This function can only be called to set virt when RVH is enabled */
590 void riscv_cpu_set_virt_enabled(CPURISCVState *env, bool enable)
591 {
592     /* Flush the TLB on all virt mode changes. */
593     if (env->virt_enabled != enable) {
594         tlb_flush(env_cpu(env));
595     }
596 
597     env->virt_enabled = enable;
598 
599     if (enable) {
600         /*
601          * The guest external interrupts from an interrupt controller are
602          * delivered only when the Guest/VM is running (i.e. V=1). This means
603          * any guest external interrupt which is triggered while the Guest/VM
604          * is not running (i.e. V=0) will be missed on QEMU resulting in guest
605          * with sluggish response to serial console input and other I/O events.
606          *
607          * To solve this, we check and inject interrupt after setting V=1.
608          */
609         riscv_cpu_update_mip(env, 0, 0);
610     }
611 }
612 
613 int riscv_cpu_claim_interrupts(RISCVCPU *cpu, uint64_t interrupts)
614 {
615     CPURISCVState *env = &cpu->env;
616     if (env->miclaim & interrupts) {
617         return -1;
618     } else {
619         env->miclaim |= interrupts;
620         return 0;
621     }
622 }
623 
624 uint64_t riscv_cpu_update_mip(CPURISCVState *env, uint64_t mask,
625                               uint64_t value)
626 {
627     CPUState *cs = env_cpu(env);
628     uint64_t gein, vsgein = 0, vstip = 0, old = env->mip;
629 
630     if (env->virt_enabled) {
631         gein = get_field(env->hstatus, HSTATUS_VGEIN);
632         vsgein = (env->hgeip & (1ULL << gein)) ? MIP_VSEIP : 0;
633     }
634 
635     vstip = env->vstime_irq ? MIP_VSTIP : 0;
636 
637     QEMU_IOTHREAD_LOCK_GUARD();
638 
639     env->mip = (env->mip & ~mask) | (value & mask);
640 
641     if (env->mip | vsgein | vstip) {
642         cpu_interrupt(cs, CPU_INTERRUPT_HARD);
643     } else {
644         cpu_reset_interrupt(cs, CPU_INTERRUPT_HARD);
645     }
646 
647     return old;
648 }
649 
650 void riscv_cpu_set_rdtime_fn(CPURISCVState *env, uint64_t (*fn)(void *),
651                              void *arg)
652 {
653     env->rdtime_fn = fn;
654     env->rdtime_fn_arg = arg;
655 }
656 
657 void riscv_cpu_set_aia_ireg_rmw_fn(CPURISCVState *env, uint32_t priv,
658                                    int (*rmw_fn)(void *arg,
659                                                  target_ulong reg,
660                                                  target_ulong *val,
661                                                  target_ulong new_val,
662                                                  target_ulong write_mask),
663                                    void *rmw_fn_arg)
664 {
665     if (priv <= PRV_M) {
666         env->aia_ireg_rmw_fn[priv] = rmw_fn;
667         env->aia_ireg_rmw_fn_arg[priv] = rmw_fn_arg;
668     }
669 }
670 
671 void riscv_cpu_set_mode(CPURISCVState *env, target_ulong newpriv)
672 {
673     g_assert(newpriv <= PRV_M && newpriv != PRV_RESERVED);
674 
675     if (icount_enabled() && newpriv != env->priv) {
676         riscv_itrigger_update_priv(env);
677     }
678     /* tlb_flush is unnecessary as mode is contained in mmu_idx */
679     env->priv = newpriv;
680     env->xl = cpu_recompute_xl(env);
681     riscv_cpu_update_mask(env);
682 
683     /*
684      * Clear the load reservation - otherwise a reservation placed in one
685      * context/process can be used by another, resulting in an SC succeeding
686      * incorrectly. Version 2.2 of the ISA specification explicitly requires
687      * this behaviour, while later revisions say that the kernel "should" use
688      * an SC instruction to force the yielding of a load reservation on a
689      * preemptive context switch. As a result, do both.
690      */
691     env->load_res = -1;
692 }
693 
694 /*
695  * get_physical_address_pmp - check PMP permission for this physical address
696  *
697  * Match the PMP region and check permission for this physical address and it's
698  * TLB page. Returns 0 if the permission checking was successful
699  *
700  * @env: CPURISCVState
701  * @prot: The returned protection attributes
702  * @addr: The physical address to be checked permission
703  * @access_type: The type of MMU access
704  * @mode: Indicates current privilege level.
705  */
706 static int get_physical_address_pmp(CPURISCVState *env, int *prot, hwaddr addr,
707                                     int size, MMUAccessType access_type,
708                                     int mode)
709 {
710     pmp_priv_t pmp_priv;
711     bool pmp_has_privs;
712 
713     if (!riscv_cpu_cfg(env)->pmp) {
714         *prot = PAGE_READ | PAGE_WRITE | PAGE_EXEC;
715         return TRANSLATE_SUCCESS;
716     }
717 
718     pmp_has_privs = pmp_hart_has_privs(env, addr, size, 1 << access_type,
719                                        &pmp_priv, mode);
720     if (!pmp_has_privs) {
721         *prot = 0;
722         return TRANSLATE_PMP_FAIL;
723     }
724 
725     *prot = pmp_priv_to_page_prot(pmp_priv);
726 
727     return TRANSLATE_SUCCESS;
728 }
729 
730 /*
731  * get_physical_address - get the physical address for this virtual address
732  *
733  * Do a page table walk to obtain the physical address corresponding to a
734  * virtual address. Returns 0 if the translation was successful
735  *
736  * Adapted from Spike's mmu_t::translate and mmu_t::walk
737  *
738  * @env: CPURISCVState
739  * @physical: This will be set to the calculated physical address
740  * @prot: The returned protection attributes
741  * @addr: The virtual address or guest physical address to be translated
742  * @fault_pte_addr: If not NULL, this will be set to fault pte address
743  *                  when a error occurs on pte address translation.
744  *                  This will already be shifted to match htval.
745  * @access_type: The type of MMU access
746  * @mmu_idx: Indicates current privilege level
747  * @first_stage: Are we in first stage translation?
748  *               Second stage is used for hypervisor guest translation
749  * @two_stage: Are we going to perform two stage translation
750  * @is_debug: Is this access from a debugger or the monitor?
751  */
752 static int get_physical_address(CPURISCVState *env, hwaddr *physical,
753                                 int *ret_prot, vaddr addr,
754                                 target_ulong *fault_pte_addr,
755                                 int access_type, int mmu_idx,
756                                 bool first_stage, bool two_stage,
757                                 bool is_debug)
758 {
759     /*
760      * NOTE: the env->pc value visible here will not be
761      * correct, but the value visible to the exception handler
762      * (riscv_cpu_do_interrupt) is correct
763      */
764     MemTxResult res;
765     MemTxAttrs attrs = MEMTXATTRS_UNSPECIFIED;
766     int mode = mmuidx_priv(mmu_idx);
767     bool use_background = false;
768     hwaddr ppn;
769     int napot_bits = 0;
770     target_ulong napot_mask;
771 
772     /*
773      * Check if we should use the background registers for the two
774      * stage translation. We don't need to check if we actually need
775      * two stage translation as that happened before this function
776      * was called. Background registers will be used if the guest has
777      * forced a two stage translation to be on (in HS or M mode).
778      */
779     if (!env->virt_enabled && two_stage) {
780         use_background = true;
781     }
782 
783     if (mode == PRV_M || !riscv_cpu_cfg(env)->mmu) {
784         *physical = addr;
785         *ret_prot = PAGE_READ | PAGE_WRITE | PAGE_EXEC;
786         return TRANSLATE_SUCCESS;
787     }
788 
789     *ret_prot = 0;
790 
791     hwaddr base;
792     int levels, ptidxbits, ptesize, vm, widened;
793 
794     if (first_stage == true) {
795         if (use_background) {
796             if (riscv_cpu_mxl(env) == MXL_RV32) {
797                 base = (hwaddr)get_field(env->vsatp, SATP32_PPN) << PGSHIFT;
798                 vm = get_field(env->vsatp, SATP32_MODE);
799             } else {
800                 base = (hwaddr)get_field(env->vsatp, SATP64_PPN) << PGSHIFT;
801                 vm = get_field(env->vsatp, SATP64_MODE);
802             }
803         } else {
804             if (riscv_cpu_mxl(env) == MXL_RV32) {
805                 base = (hwaddr)get_field(env->satp, SATP32_PPN) << PGSHIFT;
806                 vm = get_field(env->satp, SATP32_MODE);
807             } else {
808                 base = (hwaddr)get_field(env->satp, SATP64_PPN) << PGSHIFT;
809                 vm = get_field(env->satp, SATP64_MODE);
810             }
811         }
812         widened = 0;
813     } else {
814         if (riscv_cpu_mxl(env) == MXL_RV32) {
815             base = (hwaddr)get_field(env->hgatp, SATP32_PPN) << PGSHIFT;
816             vm = get_field(env->hgatp, SATP32_MODE);
817         } else {
818             base = (hwaddr)get_field(env->hgatp, SATP64_PPN) << PGSHIFT;
819             vm = get_field(env->hgatp, SATP64_MODE);
820         }
821         widened = 2;
822     }
823 
824     switch (vm) {
825     case VM_1_10_SV32:
826       levels = 2; ptidxbits = 10; ptesize = 4; break;
827     case VM_1_10_SV39:
828       levels = 3; ptidxbits = 9; ptesize = 8; break;
829     case VM_1_10_SV48:
830       levels = 4; ptidxbits = 9; ptesize = 8; break;
831     case VM_1_10_SV57:
832       levels = 5; ptidxbits = 9; ptesize = 8; break;
833     case VM_1_10_MBARE:
834         *physical = addr;
835         *ret_prot = PAGE_READ | PAGE_WRITE | PAGE_EXEC;
836         return TRANSLATE_SUCCESS;
837     default:
838       g_assert_not_reached();
839     }
840 
841     CPUState *cs = env_cpu(env);
842     int va_bits = PGSHIFT + levels * ptidxbits + widened;
843 
844     if (first_stage == true) {
845         target_ulong mask, masked_msbs;
846 
847         if (TARGET_LONG_BITS > (va_bits - 1)) {
848             mask = (1L << (TARGET_LONG_BITS - (va_bits - 1))) - 1;
849         } else {
850             mask = 0;
851         }
852         masked_msbs = (addr >> (va_bits - 1)) & mask;
853 
854         if (masked_msbs != 0 && masked_msbs != mask) {
855             return TRANSLATE_FAIL;
856         }
857     } else {
858         if (vm != VM_1_10_SV32 && addr >> va_bits != 0) {
859             return TRANSLATE_FAIL;
860         }
861     }
862 
863     bool pbmte = env->menvcfg & MENVCFG_PBMTE;
864     bool adue = env->menvcfg & MENVCFG_ADUE;
865 
866     if (first_stage && two_stage && env->virt_enabled) {
867         pbmte = pbmte && (env->henvcfg & HENVCFG_PBMTE);
868         adue = adue && (env->henvcfg & HENVCFG_ADUE);
869     }
870 
871     int ptshift = (levels - 1) * ptidxbits;
872     target_ulong pte;
873     hwaddr pte_addr;
874     int i;
875 
876 #if !TCG_OVERSIZED_GUEST
877 restart:
878 #endif
879     for (i = 0; i < levels; i++, ptshift -= ptidxbits) {
880         target_ulong idx;
881         if (i == 0) {
882             idx = (addr >> (PGSHIFT + ptshift)) &
883                            ((1 << (ptidxbits + widened)) - 1);
884         } else {
885             idx = (addr >> (PGSHIFT + ptshift)) &
886                            ((1 << ptidxbits) - 1);
887         }
888 
889         /* check that physical address of PTE is legal */
890 
891         if (two_stage && first_stage) {
892             int vbase_prot;
893             hwaddr vbase;
894 
895             /* Do the second stage translation on the base PTE address. */
896             int vbase_ret = get_physical_address(env, &vbase, &vbase_prot,
897                                                  base, NULL, MMU_DATA_LOAD,
898                                                  MMUIdx_U, false, true,
899                                                  is_debug);
900 
901             if (vbase_ret != TRANSLATE_SUCCESS) {
902                 if (fault_pte_addr) {
903                     *fault_pte_addr = (base + idx * ptesize) >> 2;
904                 }
905                 return TRANSLATE_G_STAGE_FAIL;
906             }
907 
908             pte_addr = vbase + idx * ptesize;
909         } else {
910             pte_addr = base + idx * ptesize;
911         }
912 
913         int pmp_prot;
914         int pmp_ret = get_physical_address_pmp(env, &pmp_prot, pte_addr,
915                                                sizeof(target_ulong),
916                                                MMU_DATA_LOAD, PRV_S);
917         if (pmp_ret != TRANSLATE_SUCCESS) {
918             return TRANSLATE_PMP_FAIL;
919         }
920 
921         if (riscv_cpu_mxl(env) == MXL_RV32) {
922             pte = address_space_ldl(cs->as, pte_addr, attrs, &res);
923         } else {
924             pte = address_space_ldq(cs->as, pte_addr, attrs, &res);
925         }
926 
927         if (res != MEMTX_OK) {
928             return TRANSLATE_FAIL;
929         }
930 
931         if (riscv_cpu_sxl(env) == MXL_RV32) {
932             ppn = pte >> PTE_PPN_SHIFT;
933         } else {
934             if (pte & PTE_RESERVED) {
935                 return TRANSLATE_FAIL;
936             }
937 
938             if (!pbmte && (pte & PTE_PBMT)) {
939                 return TRANSLATE_FAIL;
940             }
941 
942             if (!riscv_cpu_cfg(env)->ext_svnapot && (pte & PTE_N)) {
943                 return TRANSLATE_FAIL;
944             }
945 
946             ppn = (pte & (target_ulong)PTE_PPN_MASK) >> PTE_PPN_SHIFT;
947         }
948 
949         if (!(pte & PTE_V)) {
950             /* Invalid PTE */
951             return TRANSLATE_FAIL;
952         }
953         if (pte & (PTE_R | PTE_W | PTE_X)) {
954             goto leaf;
955         }
956 
957         /* Inner PTE, continue walking */
958         if (pte & (PTE_D | PTE_A | PTE_U | PTE_ATTR)) {
959             return TRANSLATE_FAIL;
960         }
961         base = ppn << PGSHIFT;
962     }
963 
964     /* No leaf pte at any translation level. */
965     return TRANSLATE_FAIL;
966 
967  leaf:
968     if (ppn & ((1ULL << ptshift) - 1)) {
969         /* Misaligned PPN */
970         return TRANSLATE_FAIL;
971     }
972     if (!pbmte && (pte & PTE_PBMT)) {
973         /* Reserved without Svpbmt. */
974         return TRANSLATE_FAIL;
975     }
976 
977     /* Check for reserved combinations of RWX flags. */
978     switch (pte & (PTE_R | PTE_W | PTE_X)) {
979     case PTE_W:
980     case PTE_W | PTE_X:
981         return TRANSLATE_FAIL;
982     }
983 
984     int prot = 0;
985     if (pte & PTE_R) {
986         prot |= PAGE_READ;
987     }
988     if (pte & PTE_W) {
989         prot |= PAGE_WRITE;
990     }
991     if (pte & PTE_X) {
992         bool mxr;
993 
994         if (first_stage == true) {
995             mxr = get_field(env->mstatus, MSTATUS_MXR);
996         } else {
997             mxr = get_field(env->vsstatus, MSTATUS_MXR);
998         }
999         if (mxr) {
1000             prot |= PAGE_READ;
1001         }
1002         prot |= PAGE_EXEC;
1003     }
1004 
1005     if (pte & PTE_U) {
1006         if (mode != PRV_U) {
1007             if (!mmuidx_sum(mmu_idx)) {
1008                 return TRANSLATE_FAIL;
1009             }
1010             /* SUM allows only read+write, not execute. */
1011             prot &= PAGE_READ | PAGE_WRITE;
1012         }
1013     } else if (mode != PRV_S) {
1014         /* Supervisor PTE flags when not S mode */
1015         return TRANSLATE_FAIL;
1016     }
1017 
1018     if (!((prot >> access_type) & 1)) {
1019         /* Access check failed */
1020         return TRANSLATE_FAIL;
1021     }
1022 
1023     /* If necessary, set accessed and dirty bits. */
1024     target_ulong updated_pte = pte | PTE_A |
1025                 (access_type == MMU_DATA_STORE ? PTE_D : 0);
1026 
1027     /* Page table updates need to be atomic with MTTCG enabled */
1028     if (updated_pte != pte && !is_debug) {
1029         if (!adue) {
1030             return TRANSLATE_FAIL;
1031         }
1032 
1033         /*
1034          * - if accessed or dirty bits need updating, and the PTE is
1035          *   in RAM, then we do so atomically with a compare and swap.
1036          * - if the PTE is in IO space or ROM, then it can't be updated
1037          *   and we return TRANSLATE_FAIL.
1038          * - if the PTE changed by the time we went to update it, then
1039          *   it is no longer valid and we must re-walk the page table.
1040          */
1041         MemoryRegion *mr;
1042         hwaddr l = sizeof(target_ulong), addr1;
1043         mr = address_space_translate(cs->as, pte_addr, &addr1, &l,
1044                                      false, MEMTXATTRS_UNSPECIFIED);
1045         if (memory_region_is_ram(mr)) {
1046             target_ulong *pte_pa = qemu_map_ram_ptr(mr->ram_block, addr1);
1047 #if TCG_OVERSIZED_GUEST
1048             /*
1049              * MTTCG is not enabled on oversized TCG guests so
1050              * page table updates do not need to be atomic
1051              */
1052             *pte_pa = pte = updated_pte;
1053 #else
1054             target_ulong old_pte = qatomic_cmpxchg(pte_pa, pte, updated_pte);
1055             if (old_pte != pte) {
1056                 goto restart;
1057             }
1058             pte = updated_pte;
1059 #endif
1060         } else {
1061             /*
1062              * Misconfigured PTE in ROM (AD bits are not preset) or
1063              * PTE is in IO space and can't be updated atomically.
1064              */
1065             return TRANSLATE_FAIL;
1066         }
1067     }
1068 
1069     /* For superpage mappings, make a fake leaf PTE for the TLB's benefit. */
1070     target_ulong vpn = addr >> PGSHIFT;
1071 
1072     if (riscv_cpu_cfg(env)->ext_svnapot && (pte & PTE_N)) {
1073         napot_bits = ctzl(ppn) + 1;
1074         if ((i != (levels - 1)) || (napot_bits != 4)) {
1075             return TRANSLATE_FAIL;
1076         }
1077     }
1078 
1079     napot_mask = (1 << napot_bits) - 1;
1080     *physical = (((ppn & ~napot_mask) | (vpn & napot_mask) |
1081                   (vpn & (((target_ulong)1 << ptshift) - 1))
1082                  ) << PGSHIFT) | (addr & ~TARGET_PAGE_MASK);
1083 
1084     /*
1085      * Remove write permission unless this is a store, or the page is
1086      * already dirty, so that we TLB miss on later writes to update
1087      * the dirty bit.
1088      */
1089     if (access_type != MMU_DATA_STORE && !(pte & PTE_D)) {
1090         prot &= ~PAGE_WRITE;
1091     }
1092     *ret_prot = prot;
1093 
1094     return TRANSLATE_SUCCESS;
1095 }
1096 
1097 static void raise_mmu_exception(CPURISCVState *env, target_ulong address,
1098                                 MMUAccessType access_type, bool pmp_violation,
1099                                 bool first_stage, bool two_stage,
1100                                 bool two_stage_indirect)
1101 {
1102     CPUState *cs = env_cpu(env);
1103     int page_fault_exceptions, vm;
1104     uint64_t stap_mode;
1105 
1106     if (riscv_cpu_mxl(env) == MXL_RV32) {
1107         stap_mode = SATP32_MODE;
1108     } else {
1109         stap_mode = SATP64_MODE;
1110     }
1111 
1112     if (first_stage) {
1113         vm = get_field(env->satp, stap_mode);
1114     } else {
1115         vm = get_field(env->hgatp, stap_mode);
1116     }
1117 
1118     page_fault_exceptions = vm != VM_1_10_MBARE && !pmp_violation;
1119 
1120     switch (access_type) {
1121     case MMU_INST_FETCH:
1122         if (env->virt_enabled && !first_stage) {
1123             cs->exception_index = RISCV_EXCP_INST_GUEST_PAGE_FAULT;
1124         } else {
1125             cs->exception_index = page_fault_exceptions ?
1126                 RISCV_EXCP_INST_PAGE_FAULT : RISCV_EXCP_INST_ACCESS_FAULT;
1127         }
1128         break;
1129     case MMU_DATA_LOAD:
1130         if (two_stage && !first_stage) {
1131             cs->exception_index = RISCV_EXCP_LOAD_GUEST_ACCESS_FAULT;
1132         } else {
1133             cs->exception_index = page_fault_exceptions ?
1134                 RISCV_EXCP_LOAD_PAGE_FAULT : RISCV_EXCP_LOAD_ACCESS_FAULT;
1135         }
1136         break;
1137     case MMU_DATA_STORE:
1138         if (two_stage && !first_stage) {
1139             cs->exception_index = RISCV_EXCP_STORE_GUEST_AMO_ACCESS_FAULT;
1140         } else {
1141             cs->exception_index = page_fault_exceptions ?
1142                 RISCV_EXCP_STORE_PAGE_FAULT :
1143                 RISCV_EXCP_STORE_AMO_ACCESS_FAULT;
1144         }
1145         break;
1146     default:
1147         g_assert_not_reached();
1148     }
1149     env->badaddr = address;
1150     env->two_stage_lookup = two_stage;
1151     env->two_stage_indirect_lookup = two_stage_indirect;
1152 }
1153 
1154 hwaddr riscv_cpu_get_phys_page_debug(CPUState *cs, vaddr addr)
1155 {
1156     RISCVCPU *cpu = RISCV_CPU(cs);
1157     CPURISCVState *env = &cpu->env;
1158     hwaddr phys_addr;
1159     int prot;
1160     int mmu_idx = cpu_mmu_index(&cpu->env, false);
1161 
1162     if (get_physical_address(env, &phys_addr, &prot, addr, NULL, 0, mmu_idx,
1163                              true, env->virt_enabled, true)) {
1164         return -1;
1165     }
1166 
1167     if (env->virt_enabled) {
1168         if (get_physical_address(env, &phys_addr, &prot, phys_addr, NULL,
1169                                  0, mmu_idx, false, true, true)) {
1170             return -1;
1171         }
1172     }
1173 
1174     return phys_addr & TARGET_PAGE_MASK;
1175 }
1176 
1177 void riscv_cpu_do_transaction_failed(CPUState *cs, hwaddr physaddr,
1178                                      vaddr addr, unsigned size,
1179                                      MMUAccessType access_type,
1180                                      int mmu_idx, MemTxAttrs attrs,
1181                                      MemTxResult response, uintptr_t retaddr)
1182 {
1183     RISCVCPU *cpu = RISCV_CPU(cs);
1184     CPURISCVState *env = &cpu->env;
1185 
1186     if (access_type == MMU_DATA_STORE) {
1187         cs->exception_index = RISCV_EXCP_STORE_AMO_ACCESS_FAULT;
1188     } else if (access_type == MMU_DATA_LOAD) {
1189         cs->exception_index = RISCV_EXCP_LOAD_ACCESS_FAULT;
1190     } else {
1191         cs->exception_index = RISCV_EXCP_INST_ACCESS_FAULT;
1192     }
1193 
1194     env->badaddr = addr;
1195     env->two_stage_lookup = mmuidx_2stage(mmu_idx);
1196     env->two_stage_indirect_lookup = false;
1197     cpu_loop_exit_restore(cs, retaddr);
1198 }
1199 
1200 void riscv_cpu_do_unaligned_access(CPUState *cs, vaddr addr,
1201                                    MMUAccessType access_type, int mmu_idx,
1202                                    uintptr_t retaddr)
1203 {
1204     RISCVCPU *cpu = RISCV_CPU(cs);
1205     CPURISCVState *env = &cpu->env;
1206     switch (access_type) {
1207     case MMU_INST_FETCH:
1208         cs->exception_index = RISCV_EXCP_INST_ADDR_MIS;
1209         break;
1210     case MMU_DATA_LOAD:
1211         cs->exception_index = RISCV_EXCP_LOAD_ADDR_MIS;
1212         break;
1213     case MMU_DATA_STORE:
1214         cs->exception_index = RISCV_EXCP_STORE_AMO_ADDR_MIS;
1215         break;
1216     default:
1217         g_assert_not_reached();
1218     }
1219     env->badaddr = addr;
1220     env->two_stage_lookup = mmuidx_2stage(mmu_idx);
1221     env->two_stage_indirect_lookup = false;
1222     cpu_loop_exit_restore(cs, retaddr);
1223 }
1224 
1225 
1226 static void pmu_tlb_fill_incr_ctr(RISCVCPU *cpu, MMUAccessType access_type)
1227 {
1228     enum riscv_pmu_event_idx pmu_event_type;
1229 
1230     switch (access_type) {
1231     case MMU_INST_FETCH:
1232         pmu_event_type = RISCV_PMU_EVENT_CACHE_ITLB_PREFETCH_MISS;
1233         break;
1234     case MMU_DATA_LOAD:
1235         pmu_event_type = RISCV_PMU_EVENT_CACHE_DTLB_READ_MISS;
1236         break;
1237     case MMU_DATA_STORE:
1238         pmu_event_type = RISCV_PMU_EVENT_CACHE_DTLB_WRITE_MISS;
1239         break;
1240     default:
1241         return;
1242     }
1243 
1244     riscv_pmu_incr_ctr(cpu, pmu_event_type);
1245 }
1246 
1247 bool riscv_cpu_tlb_fill(CPUState *cs, vaddr address, int size,
1248                         MMUAccessType access_type, int mmu_idx,
1249                         bool probe, uintptr_t retaddr)
1250 {
1251     RISCVCPU *cpu = RISCV_CPU(cs);
1252     CPURISCVState *env = &cpu->env;
1253     vaddr im_address;
1254     hwaddr pa = 0;
1255     int prot, prot2, prot_pmp;
1256     bool pmp_violation = false;
1257     bool first_stage_error = true;
1258     bool two_stage_lookup = mmuidx_2stage(mmu_idx);
1259     bool two_stage_indirect_error = false;
1260     int ret = TRANSLATE_FAIL;
1261     int mode = mmu_idx;
1262     /* default TLB page size */
1263     target_ulong tlb_size = TARGET_PAGE_SIZE;
1264 
1265     env->guest_phys_fault_addr = 0;
1266 
1267     qemu_log_mask(CPU_LOG_MMU, "%s ad %" VADDR_PRIx " rw %d mmu_idx %d\n",
1268                   __func__, address, access_type, mmu_idx);
1269 
1270     pmu_tlb_fill_incr_ctr(cpu, access_type);
1271     if (two_stage_lookup) {
1272         /* Two stage lookup */
1273         ret = get_physical_address(env, &pa, &prot, address,
1274                                    &env->guest_phys_fault_addr, access_type,
1275                                    mmu_idx, true, true, false);
1276 
1277         /*
1278          * A G-stage exception may be triggered during two state lookup.
1279          * And the env->guest_phys_fault_addr has already been set in
1280          * get_physical_address().
1281          */
1282         if (ret == TRANSLATE_G_STAGE_FAIL) {
1283             first_stage_error = false;
1284             two_stage_indirect_error = true;
1285         }
1286 
1287         qemu_log_mask(CPU_LOG_MMU,
1288                       "%s 1st-stage address=%" VADDR_PRIx " ret %d physical "
1289                       HWADDR_FMT_plx " prot %d\n",
1290                       __func__, address, ret, pa, prot);
1291 
1292         if (ret == TRANSLATE_SUCCESS) {
1293             /* Second stage lookup */
1294             im_address = pa;
1295 
1296             ret = get_physical_address(env, &pa, &prot2, im_address, NULL,
1297                                        access_type, MMUIdx_U, false, true,
1298                                        false);
1299 
1300             qemu_log_mask(CPU_LOG_MMU,
1301                           "%s 2nd-stage address=%" VADDR_PRIx
1302                           " ret %d physical "
1303                           HWADDR_FMT_plx " prot %d\n",
1304                           __func__, im_address, ret, pa, prot2);
1305 
1306             prot &= prot2;
1307 
1308             if (ret == TRANSLATE_SUCCESS) {
1309                 ret = get_physical_address_pmp(env, &prot_pmp, pa,
1310                                                size, access_type, mode);
1311                 tlb_size = pmp_get_tlb_size(env, pa);
1312 
1313                 qemu_log_mask(CPU_LOG_MMU,
1314                               "%s PMP address=" HWADDR_FMT_plx " ret %d prot"
1315                               " %d tlb_size " TARGET_FMT_lu "\n",
1316                               __func__, pa, ret, prot_pmp, tlb_size);
1317 
1318                 prot &= prot_pmp;
1319             }
1320 
1321             if (ret != TRANSLATE_SUCCESS) {
1322                 /*
1323                  * Guest physical address translation failed, this is a HS
1324                  * level exception
1325                  */
1326                 first_stage_error = false;
1327                 env->guest_phys_fault_addr = (im_address |
1328                                               (address &
1329                                                (TARGET_PAGE_SIZE - 1))) >> 2;
1330             }
1331         }
1332     } else {
1333         /* Single stage lookup */
1334         ret = get_physical_address(env, &pa, &prot, address, NULL,
1335                                    access_type, mmu_idx, true, false, false);
1336 
1337         qemu_log_mask(CPU_LOG_MMU,
1338                       "%s address=%" VADDR_PRIx " ret %d physical "
1339                       HWADDR_FMT_plx " prot %d\n",
1340                       __func__, address, ret, pa, prot);
1341 
1342         if (ret == TRANSLATE_SUCCESS) {
1343             ret = get_physical_address_pmp(env, &prot_pmp, pa,
1344                                            size, access_type, mode);
1345             tlb_size = pmp_get_tlb_size(env, pa);
1346 
1347             qemu_log_mask(CPU_LOG_MMU,
1348                           "%s PMP address=" HWADDR_FMT_plx " ret %d prot"
1349                           " %d tlb_size " TARGET_FMT_lu "\n",
1350                           __func__, pa, ret, prot_pmp, tlb_size);
1351 
1352             prot &= prot_pmp;
1353         }
1354     }
1355 
1356     if (ret == TRANSLATE_PMP_FAIL) {
1357         pmp_violation = true;
1358     }
1359 
1360     if (ret == TRANSLATE_SUCCESS) {
1361         tlb_set_page(cs, address & ~(tlb_size - 1), pa & ~(tlb_size - 1),
1362                      prot, mmu_idx, tlb_size);
1363         return true;
1364     } else if (probe) {
1365         return false;
1366     } else {
1367         raise_mmu_exception(env, address, access_type, pmp_violation,
1368                             first_stage_error, two_stage_lookup,
1369                             two_stage_indirect_error);
1370         cpu_loop_exit_restore(cs, retaddr);
1371     }
1372 
1373     return true;
1374 }
1375 
1376 static target_ulong riscv_transformed_insn(CPURISCVState *env,
1377                                            target_ulong insn,
1378                                            target_ulong taddr)
1379 {
1380     target_ulong xinsn = 0;
1381     target_ulong access_rs1 = 0, access_imm = 0, access_size = 0;
1382 
1383     /*
1384      * Only Quadrant 0 and Quadrant 2 of RVC instruction space need to
1385      * be uncompressed. The Quadrant 1 of RVC instruction space need
1386      * not be transformed because these instructions won't generate
1387      * any load/store trap.
1388      */
1389 
1390     if ((insn & 0x3) != 0x3) {
1391         /* Transform 16bit instruction into 32bit instruction */
1392         switch (GET_C_OP(insn)) {
1393         case OPC_RISC_C_OP_QUAD0: /* Quadrant 0 */
1394             switch (GET_C_FUNC(insn)) {
1395             case OPC_RISC_C_FUNC_FLD_LQ:
1396                 if (riscv_cpu_xlen(env) != 128) { /* C.FLD (RV32/64) */
1397                     xinsn = OPC_RISC_FLD;
1398                     xinsn = SET_RD(xinsn, GET_C_RS2S(insn));
1399                     access_rs1 = GET_C_RS1S(insn);
1400                     access_imm = GET_C_LD_IMM(insn);
1401                     access_size = 8;
1402                 }
1403                 break;
1404             case OPC_RISC_C_FUNC_LW: /* C.LW */
1405                 xinsn = OPC_RISC_LW;
1406                 xinsn = SET_RD(xinsn, GET_C_RS2S(insn));
1407                 access_rs1 = GET_C_RS1S(insn);
1408                 access_imm = GET_C_LW_IMM(insn);
1409                 access_size = 4;
1410                 break;
1411             case OPC_RISC_C_FUNC_FLW_LD:
1412                 if (riscv_cpu_xlen(env) == 32) { /* C.FLW (RV32) */
1413                     xinsn = OPC_RISC_FLW;
1414                     xinsn = SET_RD(xinsn, GET_C_RS2S(insn));
1415                     access_rs1 = GET_C_RS1S(insn);
1416                     access_imm = GET_C_LW_IMM(insn);
1417                     access_size = 4;
1418                 } else { /* C.LD (RV64/RV128) */
1419                     xinsn = OPC_RISC_LD;
1420                     xinsn = SET_RD(xinsn, GET_C_RS2S(insn));
1421                     access_rs1 = GET_C_RS1S(insn);
1422                     access_imm = GET_C_LD_IMM(insn);
1423                     access_size = 8;
1424                 }
1425                 break;
1426             case OPC_RISC_C_FUNC_FSD_SQ:
1427                 if (riscv_cpu_xlen(env) != 128) { /* C.FSD (RV32/64) */
1428                     xinsn = OPC_RISC_FSD;
1429                     xinsn = SET_RS2(xinsn, GET_C_RS2S(insn));
1430                     access_rs1 = GET_C_RS1S(insn);
1431                     access_imm = GET_C_SD_IMM(insn);
1432                     access_size = 8;
1433                 }
1434                 break;
1435             case OPC_RISC_C_FUNC_SW: /* C.SW */
1436                 xinsn = OPC_RISC_SW;
1437                 xinsn = SET_RS2(xinsn, GET_C_RS2S(insn));
1438                 access_rs1 = GET_C_RS1S(insn);
1439                 access_imm = GET_C_SW_IMM(insn);
1440                 access_size = 4;
1441                 break;
1442             case OPC_RISC_C_FUNC_FSW_SD:
1443                 if (riscv_cpu_xlen(env) == 32) { /* C.FSW (RV32) */
1444                     xinsn = OPC_RISC_FSW;
1445                     xinsn = SET_RS2(xinsn, GET_C_RS2S(insn));
1446                     access_rs1 = GET_C_RS1S(insn);
1447                     access_imm = GET_C_SW_IMM(insn);
1448                     access_size = 4;
1449                 } else { /* C.SD (RV64/RV128) */
1450                     xinsn = OPC_RISC_SD;
1451                     xinsn = SET_RS2(xinsn, GET_C_RS2S(insn));
1452                     access_rs1 = GET_C_RS1S(insn);
1453                     access_imm = GET_C_SD_IMM(insn);
1454                     access_size = 8;
1455                 }
1456                 break;
1457             default:
1458                 break;
1459             }
1460             break;
1461         case OPC_RISC_C_OP_QUAD2: /* Quadrant 2 */
1462             switch (GET_C_FUNC(insn)) {
1463             case OPC_RISC_C_FUNC_FLDSP_LQSP:
1464                 if (riscv_cpu_xlen(env) != 128) { /* C.FLDSP (RV32/64) */
1465                     xinsn = OPC_RISC_FLD;
1466                     xinsn = SET_RD(xinsn, GET_C_RD(insn));
1467                     access_rs1 = 2;
1468                     access_imm = GET_C_LDSP_IMM(insn);
1469                     access_size = 8;
1470                 }
1471                 break;
1472             case OPC_RISC_C_FUNC_LWSP: /* C.LWSP */
1473                 xinsn = OPC_RISC_LW;
1474                 xinsn = SET_RD(xinsn, GET_C_RD(insn));
1475                 access_rs1 = 2;
1476                 access_imm = GET_C_LWSP_IMM(insn);
1477                 access_size = 4;
1478                 break;
1479             case OPC_RISC_C_FUNC_FLWSP_LDSP:
1480                 if (riscv_cpu_xlen(env) == 32) { /* C.FLWSP (RV32) */
1481                     xinsn = OPC_RISC_FLW;
1482                     xinsn = SET_RD(xinsn, GET_C_RD(insn));
1483                     access_rs1 = 2;
1484                     access_imm = GET_C_LWSP_IMM(insn);
1485                     access_size = 4;
1486                 } else { /* C.LDSP (RV64/RV128) */
1487                     xinsn = OPC_RISC_LD;
1488                     xinsn = SET_RD(xinsn, GET_C_RD(insn));
1489                     access_rs1 = 2;
1490                     access_imm = GET_C_LDSP_IMM(insn);
1491                     access_size = 8;
1492                 }
1493                 break;
1494             case OPC_RISC_C_FUNC_FSDSP_SQSP:
1495                 if (riscv_cpu_xlen(env) != 128) { /* C.FSDSP (RV32/64) */
1496                     xinsn = OPC_RISC_FSD;
1497                     xinsn = SET_RS2(xinsn, GET_C_RS2(insn));
1498                     access_rs1 = 2;
1499                     access_imm = GET_C_SDSP_IMM(insn);
1500                     access_size = 8;
1501                 }
1502                 break;
1503             case OPC_RISC_C_FUNC_SWSP: /* C.SWSP */
1504                 xinsn = OPC_RISC_SW;
1505                 xinsn = SET_RS2(xinsn, GET_C_RS2(insn));
1506                 access_rs1 = 2;
1507                 access_imm = GET_C_SWSP_IMM(insn);
1508                 access_size = 4;
1509                 break;
1510             case 7:
1511                 if (riscv_cpu_xlen(env) == 32) { /* C.FSWSP (RV32) */
1512                     xinsn = OPC_RISC_FSW;
1513                     xinsn = SET_RS2(xinsn, GET_C_RS2(insn));
1514                     access_rs1 = 2;
1515                     access_imm = GET_C_SWSP_IMM(insn);
1516                     access_size = 4;
1517                 } else { /* C.SDSP (RV64/RV128) */
1518                     xinsn = OPC_RISC_SD;
1519                     xinsn = SET_RS2(xinsn, GET_C_RS2(insn));
1520                     access_rs1 = 2;
1521                     access_imm = GET_C_SDSP_IMM(insn);
1522                     access_size = 8;
1523                 }
1524                 break;
1525             default:
1526                 break;
1527             }
1528             break;
1529         default:
1530             break;
1531         }
1532 
1533         /*
1534          * Clear Bit1 of transformed instruction to indicate that
1535          * original insruction was a 16bit instruction
1536          */
1537         xinsn &= ~((target_ulong)0x2);
1538     } else {
1539         /* Transform 32bit (or wider) instructions */
1540         switch (MASK_OP_MAJOR(insn)) {
1541         case OPC_RISC_ATOMIC:
1542             xinsn = insn;
1543             access_rs1 = GET_RS1(insn);
1544             access_size = 1 << GET_FUNCT3(insn);
1545             break;
1546         case OPC_RISC_LOAD:
1547         case OPC_RISC_FP_LOAD:
1548             xinsn = SET_I_IMM(insn, 0);
1549             access_rs1 = GET_RS1(insn);
1550             access_imm = GET_IMM(insn);
1551             access_size = 1 << GET_FUNCT3(insn);
1552             break;
1553         case OPC_RISC_STORE:
1554         case OPC_RISC_FP_STORE:
1555             xinsn = SET_S_IMM(insn, 0);
1556             access_rs1 = GET_RS1(insn);
1557             access_imm = GET_STORE_IMM(insn);
1558             access_size = 1 << GET_FUNCT3(insn);
1559             break;
1560         case OPC_RISC_SYSTEM:
1561             if (MASK_OP_SYSTEM(insn) == OPC_RISC_HLVHSV) {
1562                 xinsn = insn;
1563                 access_rs1 = GET_RS1(insn);
1564                 access_size = 1 << ((GET_FUNCT7(insn) >> 1) & 0x3);
1565                 access_size = 1 << access_size;
1566             }
1567             break;
1568         default:
1569             break;
1570         }
1571     }
1572 
1573     if (access_size) {
1574         xinsn = SET_RS1(xinsn, (taddr - (env->gpr[access_rs1] + access_imm)) &
1575                                (access_size - 1));
1576     }
1577 
1578     return xinsn;
1579 }
1580 #endif /* !CONFIG_USER_ONLY */
1581 
1582 /*
1583  * Handle Traps
1584  *
1585  * Adapted from Spike's processor_t::take_trap.
1586  *
1587  */
1588 void riscv_cpu_do_interrupt(CPUState *cs)
1589 {
1590 #if !defined(CONFIG_USER_ONLY)
1591 
1592     RISCVCPU *cpu = RISCV_CPU(cs);
1593     CPURISCVState *env = &cpu->env;
1594     bool write_gva = false;
1595     uint64_t s;
1596 
1597     /*
1598      * cs->exception is 32-bits wide unlike mcause which is XLEN-bits wide
1599      * so we mask off the MSB and separate into trap type and cause.
1600      */
1601     bool async = !!(cs->exception_index & RISCV_EXCP_INT_FLAG);
1602     target_ulong cause = cs->exception_index & RISCV_EXCP_INT_MASK;
1603     uint64_t deleg = async ? env->mideleg : env->medeleg;
1604     target_ulong tval = 0;
1605     target_ulong tinst = 0;
1606     target_ulong htval = 0;
1607     target_ulong mtval2 = 0;
1608 
1609     if  (cause == RISCV_EXCP_SEMIHOST) {
1610         do_common_semihosting(cs);
1611         env->pc += 4;
1612         return;
1613     }
1614 
1615     if (!async) {
1616         /* set tval to badaddr for traps with address information */
1617         switch (cause) {
1618         case RISCV_EXCP_LOAD_GUEST_ACCESS_FAULT:
1619         case RISCV_EXCP_STORE_GUEST_AMO_ACCESS_FAULT:
1620         case RISCV_EXCP_LOAD_ADDR_MIS:
1621         case RISCV_EXCP_STORE_AMO_ADDR_MIS:
1622         case RISCV_EXCP_LOAD_ACCESS_FAULT:
1623         case RISCV_EXCP_STORE_AMO_ACCESS_FAULT:
1624         case RISCV_EXCP_LOAD_PAGE_FAULT:
1625         case RISCV_EXCP_STORE_PAGE_FAULT:
1626             write_gva = env->two_stage_lookup;
1627             tval = env->badaddr;
1628             if (env->two_stage_indirect_lookup) {
1629                 /*
1630                  * special pseudoinstruction for G-stage fault taken while
1631                  * doing VS-stage page table walk.
1632                  */
1633                 tinst = (riscv_cpu_xlen(env) == 32) ? 0x00002000 : 0x00003000;
1634             } else {
1635                 /*
1636                  * The "Addr. Offset" field in transformed instruction is
1637                  * non-zero only for misaligned access.
1638                  */
1639                 tinst = riscv_transformed_insn(env, env->bins, tval);
1640             }
1641             break;
1642         case RISCV_EXCP_INST_GUEST_PAGE_FAULT:
1643         case RISCV_EXCP_INST_ADDR_MIS:
1644         case RISCV_EXCP_INST_ACCESS_FAULT:
1645         case RISCV_EXCP_INST_PAGE_FAULT:
1646             write_gva = env->two_stage_lookup;
1647             tval = env->badaddr;
1648             if (env->two_stage_indirect_lookup) {
1649                 /*
1650                  * special pseudoinstruction for G-stage fault taken while
1651                  * doing VS-stage page table walk.
1652                  */
1653                 tinst = (riscv_cpu_xlen(env) == 32) ? 0x00002000 : 0x00003000;
1654             }
1655             break;
1656         case RISCV_EXCP_ILLEGAL_INST:
1657         case RISCV_EXCP_VIRT_INSTRUCTION_FAULT:
1658             tval = env->bins;
1659             break;
1660         case RISCV_EXCP_BREAKPOINT:
1661             if (cs->watchpoint_hit) {
1662                 tval = cs->watchpoint_hit->hitaddr;
1663                 cs->watchpoint_hit = NULL;
1664             }
1665             break;
1666         default:
1667             break;
1668         }
1669         /* ecall is dispatched as one cause so translate based on mode */
1670         if (cause == RISCV_EXCP_U_ECALL) {
1671             assert(env->priv <= 3);
1672 
1673             if (env->priv == PRV_M) {
1674                 cause = RISCV_EXCP_M_ECALL;
1675             } else if (env->priv == PRV_S && env->virt_enabled) {
1676                 cause = RISCV_EXCP_VS_ECALL;
1677             } else if (env->priv == PRV_S && !env->virt_enabled) {
1678                 cause = RISCV_EXCP_S_ECALL;
1679             } else if (env->priv == PRV_U) {
1680                 cause = RISCV_EXCP_U_ECALL;
1681             }
1682         }
1683     }
1684 
1685     trace_riscv_trap(env->mhartid, async, cause, env->pc, tval,
1686                      riscv_cpu_get_trap_name(cause, async));
1687 
1688     qemu_log_mask(CPU_LOG_INT,
1689                   "%s: hart:"TARGET_FMT_ld", async:%d, cause:"TARGET_FMT_lx", "
1690                   "epc:0x"TARGET_FMT_lx", tval:0x"TARGET_FMT_lx", desc=%s\n",
1691                   __func__, env->mhartid, async, cause, env->pc, tval,
1692                   riscv_cpu_get_trap_name(cause, async));
1693 
1694     if (env->priv <= PRV_S &&
1695             cause < TARGET_LONG_BITS && ((deleg >> cause) & 1)) {
1696         /* handle the trap in S-mode */
1697         if (riscv_has_ext(env, RVH)) {
1698             uint64_t hdeleg = async ? env->hideleg : env->hedeleg;
1699 
1700             if (env->virt_enabled && ((hdeleg >> cause) & 1)) {
1701                 /* Trap to VS mode */
1702                 /*
1703                  * See if we need to adjust cause. Yes if its VS mode interrupt
1704                  * no if hypervisor has delegated one of hs mode's interrupt
1705                  */
1706                 if (cause == IRQ_VS_TIMER || cause == IRQ_VS_SOFT ||
1707                     cause == IRQ_VS_EXT) {
1708                     cause = cause - 1;
1709                 }
1710                 write_gva = false;
1711             } else if (env->virt_enabled) {
1712                 /* Trap into HS mode, from virt */
1713                 riscv_cpu_swap_hypervisor_regs(env);
1714                 env->hstatus = set_field(env->hstatus, HSTATUS_SPVP,
1715                                          env->priv);
1716                 env->hstatus = set_field(env->hstatus, HSTATUS_SPV, true);
1717 
1718                 htval = env->guest_phys_fault_addr;
1719 
1720                 riscv_cpu_set_virt_enabled(env, 0);
1721             } else {
1722                 /* Trap into HS mode */
1723                 env->hstatus = set_field(env->hstatus, HSTATUS_SPV, false);
1724                 htval = env->guest_phys_fault_addr;
1725             }
1726             env->hstatus = set_field(env->hstatus, HSTATUS_GVA, write_gva);
1727         }
1728 
1729         s = env->mstatus;
1730         s = set_field(s, MSTATUS_SPIE, get_field(s, MSTATUS_SIE));
1731         s = set_field(s, MSTATUS_SPP, env->priv);
1732         s = set_field(s, MSTATUS_SIE, 0);
1733         env->mstatus = s;
1734         env->scause = cause | ((target_ulong)async << (TARGET_LONG_BITS - 1));
1735         env->sepc = env->pc;
1736         env->stval = tval;
1737         env->htval = htval;
1738         env->htinst = tinst;
1739         env->pc = (env->stvec >> 2 << 2) +
1740                   ((async && (env->stvec & 3) == 1) ? cause * 4 : 0);
1741         riscv_cpu_set_mode(env, PRV_S);
1742     } else {
1743         /* handle the trap in M-mode */
1744         if (riscv_has_ext(env, RVH)) {
1745             if (env->virt_enabled) {
1746                 riscv_cpu_swap_hypervisor_regs(env);
1747             }
1748             env->mstatus = set_field(env->mstatus, MSTATUS_MPV,
1749                                      env->virt_enabled);
1750             if (env->virt_enabled && tval) {
1751                 env->mstatus = set_field(env->mstatus, MSTATUS_GVA, 1);
1752             }
1753 
1754             mtval2 = env->guest_phys_fault_addr;
1755 
1756             /* Trapping to M mode, virt is disabled */
1757             riscv_cpu_set_virt_enabled(env, 0);
1758         }
1759 
1760         s = env->mstatus;
1761         s = set_field(s, MSTATUS_MPIE, get_field(s, MSTATUS_MIE));
1762         s = set_field(s, MSTATUS_MPP, env->priv);
1763         s = set_field(s, MSTATUS_MIE, 0);
1764         env->mstatus = s;
1765         env->mcause = cause | ~(((target_ulong)-1) >> async);
1766         env->mepc = env->pc;
1767         env->mtval = tval;
1768         env->mtval2 = mtval2;
1769         env->mtinst = tinst;
1770         env->pc = (env->mtvec >> 2 << 2) +
1771                   ((async && (env->mtvec & 3) == 1) ? cause * 4 : 0);
1772         riscv_cpu_set_mode(env, PRV_M);
1773     }
1774 
1775     /*
1776      * NOTE: it is not necessary to yield load reservations here. It is only
1777      * necessary for an SC from "another hart" to cause a load reservation
1778      * to be yielded. Refer to the memory consistency model section of the
1779      * RISC-V ISA Specification.
1780      */
1781 
1782     env->two_stage_lookup = false;
1783     env->two_stage_indirect_lookup = false;
1784 #endif
1785     cs->exception_index = RISCV_EXCP_NONE; /* mark handled to qemu */
1786 }
1787