1 /*
2 * RISC-V CPU helpers for qemu.
3 *
4 * Copyright (c) 2016-2017 Sagar Karandikar, sagark@eecs.berkeley.edu
5 * Copyright (c) 2017-2018 SiFive, Inc.
6 *
7 * This program is free software; you can redistribute it and/or modify it
8 * under the terms and conditions of the GNU General Public License,
9 * version 2 or later, as published by the Free Software Foundation.
10 *
11 * This program is distributed in the hope it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
14 * more details.
15 *
16 * You should have received a copy of the GNU General Public License along with
17 * this program. If not, see <http://www.gnu.org/licenses/>.
18 */
19
20 #include "qemu/osdep.h"
21 #include "qemu/log.h"
22 #include "qemu/main-loop.h"
23 #include "cpu.h"
24 #include "internals.h"
25 #include "pmu.h"
26 #include "exec/exec-all.h"
27 #include "exec/page-protection.h"
28 #include "instmap.h"
29 #include "tcg/tcg-op.h"
30 #include "hw/core/tcg-cpu-ops.h"
31 #include "trace.h"
32 #include "semihosting/common-semi.h"
33 #include "sysemu/cpu-timers.h"
34 #include "cpu_bits.h"
35 #include "debug.h"
36 #include "tcg/oversized-guest.h"
37 #include "pmp.h"
38
riscv_env_mmu_index(CPURISCVState * env,bool ifetch)39 int riscv_env_mmu_index(CPURISCVState *env, bool ifetch)
40 {
41 #ifdef CONFIG_USER_ONLY
42 return 0;
43 #else
44 bool virt = env->virt_enabled;
45 int mode = env->priv;
46
47 /* All priv -> mmu_idx mapping are here */
48 if (!ifetch) {
49 uint64_t status = env->mstatus;
50
51 if (mode == PRV_M && get_field(status, MSTATUS_MPRV)) {
52 mode = get_field(env->mstatus, MSTATUS_MPP);
53 virt = get_field(env->mstatus, MSTATUS_MPV) &&
54 (mode != PRV_M);
55 if (virt) {
56 status = env->vsstatus;
57 }
58 }
59 if (mode == PRV_S && get_field(status, MSTATUS_SUM)) {
60 mode = MMUIdx_S_SUM;
61 }
62 }
63
64 return mode | (virt ? MMU_2STAGE_BIT : 0);
65 #endif
66 }
67
cpu_get_fcfien(CPURISCVState * env)68 bool cpu_get_fcfien(CPURISCVState *env)
69 {
70 /* no cfi extension, return false */
71 if (!env_archcpu(env)->cfg.ext_zicfilp) {
72 return false;
73 }
74
75 switch (env->priv) {
76 case PRV_U:
77 if (riscv_has_ext(env, RVS)) {
78 return env->senvcfg & SENVCFG_LPE;
79 }
80 return env->menvcfg & MENVCFG_LPE;
81 #ifndef CONFIG_USER_ONLY
82 case PRV_S:
83 if (env->virt_enabled) {
84 return env->henvcfg & HENVCFG_LPE;
85 }
86 return env->menvcfg & MENVCFG_LPE;
87 case PRV_M:
88 return env->mseccfg & MSECCFG_MLPE;
89 #endif
90 default:
91 g_assert_not_reached();
92 }
93 }
94
cpu_get_bcfien(CPURISCVState * env)95 bool cpu_get_bcfien(CPURISCVState *env)
96 {
97 /* no cfi extension, return false */
98 if (!env_archcpu(env)->cfg.ext_zicfiss) {
99 return false;
100 }
101
102 switch (env->priv) {
103 case PRV_U:
104 /*
105 * If S is not implemented then shadow stack for U can't be turned on
106 * It is checked in `riscv_cpu_validate_set_extensions`, so no need to
107 * check here or assert here
108 */
109 return env->senvcfg & SENVCFG_SSE;
110 #ifndef CONFIG_USER_ONLY
111 case PRV_S:
112 if (env->virt_enabled) {
113 return env->henvcfg & HENVCFG_SSE;
114 }
115 return env->menvcfg & MENVCFG_SSE;
116 case PRV_M: /* M-mode shadow stack is always off */
117 return false;
118 #endif
119 default:
120 g_assert_not_reached();
121 }
122 }
123
cpu_get_tb_cpu_state(CPURISCVState * env,vaddr * pc,uint64_t * cs_base,uint32_t * pflags)124 void cpu_get_tb_cpu_state(CPURISCVState *env, vaddr *pc,
125 uint64_t *cs_base, uint32_t *pflags)
126 {
127 RISCVCPU *cpu = env_archcpu(env);
128 RISCVExtStatus fs, vs;
129 uint32_t flags = 0;
130
131 *pc = env->xl == MXL_RV32 ? env->pc & UINT32_MAX : env->pc;
132 *cs_base = 0;
133
134 if (cpu->cfg.ext_zve32x) {
135 /*
136 * If env->vl equals to VLMAX, we can use generic vector operation
137 * expanders (GVEC) to accerlate the vector operations.
138 * However, as LMUL could be a fractional number. The maximum
139 * vector size can be operated might be less than 8 bytes,
140 * which is not supported by GVEC. So we set vl_eq_vlmax flag to true
141 * only when maxsz >= 8 bytes.
142 */
143
144 /* lmul encoded as in DisasContext::lmul */
145 int8_t lmul = sextract32(FIELD_EX64(env->vtype, VTYPE, VLMUL), 0, 3);
146 uint32_t vsew = FIELD_EX64(env->vtype, VTYPE, VSEW);
147 uint32_t vlmax = vext_get_vlmax(cpu->cfg.vlenb, vsew, lmul);
148 uint32_t maxsz = vlmax << vsew;
149 bool vl_eq_vlmax = (env->vstart == 0) && (vlmax == env->vl) &&
150 (maxsz >= 8);
151 flags = FIELD_DP32(flags, TB_FLAGS, VILL, env->vill);
152 flags = FIELD_DP32(flags, TB_FLAGS, SEW, vsew);
153 flags = FIELD_DP32(flags, TB_FLAGS, LMUL,
154 FIELD_EX64(env->vtype, VTYPE, VLMUL));
155 flags = FIELD_DP32(flags, TB_FLAGS, VL_EQ_VLMAX, vl_eq_vlmax);
156 flags = FIELD_DP32(flags, TB_FLAGS, VTA,
157 FIELD_EX64(env->vtype, VTYPE, VTA));
158 flags = FIELD_DP32(flags, TB_FLAGS, VMA,
159 FIELD_EX64(env->vtype, VTYPE, VMA));
160 flags = FIELD_DP32(flags, TB_FLAGS, VSTART_EQ_ZERO, env->vstart == 0);
161 } else {
162 flags = FIELD_DP32(flags, TB_FLAGS, VILL, 1);
163 }
164
165 if (cpu_get_fcfien(env)) {
166 /*
167 * For Forward CFI, only the expectation of a lpad at
168 * the start of the block is tracked via env->elp. env->elp
169 * is turned on during jalr translation.
170 */
171 flags = FIELD_DP32(flags, TB_FLAGS, FCFI_LP_EXPECTED, env->elp);
172 flags = FIELD_DP32(flags, TB_FLAGS, FCFI_ENABLED, 1);
173 }
174
175 if (cpu_get_bcfien(env)) {
176 flags = FIELD_DP32(flags, TB_FLAGS, BCFI_ENABLED, 1);
177 }
178
179 #ifdef CONFIG_USER_ONLY
180 fs = EXT_STATUS_DIRTY;
181 vs = EXT_STATUS_DIRTY;
182 #else
183 flags = FIELD_DP32(flags, TB_FLAGS, PRIV, env->priv);
184
185 flags |= riscv_env_mmu_index(env, 0);
186 fs = get_field(env->mstatus, MSTATUS_FS);
187 vs = get_field(env->mstatus, MSTATUS_VS);
188
189 if (env->virt_enabled) {
190 flags = FIELD_DP32(flags, TB_FLAGS, VIRT_ENABLED, 1);
191 /*
192 * Merge DISABLED and !DIRTY states using MIN.
193 * We will set both fields when dirtying.
194 */
195 fs = MIN(fs, get_field(env->mstatus_hs, MSTATUS_FS));
196 vs = MIN(vs, get_field(env->mstatus_hs, MSTATUS_VS));
197 }
198
199 /* With Zfinx, floating point is enabled/disabled by Smstateen. */
200 if (!riscv_has_ext(env, RVF)) {
201 fs = (smstateen_acc_ok(env, 0, SMSTATEEN0_FCSR) == RISCV_EXCP_NONE)
202 ? EXT_STATUS_DIRTY : EXT_STATUS_DISABLED;
203 }
204
205 if (cpu->cfg.debug && !icount_enabled()) {
206 flags = FIELD_DP32(flags, TB_FLAGS, ITRIGGER, env->itrigger_enabled);
207 }
208 #endif
209
210 flags = FIELD_DP32(flags, TB_FLAGS, FS, fs);
211 flags = FIELD_DP32(flags, TB_FLAGS, VS, vs);
212 flags = FIELD_DP32(flags, TB_FLAGS, XL, env->xl);
213 flags = FIELD_DP32(flags, TB_FLAGS, AXL, cpu_address_xl(env));
214 if (env->cur_pmmask != 0) {
215 flags = FIELD_DP32(flags, TB_FLAGS, PM_MASK_ENABLED, 1);
216 }
217 if (env->cur_pmbase != 0) {
218 flags = FIELD_DP32(flags, TB_FLAGS, PM_BASE_ENABLED, 1);
219 }
220
221 *pflags = flags;
222 }
223
riscv_cpu_update_mask(CPURISCVState * env)224 void riscv_cpu_update_mask(CPURISCVState *env)
225 {
226 target_ulong mask = 0, base = 0;
227 RISCVMXL xl = env->xl;
228 /*
229 * TODO: Current RVJ spec does not specify
230 * how the extension interacts with XLEN.
231 */
232 #ifndef CONFIG_USER_ONLY
233 int mode = cpu_address_mode(env);
234 xl = cpu_get_xl(env, mode);
235 if (riscv_has_ext(env, RVJ)) {
236 switch (mode) {
237 case PRV_M:
238 if (env->mmte & M_PM_ENABLE) {
239 mask = env->mpmmask;
240 base = env->mpmbase;
241 }
242 break;
243 case PRV_S:
244 if (env->mmte & S_PM_ENABLE) {
245 mask = env->spmmask;
246 base = env->spmbase;
247 }
248 break;
249 case PRV_U:
250 if (env->mmte & U_PM_ENABLE) {
251 mask = env->upmmask;
252 base = env->upmbase;
253 }
254 break;
255 default:
256 g_assert_not_reached();
257 }
258 }
259 #endif
260 if (xl == MXL_RV32) {
261 env->cur_pmmask = mask & UINT32_MAX;
262 env->cur_pmbase = base & UINT32_MAX;
263 } else {
264 env->cur_pmmask = mask;
265 env->cur_pmbase = base;
266 }
267 }
268
269 #ifndef CONFIG_USER_ONLY
270
271 /*
272 * The HS-mode is allowed to configure priority only for the
273 * following VS-mode local interrupts:
274 *
275 * 0 (Reserved interrupt, reads as zero)
276 * 1 Supervisor software interrupt
277 * 4 (Reserved interrupt, reads as zero)
278 * 5 Supervisor timer interrupt
279 * 8 (Reserved interrupt, reads as zero)
280 * 13 (Reserved interrupt)
281 * 14 "
282 * 15 "
283 * 16 "
284 * 17 "
285 * 18 "
286 * 19 "
287 * 20 "
288 * 21 "
289 * 22 "
290 * 23 "
291 */
292
293 static const int hviprio_index2irq[] = {
294 0, 1, 4, 5, 8, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23 };
295 static const int hviprio_index2rdzero[] = {
296 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
297
riscv_cpu_hviprio_index2irq(int index,int * out_irq,int * out_rdzero)298 int riscv_cpu_hviprio_index2irq(int index, int *out_irq, int *out_rdzero)
299 {
300 if (index < 0 || ARRAY_SIZE(hviprio_index2irq) <= index) {
301 return -EINVAL;
302 }
303
304 if (out_irq) {
305 *out_irq = hviprio_index2irq[index];
306 }
307
308 if (out_rdzero) {
309 *out_rdzero = hviprio_index2rdzero[index];
310 }
311
312 return 0;
313 }
314
315 /*
316 * Default priorities of local interrupts are defined in the
317 * RISC-V Advanced Interrupt Architecture specification.
318 *
319 * ----------------------------------------------------------------
320 * Default |
321 * Priority | Major Interrupt Numbers
322 * ----------------------------------------------------------------
323 * Highest | 47, 23, 46, 45, 22, 44,
324 * | 43, 21, 42, 41, 20, 40
325 * |
326 * | 11 (0b), 3 (03), 7 (07)
327 * | 9 (09), 1 (01), 5 (05)
328 * | 12 (0c)
329 * | 10 (0a), 2 (02), 6 (06)
330 * |
331 * | 39, 19, 38, 37, 18, 36,
332 * Lowest | 35, 17, 34, 33, 16, 32
333 * ----------------------------------------------------------------
334 */
335 static const uint8_t default_iprio[64] = {
336 /* Custom interrupts 48 to 63 */
337 [63] = IPRIO_MMAXIPRIO,
338 [62] = IPRIO_MMAXIPRIO,
339 [61] = IPRIO_MMAXIPRIO,
340 [60] = IPRIO_MMAXIPRIO,
341 [59] = IPRIO_MMAXIPRIO,
342 [58] = IPRIO_MMAXIPRIO,
343 [57] = IPRIO_MMAXIPRIO,
344 [56] = IPRIO_MMAXIPRIO,
345 [55] = IPRIO_MMAXIPRIO,
346 [54] = IPRIO_MMAXIPRIO,
347 [53] = IPRIO_MMAXIPRIO,
348 [52] = IPRIO_MMAXIPRIO,
349 [51] = IPRIO_MMAXIPRIO,
350 [50] = IPRIO_MMAXIPRIO,
351 [49] = IPRIO_MMAXIPRIO,
352 [48] = IPRIO_MMAXIPRIO,
353
354 /* Custom interrupts 24 to 31 */
355 [31] = IPRIO_MMAXIPRIO,
356 [30] = IPRIO_MMAXIPRIO,
357 [29] = IPRIO_MMAXIPRIO,
358 [28] = IPRIO_MMAXIPRIO,
359 [27] = IPRIO_MMAXIPRIO,
360 [26] = IPRIO_MMAXIPRIO,
361 [25] = IPRIO_MMAXIPRIO,
362 [24] = IPRIO_MMAXIPRIO,
363
364 [47] = IPRIO_DEFAULT_UPPER,
365 [23] = IPRIO_DEFAULT_UPPER + 1,
366 [46] = IPRIO_DEFAULT_UPPER + 2,
367 [45] = IPRIO_DEFAULT_UPPER + 3,
368 [22] = IPRIO_DEFAULT_UPPER + 4,
369 [44] = IPRIO_DEFAULT_UPPER + 5,
370
371 [43] = IPRIO_DEFAULT_UPPER + 6,
372 [21] = IPRIO_DEFAULT_UPPER + 7,
373 [42] = IPRIO_DEFAULT_UPPER + 8,
374 [41] = IPRIO_DEFAULT_UPPER + 9,
375 [20] = IPRIO_DEFAULT_UPPER + 10,
376 [40] = IPRIO_DEFAULT_UPPER + 11,
377
378 [11] = IPRIO_DEFAULT_M,
379 [3] = IPRIO_DEFAULT_M + 1,
380 [7] = IPRIO_DEFAULT_M + 2,
381
382 [9] = IPRIO_DEFAULT_S,
383 [1] = IPRIO_DEFAULT_S + 1,
384 [5] = IPRIO_DEFAULT_S + 2,
385
386 [12] = IPRIO_DEFAULT_SGEXT,
387
388 [10] = IPRIO_DEFAULT_VS,
389 [2] = IPRIO_DEFAULT_VS + 1,
390 [6] = IPRIO_DEFAULT_VS + 2,
391
392 [39] = IPRIO_DEFAULT_LOWER,
393 [19] = IPRIO_DEFAULT_LOWER + 1,
394 [38] = IPRIO_DEFAULT_LOWER + 2,
395 [37] = IPRIO_DEFAULT_LOWER + 3,
396 [18] = IPRIO_DEFAULT_LOWER + 4,
397 [36] = IPRIO_DEFAULT_LOWER + 5,
398
399 [35] = IPRIO_DEFAULT_LOWER + 6,
400 [17] = IPRIO_DEFAULT_LOWER + 7,
401 [34] = IPRIO_DEFAULT_LOWER + 8,
402 [33] = IPRIO_DEFAULT_LOWER + 9,
403 [16] = IPRIO_DEFAULT_LOWER + 10,
404 [32] = IPRIO_DEFAULT_LOWER + 11,
405 };
406
riscv_cpu_default_priority(int irq)407 uint8_t riscv_cpu_default_priority(int irq)
408 {
409 if (irq < 0 || irq > 63) {
410 return IPRIO_MMAXIPRIO;
411 }
412
413 return default_iprio[irq] ? default_iprio[irq] : IPRIO_MMAXIPRIO;
414 };
415
riscv_cpu_pending_to_irq(CPURISCVState * env,int extirq,unsigned int extirq_def_prio,uint64_t pending,uint8_t * iprio)416 static int riscv_cpu_pending_to_irq(CPURISCVState *env,
417 int extirq, unsigned int extirq_def_prio,
418 uint64_t pending, uint8_t *iprio)
419 {
420 int irq, best_irq = RISCV_EXCP_NONE;
421 unsigned int prio, best_prio = UINT_MAX;
422
423 if (!pending) {
424 return RISCV_EXCP_NONE;
425 }
426
427 irq = ctz64(pending);
428 if (!((extirq == IRQ_M_EXT) ? riscv_cpu_cfg(env)->ext_smaia :
429 riscv_cpu_cfg(env)->ext_ssaia)) {
430 return irq;
431 }
432
433 pending = pending >> irq;
434 while (pending) {
435 prio = iprio[irq];
436 if (!prio) {
437 if (irq == extirq) {
438 prio = extirq_def_prio;
439 } else {
440 prio = (riscv_cpu_default_priority(irq) < extirq_def_prio) ?
441 1 : IPRIO_MMAXIPRIO;
442 }
443 }
444 if ((pending & 0x1) && (prio <= best_prio)) {
445 best_irq = irq;
446 best_prio = prio;
447 }
448 irq++;
449 pending = pending >> 1;
450 }
451
452 return best_irq;
453 }
454
455 /*
456 * Doesn't report interrupts inserted using mvip from M-mode firmware or
457 * using hvip bits 13:63 from HS-mode. Those are returned in
458 * riscv_cpu_sirq_pending() and riscv_cpu_vsirq_pending().
459 */
riscv_cpu_all_pending(CPURISCVState * env)460 uint64_t riscv_cpu_all_pending(CPURISCVState *env)
461 {
462 uint32_t gein = get_field(env->hstatus, HSTATUS_VGEIN);
463 uint64_t vsgein = (env->hgeip & (1ULL << gein)) ? MIP_VSEIP : 0;
464 uint64_t vstip = (env->vstime_irq) ? MIP_VSTIP : 0;
465
466 return (env->mip | vsgein | vstip) & env->mie;
467 }
468
riscv_cpu_mirq_pending(CPURISCVState * env)469 int riscv_cpu_mirq_pending(CPURISCVState *env)
470 {
471 uint64_t irqs = riscv_cpu_all_pending(env) & ~env->mideleg &
472 ~(MIP_SGEIP | MIP_VSSIP | MIP_VSTIP | MIP_VSEIP);
473
474 return riscv_cpu_pending_to_irq(env, IRQ_M_EXT, IPRIO_DEFAULT_M,
475 irqs, env->miprio);
476 }
477
riscv_cpu_sirq_pending(CPURISCVState * env)478 int riscv_cpu_sirq_pending(CPURISCVState *env)
479 {
480 uint64_t irqs = riscv_cpu_all_pending(env) & env->mideleg &
481 ~(MIP_VSSIP | MIP_VSTIP | MIP_VSEIP);
482 uint64_t irqs_f = env->mvip & env->mvien & ~env->mideleg & env->sie;
483
484 return riscv_cpu_pending_to_irq(env, IRQ_S_EXT, IPRIO_DEFAULT_S,
485 irqs | irqs_f, env->siprio);
486 }
487
riscv_cpu_vsirq_pending(CPURISCVState * env)488 int riscv_cpu_vsirq_pending(CPURISCVState *env)
489 {
490 uint64_t irqs = riscv_cpu_all_pending(env) & env->mideleg & env->hideleg;
491 uint64_t irqs_f_vs = env->hvip & env->hvien & ~env->hideleg & env->vsie;
492 uint64_t vsbits;
493
494 /* Bring VS-level bits to correct position */
495 vsbits = irqs & VS_MODE_INTERRUPTS;
496 irqs &= ~VS_MODE_INTERRUPTS;
497 irqs |= vsbits >> 1;
498
499 return riscv_cpu_pending_to_irq(env, IRQ_S_EXT, IPRIO_DEFAULT_S,
500 (irqs | irqs_f_vs), env->hviprio);
501 }
502
riscv_cpu_local_irq_pending(CPURISCVState * env)503 static int riscv_cpu_local_irq_pending(CPURISCVState *env)
504 {
505 uint64_t irqs, pending, mie, hsie, vsie, irqs_f, irqs_f_vs;
506 uint64_t vsbits, irq_delegated;
507 int virq;
508
509 /* Determine interrupt enable state of all privilege modes */
510 if (env->virt_enabled) {
511 mie = 1;
512 hsie = 1;
513 vsie = (env->priv < PRV_S) ||
514 (env->priv == PRV_S && get_field(env->mstatus, MSTATUS_SIE));
515 } else {
516 mie = (env->priv < PRV_M) ||
517 (env->priv == PRV_M && get_field(env->mstatus, MSTATUS_MIE));
518 hsie = (env->priv < PRV_S) ||
519 (env->priv == PRV_S && get_field(env->mstatus, MSTATUS_SIE));
520 vsie = 0;
521 }
522
523 /* Determine all pending interrupts */
524 pending = riscv_cpu_all_pending(env);
525
526 /* Check M-mode interrupts */
527 irqs = pending & ~env->mideleg & -mie;
528 if (irqs) {
529 return riscv_cpu_pending_to_irq(env, IRQ_M_EXT, IPRIO_DEFAULT_M,
530 irqs, env->miprio);
531 }
532
533 /* Check for virtual S-mode interrupts. */
534 irqs_f = env->mvip & (env->mvien & ~env->mideleg) & env->sie;
535
536 /* Check HS-mode interrupts */
537 irqs = ((pending & env->mideleg & ~env->hideleg) | irqs_f) & -hsie;
538 if (irqs) {
539 return riscv_cpu_pending_to_irq(env, IRQ_S_EXT, IPRIO_DEFAULT_S,
540 irqs, env->siprio);
541 }
542
543 /* Check for virtual VS-mode interrupts. */
544 irqs_f_vs = env->hvip & env->hvien & ~env->hideleg & env->vsie;
545
546 /* Check VS-mode interrupts */
547 irq_delegated = pending & env->mideleg & env->hideleg;
548
549 /* Bring VS-level bits to correct position */
550 vsbits = irq_delegated & VS_MODE_INTERRUPTS;
551 irq_delegated &= ~VS_MODE_INTERRUPTS;
552 irq_delegated |= vsbits >> 1;
553
554 irqs = (irq_delegated | irqs_f_vs) & -vsie;
555 if (irqs) {
556 virq = riscv_cpu_pending_to_irq(env, IRQ_S_EXT, IPRIO_DEFAULT_S,
557 irqs, env->hviprio);
558 if (virq <= 0 || (virq > 12 && virq <= 63)) {
559 return virq;
560 } else {
561 return virq + 1;
562 }
563 }
564
565 /* Indicate no pending interrupt */
566 return RISCV_EXCP_NONE;
567 }
568
riscv_cpu_exec_interrupt(CPUState * cs,int interrupt_request)569 bool riscv_cpu_exec_interrupt(CPUState *cs, int interrupt_request)
570 {
571 if (interrupt_request & CPU_INTERRUPT_HARD) {
572 RISCVCPU *cpu = RISCV_CPU(cs);
573 CPURISCVState *env = &cpu->env;
574 int interruptno = riscv_cpu_local_irq_pending(env);
575 if (interruptno >= 0) {
576 cs->exception_index = RISCV_EXCP_INT_FLAG | interruptno;
577 riscv_cpu_do_interrupt(cs);
578 return true;
579 }
580 }
581 return false;
582 }
583
584 /* Return true is floating point support is currently enabled */
riscv_cpu_fp_enabled(CPURISCVState * env)585 bool riscv_cpu_fp_enabled(CPURISCVState *env)
586 {
587 if (env->mstatus & MSTATUS_FS) {
588 if (env->virt_enabled && !(env->mstatus_hs & MSTATUS_FS)) {
589 return false;
590 }
591 return true;
592 }
593
594 return false;
595 }
596
597 /* Return true is vector support is currently enabled */
riscv_cpu_vector_enabled(CPURISCVState * env)598 bool riscv_cpu_vector_enabled(CPURISCVState *env)
599 {
600 if (env->mstatus & MSTATUS_VS) {
601 if (env->virt_enabled && !(env->mstatus_hs & MSTATUS_VS)) {
602 return false;
603 }
604 return true;
605 }
606
607 return false;
608 }
609
riscv_cpu_swap_hypervisor_regs(CPURISCVState * env)610 void riscv_cpu_swap_hypervisor_regs(CPURISCVState *env)
611 {
612 uint64_t mstatus_mask = MSTATUS_MXR | MSTATUS_SUM |
613 MSTATUS_SPP | MSTATUS_SPIE | MSTATUS_SIE |
614 MSTATUS64_UXL | MSTATUS_VS;
615
616 if (riscv_has_ext(env, RVF)) {
617 mstatus_mask |= MSTATUS_FS;
618 }
619 bool current_virt = env->virt_enabled;
620
621 /*
622 * If zicfilp extension available and henvcfg.LPE = 1,
623 * then apply SPELP mask on mstatus
624 */
625 if (env_archcpu(env)->cfg.ext_zicfilp &&
626 get_field(env->henvcfg, HENVCFG_LPE)) {
627 mstatus_mask |= SSTATUS_SPELP;
628 }
629
630 g_assert(riscv_has_ext(env, RVH));
631
632 if (current_virt) {
633 /* Current V=1 and we are about to change to V=0 */
634 env->vsstatus = env->mstatus & mstatus_mask;
635 env->mstatus &= ~mstatus_mask;
636 env->mstatus |= env->mstatus_hs;
637
638 env->vstvec = env->stvec;
639 env->stvec = env->stvec_hs;
640
641 env->vsscratch = env->sscratch;
642 env->sscratch = env->sscratch_hs;
643
644 env->vsepc = env->sepc;
645 env->sepc = env->sepc_hs;
646
647 env->vscause = env->scause;
648 env->scause = env->scause_hs;
649
650 env->vstval = env->stval;
651 env->stval = env->stval_hs;
652
653 env->vsatp = env->satp;
654 env->satp = env->satp_hs;
655 } else {
656 /* Current V=0 and we are about to change to V=1 */
657 env->mstatus_hs = env->mstatus & mstatus_mask;
658 env->mstatus &= ~mstatus_mask;
659 env->mstatus |= env->vsstatus;
660
661 env->stvec_hs = env->stvec;
662 env->stvec = env->vstvec;
663
664 env->sscratch_hs = env->sscratch;
665 env->sscratch = env->vsscratch;
666
667 env->sepc_hs = env->sepc;
668 env->sepc = env->vsepc;
669
670 env->scause_hs = env->scause;
671 env->scause = env->vscause;
672
673 env->stval_hs = env->stval;
674 env->stval = env->vstval;
675
676 env->satp_hs = env->satp;
677 env->satp = env->vsatp;
678 }
679 }
680
riscv_cpu_get_geilen(CPURISCVState * env)681 target_ulong riscv_cpu_get_geilen(CPURISCVState *env)
682 {
683 if (!riscv_has_ext(env, RVH)) {
684 return 0;
685 }
686
687 return env->geilen;
688 }
689
riscv_cpu_set_geilen(CPURISCVState * env,target_ulong geilen)690 void riscv_cpu_set_geilen(CPURISCVState *env, target_ulong geilen)
691 {
692 if (!riscv_has_ext(env, RVH)) {
693 return;
694 }
695
696 if (geilen > (TARGET_LONG_BITS - 1)) {
697 return;
698 }
699
700 env->geilen = geilen;
701 }
702
riscv_cpu_claim_interrupts(RISCVCPU * cpu,uint64_t interrupts)703 int riscv_cpu_claim_interrupts(RISCVCPU *cpu, uint64_t interrupts)
704 {
705 CPURISCVState *env = &cpu->env;
706 if (env->miclaim & interrupts) {
707 return -1;
708 } else {
709 env->miclaim |= interrupts;
710 return 0;
711 }
712 }
713
riscv_cpu_interrupt(CPURISCVState * env)714 void riscv_cpu_interrupt(CPURISCVState *env)
715 {
716 uint64_t gein, vsgein = 0, vstip = 0, irqf = 0;
717 CPUState *cs = env_cpu(env);
718
719 BQL_LOCK_GUARD();
720
721 if (env->virt_enabled) {
722 gein = get_field(env->hstatus, HSTATUS_VGEIN);
723 vsgein = (env->hgeip & (1ULL << gein)) ? MIP_VSEIP : 0;
724 irqf = env->hvien & env->hvip & env->vsie;
725 } else {
726 irqf = env->mvien & env->mvip & env->sie;
727 }
728
729 vstip = env->vstime_irq ? MIP_VSTIP : 0;
730
731 if (env->mip | vsgein | vstip | irqf) {
732 cpu_interrupt(cs, CPU_INTERRUPT_HARD);
733 } else {
734 cpu_reset_interrupt(cs, CPU_INTERRUPT_HARD);
735 }
736 }
737
riscv_cpu_update_mip(CPURISCVState * env,uint64_t mask,uint64_t value)738 uint64_t riscv_cpu_update_mip(CPURISCVState *env, uint64_t mask, uint64_t value)
739 {
740 uint64_t old = env->mip;
741
742 /* No need to update mip for VSTIP */
743 mask = ((mask == MIP_VSTIP) && env->vstime_irq) ? 0 : mask;
744
745 BQL_LOCK_GUARD();
746
747 env->mip = (env->mip & ~mask) | (value & mask);
748
749 riscv_cpu_interrupt(env);
750
751 return old;
752 }
753
riscv_cpu_set_rdtime_fn(CPURISCVState * env,uint64_t (* fn)(void *),void * arg)754 void riscv_cpu_set_rdtime_fn(CPURISCVState *env, uint64_t (*fn)(void *),
755 void *arg)
756 {
757 env->rdtime_fn = fn;
758 env->rdtime_fn_arg = arg;
759 }
760
riscv_cpu_set_aia_ireg_rmw_fn(CPURISCVState * env,uint32_t priv,int (* rmw_fn)(void * arg,target_ulong reg,target_ulong * val,target_ulong new_val,target_ulong write_mask),void * rmw_fn_arg)761 void riscv_cpu_set_aia_ireg_rmw_fn(CPURISCVState *env, uint32_t priv,
762 int (*rmw_fn)(void *arg,
763 target_ulong reg,
764 target_ulong *val,
765 target_ulong new_val,
766 target_ulong write_mask),
767 void *rmw_fn_arg)
768 {
769 if (priv <= PRV_M) {
770 env->aia_ireg_rmw_fn[priv] = rmw_fn;
771 env->aia_ireg_rmw_fn_arg[priv] = rmw_fn_arg;
772 }
773 }
774
riscv_cpu_set_mode(CPURISCVState * env,target_ulong newpriv,bool virt_en)775 void riscv_cpu_set_mode(CPURISCVState *env, target_ulong newpriv, bool virt_en)
776 {
777 g_assert(newpriv <= PRV_M && newpriv != PRV_RESERVED);
778
779 if (newpriv != env->priv || env->virt_enabled != virt_en) {
780 if (icount_enabled()) {
781 riscv_itrigger_update_priv(env);
782 }
783
784 riscv_pmu_update_fixed_ctrs(env, newpriv, virt_en);
785 }
786
787 /* tlb_flush is unnecessary as mode is contained in mmu_idx */
788 env->priv = newpriv;
789 env->xl = cpu_recompute_xl(env);
790 riscv_cpu_update_mask(env);
791
792 /*
793 * Clear the load reservation - otherwise a reservation placed in one
794 * context/process can be used by another, resulting in an SC succeeding
795 * incorrectly. Version 2.2 of the ISA specification explicitly requires
796 * this behaviour, while later revisions say that the kernel "should" use
797 * an SC instruction to force the yielding of a load reservation on a
798 * preemptive context switch. As a result, do both.
799 */
800 env->load_res = -1;
801
802 if (riscv_has_ext(env, RVH)) {
803 /* Flush the TLB on all virt mode changes. */
804 if (env->virt_enabled != virt_en) {
805 tlb_flush(env_cpu(env));
806 }
807
808 env->virt_enabled = virt_en;
809 if (virt_en) {
810 /*
811 * The guest external interrupts from an interrupt controller are
812 * delivered only when the Guest/VM is running (i.e. V=1). This
813 * means any guest external interrupt which is triggered while the
814 * Guest/VM is not running (i.e. V=0) will be missed on QEMU
815 * resulting in guest with sluggish response to serial console
816 * input and other I/O events.
817 *
818 * To solve this, we check and inject interrupt after setting V=1.
819 */
820 riscv_cpu_update_mip(env, 0, 0);
821 }
822 }
823 }
824
825 /*
826 * get_physical_address_pmp - check PMP permission for this physical address
827 *
828 * Match the PMP region and check permission for this physical address and it's
829 * TLB page. Returns 0 if the permission checking was successful
830 *
831 * @env: CPURISCVState
832 * @prot: The returned protection attributes
833 * @addr: The physical address to be checked permission
834 * @access_type: The type of MMU access
835 * @mode: Indicates current privilege level.
836 */
get_physical_address_pmp(CPURISCVState * env,int * prot,hwaddr addr,int size,MMUAccessType access_type,int mode)837 static int get_physical_address_pmp(CPURISCVState *env, int *prot, hwaddr addr,
838 int size, MMUAccessType access_type,
839 int mode)
840 {
841 pmp_priv_t pmp_priv;
842 bool pmp_has_privs;
843
844 if (!riscv_cpu_cfg(env)->pmp) {
845 *prot = PAGE_READ | PAGE_WRITE | PAGE_EXEC;
846 return TRANSLATE_SUCCESS;
847 }
848
849 pmp_has_privs = pmp_hart_has_privs(env, addr, size, 1 << access_type,
850 &pmp_priv, mode);
851 if (!pmp_has_privs) {
852 *prot = 0;
853 return TRANSLATE_PMP_FAIL;
854 }
855
856 *prot = pmp_priv_to_page_prot(pmp_priv);
857
858 return TRANSLATE_SUCCESS;
859 }
860
861 /*
862 * get_physical_address - get the physical address for this virtual address
863 *
864 * Do a page table walk to obtain the physical address corresponding to a
865 * virtual address. Returns 0 if the translation was successful
866 *
867 * Adapted from Spike's mmu_t::translate and mmu_t::walk
868 *
869 * @env: CPURISCVState
870 * @physical: This will be set to the calculated physical address
871 * @prot: The returned protection attributes
872 * @addr: The virtual address or guest physical address to be translated
873 * @fault_pte_addr: If not NULL, this will be set to fault pte address
874 * when a error occurs on pte address translation.
875 * This will already be shifted to match htval.
876 * @access_type: The type of MMU access
877 * @mmu_idx: Indicates current privilege level
878 * @first_stage: Are we in first stage translation?
879 * Second stage is used for hypervisor guest translation
880 * @two_stage: Are we going to perform two stage translation
881 * @is_debug: Is this access from a debugger or the monitor?
882 */
get_physical_address(CPURISCVState * env,hwaddr * physical,int * ret_prot,vaddr addr,target_ulong * fault_pte_addr,int access_type,int mmu_idx,bool first_stage,bool two_stage,bool is_debug,bool is_probe)883 static int get_physical_address(CPURISCVState *env, hwaddr *physical,
884 int *ret_prot, vaddr addr,
885 target_ulong *fault_pte_addr,
886 int access_type, int mmu_idx,
887 bool first_stage, bool two_stage,
888 bool is_debug, bool is_probe)
889 {
890 /*
891 * NOTE: the env->pc value visible here will not be
892 * correct, but the value visible to the exception handler
893 * (riscv_cpu_do_interrupt) is correct
894 */
895 MemTxResult res;
896 MemTxAttrs attrs = MEMTXATTRS_UNSPECIFIED;
897 int mode = mmuidx_priv(mmu_idx);
898 bool use_background = false;
899 hwaddr ppn;
900 int napot_bits = 0;
901 target_ulong napot_mask;
902 bool is_sstack_idx = ((mmu_idx & MMU_IDX_SS_WRITE) == MMU_IDX_SS_WRITE);
903 bool sstack_page = false;
904
905 /*
906 * Check if we should use the background registers for the two
907 * stage translation. We don't need to check if we actually need
908 * two stage translation as that happened before this function
909 * was called. Background registers will be used if the guest has
910 * forced a two stage translation to be on (in HS or M mode).
911 */
912 if (!env->virt_enabled && two_stage) {
913 use_background = true;
914 }
915
916 if (mode == PRV_M || !riscv_cpu_cfg(env)->mmu) {
917 *physical = addr;
918 *ret_prot = PAGE_READ | PAGE_WRITE | PAGE_EXEC;
919 return TRANSLATE_SUCCESS;
920 }
921
922 *ret_prot = 0;
923
924 hwaddr base;
925 int levels, ptidxbits, ptesize, vm, widened;
926
927 if (first_stage == true) {
928 if (use_background) {
929 if (riscv_cpu_mxl(env) == MXL_RV32) {
930 base = (hwaddr)get_field(env->vsatp, SATP32_PPN) << PGSHIFT;
931 vm = get_field(env->vsatp, SATP32_MODE);
932 } else {
933 base = (hwaddr)get_field(env->vsatp, SATP64_PPN) << PGSHIFT;
934 vm = get_field(env->vsatp, SATP64_MODE);
935 }
936 } else {
937 if (riscv_cpu_mxl(env) == MXL_RV32) {
938 base = (hwaddr)get_field(env->satp, SATP32_PPN) << PGSHIFT;
939 vm = get_field(env->satp, SATP32_MODE);
940 } else {
941 base = (hwaddr)get_field(env->satp, SATP64_PPN) << PGSHIFT;
942 vm = get_field(env->satp, SATP64_MODE);
943 }
944 }
945 widened = 0;
946 } else {
947 if (riscv_cpu_mxl(env) == MXL_RV32) {
948 base = (hwaddr)get_field(env->hgatp, SATP32_PPN) << PGSHIFT;
949 vm = get_field(env->hgatp, SATP32_MODE);
950 } else {
951 base = (hwaddr)get_field(env->hgatp, SATP64_PPN) << PGSHIFT;
952 vm = get_field(env->hgatp, SATP64_MODE);
953 }
954 widened = 2;
955 }
956
957 switch (vm) {
958 case VM_1_10_SV32:
959 levels = 2; ptidxbits = 10; ptesize = 4; break;
960 case VM_1_10_SV39:
961 levels = 3; ptidxbits = 9; ptesize = 8; break;
962 case VM_1_10_SV48:
963 levels = 4; ptidxbits = 9; ptesize = 8; break;
964 case VM_1_10_SV57:
965 levels = 5; ptidxbits = 9; ptesize = 8; break;
966 case VM_1_10_MBARE:
967 *physical = addr;
968 *ret_prot = PAGE_READ | PAGE_WRITE | PAGE_EXEC;
969 return TRANSLATE_SUCCESS;
970 default:
971 g_assert_not_reached();
972 }
973
974 CPUState *cs = env_cpu(env);
975 int va_bits = PGSHIFT + levels * ptidxbits + widened;
976 int sxlen = 16 << riscv_cpu_sxl(env);
977 int sxlen_bytes = sxlen / 8;
978
979 if (first_stage == true) {
980 target_ulong mask, masked_msbs;
981
982 if (sxlen > (va_bits - 1)) {
983 mask = (1L << (sxlen - (va_bits - 1))) - 1;
984 } else {
985 mask = 0;
986 }
987 masked_msbs = (addr >> (va_bits - 1)) & mask;
988
989 if (masked_msbs != 0 && masked_msbs != mask) {
990 return TRANSLATE_FAIL;
991 }
992 } else {
993 if (vm != VM_1_10_SV32 && addr >> va_bits != 0) {
994 return TRANSLATE_FAIL;
995 }
996 }
997
998 bool pbmte = env->menvcfg & MENVCFG_PBMTE;
999 bool svade = riscv_cpu_cfg(env)->ext_svade;
1000 bool svadu = riscv_cpu_cfg(env)->ext_svadu;
1001 bool adue = svadu ? env->menvcfg & MENVCFG_ADUE : !svade;
1002
1003 if (first_stage && two_stage && env->virt_enabled) {
1004 pbmte = pbmte && (env->henvcfg & HENVCFG_PBMTE);
1005 adue = adue && (env->henvcfg & HENVCFG_ADUE);
1006 }
1007
1008 int ptshift = (levels - 1) * ptidxbits;
1009 target_ulong pte;
1010 hwaddr pte_addr;
1011 int i;
1012
1013 #if !TCG_OVERSIZED_GUEST
1014 restart:
1015 #endif
1016 for (i = 0; i < levels; i++, ptshift -= ptidxbits) {
1017 target_ulong idx;
1018 if (i == 0) {
1019 idx = (addr >> (PGSHIFT + ptshift)) &
1020 ((1 << (ptidxbits + widened)) - 1);
1021 } else {
1022 idx = (addr >> (PGSHIFT + ptshift)) &
1023 ((1 << ptidxbits) - 1);
1024 }
1025
1026 /* check that physical address of PTE is legal */
1027
1028 if (two_stage && first_stage) {
1029 int vbase_prot;
1030 hwaddr vbase;
1031
1032 /* Do the second stage translation on the base PTE address. */
1033 int vbase_ret = get_physical_address(env, &vbase, &vbase_prot,
1034 base, NULL, MMU_DATA_LOAD,
1035 MMUIdx_U, false, true,
1036 is_debug, false);
1037
1038 if (vbase_ret != TRANSLATE_SUCCESS) {
1039 if (fault_pte_addr) {
1040 *fault_pte_addr = (base + idx * ptesize) >> 2;
1041 }
1042 return TRANSLATE_G_STAGE_FAIL;
1043 }
1044
1045 pte_addr = vbase + idx * ptesize;
1046 } else {
1047 pte_addr = base + idx * ptesize;
1048 }
1049
1050 int pmp_prot;
1051 int pmp_ret = get_physical_address_pmp(env, &pmp_prot, pte_addr,
1052 sxlen_bytes,
1053 MMU_DATA_LOAD, PRV_S);
1054 if (pmp_ret != TRANSLATE_SUCCESS) {
1055 return TRANSLATE_PMP_FAIL;
1056 }
1057
1058 if (riscv_cpu_mxl(env) == MXL_RV32) {
1059 pte = address_space_ldl(cs->as, pte_addr, attrs, &res);
1060 } else {
1061 pte = address_space_ldq(cs->as, pte_addr, attrs, &res);
1062 }
1063
1064 if (res != MEMTX_OK) {
1065 return TRANSLATE_FAIL;
1066 }
1067
1068 if (riscv_cpu_sxl(env) == MXL_RV32) {
1069 ppn = pte >> PTE_PPN_SHIFT;
1070 } else {
1071 if (pte & PTE_RESERVED) {
1072 return TRANSLATE_FAIL;
1073 }
1074
1075 if (!pbmte && (pte & PTE_PBMT)) {
1076 return TRANSLATE_FAIL;
1077 }
1078
1079 if (!riscv_cpu_cfg(env)->ext_svnapot && (pte & PTE_N)) {
1080 return TRANSLATE_FAIL;
1081 }
1082
1083 ppn = (pte & (target_ulong)PTE_PPN_MASK) >> PTE_PPN_SHIFT;
1084 }
1085
1086 if (!(pte & PTE_V)) {
1087 /* Invalid PTE */
1088 return TRANSLATE_FAIL;
1089 }
1090 if (pte & (PTE_R | PTE_W | PTE_X)) {
1091 goto leaf;
1092 }
1093
1094 /* Inner PTE, continue walking */
1095 if (pte & (PTE_D | PTE_A | PTE_U | PTE_ATTR)) {
1096 return TRANSLATE_FAIL;
1097 }
1098 base = ppn << PGSHIFT;
1099 }
1100
1101 /* No leaf pte at any translation level. */
1102 return TRANSLATE_FAIL;
1103
1104 leaf:
1105 if (ppn & ((1ULL << ptshift) - 1)) {
1106 /* Misaligned PPN */
1107 return TRANSLATE_FAIL;
1108 }
1109 if (!pbmte && (pte & PTE_PBMT)) {
1110 /* Reserved without Svpbmt. */
1111 return TRANSLATE_FAIL;
1112 }
1113
1114 target_ulong rwx = pte & (PTE_R | PTE_W | PTE_X);
1115 /* Check for reserved combinations of RWX flags. */
1116 switch (rwx) {
1117 case PTE_W | PTE_X:
1118 return TRANSLATE_FAIL;
1119 case PTE_W:
1120 /* if bcfi enabled, PTE_W is not reserved and shadow stack page */
1121 if (cpu_get_bcfien(env) && first_stage) {
1122 sstack_page = true;
1123 /*
1124 * if ss index, read and write allowed. else if not a probe
1125 * then only read allowed
1126 */
1127 rwx = is_sstack_idx ? (PTE_R | PTE_W) : (is_probe ? 0 : PTE_R);
1128 break;
1129 }
1130 return TRANSLATE_FAIL;
1131 case PTE_R:
1132 /*
1133 * no matter what's the `access_type`, shadow stack access to readonly
1134 * memory are always store page faults. During unwind, loads will be
1135 * promoted as store fault.
1136 */
1137 if (is_sstack_idx) {
1138 return TRANSLATE_FAIL;
1139 }
1140 break;
1141 }
1142
1143 int prot = 0;
1144 if (rwx & PTE_R) {
1145 prot |= PAGE_READ;
1146 }
1147 if (rwx & PTE_W) {
1148 prot |= PAGE_WRITE;
1149 }
1150 if (rwx & PTE_X) {
1151 bool mxr = false;
1152
1153 /*
1154 * Use mstatus for first stage or for the second stage without
1155 * virt_enabled (MPRV+MPV)
1156 */
1157 if (first_stage || !env->virt_enabled) {
1158 mxr = get_field(env->mstatus, MSTATUS_MXR);
1159 }
1160
1161 /* MPRV+MPV case, check VSSTATUS */
1162 if (first_stage && two_stage && !env->virt_enabled) {
1163 mxr |= get_field(env->vsstatus, MSTATUS_MXR);
1164 }
1165
1166 /*
1167 * Setting MXR at HS-level overrides both VS-stage and G-stage
1168 * execute-only permissions
1169 */
1170 if (env->virt_enabled) {
1171 mxr |= get_field(env->mstatus_hs, MSTATUS_MXR);
1172 }
1173
1174 if (mxr) {
1175 prot |= PAGE_READ;
1176 }
1177 prot |= PAGE_EXEC;
1178 }
1179
1180 if (pte & PTE_U) {
1181 if (mode != PRV_U) {
1182 if (!mmuidx_sum(mmu_idx)) {
1183 return TRANSLATE_FAIL;
1184 }
1185 /* SUM allows only read+write, not execute. */
1186 prot &= PAGE_READ | PAGE_WRITE;
1187 }
1188 } else if (mode != PRV_S) {
1189 /* Supervisor PTE flags when not S mode */
1190 return TRANSLATE_FAIL;
1191 }
1192
1193 if (!((prot >> access_type) & 1)) {
1194 /*
1195 * Access check failed, access check failures for shadow stack are
1196 * access faults.
1197 */
1198 return sstack_page ? TRANSLATE_PMP_FAIL : TRANSLATE_FAIL;
1199 }
1200
1201 target_ulong updated_pte = pte;
1202
1203 /*
1204 * If ADUE is enabled, set accessed and dirty bits.
1205 * Otherwise raise an exception if necessary.
1206 */
1207 if (adue) {
1208 updated_pte |= PTE_A | (access_type == MMU_DATA_STORE ? PTE_D : 0);
1209 } else if (!(pte & PTE_A) ||
1210 (access_type == MMU_DATA_STORE && !(pte & PTE_D))) {
1211 return TRANSLATE_FAIL;
1212 }
1213
1214 /* Page table updates need to be atomic with MTTCG enabled */
1215 if (updated_pte != pte && !is_debug) {
1216 if (!adue) {
1217 return TRANSLATE_FAIL;
1218 }
1219
1220 /*
1221 * - if accessed or dirty bits need updating, and the PTE is
1222 * in RAM, then we do so atomically with a compare and swap.
1223 * - if the PTE is in IO space or ROM, then it can't be updated
1224 * and we return TRANSLATE_FAIL.
1225 * - if the PTE changed by the time we went to update it, then
1226 * it is no longer valid and we must re-walk the page table.
1227 */
1228 MemoryRegion *mr;
1229 hwaddr l = sxlen_bytes, addr1;
1230 mr = address_space_translate(cs->as, pte_addr, &addr1, &l,
1231 false, MEMTXATTRS_UNSPECIFIED);
1232 if (memory_region_is_ram(mr)) {
1233 target_ulong *pte_pa = qemu_map_ram_ptr(mr->ram_block, addr1);
1234 #if TCG_OVERSIZED_GUEST
1235 /*
1236 * MTTCG is not enabled on oversized TCG guests so
1237 * page table updates do not need to be atomic
1238 */
1239 *pte_pa = pte = updated_pte;
1240 #else
1241 target_ulong old_pte;
1242 if (riscv_cpu_sxl(env) == MXL_RV32) {
1243 old_pte = qatomic_cmpxchg((uint32_t *)pte_pa, pte, updated_pte);
1244 } else {
1245 old_pte = qatomic_cmpxchg(pte_pa, pte, updated_pte);
1246 }
1247 if (old_pte != pte) {
1248 goto restart;
1249 }
1250 pte = updated_pte;
1251 #endif
1252 } else {
1253 /*
1254 * Misconfigured PTE in ROM (AD bits are not preset) or
1255 * PTE is in IO space and can't be updated atomically.
1256 */
1257 return TRANSLATE_FAIL;
1258 }
1259 }
1260
1261 /* For superpage mappings, make a fake leaf PTE for the TLB's benefit. */
1262 target_ulong vpn = addr >> PGSHIFT;
1263
1264 if (riscv_cpu_cfg(env)->ext_svnapot && (pte & PTE_N)) {
1265 napot_bits = ctzl(ppn) + 1;
1266 if ((i != (levels - 1)) || (napot_bits != 4)) {
1267 return TRANSLATE_FAIL;
1268 }
1269 }
1270
1271 napot_mask = (1 << napot_bits) - 1;
1272 *physical = (((ppn & ~napot_mask) | (vpn & napot_mask) |
1273 (vpn & (((target_ulong)1 << ptshift) - 1))
1274 ) << PGSHIFT) | (addr & ~TARGET_PAGE_MASK);
1275
1276 /*
1277 * Remove write permission unless this is a store, or the page is
1278 * already dirty, so that we TLB miss on later writes to update
1279 * the dirty bit.
1280 */
1281 if (access_type != MMU_DATA_STORE && !(pte & PTE_D)) {
1282 prot &= ~PAGE_WRITE;
1283 }
1284 *ret_prot = prot;
1285
1286 return TRANSLATE_SUCCESS;
1287 }
1288
raise_mmu_exception(CPURISCVState * env,target_ulong address,MMUAccessType access_type,bool pmp_violation,bool first_stage,bool two_stage,bool two_stage_indirect)1289 static void raise_mmu_exception(CPURISCVState *env, target_ulong address,
1290 MMUAccessType access_type, bool pmp_violation,
1291 bool first_stage, bool two_stage,
1292 bool two_stage_indirect)
1293 {
1294 CPUState *cs = env_cpu(env);
1295
1296 switch (access_type) {
1297 case MMU_INST_FETCH:
1298 if (pmp_violation) {
1299 cs->exception_index = RISCV_EXCP_INST_ACCESS_FAULT;
1300 } else if (env->virt_enabled && !first_stage) {
1301 cs->exception_index = RISCV_EXCP_INST_GUEST_PAGE_FAULT;
1302 } else {
1303 cs->exception_index = RISCV_EXCP_INST_PAGE_FAULT;
1304 }
1305 break;
1306 case MMU_DATA_LOAD:
1307 if (pmp_violation) {
1308 cs->exception_index = RISCV_EXCP_LOAD_ACCESS_FAULT;
1309 } else if (two_stage && !first_stage) {
1310 cs->exception_index = RISCV_EXCP_LOAD_GUEST_ACCESS_FAULT;
1311 } else {
1312 cs->exception_index = RISCV_EXCP_LOAD_PAGE_FAULT;
1313 }
1314 break;
1315 case MMU_DATA_STORE:
1316 if (pmp_violation) {
1317 cs->exception_index = RISCV_EXCP_STORE_AMO_ACCESS_FAULT;
1318 } else if (two_stage && !first_stage) {
1319 cs->exception_index = RISCV_EXCP_STORE_GUEST_AMO_ACCESS_FAULT;
1320 } else {
1321 cs->exception_index = RISCV_EXCP_STORE_PAGE_FAULT;
1322 }
1323 break;
1324 default:
1325 g_assert_not_reached();
1326 }
1327 env->badaddr = address;
1328 env->two_stage_lookup = two_stage;
1329 env->two_stage_indirect_lookup = two_stage_indirect;
1330 }
1331
riscv_cpu_get_phys_page_debug(CPUState * cs,vaddr addr)1332 hwaddr riscv_cpu_get_phys_page_debug(CPUState *cs, vaddr addr)
1333 {
1334 RISCVCPU *cpu = RISCV_CPU(cs);
1335 CPURISCVState *env = &cpu->env;
1336 hwaddr phys_addr;
1337 int prot;
1338 int mmu_idx = riscv_env_mmu_index(&cpu->env, false);
1339
1340 if (get_physical_address(env, &phys_addr, &prot, addr, NULL, 0, mmu_idx,
1341 true, env->virt_enabled, true, false)) {
1342 return -1;
1343 }
1344
1345 if (env->virt_enabled) {
1346 if (get_physical_address(env, &phys_addr, &prot, phys_addr, NULL,
1347 0, MMUIdx_U, false, true, true, false)) {
1348 return -1;
1349 }
1350 }
1351
1352 return phys_addr & TARGET_PAGE_MASK;
1353 }
1354
riscv_cpu_do_transaction_failed(CPUState * cs,hwaddr physaddr,vaddr addr,unsigned size,MMUAccessType access_type,int mmu_idx,MemTxAttrs attrs,MemTxResult response,uintptr_t retaddr)1355 void riscv_cpu_do_transaction_failed(CPUState *cs, hwaddr physaddr,
1356 vaddr addr, unsigned size,
1357 MMUAccessType access_type,
1358 int mmu_idx, MemTxAttrs attrs,
1359 MemTxResult response, uintptr_t retaddr)
1360 {
1361 RISCVCPU *cpu = RISCV_CPU(cs);
1362 CPURISCVState *env = &cpu->env;
1363
1364 if (access_type == MMU_DATA_STORE) {
1365 cs->exception_index = RISCV_EXCP_STORE_AMO_ACCESS_FAULT;
1366 } else if (access_type == MMU_DATA_LOAD) {
1367 cs->exception_index = RISCV_EXCP_LOAD_ACCESS_FAULT;
1368 } else {
1369 cs->exception_index = RISCV_EXCP_INST_ACCESS_FAULT;
1370 }
1371
1372 env->badaddr = addr;
1373 env->two_stage_lookup = mmuidx_2stage(mmu_idx);
1374 env->two_stage_indirect_lookup = false;
1375 cpu_loop_exit_restore(cs, retaddr);
1376 }
1377
riscv_cpu_do_unaligned_access(CPUState * cs,vaddr addr,MMUAccessType access_type,int mmu_idx,uintptr_t retaddr)1378 void riscv_cpu_do_unaligned_access(CPUState *cs, vaddr addr,
1379 MMUAccessType access_type, int mmu_idx,
1380 uintptr_t retaddr)
1381 {
1382 RISCVCPU *cpu = RISCV_CPU(cs);
1383 CPURISCVState *env = &cpu->env;
1384 switch (access_type) {
1385 case MMU_INST_FETCH:
1386 cs->exception_index = RISCV_EXCP_INST_ADDR_MIS;
1387 break;
1388 case MMU_DATA_LOAD:
1389 cs->exception_index = RISCV_EXCP_LOAD_ADDR_MIS;
1390 /* shadow stack mis aligned accesses are access faults */
1391 if (mmu_idx & MMU_IDX_SS_WRITE) {
1392 cs->exception_index = RISCV_EXCP_LOAD_ACCESS_FAULT;
1393 }
1394 break;
1395 case MMU_DATA_STORE:
1396 cs->exception_index = RISCV_EXCP_STORE_AMO_ADDR_MIS;
1397 /* shadow stack mis aligned accesses are access faults */
1398 if (mmu_idx & MMU_IDX_SS_WRITE) {
1399 cs->exception_index = RISCV_EXCP_STORE_AMO_ACCESS_FAULT;
1400 }
1401 break;
1402 default:
1403 g_assert_not_reached();
1404 }
1405 env->badaddr = addr;
1406 env->two_stage_lookup = mmuidx_2stage(mmu_idx);
1407 env->two_stage_indirect_lookup = false;
1408 cpu_loop_exit_restore(cs, retaddr);
1409 }
1410
1411
pmu_tlb_fill_incr_ctr(RISCVCPU * cpu,MMUAccessType access_type)1412 static void pmu_tlb_fill_incr_ctr(RISCVCPU *cpu, MMUAccessType access_type)
1413 {
1414 enum riscv_pmu_event_idx pmu_event_type;
1415
1416 switch (access_type) {
1417 case MMU_INST_FETCH:
1418 pmu_event_type = RISCV_PMU_EVENT_CACHE_ITLB_PREFETCH_MISS;
1419 break;
1420 case MMU_DATA_LOAD:
1421 pmu_event_type = RISCV_PMU_EVENT_CACHE_DTLB_READ_MISS;
1422 break;
1423 case MMU_DATA_STORE:
1424 pmu_event_type = RISCV_PMU_EVENT_CACHE_DTLB_WRITE_MISS;
1425 break;
1426 default:
1427 return;
1428 }
1429
1430 riscv_pmu_incr_ctr(cpu, pmu_event_type);
1431 }
1432
riscv_cpu_tlb_fill(CPUState * cs,vaddr address,int size,MMUAccessType access_type,int mmu_idx,bool probe,uintptr_t retaddr)1433 bool riscv_cpu_tlb_fill(CPUState *cs, vaddr address, int size,
1434 MMUAccessType access_type, int mmu_idx,
1435 bool probe, uintptr_t retaddr)
1436 {
1437 RISCVCPU *cpu = RISCV_CPU(cs);
1438 CPURISCVState *env = &cpu->env;
1439 vaddr im_address;
1440 hwaddr pa = 0;
1441 int prot, prot2, prot_pmp;
1442 bool pmp_violation = false;
1443 bool first_stage_error = true;
1444 bool two_stage_lookup = mmuidx_2stage(mmu_idx);
1445 bool two_stage_indirect_error = false;
1446 int ret = TRANSLATE_FAIL;
1447 int mode = mmuidx_priv(mmu_idx);
1448 /* default TLB page size */
1449 hwaddr tlb_size = TARGET_PAGE_SIZE;
1450
1451 env->guest_phys_fault_addr = 0;
1452
1453 qemu_log_mask(CPU_LOG_MMU, "%s ad %" VADDR_PRIx " rw %d mmu_idx %d\n",
1454 __func__, address, access_type, mmu_idx);
1455
1456 pmu_tlb_fill_incr_ctr(cpu, access_type);
1457 if (two_stage_lookup) {
1458 /* Two stage lookup */
1459 ret = get_physical_address(env, &pa, &prot, address,
1460 &env->guest_phys_fault_addr, access_type,
1461 mmu_idx, true, true, false, probe);
1462
1463 /*
1464 * A G-stage exception may be triggered during two state lookup.
1465 * And the env->guest_phys_fault_addr has already been set in
1466 * get_physical_address().
1467 */
1468 if (ret == TRANSLATE_G_STAGE_FAIL) {
1469 first_stage_error = false;
1470 two_stage_indirect_error = true;
1471 }
1472
1473 qemu_log_mask(CPU_LOG_MMU,
1474 "%s 1st-stage address=%" VADDR_PRIx " ret %d physical "
1475 HWADDR_FMT_plx " prot %d\n",
1476 __func__, address, ret, pa, prot);
1477
1478 if (ret == TRANSLATE_SUCCESS) {
1479 /* Second stage lookup */
1480 im_address = pa;
1481
1482 ret = get_physical_address(env, &pa, &prot2, im_address, NULL,
1483 access_type, MMUIdx_U, false, true,
1484 false, probe);
1485
1486 qemu_log_mask(CPU_LOG_MMU,
1487 "%s 2nd-stage address=%" VADDR_PRIx
1488 " ret %d physical "
1489 HWADDR_FMT_plx " prot %d\n",
1490 __func__, im_address, ret, pa, prot2);
1491
1492 prot &= prot2;
1493
1494 if (ret == TRANSLATE_SUCCESS) {
1495 ret = get_physical_address_pmp(env, &prot_pmp, pa,
1496 size, access_type, mode);
1497 tlb_size = pmp_get_tlb_size(env, pa);
1498
1499 qemu_log_mask(CPU_LOG_MMU,
1500 "%s PMP address=" HWADDR_FMT_plx " ret %d prot"
1501 " %d tlb_size %" HWADDR_PRIu "\n",
1502 __func__, pa, ret, prot_pmp, tlb_size);
1503
1504 prot &= prot_pmp;
1505 } else {
1506 /*
1507 * Guest physical address translation failed, this is a HS
1508 * level exception
1509 */
1510 first_stage_error = false;
1511 if (ret != TRANSLATE_PMP_FAIL) {
1512 env->guest_phys_fault_addr = (im_address |
1513 (address &
1514 (TARGET_PAGE_SIZE - 1))) >> 2;
1515 }
1516 }
1517 }
1518 } else {
1519 /* Single stage lookup */
1520 ret = get_physical_address(env, &pa, &prot, address, NULL,
1521 access_type, mmu_idx, true, false, false,
1522 probe);
1523
1524 qemu_log_mask(CPU_LOG_MMU,
1525 "%s address=%" VADDR_PRIx " ret %d physical "
1526 HWADDR_FMT_plx " prot %d\n",
1527 __func__, address, ret, pa, prot);
1528
1529 if (ret == TRANSLATE_SUCCESS) {
1530 ret = get_physical_address_pmp(env, &prot_pmp, pa,
1531 size, access_type, mode);
1532 tlb_size = pmp_get_tlb_size(env, pa);
1533
1534 qemu_log_mask(CPU_LOG_MMU,
1535 "%s PMP address=" HWADDR_FMT_plx " ret %d prot"
1536 " %d tlb_size %" HWADDR_PRIu "\n",
1537 __func__, pa, ret, prot_pmp, tlb_size);
1538
1539 prot &= prot_pmp;
1540 }
1541 }
1542
1543 if (ret == TRANSLATE_PMP_FAIL) {
1544 pmp_violation = true;
1545 }
1546
1547 if (ret == TRANSLATE_SUCCESS) {
1548 tlb_set_page(cs, address & ~(tlb_size - 1), pa & ~(tlb_size - 1),
1549 prot, mmu_idx, tlb_size);
1550 return true;
1551 } else if (probe) {
1552 return false;
1553 } else {
1554 int wp_access = 0;
1555
1556 if (access_type == MMU_DATA_LOAD) {
1557 wp_access |= BP_MEM_READ;
1558 } else if (access_type == MMU_DATA_STORE) {
1559 wp_access |= BP_MEM_WRITE;
1560 }
1561
1562 /*
1563 * If a watchpoint isn't found for 'addr' this will
1564 * be a no-op and we'll resume the mmu_exception path.
1565 * Otherwise we'll throw a debug exception and execution
1566 * will continue elsewhere.
1567 */
1568 cpu_check_watchpoint(cs, address, size, MEMTXATTRS_UNSPECIFIED,
1569 wp_access, retaddr);
1570
1571 raise_mmu_exception(env, address, access_type, pmp_violation,
1572 first_stage_error, two_stage_lookup,
1573 two_stage_indirect_error);
1574 cpu_loop_exit_restore(cs, retaddr);
1575 }
1576
1577 return true;
1578 }
1579
riscv_transformed_insn(CPURISCVState * env,target_ulong insn,target_ulong taddr)1580 static target_ulong riscv_transformed_insn(CPURISCVState *env,
1581 target_ulong insn,
1582 target_ulong taddr)
1583 {
1584 target_ulong xinsn = 0;
1585 target_ulong access_rs1 = 0, access_imm = 0, access_size = 0;
1586
1587 /*
1588 * Only Quadrant 0 and Quadrant 2 of RVC instruction space need to
1589 * be uncompressed. The Quadrant 1 of RVC instruction space need
1590 * not be transformed because these instructions won't generate
1591 * any load/store trap.
1592 */
1593
1594 if ((insn & 0x3) != 0x3) {
1595 /* Transform 16bit instruction into 32bit instruction */
1596 switch (GET_C_OP(insn)) {
1597 case OPC_RISC_C_OP_QUAD0: /* Quadrant 0 */
1598 switch (GET_C_FUNC(insn)) {
1599 case OPC_RISC_C_FUNC_FLD_LQ:
1600 if (riscv_cpu_xlen(env) != 128) { /* C.FLD (RV32/64) */
1601 xinsn = OPC_RISC_FLD;
1602 xinsn = SET_RD(xinsn, GET_C_RS2S(insn));
1603 access_rs1 = GET_C_RS1S(insn);
1604 access_imm = GET_C_LD_IMM(insn);
1605 access_size = 8;
1606 }
1607 break;
1608 case OPC_RISC_C_FUNC_LW: /* C.LW */
1609 xinsn = OPC_RISC_LW;
1610 xinsn = SET_RD(xinsn, GET_C_RS2S(insn));
1611 access_rs1 = GET_C_RS1S(insn);
1612 access_imm = GET_C_LW_IMM(insn);
1613 access_size = 4;
1614 break;
1615 case OPC_RISC_C_FUNC_FLW_LD:
1616 if (riscv_cpu_xlen(env) == 32) { /* C.FLW (RV32) */
1617 xinsn = OPC_RISC_FLW;
1618 xinsn = SET_RD(xinsn, GET_C_RS2S(insn));
1619 access_rs1 = GET_C_RS1S(insn);
1620 access_imm = GET_C_LW_IMM(insn);
1621 access_size = 4;
1622 } else { /* C.LD (RV64/RV128) */
1623 xinsn = OPC_RISC_LD;
1624 xinsn = SET_RD(xinsn, GET_C_RS2S(insn));
1625 access_rs1 = GET_C_RS1S(insn);
1626 access_imm = GET_C_LD_IMM(insn);
1627 access_size = 8;
1628 }
1629 break;
1630 case OPC_RISC_C_FUNC_FSD_SQ:
1631 if (riscv_cpu_xlen(env) != 128) { /* C.FSD (RV32/64) */
1632 xinsn = OPC_RISC_FSD;
1633 xinsn = SET_RS2(xinsn, GET_C_RS2S(insn));
1634 access_rs1 = GET_C_RS1S(insn);
1635 access_imm = GET_C_SD_IMM(insn);
1636 access_size = 8;
1637 }
1638 break;
1639 case OPC_RISC_C_FUNC_SW: /* C.SW */
1640 xinsn = OPC_RISC_SW;
1641 xinsn = SET_RS2(xinsn, GET_C_RS2S(insn));
1642 access_rs1 = GET_C_RS1S(insn);
1643 access_imm = GET_C_SW_IMM(insn);
1644 access_size = 4;
1645 break;
1646 case OPC_RISC_C_FUNC_FSW_SD:
1647 if (riscv_cpu_xlen(env) == 32) { /* C.FSW (RV32) */
1648 xinsn = OPC_RISC_FSW;
1649 xinsn = SET_RS2(xinsn, GET_C_RS2S(insn));
1650 access_rs1 = GET_C_RS1S(insn);
1651 access_imm = GET_C_SW_IMM(insn);
1652 access_size = 4;
1653 } else { /* C.SD (RV64/RV128) */
1654 xinsn = OPC_RISC_SD;
1655 xinsn = SET_RS2(xinsn, GET_C_RS2S(insn));
1656 access_rs1 = GET_C_RS1S(insn);
1657 access_imm = GET_C_SD_IMM(insn);
1658 access_size = 8;
1659 }
1660 break;
1661 default:
1662 break;
1663 }
1664 break;
1665 case OPC_RISC_C_OP_QUAD2: /* Quadrant 2 */
1666 switch (GET_C_FUNC(insn)) {
1667 case OPC_RISC_C_FUNC_FLDSP_LQSP:
1668 if (riscv_cpu_xlen(env) != 128) { /* C.FLDSP (RV32/64) */
1669 xinsn = OPC_RISC_FLD;
1670 xinsn = SET_RD(xinsn, GET_C_RD(insn));
1671 access_rs1 = 2;
1672 access_imm = GET_C_LDSP_IMM(insn);
1673 access_size = 8;
1674 }
1675 break;
1676 case OPC_RISC_C_FUNC_LWSP: /* C.LWSP */
1677 xinsn = OPC_RISC_LW;
1678 xinsn = SET_RD(xinsn, GET_C_RD(insn));
1679 access_rs1 = 2;
1680 access_imm = GET_C_LWSP_IMM(insn);
1681 access_size = 4;
1682 break;
1683 case OPC_RISC_C_FUNC_FLWSP_LDSP:
1684 if (riscv_cpu_xlen(env) == 32) { /* C.FLWSP (RV32) */
1685 xinsn = OPC_RISC_FLW;
1686 xinsn = SET_RD(xinsn, GET_C_RD(insn));
1687 access_rs1 = 2;
1688 access_imm = GET_C_LWSP_IMM(insn);
1689 access_size = 4;
1690 } else { /* C.LDSP (RV64/RV128) */
1691 xinsn = OPC_RISC_LD;
1692 xinsn = SET_RD(xinsn, GET_C_RD(insn));
1693 access_rs1 = 2;
1694 access_imm = GET_C_LDSP_IMM(insn);
1695 access_size = 8;
1696 }
1697 break;
1698 case OPC_RISC_C_FUNC_FSDSP_SQSP:
1699 if (riscv_cpu_xlen(env) != 128) { /* C.FSDSP (RV32/64) */
1700 xinsn = OPC_RISC_FSD;
1701 xinsn = SET_RS2(xinsn, GET_C_RS2(insn));
1702 access_rs1 = 2;
1703 access_imm = GET_C_SDSP_IMM(insn);
1704 access_size = 8;
1705 }
1706 break;
1707 case OPC_RISC_C_FUNC_SWSP: /* C.SWSP */
1708 xinsn = OPC_RISC_SW;
1709 xinsn = SET_RS2(xinsn, GET_C_RS2(insn));
1710 access_rs1 = 2;
1711 access_imm = GET_C_SWSP_IMM(insn);
1712 access_size = 4;
1713 break;
1714 case 7:
1715 if (riscv_cpu_xlen(env) == 32) { /* C.FSWSP (RV32) */
1716 xinsn = OPC_RISC_FSW;
1717 xinsn = SET_RS2(xinsn, GET_C_RS2(insn));
1718 access_rs1 = 2;
1719 access_imm = GET_C_SWSP_IMM(insn);
1720 access_size = 4;
1721 } else { /* C.SDSP (RV64/RV128) */
1722 xinsn = OPC_RISC_SD;
1723 xinsn = SET_RS2(xinsn, GET_C_RS2(insn));
1724 access_rs1 = 2;
1725 access_imm = GET_C_SDSP_IMM(insn);
1726 access_size = 8;
1727 }
1728 break;
1729 default:
1730 break;
1731 }
1732 break;
1733 default:
1734 break;
1735 }
1736
1737 /*
1738 * Clear Bit1 of transformed instruction to indicate that
1739 * original insruction was a 16bit instruction
1740 */
1741 xinsn &= ~((target_ulong)0x2);
1742 } else {
1743 /* Transform 32bit (or wider) instructions */
1744 switch (MASK_OP_MAJOR(insn)) {
1745 case OPC_RISC_ATOMIC:
1746 xinsn = insn;
1747 access_rs1 = GET_RS1(insn);
1748 access_size = 1 << GET_FUNCT3(insn);
1749 break;
1750 case OPC_RISC_LOAD:
1751 case OPC_RISC_FP_LOAD:
1752 xinsn = SET_I_IMM(insn, 0);
1753 access_rs1 = GET_RS1(insn);
1754 access_imm = GET_IMM(insn);
1755 access_size = 1 << GET_FUNCT3(insn);
1756 break;
1757 case OPC_RISC_STORE:
1758 case OPC_RISC_FP_STORE:
1759 xinsn = SET_S_IMM(insn, 0);
1760 access_rs1 = GET_RS1(insn);
1761 access_imm = GET_STORE_IMM(insn);
1762 access_size = 1 << GET_FUNCT3(insn);
1763 break;
1764 case OPC_RISC_SYSTEM:
1765 if (MASK_OP_SYSTEM(insn) == OPC_RISC_HLVHSV) {
1766 xinsn = insn;
1767 access_rs1 = GET_RS1(insn);
1768 access_size = 1 << ((GET_FUNCT7(insn) >> 1) & 0x3);
1769 access_size = 1 << access_size;
1770 }
1771 break;
1772 default:
1773 break;
1774 }
1775 }
1776
1777 if (access_size) {
1778 xinsn = SET_RS1(xinsn, (taddr - (env->gpr[access_rs1] + access_imm)) &
1779 (access_size - 1));
1780 }
1781
1782 return xinsn;
1783 }
1784
promote_load_fault(target_ulong orig_cause)1785 static target_ulong promote_load_fault(target_ulong orig_cause)
1786 {
1787 switch (orig_cause) {
1788 case RISCV_EXCP_LOAD_GUEST_ACCESS_FAULT:
1789 return RISCV_EXCP_STORE_GUEST_AMO_ACCESS_FAULT;
1790
1791 case RISCV_EXCP_LOAD_ACCESS_FAULT:
1792 return RISCV_EXCP_STORE_AMO_ACCESS_FAULT;
1793
1794 case RISCV_EXCP_LOAD_PAGE_FAULT:
1795 return RISCV_EXCP_STORE_PAGE_FAULT;
1796 }
1797
1798 /* if no promotion, return original cause */
1799 return orig_cause;
1800 }
1801 /*
1802 * Handle Traps
1803 *
1804 * Adapted from Spike's processor_t::take_trap.
1805 *
1806 */
riscv_cpu_do_interrupt(CPUState * cs)1807 void riscv_cpu_do_interrupt(CPUState *cs)
1808 {
1809 RISCVCPU *cpu = RISCV_CPU(cs);
1810 CPURISCVState *env = &cpu->env;
1811 bool virt = env->virt_enabled;
1812 bool write_gva = false;
1813 bool always_storeamo = (env->excp_uw2 & RISCV_UW2_ALWAYS_STORE_AMO);
1814 uint64_t s;
1815
1816 /*
1817 * cs->exception is 32-bits wide unlike mcause which is XLEN-bits wide
1818 * so we mask off the MSB and separate into trap type and cause.
1819 */
1820 bool async = !!(cs->exception_index & RISCV_EXCP_INT_FLAG);
1821 target_ulong cause = cs->exception_index & RISCV_EXCP_INT_MASK;
1822 uint64_t deleg = async ? env->mideleg : env->medeleg;
1823 bool s_injected = env->mvip & (1ULL << cause) & env->mvien &&
1824 !(env->mip & (1ULL << cause));
1825 bool vs_injected = env->hvip & (1ULL << cause) & env->hvien &&
1826 !(env->mip & (1ULL << cause));
1827 target_ulong tval = 0;
1828 target_ulong tinst = 0;
1829 target_ulong htval = 0;
1830 target_ulong mtval2 = 0;
1831 int sxlen = 0;
1832 int mxlen = 0;
1833
1834 if (!async) {
1835 /* set tval to badaddr for traps with address information */
1836 switch (cause) {
1837 #ifdef CONFIG_TCG
1838 case RISCV_EXCP_SEMIHOST:
1839 do_common_semihosting(cs);
1840 env->pc += 4;
1841 return;
1842 #endif
1843 case RISCV_EXCP_LOAD_GUEST_ACCESS_FAULT:
1844 case RISCV_EXCP_STORE_GUEST_AMO_ACCESS_FAULT:
1845 case RISCV_EXCP_LOAD_ADDR_MIS:
1846 case RISCV_EXCP_STORE_AMO_ADDR_MIS:
1847 case RISCV_EXCP_LOAD_ACCESS_FAULT:
1848 case RISCV_EXCP_STORE_AMO_ACCESS_FAULT:
1849 case RISCV_EXCP_LOAD_PAGE_FAULT:
1850 case RISCV_EXCP_STORE_PAGE_FAULT:
1851 if (always_storeamo) {
1852 cause = promote_load_fault(cause);
1853 }
1854 write_gva = env->two_stage_lookup;
1855 tval = env->badaddr;
1856 if (env->two_stage_indirect_lookup) {
1857 /*
1858 * special pseudoinstruction for G-stage fault taken while
1859 * doing VS-stage page table walk.
1860 */
1861 tinst = (riscv_cpu_xlen(env) == 32) ? 0x00002000 : 0x00003000;
1862 } else {
1863 /*
1864 * The "Addr. Offset" field in transformed instruction is
1865 * non-zero only for misaligned access.
1866 */
1867 tinst = riscv_transformed_insn(env, env->bins, tval);
1868 }
1869 break;
1870 case RISCV_EXCP_INST_GUEST_PAGE_FAULT:
1871 case RISCV_EXCP_INST_ADDR_MIS:
1872 case RISCV_EXCP_INST_ACCESS_FAULT:
1873 case RISCV_EXCP_INST_PAGE_FAULT:
1874 write_gva = env->two_stage_lookup;
1875 tval = env->badaddr;
1876 if (env->two_stage_indirect_lookup) {
1877 /*
1878 * special pseudoinstruction for G-stage fault taken while
1879 * doing VS-stage page table walk.
1880 */
1881 tinst = (riscv_cpu_xlen(env) == 32) ? 0x00002000 : 0x00003000;
1882 }
1883 break;
1884 case RISCV_EXCP_ILLEGAL_INST:
1885 case RISCV_EXCP_VIRT_INSTRUCTION_FAULT:
1886 tval = env->bins;
1887 break;
1888 case RISCV_EXCP_BREAKPOINT:
1889 tval = env->badaddr;
1890 if (cs->watchpoint_hit) {
1891 tval = cs->watchpoint_hit->hitaddr;
1892 cs->watchpoint_hit = NULL;
1893 }
1894 break;
1895 case RISCV_EXCP_SW_CHECK:
1896 tval = env->sw_check_code;
1897 break;
1898 default:
1899 break;
1900 }
1901 /* ecall is dispatched as one cause so translate based on mode */
1902 if (cause == RISCV_EXCP_U_ECALL) {
1903 assert(env->priv <= 3);
1904
1905 if (env->priv == PRV_M) {
1906 cause = RISCV_EXCP_M_ECALL;
1907 } else if (env->priv == PRV_S && env->virt_enabled) {
1908 cause = RISCV_EXCP_VS_ECALL;
1909 } else if (env->priv == PRV_S && !env->virt_enabled) {
1910 cause = RISCV_EXCP_S_ECALL;
1911 } else if (env->priv == PRV_U) {
1912 cause = RISCV_EXCP_U_ECALL;
1913 }
1914 }
1915 }
1916
1917 trace_riscv_trap(env->mhartid, async, cause, env->pc, tval,
1918 riscv_cpu_get_trap_name(cause, async));
1919
1920 qemu_log_mask(CPU_LOG_INT,
1921 "%s: hart:"TARGET_FMT_ld", async:%d, cause:"TARGET_FMT_lx", "
1922 "epc:0x"TARGET_FMT_lx", tval:0x"TARGET_FMT_lx", desc=%s\n",
1923 __func__, env->mhartid, async, cause, env->pc, tval,
1924 riscv_cpu_get_trap_name(cause, async));
1925
1926 if (env->priv <= PRV_S && cause < 64 &&
1927 (((deleg >> cause) & 1) || s_injected || vs_injected)) {
1928 /* handle the trap in S-mode */
1929 /* save elp status */
1930 if (cpu_get_fcfien(env)) {
1931 env->mstatus = set_field(env->mstatus, MSTATUS_SPELP, env->elp);
1932 }
1933
1934 if (riscv_has_ext(env, RVH)) {
1935 uint64_t hdeleg = async ? env->hideleg : env->hedeleg;
1936
1937 if (env->virt_enabled &&
1938 (((hdeleg >> cause) & 1) || vs_injected)) {
1939 /* Trap to VS mode */
1940 /*
1941 * See if we need to adjust cause. Yes if its VS mode interrupt
1942 * no if hypervisor has delegated one of hs mode's interrupt
1943 */
1944 if (async && (cause == IRQ_VS_TIMER || cause == IRQ_VS_SOFT ||
1945 cause == IRQ_VS_EXT)) {
1946 cause = cause - 1;
1947 }
1948 write_gva = false;
1949 } else if (env->virt_enabled) {
1950 /* Trap into HS mode, from virt */
1951 riscv_cpu_swap_hypervisor_regs(env);
1952 env->hstatus = set_field(env->hstatus, HSTATUS_SPVP,
1953 env->priv);
1954 env->hstatus = set_field(env->hstatus, HSTATUS_SPV, true);
1955
1956 htval = env->guest_phys_fault_addr;
1957
1958 virt = false;
1959 } else {
1960 /* Trap into HS mode */
1961 env->hstatus = set_field(env->hstatus, HSTATUS_SPV, false);
1962 htval = env->guest_phys_fault_addr;
1963 }
1964 env->hstatus = set_field(env->hstatus, HSTATUS_GVA, write_gva);
1965 }
1966
1967 s = env->mstatus;
1968 s = set_field(s, MSTATUS_SPIE, get_field(s, MSTATUS_SIE));
1969 s = set_field(s, MSTATUS_SPP, env->priv);
1970 s = set_field(s, MSTATUS_SIE, 0);
1971 env->mstatus = s;
1972 sxlen = 16 << riscv_cpu_sxl(env);
1973 env->scause = cause | ((target_ulong)async << (sxlen - 1));
1974 env->sepc = env->pc;
1975 env->stval = tval;
1976 env->htval = htval;
1977 env->htinst = tinst;
1978 env->pc = (env->stvec >> 2 << 2) +
1979 ((async && (env->stvec & 3) == 1) ? cause * 4 : 0);
1980 riscv_cpu_set_mode(env, PRV_S, virt);
1981 } else {
1982 /* handle the trap in M-mode */
1983 /* save elp status */
1984 if (cpu_get_fcfien(env)) {
1985 env->mstatus = set_field(env->mstatus, MSTATUS_MPELP, env->elp);
1986 }
1987
1988 if (riscv_has_ext(env, RVH)) {
1989 if (env->virt_enabled) {
1990 riscv_cpu_swap_hypervisor_regs(env);
1991 }
1992 env->mstatus = set_field(env->mstatus, MSTATUS_MPV,
1993 env->virt_enabled);
1994 if (env->virt_enabled && tval) {
1995 env->mstatus = set_field(env->mstatus, MSTATUS_GVA, 1);
1996 }
1997
1998 mtval2 = env->guest_phys_fault_addr;
1999
2000 /* Trapping to M mode, virt is disabled */
2001 virt = false;
2002 }
2003
2004 s = env->mstatus;
2005 s = set_field(s, MSTATUS_MPIE, get_field(s, MSTATUS_MIE));
2006 s = set_field(s, MSTATUS_MPP, env->priv);
2007 s = set_field(s, MSTATUS_MIE, 0);
2008 env->mstatus = s;
2009 mxlen = 16 << riscv_cpu_mxl(env);
2010 env->mcause = cause | ((target_ulong)async << (mxlen - 1));
2011 env->mepc = env->pc;
2012 env->mtval = tval;
2013 env->mtval2 = mtval2;
2014 env->mtinst = tinst;
2015 env->pc = (env->mtvec >> 2 << 2) +
2016 ((async && (env->mtvec & 3) == 1) ? cause * 4 : 0);
2017 riscv_cpu_set_mode(env, PRV_M, virt);
2018 }
2019
2020 /*
2021 * Interrupt/exception/trap delivery is asynchronous event and as per
2022 * zicfilp spec CPU should clear up the ELP state. No harm in clearing
2023 * unconditionally.
2024 */
2025 env->elp = false;
2026
2027 /*
2028 * NOTE: it is not necessary to yield load reservations here. It is only
2029 * necessary for an SC from "another hart" to cause a load reservation
2030 * to be yielded. Refer to the memory consistency model section of the
2031 * RISC-V ISA Specification.
2032 */
2033
2034 env->two_stage_lookup = false;
2035 env->two_stage_indirect_lookup = false;
2036 }
2037
2038 #endif /* !CONFIG_USER_ONLY */
2039