1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (C) 2019 Western Digital Corporation or its affiliates.
4 * Copyright (c) 2022 Ventana Micro Systems Inc.
5 */
6
7 #include <linux/bitops.h>
8 #include <linux/kvm_host.h>
9
10 #define INSN_OPCODE_MASK 0x007c
11 #define INSN_OPCODE_SHIFT 2
12 #define INSN_OPCODE_SYSTEM 28
13
14 #define INSN_MASK_WFI 0xffffffff
15 #define INSN_MATCH_WFI 0x10500073
16
17 #define INSN_MATCH_CSRRW 0x1073
18 #define INSN_MASK_CSRRW 0x707f
19 #define INSN_MATCH_CSRRS 0x2073
20 #define INSN_MASK_CSRRS 0x707f
21 #define INSN_MATCH_CSRRC 0x3073
22 #define INSN_MASK_CSRRC 0x707f
23 #define INSN_MATCH_CSRRWI 0x5073
24 #define INSN_MASK_CSRRWI 0x707f
25 #define INSN_MATCH_CSRRSI 0x6073
26 #define INSN_MASK_CSRRSI 0x707f
27 #define INSN_MATCH_CSRRCI 0x7073
28 #define INSN_MASK_CSRRCI 0x707f
29
30 #define INSN_MATCH_LB 0x3
31 #define INSN_MASK_LB 0x707f
32 #define INSN_MATCH_LH 0x1003
33 #define INSN_MASK_LH 0x707f
34 #define INSN_MATCH_LW 0x2003
35 #define INSN_MASK_LW 0x707f
36 #define INSN_MATCH_LD 0x3003
37 #define INSN_MASK_LD 0x707f
38 #define INSN_MATCH_LBU 0x4003
39 #define INSN_MASK_LBU 0x707f
40 #define INSN_MATCH_LHU 0x5003
41 #define INSN_MASK_LHU 0x707f
42 #define INSN_MATCH_LWU 0x6003
43 #define INSN_MASK_LWU 0x707f
44 #define INSN_MATCH_SB 0x23
45 #define INSN_MASK_SB 0x707f
46 #define INSN_MATCH_SH 0x1023
47 #define INSN_MASK_SH 0x707f
48 #define INSN_MATCH_SW 0x2023
49 #define INSN_MASK_SW 0x707f
50 #define INSN_MATCH_SD 0x3023
51 #define INSN_MASK_SD 0x707f
52
53 #define INSN_MATCH_C_LD 0x6000
54 #define INSN_MASK_C_LD 0xe003
55 #define INSN_MATCH_C_SD 0xe000
56 #define INSN_MASK_C_SD 0xe003
57 #define INSN_MATCH_C_LW 0x4000
58 #define INSN_MASK_C_LW 0xe003
59 #define INSN_MATCH_C_SW 0xc000
60 #define INSN_MASK_C_SW 0xe003
61 #define INSN_MATCH_C_LDSP 0x6002
62 #define INSN_MASK_C_LDSP 0xe003
63 #define INSN_MATCH_C_SDSP 0xe002
64 #define INSN_MASK_C_SDSP 0xe003
65 #define INSN_MATCH_C_LWSP 0x4002
66 #define INSN_MASK_C_LWSP 0xe003
67 #define INSN_MATCH_C_SWSP 0xc002
68 #define INSN_MASK_C_SWSP 0xe003
69
70 #define INSN_16BIT_MASK 0x3
71
72 #define INSN_IS_16BIT(insn) (((insn) & INSN_16BIT_MASK) != INSN_16BIT_MASK)
73
74 #define INSN_LEN(insn) (INSN_IS_16BIT(insn) ? 2 : 4)
75
76 #ifdef CONFIG_64BIT
77 #define LOG_REGBYTES 3
78 #else
79 #define LOG_REGBYTES 2
80 #endif
81 #define REGBYTES (1 << LOG_REGBYTES)
82
83 #define SH_RD 7
84 #define SH_RS1 15
85 #define SH_RS2 20
86 #define SH_RS2C 2
87 #define MASK_RX 0x1f
88
89 #define RV_X(x, s, n) (((x) >> (s)) & ((1 << (n)) - 1))
90 #define RVC_LW_IMM(x) ((RV_X(x, 6, 1) << 2) | \
91 (RV_X(x, 10, 3) << 3) | \
92 (RV_X(x, 5, 1) << 6))
93 #define RVC_LD_IMM(x) ((RV_X(x, 10, 3) << 3) | \
94 (RV_X(x, 5, 2) << 6))
95 #define RVC_LWSP_IMM(x) ((RV_X(x, 4, 3) << 2) | \
96 (RV_X(x, 12, 1) << 5) | \
97 (RV_X(x, 2, 2) << 6))
98 #define RVC_LDSP_IMM(x) ((RV_X(x, 5, 2) << 3) | \
99 (RV_X(x, 12, 1) << 5) | \
100 (RV_X(x, 2, 3) << 6))
101 #define RVC_SWSP_IMM(x) ((RV_X(x, 9, 4) << 2) | \
102 (RV_X(x, 7, 2) << 6))
103 #define RVC_SDSP_IMM(x) ((RV_X(x, 10, 3) << 3) | \
104 (RV_X(x, 7, 3) << 6))
105 #define RVC_RS1S(insn) (8 + RV_X(insn, SH_RD, 3))
106 #define RVC_RS2S(insn) (8 + RV_X(insn, SH_RS2C, 3))
107 #define RVC_RS2(insn) RV_X(insn, SH_RS2C, 5)
108
109 #define SHIFT_RIGHT(x, y) \
110 ((y) < 0 ? ((x) << -(y)) : ((x) >> (y)))
111
112 #define REG_MASK \
113 ((1 << (5 + LOG_REGBYTES)) - (1 << LOG_REGBYTES))
114
115 #define REG_OFFSET(insn, pos) \
116 (SHIFT_RIGHT((insn), (pos) - LOG_REGBYTES) & REG_MASK)
117
118 #define REG_PTR(insn, pos, regs) \
119 ((ulong *)((ulong)(regs) + REG_OFFSET(insn, pos)))
120
121 #define GET_FUNCT3(insn) (((insn) >> 12) & 7)
122
123 #define GET_RS1(insn, regs) (*REG_PTR(insn, SH_RS1, regs))
124 #define GET_RS2(insn, regs) (*REG_PTR(insn, SH_RS2, regs))
125 #define GET_RS1S(insn, regs) (*REG_PTR(RVC_RS1S(insn), 0, regs))
126 #define GET_RS2S(insn, regs) (*REG_PTR(RVC_RS2S(insn), 0, regs))
127 #define GET_RS2C(insn, regs) (*REG_PTR(insn, SH_RS2C, regs))
128 #define GET_SP(regs) (*REG_PTR(2, 0, regs))
129 #define SET_RD(insn, regs, val) (*REG_PTR(insn, SH_RD, regs) = (val))
130 #define IMM_I(insn) ((s32)(insn) >> 20)
131 #define IMM_S(insn) (((s32)(insn) >> 25 << 5) | \
132 (s32)(((insn) >> 7) & 0x1f))
133
134 struct insn_func {
135 unsigned long mask;
136 unsigned long match;
137 /*
138 * Possible return values are as follows:
139 * 1) Returns < 0 for error case
140 * 2) Returns 0 for exit to user-space
141 * 3) Returns 1 to continue with next sepc
142 * 4) Returns 2 to continue with same sepc
143 * 5) Returns 3 to inject illegal instruction trap and continue
144 * 6) Returns 4 to inject virtual instruction trap and continue
145 *
146 * Use enum kvm_insn_return for return values
147 */
148 int (*func)(struct kvm_vcpu *vcpu, struct kvm_run *run, ulong insn);
149 };
150
truly_illegal_insn(struct kvm_vcpu * vcpu,struct kvm_run * run,ulong insn)151 static int truly_illegal_insn(struct kvm_vcpu *vcpu, struct kvm_run *run,
152 ulong insn)
153 {
154 struct kvm_cpu_trap utrap = { 0 };
155
156 /* Redirect trap to Guest VCPU */
157 utrap.sepc = vcpu->arch.guest_context.sepc;
158 utrap.scause = EXC_INST_ILLEGAL;
159 utrap.stval = insn;
160 utrap.htval = 0;
161 utrap.htinst = 0;
162 kvm_riscv_vcpu_trap_redirect(vcpu, &utrap);
163
164 return 1;
165 }
166
truly_virtual_insn(struct kvm_vcpu * vcpu,struct kvm_run * run,ulong insn)167 static int truly_virtual_insn(struct kvm_vcpu *vcpu, struct kvm_run *run,
168 ulong insn)
169 {
170 struct kvm_cpu_trap utrap = { 0 };
171
172 /* Redirect trap to Guest VCPU */
173 utrap.sepc = vcpu->arch.guest_context.sepc;
174 utrap.scause = EXC_VIRTUAL_INST_FAULT;
175 utrap.stval = insn;
176 utrap.htval = 0;
177 utrap.htinst = 0;
178 kvm_riscv_vcpu_trap_redirect(vcpu, &utrap);
179
180 return 1;
181 }
182
183 /**
184 * kvm_riscv_vcpu_wfi -- Emulate wait for interrupt (WFI) behaviour
185 *
186 * @vcpu: The VCPU pointer
187 */
kvm_riscv_vcpu_wfi(struct kvm_vcpu * vcpu)188 void kvm_riscv_vcpu_wfi(struct kvm_vcpu *vcpu)
189 {
190 if (!kvm_arch_vcpu_runnable(vcpu)) {
191 kvm_vcpu_srcu_read_unlock(vcpu);
192 kvm_vcpu_halt(vcpu);
193 kvm_vcpu_srcu_read_lock(vcpu);
194 }
195 }
196
wfi_insn(struct kvm_vcpu * vcpu,struct kvm_run * run,ulong insn)197 static int wfi_insn(struct kvm_vcpu *vcpu, struct kvm_run *run, ulong insn)
198 {
199 vcpu->stat.wfi_exit_stat++;
200 kvm_riscv_vcpu_wfi(vcpu);
201 return KVM_INSN_CONTINUE_NEXT_SEPC;
202 }
203
204 struct csr_func {
205 unsigned int base;
206 unsigned int count;
207 /*
208 * Possible return values are as same as "func" callback in
209 * "struct insn_func".
210 */
211 int (*func)(struct kvm_vcpu *vcpu, unsigned int csr_num,
212 unsigned long *val, unsigned long new_val,
213 unsigned long wr_mask);
214 };
215
216 static const struct csr_func csr_funcs[] = {
217 KVM_RISCV_VCPU_AIA_CSR_FUNCS
218 KVM_RISCV_VCPU_HPMCOUNTER_CSR_FUNCS
219 };
220
221 /**
222 * kvm_riscv_vcpu_csr_return -- Handle CSR read/write after user space
223 * emulation or in-kernel emulation
224 *
225 * @vcpu: The VCPU pointer
226 * @run: The VCPU run struct containing the CSR data
227 *
228 * Returns > 0 upon failure and 0 upon success
229 */
kvm_riscv_vcpu_csr_return(struct kvm_vcpu * vcpu,struct kvm_run * run)230 int kvm_riscv_vcpu_csr_return(struct kvm_vcpu *vcpu, struct kvm_run *run)
231 {
232 ulong insn;
233
234 if (vcpu->arch.csr_decode.return_handled)
235 return 0;
236 vcpu->arch.csr_decode.return_handled = 1;
237
238 /* Update destination register for CSR reads */
239 insn = vcpu->arch.csr_decode.insn;
240 if ((insn >> SH_RD) & MASK_RX)
241 SET_RD(insn, &vcpu->arch.guest_context,
242 run->riscv_csr.ret_value);
243
244 /* Move to next instruction */
245 vcpu->arch.guest_context.sepc += INSN_LEN(insn);
246
247 return 0;
248 }
249
csr_insn(struct kvm_vcpu * vcpu,struct kvm_run * run,ulong insn)250 static int csr_insn(struct kvm_vcpu *vcpu, struct kvm_run *run, ulong insn)
251 {
252 int i, rc = KVM_INSN_ILLEGAL_TRAP;
253 unsigned int csr_num = insn >> SH_RS2;
254 unsigned int rs1_num = (insn >> SH_RS1) & MASK_RX;
255 ulong rs1_val = GET_RS1(insn, &vcpu->arch.guest_context);
256 const struct csr_func *tcfn, *cfn = NULL;
257 ulong val = 0, wr_mask = 0, new_val = 0;
258
259 /* Decode the CSR instruction */
260 switch (GET_FUNCT3(insn)) {
261 case GET_FUNCT3(INSN_MATCH_CSRRW):
262 wr_mask = -1UL;
263 new_val = rs1_val;
264 break;
265 case GET_FUNCT3(INSN_MATCH_CSRRS):
266 wr_mask = rs1_val;
267 new_val = -1UL;
268 break;
269 case GET_FUNCT3(INSN_MATCH_CSRRC):
270 wr_mask = rs1_val;
271 new_val = 0;
272 break;
273 case GET_FUNCT3(INSN_MATCH_CSRRWI):
274 wr_mask = -1UL;
275 new_val = rs1_num;
276 break;
277 case GET_FUNCT3(INSN_MATCH_CSRRSI):
278 wr_mask = rs1_num;
279 new_val = -1UL;
280 break;
281 case GET_FUNCT3(INSN_MATCH_CSRRCI):
282 wr_mask = rs1_num;
283 new_val = 0;
284 break;
285 default:
286 return rc;
287 }
288
289 /* Save instruction decode info */
290 vcpu->arch.csr_decode.insn = insn;
291 vcpu->arch.csr_decode.return_handled = 0;
292
293 /* Update CSR details in kvm_run struct */
294 run->riscv_csr.csr_num = csr_num;
295 run->riscv_csr.new_value = new_val;
296 run->riscv_csr.write_mask = wr_mask;
297 run->riscv_csr.ret_value = 0;
298
299 /* Find in-kernel CSR function */
300 for (i = 0; i < ARRAY_SIZE(csr_funcs); i++) {
301 tcfn = &csr_funcs[i];
302 if ((tcfn->base <= csr_num) &&
303 (csr_num < (tcfn->base + tcfn->count))) {
304 cfn = tcfn;
305 break;
306 }
307 }
308
309 /* First try in-kernel CSR emulation */
310 if (cfn && cfn->func) {
311 rc = cfn->func(vcpu, csr_num, &val, new_val, wr_mask);
312 if (rc > KVM_INSN_EXIT_TO_USER_SPACE) {
313 if (rc == KVM_INSN_CONTINUE_NEXT_SEPC) {
314 run->riscv_csr.ret_value = val;
315 vcpu->stat.csr_exit_kernel++;
316 kvm_riscv_vcpu_csr_return(vcpu, run);
317 rc = KVM_INSN_CONTINUE_SAME_SEPC;
318 }
319 return rc;
320 }
321 }
322
323 /* Exit to user-space for CSR emulation */
324 if (rc <= KVM_INSN_EXIT_TO_USER_SPACE) {
325 vcpu->stat.csr_exit_user++;
326 run->exit_reason = KVM_EXIT_RISCV_CSR;
327 }
328
329 return rc;
330 }
331
332 static const struct insn_func system_opcode_funcs[] = {
333 {
334 .mask = INSN_MASK_CSRRW,
335 .match = INSN_MATCH_CSRRW,
336 .func = csr_insn,
337 },
338 {
339 .mask = INSN_MASK_CSRRS,
340 .match = INSN_MATCH_CSRRS,
341 .func = csr_insn,
342 },
343 {
344 .mask = INSN_MASK_CSRRC,
345 .match = INSN_MATCH_CSRRC,
346 .func = csr_insn,
347 },
348 {
349 .mask = INSN_MASK_CSRRWI,
350 .match = INSN_MATCH_CSRRWI,
351 .func = csr_insn,
352 },
353 {
354 .mask = INSN_MASK_CSRRSI,
355 .match = INSN_MATCH_CSRRSI,
356 .func = csr_insn,
357 },
358 {
359 .mask = INSN_MASK_CSRRCI,
360 .match = INSN_MATCH_CSRRCI,
361 .func = csr_insn,
362 },
363 {
364 .mask = INSN_MASK_WFI,
365 .match = INSN_MATCH_WFI,
366 .func = wfi_insn,
367 },
368 };
369
system_opcode_insn(struct kvm_vcpu * vcpu,struct kvm_run * run,ulong insn)370 static int system_opcode_insn(struct kvm_vcpu *vcpu, struct kvm_run *run,
371 ulong insn)
372 {
373 int i, rc = KVM_INSN_ILLEGAL_TRAP;
374 const struct insn_func *ifn;
375
376 for (i = 0; i < ARRAY_SIZE(system_opcode_funcs); i++) {
377 ifn = &system_opcode_funcs[i];
378 if ((insn & ifn->mask) == ifn->match) {
379 rc = ifn->func(vcpu, run, insn);
380 break;
381 }
382 }
383
384 switch (rc) {
385 case KVM_INSN_ILLEGAL_TRAP:
386 return truly_illegal_insn(vcpu, run, insn);
387 case KVM_INSN_VIRTUAL_TRAP:
388 return truly_virtual_insn(vcpu, run, insn);
389 case KVM_INSN_CONTINUE_NEXT_SEPC:
390 vcpu->arch.guest_context.sepc += INSN_LEN(insn);
391 break;
392 default:
393 break;
394 }
395
396 return (rc <= 0) ? rc : 1;
397 }
398
399 /**
400 * kvm_riscv_vcpu_virtual_insn -- Handle virtual instruction trap
401 *
402 * @vcpu: The VCPU pointer
403 * @run: The VCPU run struct containing the mmio data
404 * @trap: Trap details
405 *
406 * Returns > 0 to continue run-loop
407 * Returns 0 to exit run-loop and handle in user-space.
408 * Returns < 0 to report failure and exit run-loop
409 */
kvm_riscv_vcpu_virtual_insn(struct kvm_vcpu * vcpu,struct kvm_run * run,struct kvm_cpu_trap * trap)410 int kvm_riscv_vcpu_virtual_insn(struct kvm_vcpu *vcpu, struct kvm_run *run,
411 struct kvm_cpu_trap *trap)
412 {
413 unsigned long insn = trap->stval;
414 struct kvm_cpu_trap utrap = { 0 };
415 struct kvm_cpu_context *ct;
416
417 if (unlikely(INSN_IS_16BIT(insn))) {
418 if (insn == 0) {
419 ct = &vcpu->arch.guest_context;
420 insn = kvm_riscv_vcpu_unpriv_read(vcpu, true,
421 ct->sepc,
422 &utrap);
423 if (utrap.scause) {
424 utrap.sepc = ct->sepc;
425 kvm_riscv_vcpu_trap_redirect(vcpu, &utrap);
426 return 1;
427 }
428 }
429 if (INSN_IS_16BIT(insn))
430 return truly_illegal_insn(vcpu, run, insn);
431 }
432
433 switch ((insn & INSN_OPCODE_MASK) >> INSN_OPCODE_SHIFT) {
434 case INSN_OPCODE_SYSTEM:
435 return system_opcode_insn(vcpu, run, insn);
436 default:
437 return truly_illegal_insn(vcpu, run, insn);
438 }
439 }
440
441 /**
442 * kvm_riscv_vcpu_mmio_load -- Emulate MMIO load instruction
443 *
444 * @vcpu: The VCPU pointer
445 * @run: The VCPU run struct containing the mmio data
446 * @fault_addr: Guest physical address to load
447 * @htinst: Transformed encoding of the load instruction
448 *
449 * Returns > 0 to continue run-loop
450 * Returns 0 to exit run-loop and handle in user-space.
451 * Returns < 0 to report failure and exit run-loop
452 */
kvm_riscv_vcpu_mmio_load(struct kvm_vcpu * vcpu,struct kvm_run * run,unsigned long fault_addr,unsigned long htinst)453 int kvm_riscv_vcpu_mmio_load(struct kvm_vcpu *vcpu, struct kvm_run *run,
454 unsigned long fault_addr,
455 unsigned long htinst)
456 {
457 u8 data_buf[8];
458 unsigned long insn;
459 int shift = 0, len = 0, insn_len = 0;
460 struct kvm_cpu_trap utrap = { 0 };
461 struct kvm_cpu_context *ct = &vcpu->arch.guest_context;
462
463 /* Determine trapped instruction */
464 if (htinst & 0x1) {
465 /*
466 * Bit[0] == 1 implies trapped instruction value is
467 * transformed instruction or custom instruction.
468 */
469 insn = htinst | INSN_16BIT_MASK;
470 insn_len = (htinst & BIT(1)) ? INSN_LEN(insn) : 2;
471 } else {
472 /*
473 * Bit[0] == 0 implies trapped instruction value is
474 * zero or special value.
475 */
476 insn = kvm_riscv_vcpu_unpriv_read(vcpu, true, ct->sepc,
477 &utrap);
478 if (utrap.scause) {
479 /* Redirect trap if we failed to read instruction */
480 utrap.sepc = ct->sepc;
481 kvm_riscv_vcpu_trap_redirect(vcpu, &utrap);
482 return 1;
483 }
484 insn_len = INSN_LEN(insn);
485 }
486
487 /* Decode length of MMIO and shift */
488 if ((insn & INSN_MASK_LW) == INSN_MATCH_LW) {
489 len = 4;
490 shift = 8 * (sizeof(ulong) - len);
491 } else if ((insn & INSN_MASK_LB) == INSN_MATCH_LB) {
492 len = 1;
493 shift = 8 * (sizeof(ulong) - len);
494 } else if ((insn & INSN_MASK_LBU) == INSN_MATCH_LBU) {
495 len = 1;
496 shift = 8 * (sizeof(ulong) - len);
497 #ifdef CONFIG_64BIT
498 } else if ((insn & INSN_MASK_LD) == INSN_MATCH_LD) {
499 len = 8;
500 shift = 8 * (sizeof(ulong) - len);
501 } else if ((insn & INSN_MASK_LWU) == INSN_MATCH_LWU) {
502 len = 4;
503 #endif
504 } else if ((insn & INSN_MASK_LH) == INSN_MATCH_LH) {
505 len = 2;
506 shift = 8 * (sizeof(ulong) - len);
507 } else if ((insn & INSN_MASK_LHU) == INSN_MATCH_LHU) {
508 len = 2;
509 #ifdef CONFIG_64BIT
510 } else if ((insn & INSN_MASK_C_LD) == INSN_MATCH_C_LD) {
511 len = 8;
512 shift = 8 * (sizeof(ulong) - len);
513 insn = RVC_RS2S(insn) << SH_RD;
514 } else if ((insn & INSN_MASK_C_LDSP) == INSN_MATCH_C_LDSP &&
515 ((insn >> SH_RD) & 0x1f)) {
516 len = 8;
517 shift = 8 * (sizeof(ulong) - len);
518 #endif
519 } else if ((insn & INSN_MASK_C_LW) == INSN_MATCH_C_LW) {
520 len = 4;
521 shift = 8 * (sizeof(ulong) - len);
522 insn = RVC_RS2S(insn) << SH_RD;
523 } else if ((insn & INSN_MASK_C_LWSP) == INSN_MATCH_C_LWSP &&
524 ((insn >> SH_RD) & 0x1f)) {
525 len = 4;
526 shift = 8 * (sizeof(ulong) - len);
527 } else {
528 return -EOPNOTSUPP;
529 }
530
531 /* Fault address should be aligned to length of MMIO */
532 if (fault_addr & (len - 1))
533 return -EIO;
534
535 /* Save instruction decode info */
536 vcpu->arch.mmio_decode.insn = insn;
537 vcpu->arch.mmio_decode.insn_len = insn_len;
538 vcpu->arch.mmio_decode.shift = shift;
539 vcpu->arch.mmio_decode.len = len;
540 vcpu->arch.mmio_decode.return_handled = 0;
541
542 /* Update MMIO details in kvm_run struct */
543 run->mmio.is_write = false;
544 run->mmio.phys_addr = fault_addr;
545 run->mmio.len = len;
546
547 /* Try to handle MMIO access in the kernel */
548 if (!kvm_io_bus_read(vcpu, KVM_MMIO_BUS, fault_addr, len, data_buf)) {
549 /* Successfully handled MMIO access in the kernel so resume */
550 memcpy(run->mmio.data, data_buf, len);
551 vcpu->stat.mmio_exit_kernel++;
552 kvm_riscv_vcpu_mmio_return(vcpu, run);
553 return 1;
554 }
555
556 /* Exit to userspace for MMIO emulation */
557 vcpu->stat.mmio_exit_user++;
558 run->exit_reason = KVM_EXIT_MMIO;
559
560 return 0;
561 }
562
563 /**
564 * kvm_riscv_vcpu_mmio_store -- Emulate MMIO store instruction
565 *
566 * @vcpu: The VCPU pointer
567 * @run: The VCPU run struct containing the mmio data
568 * @fault_addr: Guest physical address to store
569 * @htinst: Transformed encoding of the store instruction
570 *
571 * Returns > 0 to continue run-loop
572 * Returns 0 to exit run-loop and handle in user-space.
573 * Returns < 0 to report failure and exit run-loop
574 */
kvm_riscv_vcpu_mmio_store(struct kvm_vcpu * vcpu,struct kvm_run * run,unsigned long fault_addr,unsigned long htinst)575 int kvm_riscv_vcpu_mmio_store(struct kvm_vcpu *vcpu, struct kvm_run *run,
576 unsigned long fault_addr,
577 unsigned long htinst)
578 {
579 u8 data8;
580 u16 data16;
581 u32 data32;
582 u64 data64;
583 ulong data;
584 unsigned long insn;
585 int len = 0, insn_len = 0;
586 struct kvm_cpu_trap utrap = { 0 };
587 struct kvm_cpu_context *ct = &vcpu->arch.guest_context;
588
589 /* Determine trapped instruction */
590 if (htinst & 0x1) {
591 /*
592 * Bit[0] == 1 implies trapped instruction value is
593 * transformed instruction or custom instruction.
594 */
595 insn = htinst | INSN_16BIT_MASK;
596 insn_len = (htinst & BIT(1)) ? INSN_LEN(insn) : 2;
597 } else {
598 /*
599 * Bit[0] == 0 implies trapped instruction value is
600 * zero or special value.
601 */
602 insn = kvm_riscv_vcpu_unpriv_read(vcpu, true, ct->sepc,
603 &utrap);
604 if (utrap.scause) {
605 /* Redirect trap if we failed to read instruction */
606 utrap.sepc = ct->sepc;
607 kvm_riscv_vcpu_trap_redirect(vcpu, &utrap);
608 return 1;
609 }
610 insn_len = INSN_LEN(insn);
611 }
612
613 data = GET_RS2(insn, &vcpu->arch.guest_context);
614 data8 = data16 = data32 = data64 = data;
615
616 if ((insn & INSN_MASK_SW) == INSN_MATCH_SW) {
617 len = 4;
618 } else if ((insn & INSN_MASK_SB) == INSN_MATCH_SB) {
619 len = 1;
620 #ifdef CONFIG_64BIT
621 } else if ((insn & INSN_MASK_SD) == INSN_MATCH_SD) {
622 len = 8;
623 #endif
624 } else if ((insn & INSN_MASK_SH) == INSN_MATCH_SH) {
625 len = 2;
626 #ifdef CONFIG_64BIT
627 } else if ((insn & INSN_MASK_C_SD) == INSN_MATCH_C_SD) {
628 len = 8;
629 data64 = GET_RS2S(insn, &vcpu->arch.guest_context);
630 } else if ((insn & INSN_MASK_C_SDSP) == INSN_MATCH_C_SDSP &&
631 ((insn >> SH_RD) & 0x1f)) {
632 len = 8;
633 data64 = GET_RS2C(insn, &vcpu->arch.guest_context);
634 #endif
635 } else if ((insn & INSN_MASK_C_SW) == INSN_MATCH_C_SW) {
636 len = 4;
637 data32 = GET_RS2S(insn, &vcpu->arch.guest_context);
638 } else if ((insn & INSN_MASK_C_SWSP) == INSN_MATCH_C_SWSP &&
639 ((insn >> SH_RD) & 0x1f)) {
640 len = 4;
641 data32 = GET_RS2C(insn, &vcpu->arch.guest_context);
642 } else {
643 return -EOPNOTSUPP;
644 }
645
646 /* Fault address should be aligned to length of MMIO */
647 if (fault_addr & (len - 1))
648 return -EIO;
649
650 /* Save instruction decode info */
651 vcpu->arch.mmio_decode.insn = insn;
652 vcpu->arch.mmio_decode.insn_len = insn_len;
653 vcpu->arch.mmio_decode.shift = 0;
654 vcpu->arch.mmio_decode.len = len;
655 vcpu->arch.mmio_decode.return_handled = 0;
656
657 /* Copy data to kvm_run instance */
658 switch (len) {
659 case 1:
660 *((u8 *)run->mmio.data) = data8;
661 break;
662 case 2:
663 *((u16 *)run->mmio.data) = data16;
664 break;
665 case 4:
666 *((u32 *)run->mmio.data) = data32;
667 break;
668 case 8:
669 *((u64 *)run->mmio.data) = data64;
670 break;
671 default:
672 return -EOPNOTSUPP;
673 }
674
675 /* Update MMIO details in kvm_run struct */
676 run->mmio.is_write = true;
677 run->mmio.phys_addr = fault_addr;
678 run->mmio.len = len;
679
680 /* Try to handle MMIO access in the kernel */
681 if (!kvm_io_bus_write(vcpu, KVM_MMIO_BUS,
682 fault_addr, len, run->mmio.data)) {
683 /* Successfully handled MMIO access in the kernel so resume */
684 vcpu->stat.mmio_exit_kernel++;
685 kvm_riscv_vcpu_mmio_return(vcpu, run);
686 return 1;
687 }
688
689 /* Exit to userspace for MMIO emulation */
690 vcpu->stat.mmio_exit_user++;
691 run->exit_reason = KVM_EXIT_MMIO;
692
693 return 0;
694 }
695
696 /**
697 * kvm_riscv_vcpu_mmio_return -- Handle MMIO loads after user space emulation
698 * or in-kernel IO emulation
699 *
700 * @vcpu: The VCPU pointer
701 * @run: The VCPU run struct containing the mmio data
702 */
kvm_riscv_vcpu_mmio_return(struct kvm_vcpu * vcpu,struct kvm_run * run)703 int kvm_riscv_vcpu_mmio_return(struct kvm_vcpu *vcpu, struct kvm_run *run)
704 {
705 u8 data8;
706 u16 data16;
707 u32 data32;
708 u64 data64;
709 ulong insn;
710 int len, shift;
711
712 if (vcpu->arch.mmio_decode.return_handled)
713 return 0;
714
715 vcpu->arch.mmio_decode.return_handled = 1;
716 insn = vcpu->arch.mmio_decode.insn;
717
718 if (run->mmio.is_write)
719 goto done;
720
721 len = vcpu->arch.mmio_decode.len;
722 shift = vcpu->arch.mmio_decode.shift;
723
724 switch (len) {
725 case 1:
726 data8 = *((u8 *)run->mmio.data);
727 SET_RD(insn, &vcpu->arch.guest_context,
728 (ulong)data8 << shift >> shift);
729 break;
730 case 2:
731 data16 = *((u16 *)run->mmio.data);
732 SET_RD(insn, &vcpu->arch.guest_context,
733 (ulong)data16 << shift >> shift);
734 break;
735 case 4:
736 data32 = *((u32 *)run->mmio.data);
737 SET_RD(insn, &vcpu->arch.guest_context,
738 (ulong)data32 << shift >> shift);
739 break;
740 case 8:
741 data64 = *((u64 *)run->mmio.data);
742 SET_RD(insn, &vcpu->arch.guest_context,
743 (ulong)data64 << shift >> shift);
744 break;
745 default:
746 return -EOPNOTSUPP;
747 }
748
749 done:
750 /* Move to next instruction */
751 vcpu->arch.guest_context.sepc += vcpu->arch.mmio_decode.insn_len;
752
753 return 0;
754 }
755