xref: /openbmc/linux/arch/riscv/kvm/vcpu.c (revision 7effbd18)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright (C) 2019 Western Digital Corporation or its affiliates.
4  *
5  * Authors:
6  *     Anup Patel <anup.patel@wdc.com>
7  */
8 
9 #include <linux/bitops.h>
10 #include <linux/entry-kvm.h>
11 #include <linux/errno.h>
12 #include <linux/err.h>
13 #include <linux/kdebug.h>
14 #include <linux/module.h>
15 #include <linux/percpu.h>
16 #include <linux/uaccess.h>
17 #include <linux/vmalloc.h>
18 #include <linux/sched/signal.h>
19 #include <linux/fs.h>
20 #include <linux/kvm_host.h>
21 #include <asm/csr.h>
22 #include <asm/cacheflush.h>
23 #include <asm/hwcap.h>
24 #include <asm/sbi.h>
25 
26 const struct _kvm_stats_desc kvm_vcpu_stats_desc[] = {
27 	KVM_GENERIC_VCPU_STATS(),
28 	STATS_DESC_COUNTER(VCPU, ecall_exit_stat),
29 	STATS_DESC_COUNTER(VCPU, wfi_exit_stat),
30 	STATS_DESC_COUNTER(VCPU, mmio_exit_user),
31 	STATS_DESC_COUNTER(VCPU, mmio_exit_kernel),
32 	STATS_DESC_COUNTER(VCPU, csr_exit_user),
33 	STATS_DESC_COUNTER(VCPU, csr_exit_kernel),
34 	STATS_DESC_COUNTER(VCPU, signal_exits),
35 	STATS_DESC_COUNTER(VCPU, exits)
36 };
37 
38 const struct kvm_stats_header kvm_vcpu_stats_header = {
39 	.name_size = KVM_STATS_NAME_SIZE,
40 	.num_desc = ARRAY_SIZE(kvm_vcpu_stats_desc),
41 	.id_offset = sizeof(struct kvm_stats_header),
42 	.desc_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE,
43 	.data_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE +
44 		       sizeof(kvm_vcpu_stats_desc),
45 };
46 
47 #define KVM_RISCV_BASE_ISA_MASK		GENMASK(25, 0)
48 
49 #define KVM_ISA_EXT_ARR(ext)		[KVM_RISCV_ISA_EXT_##ext] = RISCV_ISA_EXT_##ext
50 
51 /* Mapping between KVM ISA Extension ID & Host ISA extension ID */
52 static const unsigned long kvm_isa_ext_arr[] = {
53 	[KVM_RISCV_ISA_EXT_A] = RISCV_ISA_EXT_a,
54 	[KVM_RISCV_ISA_EXT_C] = RISCV_ISA_EXT_c,
55 	[KVM_RISCV_ISA_EXT_D] = RISCV_ISA_EXT_d,
56 	[KVM_RISCV_ISA_EXT_F] = RISCV_ISA_EXT_f,
57 	[KVM_RISCV_ISA_EXT_H] = RISCV_ISA_EXT_h,
58 	[KVM_RISCV_ISA_EXT_I] = RISCV_ISA_EXT_i,
59 	[KVM_RISCV_ISA_EXT_M] = RISCV_ISA_EXT_m,
60 
61 	KVM_ISA_EXT_ARR(SSTC),
62 	KVM_ISA_EXT_ARR(SVINVAL),
63 	KVM_ISA_EXT_ARR(SVPBMT),
64 	KVM_ISA_EXT_ARR(ZIHINTPAUSE),
65 	KVM_ISA_EXT_ARR(ZICBOM),
66 };
67 
68 static unsigned long kvm_riscv_vcpu_base2isa_ext(unsigned long base_ext)
69 {
70 	unsigned long i;
71 
72 	for (i = 0; i < KVM_RISCV_ISA_EXT_MAX; i++) {
73 		if (kvm_isa_ext_arr[i] == base_ext)
74 			return i;
75 	}
76 
77 	return KVM_RISCV_ISA_EXT_MAX;
78 }
79 
80 static bool kvm_riscv_vcpu_isa_enable_allowed(unsigned long ext)
81 {
82 	switch (ext) {
83 	case KVM_RISCV_ISA_EXT_H:
84 		return false;
85 	default:
86 		break;
87 	}
88 
89 	return true;
90 }
91 
92 static bool kvm_riscv_vcpu_isa_disable_allowed(unsigned long ext)
93 {
94 	switch (ext) {
95 	case KVM_RISCV_ISA_EXT_A:
96 	case KVM_RISCV_ISA_EXT_C:
97 	case KVM_RISCV_ISA_EXT_I:
98 	case KVM_RISCV_ISA_EXT_M:
99 	case KVM_RISCV_ISA_EXT_SSTC:
100 	case KVM_RISCV_ISA_EXT_SVINVAL:
101 	case KVM_RISCV_ISA_EXT_ZIHINTPAUSE:
102 		return false;
103 	default:
104 		break;
105 	}
106 
107 	return true;
108 }
109 
110 static void kvm_riscv_reset_vcpu(struct kvm_vcpu *vcpu)
111 {
112 	struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr;
113 	struct kvm_vcpu_csr *reset_csr = &vcpu->arch.guest_reset_csr;
114 	struct kvm_cpu_context *cntx = &vcpu->arch.guest_context;
115 	struct kvm_cpu_context *reset_cntx = &vcpu->arch.guest_reset_context;
116 	bool loaded;
117 
118 	/**
119 	 * The preemption should be disabled here because it races with
120 	 * kvm_sched_out/kvm_sched_in(called from preempt notifiers) which
121 	 * also calls vcpu_load/put.
122 	 */
123 	get_cpu();
124 	loaded = (vcpu->cpu != -1);
125 	if (loaded)
126 		kvm_arch_vcpu_put(vcpu);
127 
128 	vcpu->arch.last_exit_cpu = -1;
129 
130 	memcpy(csr, reset_csr, sizeof(*csr));
131 
132 	memcpy(cntx, reset_cntx, sizeof(*cntx));
133 
134 	kvm_riscv_vcpu_fp_reset(vcpu);
135 
136 	kvm_riscv_vcpu_timer_reset(vcpu);
137 
138 	WRITE_ONCE(vcpu->arch.irqs_pending, 0);
139 	WRITE_ONCE(vcpu->arch.irqs_pending_mask, 0);
140 
141 	kvm_riscv_vcpu_pmu_reset(vcpu);
142 
143 	vcpu->arch.hfence_head = 0;
144 	vcpu->arch.hfence_tail = 0;
145 	memset(vcpu->arch.hfence_queue, 0, sizeof(vcpu->arch.hfence_queue));
146 
147 	/* Reset the guest CSRs for hotplug usecase */
148 	if (loaded)
149 		kvm_arch_vcpu_load(vcpu, smp_processor_id());
150 	put_cpu();
151 }
152 
153 int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id)
154 {
155 	return 0;
156 }
157 
158 int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu)
159 {
160 	struct kvm_cpu_context *cntx;
161 	struct kvm_vcpu_csr *reset_csr = &vcpu->arch.guest_reset_csr;
162 	unsigned long host_isa, i;
163 
164 	/* Mark this VCPU never ran */
165 	vcpu->arch.ran_atleast_once = false;
166 	vcpu->arch.mmu_page_cache.gfp_zero = __GFP_ZERO;
167 	bitmap_zero(vcpu->arch.isa, RISCV_ISA_EXT_MAX);
168 
169 	/* Setup ISA features available to VCPU */
170 	for (i = 0; i < ARRAY_SIZE(kvm_isa_ext_arr); i++) {
171 		host_isa = kvm_isa_ext_arr[i];
172 		if (__riscv_isa_extension_available(NULL, host_isa) &&
173 		    kvm_riscv_vcpu_isa_enable_allowed(i))
174 			set_bit(host_isa, vcpu->arch.isa);
175 	}
176 
177 	/* Setup vendor, arch, and implementation details */
178 	vcpu->arch.mvendorid = sbi_get_mvendorid();
179 	vcpu->arch.marchid = sbi_get_marchid();
180 	vcpu->arch.mimpid = sbi_get_mimpid();
181 
182 	/* Setup VCPU hfence queue */
183 	spin_lock_init(&vcpu->arch.hfence_lock);
184 
185 	/* Setup reset state of shadow SSTATUS and HSTATUS CSRs */
186 	cntx = &vcpu->arch.guest_reset_context;
187 	cntx->sstatus = SR_SPP | SR_SPIE;
188 	cntx->hstatus = 0;
189 	cntx->hstatus |= HSTATUS_VTW;
190 	cntx->hstatus |= HSTATUS_SPVP;
191 	cntx->hstatus |= HSTATUS_SPV;
192 
193 	/* By default, make CY, TM, and IR counters accessible in VU mode */
194 	reset_csr->scounteren = 0x7;
195 
196 	/* Setup VCPU timer */
197 	kvm_riscv_vcpu_timer_init(vcpu);
198 
199 	/* setup performance monitoring */
200 	kvm_riscv_vcpu_pmu_init(vcpu);
201 
202 	/* Reset VCPU */
203 	kvm_riscv_reset_vcpu(vcpu);
204 
205 	return 0;
206 }
207 
208 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
209 {
210 	/**
211 	 * vcpu with id 0 is the designated boot cpu.
212 	 * Keep all vcpus with non-zero id in power-off state so that
213 	 * they can be brought up using SBI HSM extension.
214 	 */
215 	if (vcpu->vcpu_idx != 0)
216 		kvm_riscv_vcpu_power_off(vcpu);
217 }
218 
219 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
220 {
221 	/* Cleanup VCPU timer */
222 	kvm_riscv_vcpu_timer_deinit(vcpu);
223 
224 	kvm_riscv_vcpu_pmu_deinit(vcpu);
225 
226 	/* Free unused pages pre-allocated for G-stage page table mappings */
227 	kvm_mmu_free_memory_cache(&vcpu->arch.mmu_page_cache);
228 }
229 
230 int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu)
231 {
232 	return kvm_riscv_vcpu_timer_pending(vcpu);
233 }
234 
235 void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu)
236 {
237 }
238 
239 void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu)
240 {
241 }
242 
243 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
244 {
245 	return (kvm_riscv_vcpu_has_interrupts(vcpu, -1UL) &&
246 		!vcpu->arch.power_off && !vcpu->arch.pause);
247 }
248 
249 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
250 {
251 	return kvm_vcpu_exiting_guest_mode(vcpu) == IN_GUEST_MODE;
252 }
253 
254 bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu)
255 {
256 	return (vcpu->arch.guest_context.sstatus & SR_SPP) ? true : false;
257 }
258 
259 vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
260 {
261 	return VM_FAULT_SIGBUS;
262 }
263 
264 static int kvm_riscv_vcpu_get_reg_config(struct kvm_vcpu *vcpu,
265 					 const struct kvm_one_reg *reg)
266 {
267 	unsigned long __user *uaddr =
268 			(unsigned long __user *)(unsigned long)reg->addr;
269 	unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK |
270 					    KVM_REG_SIZE_MASK |
271 					    KVM_REG_RISCV_CONFIG);
272 	unsigned long reg_val;
273 
274 	if (KVM_REG_SIZE(reg->id) != sizeof(unsigned long))
275 		return -EINVAL;
276 
277 	switch (reg_num) {
278 	case KVM_REG_RISCV_CONFIG_REG(isa):
279 		reg_val = vcpu->arch.isa[0] & KVM_RISCV_BASE_ISA_MASK;
280 		break;
281 	case KVM_REG_RISCV_CONFIG_REG(zicbom_block_size):
282 		if (!riscv_isa_extension_available(vcpu->arch.isa, ZICBOM))
283 			return -EINVAL;
284 		reg_val = riscv_cbom_block_size;
285 		break;
286 	case KVM_REG_RISCV_CONFIG_REG(mvendorid):
287 		reg_val = vcpu->arch.mvendorid;
288 		break;
289 	case KVM_REG_RISCV_CONFIG_REG(marchid):
290 		reg_val = vcpu->arch.marchid;
291 		break;
292 	case KVM_REG_RISCV_CONFIG_REG(mimpid):
293 		reg_val = vcpu->arch.mimpid;
294 		break;
295 	default:
296 		return -EINVAL;
297 	}
298 
299 	if (copy_to_user(uaddr, &reg_val, KVM_REG_SIZE(reg->id)))
300 		return -EFAULT;
301 
302 	return 0;
303 }
304 
305 static int kvm_riscv_vcpu_set_reg_config(struct kvm_vcpu *vcpu,
306 					 const struct kvm_one_reg *reg)
307 {
308 	unsigned long __user *uaddr =
309 			(unsigned long __user *)(unsigned long)reg->addr;
310 	unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK |
311 					    KVM_REG_SIZE_MASK |
312 					    KVM_REG_RISCV_CONFIG);
313 	unsigned long i, isa_ext, reg_val;
314 
315 	if (KVM_REG_SIZE(reg->id) != sizeof(unsigned long))
316 		return -EINVAL;
317 
318 	if (copy_from_user(&reg_val, uaddr, KVM_REG_SIZE(reg->id)))
319 		return -EFAULT;
320 
321 	switch (reg_num) {
322 	case KVM_REG_RISCV_CONFIG_REG(isa):
323 		/*
324 		 * This ONE REG interface is only defined for
325 		 * single letter extensions.
326 		 */
327 		if (fls(reg_val) >= RISCV_ISA_EXT_BASE)
328 			return -EINVAL;
329 
330 		if (!vcpu->arch.ran_atleast_once) {
331 			/* Ignore the enable/disable request for certain extensions */
332 			for (i = 0; i < RISCV_ISA_EXT_BASE; i++) {
333 				isa_ext = kvm_riscv_vcpu_base2isa_ext(i);
334 				if (isa_ext >= KVM_RISCV_ISA_EXT_MAX) {
335 					reg_val &= ~BIT(i);
336 					continue;
337 				}
338 				if (!kvm_riscv_vcpu_isa_enable_allowed(isa_ext))
339 					if (reg_val & BIT(i))
340 						reg_val &= ~BIT(i);
341 				if (!kvm_riscv_vcpu_isa_disable_allowed(isa_ext))
342 					if (!(reg_val & BIT(i)))
343 						reg_val |= BIT(i);
344 			}
345 			reg_val &= riscv_isa_extension_base(NULL);
346 			/* Do not modify anything beyond single letter extensions */
347 			reg_val = (vcpu->arch.isa[0] & ~KVM_RISCV_BASE_ISA_MASK) |
348 				  (reg_val & KVM_RISCV_BASE_ISA_MASK);
349 			vcpu->arch.isa[0] = reg_val;
350 			kvm_riscv_vcpu_fp_reset(vcpu);
351 		} else {
352 			return -EOPNOTSUPP;
353 		}
354 		break;
355 	case KVM_REG_RISCV_CONFIG_REG(zicbom_block_size):
356 		return -EOPNOTSUPP;
357 	case KVM_REG_RISCV_CONFIG_REG(mvendorid):
358 		if (!vcpu->arch.ran_atleast_once)
359 			vcpu->arch.mvendorid = reg_val;
360 		else
361 			return -EBUSY;
362 		break;
363 	case KVM_REG_RISCV_CONFIG_REG(marchid):
364 		if (!vcpu->arch.ran_atleast_once)
365 			vcpu->arch.marchid = reg_val;
366 		else
367 			return -EBUSY;
368 		break;
369 	case KVM_REG_RISCV_CONFIG_REG(mimpid):
370 		if (!vcpu->arch.ran_atleast_once)
371 			vcpu->arch.mimpid = reg_val;
372 		else
373 			return -EBUSY;
374 		break;
375 	default:
376 		return -EINVAL;
377 	}
378 
379 	return 0;
380 }
381 
382 static int kvm_riscv_vcpu_get_reg_core(struct kvm_vcpu *vcpu,
383 				       const struct kvm_one_reg *reg)
384 {
385 	struct kvm_cpu_context *cntx = &vcpu->arch.guest_context;
386 	unsigned long __user *uaddr =
387 			(unsigned long __user *)(unsigned long)reg->addr;
388 	unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK |
389 					    KVM_REG_SIZE_MASK |
390 					    KVM_REG_RISCV_CORE);
391 	unsigned long reg_val;
392 
393 	if (KVM_REG_SIZE(reg->id) != sizeof(unsigned long))
394 		return -EINVAL;
395 	if (reg_num >= sizeof(struct kvm_riscv_core) / sizeof(unsigned long))
396 		return -EINVAL;
397 
398 	if (reg_num == KVM_REG_RISCV_CORE_REG(regs.pc))
399 		reg_val = cntx->sepc;
400 	else if (KVM_REG_RISCV_CORE_REG(regs.pc) < reg_num &&
401 		 reg_num <= KVM_REG_RISCV_CORE_REG(regs.t6))
402 		reg_val = ((unsigned long *)cntx)[reg_num];
403 	else if (reg_num == KVM_REG_RISCV_CORE_REG(mode))
404 		reg_val = (cntx->sstatus & SR_SPP) ?
405 				KVM_RISCV_MODE_S : KVM_RISCV_MODE_U;
406 	else
407 		return -EINVAL;
408 
409 	if (copy_to_user(uaddr, &reg_val, KVM_REG_SIZE(reg->id)))
410 		return -EFAULT;
411 
412 	return 0;
413 }
414 
415 static int kvm_riscv_vcpu_set_reg_core(struct kvm_vcpu *vcpu,
416 				       const struct kvm_one_reg *reg)
417 {
418 	struct kvm_cpu_context *cntx = &vcpu->arch.guest_context;
419 	unsigned long __user *uaddr =
420 			(unsigned long __user *)(unsigned long)reg->addr;
421 	unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK |
422 					    KVM_REG_SIZE_MASK |
423 					    KVM_REG_RISCV_CORE);
424 	unsigned long reg_val;
425 
426 	if (KVM_REG_SIZE(reg->id) != sizeof(unsigned long))
427 		return -EINVAL;
428 	if (reg_num >= sizeof(struct kvm_riscv_core) / sizeof(unsigned long))
429 		return -EINVAL;
430 
431 	if (copy_from_user(&reg_val, uaddr, KVM_REG_SIZE(reg->id)))
432 		return -EFAULT;
433 
434 	if (reg_num == KVM_REG_RISCV_CORE_REG(regs.pc))
435 		cntx->sepc = reg_val;
436 	else if (KVM_REG_RISCV_CORE_REG(regs.pc) < reg_num &&
437 		 reg_num <= KVM_REG_RISCV_CORE_REG(regs.t6))
438 		((unsigned long *)cntx)[reg_num] = reg_val;
439 	else if (reg_num == KVM_REG_RISCV_CORE_REG(mode)) {
440 		if (reg_val == KVM_RISCV_MODE_S)
441 			cntx->sstatus |= SR_SPP;
442 		else
443 			cntx->sstatus &= ~SR_SPP;
444 	} else
445 		return -EINVAL;
446 
447 	return 0;
448 }
449 
450 static int kvm_riscv_vcpu_get_reg_csr(struct kvm_vcpu *vcpu,
451 				      const struct kvm_one_reg *reg)
452 {
453 	struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr;
454 	unsigned long __user *uaddr =
455 			(unsigned long __user *)(unsigned long)reg->addr;
456 	unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK |
457 					    KVM_REG_SIZE_MASK |
458 					    KVM_REG_RISCV_CSR);
459 	unsigned long reg_val;
460 
461 	if (KVM_REG_SIZE(reg->id) != sizeof(unsigned long))
462 		return -EINVAL;
463 	if (reg_num >= sizeof(struct kvm_riscv_csr) / sizeof(unsigned long))
464 		return -EINVAL;
465 
466 	if (reg_num == KVM_REG_RISCV_CSR_REG(sip)) {
467 		kvm_riscv_vcpu_flush_interrupts(vcpu);
468 		reg_val = (csr->hvip >> VSIP_TO_HVIP_SHIFT) & VSIP_VALID_MASK;
469 	} else
470 		reg_val = ((unsigned long *)csr)[reg_num];
471 
472 	if (copy_to_user(uaddr, &reg_val, KVM_REG_SIZE(reg->id)))
473 		return -EFAULT;
474 
475 	return 0;
476 }
477 
478 static int kvm_riscv_vcpu_set_reg_csr(struct kvm_vcpu *vcpu,
479 				      const struct kvm_one_reg *reg)
480 {
481 	struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr;
482 	unsigned long __user *uaddr =
483 			(unsigned long __user *)(unsigned long)reg->addr;
484 	unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK |
485 					    KVM_REG_SIZE_MASK |
486 					    KVM_REG_RISCV_CSR);
487 	unsigned long reg_val;
488 
489 	if (KVM_REG_SIZE(reg->id) != sizeof(unsigned long))
490 		return -EINVAL;
491 	if (reg_num >= sizeof(struct kvm_riscv_csr) / sizeof(unsigned long))
492 		return -EINVAL;
493 
494 	if (copy_from_user(&reg_val, uaddr, KVM_REG_SIZE(reg->id)))
495 		return -EFAULT;
496 
497 	if (reg_num == KVM_REG_RISCV_CSR_REG(sip)) {
498 		reg_val &= VSIP_VALID_MASK;
499 		reg_val <<= VSIP_TO_HVIP_SHIFT;
500 	}
501 
502 	((unsigned long *)csr)[reg_num] = reg_val;
503 
504 	if (reg_num == KVM_REG_RISCV_CSR_REG(sip))
505 		WRITE_ONCE(vcpu->arch.irqs_pending_mask, 0);
506 
507 	return 0;
508 }
509 
510 static int kvm_riscv_vcpu_get_reg_isa_ext(struct kvm_vcpu *vcpu,
511 					  const struct kvm_one_reg *reg)
512 {
513 	unsigned long __user *uaddr =
514 			(unsigned long __user *)(unsigned long)reg->addr;
515 	unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK |
516 					    KVM_REG_SIZE_MASK |
517 					    KVM_REG_RISCV_ISA_EXT);
518 	unsigned long reg_val = 0;
519 	unsigned long host_isa_ext;
520 
521 	if (KVM_REG_SIZE(reg->id) != sizeof(unsigned long))
522 		return -EINVAL;
523 
524 	if (reg_num >= KVM_RISCV_ISA_EXT_MAX ||
525 	    reg_num >= ARRAY_SIZE(kvm_isa_ext_arr))
526 		return -EINVAL;
527 
528 	host_isa_ext = kvm_isa_ext_arr[reg_num];
529 	if (__riscv_isa_extension_available(vcpu->arch.isa, host_isa_ext))
530 		reg_val = 1; /* Mark the given extension as available */
531 
532 	if (copy_to_user(uaddr, &reg_val, KVM_REG_SIZE(reg->id)))
533 		return -EFAULT;
534 
535 	return 0;
536 }
537 
538 static int kvm_riscv_vcpu_set_reg_isa_ext(struct kvm_vcpu *vcpu,
539 					  const struct kvm_one_reg *reg)
540 {
541 	unsigned long __user *uaddr =
542 			(unsigned long __user *)(unsigned long)reg->addr;
543 	unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK |
544 					    KVM_REG_SIZE_MASK |
545 					    KVM_REG_RISCV_ISA_EXT);
546 	unsigned long reg_val;
547 	unsigned long host_isa_ext;
548 
549 	if (KVM_REG_SIZE(reg->id) != sizeof(unsigned long))
550 		return -EINVAL;
551 
552 	if (reg_num >= KVM_RISCV_ISA_EXT_MAX ||
553 	    reg_num >= ARRAY_SIZE(kvm_isa_ext_arr))
554 		return -EINVAL;
555 
556 	if (copy_from_user(&reg_val, uaddr, KVM_REG_SIZE(reg->id)))
557 		return -EFAULT;
558 
559 	host_isa_ext = kvm_isa_ext_arr[reg_num];
560 	if (!__riscv_isa_extension_available(NULL, host_isa_ext))
561 		return	-EOPNOTSUPP;
562 
563 	if (!vcpu->arch.ran_atleast_once) {
564 		/*
565 		 * All multi-letter extension and a few single letter
566 		 * extension can be disabled
567 		 */
568 		if (reg_val == 1 &&
569 		    kvm_riscv_vcpu_isa_enable_allowed(reg_num))
570 			set_bit(host_isa_ext, vcpu->arch.isa);
571 		else if (!reg_val &&
572 			 kvm_riscv_vcpu_isa_disable_allowed(reg_num))
573 			clear_bit(host_isa_ext, vcpu->arch.isa);
574 		else
575 			return -EINVAL;
576 		kvm_riscv_vcpu_fp_reset(vcpu);
577 	} else {
578 		return -EOPNOTSUPP;
579 	}
580 
581 	return 0;
582 }
583 
584 static int kvm_riscv_vcpu_set_reg(struct kvm_vcpu *vcpu,
585 				  const struct kvm_one_reg *reg)
586 {
587 	switch (reg->id & KVM_REG_RISCV_TYPE_MASK) {
588 	case KVM_REG_RISCV_CONFIG:
589 		return kvm_riscv_vcpu_set_reg_config(vcpu, reg);
590 	case KVM_REG_RISCV_CORE:
591 		return kvm_riscv_vcpu_set_reg_core(vcpu, reg);
592 	case KVM_REG_RISCV_CSR:
593 		return kvm_riscv_vcpu_set_reg_csr(vcpu, reg);
594 	case KVM_REG_RISCV_TIMER:
595 		return kvm_riscv_vcpu_set_reg_timer(vcpu, reg);
596 	case KVM_REG_RISCV_FP_F:
597 		return kvm_riscv_vcpu_set_reg_fp(vcpu, reg,
598 						 KVM_REG_RISCV_FP_F);
599 	case KVM_REG_RISCV_FP_D:
600 		return kvm_riscv_vcpu_set_reg_fp(vcpu, reg,
601 						 KVM_REG_RISCV_FP_D);
602 	case KVM_REG_RISCV_ISA_EXT:
603 		return kvm_riscv_vcpu_set_reg_isa_ext(vcpu, reg);
604 	default:
605 		break;
606 	}
607 
608 	return -EINVAL;
609 }
610 
611 static int kvm_riscv_vcpu_get_reg(struct kvm_vcpu *vcpu,
612 				  const struct kvm_one_reg *reg)
613 {
614 	switch (reg->id & KVM_REG_RISCV_TYPE_MASK) {
615 	case KVM_REG_RISCV_CONFIG:
616 		return kvm_riscv_vcpu_get_reg_config(vcpu, reg);
617 	case KVM_REG_RISCV_CORE:
618 		return kvm_riscv_vcpu_get_reg_core(vcpu, reg);
619 	case KVM_REG_RISCV_CSR:
620 		return kvm_riscv_vcpu_get_reg_csr(vcpu, reg);
621 	case KVM_REG_RISCV_TIMER:
622 		return kvm_riscv_vcpu_get_reg_timer(vcpu, reg);
623 	case KVM_REG_RISCV_FP_F:
624 		return kvm_riscv_vcpu_get_reg_fp(vcpu, reg,
625 						 KVM_REG_RISCV_FP_F);
626 	case KVM_REG_RISCV_FP_D:
627 		return kvm_riscv_vcpu_get_reg_fp(vcpu, reg,
628 						 KVM_REG_RISCV_FP_D);
629 	case KVM_REG_RISCV_ISA_EXT:
630 		return kvm_riscv_vcpu_get_reg_isa_ext(vcpu, reg);
631 	default:
632 		break;
633 	}
634 
635 	return -EINVAL;
636 }
637 
638 long kvm_arch_vcpu_async_ioctl(struct file *filp,
639 			       unsigned int ioctl, unsigned long arg)
640 {
641 	struct kvm_vcpu *vcpu = filp->private_data;
642 	void __user *argp = (void __user *)arg;
643 
644 	if (ioctl == KVM_INTERRUPT) {
645 		struct kvm_interrupt irq;
646 
647 		if (copy_from_user(&irq, argp, sizeof(irq)))
648 			return -EFAULT;
649 
650 		if (irq.irq == KVM_INTERRUPT_SET)
651 			return kvm_riscv_vcpu_set_interrupt(vcpu, IRQ_VS_EXT);
652 		else
653 			return kvm_riscv_vcpu_unset_interrupt(vcpu, IRQ_VS_EXT);
654 	}
655 
656 	return -ENOIOCTLCMD;
657 }
658 
659 long kvm_arch_vcpu_ioctl(struct file *filp,
660 			 unsigned int ioctl, unsigned long arg)
661 {
662 	struct kvm_vcpu *vcpu = filp->private_data;
663 	void __user *argp = (void __user *)arg;
664 	long r = -EINVAL;
665 
666 	switch (ioctl) {
667 	case KVM_SET_ONE_REG:
668 	case KVM_GET_ONE_REG: {
669 		struct kvm_one_reg reg;
670 
671 		r = -EFAULT;
672 		if (copy_from_user(&reg, argp, sizeof(reg)))
673 			break;
674 
675 		if (ioctl == KVM_SET_ONE_REG)
676 			r = kvm_riscv_vcpu_set_reg(vcpu, &reg);
677 		else
678 			r = kvm_riscv_vcpu_get_reg(vcpu, &reg);
679 		break;
680 	}
681 	default:
682 		break;
683 	}
684 
685 	return r;
686 }
687 
688 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
689 				  struct kvm_sregs *sregs)
690 {
691 	return -EINVAL;
692 }
693 
694 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
695 				  struct kvm_sregs *sregs)
696 {
697 	return -EINVAL;
698 }
699 
700 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
701 {
702 	return -EINVAL;
703 }
704 
705 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
706 {
707 	return -EINVAL;
708 }
709 
710 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
711 				  struct kvm_translation *tr)
712 {
713 	return -EINVAL;
714 }
715 
716 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
717 {
718 	return -EINVAL;
719 }
720 
721 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
722 {
723 	return -EINVAL;
724 }
725 
726 void kvm_riscv_vcpu_flush_interrupts(struct kvm_vcpu *vcpu)
727 {
728 	struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr;
729 	unsigned long mask, val;
730 
731 	if (READ_ONCE(vcpu->arch.irqs_pending_mask)) {
732 		mask = xchg_acquire(&vcpu->arch.irqs_pending_mask, 0);
733 		val = READ_ONCE(vcpu->arch.irqs_pending) & mask;
734 
735 		csr->hvip &= ~mask;
736 		csr->hvip |= val;
737 	}
738 }
739 
740 void kvm_riscv_vcpu_sync_interrupts(struct kvm_vcpu *vcpu)
741 {
742 	unsigned long hvip;
743 	struct kvm_vcpu_arch *v = &vcpu->arch;
744 	struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr;
745 
746 	/* Read current HVIP and VSIE CSRs */
747 	csr->vsie = csr_read(CSR_VSIE);
748 
749 	/* Sync-up HVIP.VSSIP bit changes does by Guest */
750 	hvip = csr_read(CSR_HVIP);
751 	if ((csr->hvip ^ hvip) & (1UL << IRQ_VS_SOFT)) {
752 		if (hvip & (1UL << IRQ_VS_SOFT)) {
753 			if (!test_and_set_bit(IRQ_VS_SOFT,
754 					      &v->irqs_pending_mask))
755 				set_bit(IRQ_VS_SOFT, &v->irqs_pending);
756 		} else {
757 			if (!test_and_set_bit(IRQ_VS_SOFT,
758 					      &v->irqs_pending_mask))
759 				clear_bit(IRQ_VS_SOFT, &v->irqs_pending);
760 		}
761 	}
762 
763 	/* Sync-up timer CSRs */
764 	kvm_riscv_vcpu_timer_sync(vcpu);
765 }
766 
767 int kvm_riscv_vcpu_set_interrupt(struct kvm_vcpu *vcpu, unsigned int irq)
768 {
769 	if (irq != IRQ_VS_SOFT &&
770 	    irq != IRQ_VS_TIMER &&
771 	    irq != IRQ_VS_EXT)
772 		return -EINVAL;
773 
774 	set_bit(irq, &vcpu->arch.irqs_pending);
775 	smp_mb__before_atomic();
776 	set_bit(irq, &vcpu->arch.irqs_pending_mask);
777 
778 	kvm_vcpu_kick(vcpu);
779 
780 	return 0;
781 }
782 
783 int kvm_riscv_vcpu_unset_interrupt(struct kvm_vcpu *vcpu, unsigned int irq)
784 {
785 	if (irq != IRQ_VS_SOFT &&
786 	    irq != IRQ_VS_TIMER &&
787 	    irq != IRQ_VS_EXT)
788 		return -EINVAL;
789 
790 	clear_bit(irq, &vcpu->arch.irqs_pending);
791 	smp_mb__before_atomic();
792 	set_bit(irq, &vcpu->arch.irqs_pending_mask);
793 
794 	return 0;
795 }
796 
797 bool kvm_riscv_vcpu_has_interrupts(struct kvm_vcpu *vcpu, unsigned long mask)
798 {
799 	unsigned long ie = ((vcpu->arch.guest_csr.vsie & VSIP_VALID_MASK)
800 			    << VSIP_TO_HVIP_SHIFT) & mask;
801 
802 	return (READ_ONCE(vcpu->arch.irqs_pending) & ie) ? true : false;
803 }
804 
805 void kvm_riscv_vcpu_power_off(struct kvm_vcpu *vcpu)
806 {
807 	vcpu->arch.power_off = true;
808 	kvm_make_request(KVM_REQ_SLEEP, vcpu);
809 	kvm_vcpu_kick(vcpu);
810 }
811 
812 void kvm_riscv_vcpu_power_on(struct kvm_vcpu *vcpu)
813 {
814 	vcpu->arch.power_off = false;
815 	kvm_vcpu_wake_up(vcpu);
816 }
817 
818 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
819 				    struct kvm_mp_state *mp_state)
820 {
821 	if (vcpu->arch.power_off)
822 		mp_state->mp_state = KVM_MP_STATE_STOPPED;
823 	else
824 		mp_state->mp_state = KVM_MP_STATE_RUNNABLE;
825 
826 	return 0;
827 }
828 
829 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
830 				    struct kvm_mp_state *mp_state)
831 {
832 	int ret = 0;
833 
834 	switch (mp_state->mp_state) {
835 	case KVM_MP_STATE_RUNNABLE:
836 		vcpu->arch.power_off = false;
837 		break;
838 	case KVM_MP_STATE_STOPPED:
839 		kvm_riscv_vcpu_power_off(vcpu);
840 		break;
841 	default:
842 		ret = -EINVAL;
843 	}
844 
845 	return ret;
846 }
847 
848 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
849 					struct kvm_guest_debug *dbg)
850 {
851 	/* TODO; To be implemented later. */
852 	return -EINVAL;
853 }
854 
855 static void kvm_riscv_vcpu_update_config(const unsigned long *isa)
856 {
857 	u64 henvcfg = 0;
858 
859 	if (riscv_isa_extension_available(isa, SVPBMT))
860 		henvcfg |= ENVCFG_PBMTE;
861 
862 	if (riscv_isa_extension_available(isa, SSTC))
863 		henvcfg |= ENVCFG_STCE;
864 
865 	if (riscv_isa_extension_available(isa, ZICBOM))
866 		henvcfg |= (ENVCFG_CBIE | ENVCFG_CBCFE);
867 
868 	csr_write(CSR_HENVCFG, henvcfg);
869 #ifdef CONFIG_32BIT
870 	csr_write(CSR_HENVCFGH, henvcfg >> 32);
871 #endif
872 }
873 
874 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
875 {
876 	struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr;
877 
878 	csr_write(CSR_VSSTATUS, csr->vsstatus);
879 	csr_write(CSR_VSIE, csr->vsie);
880 	csr_write(CSR_VSTVEC, csr->vstvec);
881 	csr_write(CSR_VSSCRATCH, csr->vsscratch);
882 	csr_write(CSR_VSEPC, csr->vsepc);
883 	csr_write(CSR_VSCAUSE, csr->vscause);
884 	csr_write(CSR_VSTVAL, csr->vstval);
885 	csr_write(CSR_HVIP, csr->hvip);
886 	csr_write(CSR_VSATP, csr->vsatp);
887 
888 	kvm_riscv_vcpu_update_config(vcpu->arch.isa);
889 
890 	kvm_riscv_gstage_update_hgatp(vcpu);
891 
892 	kvm_riscv_vcpu_timer_restore(vcpu);
893 
894 	kvm_riscv_vcpu_host_fp_save(&vcpu->arch.host_context);
895 	kvm_riscv_vcpu_guest_fp_restore(&vcpu->arch.guest_context,
896 					vcpu->arch.isa);
897 
898 	vcpu->cpu = cpu;
899 }
900 
901 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
902 {
903 	struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr;
904 
905 	vcpu->cpu = -1;
906 
907 	kvm_riscv_vcpu_guest_fp_save(&vcpu->arch.guest_context,
908 				     vcpu->arch.isa);
909 	kvm_riscv_vcpu_host_fp_restore(&vcpu->arch.host_context);
910 
911 	kvm_riscv_vcpu_timer_save(vcpu);
912 
913 	csr->vsstatus = csr_read(CSR_VSSTATUS);
914 	csr->vsie = csr_read(CSR_VSIE);
915 	csr->vstvec = csr_read(CSR_VSTVEC);
916 	csr->vsscratch = csr_read(CSR_VSSCRATCH);
917 	csr->vsepc = csr_read(CSR_VSEPC);
918 	csr->vscause = csr_read(CSR_VSCAUSE);
919 	csr->vstval = csr_read(CSR_VSTVAL);
920 	csr->hvip = csr_read(CSR_HVIP);
921 	csr->vsatp = csr_read(CSR_VSATP);
922 }
923 
924 static void kvm_riscv_check_vcpu_requests(struct kvm_vcpu *vcpu)
925 {
926 	struct rcuwait *wait = kvm_arch_vcpu_get_wait(vcpu);
927 
928 	if (kvm_request_pending(vcpu)) {
929 		if (kvm_check_request(KVM_REQ_SLEEP, vcpu)) {
930 			kvm_vcpu_srcu_read_unlock(vcpu);
931 			rcuwait_wait_event(wait,
932 				(!vcpu->arch.power_off) && (!vcpu->arch.pause),
933 				TASK_INTERRUPTIBLE);
934 			kvm_vcpu_srcu_read_lock(vcpu);
935 
936 			if (vcpu->arch.power_off || vcpu->arch.pause) {
937 				/*
938 				 * Awaken to handle a signal, request to
939 				 * sleep again later.
940 				 */
941 				kvm_make_request(KVM_REQ_SLEEP, vcpu);
942 			}
943 		}
944 
945 		if (kvm_check_request(KVM_REQ_VCPU_RESET, vcpu))
946 			kvm_riscv_reset_vcpu(vcpu);
947 
948 		if (kvm_check_request(KVM_REQ_UPDATE_HGATP, vcpu))
949 			kvm_riscv_gstage_update_hgatp(vcpu);
950 
951 		if (kvm_check_request(KVM_REQ_FENCE_I, vcpu))
952 			kvm_riscv_fence_i_process(vcpu);
953 
954 		/*
955 		 * The generic KVM_REQ_TLB_FLUSH is same as
956 		 * KVM_REQ_HFENCE_GVMA_VMID_ALL
957 		 */
958 		if (kvm_check_request(KVM_REQ_HFENCE_GVMA_VMID_ALL, vcpu))
959 			kvm_riscv_hfence_gvma_vmid_all_process(vcpu);
960 
961 		if (kvm_check_request(KVM_REQ_HFENCE_VVMA_ALL, vcpu))
962 			kvm_riscv_hfence_vvma_all_process(vcpu);
963 
964 		if (kvm_check_request(KVM_REQ_HFENCE, vcpu))
965 			kvm_riscv_hfence_process(vcpu);
966 	}
967 }
968 
969 static void kvm_riscv_update_hvip(struct kvm_vcpu *vcpu)
970 {
971 	struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr;
972 
973 	csr_write(CSR_HVIP, csr->hvip);
974 }
975 
976 /*
977  * Actually run the vCPU, entering an RCU extended quiescent state (EQS) while
978  * the vCPU is running.
979  *
980  * This must be noinstr as instrumentation may make use of RCU, and this is not
981  * safe during the EQS.
982  */
983 static void noinstr kvm_riscv_vcpu_enter_exit(struct kvm_vcpu *vcpu)
984 {
985 	guest_state_enter_irqoff();
986 	__kvm_riscv_switch_to(&vcpu->arch);
987 	vcpu->arch.last_exit_cpu = vcpu->cpu;
988 	guest_state_exit_irqoff();
989 }
990 
991 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)
992 {
993 	int ret;
994 	struct kvm_cpu_trap trap;
995 	struct kvm_run *run = vcpu->run;
996 
997 	/* Mark this VCPU ran at least once */
998 	vcpu->arch.ran_atleast_once = true;
999 
1000 	kvm_vcpu_srcu_read_lock(vcpu);
1001 
1002 	switch (run->exit_reason) {
1003 	case KVM_EXIT_MMIO:
1004 		/* Process MMIO value returned from user-space */
1005 		ret = kvm_riscv_vcpu_mmio_return(vcpu, vcpu->run);
1006 		break;
1007 	case KVM_EXIT_RISCV_SBI:
1008 		/* Process SBI value returned from user-space */
1009 		ret = kvm_riscv_vcpu_sbi_return(vcpu, vcpu->run);
1010 		break;
1011 	case KVM_EXIT_RISCV_CSR:
1012 		/* Process CSR value returned from user-space */
1013 		ret = kvm_riscv_vcpu_csr_return(vcpu, vcpu->run);
1014 		break;
1015 	default:
1016 		ret = 0;
1017 		break;
1018 	}
1019 	if (ret) {
1020 		kvm_vcpu_srcu_read_unlock(vcpu);
1021 		return ret;
1022 	}
1023 
1024 	if (run->immediate_exit) {
1025 		kvm_vcpu_srcu_read_unlock(vcpu);
1026 		return -EINTR;
1027 	}
1028 
1029 	vcpu_load(vcpu);
1030 
1031 	kvm_sigset_activate(vcpu);
1032 
1033 	ret = 1;
1034 	run->exit_reason = KVM_EXIT_UNKNOWN;
1035 	while (ret > 0) {
1036 		/* Check conditions before entering the guest */
1037 		ret = xfer_to_guest_mode_handle_work(vcpu);
1038 		if (ret)
1039 			continue;
1040 		ret = 1;
1041 
1042 		kvm_riscv_gstage_vmid_update(vcpu);
1043 
1044 		kvm_riscv_check_vcpu_requests(vcpu);
1045 
1046 		local_irq_disable();
1047 
1048 		/*
1049 		 * Ensure we set mode to IN_GUEST_MODE after we disable
1050 		 * interrupts and before the final VCPU requests check.
1051 		 * See the comment in kvm_vcpu_exiting_guest_mode() and
1052 		 * Documentation/virt/kvm/vcpu-requests.rst
1053 		 */
1054 		vcpu->mode = IN_GUEST_MODE;
1055 
1056 		kvm_vcpu_srcu_read_unlock(vcpu);
1057 		smp_mb__after_srcu_read_unlock();
1058 
1059 		/*
1060 		 * We might have got VCPU interrupts updated asynchronously
1061 		 * so update it in HW.
1062 		 */
1063 		kvm_riscv_vcpu_flush_interrupts(vcpu);
1064 
1065 		/* Update HVIP CSR for current CPU */
1066 		kvm_riscv_update_hvip(vcpu);
1067 
1068 		if (ret <= 0 ||
1069 		    kvm_riscv_gstage_vmid_ver_changed(&vcpu->kvm->arch.vmid) ||
1070 		    kvm_request_pending(vcpu) ||
1071 		    xfer_to_guest_mode_work_pending()) {
1072 			vcpu->mode = OUTSIDE_GUEST_MODE;
1073 			local_irq_enable();
1074 			kvm_vcpu_srcu_read_lock(vcpu);
1075 			continue;
1076 		}
1077 
1078 		/*
1079 		 * Cleanup stale TLB enteries
1080 		 *
1081 		 * Note: This should be done after G-stage VMID has been
1082 		 * updated using kvm_riscv_gstage_vmid_ver_changed()
1083 		 */
1084 		kvm_riscv_local_tlb_sanitize(vcpu);
1085 
1086 		guest_timing_enter_irqoff();
1087 
1088 		kvm_riscv_vcpu_enter_exit(vcpu);
1089 
1090 		vcpu->mode = OUTSIDE_GUEST_MODE;
1091 		vcpu->stat.exits++;
1092 
1093 		/*
1094 		 * Save SCAUSE, STVAL, HTVAL, and HTINST because we might
1095 		 * get an interrupt between __kvm_riscv_switch_to() and
1096 		 * local_irq_enable() which can potentially change CSRs.
1097 		 */
1098 		trap.sepc = vcpu->arch.guest_context.sepc;
1099 		trap.scause = csr_read(CSR_SCAUSE);
1100 		trap.stval = csr_read(CSR_STVAL);
1101 		trap.htval = csr_read(CSR_HTVAL);
1102 		trap.htinst = csr_read(CSR_HTINST);
1103 
1104 		/* Syncup interrupts state with HW */
1105 		kvm_riscv_vcpu_sync_interrupts(vcpu);
1106 
1107 		preempt_disable();
1108 
1109 		/*
1110 		 * We must ensure that any pending interrupts are taken before
1111 		 * we exit guest timing so that timer ticks are accounted as
1112 		 * guest time. Transiently unmask interrupts so that any
1113 		 * pending interrupts are taken.
1114 		 *
1115 		 * There's no barrier which ensures that pending interrupts are
1116 		 * recognised, so we just hope that the CPU takes any pending
1117 		 * interrupts between the enable and disable.
1118 		 */
1119 		local_irq_enable();
1120 		local_irq_disable();
1121 
1122 		guest_timing_exit_irqoff();
1123 
1124 		local_irq_enable();
1125 
1126 		preempt_enable();
1127 
1128 		kvm_vcpu_srcu_read_lock(vcpu);
1129 
1130 		ret = kvm_riscv_vcpu_exit(vcpu, run, &trap);
1131 	}
1132 
1133 	kvm_sigset_deactivate(vcpu);
1134 
1135 	vcpu_put(vcpu);
1136 
1137 	kvm_vcpu_srcu_read_unlock(vcpu);
1138 
1139 	return ret;
1140 }
1141