xref: /openbmc/linux/arch/mips/kvm/mips.c (revision fa7f32422ea1ac276b45b96a540ed5981caaa61f)
1 /*
2  * This file is subject to the terms and conditions of the GNU General Public
3  * License.  See the file "COPYING" in the main directory of this archive
4  * for more details.
5  *
6  * KVM/MIPS: MIPS specific KVM APIs
7  *
8  * Copyright (C) 2012  MIPS Technologies, Inc.  All rights reserved.
9  * Authors: Sanjay Lal <sanjayl@kymasys.com>
10  */
11 
12 #include <linux/bitops.h>
13 #include <linux/errno.h>
14 #include <linux/err.h>
15 #include <linux/kdebug.h>
16 #include <linux/module.h>
17 #include <linux/vmalloc.h>
18 #include <linux/fs.h>
19 #include <linux/bootmem.h>
20 #include <asm/fpu.h>
21 #include <asm/page.h>
22 #include <asm/cacheflush.h>
23 #include <asm/mmu_context.h>
24 #include <asm/pgtable.h>
25 
26 #include <linux/kvm_host.h>
27 
28 #include "interrupt.h"
29 #include "commpage.h"
30 
31 #define CREATE_TRACE_POINTS
32 #include "trace.h"
33 
34 #ifndef VECTORSPACING
35 #define VECTORSPACING 0x100	/* for EI/VI mode */
36 #endif
37 
38 #define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x)
39 struct kvm_stats_debugfs_item debugfs_entries[] = {
40 	{ "wait",	  VCPU_STAT(wait_exits),	 KVM_STAT_VCPU },
41 	{ "cache",	  VCPU_STAT(cache_exits),	 KVM_STAT_VCPU },
42 	{ "signal",	  VCPU_STAT(signal_exits),	 KVM_STAT_VCPU },
43 	{ "interrupt",	  VCPU_STAT(int_exits),		 KVM_STAT_VCPU },
44 	{ "cop_unsuable", VCPU_STAT(cop_unusable_exits), KVM_STAT_VCPU },
45 	{ "tlbmod",	  VCPU_STAT(tlbmod_exits),	 KVM_STAT_VCPU },
46 	{ "tlbmiss_ld",	  VCPU_STAT(tlbmiss_ld_exits),	 KVM_STAT_VCPU },
47 	{ "tlbmiss_st",	  VCPU_STAT(tlbmiss_st_exits),	 KVM_STAT_VCPU },
48 	{ "addrerr_st",	  VCPU_STAT(addrerr_st_exits),	 KVM_STAT_VCPU },
49 	{ "addrerr_ld",	  VCPU_STAT(addrerr_ld_exits),	 KVM_STAT_VCPU },
50 	{ "syscall",	  VCPU_STAT(syscall_exits),	 KVM_STAT_VCPU },
51 	{ "resvd_inst",	  VCPU_STAT(resvd_inst_exits),	 KVM_STAT_VCPU },
52 	{ "break_inst",	  VCPU_STAT(break_inst_exits),	 KVM_STAT_VCPU },
53 	{ "trap_inst",	  VCPU_STAT(trap_inst_exits),	 KVM_STAT_VCPU },
54 	{ "msa_fpe",	  VCPU_STAT(msa_fpe_exits),	 KVM_STAT_VCPU },
55 	{ "fpe",	  VCPU_STAT(fpe_exits),		 KVM_STAT_VCPU },
56 	{ "msa_disabled", VCPU_STAT(msa_disabled_exits), KVM_STAT_VCPU },
57 	{ "flush_dcache", VCPU_STAT(flush_dcache_exits), KVM_STAT_VCPU },
58 	{ "halt_successful_poll", VCPU_STAT(halt_successful_poll), KVM_STAT_VCPU },
59 	{ "halt_attempted_poll", VCPU_STAT(halt_attempted_poll), KVM_STAT_VCPU },
60 	{ "halt_poll_invalid", VCPU_STAT(halt_poll_invalid), KVM_STAT_VCPU },
61 	{ "halt_wakeup",  VCPU_STAT(halt_wakeup),	 KVM_STAT_VCPU },
62 	{NULL}
63 };
64 
65 static int kvm_mips_reset_vcpu(struct kvm_vcpu *vcpu)
66 {
67 	int i;
68 
69 	for_each_possible_cpu(i) {
70 		vcpu->arch.guest_kernel_asid[i] = 0;
71 		vcpu->arch.guest_user_asid[i] = 0;
72 	}
73 
74 	return 0;
75 }
76 
77 /*
78  * XXXKYMA: We are simulatoring a processor that has the WII bit set in
79  * Config7, so we are "runnable" if interrupts are pending
80  */
81 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
82 {
83 	return !!(vcpu->arch.pending_exceptions);
84 }
85 
86 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
87 {
88 	return 1;
89 }
90 
91 int kvm_arch_hardware_enable(void)
92 {
93 	return 0;
94 }
95 
96 int kvm_arch_hardware_setup(void)
97 {
98 	return 0;
99 }
100 
101 void kvm_arch_check_processor_compat(void *rtn)
102 {
103 	*(int *)rtn = 0;
104 }
105 
106 static void kvm_mips_init_tlbs(struct kvm *kvm)
107 {
108 	unsigned long wired;
109 
110 	/*
111 	 * Add a wired entry to the TLB, it is used to map the commpage to
112 	 * the Guest kernel
113 	 */
114 	wired = read_c0_wired();
115 	write_c0_wired(wired + 1);
116 	mtc0_tlbw_hazard();
117 	kvm->arch.commpage_tlb = wired;
118 
119 	kvm_debug("[%d] commpage TLB: %d\n", smp_processor_id(),
120 		  kvm->arch.commpage_tlb);
121 }
122 
123 static void kvm_mips_init_vm_percpu(void *arg)
124 {
125 	struct kvm *kvm = (struct kvm *)arg;
126 
127 	kvm_mips_init_tlbs(kvm);
128 	kvm_mips_callbacks->vm_init(kvm);
129 
130 }
131 
132 int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
133 {
134 	if (atomic_inc_return(&kvm_mips_instance) == 1) {
135 		kvm_debug("%s: 1st KVM instance, setup host TLB parameters\n",
136 			  __func__);
137 		on_each_cpu(kvm_mips_init_vm_percpu, kvm, 1);
138 	}
139 
140 	return 0;
141 }
142 
143 bool kvm_arch_has_vcpu_debugfs(void)
144 {
145 	return false;
146 }
147 
148 int kvm_arch_create_vcpu_debugfs(struct kvm_vcpu *vcpu)
149 {
150 	return 0;
151 }
152 
153 void kvm_mips_free_vcpus(struct kvm *kvm)
154 {
155 	unsigned int i;
156 	struct kvm_vcpu *vcpu;
157 
158 	/* Put the pages we reserved for the guest pmap */
159 	for (i = 0; i < kvm->arch.guest_pmap_npages; i++) {
160 		if (kvm->arch.guest_pmap[i] != KVM_INVALID_PAGE)
161 			kvm_release_pfn_clean(kvm->arch.guest_pmap[i]);
162 	}
163 	kfree(kvm->arch.guest_pmap);
164 
165 	kvm_for_each_vcpu(i, vcpu, kvm) {
166 		kvm_arch_vcpu_free(vcpu);
167 	}
168 
169 	mutex_lock(&kvm->lock);
170 
171 	for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
172 		kvm->vcpus[i] = NULL;
173 
174 	atomic_set(&kvm->online_vcpus, 0);
175 
176 	mutex_unlock(&kvm->lock);
177 }
178 
179 static void kvm_mips_uninit_tlbs(void *arg)
180 {
181 	/* Restore wired count */
182 	write_c0_wired(0);
183 	mtc0_tlbw_hazard();
184 	/* Clear out all the TLBs */
185 	kvm_local_flush_tlb_all();
186 }
187 
188 void kvm_arch_destroy_vm(struct kvm *kvm)
189 {
190 	kvm_mips_free_vcpus(kvm);
191 
192 	/* If this is the last instance, restore wired count */
193 	if (atomic_dec_return(&kvm_mips_instance) == 0) {
194 		kvm_debug("%s: last KVM instance, restoring TLB parameters\n",
195 			  __func__);
196 		on_each_cpu(kvm_mips_uninit_tlbs, NULL, 1);
197 	}
198 }
199 
200 long kvm_arch_dev_ioctl(struct file *filp, unsigned int ioctl,
201 			unsigned long arg)
202 {
203 	return -ENOIOCTLCMD;
204 }
205 
206 int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot,
207 			    unsigned long npages)
208 {
209 	return 0;
210 }
211 
212 int kvm_arch_prepare_memory_region(struct kvm *kvm,
213 				   struct kvm_memory_slot *memslot,
214 				   const struct kvm_userspace_memory_region *mem,
215 				   enum kvm_mr_change change)
216 {
217 	return 0;
218 }
219 
220 void kvm_arch_commit_memory_region(struct kvm *kvm,
221 				   const struct kvm_userspace_memory_region *mem,
222 				   const struct kvm_memory_slot *old,
223 				   const struct kvm_memory_slot *new,
224 				   enum kvm_mr_change change)
225 {
226 	unsigned long npages = 0;
227 	int i;
228 
229 	kvm_debug("%s: kvm: %p slot: %d, GPA: %llx, size: %llx, QVA: %llx\n",
230 		  __func__, kvm, mem->slot, mem->guest_phys_addr,
231 		  mem->memory_size, mem->userspace_addr);
232 
233 	/* Setup Guest PMAP table */
234 	if (!kvm->arch.guest_pmap) {
235 		if (mem->slot == 0)
236 			npages = mem->memory_size >> PAGE_SHIFT;
237 
238 		if (npages) {
239 			kvm->arch.guest_pmap_npages = npages;
240 			kvm->arch.guest_pmap =
241 			    kzalloc(npages * sizeof(unsigned long), GFP_KERNEL);
242 
243 			if (!kvm->arch.guest_pmap) {
244 				kvm_err("Failed to allocate guest PMAP\n");
245 				return;
246 			}
247 
248 			kvm_debug("Allocated space for Guest PMAP Table (%ld pages) @ %p\n",
249 				  npages, kvm->arch.guest_pmap);
250 
251 			/* Now setup the page table */
252 			for (i = 0; i < npages; i++)
253 				kvm->arch.guest_pmap[i] = KVM_INVALID_PAGE;
254 		}
255 	}
256 }
257 
258 static inline void dump_handler(const char *symbol, void *start, void *end)
259 {
260 	u32 *p;
261 
262 	pr_debug("LEAF(%s)\n", symbol);
263 
264 	pr_debug("\t.set push\n");
265 	pr_debug("\t.set noreorder\n");
266 
267 	for (p = start; p < (u32 *)end; ++p)
268 		pr_debug("\t.word\t0x%08x\t\t# %p\n", *p, p);
269 
270 	pr_debug("\t.set\tpop\n");
271 
272 	pr_debug("\tEND(%s)\n", symbol);
273 }
274 
275 struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm, unsigned int id)
276 {
277 	int err, size;
278 	void *gebase, *p, *handler;
279 	int i;
280 
281 	struct kvm_vcpu *vcpu = kzalloc(sizeof(struct kvm_vcpu), GFP_KERNEL);
282 
283 	if (!vcpu) {
284 		err = -ENOMEM;
285 		goto out;
286 	}
287 
288 	err = kvm_vcpu_init(vcpu, kvm, id);
289 
290 	if (err)
291 		goto out_free_cpu;
292 
293 	kvm_debug("kvm @ %p: create cpu %d at %p\n", kvm, id, vcpu);
294 
295 	/*
296 	 * Allocate space for host mode exception handlers that handle
297 	 * guest mode exits
298 	 */
299 	if (cpu_has_veic || cpu_has_vint)
300 		size = 0x200 + VECTORSPACING * 64;
301 	else
302 		size = 0x4000;
303 
304 	gebase = kzalloc(ALIGN(size, PAGE_SIZE), GFP_KERNEL);
305 
306 	if (!gebase) {
307 		err = -ENOMEM;
308 		goto out_uninit_cpu;
309 	}
310 	kvm_debug("Allocated %d bytes for KVM Exception Handlers @ %p\n",
311 		  ALIGN(size, PAGE_SIZE), gebase);
312 
313 	/*
314 	 * Check new ebase actually fits in CP0_EBase. The lack of a write gate
315 	 * limits us to the low 512MB of physical address space. If the memory
316 	 * we allocate is out of range, just give up now.
317 	 */
318 	if (!cpu_has_ebase_wg && virt_to_phys(gebase) >= 0x20000000) {
319 		kvm_err("CP0_EBase.WG required for guest exception base %pK\n",
320 			gebase);
321 		err = -ENOMEM;
322 		goto out_free_gebase;
323 	}
324 
325 	/* Save new ebase */
326 	vcpu->arch.guest_ebase = gebase;
327 
328 	/* Build guest exception vectors dynamically in unmapped memory */
329 	handler = gebase + 0x2000;
330 
331 	/* TLB Refill, EXL = 0 */
332 	kvm_mips_build_exception(gebase, handler);
333 
334 	/* General Exception Entry point */
335 	kvm_mips_build_exception(gebase + 0x180, handler);
336 
337 	/* For vectored interrupts poke the exception code @ all offsets 0-7 */
338 	for (i = 0; i < 8; i++) {
339 		kvm_debug("L1 Vectored handler @ %p\n",
340 			  gebase + 0x200 + (i * VECTORSPACING));
341 		kvm_mips_build_exception(gebase + 0x200 + i * VECTORSPACING,
342 					 handler);
343 	}
344 
345 	/* General exit handler */
346 	p = handler;
347 	p = kvm_mips_build_exit(p);
348 
349 	/* Guest entry routine */
350 	vcpu->arch.vcpu_run = p;
351 	p = kvm_mips_build_vcpu_run(p);
352 
353 	/* Dump the generated code */
354 	pr_debug("#include <asm/asm.h>\n");
355 	pr_debug("#include <asm/regdef.h>\n");
356 	pr_debug("\n");
357 	dump_handler("kvm_vcpu_run", vcpu->arch.vcpu_run, p);
358 	dump_handler("kvm_gen_exc", gebase + 0x180, gebase + 0x200);
359 	dump_handler("kvm_exit", gebase + 0x2000, vcpu->arch.vcpu_run);
360 
361 	/* Invalidate the icache for these ranges */
362 	local_flush_icache_range((unsigned long)gebase,
363 				(unsigned long)gebase + ALIGN(size, PAGE_SIZE));
364 
365 	/*
366 	 * Allocate comm page for guest kernel, a TLB will be reserved for
367 	 * mapping GVA @ 0xFFFF8000 to this page
368 	 */
369 	vcpu->arch.kseg0_commpage = kzalloc(PAGE_SIZE << 1, GFP_KERNEL);
370 
371 	if (!vcpu->arch.kseg0_commpage) {
372 		err = -ENOMEM;
373 		goto out_free_gebase;
374 	}
375 
376 	kvm_debug("Allocated COMM page @ %p\n", vcpu->arch.kseg0_commpage);
377 	kvm_mips_commpage_init(vcpu);
378 
379 	/* Init */
380 	vcpu->arch.last_sched_cpu = -1;
381 
382 	/* Start off the timer */
383 	kvm_mips_init_count(vcpu);
384 
385 	return vcpu;
386 
387 out_free_gebase:
388 	kfree(gebase);
389 
390 out_uninit_cpu:
391 	kvm_vcpu_uninit(vcpu);
392 
393 out_free_cpu:
394 	kfree(vcpu);
395 
396 out:
397 	return ERR_PTR(err);
398 }
399 
400 void kvm_arch_vcpu_free(struct kvm_vcpu *vcpu)
401 {
402 	hrtimer_cancel(&vcpu->arch.comparecount_timer);
403 
404 	kvm_vcpu_uninit(vcpu);
405 
406 	kvm_mips_dump_stats(vcpu);
407 
408 	kfree(vcpu->arch.guest_ebase);
409 	kfree(vcpu->arch.kseg0_commpage);
410 	kfree(vcpu);
411 }
412 
413 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
414 {
415 	kvm_arch_vcpu_free(vcpu);
416 }
417 
418 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
419 					struct kvm_guest_debug *dbg)
420 {
421 	return -ENOIOCTLCMD;
422 }
423 
424 /* Must be called with preemption disabled, just before entering guest */
425 static void kvm_mips_check_asids(struct kvm_vcpu *vcpu)
426 {
427 	struct mips_coproc *cop0 = vcpu->arch.cop0;
428 	int cpu = smp_processor_id();
429 	unsigned int gasid;
430 
431 	/*
432 	 * Lazy host ASID regeneration for guest user mode.
433 	 * If the guest ASID has changed since the last guest usermode
434 	 * execution, regenerate the host ASID so as to invalidate stale TLB
435 	 * entries.
436 	 */
437 	if (!KVM_GUEST_KERNEL_MODE(vcpu)) {
438 		gasid = kvm_read_c0_guest_entryhi(cop0) & KVM_ENTRYHI_ASID;
439 		if (gasid != vcpu->arch.last_user_gasid) {
440 			kvm_get_new_mmu_context(&vcpu->arch.guest_user_mm, cpu,
441 						vcpu);
442 			vcpu->arch.guest_user_asid[cpu] =
443 				vcpu->arch.guest_user_mm.context.asid[cpu];
444 			vcpu->arch.last_user_gasid = gasid;
445 		}
446 	}
447 }
448 
449 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *run)
450 {
451 	int r = 0;
452 	sigset_t sigsaved;
453 
454 	if (vcpu->sigset_active)
455 		sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);
456 
457 	if (vcpu->mmio_needed) {
458 		if (!vcpu->mmio_is_write)
459 			kvm_mips_complete_mmio_load(vcpu, run);
460 		vcpu->mmio_needed = 0;
461 	}
462 
463 	lose_fpu(1);
464 
465 	local_irq_disable();
466 	/* Check if we have any exceptions/interrupts pending */
467 	kvm_mips_deliver_interrupts(vcpu,
468 				    kvm_read_c0_guest_cause(vcpu->arch.cop0));
469 
470 	guest_enter_irqoff();
471 
472 	/* Disable hardware page table walking while in guest */
473 	htw_stop();
474 
475 	trace_kvm_enter(vcpu);
476 
477 	kvm_mips_check_asids(vcpu);
478 
479 	r = vcpu->arch.vcpu_run(run, vcpu);
480 	trace_kvm_out(vcpu);
481 
482 	/* Re-enable HTW before enabling interrupts */
483 	htw_start();
484 
485 	guest_exit_irqoff();
486 	local_irq_enable();
487 
488 	if (vcpu->sigset_active)
489 		sigprocmask(SIG_SETMASK, &sigsaved, NULL);
490 
491 	return r;
492 }
493 
494 int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu,
495 			     struct kvm_mips_interrupt *irq)
496 {
497 	int intr = (int)irq->irq;
498 	struct kvm_vcpu *dvcpu = NULL;
499 
500 	if (intr == 3 || intr == -3 || intr == 4 || intr == -4)
501 		kvm_debug("%s: CPU: %d, INTR: %d\n", __func__, irq->cpu,
502 			  (int)intr);
503 
504 	if (irq->cpu == -1)
505 		dvcpu = vcpu;
506 	else
507 		dvcpu = vcpu->kvm->vcpus[irq->cpu];
508 
509 	if (intr == 2 || intr == 3 || intr == 4) {
510 		kvm_mips_callbacks->queue_io_int(dvcpu, irq);
511 
512 	} else if (intr == -2 || intr == -3 || intr == -4) {
513 		kvm_mips_callbacks->dequeue_io_int(dvcpu, irq);
514 	} else {
515 		kvm_err("%s: invalid interrupt ioctl (%d:%d)\n", __func__,
516 			irq->cpu, irq->irq);
517 		return -EINVAL;
518 	}
519 
520 	dvcpu->arch.wait = 0;
521 
522 	if (swait_active(&dvcpu->wq))
523 		swake_up(&dvcpu->wq);
524 
525 	return 0;
526 }
527 
528 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
529 				    struct kvm_mp_state *mp_state)
530 {
531 	return -ENOIOCTLCMD;
532 }
533 
534 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
535 				    struct kvm_mp_state *mp_state)
536 {
537 	return -ENOIOCTLCMD;
538 }
539 
540 static u64 kvm_mips_get_one_regs[] = {
541 	KVM_REG_MIPS_R0,
542 	KVM_REG_MIPS_R1,
543 	KVM_REG_MIPS_R2,
544 	KVM_REG_MIPS_R3,
545 	KVM_REG_MIPS_R4,
546 	KVM_REG_MIPS_R5,
547 	KVM_REG_MIPS_R6,
548 	KVM_REG_MIPS_R7,
549 	KVM_REG_MIPS_R8,
550 	KVM_REG_MIPS_R9,
551 	KVM_REG_MIPS_R10,
552 	KVM_REG_MIPS_R11,
553 	KVM_REG_MIPS_R12,
554 	KVM_REG_MIPS_R13,
555 	KVM_REG_MIPS_R14,
556 	KVM_REG_MIPS_R15,
557 	KVM_REG_MIPS_R16,
558 	KVM_REG_MIPS_R17,
559 	KVM_REG_MIPS_R18,
560 	KVM_REG_MIPS_R19,
561 	KVM_REG_MIPS_R20,
562 	KVM_REG_MIPS_R21,
563 	KVM_REG_MIPS_R22,
564 	KVM_REG_MIPS_R23,
565 	KVM_REG_MIPS_R24,
566 	KVM_REG_MIPS_R25,
567 	KVM_REG_MIPS_R26,
568 	KVM_REG_MIPS_R27,
569 	KVM_REG_MIPS_R28,
570 	KVM_REG_MIPS_R29,
571 	KVM_REG_MIPS_R30,
572 	KVM_REG_MIPS_R31,
573 
574 #ifndef CONFIG_CPU_MIPSR6
575 	KVM_REG_MIPS_HI,
576 	KVM_REG_MIPS_LO,
577 #endif
578 	KVM_REG_MIPS_PC,
579 
580 	KVM_REG_MIPS_CP0_INDEX,
581 	KVM_REG_MIPS_CP0_CONTEXT,
582 	KVM_REG_MIPS_CP0_USERLOCAL,
583 	KVM_REG_MIPS_CP0_PAGEMASK,
584 	KVM_REG_MIPS_CP0_WIRED,
585 	KVM_REG_MIPS_CP0_HWRENA,
586 	KVM_REG_MIPS_CP0_BADVADDR,
587 	KVM_REG_MIPS_CP0_COUNT,
588 	KVM_REG_MIPS_CP0_ENTRYHI,
589 	KVM_REG_MIPS_CP0_COMPARE,
590 	KVM_REG_MIPS_CP0_STATUS,
591 	KVM_REG_MIPS_CP0_CAUSE,
592 	KVM_REG_MIPS_CP0_EPC,
593 	KVM_REG_MIPS_CP0_PRID,
594 	KVM_REG_MIPS_CP0_CONFIG,
595 	KVM_REG_MIPS_CP0_CONFIG1,
596 	KVM_REG_MIPS_CP0_CONFIG2,
597 	KVM_REG_MIPS_CP0_CONFIG3,
598 	KVM_REG_MIPS_CP0_CONFIG4,
599 	KVM_REG_MIPS_CP0_CONFIG5,
600 	KVM_REG_MIPS_CP0_CONFIG7,
601 	KVM_REG_MIPS_CP0_ERROREPC,
602 
603 	KVM_REG_MIPS_COUNT_CTL,
604 	KVM_REG_MIPS_COUNT_RESUME,
605 	KVM_REG_MIPS_COUNT_HZ,
606 };
607 
608 static u64 kvm_mips_get_one_regs_fpu[] = {
609 	KVM_REG_MIPS_FCR_IR,
610 	KVM_REG_MIPS_FCR_CSR,
611 };
612 
613 static u64 kvm_mips_get_one_regs_msa[] = {
614 	KVM_REG_MIPS_MSA_IR,
615 	KVM_REG_MIPS_MSA_CSR,
616 };
617 
618 static u64 kvm_mips_get_one_regs_kscratch[] = {
619 	KVM_REG_MIPS_CP0_KSCRATCH1,
620 	KVM_REG_MIPS_CP0_KSCRATCH2,
621 	KVM_REG_MIPS_CP0_KSCRATCH3,
622 	KVM_REG_MIPS_CP0_KSCRATCH4,
623 	KVM_REG_MIPS_CP0_KSCRATCH5,
624 	KVM_REG_MIPS_CP0_KSCRATCH6,
625 };
626 
627 static unsigned long kvm_mips_num_regs(struct kvm_vcpu *vcpu)
628 {
629 	unsigned long ret;
630 
631 	ret = ARRAY_SIZE(kvm_mips_get_one_regs);
632 	if (kvm_mips_guest_can_have_fpu(&vcpu->arch)) {
633 		ret += ARRAY_SIZE(kvm_mips_get_one_regs_fpu) + 48;
634 		/* odd doubles */
635 		if (boot_cpu_data.fpu_id & MIPS_FPIR_F64)
636 			ret += 16;
637 	}
638 	if (kvm_mips_guest_can_have_msa(&vcpu->arch))
639 		ret += ARRAY_SIZE(kvm_mips_get_one_regs_msa) + 32;
640 	ret += __arch_hweight8(vcpu->arch.kscratch_enabled);
641 	ret += kvm_mips_callbacks->num_regs(vcpu);
642 
643 	return ret;
644 }
645 
646 static int kvm_mips_copy_reg_indices(struct kvm_vcpu *vcpu, u64 __user *indices)
647 {
648 	u64 index;
649 	unsigned int i;
650 
651 	if (copy_to_user(indices, kvm_mips_get_one_regs,
652 			 sizeof(kvm_mips_get_one_regs)))
653 		return -EFAULT;
654 	indices += ARRAY_SIZE(kvm_mips_get_one_regs);
655 
656 	if (kvm_mips_guest_can_have_fpu(&vcpu->arch)) {
657 		if (copy_to_user(indices, kvm_mips_get_one_regs_fpu,
658 				 sizeof(kvm_mips_get_one_regs_fpu)))
659 			return -EFAULT;
660 		indices += ARRAY_SIZE(kvm_mips_get_one_regs_fpu);
661 
662 		for (i = 0; i < 32; ++i) {
663 			index = KVM_REG_MIPS_FPR_32(i);
664 			if (copy_to_user(indices, &index, sizeof(index)))
665 				return -EFAULT;
666 			++indices;
667 
668 			/* skip odd doubles if no F64 */
669 			if (i & 1 && !(boot_cpu_data.fpu_id & MIPS_FPIR_F64))
670 				continue;
671 
672 			index = KVM_REG_MIPS_FPR_64(i);
673 			if (copy_to_user(indices, &index, sizeof(index)))
674 				return -EFAULT;
675 			++indices;
676 		}
677 	}
678 
679 	if (kvm_mips_guest_can_have_msa(&vcpu->arch)) {
680 		if (copy_to_user(indices, kvm_mips_get_one_regs_msa,
681 				 sizeof(kvm_mips_get_one_regs_msa)))
682 			return -EFAULT;
683 		indices += ARRAY_SIZE(kvm_mips_get_one_regs_msa);
684 
685 		for (i = 0; i < 32; ++i) {
686 			index = KVM_REG_MIPS_VEC_128(i);
687 			if (copy_to_user(indices, &index, sizeof(index)))
688 				return -EFAULT;
689 			++indices;
690 		}
691 	}
692 
693 	for (i = 0; i < 6; ++i) {
694 		if (!(vcpu->arch.kscratch_enabled & BIT(i + 2)))
695 			continue;
696 
697 		if (copy_to_user(indices, &kvm_mips_get_one_regs_kscratch[i],
698 				 sizeof(kvm_mips_get_one_regs_kscratch[i])))
699 			return -EFAULT;
700 		++indices;
701 	}
702 
703 	return kvm_mips_callbacks->copy_reg_indices(vcpu, indices);
704 }
705 
706 static int kvm_mips_get_reg(struct kvm_vcpu *vcpu,
707 			    const struct kvm_one_reg *reg)
708 {
709 	struct mips_coproc *cop0 = vcpu->arch.cop0;
710 	struct mips_fpu_struct *fpu = &vcpu->arch.fpu;
711 	int ret;
712 	s64 v;
713 	s64 vs[2];
714 	unsigned int idx;
715 
716 	switch (reg->id) {
717 	/* General purpose registers */
718 	case KVM_REG_MIPS_R0 ... KVM_REG_MIPS_R31:
719 		v = (long)vcpu->arch.gprs[reg->id - KVM_REG_MIPS_R0];
720 		break;
721 #ifndef CONFIG_CPU_MIPSR6
722 	case KVM_REG_MIPS_HI:
723 		v = (long)vcpu->arch.hi;
724 		break;
725 	case KVM_REG_MIPS_LO:
726 		v = (long)vcpu->arch.lo;
727 		break;
728 #endif
729 	case KVM_REG_MIPS_PC:
730 		v = (long)vcpu->arch.pc;
731 		break;
732 
733 	/* Floating point registers */
734 	case KVM_REG_MIPS_FPR_32(0) ... KVM_REG_MIPS_FPR_32(31):
735 		if (!kvm_mips_guest_has_fpu(&vcpu->arch))
736 			return -EINVAL;
737 		idx = reg->id - KVM_REG_MIPS_FPR_32(0);
738 		/* Odd singles in top of even double when FR=0 */
739 		if (kvm_read_c0_guest_status(cop0) & ST0_FR)
740 			v = get_fpr32(&fpu->fpr[idx], 0);
741 		else
742 			v = get_fpr32(&fpu->fpr[idx & ~1], idx & 1);
743 		break;
744 	case KVM_REG_MIPS_FPR_64(0) ... KVM_REG_MIPS_FPR_64(31):
745 		if (!kvm_mips_guest_has_fpu(&vcpu->arch))
746 			return -EINVAL;
747 		idx = reg->id - KVM_REG_MIPS_FPR_64(0);
748 		/* Can't access odd doubles in FR=0 mode */
749 		if (idx & 1 && !(kvm_read_c0_guest_status(cop0) & ST0_FR))
750 			return -EINVAL;
751 		v = get_fpr64(&fpu->fpr[idx], 0);
752 		break;
753 	case KVM_REG_MIPS_FCR_IR:
754 		if (!kvm_mips_guest_has_fpu(&vcpu->arch))
755 			return -EINVAL;
756 		v = boot_cpu_data.fpu_id;
757 		break;
758 	case KVM_REG_MIPS_FCR_CSR:
759 		if (!kvm_mips_guest_has_fpu(&vcpu->arch))
760 			return -EINVAL;
761 		v = fpu->fcr31;
762 		break;
763 
764 	/* MIPS SIMD Architecture (MSA) registers */
765 	case KVM_REG_MIPS_VEC_128(0) ... KVM_REG_MIPS_VEC_128(31):
766 		if (!kvm_mips_guest_has_msa(&vcpu->arch))
767 			return -EINVAL;
768 		/* Can't access MSA registers in FR=0 mode */
769 		if (!(kvm_read_c0_guest_status(cop0) & ST0_FR))
770 			return -EINVAL;
771 		idx = reg->id - KVM_REG_MIPS_VEC_128(0);
772 #ifdef CONFIG_CPU_LITTLE_ENDIAN
773 		/* least significant byte first */
774 		vs[0] = get_fpr64(&fpu->fpr[idx], 0);
775 		vs[1] = get_fpr64(&fpu->fpr[idx], 1);
776 #else
777 		/* most significant byte first */
778 		vs[0] = get_fpr64(&fpu->fpr[idx], 1);
779 		vs[1] = get_fpr64(&fpu->fpr[idx], 0);
780 #endif
781 		break;
782 	case KVM_REG_MIPS_MSA_IR:
783 		if (!kvm_mips_guest_has_msa(&vcpu->arch))
784 			return -EINVAL;
785 		v = boot_cpu_data.msa_id;
786 		break;
787 	case KVM_REG_MIPS_MSA_CSR:
788 		if (!kvm_mips_guest_has_msa(&vcpu->arch))
789 			return -EINVAL;
790 		v = fpu->msacsr;
791 		break;
792 
793 	/* Co-processor 0 registers */
794 	case KVM_REG_MIPS_CP0_INDEX:
795 		v = (long)kvm_read_c0_guest_index(cop0);
796 		break;
797 	case KVM_REG_MIPS_CP0_CONTEXT:
798 		v = (long)kvm_read_c0_guest_context(cop0);
799 		break;
800 	case KVM_REG_MIPS_CP0_USERLOCAL:
801 		v = (long)kvm_read_c0_guest_userlocal(cop0);
802 		break;
803 	case KVM_REG_MIPS_CP0_PAGEMASK:
804 		v = (long)kvm_read_c0_guest_pagemask(cop0);
805 		break;
806 	case KVM_REG_MIPS_CP0_WIRED:
807 		v = (long)kvm_read_c0_guest_wired(cop0);
808 		break;
809 	case KVM_REG_MIPS_CP0_HWRENA:
810 		v = (long)kvm_read_c0_guest_hwrena(cop0);
811 		break;
812 	case KVM_REG_MIPS_CP0_BADVADDR:
813 		v = (long)kvm_read_c0_guest_badvaddr(cop0);
814 		break;
815 	case KVM_REG_MIPS_CP0_ENTRYHI:
816 		v = (long)kvm_read_c0_guest_entryhi(cop0);
817 		break;
818 	case KVM_REG_MIPS_CP0_COMPARE:
819 		v = (long)kvm_read_c0_guest_compare(cop0);
820 		break;
821 	case KVM_REG_MIPS_CP0_STATUS:
822 		v = (long)kvm_read_c0_guest_status(cop0);
823 		break;
824 	case KVM_REG_MIPS_CP0_CAUSE:
825 		v = (long)kvm_read_c0_guest_cause(cop0);
826 		break;
827 	case KVM_REG_MIPS_CP0_EPC:
828 		v = (long)kvm_read_c0_guest_epc(cop0);
829 		break;
830 	case KVM_REG_MIPS_CP0_PRID:
831 		v = (long)kvm_read_c0_guest_prid(cop0);
832 		break;
833 	case KVM_REG_MIPS_CP0_CONFIG:
834 		v = (long)kvm_read_c0_guest_config(cop0);
835 		break;
836 	case KVM_REG_MIPS_CP0_CONFIG1:
837 		v = (long)kvm_read_c0_guest_config1(cop0);
838 		break;
839 	case KVM_REG_MIPS_CP0_CONFIG2:
840 		v = (long)kvm_read_c0_guest_config2(cop0);
841 		break;
842 	case KVM_REG_MIPS_CP0_CONFIG3:
843 		v = (long)kvm_read_c0_guest_config3(cop0);
844 		break;
845 	case KVM_REG_MIPS_CP0_CONFIG4:
846 		v = (long)kvm_read_c0_guest_config4(cop0);
847 		break;
848 	case KVM_REG_MIPS_CP0_CONFIG5:
849 		v = (long)kvm_read_c0_guest_config5(cop0);
850 		break;
851 	case KVM_REG_MIPS_CP0_CONFIG7:
852 		v = (long)kvm_read_c0_guest_config7(cop0);
853 		break;
854 	case KVM_REG_MIPS_CP0_ERROREPC:
855 		v = (long)kvm_read_c0_guest_errorepc(cop0);
856 		break;
857 	case KVM_REG_MIPS_CP0_KSCRATCH1 ... KVM_REG_MIPS_CP0_KSCRATCH6:
858 		idx = reg->id - KVM_REG_MIPS_CP0_KSCRATCH1 + 2;
859 		if (!(vcpu->arch.kscratch_enabled & BIT(idx)))
860 			return -EINVAL;
861 		switch (idx) {
862 		case 2:
863 			v = (long)kvm_read_c0_guest_kscratch1(cop0);
864 			break;
865 		case 3:
866 			v = (long)kvm_read_c0_guest_kscratch2(cop0);
867 			break;
868 		case 4:
869 			v = (long)kvm_read_c0_guest_kscratch3(cop0);
870 			break;
871 		case 5:
872 			v = (long)kvm_read_c0_guest_kscratch4(cop0);
873 			break;
874 		case 6:
875 			v = (long)kvm_read_c0_guest_kscratch5(cop0);
876 			break;
877 		case 7:
878 			v = (long)kvm_read_c0_guest_kscratch6(cop0);
879 			break;
880 		}
881 		break;
882 	/* registers to be handled specially */
883 	default:
884 		ret = kvm_mips_callbacks->get_one_reg(vcpu, reg, &v);
885 		if (ret)
886 			return ret;
887 		break;
888 	}
889 	if ((reg->id & KVM_REG_SIZE_MASK) == KVM_REG_SIZE_U64) {
890 		u64 __user *uaddr64 = (u64 __user *)(long)reg->addr;
891 
892 		return put_user(v, uaddr64);
893 	} else if ((reg->id & KVM_REG_SIZE_MASK) == KVM_REG_SIZE_U32) {
894 		u32 __user *uaddr32 = (u32 __user *)(long)reg->addr;
895 		u32 v32 = (u32)v;
896 
897 		return put_user(v32, uaddr32);
898 	} else if ((reg->id & KVM_REG_SIZE_MASK) == KVM_REG_SIZE_U128) {
899 		void __user *uaddr = (void __user *)(long)reg->addr;
900 
901 		return copy_to_user(uaddr, vs, 16) ? -EFAULT : 0;
902 	} else {
903 		return -EINVAL;
904 	}
905 }
906 
907 static int kvm_mips_set_reg(struct kvm_vcpu *vcpu,
908 			    const struct kvm_one_reg *reg)
909 {
910 	struct mips_coproc *cop0 = vcpu->arch.cop0;
911 	struct mips_fpu_struct *fpu = &vcpu->arch.fpu;
912 	s64 v;
913 	s64 vs[2];
914 	unsigned int idx;
915 
916 	if ((reg->id & KVM_REG_SIZE_MASK) == KVM_REG_SIZE_U64) {
917 		u64 __user *uaddr64 = (u64 __user *)(long)reg->addr;
918 
919 		if (get_user(v, uaddr64) != 0)
920 			return -EFAULT;
921 	} else if ((reg->id & KVM_REG_SIZE_MASK) == KVM_REG_SIZE_U32) {
922 		u32 __user *uaddr32 = (u32 __user *)(long)reg->addr;
923 		s32 v32;
924 
925 		if (get_user(v32, uaddr32) != 0)
926 			return -EFAULT;
927 		v = (s64)v32;
928 	} else if ((reg->id & KVM_REG_SIZE_MASK) == KVM_REG_SIZE_U128) {
929 		void __user *uaddr = (void __user *)(long)reg->addr;
930 
931 		return copy_from_user(vs, uaddr, 16) ? -EFAULT : 0;
932 	} else {
933 		return -EINVAL;
934 	}
935 
936 	switch (reg->id) {
937 	/* General purpose registers */
938 	case KVM_REG_MIPS_R0:
939 		/* Silently ignore requests to set $0 */
940 		break;
941 	case KVM_REG_MIPS_R1 ... KVM_REG_MIPS_R31:
942 		vcpu->arch.gprs[reg->id - KVM_REG_MIPS_R0] = v;
943 		break;
944 #ifndef CONFIG_CPU_MIPSR6
945 	case KVM_REG_MIPS_HI:
946 		vcpu->arch.hi = v;
947 		break;
948 	case KVM_REG_MIPS_LO:
949 		vcpu->arch.lo = v;
950 		break;
951 #endif
952 	case KVM_REG_MIPS_PC:
953 		vcpu->arch.pc = v;
954 		break;
955 
956 	/* Floating point registers */
957 	case KVM_REG_MIPS_FPR_32(0) ... KVM_REG_MIPS_FPR_32(31):
958 		if (!kvm_mips_guest_has_fpu(&vcpu->arch))
959 			return -EINVAL;
960 		idx = reg->id - KVM_REG_MIPS_FPR_32(0);
961 		/* Odd singles in top of even double when FR=0 */
962 		if (kvm_read_c0_guest_status(cop0) & ST0_FR)
963 			set_fpr32(&fpu->fpr[idx], 0, v);
964 		else
965 			set_fpr32(&fpu->fpr[idx & ~1], idx & 1, v);
966 		break;
967 	case KVM_REG_MIPS_FPR_64(0) ... KVM_REG_MIPS_FPR_64(31):
968 		if (!kvm_mips_guest_has_fpu(&vcpu->arch))
969 			return -EINVAL;
970 		idx = reg->id - KVM_REG_MIPS_FPR_64(0);
971 		/* Can't access odd doubles in FR=0 mode */
972 		if (idx & 1 && !(kvm_read_c0_guest_status(cop0) & ST0_FR))
973 			return -EINVAL;
974 		set_fpr64(&fpu->fpr[idx], 0, v);
975 		break;
976 	case KVM_REG_MIPS_FCR_IR:
977 		if (!kvm_mips_guest_has_fpu(&vcpu->arch))
978 			return -EINVAL;
979 		/* Read-only */
980 		break;
981 	case KVM_REG_MIPS_FCR_CSR:
982 		if (!kvm_mips_guest_has_fpu(&vcpu->arch))
983 			return -EINVAL;
984 		fpu->fcr31 = v;
985 		break;
986 
987 	/* MIPS SIMD Architecture (MSA) registers */
988 	case KVM_REG_MIPS_VEC_128(0) ... KVM_REG_MIPS_VEC_128(31):
989 		if (!kvm_mips_guest_has_msa(&vcpu->arch))
990 			return -EINVAL;
991 		idx = reg->id - KVM_REG_MIPS_VEC_128(0);
992 #ifdef CONFIG_CPU_LITTLE_ENDIAN
993 		/* least significant byte first */
994 		set_fpr64(&fpu->fpr[idx], 0, vs[0]);
995 		set_fpr64(&fpu->fpr[idx], 1, vs[1]);
996 #else
997 		/* most significant byte first */
998 		set_fpr64(&fpu->fpr[idx], 1, vs[0]);
999 		set_fpr64(&fpu->fpr[idx], 0, vs[1]);
1000 #endif
1001 		break;
1002 	case KVM_REG_MIPS_MSA_IR:
1003 		if (!kvm_mips_guest_has_msa(&vcpu->arch))
1004 			return -EINVAL;
1005 		/* Read-only */
1006 		break;
1007 	case KVM_REG_MIPS_MSA_CSR:
1008 		if (!kvm_mips_guest_has_msa(&vcpu->arch))
1009 			return -EINVAL;
1010 		fpu->msacsr = v;
1011 		break;
1012 
1013 	/* Co-processor 0 registers */
1014 	case KVM_REG_MIPS_CP0_INDEX:
1015 		kvm_write_c0_guest_index(cop0, v);
1016 		break;
1017 	case KVM_REG_MIPS_CP0_CONTEXT:
1018 		kvm_write_c0_guest_context(cop0, v);
1019 		break;
1020 	case KVM_REG_MIPS_CP0_USERLOCAL:
1021 		kvm_write_c0_guest_userlocal(cop0, v);
1022 		break;
1023 	case KVM_REG_MIPS_CP0_PAGEMASK:
1024 		kvm_write_c0_guest_pagemask(cop0, v);
1025 		break;
1026 	case KVM_REG_MIPS_CP0_WIRED:
1027 		kvm_write_c0_guest_wired(cop0, v);
1028 		break;
1029 	case KVM_REG_MIPS_CP0_HWRENA:
1030 		kvm_write_c0_guest_hwrena(cop0, v);
1031 		break;
1032 	case KVM_REG_MIPS_CP0_BADVADDR:
1033 		kvm_write_c0_guest_badvaddr(cop0, v);
1034 		break;
1035 	case KVM_REG_MIPS_CP0_ENTRYHI:
1036 		kvm_write_c0_guest_entryhi(cop0, v);
1037 		break;
1038 	case KVM_REG_MIPS_CP0_STATUS:
1039 		kvm_write_c0_guest_status(cop0, v);
1040 		break;
1041 	case KVM_REG_MIPS_CP0_EPC:
1042 		kvm_write_c0_guest_epc(cop0, v);
1043 		break;
1044 	case KVM_REG_MIPS_CP0_PRID:
1045 		kvm_write_c0_guest_prid(cop0, v);
1046 		break;
1047 	case KVM_REG_MIPS_CP0_ERROREPC:
1048 		kvm_write_c0_guest_errorepc(cop0, v);
1049 		break;
1050 	case KVM_REG_MIPS_CP0_KSCRATCH1 ... KVM_REG_MIPS_CP0_KSCRATCH6:
1051 		idx = reg->id - KVM_REG_MIPS_CP0_KSCRATCH1 + 2;
1052 		if (!(vcpu->arch.kscratch_enabled & BIT(idx)))
1053 			return -EINVAL;
1054 		switch (idx) {
1055 		case 2:
1056 			kvm_write_c0_guest_kscratch1(cop0, v);
1057 			break;
1058 		case 3:
1059 			kvm_write_c0_guest_kscratch2(cop0, v);
1060 			break;
1061 		case 4:
1062 			kvm_write_c0_guest_kscratch3(cop0, v);
1063 			break;
1064 		case 5:
1065 			kvm_write_c0_guest_kscratch4(cop0, v);
1066 			break;
1067 		case 6:
1068 			kvm_write_c0_guest_kscratch5(cop0, v);
1069 			break;
1070 		case 7:
1071 			kvm_write_c0_guest_kscratch6(cop0, v);
1072 			break;
1073 		}
1074 		break;
1075 	/* registers to be handled specially */
1076 	default:
1077 		return kvm_mips_callbacks->set_one_reg(vcpu, reg, v);
1078 	}
1079 	return 0;
1080 }
1081 
1082 static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
1083 				     struct kvm_enable_cap *cap)
1084 {
1085 	int r = 0;
1086 
1087 	if (!kvm_vm_ioctl_check_extension(vcpu->kvm, cap->cap))
1088 		return -EINVAL;
1089 	if (cap->flags)
1090 		return -EINVAL;
1091 	if (cap->args[0])
1092 		return -EINVAL;
1093 
1094 	switch (cap->cap) {
1095 	case KVM_CAP_MIPS_FPU:
1096 		vcpu->arch.fpu_enabled = true;
1097 		break;
1098 	case KVM_CAP_MIPS_MSA:
1099 		vcpu->arch.msa_enabled = true;
1100 		break;
1101 	default:
1102 		r = -EINVAL;
1103 		break;
1104 	}
1105 
1106 	return r;
1107 }
1108 
1109 long kvm_arch_vcpu_ioctl(struct file *filp, unsigned int ioctl,
1110 			 unsigned long arg)
1111 {
1112 	struct kvm_vcpu *vcpu = filp->private_data;
1113 	void __user *argp = (void __user *)arg;
1114 	long r;
1115 
1116 	switch (ioctl) {
1117 	case KVM_SET_ONE_REG:
1118 	case KVM_GET_ONE_REG: {
1119 		struct kvm_one_reg reg;
1120 
1121 		if (copy_from_user(&reg, argp, sizeof(reg)))
1122 			return -EFAULT;
1123 		if (ioctl == KVM_SET_ONE_REG)
1124 			return kvm_mips_set_reg(vcpu, &reg);
1125 		else
1126 			return kvm_mips_get_reg(vcpu, &reg);
1127 	}
1128 	case KVM_GET_REG_LIST: {
1129 		struct kvm_reg_list __user *user_list = argp;
1130 		struct kvm_reg_list reg_list;
1131 		unsigned n;
1132 
1133 		if (copy_from_user(&reg_list, user_list, sizeof(reg_list)))
1134 			return -EFAULT;
1135 		n = reg_list.n;
1136 		reg_list.n = kvm_mips_num_regs(vcpu);
1137 		if (copy_to_user(user_list, &reg_list, sizeof(reg_list)))
1138 			return -EFAULT;
1139 		if (n < reg_list.n)
1140 			return -E2BIG;
1141 		return kvm_mips_copy_reg_indices(vcpu, user_list->reg);
1142 	}
1143 	case KVM_NMI:
1144 		/* Treat the NMI as a CPU reset */
1145 		r = kvm_mips_reset_vcpu(vcpu);
1146 		break;
1147 	case KVM_INTERRUPT:
1148 		{
1149 			struct kvm_mips_interrupt irq;
1150 
1151 			r = -EFAULT;
1152 			if (copy_from_user(&irq, argp, sizeof(irq)))
1153 				goto out;
1154 
1155 			kvm_debug("[%d] %s: irq: %d\n", vcpu->vcpu_id, __func__,
1156 				  irq.irq);
1157 
1158 			r = kvm_vcpu_ioctl_interrupt(vcpu, &irq);
1159 			break;
1160 		}
1161 	case KVM_ENABLE_CAP: {
1162 		struct kvm_enable_cap cap;
1163 
1164 		r = -EFAULT;
1165 		if (copy_from_user(&cap, argp, sizeof(cap)))
1166 			goto out;
1167 		r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
1168 		break;
1169 	}
1170 	default:
1171 		r = -ENOIOCTLCMD;
1172 	}
1173 
1174 out:
1175 	return r;
1176 }
1177 
1178 /* Get (and clear) the dirty memory log for a memory slot. */
1179 int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm, struct kvm_dirty_log *log)
1180 {
1181 	struct kvm_memslots *slots;
1182 	struct kvm_memory_slot *memslot;
1183 	unsigned long ga, ga_end;
1184 	int is_dirty = 0;
1185 	int r;
1186 	unsigned long n;
1187 
1188 	mutex_lock(&kvm->slots_lock);
1189 
1190 	r = kvm_get_dirty_log(kvm, log, &is_dirty);
1191 	if (r)
1192 		goto out;
1193 
1194 	/* If nothing is dirty, don't bother messing with page tables. */
1195 	if (is_dirty) {
1196 		slots = kvm_memslots(kvm);
1197 		memslot = id_to_memslot(slots, log->slot);
1198 
1199 		ga = memslot->base_gfn << PAGE_SHIFT;
1200 		ga_end = ga + (memslot->npages << PAGE_SHIFT);
1201 
1202 		kvm_info("%s: dirty, ga: %#lx, ga_end %#lx\n", __func__, ga,
1203 			 ga_end);
1204 
1205 		n = kvm_dirty_bitmap_bytes(memslot);
1206 		memset(memslot->dirty_bitmap, 0, n);
1207 	}
1208 
1209 	r = 0;
1210 out:
1211 	mutex_unlock(&kvm->slots_lock);
1212 	return r;
1213 
1214 }
1215 
1216 long kvm_arch_vm_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg)
1217 {
1218 	long r;
1219 
1220 	switch (ioctl) {
1221 	default:
1222 		r = -ENOIOCTLCMD;
1223 	}
1224 
1225 	return r;
1226 }
1227 
1228 int kvm_arch_init(void *opaque)
1229 {
1230 	if (kvm_mips_callbacks) {
1231 		kvm_err("kvm: module already exists\n");
1232 		return -EEXIST;
1233 	}
1234 
1235 	return kvm_mips_emulation_init(&kvm_mips_callbacks);
1236 }
1237 
1238 void kvm_arch_exit(void)
1239 {
1240 	kvm_mips_callbacks = NULL;
1241 }
1242 
1243 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
1244 				  struct kvm_sregs *sregs)
1245 {
1246 	return -ENOIOCTLCMD;
1247 }
1248 
1249 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
1250 				  struct kvm_sregs *sregs)
1251 {
1252 	return -ENOIOCTLCMD;
1253 }
1254 
1255 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
1256 {
1257 }
1258 
1259 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
1260 {
1261 	return -ENOIOCTLCMD;
1262 }
1263 
1264 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
1265 {
1266 	return -ENOIOCTLCMD;
1267 }
1268 
1269 int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
1270 {
1271 	return VM_FAULT_SIGBUS;
1272 }
1273 
1274 int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
1275 {
1276 	int r;
1277 
1278 	switch (ext) {
1279 	case KVM_CAP_ONE_REG:
1280 	case KVM_CAP_ENABLE_CAP:
1281 		r = 1;
1282 		break;
1283 	case KVM_CAP_COALESCED_MMIO:
1284 		r = KVM_COALESCED_MMIO_PAGE_OFFSET;
1285 		break;
1286 	case KVM_CAP_MIPS_FPU:
1287 		/* We don't handle systems with inconsistent cpu_has_fpu */
1288 		r = !!raw_cpu_has_fpu;
1289 		break;
1290 	case KVM_CAP_MIPS_MSA:
1291 		/*
1292 		 * We don't support MSA vector partitioning yet:
1293 		 * 1) It would require explicit support which can't be tested
1294 		 *    yet due to lack of support in current hardware.
1295 		 * 2) It extends the state that would need to be saved/restored
1296 		 *    by e.g. QEMU for migration.
1297 		 *
1298 		 * When vector partitioning hardware becomes available, support
1299 		 * could be added by requiring a flag when enabling
1300 		 * KVM_CAP_MIPS_MSA capability to indicate that userland knows
1301 		 * to save/restore the appropriate extra state.
1302 		 */
1303 		r = cpu_has_msa && !(boot_cpu_data.msa_id & MSA_IR_WRPF);
1304 		break;
1305 	default:
1306 		r = 0;
1307 		break;
1308 	}
1309 	return r;
1310 }
1311 
1312 int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu)
1313 {
1314 	return kvm_mips_pending_timer(vcpu);
1315 }
1316 
1317 int kvm_arch_vcpu_dump_regs(struct kvm_vcpu *vcpu)
1318 {
1319 	int i;
1320 	struct mips_coproc *cop0;
1321 
1322 	if (!vcpu)
1323 		return -1;
1324 
1325 	kvm_debug("VCPU Register Dump:\n");
1326 	kvm_debug("\tpc = 0x%08lx\n", vcpu->arch.pc);
1327 	kvm_debug("\texceptions: %08lx\n", vcpu->arch.pending_exceptions);
1328 
1329 	for (i = 0; i < 32; i += 4) {
1330 		kvm_debug("\tgpr%02d: %08lx %08lx %08lx %08lx\n", i,
1331 		       vcpu->arch.gprs[i],
1332 		       vcpu->arch.gprs[i + 1],
1333 		       vcpu->arch.gprs[i + 2], vcpu->arch.gprs[i + 3]);
1334 	}
1335 	kvm_debug("\thi: 0x%08lx\n", vcpu->arch.hi);
1336 	kvm_debug("\tlo: 0x%08lx\n", vcpu->arch.lo);
1337 
1338 	cop0 = vcpu->arch.cop0;
1339 	kvm_debug("\tStatus: 0x%08lx, Cause: 0x%08lx\n",
1340 		  kvm_read_c0_guest_status(cop0),
1341 		  kvm_read_c0_guest_cause(cop0));
1342 
1343 	kvm_debug("\tEPC: 0x%08lx\n", kvm_read_c0_guest_epc(cop0));
1344 
1345 	return 0;
1346 }
1347 
1348 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
1349 {
1350 	int i;
1351 
1352 	for (i = 1; i < ARRAY_SIZE(vcpu->arch.gprs); i++)
1353 		vcpu->arch.gprs[i] = regs->gpr[i];
1354 	vcpu->arch.gprs[0] = 0; /* zero is special, and cannot be set. */
1355 	vcpu->arch.hi = regs->hi;
1356 	vcpu->arch.lo = regs->lo;
1357 	vcpu->arch.pc = regs->pc;
1358 
1359 	return 0;
1360 }
1361 
1362 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
1363 {
1364 	int i;
1365 
1366 	for (i = 0; i < ARRAY_SIZE(vcpu->arch.gprs); i++)
1367 		regs->gpr[i] = vcpu->arch.gprs[i];
1368 
1369 	regs->hi = vcpu->arch.hi;
1370 	regs->lo = vcpu->arch.lo;
1371 	regs->pc = vcpu->arch.pc;
1372 
1373 	return 0;
1374 }
1375 
1376 static void kvm_mips_comparecount_func(unsigned long data)
1377 {
1378 	struct kvm_vcpu *vcpu = (struct kvm_vcpu *)data;
1379 
1380 	kvm_mips_callbacks->queue_timer_int(vcpu);
1381 
1382 	vcpu->arch.wait = 0;
1383 	if (swait_active(&vcpu->wq))
1384 		swake_up(&vcpu->wq);
1385 }
1386 
1387 /* low level hrtimer wake routine */
1388 static enum hrtimer_restart kvm_mips_comparecount_wakeup(struct hrtimer *timer)
1389 {
1390 	struct kvm_vcpu *vcpu;
1391 
1392 	vcpu = container_of(timer, struct kvm_vcpu, arch.comparecount_timer);
1393 	kvm_mips_comparecount_func((unsigned long) vcpu);
1394 	return kvm_mips_count_timeout(vcpu);
1395 }
1396 
1397 int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
1398 {
1399 	kvm_mips_callbacks->vcpu_init(vcpu);
1400 	hrtimer_init(&vcpu->arch.comparecount_timer, CLOCK_MONOTONIC,
1401 		     HRTIMER_MODE_REL);
1402 	vcpu->arch.comparecount_timer.function = kvm_mips_comparecount_wakeup;
1403 	return 0;
1404 }
1405 
1406 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
1407 				  struct kvm_translation *tr)
1408 {
1409 	return 0;
1410 }
1411 
1412 /* Initial guest state */
1413 int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
1414 {
1415 	return kvm_mips_callbacks->vcpu_setup(vcpu);
1416 }
1417 
1418 static void kvm_mips_set_c0_status(void)
1419 {
1420 	u32 status = read_c0_status();
1421 
1422 	if (cpu_has_dsp)
1423 		status |= (ST0_MX);
1424 
1425 	write_c0_status(status);
1426 	ehb();
1427 }
1428 
1429 /*
1430  * Return value is in the form (errcode<<2 | RESUME_FLAG_HOST | RESUME_FLAG_NV)
1431  */
1432 int kvm_mips_handle_exit(struct kvm_run *run, struct kvm_vcpu *vcpu)
1433 {
1434 	u32 cause = vcpu->arch.host_cp0_cause;
1435 	u32 exccode = (cause >> CAUSEB_EXCCODE) & 0x1f;
1436 	u32 __user *opc = (u32 __user *) vcpu->arch.pc;
1437 	unsigned long badvaddr = vcpu->arch.host_cp0_badvaddr;
1438 	enum emulation_result er = EMULATE_DONE;
1439 	int ret = RESUME_GUEST;
1440 
1441 	/* re-enable HTW before enabling interrupts */
1442 	htw_start();
1443 
1444 	/* Set a default exit reason */
1445 	run->exit_reason = KVM_EXIT_UNKNOWN;
1446 	run->ready_for_interrupt_injection = 1;
1447 
1448 	/*
1449 	 * Set the appropriate status bits based on host CPU features,
1450 	 * before we hit the scheduler
1451 	 */
1452 	kvm_mips_set_c0_status();
1453 
1454 	local_irq_enable();
1455 
1456 	kvm_debug("kvm_mips_handle_exit: cause: %#x, PC: %p, kvm_run: %p, kvm_vcpu: %p\n",
1457 			cause, opc, run, vcpu);
1458 	trace_kvm_exit(vcpu, exccode);
1459 
1460 	/*
1461 	 * Do a privilege check, if in UM most of these exit conditions end up
1462 	 * causing an exception to be delivered to the Guest Kernel
1463 	 */
1464 	er = kvm_mips_check_privilege(cause, opc, run, vcpu);
1465 	if (er == EMULATE_PRIV_FAIL) {
1466 		goto skip_emul;
1467 	} else if (er == EMULATE_FAIL) {
1468 		run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
1469 		ret = RESUME_HOST;
1470 		goto skip_emul;
1471 	}
1472 
1473 	switch (exccode) {
1474 	case EXCCODE_INT:
1475 		kvm_debug("[%d]EXCCODE_INT @ %p\n", vcpu->vcpu_id, opc);
1476 
1477 		++vcpu->stat.int_exits;
1478 
1479 		if (need_resched())
1480 			cond_resched();
1481 
1482 		ret = RESUME_GUEST;
1483 		break;
1484 
1485 	case EXCCODE_CPU:
1486 		kvm_debug("EXCCODE_CPU: @ PC: %p\n", opc);
1487 
1488 		++vcpu->stat.cop_unusable_exits;
1489 		ret = kvm_mips_callbacks->handle_cop_unusable(vcpu);
1490 		/* XXXKYMA: Might need to return to user space */
1491 		if (run->exit_reason == KVM_EXIT_IRQ_WINDOW_OPEN)
1492 			ret = RESUME_HOST;
1493 		break;
1494 
1495 	case EXCCODE_MOD:
1496 		++vcpu->stat.tlbmod_exits;
1497 		ret = kvm_mips_callbacks->handle_tlb_mod(vcpu);
1498 		break;
1499 
1500 	case EXCCODE_TLBS:
1501 		kvm_debug("TLB ST fault:  cause %#x, status %#lx, PC: %p, BadVaddr: %#lx\n",
1502 			  cause, kvm_read_c0_guest_status(vcpu->arch.cop0), opc,
1503 			  badvaddr);
1504 
1505 		++vcpu->stat.tlbmiss_st_exits;
1506 		ret = kvm_mips_callbacks->handle_tlb_st_miss(vcpu);
1507 		break;
1508 
1509 	case EXCCODE_TLBL:
1510 		kvm_debug("TLB LD fault: cause %#x, PC: %p, BadVaddr: %#lx\n",
1511 			  cause, opc, badvaddr);
1512 
1513 		++vcpu->stat.tlbmiss_ld_exits;
1514 		ret = kvm_mips_callbacks->handle_tlb_ld_miss(vcpu);
1515 		break;
1516 
1517 	case EXCCODE_ADES:
1518 		++vcpu->stat.addrerr_st_exits;
1519 		ret = kvm_mips_callbacks->handle_addr_err_st(vcpu);
1520 		break;
1521 
1522 	case EXCCODE_ADEL:
1523 		++vcpu->stat.addrerr_ld_exits;
1524 		ret = kvm_mips_callbacks->handle_addr_err_ld(vcpu);
1525 		break;
1526 
1527 	case EXCCODE_SYS:
1528 		++vcpu->stat.syscall_exits;
1529 		ret = kvm_mips_callbacks->handle_syscall(vcpu);
1530 		break;
1531 
1532 	case EXCCODE_RI:
1533 		++vcpu->stat.resvd_inst_exits;
1534 		ret = kvm_mips_callbacks->handle_res_inst(vcpu);
1535 		break;
1536 
1537 	case EXCCODE_BP:
1538 		++vcpu->stat.break_inst_exits;
1539 		ret = kvm_mips_callbacks->handle_break(vcpu);
1540 		break;
1541 
1542 	case EXCCODE_TR:
1543 		++vcpu->stat.trap_inst_exits;
1544 		ret = kvm_mips_callbacks->handle_trap(vcpu);
1545 		break;
1546 
1547 	case EXCCODE_MSAFPE:
1548 		++vcpu->stat.msa_fpe_exits;
1549 		ret = kvm_mips_callbacks->handle_msa_fpe(vcpu);
1550 		break;
1551 
1552 	case EXCCODE_FPE:
1553 		++vcpu->stat.fpe_exits;
1554 		ret = kvm_mips_callbacks->handle_fpe(vcpu);
1555 		break;
1556 
1557 	case EXCCODE_MSADIS:
1558 		++vcpu->stat.msa_disabled_exits;
1559 		ret = kvm_mips_callbacks->handle_msa_disabled(vcpu);
1560 		break;
1561 
1562 	default:
1563 		kvm_err("Exception Code: %d, not yet handled, @ PC: %p, inst: 0x%08x  BadVaddr: %#lx Status: %#lx\n",
1564 			exccode, opc, kvm_get_inst(opc, vcpu), badvaddr,
1565 			kvm_read_c0_guest_status(vcpu->arch.cop0));
1566 		kvm_arch_vcpu_dump_regs(vcpu);
1567 		run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
1568 		ret = RESUME_HOST;
1569 		break;
1570 
1571 	}
1572 
1573 skip_emul:
1574 	local_irq_disable();
1575 
1576 	if (er == EMULATE_DONE && !(ret & RESUME_HOST))
1577 		kvm_mips_deliver_interrupts(vcpu, cause);
1578 
1579 	if (!(ret & RESUME_HOST)) {
1580 		/* Only check for signals if not already exiting to userspace */
1581 		if (signal_pending(current)) {
1582 			run->exit_reason = KVM_EXIT_INTR;
1583 			ret = (-EINTR << 2) | RESUME_HOST;
1584 			++vcpu->stat.signal_exits;
1585 			trace_kvm_exit(vcpu, KVM_TRACE_EXIT_SIGNAL);
1586 		}
1587 	}
1588 
1589 	if (ret == RESUME_GUEST) {
1590 		trace_kvm_reenter(vcpu);
1591 
1592 		kvm_mips_check_asids(vcpu);
1593 
1594 		/*
1595 		 * If FPU / MSA are enabled (i.e. the guest's FPU / MSA context
1596 		 * is live), restore FCR31 / MSACSR.
1597 		 *
1598 		 * This should be before returning to the guest exception
1599 		 * vector, as it may well cause an [MSA] FP exception if there
1600 		 * are pending exception bits unmasked. (see
1601 		 * kvm_mips_csr_die_notifier() for how that is handled).
1602 		 */
1603 		if (kvm_mips_guest_has_fpu(&vcpu->arch) &&
1604 		    read_c0_status() & ST0_CU1)
1605 			__kvm_restore_fcsr(&vcpu->arch);
1606 
1607 		if (kvm_mips_guest_has_msa(&vcpu->arch) &&
1608 		    read_c0_config5() & MIPS_CONF5_MSAEN)
1609 			__kvm_restore_msacsr(&vcpu->arch);
1610 	}
1611 
1612 	/* Disable HTW before returning to guest or host */
1613 	htw_stop();
1614 
1615 	return ret;
1616 }
1617 
1618 /* Enable FPU for guest and restore context */
1619 void kvm_own_fpu(struct kvm_vcpu *vcpu)
1620 {
1621 	struct mips_coproc *cop0 = vcpu->arch.cop0;
1622 	unsigned int sr, cfg5;
1623 
1624 	preempt_disable();
1625 
1626 	sr = kvm_read_c0_guest_status(cop0);
1627 
1628 	/*
1629 	 * If MSA state is already live, it is undefined how it interacts with
1630 	 * FR=0 FPU state, and we don't want to hit reserved instruction
1631 	 * exceptions trying to save the MSA state later when CU=1 && FR=1, so
1632 	 * play it safe and save it first.
1633 	 *
1634 	 * In theory we shouldn't ever hit this case since kvm_lose_fpu() should
1635 	 * get called when guest CU1 is set, however we can't trust the guest
1636 	 * not to clobber the status register directly via the commpage.
1637 	 */
1638 	if (cpu_has_msa && sr & ST0_CU1 && !(sr & ST0_FR) &&
1639 	    vcpu->arch.aux_inuse & KVM_MIPS_AUX_MSA)
1640 		kvm_lose_fpu(vcpu);
1641 
1642 	/*
1643 	 * Enable FPU for guest
1644 	 * We set FR and FRE according to guest context
1645 	 */
1646 	change_c0_status(ST0_CU1 | ST0_FR, sr);
1647 	if (cpu_has_fre) {
1648 		cfg5 = kvm_read_c0_guest_config5(cop0);
1649 		change_c0_config5(MIPS_CONF5_FRE, cfg5);
1650 	}
1651 	enable_fpu_hazard();
1652 
1653 	/* If guest FPU state not active, restore it now */
1654 	if (!(vcpu->arch.aux_inuse & KVM_MIPS_AUX_FPU)) {
1655 		__kvm_restore_fpu(&vcpu->arch);
1656 		vcpu->arch.aux_inuse |= KVM_MIPS_AUX_FPU;
1657 		trace_kvm_aux(vcpu, KVM_TRACE_AUX_RESTORE, KVM_TRACE_AUX_FPU);
1658 	} else {
1659 		trace_kvm_aux(vcpu, KVM_TRACE_AUX_ENABLE, KVM_TRACE_AUX_FPU);
1660 	}
1661 
1662 	preempt_enable();
1663 }
1664 
1665 #ifdef CONFIG_CPU_HAS_MSA
1666 /* Enable MSA for guest and restore context */
1667 void kvm_own_msa(struct kvm_vcpu *vcpu)
1668 {
1669 	struct mips_coproc *cop0 = vcpu->arch.cop0;
1670 	unsigned int sr, cfg5;
1671 
1672 	preempt_disable();
1673 
1674 	/*
1675 	 * Enable FPU if enabled in guest, since we're restoring FPU context
1676 	 * anyway. We set FR and FRE according to guest context.
1677 	 */
1678 	if (kvm_mips_guest_has_fpu(&vcpu->arch)) {
1679 		sr = kvm_read_c0_guest_status(cop0);
1680 
1681 		/*
1682 		 * If FR=0 FPU state is already live, it is undefined how it
1683 		 * interacts with MSA state, so play it safe and save it first.
1684 		 */
1685 		if (!(sr & ST0_FR) &&
1686 		    (vcpu->arch.aux_inuse & (KVM_MIPS_AUX_FPU |
1687 				KVM_MIPS_AUX_MSA)) == KVM_MIPS_AUX_FPU)
1688 			kvm_lose_fpu(vcpu);
1689 
1690 		change_c0_status(ST0_CU1 | ST0_FR, sr);
1691 		if (sr & ST0_CU1 && cpu_has_fre) {
1692 			cfg5 = kvm_read_c0_guest_config5(cop0);
1693 			change_c0_config5(MIPS_CONF5_FRE, cfg5);
1694 		}
1695 	}
1696 
1697 	/* Enable MSA for guest */
1698 	set_c0_config5(MIPS_CONF5_MSAEN);
1699 	enable_fpu_hazard();
1700 
1701 	switch (vcpu->arch.aux_inuse & (KVM_MIPS_AUX_FPU | KVM_MIPS_AUX_MSA)) {
1702 	case KVM_MIPS_AUX_FPU:
1703 		/*
1704 		 * Guest FPU state already loaded, only restore upper MSA state
1705 		 */
1706 		__kvm_restore_msa_upper(&vcpu->arch);
1707 		vcpu->arch.aux_inuse |= KVM_MIPS_AUX_MSA;
1708 		trace_kvm_aux(vcpu, KVM_TRACE_AUX_RESTORE, KVM_TRACE_AUX_MSA);
1709 		break;
1710 	case 0:
1711 		/* Neither FPU or MSA already active, restore full MSA state */
1712 		__kvm_restore_msa(&vcpu->arch);
1713 		vcpu->arch.aux_inuse |= KVM_MIPS_AUX_MSA;
1714 		if (kvm_mips_guest_has_fpu(&vcpu->arch))
1715 			vcpu->arch.aux_inuse |= KVM_MIPS_AUX_FPU;
1716 		trace_kvm_aux(vcpu, KVM_TRACE_AUX_RESTORE,
1717 			      KVM_TRACE_AUX_FPU_MSA);
1718 		break;
1719 	default:
1720 		trace_kvm_aux(vcpu, KVM_TRACE_AUX_ENABLE, KVM_TRACE_AUX_MSA);
1721 		break;
1722 	}
1723 
1724 	preempt_enable();
1725 }
1726 #endif
1727 
1728 /* Drop FPU & MSA without saving it */
1729 void kvm_drop_fpu(struct kvm_vcpu *vcpu)
1730 {
1731 	preempt_disable();
1732 	if (cpu_has_msa && vcpu->arch.aux_inuse & KVM_MIPS_AUX_MSA) {
1733 		disable_msa();
1734 		trace_kvm_aux(vcpu, KVM_TRACE_AUX_DISCARD, KVM_TRACE_AUX_MSA);
1735 		vcpu->arch.aux_inuse &= ~KVM_MIPS_AUX_MSA;
1736 	}
1737 	if (vcpu->arch.aux_inuse & KVM_MIPS_AUX_FPU) {
1738 		clear_c0_status(ST0_CU1 | ST0_FR);
1739 		trace_kvm_aux(vcpu, KVM_TRACE_AUX_DISCARD, KVM_TRACE_AUX_FPU);
1740 		vcpu->arch.aux_inuse &= ~KVM_MIPS_AUX_FPU;
1741 	}
1742 	preempt_enable();
1743 }
1744 
1745 /* Save and disable FPU & MSA */
1746 void kvm_lose_fpu(struct kvm_vcpu *vcpu)
1747 {
1748 	/*
1749 	 * FPU & MSA get disabled in root context (hardware) when it is disabled
1750 	 * in guest context (software), but the register state in the hardware
1751 	 * may still be in use. This is why we explicitly re-enable the hardware
1752 	 * before saving.
1753 	 */
1754 
1755 	preempt_disable();
1756 	if (cpu_has_msa && vcpu->arch.aux_inuse & KVM_MIPS_AUX_MSA) {
1757 		set_c0_config5(MIPS_CONF5_MSAEN);
1758 		enable_fpu_hazard();
1759 
1760 		__kvm_save_msa(&vcpu->arch);
1761 		trace_kvm_aux(vcpu, KVM_TRACE_AUX_SAVE, KVM_TRACE_AUX_FPU_MSA);
1762 
1763 		/* Disable MSA & FPU */
1764 		disable_msa();
1765 		if (vcpu->arch.aux_inuse & KVM_MIPS_AUX_FPU) {
1766 			clear_c0_status(ST0_CU1 | ST0_FR);
1767 			disable_fpu_hazard();
1768 		}
1769 		vcpu->arch.aux_inuse &= ~(KVM_MIPS_AUX_FPU | KVM_MIPS_AUX_MSA);
1770 	} else if (vcpu->arch.aux_inuse & KVM_MIPS_AUX_FPU) {
1771 		set_c0_status(ST0_CU1);
1772 		enable_fpu_hazard();
1773 
1774 		__kvm_save_fpu(&vcpu->arch);
1775 		vcpu->arch.aux_inuse &= ~KVM_MIPS_AUX_FPU;
1776 		trace_kvm_aux(vcpu, KVM_TRACE_AUX_SAVE, KVM_TRACE_AUX_FPU);
1777 
1778 		/* Disable FPU */
1779 		clear_c0_status(ST0_CU1 | ST0_FR);
1780 		disable_fpu_hazard();
1781 	}
1782 	preempt_enable();
1783 }
1784 
1785 /*
1786  * Step over a specific ctc1 to FCSR and a specific ctcmsa to MSACSR which are
1787  * used to restore guest FCSR/MSACSR state and may trigger a "harmless" FP/MSAFP
1788  * exception if cause bits are set in the value being written.
1789  */
1790 static int kvm_mips_csr_die_notify(struct notifier_block *self,
1791 				   unsigned long cmd, void *ptr)
1792 {
1793 	struct die_args *args = (struct die_args *)ptr;
1794 	struct pt_regs *regs = args->regs;
1795 	unsigned long pc;
1796 
1797 	/* Only interested in FPE and MSAFPE */
1798 	if (cmd != DIE_FP && cmd != DIE_MSAFP)
1799 		return NOTIFY_DONE;
1800 
1801 	/* Return immediately if guest context isn't active */
1802 	if (!(current->flags & PF_VCPU))
1803 		return NOTIFY_DONE;
1804 
1805 	/* Should never get here from user mode */
1806 	BUG_ON(user_mode(regs));
1807 
1808 	pc = instruction_pointer(regs);
1809 	switch (cmd) {
1810 	case DIE_FP:
1811 		/* match 2nd instruction in __kvm_restore_fcsr */
1812 		if (pc != (unsigned long)&__kvm_restore_fcsr + 4)
1813 			return NOTIFY_DONE;
1814 		break;
1815 	case DIE_MSAFP:
1816 		/* match 2nd/3rd instruction in __kvm_restore_msacsr */
1817 		if (!cpu_has_msa ||
1818 		    pc < (unsigned long)&__kvm_restore_msacsr + 4 ||
1819 		    pc > (unsigned long)&__kvm_restore_msacsr + 8)
1820 			return NOTIFY_DONE;
1821 		break;
1822 	}
1823 
1824 	/* Move PC forward a little and continue executing */
1825 	instruction_pointer(regs) += 4;
1826 
1827 	return NOTIFY_STOP;
1828 }
1829 
1830 static struct notifier_block kvm_mips_csr_die_notifier = {
1831 	.notifier_call = kvm_mips_csr_die_notify,
1832 };
1833 
1834 static int __init kvm_mips_init(void)
1835 {
1836 	int ret;
1837 
1838 	ret = kvm_mips_entry_setup();
1839 	if (ret)
1840 		return ret;
1841 
1842 	ret = kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
1843 
1844 	if (ret)
1845 		return ret;
1846 
1847 	register_die_notifier(&kvm_mips_csr_die_notifier);
1848 
1849 	return 0;
1850 }
1851 
1852 static void __exit kvm_mips_exit(void)
1853 {
1854 	kvm_exit();
1855 
1856 	unregister_die_notifier(&kvm_mips_csr_die_notifier);
1857 }
1858 
1859 module_init(kvm_mips_init);
1860 module_exit(kvm_mips_exit);
1861 
1862 EXPORT_TRACEPOINT_SYMBOL(kvm_exit);
1863