1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright (c) 2017-2019, IBM Corporation.
4  */
5 
6 #define pr_fmt(fmt) "xive-kvm: " fmt
7 
8 #include <linux/kernel.h>
9 #include <linux/kvm_host.h>
10 #include <linux/err.h>
11 #include <linux/gfp.h>
12 #include <linux/spinlock.h>
13 #include <linux/delay.h>
14 #include <linux/file.h>
15 #include <linux/irqdomain.h>
16 #include <asm/uaccess.h>
17 #include <asm/kvm_book3s.h>
18 #include <asm/kvm_ppc.h>
19 #include <asm/hvcall.h>
20 #include <asm/xive.h>
21 #include <asm/xive-regs.h>
22 #include <asm/debug.h>
23 #include <asm/opal.h>
24 
25 #include <linux/debugfs.h>
26 #include <linux/seq_file.h>
27 
28 #include "book3s_xive.h"
29 
30 static u8 xive_vm_esb_load(struct xive_irq_data *xd, u32 offset)
31 {
32 	u64 val;
33 
34 	/*
35 	 * The KVM XIVE native device does not use the XIVE_ESB_SET_PQ_10
36 	 * load operation, so there is no need to enforce load-after-store
37 	 * ordering.
38 	 */
39 
40 	val = in_be64(xd->eoi_mmio + offset);
41 	return (u8)val;
42 }
43 
44 static void kvmppc_xive_native_cleanup_queue(struct kvm_vcpu *vcpu, int prio)
45 {
46 	struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
47 	struct xive_q *q = &xc->queues[prio];
48 
49 	xive_native_disable_queue(xc->vp_id, q, prio);
50 	if (q->qpage) {
51 		put_page(virt_to_page(q->qpage));
52 		q->qpage = NULL;
53 	}
54 }
55 
56 static int kvmppc_xive_native_configure_queue(u32 vp_id, struct xive_q *q,
57 					      u8 prio, __be32 *qpage,
58 					      u32 order, bool can_escalate)
59 {
60 	int rc;
61 	__be32 *qpage_prev = q->qpage;
62 
63 	rc = xive_native_configure_queue(vp_id, q, prio, qpage, order,
64 					 can_escalate);
65 	if (rc)
66 		return rc;
67 
68 	if (qpage_prev)
69 		put_page(virt_to_page(qpage_prev));
70 
71 	return rc;
72 }
73 
74 void kvmppc_xive_native_cleanup_vcpu(struct kvm_vcpu *vcpu)
75 {
76 	struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
77 	int i;
78 
79 	if (!kvmppc_xive_enabled(vcpu))
80 		return;
81 
82 	if (!xc)
83 		return;
84 
85 	pr_devel("native_cleanup_vcpu(cpu=%d)\n", xc->server_num);
86 
87 	/* Ensure no interrupt is still routed to that VP */
88 	xc->valid = false;
89 	kvmppc_xive_disable_vcpu_interrupts(vcpu);
90 
91 	/* Free escalations */
92 	for (i = 0; i < KVMPPC_XIVE_Q_COUNT; i++) {
93 		/* Free the escalation irq */
94 		if (xc->esc_virq[i]) {
95 			if (kvmppc_xive_has_single_escalation(xc->xive))
96 				xive_cleanup_single_escalation(vcpu, xc,
97 							xc->esc_virq[i]);
98 			free_irq(xc->esc_virq[i], vcpu);
99 			irq_dispose_mapping(xc->esc_virq[i]);
100 			kfree(xc->esc_virq_names[i]);
101 			xc->esc_virq[i] = 0;
102 		}
103 	}
104 
105 	/* Disable the VP */
106 	xive_native_disable_vp(xc->vp_id);
107 
108 	/* Clear the cam word so guest entry won't try to push context */
109 	vcpu->arch.xive_cam_word = 0;
110 
111 	/* Free the queues */
112 	for (i = 0; i < KVMPPC_XIVE_Q_COUNT; i++) {
113 		kvmppc_xive_native_cleanup_queue(vcpu, i);
114 	}
115 
116 	/* Free the VP */
117 	kfree(xc);
118 
119 	/* Cleanup the vcpu */
120 	vcpu->arch.irq_type = KVMPPC_IRQ_DEFAULT;
121 	vcpu->arch.xive_vcpu = NULL;
122 }
123 
124 int kvmppc_xive_native_connect_vcpu(struct kvm_device *dev,
125 				    struct kvm_vcpu *vcpu, u32 server_num)
126 {
127 	struct kvmppc_xive *xive = dev->private;
128 	struct kvmppc_xive_vcpu *xc = NULL;
129 	int rc;
130 	u32 vp_id;
131 
132 	pr_devel("native_connect_vcpu(server=%d)\n", server_num);
133 
134 	if (dev->ops != &kvm_xive_native_ops) {
135 		pr_devel("Wrong ops !\n");
136 		return -EPERM;
137 	}
138 	if (xive->kvm != vcpu->kvm)
139 		return -EPERM;
140 	if (vcpu->arch.irq_type != KVMPPC_IRQ_DEFAULT)
141 		return -EBUSY;
142 
143 	mutex_lock(&xive->lock);
144 
145 	rc = kvmppc_xive_compute_vp_id(xive, server_num, &vp_id);
146 	if (rc)
147 		goto bail;
148 
149 	xc = kzalloc(sizeof(*xc), GFP_KERNEL);
150 	if (!xc) {
151 		rc = -ENOMEM;
152 		goto bail;
153 	}
154 
155 	vcpu->arch.xive_vcpu = xc;
156 	xc->xive = xive;
157 	xc->vcpu = vcpu;
158 	xc->server_num = server_num;
159 
160 	xc->vp_id = vp_id;
161 	xc->valid = true;
162 	vcpu->arch.irq_type = KVMPPC_IRQ_XIVE;
163 
164 	rc = xive_native_get_vp_info(xc->vp_id, &xc->vp_cam, &xc->vp_chip_id);
165 	if (rc) {
166 		pr_err("Failed to get VP info from OPAL: %d\n", rc);
167 		goto bail;
168 	}
169 
170 	if (!kvmppc_xive_check_save_restore(vcpu)) {
171 		pr_err("inconsistent save-restore setup for VCPU %d\n", server_num);
172 		rc = -EIO;
173 		goto bail;
174 	}
175 
176 	/*
177 	 * Enable the VP first as the single escalation mode will
178 	 * affect escalation interrupts numbering
179 	 */
180 	rc = xive_native_enable_vp(xc->vp_id, kvmppc_xive_has_single_escalation(xive));
181 	if (rc) {
182 		pr_err("Failed to enable VP in OPAL: %d\n", rc);
183 		goto bail;
184 	}
185 
186 	/* Configure VCPU fields for use by assembly push/pull */
187 	vcpu->arch.xive_saved_state.w01 = cpu_to_be64(0xff000000);
188 	vcpu->arch.xive_cam_word = cpu_to_be32(xc->vp_cam | TM_QW1W2_VO);
189 
190 	/* TODO: reset all queues to a clean state ? */
191 bail:
192 	mutex_unlock(&xive->lock);
193 	if (rc)
194 		kvmppc_xive_native_cleanup_vcpu(vcpu);
195 
196 	return rc;
197 }
198 
199 /*
200  * Device passthrough support
201  */
202 static int kvmppc_xive_native_reset_mapped(struct kvm *kvm, unsigned long irq)
203 {
204 	struct kvmppc_xive *xive = kvm->arch.xive;
205 	pgoff_t esb_pgoff = KVM_XIVE_ESB_PAGE_OFFSET + irq * 2;
206 
207 	if (irq >= KVMPPC_XIVE_NR_IRQS)
208 		return -EINVAL;
209 
210 	/*
211 	 * Clear the ESB pages of the IRQ number being mapped (or
212 	 * unmapped) into the guest and let the VM fault handler
213 	 * repopulate with the appropriate ESB pages (device or IC)
214 	 */
215 	pr_debug("clearing esb pages for girq 0x%lx\n", irq);
216 	mutex_lock(&xive->mapping_lock);
217 	if (xive->mapping)
218 		unmap_mapping_range(xive->mapping,
219 				    esb_pgoff << PAGE_SHIFT,
220 				    2ull << PAGE_SHIFT, 1);
221 	mutex_unlock(&xive->mapping_lock);
222 	return 0;
223 }
224 
225 static struct kvmppc_xive_ops kvmppc_xive_native_ops =  {
226 	.reset_mapped = kvmppc_xive_native_reset_mapped,
227 };
228 
229 static vm_fault_t xive_native_esb_fault(struct vm_fault *vmf)
230 {
231 	struct vm_area_struct *vma = vmf->vma;
232 	struct kvm_device *dev = vma->vm_file->private_data;
233 	struct kvmppc_xive *xive = dev->private;
234 	struct kvmppc_xive_src_block *sb;
235 	struct kvmppc_xive_irq_state *state;
236 	struct xive_irq_data *xd;
237 	u32 hw_num;
238 	u16 src;
239 	u64 page;
240 	unsigned long irq;
241 	u64 page_offset;
242 
243 	/*
244 	 * Linux/KVM uses a two pages ESB setting, one for trigger and
245 	 * one for EOI
246 	 */
247 	page_offset = vmf->pgoff - vma->vm_pgoff;
248 	irq = page_offset / 2;
249 
250 	sb = kvmppc_xive_find_source(xive, irq, &src);
251 	if (!sb) {
252 		pr_devel("%s: source %lx not found !\n", __func__, irq);
253 		return VM_FAULT_SIGBUS;
254 	}
255 
256 	state = &sb->irq_state[src];
257 
258 	/* Some sanity checking */
259 	if (!state->valid) {
260 		pr_devel("%s: source %lx invalid !\n", __func__, irq);
261 		return VM_FAULT_SIGBUS;
262 	}
263 
264 	kvmppc_xive_select_irq(state, &hw_num, &xd);
265 
266 	arch_spin_lock(&sb->lock);
267 
268 	/*
269 	 * first/even page is for trigger
270 	 * second/odd page is for EOI and management.
271 	 */
272 	page = page_offset % 2 ? xd->eoi_page : xd->trig_page;
273 	arch_spin_unlock(&sb->lock);
274 
275 	if (WARN_ON(!page)) {
276 		pr_err("%s: accessing invalid ESB page for source %lx !\n",
277 		       __func__, irq);
278 		return VM_FAULT_SIGBUS;
279 	}
280 
281 	vmf_insert_pfn(vma, vmf->address, page >> PAGE_SHIFT);
282 	return VM_FAULT_NOPAGE;
283 }
284 
285 static const struct vm_operations_struct xive_native_esb_vmops = {
286 	.fault = xive_native_esb_fault,
287 };
288 
289 static vm_fault_t xive_native_tima_fault(struct vm_fault *vmf)
290 {
291 	struct vm_area_struct *vma = vmf->vma;
292 
293 	switch (vmf->pgoff - vma->vm_pgoff) {
294 	case 0: /* HW - forbid access */
295 	case 1: /* HV - forbid access */
296 		return VM_FAULT_SIGBUS;
297 	case 2: /* OS */
298 		vmf_insert_pfn(vma, vmf->address, xive_tima_os >> PAGE_SHIFT);
299 		return VM_FAULT_NOPAGE;
300 	case 3: /* USER - TODO */
301 	default:
302 		return VM_FAULT_SIGBUS;
303 	}
304 }
305 
306 static const struct vm_operations_struct xive_native_tima_vmops = {
307 	.fault = xive_native_tima_fault,
308 };
309 
310 static int kvmppc_xive_native_mmap(struct kvm_device *dev,
311 				   struct vm_area_struct *vma)
312 {
313 	struct kvmppc_xive *xive = dev->private;
314 
315 	/* We only allow mappings at fixed offset for now */
316 	if (vma->vm_pgoff == KVM_XIVE_TIMA_PAGE_OFFSET) {
317 		if (vma_pages(vma) > 4)
318 			return -EINVAL;
319 		vma->vm_ops = &xive_native_tima_vmops;
320 	} else if (vma->vm_pgoff == KVM_XIVE_ESB_PAGE_OFFSET) {
321 		if (vma_pages(vma) > KVMPPC_XIVE_NR_IRQS * 2)
322 			return -EINVAL;
323 		vma->vm_ops = &xive_native_esb_vmops;
324 	} else {
325 		return -EINVAL;
326 	}
327 
328 	vma->vm_flags |= VM_IO | VM_PFNMAP;
329 	vma->vm_page_prot = pgprot_noncached_wc(vma->vm_page_prot);
330 
331 	/*
332 	 * Grab the KVM device file address_space to be able to clear
333 	 * the ESB pages mapping when a device is passed-through into
334 	 * the guest.
335 	 */
336 	xive->mapping = vma->vm_file->f_mapping;
337 	return 0;
338 }
339 
340 static int kvmppc_xive_native_set_source(struct kvmppc_xive *xive, long irq,
341 					 u64 addr)
342 {
343 	struct kvmppc_xive_src_block *sb;
344 	struct kvmppc_xive_irq_state *state;
345 	u64 __user *ubufp = (u64 __user *) addr;
346 	u64 val;
347 	u16 idx;
348 	int rc;
349 
350 	pr_devel("%s irq=0x%lx\n", __func__, irq);
351 
352 	if (irq < KVMPPC_XIVE_FIRST_IRQ || irq >= KVMPPC_XIVE_NR_IRQS)
353 		return -E2BIG;
354 
355 	sb = kvmppc_xive_find_source(xive, irq, &idx);
356 	if (!sb) {
357 		pr_debug("No source, creating source block...\n");
358 		sb = kvmppc_xive_create_src_block(xive, irq);
359 		if (!sb) {
360 			pr_err("Failed to create block...\n");
361 			return -ENOMEM;
362 		}
363 	}
364 	state = &sb->irq_state[idx];
365 
366 	if (get_user(val, ubufp)) {
367 		pr_err("fault getting user info !\n");
368 		return -EFAULT;
369 	}
370 
371 	arch_spin_lock(&sb->lock);
372 
373 	/*
374 	 * If the source doesn't already have an IPI, allocate
375 	 * one and get the corresponding data
376 	 */
377 	if (!state->ipi_number) {
378 		state->ipi_number = xive_native_alloc_irq();
379 		if (state->ipi_number == 0) {
380 			pr_err("Failed to allocate IRQ !\n");
381 			rc = -ENXIO;
382 			goto unlock;
383 		}
384 		xive_native_populate_irq_data(state->ipi_number,
385 					      &state->ipi_data);
386 		pr_debug("%s allocated hw_irq=0x%x for irq=0x%lx\n", __func__,
387 			 state->ipi_number, irq);
388 	}
389 
390 	/* Restore LSI state */
391 	if (val & KVM_XIVE_LEVEL_SENSITIVE) {
392 		state->lsi = true;
393 		if (val & KVM_XIVE_LEVEL_ASSERTED)
394 			state->asserted = true;
395 		pr_devel("  LSI ! Asserted=%d\n", state->asserted);
396 	}
397 
398 	/* Mask IRQ to start with */
399 	state->act_server = 0;
400 	state->act_priority = MASKED;
401 	xive_vm_esb_load(&state->ipi_data, XIVE_ESB_SET_PQ_01);
402 	xive_native_configure_irq(state->ipi_number, 0, MASKED, 0);
403 
404 	/* Increment the number of valid sources and mark this one valid */
405 	if (!state->valid)
406 		xive->src_count++;
407 	state->valid = true;
408 
409 	rc = 0;
410 
411 unlock:
412 	arch_spin_unlock(&sb->lock);
413 
414 	return rc;
415 }
416 
417 static int kvmppc_xive_native_update_source_config(struct kvmppc_xive *xive,
418 					struct kvmppc_xive_src_block *sb,
419 					struct kvmppc_xive_irq_state *state,
420 					u32 server, u8 priority, bool masked,
421 					u32 eisn)
422 {
423 	struct kvm *kvm = xive->kvm;
424 	u32 hw_num;
425 	int rc = 0;
426 
427 	arch_spin_lock(&sb->lock);
428 
429 	if (state->act_server == server && state->act_priority == priority &&
430 	    state->eisn == eisn)
431 		goto unlock;
432 
433 	pr_devel("new_act_prio=%d new_act_server=%d mask=%d act_server=%d act_prio=%d\n",
434 		 priority, server, masked, state->act_server,
435 		 state->act_priority);
436 
437 	kvmppc_xive_select_irq(state, &hw_num, NULL);
438 
439 	if (priority != MASKED && !masked) {
440 		rc = kvmppc_xive_select_target(kvm, &server, priority);
441 		if (rc)
442 			goto unlock;
443 
444 		state->act_priority = priority;
445 		state->act_server = server;
446 		state->eisn = eisn;
447 
448 		rc = xive_native_configure_irq(hw_num,
449 					       kvmppc_xive_vp(xive, server),
450 					       priority, eisn);
451 	} else {
452 		state->act_priority = MASKED;
453 		state->act_server = 0;
454 		state->eisn = 0;
455 
456 		rc = xive_native_configure_irq(hw_num, 0, MASKED, 0);
457 	}
458 
459 unlock:
460 	arch_spin_unlock(&sb->lock);
461 	return rc;
462 }
463 
464 static int kvmppc_xive_native_set_source_config(struct kvmppc_xive *xive,
465 						long irq, u64 addr)
466 {
467 	struct kvmppc_xive_src_block *sb;
468 	struct kvmppc_xive_irq_state *state;
469 	u64 __user *ubufp = (u64 __user *) addr;
470 	u16 src;
471 	u64 kvm_cfg;
472 	u32 server;
473 	u8 priority;
474 	bool masked;
475 	u32 eisn;
476 
477 	sb = kvmppc_xive_find_source(xive, irq, &src);
478 	if (!sb)
479 		return -ENOENT;
480 
481 	state = &sb->irq_state[src];
482 
483 	if (!state->valid)
484 		return -EINVAL;
485 
486 	if (get_user(kvm_cfg, ubufp))
487 		return -EFAULT;
488 
489 	pr_devel("%s irq=0x%lx cfg=%016llx\n", __func__, irq, kvm_cfg);
490 
491 	priority = (kvm_cfg & KVM_XIVE_SOURCE_PRIORITY_MASK) >>
492 		KVM_XIVE_SOURCE_PRIORITY_SHIFT;
493 	server = (kvm_cfg & KVM_XIVE_SOURCE_SERVER_MASK) >>
494 		KVM_XIVE_SOURCE_SERVER_SHIFT;
495 	masked = (kvm_cfg & KVM_XIVE_SOURCE_MASKED_MASK) >>
496 		KVM_XIVE_SOURCE_MASKED_SHIFT;
497 	eisn = (kvm_cfg & KVM_XIVE_SOURCE_EISN_MASK) >>
498 		KVM_XIVE_SOURCE_EISN_SHIFT;
499 
500 	if (priority != xive_prio_from_guest(priority)) {
501 		pr_err("invalid priority for queue %d for VCPU %d\n",
502 		       priority, server);
503 		return -EINVAL;
504 	}
505 
506 	return kvmppc_xive_native_update_source_config(xive, sb, state, server,
507 						       priority, masked, eisn);
508 }
509 
510 static int kvmppc_xive_native_sync_source(struct kvmppc_xive *xive,
511 					  long irq, u64 addr)
512 {
513 	struct kvmppc_xive_src_block *sb;
514 	struct kvmppc_xive_irq_state *state;
515 	struct xive_irq_data *xd;
516 	u32 hw_num;
517 	u16 src;
518 	int rc = 0;
519 
520 	pr_devel("%s irq=0x%lx", __func__, irq);
521 
522 	sb = kvmppc_xive_find_source(xive, irq, &src);
523 	if (!sb)
524 		return -ENOENT;
525 
526 	state = &sb->irq_state[src];
527 
528 	rc = -EINVAL;
529 
530 	arch_spin_lock(&sb->lock);
531 
532 	if (state->valid) {
533 		kvmppc_xive_select_irq(state, &hw_num, &xd);
534 		xive_native_sync_source(hw_num);
535 		rc = 0;
536 	}
537 
538 	arch_spin_unlock(&sb->lock);
539 	return rc;
540 }
541 
542 static int xive_native_validate_queue_size(u32 qshift)
543 {
544 	/*
545 	 * We only support 64K pages for the moment. This is also
546 	 * advertised in the DT property "ibm,xive-eq-sizes"
547 	 */
548 	switch (qshift) {
549 	case 0: /* EQ reset */
550 	case 16:
551 		return 0;
552 	case 12:
553 	case 21:
554 	case 24:
555 	default:
556 		return -EINVAL;
557 	}
558 }
559 
560 static int kvmppc_xive_native_set_queue_config(struct kvmppc_xive *xive,
561 					       long eq_idx, u64 addr)
562 {
563 	struct kvm *kvm = xive->kvm;
564 	struct kvm_vcpu *vcpu;
565 	struct kvmppc_xive_vcpu *xc;
566 	void __user *ubufp = (void __user *) addr;
567 	u32 server;
568 	u8 priority;
569 	struct kvm_ppc_xive_eq kvm_eq;
570 	int rc;
571 	__be32 *qaddr = 0;
572 	struct page *page;
573 	struct xive_q *q;
574 	gfn_t gfn;
575 	unsigned long page_size;
576 	int srcu_idx;
577 
578 	/*
579 	 * Demangle priority/server tuple from the EQ identifier
580 	 */
581 	priority = (eq_idx & KVM_XIVE_EQ_PRIORITY_MASK) >>
582 		KVM_XIVE_EQ_PRIORITY_SHIFT;
583 	server = (eq_idx & KVM_XIVE_EQ_SERVER_MASK) >>
584 		KVM_XIVE_EQ_SERVER_SHIFT;
585 
586 	if (copy_from_user(&kvm_eq, ubufp, sizeof(kvm_eq)))
587 		return -EFAULT;
588 
589 	vcpu = kvmppc_xive_find_server(kvm, server);
590 	if (!vcpu) {
591 		pr_err("Can't find server %d\n", server);
592 		return -ENOENT;
593 	}
594 	xc = vcpu->arch.xive_vcpu;
595 
596 	if (priority != xive_prio_from_guest(priority)) {
597 		pr_err("Trying to restore invalid queue %d for VCPU %d\n",
598 		       priority, server);
599 		return -EINVAL;
600 	}
601 	q = &xc->queues[priority];
602 
603 	pr_devel("%s VCPU %d priority %d fl:%x shift:%d addr:%llx g:%d idx:%d\n",
604 		 __func__, server, priority, kvm_eq.flags,
605 		 kvm_eq.qshift, kvm_eq.qaddr, kvm_eq.qtoggle, kvm_eq.qindex);
606 
607 	/* reset queue and disable queueing */
608 	if (!kvm_eq.qshift) {
609 		q->guest_qaddr  = 0;
610 		q->guest_qshift = 0;
611 
612 		rc = kvmppc_xive_native_configure_queue(xc->vp_id, q, priority,
613 							NULL, 0, true);
614 		if (rc) {
615 			pr_err("Failed to reset queue %d for VCPU %d: %d\n",
616 			       priority, xc->server_num, rc);
617 			return rc;
618 		}
619 
620 		return 0;
621 	}
622 
623 	/*
624 	 * sPAPR specifies a "Unconditional Notify (n) flag" for the
625 	 * H_INT_SET_QUEUE_CONFIG hcall which forces notification
626 	 * without using the coalescing mechanisms provided by the
627 	 * XIVE END ESBs. This is required on KVM as notification
628 	 * using the END ESBs is not supported.
629 	 */
630 	if (kvm_eq.flags != KVM_XIVE_EQ_ALWAYS_NOTIFY) {
631 		pr_err("invalid flags %d\n", kvm_eq.flags);
632 		return -EINVAL;
633 	}
634 
635 	rc = xive_native_validate_queue_size(kvm_eq.qshift);
636 	if (rc) {
637 		pr_err("invalid queue size %d\n", kvm_eq.qshift);
638 		return rc;
639 	}
640 
641 	if (kvm_eq.qaddr & ((1ull << kvm_eq.qshift) - 1)) {
642 		pr_err("queue page is not aligned %llx/%llx\n", kvm_eq.qaddr,
643 		       1ull << kvm_eq.qshift);
644 		return -EINVAL;
645 	}
646 
647 	srcu_idx = srcu_read_lock(&kvm->srcu);
648 	gfn = gpa_to_gfn(kvm_eq.qaddr);
649 
650 	page_size = kvm_host_page_size(vcpu, gfn);
651 	if (1ull << kvm_eq.qshift > page_size) {
652 		srcu_read_unlock(&kvm->srcu, srcu_idx);
653 		pr_warn("Incompatible host page size %lx!\n", page_size);
654 		return -EINVAL;
655 	}
656 
657 	page = gfn_to_page(kvm, gfn);
658 	if (is_error_page(page)) {
659 		srcu_read_unlock(&kvm->srcu, srcu_idx);
660 		pr_err("Couldn't get queue page %llx!\n", kvm_eq.qaddr);
661 		return -EINVAL;
662 	}
663 
664 	qaddr = page_to_virt(page) + (kvm_eq.qaddr & ~PAGE_MASK);
665 	srcu_read_unlock(&kvm->srcu, srcu_idx);
666 
667 	/*
668 	 * Backup the queue page guest address to the mark EQ page
669 	 * dirty for migration.
670 	 */
671 	q->guest_qaddr  = kvm_eq.qaddr;
672 	q->guest_qshift = kvm_eq.qshift;
673 
674 	 /*
675 	  * Unconditional Notification is forced by default at the
676 	  * OPAL level because the use of END ESBs is not supported by
677 	  * Linux.
678 	  */
679 	rc = kvmppc_xive_native_configure_queue(xc->vp_id, q, priority,
680 					(__be32 *) qaddr, kvm_eq.qshift, true);
681 	if (rc) {
682 		pr_err("Failed to configure queue %d for VCPU %d: %d\n",
683 		       priority, xc->server_num, rc);
684 		put_page(page);
685 		return rc;
686 	}
687 
688 	/*
689 	 * Only restore the queue state when needed. When doing the
690 	 * H_INT_SET_SOURCE_CONFIG hcall, it should not.
691 	 */
692 	if (kvm_eq.qtoggle != 1 || kvm_eq.qindex != 0) {
693 		rc = xive_native_set_queue_state(xc->vp_id, priority,
694 						 kvm_eq.qtoggle,
695 						 kvm_eq.qindex);
696 		if (rc)
697 			goto error;
698 	}
699 
700 	rc = kvmppc_xive_attach_escalation(vcpu, priority,
701 					   kvmppc_xive_has_single_escalation(xive));
702 error:
703 	if (rc)
704 		kvmppc_xive_native_cleanup_queue(vcpu, priority);
705 	return rc;
706 }
707 
708 static int kvmppc_xive_native_get_queue_config(struct kvmppc_xive *xive,
709 					       long eq_idx, u64 addr)
710 {
711 	struct kvm *kvm = xive->kvm;
712 	struct kvm_vcpu *vcpu;
713 	struct kvmppc_xive_vcpu *xc;
714 	struct xive_q *q;
715 	void __user *ubufp = (u64 __user *) addr;
716 	u32 server;
717 	u8 priority;
718 	struct kvm_ppc_xive_eq kvm_eq;
719 	u64 qaddr;
720 	u64 qshift;
721 	u64 qeoi_page;
722 	u32 escalate_irq;
723 	u64 qflags;
724 	int rc;
725 
726 	/*
727 	 * Demangle priority/server tuple from the EQ identifier
728 	 */
729 	priority = (eq_idx & KVM_XIVE_EQ_PRIORITY_MASK) >>
730 		KVM_XIVE_EQ_PRIORITY_SHIFT;
731 	server = (eq_idx & KVM_XIVE_EQ_SERVER_MASK) >>
732 		KVM_XIVE_EQ_SERVER_SHIFT;
733 
734 	vcpu = kvmppc_xive_find_server(kvm, server);
735 	if (!vcpu) {
736 		pr_err("Can't find server %d\n", server);
737 		return -ENOENT;
738 	}
739 	xc = vcpu->arch.xive_vcpu;
740 
741 	if (priority != xive_prio_from_guest(priority)) {
742 		pr_err("invalid priority for queue %d for VCPU %d\n",
743 		       priority, server);
744 		return -EINVAL;
745 	}
746 	q = &xc->queues[priority];
747 
748 	memset(&kvm_eq, 0, sizeof(kvm_eq));
749 
750 	if (!q->qpage)
751 		return 0;
752 
753 	rc = xive_native_get_queue_info(xc->vp_id, priority, &qaddr, &qshift,
754 					&qeoi_page, &escalate_irq, &qflags);
755 	if (rc)
756 		return rc;
757 
758 	kvm_eq.flags = 0;
759 	if (qflags & OPAL_XIVE_EQ_ALWAYS_NOTIFY)
760 		kvm_eq.flags |= KVM_XIVE_EQ_ALWAYS_NOTIFY;
761 
762 	kvm_eq.qshift = q->guest_qshift;
763 	kvm_eq.qaddr  = q->guest_qaddr;
764 
765 	rc = xive_native_get_queue_state(xc->vp_id, priority, &kvm_eq.qtoggle,
766 					 &kvm_eq.qindex);
767 	if (rc)
768 		return rc;
769 
770 	pr_devel("%s VCPU %d priority %d fl:%x shift:%d addr:%llx g:%d idx:%d\n",
771 		 __func__, server, priority, kvm_eq.flags,
772 		 kvm_eq.qshift, kvm_eq.qaddr, kvm_eq.qtoggle, kvm_eq.qindex);
773 
774 	if (copy_to_user(ubufp, &kvm_eq, sizeof(kvm_eq)))
775 		return -EFAULT;
776 
777 	return 0;
778 }
779 
780 static void kvmppc_xive_reset_sources(struct kvmppc_xive_src_block *sb)
781 {
782 	int i;
783 
784 	for (i = 0; i < KVMPPC_XICS_IRQ_PER_ICS; i++) {
785 		struct kvmppc_xive_irq_state *state = &sb->irq_state[i];
786 
787 		if (!state->valid)
788 			continue;
789 
790 		if (state->act_priority == MASKED)
791 			continue;
792 
793 		state->eisn = 0;
794 		state->act_server = 0;
795 		state->act_priority = MASKED;
796 		xive_vm_esb_load(&state->ipi_data, XIVE_ESB_SET_PQ_01);
797 		xive_native_configure_irq(state->ipi_number, 0, MASKED, 0);
798 		if (state->pt_number) {
799 			xive_vm_esb_load(state->pt_data, XIVE_ESB_SET_PQ_01);
800 			xive_native_configure_irq(state->pt_number,
801 						  0, MASKED, 0);
802 		}
803 	}
804 }
805 
806 static int kvmppc_xive_reset(struct kvmppc_xive *xive)
807 {
808 	struct kvm *kvm = xive->kvm;
809 	struct kvm_vcpu *vcpu;
810 	unsigned long i;
811 
812 	pr_devel("%s\n", __func__);
813 
814 	mutex_lock(&xive->lock);
815 
816 	kvm_for_each_vcpu(i, vcpu, kvm) {
817 		struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
818 		unsigned int prio;
819 
820 		if (!xc)
821 			continue;
822 
823 		kvmppc_xive_disable_vcpu_interrupts(vcpu);
824 
825 		for (prio = 0; prio < KVMPPC_XIVE_Q_COUNT; prio++) {
826 
827 			/* Single escalation, no queue 7 */
828 			if (prio == 7 && kvmppc_xive_has_single_escalation(xive))
829 				break;
830 
831 			if (xc->esc_virq[prio]) {
832 				free_irq(xc->esc_virq[prio], vcpu);
833 				irq_dispose_mapping(xc->esc_virq[prio]);
834 				kfree(xc->esc_virq_names[prio]);
835 				xc->esc_virq[prio] = 0;
836 			}
837 
838 			kvmppc_xive_native_cleanup_queue(vcpu, prio);
839 		}
840 	}
841 
842 	for (i = 0; i <= xive->max_sbid; i++) {
843 		struct kvmppc_xive_src_block *sb = xive->src_blocks[i];
844 
845 		if (sb) {
846 			arch_spin_lock(&sb->lock);
847 			kvmppc_xive_reset_sources(sb);
848 			arch_spin_unlock(&sb->lock);
849 		}
850 	}
851 
852 	mutex_unlock(&xive->lock);
853 
854 	return 0;
855 }
856 
857 static void kvmppc_xive_native_sync_sources(struct kvmppc_xive_src_block *sb)
858 {
859 	int j;
860 
861 	for (j = 0; j < KVMPPC_XICS_IRQ_PER_ICS; j++) {
862 		struct kvmppc_xive_irq_state *state = &sb->irq_state[j];
863 		struct xive_irq_data *xd;
864 		u32 hw_num;
865 
866 		if (!state->valid)
867 			continue;
868 
869 		/*
870 		 * The struct kvmppc_xive_irq_state reflects the state
871 		 * of the EAS configuration and not the state of the
872 		 * source. The source is masked setting the PQ bits to
873 		 * '-Q', which is what is being done before calling
874 		 * the KVM_DEV_XIVE_EQ_SYNC control.
875 		 *
876 		 * If a source EAS is configured, OPAL syncs the XIVE
877 		 * IC of the source and the XIVE IC of the previous
878 		 * target if any.
879 		 *
880 		 * So it should be fine ignoring MASKED sources as
881 		 * they have been synced already.
882 		 */
883 		if (state->act_priority == MASKED)
884 			continue;
885 
886 		kvmppc_xive_select_irq(state, &hw_num, &xd);
887 		xive_native_sync_source(hw_num);
888 		xive_native_sync_queue(hw_num);
889 	}
890 }
891 
892 static int kvmppc_xive_native_vcpu_eq_sync(struct kvm_vcpu *vcpu)
893 {
894 	struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
895 	unsigned int prio;
896 	int srcu_idx;
897 
898 	if (!xc)
899 		return -ENOENT;
900 
901 	for (prio = 0; prio < KVMPPC_XIVE_Q_COUNT; prio++) {
902 		struct xive_q *q = &xc->queues[prio];
903 
904 		if (!q->qpage)
905 			continue;
906 
907 		/* Mark EQ page dirty for migration */
908 		srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
909 		mark_page_dirty(vcpu->kvm, gpa_to_gfn(q->guest_qaddr));
910 		srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);
911 	}
912 	return 0;
913 }
914 
915 static int kvmppc_xive_native_eq_sync(struct kvmppc_xive *xive)
916 {
917 	struct kvm *kvm = xive->kvm;
918 	struct kvm_vcpu *vcpu;
919 	unsigned long i;
920 
921 	pr_devel("%s\n", __func__);
922 
923 	mutex_lock(&xive->lock);
924 	for (i = 0; i <= xive->max_sbid; i++) {
925 		struct kvmppc_xive_src_block *sb = xive->src_blocks[i];
926 
927 		if (sb) {
928 			arch_spin_lock(&sb->lock);
929 			kvmppc_xive_native_sync_sources(sb);
930 			arch_spin_unlock(&sb->lock);
931 		}
932 	}
933 
934 	kvm_for_each_vcpu(i, vcpu, kvm) {
935 		kvmppc_xive_native_vcpu_eq_sync(vcpu);
936 	}
937 	mutex_unlock(&xive->lock);
938 
939 	return 0;
940 }
941 
942 static int kvmppc_xive_native_set_attr(struct kvm_device *dev,
943 				       struct kvm_device_attr *attr)
944 {
945 	struct kvmppc_xive *xive = dev->private;
946 
947 	switch (attr->group) {
948 	case KVM_DEV_XIVE_GRP_CTRL:
949 		switch (attr->attr) {
950 		case KVM_DEV_XIVE_RESET:
951 			return kvmppc_xive_reset(xive);
952 		case KVM_DEV_XIVE_EQ_SYNC:
953 			return kvmppc_xive_native_eq_sync(xive);
954 		case KVM_DEV_XIVE_NR_SERVERS:
955 			return kvmppc_xive_set_nr_servers(xive, attr->addr);
956 		}
957 		break;
958 	case KVM_DEV_XIVE_GRP_SOURCE:
959 		return kvmppc_xive_native_set_source(xive, attr->attr,
960 						     attr->addr);
961 	case KVM_DEV_XIVE_GRP_SOURCE_CONFIG:
962 		return kvmppc_xive_native_set_source_config(xive, attr->attr,
963 							    attr->addr);
964 	case KVM_DEV_XIVE_GRP_EQ_CONFIG:
965 		return kvmppc_xive_native_set_queue_config(xive, attr->attr,
966 							   attr->addr);
967 	case KVM_DEV_XIVE_GRP_SOURCE_SYNC:
968 		return kvmppc_xive_native_sync_source(xive, attr->attr,
969 						      attr->addr);
970 	}
971 	return -ENXIO;
972 }
973 
974 static int kvmppc_xive_native_get_attr(struct kvm_device *dev,
975 				       struct kvm_device_attr *attr)
976 {
977 	struct kvmppc_xive *xive = dev->private;
978 
979 	switch (attr->group) {
980 	case KVM_DEV_XIVE_GRP_EQ_CONFIG:
981 		return kvmppc_xive_native_get_queue_config(xive, attr->attr,
982 							   attr->addr);
983 	}
984 	return -ENXIO;
985 }
986 
987 static int kvmppc_xive_native_has_attr(struct kvm_device *dev,
988 				       struct kvm_device_attr *attr)
989 {
990 	switch (attr->group) {
991 	case KVM_DEV_XIVE_GRP_CTRL:
992 		switch (attr->attr) {
993 		case KVM_DEV_XIVE_RESET:
994 		case KVM_DEV_XIVE_EQ_SYNC:
995 		case KVM_DEV_XIVE_NR_SERVERS:
996 			return 0;
997 		}
998 		break;
999 	case KVM_DEV_XIVE_GRP_SOURCE:
1000 	case KVM_DEV_XIVE_GRP_SOURCE_CONFIG:
1001 	case KVM_DEV_XIVE_GRP_SOURCE_SYNC:
1002 		if (attr->attr >= KVMPPC_XIVE_FIRST_IRQ &&
1003 		    attr->attr < KVMPPC_XIVE_NR_IRQS)
1004 			return 0;
1005 		break;
1006 	case KVM_DEV_XIVE_GRP_EQ_CONFIG:
1007 		return 0;
1008 	}
1009 	return -ENXIO;
1010 }
1011 
1012 /*
1013  * Called when device fd is closed.  kvm->lock is held.
1014  */
1015 static void kvmppc_xive_native_release(struct kvm_device *dev)
1016 {
1017 	struct kvmppc_xive *xive = dev->private;
1018 	struct kvm *kvm = xive->kvm;
1019 	struct kvm_vcpu *vcpu;
1020 	unsigned long i;
1021 
1022 	pr_devel("Releasing xive native device\n");
1023 
1024 	/*
1025 	 * Clear the KVM device file address_space which is used to
1026 	 * unmap the ESB pages when a device is passed-through.
1027 	 */
1028 	mutex_lock(&xive->mapping_lock);
1029 	xive->mapping = NULL;
1030 	mutex_unlock(&xive->mapping_lock);
1031 
1032 	/*
1033 	 * Since this is the device release function, we know that
1034 	 * userspace does not have any open fd or mmap referring to
1035 	 * the device.  Therefore there can not be any of the
1036 	 * device attribute set/get, mmap, or page fault functions
1037 	 * being executed concurrently, and similarly, the
1038 	 * connect_vcpu and set/clr_mapped functions also cannot
1039 	 * be being executed.
1040 	 */
1041 
1042 	debugfs_remove(xive->dentry);
1043 
1044 	/*
1045 	 * We should clean up the vCPU interrupt presenters first.
1046 	 */
1047 	kvm_for_each_vcpu(i, vcpu, kvm) {
1048 		/*
1049 		 * Take vcpu->mutex to ensure that no one_reg get/set ioctl
1050 		 * (i.e. kvmppc_xive_native_[gs]et_vp) can be being done.
1051 		 * Holding the vcpu->mutex also means that the vcpu cannot
1052 		 * be executing the KVM_RUN ioctl, and therefore it cannot
1053 		 * be executing the XIVE push or pull code or accessing
1054 		 * the XIVE MMIO regions.
1055 		 */
1056 		mutex_lock(&vcpu->mutex);
1057 		kvmppc_xive_native_cleanup_vcpu(vcpu);
1058 		mutex_unlock(&vcpu->mutex);
1059 	}
1060 
1061 	/*
1062 	 * Now that we have cleared vcpu->arch.xive_vcpu, vcpu->arch.irq_type
1063 	 * and vcpu->arch.xive_esc_[vr]addr on each vcpu, we are safe
1064 	 * against xive code getting called during vcpu execution or
1065 	 * set/get one_reg operations.
1066 	 */
1067 	kvm->arch.xive = NULL;
1068 
1069 	for (i = 0; i <= xive->max_sbid; i++) {
1070 		if (xive->src_blocks[i])
1071 			kvmppc_xive_free_sources(xive->src_blocks[i]);
1072 		kfree(xive->src_blocks[i]);
1073 		xive->src_blocks[i] = NULL;
1074 	}
1075 
1076 	if (xive->vp_base != XIVE_INVALID_VP)
1077 		xive_native_free_vp_block(xive->vp_base);
1078 
1079 	/*
1080 	 * A reference of the kvmppc_xive pointer is now kept under
1081 	 * the xive_devices struct of the machine for reuse. It is
1082 	 * freed when the VM is destroyed for now until we fix all the
1083 	 * execution paths.
1084 	 */
1085 
1086 	kfree(dev);
1087 }
1088 
1089 /*
1090  * Create a XIVE device.  kvm->lock is held.
1091  */
1092 static int kvmppc_xive_native_create(struct kvm_device *dev, u32 type)
1093 {
1094 	struct kvmppc_xive *xive;
1095 	struct kvm *kvm = dev->kvm;
1096 
1097 	pr_devel("Creating xive native device\n");
1098 
1099 	if (kvm->arch.xive)
1100 		return -EEXIST;
1101 
1102 	xive = kvmppc_xive_get_device(kvm, type);
1103 	if (!xive)
1104 		return -ENOMEM;
1105 
1106 	dev->private = xive;
1107 	xive->dev = dev;
1108 	xive->kvm = kvm;
1109 	mutex_init(&xive->mapping_lock);
1110 	mutex_init(&xive->lock);
1111 
1112 	/* VP allocation is delayed to the first call to connect_vcpu */
1113 	xive->vp_base = XIVE_INVALID_VP;
1114 	/* KVM_MAX_VCPUS limits the number of VMs to roughly 64 per sockets
1115 	 * on a POWER9 system.
1116 	 */
1117 	xive->nr_servers = KVM_MAX_VCPUS;
1118 
1119 	if (xive_native_has_single_escalation())
1120 		xive->flags |= KVMPPC_XIVE_FLAG_SINGLE_ESCALATION;
1121 
1122 	if (xive_native_has_save_restore())
1123 		xive->flags |= KVMPPC_XIVE_FLAG_SAVE_RESTORE;
1124 
1125 	xive->ops = &kvmppc_xive_native_ops;
1126 
1127 	kvm->arch.xive = xive;
1128 	return 0;
1129 }
1130 
1131 /*
1132  * Interrupt Pending Buffer (IPB) offset
1133  */
1134 #define TM_IPB_SHIFT 40
1135 #define TM_IPB_MASK  (((u64) 0xFF) << TM_IPB_SHIFT)
1136 
1137 int kvmppc_xive_native_get_vp(struct kvm_vcpu *vcpu, union kvmppc_one_reg *val)
1138 {
1139 	struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
1140 	u64 opal_state;
1141 	int rc;
1142 
1143 	if (!kvmppc_xive_enabled(vcpu))
1144 		return -EPERM;
1145 
1146 	if (!xc)
1147 		return -ENOENT;
1148 
1149 	/* Thread context registers. We only care about IPB and CPPR */
1150 	val->xive_timaval[0] = vcpu->arch.xive_saved_state.w01;
1151 
1152 	/* Get the VP state from OPAL */
1153 	rc = xive_native_get_vp_state(xc->vp_id, &opal_state);
1154 	if (rc)
1155 		return rc;
1156 
1157 	/*
1158 	 * Capture the backup of IPB register in the NVT structure and
1159 	 * merge it in our KVM VP state.
1160 	 */
1161 	val->xive_timaval[0] |= cpu_to_be64(opal_state & TM_IPB_MASK);
1162 
1163 	pr_devel("%s NSR=%02x CPPR=%02x IBP=%02x PIPR=%02x w01=%016llx w2=%08x opal=%016llx\n",
1164 		 __func__,
1165 		 vcpu->arch.xive_saved_state.nsr,
1166 		 vcpu->arch.xive_saved_state.cppr,
1167 		 vcpu->arch.xive_saved_state.ipb,
1168 		 vcpu->arch.xive_saved_state.pipr,
1169 		 vcpu->arch.xive_saved_state.w01,
1170 		 (u32) vcpu->arch.xive_cam_word, opal_state);
1171 
1172 	return 0;
1173 }
1174 
1175 int kvmppc_xive_native_set_vp(struct kvm_vcpu *vcpu, union kvmppc_one_reg *val)
1176 {
1177 	struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
1178 	struct kvmppc_xive *xive = vcpu->kvm->arch.xive;
1179 
1180 	pr_devel("%s w01=%016llx vp=%016llx\n", __func__,
1181 		 val->xive_timaval[0], val->xive_timaval[1]);
1182 
1183 	if (!kvmppc_xive_enabled(vcpu))
1184 		return -EPERM;
1185 
1186 	if (!xc || !xive)
1187 		return -ENOENT;
1188 
1189 	/* We can't update the state of a "pushed" VCPU	 */
1190 	if (WARN_ON(vcpu->arch.xive_pushed))
1191 		return -EBUSY;
1192 
1193 	/*
1194 	 * Restore the thread context registers. IPB and CPPR should
1195 	 * be the only ones that matter.
1196 	 */
1197 	vcpu->arch.xive_saved_state.w01 = val->xive_timaval[0];
1198 
1199 	/*
1200 	 * There is no need to restore the XIVE internal state (IPB
1201 	 * stored in the NVT) as the IPB register was merged in KVM VP
1202 	 * state when captured.
1203 	 */
1204 	return 0;
1205 }
1206 
1207 bool kvmppc_xive_native_supported(void)
1208 {
1209 	return xive_native_has_queue_state_support();
1210 }
1211 
1212 static int xive_native_debug_show(struct seq_file *m, void *private)
1213 {
1214 	struct kvmppc_xive *xive = m->private;
1215 	struct kvm *kvm = xive->kvm;
1216 	struct kvm_vcpu *vcpu;
1217 	unsigned long i;
1218 
1219 	if (!kvm)
1220 		return 0;
1221 
1222 	seq_puts(m, "=========\nVCPU state\n=========\n");
1223 
1224 	kvm_for_each_vcpu(i, vcpu, kvm) {
1225 		struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
1226 
1227 		if (!xc)
1228 			continue;
1229 
1230 		seq_printf(m, "VCPU %d: VP=%#x/%02x\n"
1231 			   "    NSR=%02x CPPR=%02x IBP=%02x PIPR=%02x w01=%016llx w2=%08x\n",
1232 			   xc->server_num, xc->vp_id, xc->vp_chip_id,
1233 			   vcpu->arch.xive_saved_state.nsr,
1234 			   vcpu->arch.xive_saved_state.cppr,
1235 			   vcpu->arch.xive_saved_state.ipb,
1236 			   vcpu->arch.xive_saved_state.pipr,
1237 			   be64_to_cpu(vcpu->arch.xive_saved_state.w01),
1238 			   be32_to_cpu(vcpu->arch.xive_cam_word));
1239 
1240 		kvmppc_xive_debug_show_queues(m, vcpu);
1241 	}
1242 
1243 	seq_puts(m, "=========\nSources\n=========\n");
1244 
1245 	for (i = 0; i <= xive->max_sbid; i++) {
1246 		struct kvmppc_xive_src_block *sb = xive->src_blocks[i];
1247 
1248 		if (sb) {
1249 			arch_spin_lock(&sb->lock);
1250 			kvmppc_xive_debug_show_sources(m, sb);
1251 			arch_spin_unlock(&sb->lock);
1252 		}
1253 	}
1254 
1255 	return 0;
1256 }
1257 
1258 DEFINE_SHOW_ATTRIBUTE(xive_native_debug);
1259 
1260 static void xive_native_debugfs_init(struct kvmppc_xive *xive)
1261 {
1262 	xive->dentry = debugfs_create_file("xive", 0444, xive->kvm->debugfs_dentry,
1263 					   xive, &xive_native_debug_fops);
1264 
1265 	pr_debug("%s: created\n", __func__);
1266 }
1267 
1268 static void kvmppc_xive_native_init(struct kvm_device *dev)
1269 {
1270 	struct kvmppc_xive *xive = dev->private;
1271 
1272 	/* Register some debug interfaces */
1273 	xive_native_debugfs_init(xive);
1274 }
1275 
1276 struct kvm_device_ops kvm_xive_native_ops = {
1277 	.name = "kvm-xive-native",
1278 	.create = kvmppc_xive_native_create,
1279 	.init = kvmppc_xive_native_init,
1280 	.release = kvmppc_xive_native_release,
1281 	.set_attr = kvmppc_xive_native_set_attr,
1282 	.get_attr = kvmppc_xive_native_get_attr,
1283 	.has_attr = kvmppc_xive_native_has_attr,
1284 	.mmap = kvmppc_xive_native_mmap,
1285 };
1286