1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright (c) 2017-2019, IBM Corporation.
4  */
5 
6 #define pr_fmt(fmt) "xive-kvm: " fmt
7 
8 #include <linux/kernel.h>
9 #include <linux/kvm_host.h>
10 #include <linux/err.h>
11 #include <linux/gfp.h>
12 #include <linux/spinlock.h>
13 #include <linux/delay.h>
14 #include <linux/file.h>
15 #include <asm/uaccess.h>
16 #include <asm/kvm_book3s.h>
17 #include <asm/kvm_ppc.h>
18 #include <asm/hvcall.h>
19 #include <asm/xive.h>
20 #include <asm/xive-regs.h>
21 #include <asm/debug.h>
22 #include <asm/debugfs.h>
23 #include <asm/opal.h>
24 
25 #include <linux/debugfs.h>
26 #include <linux/seq_file.h>
27 
28 #include "book3s_xive.h"
29 
30 static u8 xive_vm_esb_load(struct xive_irq_data *xd, u32 offset)
31 {
32 	u64 val;
33 
34 	/*
35 	 * The KVM XIVE native device does not use the XIVE_ESB_SET_PQ_10
36 	 * load operation, so there is no need to enforce load-after-store
37 	 * ordering.
38 	 */
39 
40 	if (xd->flags & XIVE_IRQ_FLAG_SHIFT_BUG)
41 		offset |= offset << 4;
42 
43 	val = in_be64(xd->eoi_mmio + offset);
44 	return (u8)val;
45 }
46 
47 static void kvmppc_xive_native_cleanup_queue(struct kvm_vcpu *vcpu, int prio)
48 {
49 	struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
50 	struct xive_q *q = &xc->queues[prio];
51 
52 	xive_native_disable_queue(xc->vp_id, q, prio);
53 	if (q->qpage) {
54 		put_page(virt_to_page(q->qpage));
55 		q->qpage = NULL;
56 	}
57 }
58 
59 static int kvmppc_xive_native_configure_queue(u32 vp_id, struct xive_q *q,
60 					      u8 prio, __be32 *qpage,
61 					      u32 order, bool can_escalate)
62 {
63 	int rc;
64 	__be32 *qpage_prev = q->qpage;
65 
66 	rc = xive_native_configure_queue(vp_id, q, prio, qpage, order,
67 					 can_escalate);
68 	if (rc)
69 		return rc;
70 
71 	if (qpage_prev)
72 		put_page(virt_to_page(qpage_prev));
73 
74 	return rc;
75 }
76 
77 void kvmppc_xive_native_cleanup_vcpu(struct kvm_vcpu *vcpu)
78 {
79 	struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
80 	int i;
81 
82 	if (!kvmppc_xive_enabled(vcpu))
83 		return;
84 
85 	if (!xc)
86 		return;
87 
88 	pr_devel("native_cleanup_vcpu(cpu=%d)\n", xc->server_num);
89 
90 	/* Ensure no interrupt is still routed to that VP */
91 	xc->valid = false;
92 	kvmppc_xive_disable_vcpu_interrupts(vcpu);
93 
94 	/* Free escalations */
95 	for (i = 0; i < KVMPPC_XIVE_Q_COUNT; i++) {
96 		/* Free the escalation irq */
97 		if (xc->esc_virq[i]) {
98 			if (xc->xive->single_escalation)
99 				xive_cleanup_single_escalation(vcpu, xc,
100 							xc->esc_virq[i]);
101 			free_irq(xc->esc_virq[i], vcpu);
102 			irq_dispose_mapping(xc->esc_virq[i]);
103 			kfree(xc->esc_virq_names[i]);
104 			xc->esc_virq[i] = 0;
105 		}
106 	}
107 
108 	/* Disable the VP */
109 	xive_native_disable_vp(xc->vp_id);
110 
111 	/* Clear the cam word so guest entry won't try to push context */
112 	vcpu->arch.xive_cam_word = 0;
113 
114 	/* Free the queues */
115 	for (i = 0; i < KVMPPC_XIVE_Q_COUNT; i++) {
116 		kvmppc_xive_native_cleanup_queue(vcpu, i);
117 	}
118 
119 	/* Free the VP */
120 	kfree(xc);
121 
122 	/* Cleanup the vcpu */
123 	vcpu->arch.irq_type = KVMPPC_IRQ_DEFAULT;
124 	vcpu->arch.xive_vcpu = NULL;
125 }
126 
127 int kvmppc_xive_native_connect_vcpu(struct kvm_device *dev,
128 				    struct kvm_vcpu *vcpu, u32 server_num)
129 {
130 	struct kvmppc_xive *xive = dev->private;
131 	struct kvmppc_xive_vcpu *xc = NULL;
132 	int rc;
133 	u32 vp_id;
134 
135 	pr_devel("native_connect_vcpu(server=%d)\n", server_num);
136 
137 	if (dev->ops != &kvm_xive_native_ops) {
138 		pr_devel("Wrong ops !\n");
139 		return -EPERM;
140 	}
141 	if (xive->kvm != vcpu->kvm)
142 		return -EPERM;
143 	if (vcpu->arch.irq_type != KVMPPC_IRQ_DEFAULT)
144 		return -EBUSY;
145 
146 	mutex_lock(&xive->lock);
147 
148 	rc = kvmppc_xive_compute_vp_id(xive, server_num, &vp_id);
149 	if (rc)
150 		goto bail;
151 
152 	xc = kzalloc(sizeof(*xc), GFP_KERNEL);
153 	if (!xc) {
154 		rc = -ENOMEM;
155 		goto bail;
156 	}
157 
158 	vcpu->arch.xive_vcpu = xc;
159 	xc->xive = xive;
160 	xc->vcpu = vcpu;
161 	xc->server_num = server_num;
162 
163 	xc->vp_id = vp_id;
164 	xc->valid = true;
165 	vcpu->arch.irq_type = KVMPPC_IRQ_XIVE;
166 
167 	rc = xive_native_get_vp_info(xc->vp_id, &xc->vp_cam, &xc->vp_chip_id);
168 	if (rc) {
169 		pr_err("Failed to get VP info from OPAL: %d\n", rc);
170 		goto bail;
171 	}
172 
173 	/*
174 	 * Enable the VP first as the single escalation mode will
175 	 * affect escalation interrupts numbering
176 	 */
177 	rc = xive_native_enable_vp(xc->vp_id, xive->single_escalation);
178 	if (rc) {
179 		pr_err("Failed to enable VP in OPAL: %d\n", rc);
180 		goto bail;
181 	}
182 
183 	/* Configure VCPU fields for use by assembly push/pull */
184 	vcpu->arch.xive_saved_state.w01 = cpu_to_be64(0xff000000);
185 	vcpu->arch.xive_cam_word = cpu_to_be32(xc->vp_cam | TM_QW1W2_VO);
186 
187 	/* TODO: reset all queues to a clean state ? */
188 bail:
189 	mutex_unlock(&xive->lock);
190 	if (rc)
191 		kvmppc_xive_native_cleanup_vcpu(vcpu);
192 
193 	return rc;
194 }
195 
196 /*
197  * Device passthrough support
198  */
199 static int kvmppc_xive_native_reset_mapped(struct kvm *kvm, unsigned long irq)
200 {
201 	struct kvmppc_xive *xive = kvm->arch.xive;
202 	pgoff_t esb_pgoff = KVM_XIVE_ESB_PAGE_OFFSET + irq * 2;
203 
204 	if (irq >= KVMPPC_XIVE_NR_IRQS)
205 		return -EINVAL;
206 
207 	/*
208 	 * Clear the ESB pages of the IRQ number being mapped (or
209 	 * unmapped) into the guest and let the the VM fault handler
210 	 * repopulate with the appropriate ESB pages (device or IC)
211 	 */
212 	pr_debug("clearing esb pages for girq 0x%lx\n", irq);
213 	mutex_lock(&xive->mapping_lock);
214 	if (xive->mapping)
215 		unmap_mapping_range(xive->mapping,
216 				    esb_pgoff << PAGE_SHIFT,
217 				    2ull << PAGE_SHIFT, 1);
218 	mutex_unlock(&xive->mapping_lock);
219 	return 0;
220 }
221 
222 static struct kvmppc_xive_ops kvmppc_xive_native_ops =  {
223 	.reset_mapped = kvmppc_xive_native_reset_mapped,
224 };
225 
226 static vm_fault_t xive_native_esb_fault(struct vm_fault *vmf)
227 {
228 	struct vm_area_struct *vma = vmf->vma;
229 	struct kvm_device *dev = vma->vm_file->private_data;
230 	struct kvmppc_xive *xive = dev->private;
231 	struct kvmppc_xive_src_block *sb;
232 	struct kvmppc_xive_irq_state *state;
233 	struct xive_irq_data *xd;
234 	u32 hw_num;
235 	u16 src;
236 	u64 page;
237 	unsigned long irq;
238 	u64 page_offset;
239 
240 	/*
241 	 * Linux/KVM uses a two pages ESB setting, one for trigger and
242 	 * one for EOI
243 	 */
244 	page_offset = vmf->pgoff - vma->vm_pgoff;
245 	irq = page_offset / 2;
246 
247 	sb = kvmppc_xive_find_source(xive, irq, &src);
248 	if (!sb) {
249 		pr_devel("%s: source %lx not found !\n", __func__, irq);
250 		return VM_FAULT_SIGBUS;
251 	}
252 
253 	state = &sb->irq_state[src];
254 	kvmppc_xive_select_irq(state, &hw_num, &xd);
255 
256 	arch_spin_lock(&sb->lock);
257 
258 	/*
259 	 * first/even page is for trigger
260 	 * second/odd page is for EOI and management.
261 	 */
262 	page = page_offset % 2 ? xd->eoi_page : xd->trig_page;
263 	arch_spin_unlock(&sb->lock);
264 
265 	if (WARN_ON(!page)) {
266 		pr_err("%s: accessing invalid ESB page for source %lx !\n",
267 		       __func__, irq);
268 		return VM_FAULT_SIGBUS;
269 	}
270 
271 	vmf_insert_pfn(vma, vmf->address, page >> PAGE_SHIFT);
272 	return VM_FAULT_NOPAGE;
273 }
274 
275 static const struct vm_operations_struct xive_native_esb_vmops = {
276 	.fault = xive_native_esb_fault,
277 };
278 
279 static vm_fault_t xive_native_tima_fault(struct vm_fault *vmf)
280 {
281 	struct vm_area_struct *vma = vmf->vma;
282 
283 	switch (vmf->pgoff - vma->vm_pgoff) {
284 	case 0: /* HW - forbid access */
285 	case 1: /* HV - forbid access */
286 		return VM_FAULT_SIGBUS;
287 	case 2: /* OS */
288 		vmf_insert_pfn(vma, vmf->address, xive_tima_os >> PAGE_SHIFT);
289 		return VM_FAULT_NOPAGE;
290 	case 3: /* USER - TODO */
291 	default:
292 		return VM_FAULT_SIGBUS;
293 	}
294 }
295 
296 static const struct vm_operations_struct xive_native_tima_vmops = {
297 	.fault = xive_native_tima_fault,
298 };
299 
300 static int kvmppc_xive_native_mmap(struct kvm_device *dev,
301 				   struct vm_area_struct *vma)
302 {
303 	struct kvmppc_xive *xive = dev->private;
304 
305 	/* We only allow mappings at fixed offset for now */
306 	if (vma->vm_pgoff == KVM_XIVE_TIMA_PAGE_OFFSET) {
307 		if (vma_pages(vma) > 4)
308 			return -EINVAL;
309 		vma->vm_ops = &xive_native_tima_vmops;
310 	} else if (vma->vm_pgoff == KVM_XIVE_ESB_PAGE_OFFSET) {
311 		if (vma_pages(vma) > KVMPPC_XIVE_NR_IRQS * 2)
312 			return -EINVAL;
313 		vma->vm_ops = &xive_native_esb_vmops;
314 	} else {
315 		return -EINVAL;
316 	}
317 
318 	vma->vm_flags |= VM_IO | VM_PFNMAP;
319 	vma->vm_page_prot = pgprot_noncached_wc(vma->vm_page_prot);
320 
321 	/*
322 	 * Grab the KVM device file address_space to be able to clear
323 	 * the ESB pages mapping when a device is passed-through into
324 	 * the guest.
325 	 */
326 	xive->mapping = vma->vm_file->f_mapping;
327 	return 0;
328 }
329 
330 static int kvmppc_xive_native_set_source(struct kvmppc_xive *xive, long irq,
331 					 u64 addr)
332 {
333 	struct kvmppc_xive_src_block *sb;
334 	struct kvmppc_xive_irq_state *state;
335 	u64 __user *ubufp = (u64 __user *) addr;
336 	u64 val;
337 	u16 idx;
338 	int rc;
339 
340 	pr_devel("%s irq=0x%lx\n", __func__, irq);
341 
342 	if (irq < KVMPPC_XIVE_FIRST_IRQ || irq >= KVMPPC_XIVE_NR_IRQS)
343 		return -E2BIG;
344 
345 	sb = kvmppc_xive_find_source(xive, irq, &idx);
346 	if (!sb) {
347 		pr_debug("No source, creating source block...\n");
348 		sb = kvmppc_xive_create_src_block(xive, irq);
349 		if (!sb) {
350 			pr_err("Failed to create block...\n");
351 			return -ENOMEM;
352 		}
353 	}
354 	state = &sb->irq_state[idx];
355 
356 	if (get_user(val, ubufp)) {
357 		pr_err("fault getting user info !\n");
358 		return -EFAULT;
359 	}
360 
361 	arch_spin_lock(&sb->lock);
362 
363 	/*
364 	 * If the source doesn't already have an IPI, allocate
365 	 * one and get the corresponding data
366 	 */
367 	if (!state->ipi_number) {
368 		state->ipi_number = xive_native_alloc_irq();
369 		if (state->ipi_number == 0) {
370 			pr_err("Failed to allocate IRQ !\n");
371 			rc = -ENXIO;
372 			goto unlock;
373 		}
374 		xive_native_populate_irq_data(state->ipi_number,
375 					      &state->ipi_data);
376 		pr_debug("%s allocated hw_irq=0x%x for irq=0x%lx\n", __func__,
377 			 state->ipi_number, irq);
378 	}
379 
380 	/* Restore LSI state */
381 	if (val & KVM_XIVE_LEVEL_SENSITIVE) {
382 		state->lsi = true;
383 		if (val & KVM_XIVE_LEVEL_ASSERTED)
384 			state->asserted = true;
385 		pr_devel("  LSI ! Asserted=%d\n", state->asserted);
386 	}
387 
388 	/* Mask IRQ to start with */
389 	state->act_server = 0;
390 	state->act_priority = MASKED;
391 	xive_vm_esb_load(&state->ipi_data, XIVE_ESB_SET_PQ_01);
392 	xive_native_configure_irq(state->ipi_number, 0, MASKED, 0);
393 
394 	/* Increment the number of valid sources and mark this one valid */
395 	if (!state->valid)
396 		xive->src_count++;
397 	state->valid = true;
398 
399 	rc = 0;
400 
401 unlock:
402 	arch_spin_unlock(&sb->lock);
403 
404 	return rc;
405 }
406 
407 static int kvmppc_xive_native_update_source_config(struct kvmppc_xive *xive,
408 					struct kvmppc_xive_src_block *sb,
409 					struct kvmppc_xive_irq_state *state,
410 					u32 server, u8 priority, bool masked,
411 					u32 eisn)
412 {
413 	struct kvm *kvm = xive->kvm;
414 	u32 hw_num;
415 	int rc = 0;
416 
417 	arch_spin_lock(&sb->lock);
418 
419 	if (state->act_server == server && state->act_priority == priority &&
420 	    state->eisn == eisn)
421 		goto unlock;
422 
423 	pr_devel("new_act_prio=%d new_act_server=%d mask=%d act_server=%d act_prio=%d\n",
424 		 priority, server, masked, state->act_server,
425 		 state->act_priority);
426 
427 	kvmppc_xive_select_irq(state, &hw_num, NULL);
428 
429 	if (priority != MASKED && !masked) {
430 		rc = kvmppc_xive_select_target(kvm, &server, priority);
431 		if (rc)
432 			goto unlock;
433 
434 		state->act_priority = priority;
435 		state->act_server = server;
436 		state->eisn = eisn;
437 
438 		rc = xive_native_configure_irq(hw_num,
439 					       kvmppc_xive_vp(xive, server),
440 					       priority, eisn);
441 	} else {
442 		state->act_priority = MASKED;
443 		state->act_server = 0;
444 		state->eisn = 0;
445 
446 		rc = xive_native_configure_irq(hw_num, 0, MASKED, 0);
447 	}
448 
449 unlock:
450 	arch_spin_unlock(&sb->lock);
451 	return rc;
452 }
453 
454 static int kvmppc_xive_native_set_source_config(struct kvmppc_xive *xive,
455 						long irq, u64 addr)
456 {
457 	struct kvmppc_xive_src_block *sb;
458 	struct kvmppc_xive_irq_state *state;
459 	u64 __user *ubufp = (u64 __user *) addr;
460 	u16 src;
461 	u64 kvm_cfg;
462 	u32 server;
463 	u8 priority;
464 	bool masked;
465 	u32 eisn;
466 
467 	sb = kvmppc_xive_find_source(xive, irq, &src);
468 	if (!sb)
469 		return -ENOENT;
470 
471 	state = &sb->irq_state[src];
472 
473 	if (!state->valid)
474 		return -EINVAL;
475 
476 	if (get_user(kvm_cfg, ubufp))
477 		return -EFAULT;
478 
479 	pr_devel("%s irq=0x%lx cfg=%016llx\n", __func__, irq, kvm_cfg);
480 
481 	priority = (kvm_cfg & KVM_XIVE_SOURCE_PRIORITY_MASK) >>
482 		KVM_XIVE_SOURCE_PRIORITY_SHIFT;
483 	server = (kvm_cfg & KVM_XIVE_SOURCE_SERVER_MASK) >>
484 		KVM_XIVE_SOURCE_SERVER_SHIFT;
485 	masked = (kvm_cfg & KVM_XIVE_SOURCE_MASKED_MASK) >>
486 		KVM_XIVE_SOURCE_MASKED_SHIFT;
487 	eisn = (kvm_cfg & KVM_XIVE_SOURCE_EISN_MASK) >>
488 		KVM_XIVE_SOURCE_EISN_SHIFT;
489 
490 	if (priority != xive_prio_from_guest(priority)) {
491 		pr_err("invalid priority for queue %d for VCPU %d\n",
492 		       priority, server);
493 		return -EINVAL;
494 	}
495 
496 	return kvmppc_xive_native_update_source_config(xive, sb, state, server,
497 						       priority, masked, eisn);
498 }
499 
500 static int kvmppc_xive_native_sync_source(struct kvmppc_xive *xive,
501 					  long irq, u64 addr)
502 {
503 	struct kvmppc_xive_src_block *sb;
504 	struct kvmppc_xive_irq_state *state;
505 	struct xive_irq_data *xd;
506 	u32 hw_num;
507 	u16 src;
508 	int rc = 0;
509 
510 	pr_devel("%s irq=0x%lx", __func__, irq);
511 
512 	sb = kvmppc_xive_find_source(xive, irq, &src);
513 	if (!sb)
514 		return -ENOENT;
515 
516 	state = &sb->irq_state[src];
517 
518 	rc = -EINVAL;
519 
520 	arch_spin_lock(&sb->lock);
521 
522 	if (state->valid) {
523 		kvmppc_xive_select_irq(state, &hw_num, &xd);
524 		xive_native_sync_source(hw_num);
525 		rc = 0;
526 	}
527 
528 	arch_spin_unlock(&sb->lock);
529 	return rc;
530 }
531 
532 static int xive_native_validate_queue_size(u32 qshift)
533 {
534 	/*
535 	 * We only support 64K pages for the moment. This is also
536 	 * advertised in the DT property "ibm,xive-eq-sizes"
537 	 */
538 	switch (qshift) {
539 	case 0: /* EQ reset */
540 	case 16:
541 		return 0;
542 	case 12:
543 	case 21:
544 	case 24:
545 	default:
546 		return -EINVAL;
547 	}
548 }
549 
550 static int kvmppc_xive_native_set_queue_config(struct kvmppc_xive *xive,
551 					       long eq_idx, u64 addr)
552 {
553 	struct kvm *kvm = xive->kvm;
554 	struct kvm_vcpu *vcpu;
555 	struct kvmppc_xive_vcpu *xc;
556 	void __user *ubufp = (void __user *) addr;
557 	u32 server;
558 	u8 priority;
559 	struct kvm_ppc_xive_eq kvm_eq;
560 	int rc;
561 	__be32 *qaddr = 0;
562 	struct page *page;
563 	struct xive_q *q;
564 	gfn_t gfn;
565 	unsigned long page_size;
566 	int srcu_idx;
567 
568 	/*
569 	 * Demangle priority/server tuple from the EQ identifier
570 	 */
571 	priority = (eq_idx & KVM_XIVE_EQ_PRIORITY_MASK) >>
572 		KVM_XIVE_EQ_PRIORITY_SHIFT;
573 	server = (eq_idx & KVM_XIVE_EQ_SERVER_MASK) >>
574 		KVM_XIVE_EQ_SERVER_SHIFT;
575 
576 	if (copy_from_user(&kvm_eq, ubufp, sizeof(kvm_eq)))
577 		return -EFAULT;
578 
579 	vcpu = kvmppc_xive_find_server(kvm, server);
580 	if (!vcpu) {
581 		pr_err("Can't find server %d\n", server);
582 		return -ENOENT;
583 	}
584 	xc = vcpu->arch.xive_vcpu;
585 
586 	if (priority != xive_prio_from_guest(priority)) {
587 		pr_err("Trying to restore invalid queue %d for VCPU %d\n",
588 		       priority, server);
589 		return -EINVAL;
590 	}
591 	q = &xc->queues[priority];
592 
593 	pr_devel("%s VCPU %d priority %d fl:%x shift:%d addr:%llx g:%d idx:%d\n",
594 		 __func__, server, priority, kvm_eq.flags,
595 		 kvm_eq.qshift, kvm_eq.qaddr, kvm_eq.qtoggle, kvm_eq.qindex);
596 
597 	/* reset queue and disable queueing */
598 	if (!kvm_eq.qshift) {
599 		q->guest_qaddr  = 0;
600 		q->guest_qshift = 0;
601 
602 		rc = kvmppc_xive_native_configure_queue(xc->vp_id, q, priority,
603 							NULL, 0, true);
604 		if (rc) {
605 			pr_err("Failed to reset queue %d for VCPU %d: %d\n",
606 			       priority, xc->server_num, rc);
607 			return rc;
608 		}
609 
610 		return 0;
611 	}
612 
613 	/*
614 	 * sPAPR specifies a "Unconditional Notify (n) flag" for the
615 	 * H_INT_SET_QUEUE_CONFIG hcall which forces notification
616 	 * without using the coalescing mechanisms provided by the
617 	 * XIVE END ESBs. This is required on KVM as notification
618 	 * using the END ESBs is not supported.
619 	 */
620 	if (kvm_eq.flags != KVM_XIVE_EQ_ALWAYS_NOTIFY) {
621 		pr_err("invalid flags %d\n", kvm_eq.flags);
622 		return -EINVAL;
623 	}
624 
625 	rc = xive_native_validate_queue_size(kvm_eq.qshift);
626 	if (rc) {
627 		pr_err("invalid queue size %d\n", kvm_eq.qshift);
628 		return rc;
629 	}
630 
631 	if (kvm_eq.qaddr & ((1ull << kvm_eq.qshift) - 1)) {
632 		pr_err("queue page is not aligned %llx/%llx\n", kvm_eq.qaddr,
633 		       1ull << kvm_eq.qshift);
634 		return -EINVAL;
635 	}
636 
637 	srcu_idx = srcu_read_lock(&kvm->srcu);
638 	gfn = gpa_to_gfn(kvm_eq.qaddr);
639 
640 	page_size = kvm_host_page_size(vcpu, gfn);
641 	if (1ull << kvm_eq.qshift > page_size) {
642 		srcu_read_unlock(&kvm->srcu, srcu_idx);
643 		pr_warn("Incompatible host page size %lx!\n", page_size);
644 		return -EINVAL;
645 	}
646 
647 	page = gfn_to_page(kvm, gfn);
648 	if (is_error_page(page)) {
649 		srcu_read_unlock(&kvm->srcu, srcu_idx);
650 		pr_err("Couldn't get queue page %llx!\n", kvm_eq.qaddr);
651 		return -EINVAL;
652 	}
653 
654 	qaddr = page_to_virt(page) + (kvm_eq.qaddr & ~PAGE_MASK);
655 	srcu_read_unlock(&kvm->srcu, srcu_idx);
656 
657 	/*
658 	 * Backup the queue page guest address to the mark EQ page
659 	 * dirty for migration.
660 	 */
661 	q->guest_qaddr  = kvm_eq.qaddr;
662 	q->guest_qshift = kvm_eq.qshift;
663 
664 	 /*
665 	  * Unconditional Notification is forced by default at the
666 	  * OPAL level because the use of END ESBs is not supported by
667 	  * Linux.
668 	  */
669 	rc = kvmppc_xive_native_configure_queue(xc->vp_id, q, priority,
670 					(__be32 *) qaddr, kvm_eq.qshift, true);
671 	if (rc) {
672 		pr_err("Failed to configure queue %d for VCPU %d: %d\n",
673 		       priority, xc->server_num, rc);
674 		put_page(page);
675 		return rc;
676 	}
677 
678 	/*
679 	 * Only restore the queue state when needed. When doing the
680 	 * H_INT_SET_SOURCE_CONFIG hcall, it should not.
681 	 */
682 	if (kvm_eq.qtoggle != 1 || kvm_eq.qindex != 0) {
683 		rc = xive_native_set_queue_state(xc->vp_id, priority,
684 						 kvm_eq.qtoggle,
685 						 kvm_eq.qindex);
686 		if (rc)
687 			goto error;
688 	}
689 
690 	rc = kvmppc_xive_attach_escalation(vcpu, priority,
691 					   xive->single_escalation);
692 error:
693 	if (rc)
694 		kvmppc_xive_native_cleanup_queue(vcpu, priority);
695 	return rc;
696 }
697 
698 static int kvmppc_xive_native_get_queue_config(struct kvmppc_xive *xive,
699 					       long eq_idx, u64 addr)
700 {
701 	struct kvm *kvm = xive->kvm;
702 	struct kvm_vcpu *vcpu;
703 	struct kvmppc_xive_vcpu *xc;
704 	struct xive_q *q;
705 	void __user *ubufp = (u64 __user *) addr;
706 	u32 server;
707 	u8 priority;
708 	struct kvm_ppc_xive_eq kvm_eq;
709 	u64 qaddr;
710 	u64 qshift;
711 	u64 qeoi_page;
712 	u32 escalate_irq;
713 	u64 qflags;
714 	int rc;
715 
716 	/*
717 	 * Demangle priority/server tuple from the EQ identifier
718 	 */
719 	priority = (eq_idx & KVM_XIVE_EQ_PRIORITY_MASK) >>
720 		KVM_XIVE_EQ_PRIORITY_SHIFT;
721 	server = (eq_idx & KVM_XIVE_EQ_SERVER_MASK) >>
722 		KVM_XIVE_EQ_SERVER_SHIFT;
723 
724 	vcpu = kvmppc_xive_find_server(kvm, server);
725 	if (!vcpu) {
726 		pr_err("Can't find server %d\n", server);
727 		return -ENOENT;
728 	}
729 	xc = vcpu->arch.xive_vcpu;
730 
731 	if (priority != xive_prio_from_guest(priority)) {
732 		pr_err("invalid priority for queue %d for VCPU %d\n",
733 		       priority, server);
734 		return -EINVAL;
735 	}
736 	q = &xc->queues[priority];
737 
738 	memset(&kvm_eq, 0, sizeof(kvm_eq));
739 
740 	if (!q->qpage)
741 		return 0;
742 
743 	rc = xive_native_get_queue_info(xc->vp_id, priority, &qaddr, &qshift,
744 					&qeoi_page, &escalate_irq, &qflags);
745 	if (rc)
746 		return rc;
747 
748 	kvm_eq.flags = 0;
749 	if (qflags & OPAL_XIVE_EQ_ALWAYS_NOTIFY)
750 		kvm_eq.flags |= KVM_XIVE_EQ_ALWAYS_NOTIFY;
751 
752 	kvm_eq.qshift = q->guest_qshift;
753 	kvm_eq.qaddr  = q->guest_qaddr;
754 
755 	rc = xive_native_get_queue_state(xc->vp_id, priority, &kvm_eq.qtoggle,
756 					 &kvm_eq.qindex);
757 	if (rc)
758 		return rc;
759 
760 	pr_devel("%s VCPU %d priority %d fl:%x shift:%d addr:%llx g:%d idx:%d\n",
761 		 __func__, server, priority, kvm_eq.flags,
762 		 kvm_eq.qshift, kvm_eq.qaddr, kvm_eq.qtoggle, kvm_eq.qindex);
763 
764 	if (copy_to_user(ubufp, &kvm_eq, sizeof(kvm_eq)))
765 		return -EFAULT;
766 
767 	return 0;
768 }
769 
770 static void kvmppc_xive_reset_sources(struct kvmppc_xive_src_block *sb)
771 {
772 	int i;
773 
774 	for (i = 0; i < KVMPPC_XICS_IRQ_PER_ICS; i++) {
775 		struct kvmppc_xive_irq_state *state = &sb->irq_state[i];
776 
777 		if (!state->valid)
778 			continue;
779 
780 		if (state->act_priority == MASKED)
781 			continue;
782 
783 		state->eisn = 0;
784 		state->act_server = 0;
785 		state->act_priority = MASKED;
786 		xive_vm_esb_load(&state->ipi_data, XIVE_ESB_SET_PQ_01);
787 		xive_native_configure_irq(state->ipi_number, 0, MASKED, 0);
788 		if (state->pt_number) {
789 			xive_vm_esb_load(state->pt_data, XIVE_ESB_SET_PQ_01);
790 			xive_native_configure_irq(state->pt_number,
791 						  0, MASKED, 0);
792 		}
793 	}
794 }
795 
796 static int kvmppc_xive_reset(struct kvmppc_xive *xive)
797 {
798 	struct kvm *kvm = xive->kvm;
799 	struct kvm_vcpu *vcpu;
800 	unsigned int i;
801 
802 	pr_devel("%s\n", __func__);
803 
804 	mutex_lock(&xive->lock);
805 
806 	kvm_for_each_vcpu(i, vcpu, kvm) {
807 		struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
808 		unsigned int prio;
809 
810 		if (!xc)
811 			continue;
812 
813 		kvmppc_xive_disable_vcpu_interrupts(vcpu);
814 
815 		for (prio = 0; prio < KVMPPC_XIVE_Q_COUNT; prio++) {
816 
817 			/* Single escalation, no queue 7 */
818 			if (prio == 7 && xive->single_escalation)
819 				break;
820 
821 			if (xc->esc_virq[prio]) {
822 				free_irq(xc->esc_virq[prio], vcpu);
823 				irq_dispose_mapping(xc->esc_virq[prio]);
824 				kfree(xc->esc_virq_names[prio]);
825 				xc->esc_virq[prio] = 0;
826 			}
827 
828 			kvmppc_xive_native_cleanup_queue(vcpu, prio);
829 		}
830 	}
831 
832 	for (i = 0; i <= xive->max_sbid; i++) {
833 		struct kvmppc_xive_src_block *sb = xive->src_blocks[i];
834 
835 		if (sb) {
836 			arch_spin_lock(&sb->lock);
837 			kvmppc_xive_reset_sources(sb);
838 			arch_spin_unlock(&sb->lock);
839 		}
840 	}
841 
842 	mutex_unlock(&xive->lock);
843 
844 	return 0;
845 }
846 
847 static void kvmppc_xive_native_sync_sources(struct kvmppc_xive_src_block *sb)
848 {
849 	int j;
850 
851 	for (j = 0; j < KVMPPC_XICS_IRQ_PER_ICS; j++) {
852 		struct kvmppc_xive_irq_state *state = &sb->irq_state[j];
853 		struct xive_irq_data *xd;
854 		u32 hw_num;
855 
856 		if (!state->valid)
857 			continue;
858 
859 		/*
860 		 * The struct kvmppc_xive_irq_state reflects the state
861 		 * of the EAS configuration and not the state of the
862 		 * source. The source is masked setting the PQ bits to
863 		 * '-Q', which is what is being done before calling
864 		 * the KVM_DEV_XIVE_EQ_SYNC control.
865 		 *
866 		 * If a source EAS is configured, OPAL syncs the XIVE
867 		 * IC of the source and the XIVE IC of the previous
868 		 * target if any.
869 		 *
870 		 * So it should be fine ignoring MASKED sources as
871 		 * they have been synced already.
872 		 */
873 		if (state->act_priority == MASKED)
874 			continue;
875 
876 		kvmppc_xive_select_irq(state, &hw_num, &xd);
877 		xive_native_sync_source(hw_num);
878 		xive_native_sync_queue(hw_num);
879 	}
880 }
881 
882 static int kvmppc_xive_native_vcpu_eq_sync(struct kvm_vcpu *vcpu)
883 {
884 	struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
885 	unsigned int prio;
886 	int srcu_idx;
887 
888 	if (!xc)
889 		return -ENOENT;
890 
891 	for (prio = 0; prio < KVMPPC_XIVE_Q_COUNT; prio++) {
892 		struct xive_q *q = &xc->queues[prio];
893 
894 		if (!q->qpage)
895 			continue;
896 
897 		/* Mark EQ page dirty for migration */
898 		srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
899 		mark_page_dirty(vcpu->kvm, gpa_to_gfn(q->guest_qaddr));
900 		srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);
901 	}
902 	return 0;
903 }
904 
905 static int kvmppc_xive_native_eq_sync(struct kvmppc_xive *xive)
906 {
907 	struct kvm *kvm = xive->kvm;
908 	struct kvm_vcpu *vcpu;
909 	unsigned int i;
910 
911 	pr_devel("%s\n", __func__);
912 
913 	mutex_lock(&xive->lock);
914 	for (i = 0; i <= xive->max_sbid; i++) {
915 		struct kvmppc_xive_src_block *sb = xive->src_blocks[i];
916 
917 		if (sb) {
918 			arch_spin_lock(&sb->lock);
919 			kvmppc_xive_native_sync_sources(sb);
920 			arch_spin_unlock(&sb->lock);
921 		}
922 	}
923 
924 	kvm_for_each_vcpu(i, vcpu, kvm) {
925 		kvmppc_xive_native_vcpu_eq_sync(vcpu);
926 	}
927 	mutex_unlock(&xive->lock);
928 
929 	return 0;
930 }
931 
932 static int kvmppc_xive_native_set_attr(struct kvm_device *dev,
933 				       struct kvm_device_attr *attr)
934 {
935 	struct kvmppc_xive *xive = dev->private;
936 
937 	switch (attr->group) {
938 	case KVM_DEV_XIVE_GRP_CTRL:
939 		switch (attr->attr) {
940 		case KVM_DEV_XIVE_RESET:
941 			return kvmppc_xive_reset(xive);
942 		case KVM_DEV_XIVE_EQ_SYNC:
943 			return kvmppc_xive_native_eq_sync(xive);
944 		case KVM_DEV_XIVE_NR_SERVERS:
945 			return kvmppc_xive_set_nr_servers(xive, attr->addr);
946 		}
947 		break;
948 	case KVM_DEV_XIVE_GRP_SOURCE:
949 		return kvmppc_xive_native_set_source(xive, attr->attr,
950 						     attr->addr);
951 	case KVM_DEV_XIVE_GRP_SOURCE_CONFIG:
952 		return kvmppc_xive_native_set_source_config(xive, attr->attr,
953 							    attr->addr);
954 	case KVM_DEV_XIVE_GRP_EQ_CONFIG:
955 		return kvmppc_xive_native_set_queue_config(xive, attr->attr,
956 							   attr->addr);
957 	case KVM_DEV_XIVE_GRP_SOURCE_SYNC:
958 		return kvmppc_xive_native_sync_source(xive, attr->attr,
959 						      attr->addr);
960 	}
961 	return -ENXIO;
962 }
963 
964 static int kvmppc_xive_native_get_attr(struct kvm_device *dev,
965 				       struct kvm_device_attr *attr)
966 {
967 	struct kvmppc_xive *xive = dev->private;
968 
969 	switch (attr->group) {
970 	case KVM_DEV_XIVE_GRP_EQ_CONFIG:
971 		return kvmppc_xive_native_get_queue_config(xive, attr->attr,
972 							   attr->addr);
973 	}
974 	return -ENXIO;
975 }
976 
977 static int kvmppc_xive_native_has_attr(struct kvm_device *dev,
978 				       struct kvm_device_attr *attr)
979 {
980 	switch (attr->group) {
981 	case KVM_DEV_XIVE_GRP_CTRL:
982 		switch (attr->attr) {
983 		case KVM_DEV_XIVE_RESET:
984 		case KVM_DEV_XIVE_EQ_SYNC:
985 		case KVM_DEV_XIVE_NR_SERVERS:
986 			return 0;
987 		}
988 		break;
989 	case KVM_DEV_XIVE_GRP_SOURCE:
990 	case KVM_DEV_XIVE_GRP_SOURCE_CONFIG:
991 	case KVM_DEV_XIVE_GRP_SOURCE_SYNC:
992 		if (attr->attr >= KVMPPC_XIVE_FIRST_IRQ &&
993 		    attr->attr < KVMPPC_XIVE_NR_IRQS)
994 			return 0;
995 		break;
996 	case KVM_DEV_XIVE_GRP_EQ_CONFIG:
997 		return 0;
998 	}
999 	return -ENXIO;
1000 }
1001 
1002 /*
1003  * Called when device fd is closed.  kvm->lock is held.
1004  */
1005 static void kvmppc_xive_native_release(struct kvm_device *dev)
1006 {
1007 	struct kvmppc_xive *xive = dev->private;
1008 	struct kvm *kvm = xive->kvm;
1009 	struct kvm_vcpu *vcpu;
1010 	int i;
1011 
1012 	pr_devel("Releasing xive native device\n");
1013 
1014 	/*
1015 	 * Clear the KVM device file address_space which is used to
1016 	 * unmap the ESB pages when a device is passed-through.
1017 	 */
1018 	mutex_lock(&xive->mapping_lock);
1019 	xive->mapping = NULL;
1020 	mutex_unlock(&xive->mapping_lock);
1021 
1022 	/*
1023 	 * Since this is the device release function, we know that
1024 	 * userspace does not have any open fd or mmap referring to
1025 	 * the device.  Therefore there can not be any of the
1026 	 * device attribute set/get, mmap, or page fault functions
1027 	 * being executed concurrently, and similarly, the
1028 	 * connect_vcpu and set/clr_mapped functions also cannot
1029 	 * be being executed.
1030 	 */
1031 
1032 	debugfs_remove(xive->dentry);
1033 
1034 	/*
1035 	 * We should clean up the vCPU interrupt presenters first.
1036 	 */
1037 	kvm_for_each_vcpu(i, vcpu, kvm) {
1038 		/*
1039 		 * Take vcpu->mutex to ensure that no one_reg get/set ioctl
1040 		 * (i.e. kvmppc_xive_native_[gs]et_vp) can be being done.
1041 		 * Holding the vcpu->mutex also means that the vcpu cannot
1042 		 * be executing the KVM_RUN ioctl, and therefore it cannot
1043 		 * be executing the XIVE push or pull code or accessing
1044 		 * the XIVE MMIO regions.
1045 		 */
1046 		mutex_lock(&vcpu->mutex);
1047 		kvmppc_xive_native_cleanup_vcpu(vcpu);
1048 		mutex_unlock(&vcpu->mutex);
1049 	}
1050 
1051 	/*
1052 	 * Now that we have cleared vcpu->arch.xive_vcpu, vcpu->arch.irq_type
1053 	 * and vcpu->arch.xive_esc_[vr]addr on each vcpu, we are safe
1054 	 * against xive code getting called during vcpu execution or
1055 	 * set/get one_reg operations.
1056 	 */
1057 	kvm->arch.xive = NULL;
1058 
1059 	for (i = 0; i <= xive->max_sbid; i++) {
1060 		if (xive->src_blocks[i])
1061 			kvmppc_xive_free_sources(xive->src_blocks[i]);
1062 		kfree(xive->src_blocks[i]);
1063 		xive->src_blocks[i] = NULL;
1064 	}
1065 
1066 	if (xive->vp_base != XIVE_INVALID_VP)
1067 		xive_native_free_vp_block(xive->vp_base);
1068 
1069 	/*
1070 	 * A reference of the kvmppc_xive pointer is now kept under
1071 	 * the xive_devices struct of the machine for reuse. It is
1072 	 * freed when the VM is destroyed for now until we fix all the
1073 	 * execution paths.
1074 	 */
1075 
1076 	kfree(dev);
1077 }
1078 
1079 /*
1080  * Create a XIVE device.  kvm->lock is held.
1081  */
1082 static int kvmppc_xive_native_create(struct kvm_device *dev, u32 type)
1083 {
1084 	struct kvmppc_xive *xive;
1085 	struct kvm *kvm = dev->kvm;
1086 
1087 	pr_devel("Creating xive native device\n");
1088 
1089 	if (kvm->arch.xive)
1090 		return -EEXIST;
1091 
1092 	xive = kvmppc_xive_get_device(kvm, type);
1093 	if (!xive)
1094 		return -ENOMEM;
1095 
1096 	dev->private = xive;
1097 	xive->dev = dev;
1098 	xive->kvm = kvm;
1099 	mutex_init(&xive->mapping_lock);
1100 	mutex_init(&xive->lock);
1101 
1102 	/* VP allocation is delayed to the first call to connect_vcpu */
1103 	xive->vp_base = XIVE_INVALID_VP;
1104 	/* KVM_MAX_VCPUS limits the number of VMs to roughly 64 per sockets
1105 	 * on a POWER9 system.
1106 	 */
1107 	xive->nr_servers = KVM_MAX_VCPUS;
1108 
1109 	xive->single_escalation = xive_native_has_single_escalation();
1110 	xive->ops = &kvmppc_xive_native_ops;
1111 
1112 	kvm->arch.xive = xive;
1113 	return 0;
1114 }
1115 
1116 /*
1117  * Interrupt Pending Buffer (IPB) offset
1118  */
1119 #define TM_IPB_SHIFT 40
1120 #define TM_IPB_MASK  (((u64) 0xFF) << TM_IPB_SHIFT)
1121 
1122 int kvmppc_xive_native_get_vp(struct kvm_vcpu *vcpu, union kvmppc_one_reg *val)
1123 {
1124 	struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
1125 	u64 opal_state;
1126 	int rc;
1127 
1128 	if (!kvmppc_xive_enabled(vcpu))
1129 		return -EPERM;
1130 
1131 	if (!xc)
1132 		return -ENOENT;
1133 
1134 	/* Thread context registers. We only care about IPB and CPPR */
1135 	val->xive_timaval[0] = vcpu->arch.xive_saved_state.w01;
1136 
1137 	/* Get the VP state from OPAL */
1138 	rc = xive_native_get_vp_state(xc->vp_id, &opal_state);
1139 	if (rc)
1140 		return rc;
1141 
1142 	/*
1143 	 * Capture the backup of IPB register in the NVT structure and
1144 	 * merge it in our KVM VP state.
1145 	 */
1146 	val->xive_timaval[0] |= cpu_to_be64(opal_state & TM_IPB_MASK);
1147 
1148 	pr_devel("%s NSR=%02x CPPR=%02x IBP=%02x PIPR=%02x w01=%016llx w2=%08x opal=%016llx\n",
1149 		 __func__,
1150 		 vcpu->arch.xive_saved_state.nsr,
1151 		 vcpu->arch.xive_saved_state.cppr,
1152 		 vcpu->arch.xive_saved_state.ipb,
1153 		 vcpu->arch.xive_saved_state.pipr,
1154 		 vcpu->arch.xive_saved_state.w01,
1155 		 (u32) vcpu->arch.xive_cam_word, opal_state);
1156 
1157 	return 0;
1158 }
1159 
1160 int kvmppc_xive_native_set_vp(struct kvm_vcpu *vcpu, union kvmppc_one_reg *val)
1161 {
1162 	struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
1163 	struct kvmppc_xive *xive = vcpu->kvm->arch.xive;
1164 
1165 	pr_devel("%s w01=%016llx vp=%016llx\n", __func__,
1166 		 val->xive_timaval[0], val->xive_timaval[1]);
1167 
1168 	if (!kvmppc_xive_enabled(vcpu))
1169 		return -EPERM;
1170 
1171 	if (!xc || !xive)
1172 		return -ENOENT;
1173 
1174 	/* We can't update the state of a "pushed" VCPU	 */
1175 	if (WARN_ON(vcpu->arch.xive_pushed))
1176 		return -EBUSY;
1177 
1178 	/*
1179 	 * Restore the thread context registers. IPB and CPPR should
1180 	 * be the only ones that matter.
1181 	 */
1182 	vcpu->arch.xive_saved_state.w01 = val->xive_timaval[0];
1183 
1184 	/*
1185 	 * There is no need to restore the XIVE internal state (IPB
1186 	 * stored in the NVT) as the IPB register was merged in KVM VP
1187 	 * state when captured.
1188 	 */
1189 	return 0;
1190 }
1191 
1192 bool kvmppc_xive_native_supported(void)
1193 {
1194 	return xive_native_has_queue_state_support();
1195 }
1196 
1197 static int xive_native_debug_show(struct seq_file *m, void *private)
1198 {
1199 	struct kvmppc_xive *xive = m->private;
1200 	struct kvm *kvm = xive->kvm;
1201 	struct kvm_vcpu *vcpu;
1202 	unsigned int i;
1203 
1204 	if (!kvm)
1205 		return 0;
1206 
1207 	seq_puts(m, "=========\nVCPU state\n=========\n");
1208 
1209 	kvm_for_each_vcpu(i, vcpu, kvm) {
1210 		struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
1211 
1212 		if (!xc)
1213 			continue;
1214 
1215 		seq_printf(m, "cpu server %#x VP=%#x NSR=%02x CPPR=%02x IBP=%02x PIPR=%02x w01=%016llx w2=%08x\n",
1216 			   xc->server_num, xc->vp_id,
1217 			   vcpu->arch.xive_saved_state.nsr,
1218 			   vcpu->arch.xive_saved_state.cppr,
1219 			   vcpu->arch.xive_saved_state.ipb,
1220 			   vcpu->arch.xive_saved_state.pipr,
1221 			   vcpu->arch.xive_saved_state.w01,
1222 			   (u32) vcpu->arch.xive_cam_word);
1223 
1224 		kvmppc_xive_debug_show_queues(m, vcpu);
1225 	}
1226 
1227 	return 0;
1228 }
1229 
1230 static int xive_native_debug_open(struct inode *inode, struct file *file)
1231 {
1232 	return single_open(file, xive_native_debug_show, inode->i_private);
1233 }
1234 
1235 static const struct file_operations xive_native_debug_fops = {
1236 	.open = xive_native_debug_open,
1237 	.read = seq_read,
1238 	.llseek = seq_lseek,
1239 	.release = single_release,
1240 };
1241 
1242 static void xive_native_debugfs_init(struct kvmppc_xive *xive)
1243 {
1244 	char *name;
1245 
1246 	name = kasprintf(GFP_KERNEL, "kvm-xive-%p", xive);
1247 	if (!name) {
1248 		pr_err("%s: no memory for name\n", __func__);
1249 		return;
1250 	}
1251 
1252 	xive->dentry = debugfs_create_file(name, 0444, powerpc_debugfs_root,
1253 					   xive, &xive_native_debug_fops);
1254 
1255 	pr_debug("%s: created %s\n", __func__, name);
1256 	kfree(name);
1257 }
1258 
1259 static void kvmppc_xive_native_init(struct kvm_device *dev)
1260 {
1261 	struct kvmppc_xive *xive = (struct kvmppc_xive *)dev->private;
1262 
1263 	/* Register some debug interfaces */
1264 	xive_native_debugfs_init(xive);
1265 }
1266 
1267 struct kvm_device_ops kvm_xive_native_ops = {
1268 	.name = "kvm-xive-native",
1269 	.create = kvmppc_xive_native_create,
1270 	.init = kvmppc_xive_native_init,
1271 	.release = kvmppc_xive_native_release,
1272 	.set_attr = kvmppc_xive_native_set_attr,
1273 	.get_attr = kvmppc_xive_native_get_attr,
1274 	.has_attr = kvmppc_xive_native_has_attr,
1275 	.mmap = kvmppc_xive_native_mmap,
1276 };
1277 
1278 void kvmppc_xive_native_init_module(void)
1279 {
1280 	;
1281 }
1282 
1283 void kvmppc_xive_native_exit_module(void)
1284 {
1285 	;
1286 }
1287