xref: /openbmc/linux/arch/x86/xen/smp.c (revision 609e478b)
1 /*
2  * Xen SMP support
3  *
4  * This file implements the Xen versions of smp_ops.  SMP under Xen is
5  * very straightforward.  Bringing a CPU up is simply a matter of
6  * loading its initial context and setting it running.
7  *
8  * IPIs are handled through the Xen event mechanism.
9  *
10  * Because virtual CPUs can be scheduled onto any real CPU, there's no
11  * useful topology information for the kernel to make use of.  As a
12  * result, all CPUs are treated as if they're single-core and
13  * single-threaded.
14  */
15 #include <linux/sched.h>
16 #include <linux/err.h>
17 #include <linux/slab.h>
18 #include <linux/smp.h>
19 #include <linux/irq_work.h>
20 #include <linux/tick.h>
21 
22 #include <asm/paravirt.h>
23 #include <asm/desc.h>
24 #include <asm/pgtable.h>
25 #include <asm/cpu.h>
26 
27 #include <xen/interface/xen.h>
28 #include <xen/interface/vcpu.h>
29 
30 #include <asm/xen/interface.h>
31 #include <asm/xen/hypercall.h>
32 
33 #include <xen/xen.h>
34 #include <xen/page.h>
35 #include <xen/events.h>
36 
37 #include <xen/hvc-console.h>
38 #include "xen-ops.h"
39 #include "mmu.h"
40 #include "smp.h"
41 
42 cpumask_var_t xen_cpu_initialized_map;
43 
44 struct xen_common_irq {
45 	int irq;
46 	char *name;
47 };
48 static DEFINE_PER_CPU(struct xen_common_irq, xen_resched_irq) = { .irq = -1 };
49 static DEFINE_PER_CPU(struct xen_common_irq, xen_callfunc_irq) = { .irq = -1 };
50 static DEFINE_PER_CPU(struct xen_common_irq, xen_callfuncsingle_irq) = { .irq = -1 };
51 static DEFINE_PER_CPU(struct xen_common_irq, xen_irq_work) = { .irq = -1 };
52 static DEFINE_PER_CPU(struct xen_common_irq, xen_debug_irq) = { .irq = -1 };
53 
54 static irqreturn_t xen_call_function_interrupt(int irq, void *dev_id);
55 static irqreturn_t xen_call_function_single_interrupt(int irq, void *dev_id);
56 static irqreturn_t xen_irq_work_interrupt(int irq, void *dev_id);
57 
58 /*
59  * Reschedule call back.
60  */
61 static irqreturn_t xen_reschedule_interrupt(int irq, void *dev_id)
62 {
63 	inc_irq_stat(irq_resched_count);
64 	scheduler_ipi();
65 
66 	return IRQ_HANDLED;
67 }
68 
69 static void cpu_bringup(void)
70 {
71 	int cpu;
72 
73 	cpu_init();
74 	touch_softlockup_watchdog();
75 	preempt_disable();
76 
77 	/* PVH runs in ring 0 and allows us to do native syscalls. Yay! */
78 	if (!xen_feature(XENFEAT_supervisor_mode_kernel)) {
79 		xen_enable_sysenter();
80 		xen_enable_syscall();
81 	}
82 	cpu = smp_processor_id();
83 	smp_store_cpu_info(cpu);
84 	cpu_data(cpu).x86_max_cores = 1;
85 	set_cpu_sibling_map(cpu);
86 
87 	xen_setup_cpu_clockevents();
88 
89 	notify_cpu_starting(cpu);
90 
91 	set_cpu_online(cpu, true);
92 
93 	this_cpu_write(cpu_state, CPU_ONLINE);
94 
95 	wmb();
96 
97 	/* We can take interrupts now: we're officially "up". */
98 	local_irq_enable();
99 
100 	wmb();			/* make sure everything is out */
101 }
102 
103 /*
104  * Note: cpu parameter is only relevant for PVH. The reason for passing it
105  * is we can't do smp_processor_id until the percpu segments are loaded, for
106  * which we need the cpu number! So we pass it in rdi as first parameter.
107  */
108 asmlinkage __visible void cpu_bringup_and_idle(int cpu)
109 {
110 #ifdef CONFIG_XEN_PVH
111 	if (xen_feature(XENFEAT_auto_translated_physmap) &&
112 	    xen_feature(XENFEAT_supervisor_mode_kernel))
113 		xen_pvh_secondary_vcpu_init(cpu);
114 #endif
115 	cpu_bringup();
116 	cpu_startup_entry(CPUHP_ONLINE);
117 }
118 
119 static void xen_smp_intr_free(unsigned int cpu)
120 {
121 	if (per_cpu(xen_resched_irq, cpu).irq >= 0) {
122 		unbind_from_irqhandler(per_cpu(xen_resched_irq, cpu).irq, NULL);
123 		per_cpu(xen_resched_irq, cpu).irq = -1;
124 		kfree(per_cpu(xen_resched_irq, cpu).name);
125 		per_cpu(xen_resched_irq, cpu).name = NULL;
126 	}
127 	if (per_cpu(xen_callfunc_irq, cpu).irq >= 0) {
128 		unbind_from_irqhandler(per_cpu(xen_callfunc_irq, cpu).irq, NULL);
129 		per_cpu(xen_callfunc_irq, cpu).irq = -1;
130 		kfree(per_cpu(xen_callfunc_irq, cpu).name);
131 		per_cpu(xen_callfunc_irq, cpu).name = NULL;
132 	}
133 	if (per_cpu(xen_debug_irq, cpu).irq >= 0) {
134 		unbind_from_irqhandler(per_cpu(xen_debug_irq, cpu).irq, NULL);
135 		per_cpu(xen_debug_irq, cpu).irq = -1;
136 		kfree(per_cpu(xen_debug_irq, cpu).name);
137 		per_cpu(xen_debug_irq, cpu).name = NULL;
138 	}
139 	if (per_cpu(xen_callfuncsingle_irq, cpu).irq >= 0) {
140 		unbind_from_irqhandler(per_cpu(xen_callfuncsingle_irq, cpu).irq,
141 				       NULL);
142 		per_cpu(xen_callfuncsingle_irq, cpu).irq = -1;
143 		kfree(per_cpu(xen_callfuncsingle_irq, cpu).name);
144 		per_cpu(xen_callfuncsingle_irq, cpu).name = NULL;
145 	}
146 	if (xen_hvm_domain())
147 		return;
148 
149 	if (per_cpu(xen_irq_work, cpu).irq >= 0) {
150 		unbind_from_irqhandler(per_cpu(xen_irq_work, cpu).irq, NULL);
151 		per_cpu(xen_irq_work, cpu).irq = -1;
152 		kfree(per_cpu(xen_irq_work, cpu).name);
153 		per_cpu(xen_irq_work, cpu).name = NULL;
154 	}
155 };
156 static int xen_smp_intr_init(unsigned int cpu)
157 {
158 	int rc;
159 	char *resched_name, *callfunc_name, *debug_name;
160 
161 	resched_name = kasprintf(GFP_KERNEL, "resched%d", cpu);
162 	rc = bind_ipi_to_irqhandler(XEN_RESCHEDULE_VECTOR,
163 				    cpu,
164 				    xen_reschedule_interrupt,
165 				    IRQF_PERCPU|IRQF_NOBALANCING,
166 				    resched_name,
167 				    NULL);
168 	if (rc < 0)
169 		goto fail;
170 	per_cpu(xen_resched_irq, cpu).irq = rc;
171 	per_cpu(xen_resched_irq, cpu).name = resched_name;
172 
173 	callfunc_name = kasprintf(GFP_KERNEL, "callfunc%d", cpu);
174 	rc = bind_ipi_to_irqhandler(XEN_CALL_FUNCTION_VECTOR,
175 				    cpu,
176 				    xen_call_function_interrupt,
177 				    IRQF_PERCPU|IRQF_NOBALANCING,
178 				    callfunc_name,
179 				    NULL);
180 	if (rc < 0)
181 		goto fail;
182 	per_cpu(xen_callfunc_irq, cpu).irq = rc;
183 	per_cpu(xen_callfunc_irq, cpu).name = callfunc_name;
184 
185 	debug_name = kasprintf(GFP_KERNEL, "debug%d", cpu);
186 	rc = bind_virq_to_irqhandler(VIRQ_DEBUG, cpu, xen_debug_interrupt,
187 				     IRQF_PERCPU | IRQF_NOBALANCING,
188 				     debug_name, NULL);
189 	if (rc < 0)
190 		goto fail;
191 	per_cpu(xen_debug_irq, cpu).irq = rc;
192 	per_cpu(xen_debug_irq, cpu).name = debug_name;
193 
194 	callfunc_name = kasprintf(GFP_KERNEL, "callfuncsingle%d", cpu);
195 	rc = bind_ipi_to_irqhandler(XEN_CALL_FUNCTION_SINGLE_VECTOR,
196 				    cpu,
197 				    xen_call_function_single_interrupt,
198 				    IRQF_PERCPU|IRQF_NOBALANCING,
199 				    callfunc_name,
200 				    NULL);
201 	if (rc < 0)
202 		goto fail;
203 	per_cpu(xen_callfuncsingle_irq, cpu).irq = rc;
204 	per_cpu(xen_callfuncsingle_irq, cpu).name = callfunc_name;
205 
206 	/*
207 	 * The IRQ worker on PVHVM goes through the native path and uses the
208 	 * IPI mechanism.
209 	 */
210 	if (xen_hvm_domain())
211 		return 0;
212 
213 	callfunc_name = kasprintf(GFP_KERNEL, "irqwork%d", cpu);
214 	rc = bind_ipi_to_irqhandler(XEN_IRQ_WORK_VECTOR,
215 				    cpu,
216 				    xen_irq_work_interrupt,
217 				    IRQF_PERCPU|IRQF_NOBALANCING,
218 				    callfunc_name,
219 				    NULL);
220 	if (rc < 0)
221 		goto fail;
222 	per_cpu(xen_irq_work, cpu).irq = rc;
223 	per_cpu(xen_irq_work, cpu).name = callfunc_name;
224 
225 	return 0;
226 
227  fail:
228 	xen_smp_intr_free(cpu);
229 	return rc;
230 }
231 
232 static void __init xen_fill_possible_map(void)
233 {
234 	int i, rc;
235 
236 	if (xen_initial_domain())
237 		return;
238 
239 	for (i = 0; i < nr_cpu_ids; i++) {
240 		rc = HYPERVISOR_vcpu_op(VCPUOP_is_up, i, NULL);
241 		if (rc >= 0) {
242 			num_processors++;
243 			set_cpu_possible(i, true);
244 		}
245 	}
246 }
247 
248 static void __init xen_filter_cpu_maps(void)
249 {
250 	int i, rc;
251 	unsigned int subtract = 0;
252 
253 	if (!xen_initial_domain())
254 		return;
255 
256 	num_processors = 0;
257 	disabled_cpus = 0;
258 	for (i = 0; i < nr_cpu_ids; i++) {
259 		rc = HYPERVISOR_vcpu_op(VCPUOP_is_up, i, NULL);
260 		if (rc >= 0) {
261 			num_processors++;
262 			set_cpu_possible(i, true);
263 		} else {
264 			set_cpu_possible(i, false);
265 			set_cpu_present(i, false);
266 			subtract++;
267 		}
268 	}
269 #ifdef CONFIG_HOTPLUG_CPU
270 	/* This is akin to using 'nr_cpus' on the Linux command line.
271 	 * Which is OK as when we use 'dom0_max_vcpus=X' we can only
272 	 * have up to X, while nr_cpu_ids is greater than X. This
273 	 * normally is not a problem, except when CPU hotplugging
274 	 * is involved and then there might be more than X CPUs
275 	 * in the guest - which will not work as there is no
276 	 * hypercall to expand the max number of VCPUs an already
277 	 * running guest has. So cap it up to X. */
278 	if (subtract)
279 		nr_cpu_ids = nr_cpu_ids - subtract;
280 #endif
281 
282 }
283 
284 static void __init xen_smp_prepare_boot_cpu(void)
285 {
286 	BUG_ON(smp_processor_id() != 0);
287 	native_smp_prepare_boot_cpu();
288 
289 	if (xen_pv_domain()) {
290 		if (!xen_feature(XENFEAT_writable_page_tables))
291 			/* We've switched to the "real" per-cpu gdt, so make
292 			 * sure the old memory can be recycled. */
293 			make_lowmem_page_readwrite(xen_initial_gdt);
294 
295 #ifdef CONFIG_X86_32
296 		/*
297 		 * Xen starts us with XEN_FLAT_RING1_DS, but linux code
298 		 * expects __USER_DS
299 		 */
300 		loadsegment(ds, __USER_DS);
301 		loadsegment(es, __USER_DS);
302 #endif
303 
304 		xen_filter_cpu_maps();
305 		xen_setup_vcpu_info_placement();
306 	}
307 	/*
308 	 * The alternative logic (which patches the unlock/lock) runs before
309 	 * the smp bootup up code is activated. Hence we need to set this up
310 	 * the core kernel is being patched. Otherwise we will have only
311 	 * modules patched but not core code.
312 	 */
313 	xen_init_spinlocks();
314 }
315 
316 static void __init xen_smp_prepare_cpus(unsigned int max_cpus)
317 {
318 	unsigned cpu;
319 	unsigned int i;
320 
321 	if (skip_ioapic_setup) {
322 		char *m = (max_cpus == 0) ?
323 			"The nosmp parameter is incompatible with Xen; " \
324 			"use Xen dom0_max_vcpus=1 parameter" :
325 			"The noapic parameter is incompatible with Xen";
326 
327 		xen_raw_printk(m);
328 		panic(m);
329 	}
330 	xen_init_lock_cpu(0);
331 
332 	smp_store_boot_cpu_info();
333 	cpu_data(0).x86_max_cores = 1;
334 
335 	for_each_possible_cpu(i) {
336 		zalloc_cpumask_var(&per_cpu(cpu_sibling_map, i), GFP_KERNEL);
337 		zalloc_cpumask_var(&per_cpu(cpu_core_map, i), GFP_KERNEL);
338 		zalloc_cpumask_var(&per_cpu(cpu_llc_shared_map, i), GFP_KERNEL);
339 	}
340 	set_cpu_sibling_map(0);
341 
342 	if (xen_smp_intr_init(0))
343 		BUG();
344 
345 	if (!alloc_cpumask_var(&xen_cpu_initialized_map, GFP_KERNEL))
346 		panic("could not allocate xen_cpu_initialized_map\n");
347 
348 	cpumask_copy(xen_cpu_initialized_map, cpumask_of(0));
349 
350 	/* Restrict the possible_map according to max_cpus. */
351 	while ((num_possible_cpus() > 1) && (num_possible_cpus() > max_cpus)) {
352 		for (cpu = nr_cpu_ids - 1; !cpu_possible(cpu); cpu--)
353 			continue;
354 		set_cpu_possible(cpu, false);
355 	}
356 
357 	for_each_possible_cpu(cpu)
358 		set_cpu_present(cpu, true);
359 }
360 
361 static int
362 cpu_initialize_context(unsigned int cpu, struct task_struct *idle)
363 {
364 	struct vcpu_guest_context *ctxt;
365 	struct desc_struct *gdt;
366 	unsigned long gdt_mfn;
367 
368 	/* used to tell cpu_init() that it can proceed with initialization */
369 	cpumask_set_cpu(cpu, cpu_callout_mask);
370 	if (cpumask_test_and_set_cpu(cpu, xen_cpu_initialized_map))
371 		return 0;
372 
373 	ctxt = kzalloc(sizeof(*ctxt), GFP_KERNEL);
374 	if (ctxt == NULL)
375 		return -ENOMEM;
376 
377 	gdt = get_cpu_gdt_table(cpu);
378 
379 #ifdef CONFIG_X86_32
380 	/* Note: PVH is not yet supported on x86_32. */
381 	ctxt->user_regs.fs = __KERNEL_PERCPU;
382 	ctxt->user_regs.gs = __KERNEL_STACK_CANARY;
383 #endif
384 	memset(&ctxt->fpu_ctxt, 0, sizeof(ctxt->fpu_ctxt));
385 
386 	if (!xen_feature(XENFEAT_auto_translated_physmap)) {
387 		ctxt->user_regs.eip = (unsigned long)cpu_bringup_and_idle;
388 		ctxt->flags = VGCF_IN_KERNEL;
389 		ctxt->user_regs.eflags = 0x1000; /* IOPL_RING1 */
390 		ctxt->user_regs.ds = __USER_DS;
391 		ctxt->user_regs.es = __USER_DS;
392 		ctxt->user_regs.ss = __KERNEL_DS;
393 
394 		xen_copy_trap_info(ctxt->trap_ctxt);
395 
396 		ctxt->ldt_ents = 0;
397 
398 		BUG_ON((unsigned long)gdt & ~PAGE_MASK);
399 
400 		gdt_mfn = arbitrary_virt_to_mfn(gdt);
401 		make_lowmem_page_readonly(gdt);
402 		make_lowmem_page_readonly(mfn_to_virt(gdt_mfn));
403 
404 		ctxt->gdt_frames[0] = gdt_mfn;
405 		ctxt->gdt_ents      = GDT_ENTRIES;
406 
407 		ctxt->kernel_ss = __KERNEL_DS;
408 		ctxt->kernel_sp = idle->thread.sp0;
409 
410 #ifdef CONFIG_X86_32
411 		ctxt->event_callback_cs     = __KERNEL_CS;
412 		ctxt->failsafe_callback_cs  = __KERNEL_CS;
413 #else
414 		ctxt->gs_base_kernel = per_cpu_offset(cpu);
415 #endif
416 		ctxt->event_callback_eip    =
417 					(unsigned long)xen_hypervisor_callback;
418 		ctxt->failsafe_callback_eip =
419 					(unsigned long)xen_failsafe_callback;
420 		ctxt->user_regs.cs = __KERNEL_CS;
421 		per_cpu(xen_cr3, cpu) = __pa(swapper_pg_dir);
422 	}
423 #ifdef CONFIG_XEN_PVH
424 	else {
425 		/*
426 		 * The vcpu comes on kernel page tables which have the NX pte
427 		 * bit set. This means before DS/SS is touched, NX in
428 		 * EFER must be set. Hence the following assembly glue code.
429 		 */
430 		ctxt->user_regs.eip = (unsigned long)xen_pvh_early_cpu_init;
431 		ctxt->user_regs.rdi = cpu;
432 		ctxt->user_regs.rsi = true;  /* entry == true */
433 	}
434 #endif
435 	ctxt->user_regs.esp = idle->thread.sp0 - sizeof(struct pt_regs);
436 	ctxt->ctrlreg[3] = xen_pfn_to_cr3(virt_to_mfn(swapper_pg_dir));
437 	if (HYPERVISOR_vcpu_op(VCPUOP_initialise, cpu, ctxt))
438 		BUG();
439 
440 	kfree(ctxt);
441 	return 0;
442 }
443 
444 static int xen_cpu_up(unsigned int cpu, struct task_struct *idle)
445 {
446 	int rc;
447 
448 	per_cpu(current_task, cpu) = idle;
449 #ifdef CONFIG_X86_32
450 	irq_ctx_init(cpu);
451 #else
452 	clear_tsk_thread_flag(idle, TIF_FORK);
453 #endif
454 	per_cpu(kernel_stack, cpu) =
455 		(unsigned long)task_stack_page(idle) -
456 		KERNEL_STACK_OFFSET + THREAD_SIZE;
457 
458 	xen_setup_runstate_info(cpu);
459 	xen_setup_timer(cpu);
460 	xen_init_lock_cpu(cpu);
461 
462 	per_cpu(cpu_state, cpu) = CPU_UP_PREPARE;
463 
464 	/* make sure interrupts start blocked */
465 	per_cpu(xen_vcpu, cpu)->evtchn_upcall_mask = 1;
466 
467 	rc = cpu_initialize_context(cpu, idle);
468 	if (rc)
469 		return rc;
470 
471 	if (num_online_cpus() == 1)
472 		/* Just in case we booted with a single CPU. */
473 		alternatives_enable_smp();
474 
475 	rc = xen_smp_intr_init(cpu);
476 	if (rc)
477 		return rc;
478 
479 	rc = HYPERVISOR_vcpu_op(VCPUOP_up, cpu, NULL);
480 	BUG_ON(rc);
481 
482 	while(per_cpu(cpu_state, cpu) != CPU_ONLINE) {
483 		HYPERVISOR_sched_op(SCHEDOP_yield, NULL);
484 		barrier();
485 	}
486 
487 	return 0;
488 }
489 
490 static void xen_smp_cpus_done(unsigned int max_cpus)
491 {
492 }
493 
494 #ifdef CONFIG_HOTPLUG_CPU
495 static int xen_cpu_disable(void)
496 {
497 	unsigned int cpu = smp_processor_id();
498 	if (cpu == 0)
499 		return -EBUSY;
500 
501 	cpu_disable_common();
502 
503 	load_cr3(swapper_pg_dir);
504 	return 0;
505 }
506 
507 static void xen_cpu_die(unsigned int cpu)
508 {
509 	while (xen_pv_domain() && HYPERVISOR_vcpu_op(VCPUOP_is_up, cpu, NULL)) {
510 		current->state = TASK_UNINTERRUPTIBLE;
511 		schedule_timeout(HZ/10);
512 	}
513 	xen_smp_intr_free(cpu);
514 	xen_uninit_lock_cpu(cpu);
515 	xen_teardown_timer(cpu);
516 }
517 
518 static void xen_play_dead(void) /* used only with HOTPLUG_CPU */
519 {
520 	play_dead_common();
521 	HYPERVISOR_vcpu_op(VCPUOP_down, smp_processor_id(), NULL);
522 	cpu_bringup();
523 	/*
524 	 * commit 4b0c0f294 (tick: Cleanup NOHZ per cpu data on cpu down)
525 	 * clears certain data that the cpu_idle loop (which called us
526 	 * and that we return from) expects. The only way to get that
527 	 * data back is to call:
528 	 */
529 	tick_nohz_idle_enter();
530 }
531 
532 #else /* !CONFIG_HOTPLUG_CPU */
533 static int xen_cpu_disable(void)
534 {
535 	return -ENOSYS;
536 }
537 
538 static void xen_cpu_die(unsigned int cpu)
539 {
540 	BUG();
541 }
542 
543 static void xen_play_dead(void)
544 {
545 	BUG();
546 }
547 
548 #endif
549 static void stop_self(void *v)
550 {
551 	int cpu = smp_processor_id();
552 
553 	/* make sure we're not pinning something down */
554 	load_cr3(swapper_pg_dir);
555 	/* should set up a minimal gdt */
556 
557 	set_cpu_online(cpu, false);
558 
559 	HYPERVISOR_vcpu_op(VCPUOP_down, cpu, NULL);
560 	BUG();
561 }
562 
563 static void xen_stop_other_cpus(int wait)
564 {
565 	smp_call_function(stop_self, NULL, wait);
566 }
567 
568 static void xen_smp_send_reschedule(int cpu)
569 {
570 	xen_send_IPI_one(cpu, XEN_RESCHEDULE_VECTOR);
571 }
572 
573 static void __xen_send_IPI_mask(const struct cpumask *mask,
574 			      int vector)
575 {
576 	unsigned cpu;
577 
578 	for_each_cpu_and(cpu, mask, cpu_online_mask)
579 		xen_send_IPI_one(cpu, vector);
580 }
581 
582 static void xen_smp_send_call_function_ipi(const struct cpumask *mask)
583 {
584 	int cpu;
585 
586 	__xen_send_IPI_mask(mask, XEN_CALL_FUNCTION_VECTOR);
587 
588 	/* Make sure other vcpus get a chance to run if they need to. */
589 	for_each_cpu(cpu, mask) {
590 		if (xen_vcpu_stolen(cpu)) {
591 			HYPERVISOR_sched_op(SCHEDOP_yield, NULL);
592 			break;
593 		}
594 	}
595 }
596 
597 static void xen_smp_send_call_function_single_ipi(int cpu)
598 {
599 	__xen_send_IPI_mask(cpumask_of(cpu),
600 			  XEN_CALL_FUNCTION_SINGLE_VECTOR);
601 }
602 
603 static inline int xen_map_vector(int vector)
604 {
605 	int xen_vector;
606 
607 	switch (vector) {
608 	case RESCHEDULE_VECTOR:
609 		xen_vector = XEN_RESCHEDULE_VECTOR;
610 		break;
611 	case CALL_FUNCTION_VECTOR:
612 		xen_vector = XEN_CALL_FUNCTION_VECTOR;
613 		break;
614 	case CALL_FUNCTION_SINGLE_VECTOR:
615 		xen_vector = XEN_CALL_FUNCTION_SINGLE_VECTOR;
616 		break;
617 	case IRQ_WORK_VECTOR:
618 		xen_vector = XEN_IRQ_WORK_VECTOR;
619 		break;
620 #ifdef CONFIG_X86_64
621 	case NMI_VECTOR:
622 	case APIC_DM_NMI: /* Some use that instead of NMI_VECTOR */
623 		xen_vector = XEN_NMI_VECTOR;
624 		break;
625 #endif
626 	default:
627 		xen_vector = -1;
628 		printk(KERN_ERR "xen: vector 0x%x is not implemented\n",
629 			vector);
630 	}
631 
632 	return xen_vector;
633 }
634 
635 void xen_send_IPI_mask(const struct cpumask *mask,
636 			      int vector)
637 {
638 	int xen_vector = xen_map_vector(vector);
639 
640 	if (xen_vector >= 0)
641 		__xen_send_IPI_mask(mask, xen_vector);
642 }
643 
644 void xen_send_IPI_all(int vector)
645 {
646 	int xen_vector = xen_map_vector(vector);
647 
648 	if (xen_vector >= 0)
649 		__xen_send_IPI_mask(cpu_online_mask, xen_vector);
650 }
651 
652 void xen_send_IPI_self(int vector)
653 {
654 	int xen_vector = xen_map_vector(vector);
655 
656 	if (xen_vector >= 0)
657 		xen_send_IPI_one(smp_processor_id(), xen_vector);
658 }
659 
660 void xen_send_IPI_mask_allbutself(const struct cpumask *mask,
661 				int vector)
662 {
663 	unsigned cpu;
664 	unsigned int this_cpu = smp_processor_id();
665 	int xen_vector = xen_map_vector(vector);
666 
667 	if (!(num_online_cpus() > 1) || (xen_vector < 0))
668 		return;
669 
670 	for_each_cpu_and(cpu, mask, cpu_online_mask) {
671 		if (this_cpu == cpu)
672 			continue;
673 
674 		xen_send_IPI_one(cpu, xen_vector);
675 	}
676 }
677 
678 void xen_send_IPI_allbutself(int vector)
679 {
680 	xen_send_IPI_mask_allbutself(cpu_online_mask, vector);
681 }
682 
683 static irqreturn_t xen_call_function_interrupt(int irq, void *dev_id)
684 {
685 	irq_enter();
686 	generic_smp_call_function_interrupt();
687 	inc_irq_stat(irq_call_count);
688 	irq_exit();
689 
690 	return IRQ_HANDLED;
691 }
692 
693 static irqreturn_t xen_call_function_single_interrupt(int irq, void *dev_id)
694 {
695 	irq_enter();
696 	generic_smp_call_function_single_interrupt();
697 	inc_irq_stat(irq_call_count);
698 	irq_exit();
699 
700 	return IRQ_HANDLED;
701 }
702 
703 static irqreturn_t xen_irq_work_interrupt(int irq, void *dev_id)
704 {
705 	irq_enter();
706 	irq_work_run();
707 	inc_irq_stat(apic_irq_work_irqs);
708 	irq_exit();
709 
710 	return IRQ_HANDLED;
711 }
712 
713 static const struct smp_ops xen_smp_ops __initconst = {
714 	.smp_prepare_boot_cpu = xen_smp_prepare_boot_cpu,
715 	.smp_prepare_cpus = xen_smp_prepare_cpus,
716 	.smp_cpus_done = xen_smp_cpus_done,
717 
718 	.cpu_up = xen_cpu_up,
719 	.cpu_die = xen_cpu_die,
720 	.cpu_disable = xen_cpu_disable,
721 	.play_dead = xen_play_dead,
722 
723 	.stop_other_cpus = xen_stop_other_cpus,
724 	.smp_send_reschedule = xen_smp_send_reschedule,
725 
726 	.send_call_func_ipi = xen_smp_send_call_function_ipi,
727 	.send_call_func_single_ipi = xen_smp_send_call_function_single_ipi,
728 };
729 
730 void __init xen_smp_init(void)
731 {
732 	smp_ops = xen_smp_ops;
733 	xen_fill_possible_map();
734 }
735 
736 static void __init xen_hvm_smp_prepare_cpus(unsigned int max_cpus)
737 {
738 	native_smp_prepare_cpus(max_cpus);
739 	WARN_ON(xen_smp_intr_init(0));
740 
741 	xen_init_lock_cpu(0);
742 }
743 
744 static int xen_hvm_cpu_up(unsigned int cpu, struct task_struct *tidle)
745 {
746 	int rc;
747 	/*
748 	 * xen_smp_intr_init() needs to run before native_cpu_up()
749 	 * so that IPI vectors are set up on the booting CPU before
750 	 * it is marked online in native_cpu_up().
751 	*/
752 	rc = xen_smp_intr_init(cpu);
753 	WARN_ON(rc);
754 	if (!rc)
755 		rc =  native_cpu_up(cpu, tidle);
756 
757 	/*
758 	 * We must initialize the slowpath CPU kicker _after_ the native
759 	 * path has executed. If we initialized it before none of the
760 	 * unlocker IPI kicks would reach the booting CPU as the booting
761 	 * CPU had not set itself 'online' in cpu_online_mask. That mask
762 	 * is checked when IPIs are sent (on HVM at least).
763 	 */
764 	xen_init_lock_cpu(cpu);
765 	return rc;
766 }
767 
768 static void xen_hvm_cpu_die(unsigned int cpu)
769 {
770 	xen_cpu_die(cpu);
771 	native_cpu_die(cpu);
772 }
773 
774 void __init xen_hvm_smp_init(void)
775 {
776 	if (!xen_have_vector_callback)
777 		return;
778 	smp_ops.smp_prepare_cpus = xen_hvm_smp_prepare_cpus;
779 	smp_ops.smp_send_reschedule = xen_smp_send_reschedule;
780 	smp_ops.cpu_up = xen_hvm_cpu_up;
781 	smp_ops.cpu_die = xen_hvm_cpu_die;
782 	smp_ops.send_call_func_ipi = xen_smp_send_call_function_ipi;
783 	smp_ops.send_call_func_single_ipi = xen_smp_send_call_function_single_ipi;
784 	smp_ops.smp_prepare_boot_cpu = xen_smp_prepare_boot_cpu;
785 }
786