xref: /openbmc/linux/arch/x86/xen/smp.c (revision 81d67439)
1 /*
2  * Xen SMP support
3  *
4  * This file implements the Xen versions of smp_ops.  SMP under Xen is
5  * very straightforward.  Bringing a CPU up is simply a matter of
6  * loading its initial context and setting it running.
7  *
8  * IPIs are handled through the Xen event mechanism.
9  *
10  * Because virtual CPUs can be scheduled onto any real CPU, there's no
11  * useful topology information for the kernel to make use of.  As a
12  * result, all CPUs are treated as if they're single-core and
13  * single-threaded.
14  */
15 #include <linux/sched.h>
16 #include <linux/err.h>
17 #include <linux/slab.h>
18 #include <linux/smp.h>
19 
20 #include <asm/paravirt.h>
21 #include <asm/desc.h>
22 #include <asm/pgtable.h>
23 #include <asm/cpu.h>
24 
25 #include <xen/interface/xen.h>
26 #include <xen/interface/vcpu.h>
27 
28 #include <asm/xen/interface.h>
29 #include <asm/xen/hypercall.h>
30 
31 #include <xen/xen.h>
32 #include <xen/page.h>
33 #include <xen/events.h>
34 
35 #include "xen-ops.h"
36 #include "mmu.h"
37 
38 cpumask_var_t xen_cpu_initialized_map;
39 
40 static DEFINE_PER_CPU(int, xen_resched_irq);
41 static DEFINE_PER_CPU(int, xen_callfunc_irq);
42 static DEFINE_PER_CPU(int, xen_callfuncsingle_irq);
43 static DEFINE_PER_CPU(int, xen_debug_irq) = -1;
44 
45 static irqreturn_t xen_call_function_interrupt(int irq, void *dev_id);
46 static irqreturn_t xen_call_function_single_interrupt(int irq, void *dev_id);
47 
48 /*
49  * Reschedule call back.
50  */
51 static irqreturn_t xen_reschedule_interrupt(int irq, void *dev_id)
52 {
53 	inc_irq_stat(irq_resched_count);
54 	scheduler_ipi();
55 
56 	return IRQ_HANDLED;
57 }
58 
59 static void __cpuinit cpu_bringup(void)
60 {
61 	int cpu = smp_processor_id();
62 
63 	cpu_init();
64 	touch_softlockup_watchdog();
65 	preempt_disable();
66 
67 	xen_enable_sysenter();
68 	xen_enable_syscall();
69 
70 	cpu = smp_processor_id();
71 	smp_store_cpu_info(cpu);
72 	cpu_data(cpu).x86_max_cores = 1;
73 	set_cpu_sibling_map(cpu);
74 
75 	xen_setup_cpu_clockevents();
76 
77 	set_cpu_online(cpu, true);
78 	percpu_write(cpu_state, CPU_ONLINE);
79 	wmb();
80 
81 	/* We can take interrupts now: we're officially "up". */
82 	local_irq_enable();
83 
84 	wmb();			/* make sure everything is out */
85 }
86 
87 static void __cpuinit cpu_bringup_and_idle(void)
88 {
89 	cpu_bringup();
90 	cpu_idle();
91 }
92 
93 static int xen_smp_intr_init(unsigned int cpu)
94 {
95 	int rc;
96 	const char *resched_name, *callfunc_name, *debug_name;
97 
98 	resched_name = kasprintf(GFP_KERNEL, "resched%d", cpu);
99 	rc = bind_ipi_to_irqhandler(XEN_RESCHEDULE_VECTOR,
100 				    cpu,
101 				    xen_reschedule_interrupt,
102 				    IRQF_DISABLED|IRQF_PERCPU|IRQF_NOBALANCING,
103 				    resched_name,
104 				    NULL);
105 	if (rc < 0)
106 		goto fail;
107 	per_cpu(xen_resched_irq, cpu) = rc;
108 
109 	callfunc_name = kasprintf(GFP_KERNEL, "callfunc%d", cpu);
110 	rc = bind_ipi_to_irqhandler(XEN_CALL_FUNCTION_VECTOR,
111 				    cpu,
112 				    xen_call_function_interrupt,
113 				    IRQF_DISABLED|IRQF_PERCPU|IRQF_NOBALANCING,
114 				    callfunc_name,
115 				    NULL);
116 	if (rc < 0)
117 		goto fail;
118 	per_cpu(xen_callfunc_irq, cpu) = rc;
119 
120 	debug_name = kasprintf(GFP_KERNEL, "debug%d", cpu);
121 	rc = bind_virq_to_irqhandler(VIRQ_DEBUG, cpu, xen_debug_interrupt,
122 				     IRQF_DISABLED | IRQF_PERCPU | IRQF_NOBALANCING,
123 				     debug_name, NULL);
124 	if (rc < 0)
125 		goto fail;
126 	per_cpu(xen_debug_irq, cpu) = rc;
127 
128 	callfunc_name = kasprintf(GFP_KERNEL, "callfuncsingle%d", cpu);
129 	rc = bind_ipi_to_irqhandler(XEN_CALL_FUNCTION_SINGLE_VECTOR,
130 				    cpu,
131 				    xen_call_function_single_interrupt,
132 				    IRQF_DISABLED|IRQF_PERCPU|IRQF_NOBALANCING,
133 				    callfunc_name,
134 				    NULL);
135 	if (rc < 0)
136 		goto fail;
137 	per_cpu(xen_callfuncsingle_irq, cpu) = rc;
138 
139 	return 0;
140 
141  fail:
142 	if (per_cpu(xen_resched_irq, cpu) >= 0)
143 		unbind_from_irqhandler(per_cpu(xen_resched_irq, cpu), NULL);
144 	if (per_cpu(xen_callfunc_irq, cpu) >= 0)
145 		unbind_from_irqhandler(per_cpu(xen_callfunc_irq, cpu), NULL);
146 	if (per_cpu(xen_debug_irq, cpu) >= 0)
147 		unbind_from_irqhandler(per_cpu(xen_debug_irq, cpu), NULL);
148 	if (per_cpu(xen_callfuncsingle_irq, cpu) >= 0)
149 		unbind_from_irqhandler(per_cpu(xen_callfuncsingle_irq, cpu),
150 				       NULL);
151 
152 	return rc;
153 }
154 
155 static void __init xen_fill_possible_map(void)
156 {
157 	int i, rc;
158 
159 	if (xen_initial_domain())
160 		return;
161 
162 	for (i = 0; i < nr_cpu_ids; i++) {
163 		rc = HYPERVISOR_vcpu_op(VCPUOP_is_up, i, NULL);
164 		if (rc >= 0) {
165 			num_processors++;
166 			set_cpu_possible(i, true);
167 		}
168 	}
169 }
170 
171 static void __init xen_filter_cpu_maps(void)
172 {
173 	int i, rc;
174 
175 	if (!xen_initial_domain())
176 		return;
177 
178 	num_processors = 0;
179 	disabled_cpus = 0;
180 	for (i = 0; i < nr_cpu_ids; i++) {
181 		rc = HYPERVISOR_vcpu_op(VCPUOP_is_up, i, NULL);
182 		if (rc >= 0) {
183 			num_processors++;
184 			set_cpu_possible(i, true);
185 		} else {
186 			set_cpu_possible(i, false);
187 			set_cpu_present(i, false);
188 		}
189 	}
190 }
191 
192 static void __init xen_smp_prepare_boot_cpu(void)
193 {
194 	BUG_ON(smp_processor_id() != 0);
195 	native_smp_prepare_boot_cpu();
196 
197 	/* We've switched to the "real" per-cpu gdt, so make sure the
198 	   old memory can be recycled */
199 	make_lowmem_page_readwrite(xen_initial_gdt);
200 
201 	xen_filter_cpu_maps();
202 	xen_setup_vcpu_info_placement();
203 }
204 
205 static void __init xen_smp_prepare_cpus(unsigned int max_cpus)
206 {
207 	unsigned cpu;
208 	unsigned int i;
209 
210 	xen_init_lock_cpu(0);
211 
212 	smp_store_cpu_info(0);
213 	cpu_data(0).x86_max_cores = 1;
214 
215 	for_each_possible_cpu(i) {
216 		zalloc_cpumask_var(&per_cpu(cpu_sibling_map, i), GFP_KERNEL);
217 		zalloc_cpumask_var(&per_cpu(cpu_core_map, i), GFP_KERNEL);
218 		zalloc_cpumask_var(&per_cpu(cpu_llc_shared_map, i), GFP_KERNEL);
219 	}
220 	set_cpu_sibling_map(0);
221 
222 	if (xen_smp_intr_init(0))
223 		BUG();
224 
225 	if (!alloc_cpumask_var(&xen_cpu_initialized_map, GFP_KERNEL))
226 		panic("could not allocate xen_cpu_initialized_map\n");
227 
228 	cpumask_copy(xen_cpu_initialized_map, cpumask_of(0));
229 
230 	/* Restrict the possible_map according to max_cpus. */
231 	while ((num_possible_cpus() > 1) && (num_possible_cpus() > max_cpus)) {
232 		for (cpu = nr_cpu_ids - 1; !cpu_possible(cpu); cpu--)
233 			continue;
234 		set_cpu_possible(cpu, false);
235 	}
236 
237 	for_each_possible_cpu (cpu) {
238 		struct task_struct *idle;
239 
240 		if (cpu == 0)
241 			continue;
242 
243 		idle = fork_idle(cpu);
244 		if (IS_ERR(idle))
245 			panic("failed fork for CPU %d", cpu);
246 
247 		set_cpu_present(cpu, true);
248 	}
249 }
250 
251 static int __cpuinit
252 cpu_initialize_context(unsigned int cpu, struct task_struct *idle)
253 {
254 	struct vcpu_guest_context *ctxt;
255 	struct desc_struct *gdt;
256 	unsigned long gdt_mfn;
257 
258 	if (cpumask_test_and_set_cpu(cpu, xen_cpu_initialized_map))
259 		return 0;
260 
261 	ctxt = kzalloc(sizeof(*ctxt), GFP_KERNEL);
262 	if (ctxt == NULL)
263 		return -ENOMEM;
264 
265 	gdt = get_cpu_gdt_table(cpu);
266 
267 	ctxt->flags = VGCF_IN_KERNEL;
268 	ctxt->user_regs.ds = __USER_DS;
269 	ctxt->user_regs.es = __USER_DS;
270 	ctxt->user_regs.ss = __KERNEL_DS;
271 #ifdef CONFIG_X86_32
272 	ctxt->user_regs.fs = __KERNEL_PERCPU;
273 	ctxt->user_regs.gs = __KERNEL_STACK_CANARY;
274 #else
275 	ctxt->gs_base_kernel = per_cpu_offset(cpu);
276 #endif
277 	ctxt->user_regs.eip = (unsigned long)cpu_bringup_and_idle;
278 	ctxt->user_regs.eflags = 0x1000; /* IOPL_RING1 */
279 
280 	memset(&ctxt->fpu_ctxt, 0, sizeof(ctxt->fpu_ctxt));
281 
282 	xen_copy_trap_info(ctxt->trap_ctxt);
283 
284 	ctxt->ldt_ents = 0;
285 
286 	BUG_ON((unsigned long)gdt & ~PAGE_MASK);
287 
288 	gdt_mfn = arbitrary_virt_to_mfn(gdt);
289 	make_lowmem_page_readonly(gdt);
290 	make_lowmem_page_readonly(mfn_to_virt(gdt_mfn));
291 
292 	ctxt->gdt_frames[0] = gdt_mfn;
293 	ctxt->gdt_ents      = GDT_ENTRIES;
294 
295 	ctxt->user_regs.cs = __KERNEL_CS;
296 	ctxt->user_regs.esp = idle->thread.sp0 - sizeof(struct pt_regs);
297 
298 	ctxt->kernel_ss = __KERNEL_DS;
299 	ctxt->kernel_sp = idle->thread.sp0;
300 
301 #ifdef CONFIG_X86_32
302 	ctxt->event_callback_cs     = __KERNEL_CS;
303 	ctxt->failsafe_callback_cs  = __KERNEL_CS;
304 #endif
305 	ctxt->event_callback_eip    = (unsigned long)xen_hypervisor_callback;
306 	ctxt->failsafe_callback_eip = (unsigned long)xen_failsafe_callback;
307 
308 	per_cpu(xen_cr3, cpu) = __pa(swapper_pg_dir);
309 	ctxt->ctrlreg[3] = xen_pfn_to_cr3(virt_to_mfn(swapper_pg_dir));
310 
311 	if (HYPERVISOR_vcpu_op(VCPUOP_initialise, cpu, ctxt))
312 		BUG();
313 
314 	kfree(ctxt);
315 	return 0;
316 }
317 
318 static int __cpuinit xen_cpu_up(unsigned int cpu)
319 {
320 	struct task_struct *idle = idle_task(cpu);
321 	int rc;
322 
323 	per_cpu(current_task, cpu) = idle;
324 #ifdef CONFIG_X86_32
325 	irq_ctx_init(cpu);
326 #else
327 	clear_tsk_thread_flag(idle, TIF_FORK);
328 	per_cpu(kernel_stack, cpu) =
329 		(unsigned long)task_stack_page(idle) -
330 		KERNEL_STACK_OFFSET + THREAD_SIZE;
331 #endif
332 	xen_setup_runstate_info(cpu);
333 	xen_setup_timer(cpu);
334 	xen_init_lock_cpu(cpu);
335 
336 	per_cpu(cpu_state, cpu) = CPU_UP_PREPARE;
337 
338 	/* make sure interrupts start blocked */
339 	per_cpu(xen_vcpu, cpu)->evtchn_upcall_mask = 1;
340 
341 	rc = cpu_initialize_context(cpu, idle);
342 	if (rc)
343 		return rc;
344 
345 	if (num_online_cpus() == 1)
346 		alternatives_smp_switch(1);
347 
348 	rc = xen_smp_intr_init(cpu);
349 	if (rc)
350 		return rc;
351 
352 	rc = HYPERVISOR_vcpu_op(VCPUOP_up, cpu, NULL);
353 	BUG_ON(rc);
354 
355 	while(per_cpu(cpu_state, cpu) != CPU_ONLINE) {
356 		HYPERVISOR_sched_op(SCHEDOP_yield, NULL);
357 		barrier();
358 	}
359 
360 	return 0;
361 }
362 
363 static void xen_smp_cpus_done(unsigned int max_cpus)
364 {
365 }
366 
367 #ifdef CONFIG_HOTPLUG_CPU
368 static int xen_cpu_disable(void)
369 {
370 	unsigned int cpu = smp_processor_id();
371 	if (cpu == 0)
372 		return -EBUSY;
373 
374 	cpu_disable_common();
375 
376 	load_cr3(swapper_pg_dir);
377 	return 0;
378 }
379 
380 static void xen_cpu_die(unsigned int cpu)
381 {
382 	while (HYPERVISOR_vcpu_op(VCPUOP_is_up, cpu, NULL)) {
383 		current->state = TASK_UNINTERRUPTIBLE;
384 		schedule_timeout(HZ/10);
385 	}
386 	unbind_from_irqhandler(per_cpu(xen_resched_irq, cpu), NULL);
387 	unbind_from_irqhandler(per_cpu(xen_callfunc_irq, cpu), NULL);
388 	unbind_from_irqhandler(per_cpu(xen_debug_irq, cpu), NULL);
389 	unbind_from_irqhandler(per_cpu(xen_callfuncsingle_irq, cpu), NULL);
390 	xen_uninit_lock_cpu(cpu);
391 	xen_teardown_timer(cpu);
392 
393 	if (num_online_cpus() == 1)
394 		alternatives_smp_switch(0);
395 }
396 
397 static void __cpuinit xen_play_dead(void) /* used only with HOTPLUG_CPU */
398 {
399 	play_dead_common();
400 	HYPERVISOR_vcpu_op(VCPUOP_down, smp_processor_id(), NULL);
401 	cpu_bringup();
402 }
403 
404 #else /* !CONFIG_HOTPLUG_CPU */
405 static int xen_cpu_disable(void)
406 {
407 	return -ENOSYS;
408 }
409 
410 static void xen_cpu_die(unsigned int cpu)
411 {
412 	BUG();
413 }
414 
415 static void xen_play_dead(void)
416 {
417 	BUG();
418 }
419 
420 #endif
421 static void stop_self(void *v)
422 {
423 	int cpu = smp_processor_id();
424 
425 	/* make sure we're not pinning something down */
426 	load_cr3(swapper_pg_dir);
427 	/* should set up a minimal gdt */
428 
429 	set_cpu_online(cpu, false);
430 
431 	HYPERVISOR_vcpu_op(VCPUOP_down, cpu, NULL);
432 	BUG();
433 }
434 
435 static void xen_stop_other_cpus(int wait)
436 {
437 	smp_call_function(stop_self, NULL, wait);
438 }
439 
440 static void xen_smp_send_reschedule(int cpu)
441 {
442 	xen_send_IPI_one(cpu, XEN_RESCHEDULE_VECTOR);
443 }
444 
445 static void xen_send_IPI_mask(const struct cpumask *mask,
446 			      enum ipi_vector vector)
447 {
448 	unsigned cpu;
449 
450 	for_each_cpu_and(cpu, mask, cpu_online_mask)
451 		xen_send_IPI_one(cpu, vector);
452 }
453 
454 static void xen_smp_send_call_function_ipi(const struct cpumask *mask)
455 {
456 	int cpu;
457 
458 	xen_send_IPI_mask(mask, XEN_CALL_FUNCTION_VECTOR);
459 
460 	/* Make sure other vcpus get a chance to run if they need to. */
461 	for_each_cpu(cpu, mask) {
462 		if (xen_vcpu_stolen(cpu)) {
463 			HYPERVISOR_sched_op(SCHEDOP_yield, NULL);
464 			break;
465 		}
466 	}
467 }
468 
469 static void xen_smp_send_call_function_single_ipi(int cpu)
470 {
471 	xen_send_IPI_mask(cpumask_of(cpu),
472 			  XEN_CALL_FUNCTION_SINGLE_VECTOR);
473 }
474 
475 static irqreturn_t xen_call_function_interrupt(int irq, void *dev_id)
476 {
477 	irq_enter();
478 	generic_smp_call_function_interrupt();
479 	inc_irq_stat(irq_call_count);
480 	irq_exit();
481 
482 	return IRQ_HANDLED;
483 }
484 
485 static irqreturn_t xen_call_function_single_interrupt(int irq, void *dev_id)
486 {
487 	irq_enter();
488 	generic_smp_call_function_single_interrupt();
489 	inc_irq_stat(irq_call_count);
490 	irq_exit();
491 
492 	return IRQ_HANDLED;
493 }
494 
495 static const struct smp_ops xen_smp_ops __initconst = {
496 	.smp_prepare_boot_cpu = xen_smp_prepare_boot_cpu,
497 	.smp_prepare_cpus = xen_smp_prepare_cpus,
498 	.smp_cpus_done = xen_smp_cpus_done,
499 
500 	.cpu_up = xen_cpu_up,
501 	.cpu_die = xen_cpu_die,
502 	.cpu_disable = xen_cpu_disable,
503 	.play_dead = xen_play_dead,
504 
505 	.stop_other_cpus = xen_stop_other_cpus,
506 	.smp_send_reschedule = xen_smp_send_reschedule,
507 
508 	.send_call_func_ipi = xen_smp_send_call_function_ipi,
509 	.send_call_func_single_ipi = xen_smp_send_call_function_single_ipi,
510 };
511 
512 void __init xen_smp_init(void)
513 {
514 	smp_ops = xen_smp_ops;
515 	xen_fill_possible_map();
516 	xen_init_spinlocks();
517 }
518 
519 static void __init xen_hvm_smp_prepare_cpus(unsigned int max_cpus)
520 {
521 	native_smp_prepare_cpus(max_cpus);
522 	WARN_ON(xen_smp_intr_init(0));
523 
524 	if (!xen_have_vector_callback)
525 		return;
526 	xen_init_lock_cpu(0);
527 	xen_init_spinlocks();
528 }
529 
530 static int __cpuinit xen_hvm_cpu_up(unsigned int cpu)
531 {
532 	int rc;
533 	rc = native_cpu_up(cpu);
534 	WARN_ON (xen_smp_intr_init(cpu));
535 	return rc;
536 }
537 
538 static void xen_hvm_cpu_die(unsigned int cpu)
539 {
540 	unbind_from_irqhandler(per_cpu(xen_resched_irq, cpu), NULL);
541 	unbind_from_irqhandler(per_cpu(xen_callfunc_irq, cpu), NULL);
542 	unbind_from_irqhandler(per_cpu(xen_debug_irq, cpu), NULL);
543 	unbind_from_irqhandler(per_cpu(xen_callfuncsingle_irq, cpu), NULL);
544 	native_cpu_die(cpu);
545 }
546 
547 void __init xen_hvm_smp_init(void)
548 {
549 	smp_ops.smp_prepare_cpus = xen_hvm_smp_prepare_cpus;
550 	smp_ops.smp_send_reschedule = xen_smp_send_reschedule;
551 	smp_ops.cpu_up = xen_hvm_cpu_up;
552 	smp_ops.cpu_die = xen_hvm_cpu_die;
553 	smp_ops.send_call_func_ipi = xen_smp_send_call_function_ipi;
554 	smp_ops.send_call_func_single_ipi = xen_smp_send_call_function_single_ipi;
555 }
556