xref: /openbmc/linux/arch/s390/kernel/smp.c (revision e2f1cf25)
1 /*
2  *  SMP related functions
3  *
4  *    Copyright IBM Corp. 1999, 2012
5  *    Author(s): Denis Joseph Barrow,
6  *		 Martin Schwidefsky <schwidefsky@de.ibm.com>,
7  *		 Heiko Carstens <heiko.carstens@de.ibm.com>,
8  *
9  *  based on other smp stuff by
10  *    (c) 1995 Alan Cox, CymruNET Ltd  <alan@cymru.net>
11  *    (c) 1998 Ingo Molnar
12  *
13  * The code outside of smp.c uses logical cpu numbers, only smp.c does
14  * the translation of logical to physical cpu ids. All new code that
15  * operates on physical cpu numbers needs to go into smp.c.
16  */
17 
18 #define KMSG_COMPONENT "cpu"
19 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
20 
21 #include <linux/workqueue.h>
22 #include <linux/module.h>
23 #include <linux/init.h>
24 #include <linux/mm.h>
25 #include <linux/err.h>
26 #include <linux/spinlock.h>
27 #include <linux/kernel_stat.h>
28 #include <linux/delay.h>
29 #include <linux/interrupt.h>
30 #include <linux/irqflags.h>
31 #include <linux/cpu.h>
32 #include <linux/slab.h>
33 #include <linux/crash_dump.h>
34 #include <linux/memblock.h>
35 #include <asm/asm-offsets.h>
36 #include <asm/switch_to.h>
37 #include <asm/facility.h>
38 #include <asm/ipl.h>
39 #include <asm/setup.h>
40 #include <asm/irq.h>
41 #include <asm/tlbflush.h>
42 #include <asm/vtimer.h>
43 #include <asm/lowcore.h>
44 #include <asm/sclp.h>
45 #include <asm/vdso.h>
46 #include <asm/debug.h>
47 #include <asm/os_info.h>
48 #include <asm/sigp.h>
49 #include <asm/idle.h>
50 #include "entry.h"
51 
52 enum {
53 	ec_schedule = 0,
54 	ec_call_function_single,
55 	ec_stop_cpu,
56 };
57 
58 enum {
59 	CPU_STATE_STANDBY,
60 	CPU_STATE_CONFIGURED,
61 };
62 
63 static DEFINE_PER_CPU(struct cpu *, cpu_device);
64 
65 struct pcpu {
66 	struct _lowcore *lowcore;	/* lowcore page(s) for the cpu */
67 	unsigned long ec_mask;		/* bit mask for ec_xxx functions */
68 	signed char state;		/* physical cpu state */
69 	signed char polarization;	/* physical polarization */
70 	u16 address;			/* physical cpu address */
71 };
72 
73 static u8 boot_core_type;
74 static struct pcpu pcpu_devices[NR_CPUS];
75 
76 unsigned int smp_cpu_mt_shift;
77 EXPORT_SYMBOL(smp_cpu_mt_shift);
78 
79 unsigned int smp_cpu_mtid;
80 EXPORT_SYMBOL(smp_cpu_mtid);
81 
82 static unsigned int smp_max_threads __initdata = -1U;
83 
84 static int __init early_nosmt(char *s)
85 {
86 	smp_max_threads = 1;
87 	return 0;
88 }
89 early_param("nosmt", early_nosmt);
90 
91 static int __init early_smt(char *s)
92 {
93 	get_option(&s, &smp_max_threads);
94 	return 0;
95 }
96 early_param("smt", early_smt);
97 
98 /*
99  * The smp_cpu_state_mutex must be held when changing the state or polarization
100  * member of a pcpu data structure within the pcpu_devices arreay.
101  */
102 DEFINE_MUTEX(smp_cpu_state_mutex);
103 
104 /*
105  * Signal processor helper functions.
106  */
107 static inline int __pcpu_sigp_relax(u16 addr, u8 order, unsigned long parm,
108 				    u32 *status)
109 {
110 	int cc;
111 
112 	while (1) {
113 		cc = __pcpu_sigp(addr, order, parm, NULL);
114 		if (cc != SIGP_CC_BUSY)
115 			return cc;
116 		cpu_relax();
117 	}
118 }
119 
120 static int pcpu_sigp_retry(struct pcpu *pcpu, u8 order, u32 parm)
121 {
122 	int cc, retry;
123 
124 	for (retry = 0; ; retry++) {
125 		cc = __pcpu_sigp(pcpu->address, order, parm, NULL);
126 		if (cc != SIGP_CC_BUSY)
127 			break;
128 		if (retry >= 3)
129 			udelay(10);
130 	}
131 	return cc;
132 }
133 
134 static inline int pcpu_stopped(struct pcpu *pcpu)
135 {
136 	u32 uninitialized_var(status);
137 
138 	if (__pcpu_sigp(pcpu->address, SIGP_SENSE,
139 			0, &status) != SIGP_CC_STATUS_STORED)
140 		return 0;
141 	return !!(status & (SIGP_STATUS_CHECK_STOP|SIGP_STATUS_STOPPED));
142 }
143 
144 static inline int pcpu_running(struct pcpu *pcpu)
145 {
146 	if (__pcpu_sigp(pcpu->address, SIGP_SENSE_RUNNING,
147 			0, NULL) != SIGP_CC_STATUS_STORED)
148 		return 1;
149 	/* Status stored condition code is equivalent to cpu not running. */
150 	return 0;
151 }
152 
153 /*
154  * Find struct pcpu by cpu address.
155  */
156 static struct pcpu *pcpu_find_address(const struct cpumask *mask, u16 address)
157 {
158 	int cpu;
159 
160 	for_each_cpu(cpu, mask)
161 		if (pcpu_devices[cpu].address == address)
162 			return pcpu_devices + cpu;
163 	return NULL;
164 }
165 
166 static void pcpu_ec_call(struct pcpu *pcpu, int ec_bit)
167 {
168 	int order;
169 
170 	if (test_and_set_bit(ec_bit, &pcpu->ec_mask))
171 		return;
172 	order = pcpu_running(pcpu) ? SIGP_EXTERNAL_CALL : SIGP_EMERGENCY_SIGNAL;
173 	pcpu_sigp_retry(pcpu, order, 0);
174 }
175 
176 #define ASYNC_FRAME_OFFSET (ASYNC_SIZE - STACK_FRAME_OVERHEAD - __PT_SIZE)
177 #define PANIC_FRAME_OFFSET (PAGE_SIZE - STACK_FRAME_OVERHEAD - __PT_SIZE)
178 
179 static int pcpu_alloc_lowcore(struct pcpu *pcpu, int cpu)
180 {
181 	unsigned long async_stack, panic_stack;
182 	struct _lowcore *lc;
183 
184 	if (pcpu != &pcpu_devices[0]) {
185 		pcpu->lowcore =	(struct _lowcore *)
186 			__get_free_pages(GFP_KERNEL | GFP_DMA, LC_ORDER);
187 		async_stack = __get_free_pages(GFP_KERNEL, ASYNC_ORDER);
188 		panic_stack = __get_free_page(GFP_KERNEL);
189 		if (!pcpu->lowcore || !panic_stack || !async_stack)
190 			goto out;
191 	} else {
192 		async_stack = pcpu->lowcore->async_stack - ASYNC_FRAME_OFFSET;
193 		panic_stack = pcpu->lowcore->panic_stack - PANIC_FRAME_OFFSET;
194 	}
195 	lc = pcpu->lowcore;
196 	memcpy(lc, &S390_lowcore, 512);
197 	memset((char *) lc + 512, 0, sizeof(*lc) - 512);
198 	lc->async_stack = async_stack + ASYNC_FRAME_OFFSET;
199 	lc->panic_stack = panic_stack + PANIC_FRAME_OFFSET;
200 	lc->cpu_nr = cpu;
201 	lc->spinlock_lockval = arch_spin_lockval(cpu);
202 	if (MACHINE_HAS_VX)
203 		lc->vector_save_area_addr =
204 			(unsigned long) &lc->vector_save_area;
205 	if (vdso_alloc_per_cpu(lc))
206 		goto out;
207 	lowcore_ptr[cpu] = lc;
208 	pcpu_sigp_retry(pcpu, SIGP_SET_PREFIX, (u32)(unsigned long) lc);
209 	return 0;
210 out:
211 	if (pcpu != &pcpu_devices[0]) {
212 		free_page(panic_stack);
213 		free_pages(async_stack, ASYNC_ORDER);
214 		free_pages((unsigned long) pcpu->lowcore, LC_ORDER);
215 	}
216 	return -ENOMEM;
217 }
218 
219 #ifdef CONFIG_HOTPLUG_CPU
220 
221 static void pcpu_free_lowcore(struct pcpu *pcpu)
222 {
223 	pcpu_sigp_retry(pcpu, SIGP_SET_PREFIX, 0);
224 	lowcore_ptr[pcpu - pcpu_devices] = NULL;
225 	vdso_free_per_cpu(pcpu->lowcore);
226 	if (pcpu == &pcpu_devices[0])
227 		return;
228 	free_page(pcpu->lowcore->panic_stack-PANIC_FRAME_OFFSET);
229 	free_pages(pcpu->lowcore->async_stack-ASYNC_FRAME_OFFSET, ASYNC_ORDER);
230 	free_pages((unsigned long) pcpu->lowcore, LC_ORDER);
231 }
232 
233 #endif /* CONFIG_HOTPLUG_CPU */
234 
235 static void pcpu_prepare_secondary(struct pcpu *pcpu, int cpu)
236 {
237 	struct _lowcore *lc = pcpu->lowcore;
238 
239 	if (MACHINE_HAS_TLB_LC)
240 		cpumask_set_cpu(cpu, &init_mm.context.cpu_attach_mask);
241 	cpumask_set_cpu(cpu, mm_cpumask(&init_mm));
242 	atomic_inc(&init_mm.context.attach_count);
243 	lc->cpu_nr = cpu;
244 	lc->spinlock_lockval = arch_spin_lockval(cpu);
245 	lc->percpu_offset = __per_cpu_offset[cpu];
246 	lc->kernel_asce = S390_lowcore.kernel_asce;
247 	lc->machine_flags = S390_lowcore.machine_flags;
248 	lc->user_timer = lc->system_timer = lc->steal_timer = 0;
249 	__ctl_store(lc->cregs_save_area, 0, 15);
250 	save_access_regs((unsigned int *) lc->access_regs_save_area);
251 	memcpy(lc->stfle_fac_list, S390_lowcore.stfle_fac_list,
252 	       MAX_FACILITY_BIT/8);
253 }
254 
255 static void pcpu_attach_task(struct pcpu *pcpu, struct task_struct *tsk)
256 {
257 	struct _lowcore *lc = pcpu->lowcore;
258 	struct thread_info *ti = task_thread_info(tsk);
259 
260 	lc->kernel_stack = (unsigned long) task_stack_page(tsk)
261 		+ THREAD_SIZE - STACK_FRAME_OVERHEAD - sizeof(struct pt_regs);
262 	lc->thread_info = (unsigned long) task_thread_info(tsk);
263 	lc->current_task = (unsigned long) tsk;
264 	lc->user_timer = ti->user_timer;
265 	lc->system_timer = ti->system_timer;
266 	lc->steal_timer = 0;
267 }
268 
269 static void pcpu_start_fn(struct pcpu *pcpu, void (*func)(void *), void *data)
270 {
271 	struct _lowcore *lc = pcpu->lowcore;
272 
273 	lc->restart_stack = lc->kernel_stack;
274 	lc->restart_fn = (unsigned long) func;
275 	lc->restart_data = (unsigned long) data;
276 	lc->restart_source = -1UL;
277 	pcpu_sigp_retry(pcpu, SIGP_RESTART, 0);
278 }
279 
280 /*
281  * Call function via PSW restart on pcpu and stop the current cpu.
282  */
283 static void pcpu_delegate(struct pcpu *pcpu, void (*func)(void *),
284 			  void *data, unsigned long stack)
285 {
286 	struct _lowcore *lc = lowcore_ptr[pcpu - pcpu_devices];
287 	unsigned long source_cpu = stap();
288 
289 	__load_psw_mask(PSW_KERNEL_BITS);
290 	if (pcpu->address == source_cpu)
291 		func(data);	/* should not return */
292 	/* Stop target cpu (if func returns this stops the current cpu). */
293 	pcpu_sigp_retry(pcpu, SIGP_STOP, 0);
294 	/* Restart func on the target cpu and stop the current cpu. */
295 	mem_assign_absolute(lc->restart_stack, stack);
296 	mem_assign_absolute(lc->restart_fn, (unsigned long) func);
297 	mem_assign_absolute(lc->restart_data, (unsigned long) data);
298 	mem_assign_absolute(lc->restart_source, source_cpu);
299 	asm volatile(
300 		"0:	sigp	0,%0,%2	# sigp restart to target cpu\n"
301 		"	brc	2,0b	# busy, try again\n"
302 		"1:	sigp	0,%1,%3	# sigp stop to current cpu\n"
303 		"	brc	2,1b	# busy, try again\n"
304 		: : "d" (pcpu->address), "d" (source_cpu),
305 		    "K" (SIGP_RESTART), "K" (SIGP_STOP)
306 		: "0", "1", "cc");
307 	for (;;) ;
308 }
309 
310 /*
311  * Enable additional logical cpus for multi-threading.
312  */
313 static int pcpu_set_smt(unsigned int mtid)
314 {
315 	register unsigned long reg1 asm ("1") = (unsigned long) mtid;
316 	int cc;
317 
318 	if (smp_cpu_mtid == mtid)
319 		return 0;
320 	asm volatile(
321 		"	sigp	%1,0,%2	# sigp set multi-threading\n"
322 		"	ipm	%0\n"
323 		"	srl	%0,28\n"
324 		: "=d" (cc) : "d" (reg1), "K" (SIGP_SET_MULTI_THREADING)
325 		: "cc");
326 	if (cc == 0) {
327 		smp_cpu_mtid = mtid;
328 		smp_cpu_mt_shift = 0;
329 		while (smp_cpu_mtid >= (1U << smp_cpu_mt_shift))
330 			smp_cpu_mt_shift++;
331 		pcpu_devices[0].address = stap();
332 	}
333 	return cc;
334 }
335 
336 /*
337  * Call function on an online CPU.
338  */
339 void smp_call_online_cpu(void (*func)(void *), void *data)
340 {
341 	struct pcpu *pcpu;
342 
343 	/* Use the current cpu if it is online. */
344 	pcpu = pcpu_find_address(cpu_online_mask, stap());
345 	if (!pcpu)
346 		/* Use the first online cpu. */
347 		pcpu = pcpu_devices + cpumask_first(cpu_online_mask);
348 	pcpu_delegate(pcpu, func, data, (unsigned long) restart_stack);
349 }
350 
351 /*
352  * Call function on the ipl CPU.
353  */
354 void smp_call_ipl_cpu(void (*func)(void *), void *data)
355 {
356 	pcpu_delegate(&pcpu_devices[0], func, data,
357 		      pcpu_devices->lowcore->panic_stack -
358 		      PANIC_FRAME_OFFSET + PAGE_SIZE);
359 }
360 
361 int smp_find_processor_id(u16 address)
362 {
363 	int cpu;
364 
365 	for_each_present_cpu(cpu)
366 		if (pcpu_devices[cpu].address == address)
367 			return cpu;
368 	return -1;
369 }
370 
371 int smp_vcpu_scheduled(int cpu)
372 {
373 	return pcpu_running(pcpu_devices + cpu);
374 }
375 
376 void smp_yield_cpu(int cpu)
377 {
378 	if (MACHINE_HAS_DIAG9C)
379 		asm volatile("diag %0,0,0x9c"
380 			     : : "d" (pcpu_devices[cpu].address));
381 	else if (MACHINE_HAS_DIAG44)
382 		asm volatile("diag 0,0,0x44");
383 }
384 
385 /*
386  * Send cpus emergency shutdown signal. This gives the cpus the
387  * opportunity to complete outstanding interrupts.
388  */
389 static void smp_emergency_stop(cpumask_t *cpumask)
390 {
391 	u64 end;
392 	int cpu;
393 
394 	end = get_tod_clock() + (1000000UL << 12);
395 	for_each_cpu(cpu, cpumask) {
396 		struct pcpu *pcpu = pcpu_devices + cpu;
397 		set_bit(ec_stop_cpu, &pcpu->ec_mask);
398 		while (__pcpu_sigp(pcpu->address, SIGP_EMERGENCY_SIGNAL,
399 				   0, NULL) == SIGP_CC_BUSY &&
400 		       get_tod_clock() < end)
401 			cpu_relax();
402 	}
403 	while (get_tod_clock() < end) {
404 		for_each_cpu(cpu, cpumask)
405 			if (pcpu_stopped(pcpu_devices + cpu))
406 				cpumask_clear_cpu(cpu, cpumask);
407 		if (cpumask_empty(cpumask))
408 			break;
409 		cpu_relax();
410 	}
411 }
412 
413 /*
414  * Stop all cpus but the current one.
415  */
416 void smp_send_stop(void)
417 {
418 	cpumask_t cpumask;
419 	int cpu;
420 
421 	/* Disable all interrupts/machine checks */
422 	__load_psw_mask(PSW_KERNEL_BITS | PSW_MASK_DAT);
423 	trace_hardirqs_off();
424 
425 	debug_set_critical();
426 	cpumask_copy(&cpumask, cpu_online_mask);
427 	cpumask_clear_cpu(smp_processor_id(), &cpumask);
428 
429 	if (oops_in_progress)
430 		smp_emergency_stop(&cpumask);
431 
432 	/* stop all processors */
433 	for_each_cpu(cpu, &cpumask) {
434 		struct pcpu *pcpu = pcpu_devices + cpu;
435 		pcpu_sigp_retry(pcpu, SIGP_STOP, 0);
436 		while (!pcpu_stopped(pcpu))
437 			cpu_relax();
438 	}
439 }
440 
441 /*
442  * This is the main routine where commands issued by other
443  * cpus are handled.
444  */
445 static void smp_handle_ext_call(void)
446 {
447 	unsigned long bits;
448 
449 	/* handle bit signal external calls */
450 	bits = xchg(&pcpu_devices[smp_processor_id()].ec_mask, 0);
451 	if (test_bit(ec_stop_cpu, &bits))
452 		smp_stop_cpu();
453 	if (test_bit(ec_schedule, &bits))
454 		scheduler_ipi();
455 	if (test_bit(ec_call_function_single, &bits))
456 		generic_smp_call_function_single_interrupt();
457 }
458 
459 static void do_ext_call_interrupt(struct ext_code ext_code,
460 				  unsigned int param32, unsigned long param64)
461 {
462 	inc_irq_stat(ext_code.code == 0x1202 ? IRQEXT_EXC : IRQEXT_EMS);
463 	smp_handle_ext_call();
464 }
465 
466 void arch_send_call_function_ipi_mask(const struct cpumask *mask)
467 {
468 	int cpu;
469 
470 	for_each_cpu(cpu, mask)
471 		pcpu_ec_call(pcpu_devices + cpu, ec_call_function_single);
472 }
473 
474 void arch_send_call_function_single_ipi(int cpu)
475 {
476 	pcpu_ec_call(pcpu_devices + cpu, ec_call_function_single);
477 }
478 
479 /*
480  * this function sends a 'reschedule' IPI to another CPU.
481  * it goes straight through and wastes no time serializing
482  * anything. Worst case is that we lose a reschedule ...
483  */
484 void smp_send_reschedule(int cpu)
485 {
486 	pcpu_ec_call(pcpu_devices + cpu, ec_schedule);
487 }
488 
489 /*
490  * parameter area for the set/clear control bit callbacks
491  */
492 struct ec_creg_mask_parms {
493 	unsigned long orval;
494 	unsigned long andval;
495 	int cr;
496 };
497 
498 /*
499  * callback for setting/clearing control bits
500  */
501 static void smp_ctl_bit_callback(void *info)
502 {
503 	struct ec_creg_mask_parms *pp = info;
504 	unsigned long cregs[16];
505 
506 	__ctl_store(cregs, 0, 15);
507 	cregs[pp->cr] = (cregs[pp->cr] & pp->andval) | pp->orval;
508 	__ctl_load(cregs, 0, 15);
509 }
510 
511 /*
512  * Set a bit in a control register of all cpus
513  */
514 void smp_ctl_set_bit(int cr, int bit)
515 {
516 	struct ec_creg_mask_parms parms = { 1UL << bit, -1UL, cr };
517 
518 	on_each_cpu(smp_ctl_bit_callback, &parms, 1);
519 }
520 EXPORT_SYMBOL(smp_ctl_set_bit);
521 
522 /*
523  * Clear a bit in a control register of all cpus
524  */
525 void smp_ctl_clear_bit(int cr, int bit)
526 {
527 	struct ec_creg_mask_parms parms = { 0, ~(1UL << bit), cr };
528 
529 	on_each_cpu(smp_ctl_bit_callback, &parms, 1);
530 }
531 EXPORT_SYMBOL(smp_ctl_clear_bit);
532 
533 #ifdef CONFIG_CRASH_DUMP
534 
535 static void __smp_store_cpu_state(struct save_area_ext *sa_ext, u16 address,
536 				  int is_boot_cpu)
537 {
538 	void *lc = (void *)(unsigned long) store_prefix();
539 	unsigned long vx_sa;
540 
541 	if (is_boot_cpu) {
542 		/* Copy the registers of the boot CPU. */
543 		copy_oldmem_page(1, (void *) &sa_ext->sa, sizeof(sa_ext->sa),
544 				 SAVE_AREA_BASE - PAGE_SIZE, 0);
545 		if (MACHINE_HAS_VX)
546 			save_vx_regs_safe(sa_ext->vx_regs);
547 		return;
548 	}
549 	/* Get the registers of a non-boot cpu. */
550 	__pcpu_sigp_relax(address, SIGP_STOP_AND_STORE_STATUS, 0, NULL);
551 	memcpy_real(&sa_ext->sa, lc + SAVE_AREA_BASE, sizeof(sa_ext->sa));
552 	if (!MACHINE_HAS_VX)
553 		return;
554 	/* Get the VX registers */
555 	vx_sa = memblock_alloc(PAGE_SIZE, PAGE_SIZE);
556 	if (!vx_sa)
557 		panic("could not allocate memory for VX save area\n");
558 	__pcpu_sigp_relax(address, SIGP_STORE_ADDITIONAL_STATUS, vx_sa, NULL);
559 	memcpy(sa_ext->vx_regs, (void *) vx_sa, sizeof(sa_ext->vx_regs));
560 	memblock_free(vx_sa, PAGE_SIZE);
561 }
562 
563 int smp_store_status(int cpu)
564 {
565 	unsigned long vx_sa;
566 	struct pcpu *pcpu;
567 
568 	pcpu = pcpu_devices + cpu;
569 	if (__pcpu_sigp_relax(pcpu->address, SIGP_STOP_AND_STORE_STATUS,
570 			      0, NULL) != SIGP_CC_ORDER_CODE_ACCEPTED)
571 		return -EIO;
572 	if (!MACHINE_HAS_VX)
573 		return 0;
574 	vx_sa = __pa(pcpu->lowcore->vector_save_area_addr);
575 	__pcpu_sigp_relax(pcpu->address, SIGP_STORE_ADDITIONAL_STATUS,
576 			  vx_sa, NULL);
577 	return 0;
578 }
579 
580 #endif /* CONFIG_CRASH_DUMP */
581 
582 /*
583  * Collect CPU state of the previous, crashed system.
584  * There are four cases:
585  * 1) standard zfcp dump
586  *    condition: OLDMEM_BASE == NULL && ipl_info.type == IPL_TYPE_FCP_DUMP
587  *    The state for all CPUs except the boot CPU needs to be collected
588  *    with sigp stop-and-store-status. The boot CPU state is located in
589  *    the absolute lowcore of the memory stored in the HSA. The zcore code
590  *    will allocate the save area and copy the boot CPU state from the HSA.
591  * 2) stand-alone kdump for SCSI (zfcp dump with swapped memory)
592  *    condition: OLDMEM_BASE != NULL && ipl_info.type == IPL_TYPE_FCP_DUMP
593  *    The state for all CPUs except the boot CPU needs to be collected
594  *    with sigp stop-and-store-status. The firmware or the boot-loader
595  *    stored the registers of the boot CPU in the absolute lowcore in the
596  *    memory of the old system.
597  * 3) kdump and the old kernel did not store the CPU state,
598  *    or stand-alone kdump for DASD
599  *    condition: OLDMEM_BASE != NULL && !is_kdump_kernel()
600  *    The state for all CPUs except the boot CPU needs to be collected
601  *    with sigp stop-and-store-status. The kexec code or the boot-loader
602  *    stored the registers of the boot CPU in the memory of the old system.
603  * 4) kdump and the old kernel stored the CPU state
604  *    condition: OLDMEM_BASE != NULL && is_kdump_kernel()
605  *    The state of all CPUs is stored in ELF sections in the memory of the
606  *    old system. The ELF sections are picked up by the crash_dump code
607  *    via elfcorehdr_addr.
608  */
609 void __init smp_save_dump_cpus(void)
610 {
611 #ifdef CONFIG_CRASH_DUMP
612 	int addr, cpu, boot_cpu_addr, max_cpu_addr;
613 	struct save_area_ext *sa_ext;
614 	bool is_boot_cpu;
615 
616 	if (is_kdump_kernel())
617 		/* Previous system stored the CPU states. Nothing to do. */
618 		return;
619 	if (!(OLDMEM_BASE || ipl_info.type == IPL_TYPE_FCP_DUMP))
620 		/* No previous system present, normal boot. */
621 		return;
622 	/* Set multi-threading state to the previous system. */
623 	pcpu_set_smt(sclp.mtid_prev);
624 	max_cpu_addr = SCLP_MAX_CORES << sclp.mtid_prev;
625 	for (cpu = 0, addr = 0; addr <= max_cpu_addr; addr++) {
626 		if (__pcpu_sigp_relax(addr, SIGP_SENSE, 0, NULL) ==
627 		    SIGP_CC_NOT_OPERATIONAL)
628 			continue;
629 		cpu += 1;
630 	}
631 	dump_save_areas.areas = (void *)memblock_alloc(sizeof(void *) * cpu, 8);
632 	dump_save_areas.count = cpu;
633 	boot_cpu_addr = stap();
634 	for (cpu = 0, addr = 0; addr <= max_cpu_addr; addr++) {
635 		if (__pcpu_sigp_relax(addr, SIGP_SENSE, 0, NULL) ==
636 		    SIGP_CC_NOT_OPERATIONAL)
637 			continue;
638 		sa_ext = (void *) memblock_alloc(sizeof(*sa_ext), 8);
639 		dump_save_areas.areas[cpu] = sa_ext;
640 		if (!sa_ext)
641 			panic("could not allocate memory for save area\n");
642 		is_boot_cpu = (addr == boot_cpu_addr);
643 		cpu += 1;
644 		if (is_boot_cpu && !OLDMEM_BASE)
645 			/* Skip boot CPU for standard zfcp dump. */
646 			continue;
647 		/* Get state for this CPU. */
648 		__smp_store_cpu_state(sa_ext, addr, is_boot_cpu);
649 	}
650 	diag308_reset();
651 	pcpu_set_smt(0);
652 #endif /* CONFIG_CRASH_DUMP */
653 }
654 
655 void smp_cpu_set_polarization(int cpu, int val)
656 {
657 	pcpu_devices[cpu].polarization = val;
658 }
659 
660 int smp_cpu_get_polarization(int cpu)
661 {
662 	return pcpu_devices[cpu].polarization;
663 }
664 
665 static struct sclp_core_info *smp_get_core_info(void)
666 {
667 	static int use_sigp_detection;
668 	struct sclp_core_info *info;
669 	int address;
670 
671 	info = kzalloc(sizeof(*info), GFP_KERNEL);
672 	if (info && (use_sigp_detection || sclp_get_core_info(info))) {
673 		use_sigp_detection = 1;
674 		for (address = 0;
675 		     address < (SCLP_MAX_CORES << smp_cpu_mt_shift);
676 		     address += (1U << smp_cpu_mt_shift)) {
677 			if (__pcpu_sigp_relax(address, SIGP_SENSE, 0, NULL) ==
678 			    SIGP_CC_NOT_OPERATIONAL)
679 				continue;
680 			info->core[info->configured].core_id =
681 				address >> smp_cpu_mt_shift;
682 			info->configured++;
683 		}
684 		info->combined = info->configured;
685 	}
686 	return info;
687 }
688 
689 static int smp_add_present_cpu(int cpu);
690 
691 static int __smp_rescan_cpus(struct sclp_core_info *info, int sysfs_add)
692 {
693 	struct pcpu *pcpu;
694 	cpumask_t avail;
695 	int cpu, nr, i, j;
696 	u16 address;
697 
698 	nr = 0;
699 	cpumask_xor(&avail, cpu_possible_mask, cpu_present_mask);
700 	cpu = cpumask_first(&avail);
701 	for (i = 0; (i < info->combined) && (cpu < nr_cpu_ids); i++) {
702 		if (sclp.has_core_type && info->core[i].type != boot_core_type)
703 			continue;
704 		address = info->core[i].core_id << smp_cpu_mt_shift;
705 		for (j = 0; j <= smp_cpu_mtid; j++) {
706 			if (pcpu_find_address(cpu_present_mask, address + j))
707 				continue;
708 			pcpu = pcpu_devices + cpu;
709 			pcpu->address = address + j;
710 			pcpu->state =
711 				(cpu >= info->configured*(smp_cpu_mtid + 1)) ?
712 				CPU_STATE_STANDBY : CPU_STATE_CONFIGURED;
713 			smp_cpu_set_polarization(cpu, POLARIZATION_UNKNOWN);
714 			set_cpu_present(cpu, true);
715 			if (sysfs_add && smp_add_present_cpu(cpu) != 0)
716 				set_cpu_present(cpu, false);
717 			else
718 				nr++;
719 			cpu = cpumask_next(cpu, &avail);
720 			if (cpu >= nr_cpu_ids)
721 				break;
722 		}
723 	}
724 	return nr;
725 }
726 
727 static void __init smp_detect_cpus(void)
728 {
729 	unsigned int cpu, mtid, c_cpus, s_cpus;
730 	struct sclp_core_info *info;
731 	u16 address;
732 
733 	/* Get CPU information */
734 	info = smp_get_core_info();
735 	if (!info)
736 		panic("smp_detect_cpus failed to allocate memory\n");
737 
738 	/* Find boot CPU type */
739 	if (sclp.has_core_type) {
740 		address = stap();
741 		for (cpu = 0; cpu < info->combined; cpu++)
742 			if (info->core[cpu].core_id == address) {
743 				/* The boot cpu dictates the cpu type. */
744 				boot_core_type = info->core[cpu].type;
745 				break;
746 			}
747 		if (cpu >= info->combined)
748 			panic("Could not find boot CPU type");
749 	}
750 
751 	/* Set multi-threading state for the current system */
752 	mtid = boot_core_type ? sclp.mtid : sclp.mtid_cp;
753 	mtid = (mtid < smp_max_threads) ? mtid : smp_max_threads - 1;
754 	pcpu_set_smt(mtid);
755 
756 	/* Print number of CPUs */
757 	c_cpus = s_cpus = 0;
758 	for (cpu = 0; cpu < info->combined; cpu++) {
759 		if (sclp.has_core_type &&
760 		    info->core[cpu].type != boot_core_type)
761 			continue;
762 		if (cpu < info->configured)
763 			c_cpus += smp_cpu_mtid + 1;
764 		else
765 			s_cpus += smp_cpu_mtid + 1;
766 	}
767 	pr_info("%d configured CPUs, %d standby CPUs\n", c_cpus, s_cpus);
768 
769 	/* Add CPUs present at boot */
770 	get_online_cpus();
771 	__smp_rescan_cpus(info, 0);
772 	put_online_cpus();
773 	kfree(info);
774 }
775 
776 /*
777  *	Activate a secondary processor.
778  */
779 static void smp_start_secondary(void *cpuvoid)
780 {
781 	S390_lowcore.last_update_clock = get_tod_clock();
782 	S390_lowcore.restart_stack = (unsigned long) restart_stack;
783 	S390_lowcore.restart_fn = (unsigned long) do_restart;
784 	S390_lowcore.restart_data = 0;
785 	S390_lowcore.restart_source = -1UL;
786 	restore_access_regs(S390_lowcore.access_regs_save_area);
787 	__ctl_load(S390_lowcore.cregs_save_area, 0, 15);
788 	__load_psw_mask(PSW_KERNEL_BITS | PSW_MASK_DAT);
789 	cpu_init();
790 	preempt_disable();
791 	init_cpu_timer();
792 	vtime_init();
793 	pfault_init();
794 	notify_cpu_starting(smp_processor_id());
795 	set_cpu_online(smp_processor_id(), true);
796 	inc_irq_stat(CPU_RST);
797 	local_irq_enable();
798 	cpu_startup_entry(CPUHP_ONLINE);
799 }
800 
801 /* Upping and downing of CPUs */
802 int __cpu_up(unsigned int cpu, struct task_struct *tidle)
803 {
804 	struct pcpu *pcpu;
805 	int base, i, rc;
806 
807 	pcpu = pcpu_devices + cpu;
808 	if (pcpu->state != CPU_STATE_CONFIGURED)
809 		return -EIO;
810 	base = cpu - (cpu % (smp_cpu_mtid + 1));
811 	for (i = 0; i <= smp_cpu_mtid; i++) {
812 		if (base + i < nr_cpu_ids)
813 			if (cpu_online(base + i))
814 				break;
815 	}
816 	/*
817 	 * If this is the first CPU of the core to get online
818 	 * do an initial CPU reset.
819 	 */
820 	if (i > smp_cpu_mtid &&
821 	    pcpu_sigp_retry(pcpu_devices + base, SIGP_INITIAL_CPU_RESET, 0) !=
822 	    SIGP_CC_ORDER_CODE_ACCEPTED)
823 		return -EIO;
824 
825 	rc = pcpu_alloc_lowcore(pcpu, cpu);
826 	if (rc)
827 		return rc;
828 	pcpu_prepare_secondary(pcpu, cpu);
829 	pcpu_attach_task(pcpu, tidle);
830 	pcpu_start_fn(pcpu, smp_start_secondary, NULL);
831 	/* Wait until cpu puts itself in the online & active maps */
832 	while (!cpu_online(cpu) || !cpu_active(cpu))
833 		cpu_relax();
834 	return 0;
835 }
836 
837 static unsigned int setup_possible_cpus __initdata;
838 
839 static int __init _setup_possible_cpus(char *s)
840 {
841 	get_option(&s, &setup_possible_cpus);
842 	return 0;
843 }
844 early_param("possible_cpus", _setup_possible_cpus);
845 
846 #ifdef CONFIG_HOTPLUG_CPU
847 
848 int __cpu_disable(void)
849 {
850 	unsigned long cregs[16];
851 
852 	/* Handle possible pending IPIs */
853 	smp_handle_ext_call();
854 	set_cpu_online(smp_processor_id(), false);
855 	/* Disable pseudo page faults on this cpu. */
856 	pfault_fini();
857 	/* Disable interrupt sources via control register. */
858 	__ctl_store(cregs, 0, 15);
859 	cregs[0]  &= ~0x0000ee70UL;	/* disable all external interrupts */
860 	cregs[6]  &= ~0xff000000UL;	/* disable all I/O interrupts */
861 	cregs[14] &= ~0x1f000000UL;	/* disable most machine checks */
862 	__ctl_load(cregs, 0, 15);
863 	clear_cpu_flag(CIF_NOHZ_DELAY);
864 	return 0;
865 }
866 
867 void __cpu_die(unsigned int cpu)
868 {
869 	struct pcpu *pcpu;
870 
871 	/* Wait until target cpu is down */
872 	pcpu = pcpu_devices + cpu;
873 	while (!pcpu_stopped(pcpu))
874 		cpu_relax();
875 	pcpu_free_lowcore(pcpu);
876 	atomic_dec(&init_mm.context.attach_count);
877 	cpumask_clear_cpu(cpu, mm_cpumask(&init_mm));
878 	if (MACHINE_HAS_TLB_LC)
879 		cpumask_clear_cpu(cpu, &init_mm.context.cpu_attach_mask);
880 }
881 
882 void __noreturn cpu_die(void)
883 {
884 	idle_task_exit();
885 	pcpu_sigp_retry(pcpu_devices + smp_processor_id(), SIGP_STOP, 0);
886 	for (;;) ;
887 }
888 
889 #endif /* CONFIG_HOTPLUG_CPU */
890 
891 void __init smp_fill_possible_mask(void)
892 {
893 	unsigned int possible, sclp_max, cpu;
894 
895 	sclp_max = max(sclp.mtid, sclp.mtid_cp) + 1;
896 	sclp_max = min(smp_max_threads, sclp_max);
897 	sclp_max = sclp.max_cores * sclp_max ?: nr_cpu_ids;
898 	possible = setup_possible_cpus ?: nr_cpu_ids;
899 	possible = min(possible, sclp_max);
900 	for (cpu = 0; cpu < possible && cpu < nr_cpu_ids; cpu++)
901 		set_cpu_possible(cpu, true);
902 }
903 
904 void __init smp_prepare_cpus(unsigned int max_cpus)
905 {
906 	/* request the 0x1201 emergency signal external interrupt */
907 	if (register_external_irq(EXT_IRQ_EMERGENCY_SIG, do_ext_call_interrupt))
908 		panic("Couldn't request external interrupt 0x1201");
909 	/* request the 0x1202 external call external interrupt */
910 	if (register_external_irq(EXT_IRQ_EXTERNAL_CALL, do_ext_call_interrupt))
911 		panic("Couldn't request external interrupt 0x1202");
912 	smp_detect_cpus();
913 }
914 
915 void __init smp_prepare_boot_cpu(void)
916 {
917 	struct pcpu *pcpu = pcpu_devices;
918 
919 	pcpu->state = CPU_STATE_CONFIGURED;
920 	pcpu->address = stap();
921 	pcpu->lowcore = (struct _lowcore *)(unsigned long) store_prefix();
922 	S390_lowcore.percpu_offset = __per_cpu_offset[0];
923 	smp_cpu_set_polarization(0, POLARIZATION_UNKNOWN);
924 	set_cpu_present(0, true);
925 	set_cpu_online(0, true);
926 }
927 
928 void __init smp_cpus_done(unsigned int max_cpus)
929 {
930 }
931 
932 void __init smp_setup_processor_id(void)
933 {
934 	S390_lowcore.cpu_nr = 0;
935 	S390_lowcore.spinlock_lockval = arch_spin_lockval(0);
936 }
937 
938 /*
939  * the frequency of the profiling timer can be changed
940  * by writing a multiplier value into /proc/profile.
941  *
942  * usually you want to run this on all CPUs ;)
943  */
944 int setup_profiling_timer(unsigned int multiplier)
945 {
946 	return 0;
947 }
948 
949 #ifdef CONFIG_HOTPLUG_CPU
950 static ssize_t cpu_configure_show(struct device *dev,
951 				  struct device_attribute *attr, char *buf)
952 {
953 	ssize_t count;
954 
955 	mutex_lock(&smp_cpu_state_mutex);
956 	count = sprintf(buf, "%d\n", pcpu_devices[dev->id].state);
957 	mutex_unlock(&smp_cpu_state_mutex);
958 	return count;
959 }
960 
961 static ssize_t cpu_configure_store(struct device *dev,
962 				   struct device_attribute *attr,
963 				   const char *buf, size_t count)
964 {
965 	struct pcpu *pcpu;
966 	int cpu, val, rc, i;
967 	char delim;
968 
969 	if (sscanf(buf, "%d %c", &val, &delim) != 1)
970 		return -EINVAL;
971 	if (val != 0 && val != 1)
972 		return -EINVAL;
973 	get_online_cpus();
974 	mutex_lock(&smp_cpu_state_mutex);
975 	rc = -EBUSY;
976 	/* disallow configuration changes of online cpus and cpu 0 */
977 	cpu = dev->id;
978 	cpu -= cpu % (smp_cpu_mtid + 1);
979 	if (cpu == 0)
980 		goto out;
981 	for (i = 0; i <= smp_cpu_mtid; i++)
982 		if (cpu_online(cpu + i))
983 			goto out;
984 	pcpu = pcpu_devices + cpu;
985 	rc = 0;
986 	switch (val) {
987 	case 0:
988 		if (pcpu->state != CPU_STATE_CONFIGURED)
989 			break;
990 		rc = sclp_core_deconfigure(pcpu->address >> smp_cpu_mt_shift);
991 		if (rc)
992 			break;
993 		for (i = 0; i <= smp_cpu_mtid; i++) {
994 			if (cpu + i >= nr_cpu_ids || !cpu_present(cpu + i))
995 				continue;
996 			pcpu[i].state = CPU_STATE_STANDBY;
997 			smp_cpu_set_polarization(cpu + i,
998 						 POLARIZATION_UNKNOWN);
999 		}
1000 		topology_expect_change();
1001 		break;
1002 	case 1:
1003 		if (pcpu->state != CPU_STATE_STANDBY)
1004 			break;
1005 		rc = sclp_core_configure(pcpu->address >> smp_cpu_mt_shift);
1006 		if (rc)
1007 			break;
1008 		for (i = 0; i <= smp_cpu_mtid; i++) {
1009 			if (cpu + i >= nr_cpu_ids || !cpu_present(cpu + i))
1010 				continue;
1011 			pcpu[i].state = CPU_STATE_CONFIGURED;
1012 			smp_cpu_set_polarization(cpu + i,
1013 						 POLARIZATION_UNKNOWN);
1014 		}
1015 		topology_expect_change();
1016 		break;
1017 	default:
1018 		break;
1019 	}
1020 out:
1021 	mutex_unlock(&smp_cpu_state_mutex);
1022 	put_online_cpus();
1023 	return rc ? rc : count;
1024 }
1025 static DEVICE_ATTR(configure, 0644, cpu_configure_show, cpu_configure_store);
1026 #endif /* CONFIG_HOTPLUG_CPU */
1027 
1028 static ssize_t show_cpu_address(struct device *dev,
1029 				struct device_attribute *attr, char *buf)
1030 {
1031 	return sprintf(buf, "%d\n", pcpu_devices[dev->id].address);
1032 }
1033 static DEVICE_ATTR(address, 0444, show_cpu_address, NULL);
1034 
1035 static struct attribute *cpu_common_attrs[] = {
1036 #ifdef CONFIG_HOTPLUG_CPU
1037 	&dev_attr_configure.attr,
1038 #endif
1039 	&dev_attr_address.attr,
1040 	NULL,
1041 };
1042 
1043 static struct attribute_group cpu_common_attr_group = {
1044 	.attrs = cpu_common_attrs,
1045 };
1046 
1047 static struct attribute *cpu_online_attrs[] = {
1048 	&dev_attr_idle_count.attr,
1049 	&dev_attr_idle_time_us.attr,
1050 	NULL,
1051 };
1052 
1053 static struct attribute_group cpu_online_attr_group = {
1054 	.attrs = cpu_online_attrs,
1055 };
1056 
1057 static int smp_cpu_notify(struct notifier_block *self, unsigned long action,
1058 			  void *hcpu)
1059 {
1060 	unsigned int cpu = (unsigned int)(long)hcpu;
1061 	struct device *s = &per_cpu(cpu_device, cpu)->dev;
1062 	int err = 0;
1063 
1064 	switch (action & ~CPU_TASKS_FROZEN) {
1065 	case CPU_ONLINE:
1066 		err = sysfs_create_group(&s->kobj, &cpu_online_attr_group);
1067 		break;
1068 	case CPU_DEAD:
1069 		sysfs_remove_group(&s->kobj, &cpu_online_attr_group);
1070 		break;
1071 	}
1072 	return notifier_from_errno(err);
1073 }
1074 
1075 static int smp_add_present_cpu(int cpu)
1076 {
1077 	struct device *s;
1078 	struct cpu *c;
1079 	int rc;
1080 
1081 	c = kzalloc(sizeof(*c), GFP_KERNEL);
1082 	if (!c)
1083 		return -ENOMEM;
1084 	per_cpu(cpu_device, cpu) = c;
1085 	s = &c->dev;
1086 	c->hotpluggable = 1;
1087 	rc = register_cpu(c, cpu);
1088 	if (rc)
1089 		goto out;
1090 	rc = sysfs_create_group(&s->kobj, &cpu_common_attr_group);
1091 	if (rc)
1092 		goto out_cpu;
1093 	if (cpu_online(cpu)) {
1094 		rc = sysfs_create_group(&s->kobj, &cpu_online_attr_group);
1095 		if (rc)
1096 			goto out_online;
1097 	}
1098 	rc = topology_cpu_init(c);
1099 	if (rc)
1100 		goto out_topology;
1101 	return 0;
1102 
1103 out_topology:
1104 	if (cpu_online(cpu))
1105 		sysfs_remove_group(&s->kobj, &cpu_online_attr_group);
1106 out_online:
1107 	sysfs_remove_group(&s->kobj, &cpu_common_attr_group);
1108 out_cpu:
1109 #ifdef CONFIG_HOTPLUG_CPU
1110 	unregister_cpu(c);
1111 #endif
1112 out:
1113 	return rc;
1114 }
1115 
1116 #ifdef CONFIG_HOTPLUG_CPU
1117 
1118 int __ref smp_rescan_cpus(void)
1119 {
1120 	struct sclp_core_info *info;
1121 	int nr;
1122 
1123 	info = smp_get_core_info();
1124 	if (!info)
1125 		return -ENOMEM;
1126 	get_online_cpus();
1127 	mutex_lock(&smp_cpu_state_mutex);
1128 	nr = __smp_rescan_cpus(info, 1);
1129 	mutex_unlock(&smp_cpu_state_mutex);
1130 	put_online_cpus();
1131 	kfree(info);
1132 	if (nr)
1133 		topology_schedule_update();
1134 	return 0;
1135 }
1136 
1137 static ssize_t __ref rescan_store(struct device *dev,
1138 				  struct device_attribute *attr,
1139 				  const char *buf,
1140 				  size_t count)
1141 {
1142 	int rc;
1143 
1144 	rc = smp_rescan_cpus();
1145 	return rc ? rc : count;
1146 }
1147 static DEVICE_ATTR(rescan, 0200, NULL, rescan_store);
1148 #endif /* CONFIG_HOTPLUG_CPU */
1149 
1150 static int __init s390_smp_init(void)
1151 {
1152 	int cpu, rc = 0;
1153 
1154 #ifdef CONFIG_HOTPLUG_CPU
1155 	rc = device_create_file(cpu_subsys.dev_root, &dev_attr_rescan);
1156 	if (rc)
1157 		return rc;
1158 #endif
1159 	cpu_notifier_register_begin();
1160 	for_each_present_cpu(cpu) {
1161 		rc = smp_add_present_cpu(cpu);
1162 		if (rc)
1163 			goto out;
1164 	}
1165 
1166 	__hotcpu_notifier(smp_cpu_notify, 0);
1167 
1168 out:
1169 	cpu_notifier_register_done();
1170 	return rc;
1171 }
1172 subsys_initcall(s390_smp_init);
1173