1 /*
2  * Copyright 2013, Michael (Ellerman|Neuling), IBM Corporation.
3  *
4  * This program is free software; you can redistribute it and/or
5  * modify it under the terms of the GNU General Public License
6  * as published by the Free Software Foundation; either version
7  * 2 of the License, or (at your option) any later version.
8  */
9 
10 #define pr_fmt(fmt)	"powernv: " fmt
11 
12 #include <linux/kernel.h>
13 #include <linux/cpu.h>
14 #include <linux/cpumask.h>
15 #include <linux/device.h>
16 #include <linux/gfp.h>
17 #include <linux/smp.h>
18 #include <linux/stop_machine.h>
19 
20 #include <asm/cputhreads.h>
21 #include <asm/cpuidle.h>
22 #include <asm/kvm_ppc.h>
23 #include <asm/machdep.h>
24 #include <asm/opal.h>
25 #include <asm/smp.h>
26 
27 #include "subcore.h"
28 #include "powernv.h"
29 
30 
31 /*
32  * Split/unsplit procedure:
33  *
34  * A core can be in one of three states, unsplit, 2-way split, and 4-way split.
35  *
36  * The mapping to subcores_per_core is simple:
37  *
38  *  State       | subcores_per_core
39  *  ------------|------------------
40  *  Unsplit     |        1
41  *  2-way split |        2
42  *  4-way split |        4
43  *
44  * The core is split along thread boundaries, the mapping between subcores and
45  * threads is as follows:
46  *
47  *  Unsplit:
48  *          ----------------------------
49  *  Subcore |            0             |
50  *          ----------------------------
51  *  Thread  |  0  1  2  3  4  5  6  7  |
52  *          ----------------------------
53  *
54  *  2-way split:
55  *          -------------------------------------
56  *  Subcore |        0        |        1        |
57  *          -------------------------------------
58  *  Thread  |  0   1   2   3  |  4   5   6   7  |
59  *          -------------------------------------
60  *
61  *  4-way split:
62  *          -----------------------------------------
63  *  Subcore |    0    |    1    |    2    |    3    |
64  *          -----------------------------------------
65  *  Thread  |  0   1  |  2   3  |  4   5  |  6   7  |
66  *          -----------------------------------------
67  *
68  *
69  * Transitions
70  * -----------
71  *
72  * It is not possible to transition between either of the split states, the
73  * core must first be unsplit. The legal transitions are:
74  *
75  *  -----------          ---------------
76  *  |         |  <---->  | 2-way split |
77  *  |         |          ---------------
78  *  | Unsplit |
79  *  |         |          ---------------
80  *  |         |  <---->  | 4-way split |
81  *  -----------          ---------------
82  *
83  * Unsplitting
84  * -----------
85  *
86  * Unsplitting is the simpler procedure. It requires thread 0 to request the
87  * unsplit while all other threads NAP.
88  *
89  * Thread 0 clears HID0_POWER8_DYNLPARDIS (Dynamic LPAR Disable). This tells
90  * the hardware that if all threads except 0 are napping, the hardware should
91  * unsplit the core.
92  *
93  * Non-zero threads are sent to a NAP loop, they don't exit the loop until they
94  * see the core unsplit.
95  *
96  * Core 0 spins waiting for the hardware to see all the other threads napping
97  * and perform the unsplit.
98  *
99  * Once thread 0 sees the unsplit, it IPIs the secondary threads to wake them
100  * out of NAP. They will then see the core unsplit and exit the NAP loop.
101  *
102  * Splitting
103  * ---------
104  *
105  * The basic splitting procedure is fairly straight forward. However it is
106  * complicated by the fact that after the split occurs, the newly created
107  * subcores are not in a fully initialised state.
108  *
109  * Most notably the subcores do not have the correct value for SDR1, which
110  * means they must not be running in virtual mode when the split occurs. The
111  * subcores have separate timebases SPRs but these are pre-synchronised by
112  * opal.
113  *
114  * To begin with secondary threads are sent to an assembly routine. There they
115  * switch to real mode, so they are immune to the uninitialised SDR1 value.
116  * Once in real mode they indicate that they are in real mode, and spin waiting
117  * to see the core split.
118  *
119  * Thread 0 waits to see that all secondaries are in real mode, and then begins
120  * the splitting procedure. It firstly sets HID0_POWER8_DYNLPARDIS, which
121  * prevents the hardware from unsplitting. Then it sets the appropriate HID bit
122  * to request the split, and spins waiting to see that the split has happened.
123  *
124  * Concurrently the secondaries will notice the split. When they do they set up
125  * their SPRs, notably SDR1, and then they can return to virtual mode and exit
126  * the procedure.
127  */
128 
129 /* Initialised at boot by subcore_init() */
130 static int subcores_per_core;
131 
132 /*
133  * Used to communicate to offline cpus that we want them to pop out of the
134  * offline loop and do a split or unsplit.
135  *
136  * 0 - no split happening
137  * 1 - unsplit in progress
138  * 2 - split to 2 in progress
139  * 4 - split to 4 in progress
140  */
141 static int new_split_mode;
142 
143 static cpumask_var_t cpu_offline_mask;
144 
145 struct split_state {
146 	u8 step;
147 	u8 master;
148 };
149 
150 static DEFINE_PER_CPU(struct split_state, split_state);
151 
152 static void wait_for_sync_step(int step)
153 {
154 	int i, cpu = smp_processor_id();
155 
156 	for (i = cpu + 1; i < cpu + threads_per_core; i++)
157 		while(per_cpu(split_state, i).step < step)
158 			barrier();
159 
160 	/* Order the wait loop vs any subsequent loads/stores. */
161 	mb();
162 }
163 
164 static void update_hid_in_slw(u64 hid0)
165 {
166 	u64 idle_states = pnv_get_supported_cpuidle_states();
167 
168 	if (idle_states & OPAL_PM_WINKLE_ENABLED) {
169 		/* OPAL call to patch slw with the new HID0 value */
170 		u64 cpu_pir = hard_smp_processor_id();
171 
172 		opal_slw_set_reg(cpu_pir, SPRN_HID0, hid0);
173 	}
174 }
175 
176 static void unsplit_core(void)
177 {
178 	u64 hid0, mask;
179 	int i, cpu;
180 
181 	mask = HID0_POWER8_2LPARMODE | HID0_POWER8_4LPARMODE;
182 
183 	cpu = smp_processor_id();
184 	if (cpu_thread_in_core(cpu) != 0) {
185 		while (mfspr(SPRN_HID0) & mask)
186 			power7_idle_insn(PNV_THREAD_NAP);
187 
188 		per_cpu(split_state, cpu).step = SYNC_STEP_UNSPLIT;
189 		return;
190 	}
191 
192 	hid0 = mfspr(SPRN_HID0);
193 	hid0 &= ~HID0_POWER8_DYNLPARDIS;
194 	update_power8_hid0(hid0);
195 	update_hid_in_slw(hid0);
196 
197 	while (mfspr(SPRN_HID0) & mask)
198 		cpu_relax();
199 
200 	/* Wake secondaries out of NAP */
201 	for (i = cpu + 1; i < cpu + threads_per_core; i++)
202 		smp_send_reschedule(i);
203 
204 	wait_for_sync_step(SYNC_STEP_UNSPLIT);
205 }
206 
207 static void split_core(int new_mode)
208 {
209 	struct {  u64 value; u64 mask; } split_parms[2] = {
210 		{ HID0_POWER8_1TO2LPAR, HID0_POWER8_2LPARMODE },
211 		{ HID0_POWER8_1TO4LPAR, HID0_POWER8_4LPARMODE }
212 	};
213 	int i, cpu;
214 	u64 hid0;
215 
216 	/* Convert new_mode (2 or 4) into an index into our parms array */
217 	i = (new_mode >> 1) - 1;
218 	BUG_ON(i < 0 || i > 1);
219 
220 	cpu = smp_processor_id();
221 	if (cpu_thread_in_core(cpu) != 0) {
222 		split_core_secondary_loop(&per_cpu(split_state, cpu).step);
223 		return;
224 	}
225 
226 	wait_for_sync_step(SYNC_STEP_REAL_MODE);
227 
228 	/* Write new mode */
229 	hid0  = mfspr(SPRN_HID0);
230 	hid0 |= HID0_POWER8_DYNLPARDIS | split_parms[i].value;
231 	update_power8_hid0(hid0);
232 	update_hid_in_slw(hid0);
233 
234 	/* Wait for it to happen */
235 	while (!(mfspr(SPRN_HID0) & split_parms[i].mask))
236 		cpu_relax();
237 }
238 
239 static void cpu_do_split(int new_mode)
240 {
241 	/*
242 	 * At boot subcores_per_core will be 0, so we will always unsplit at
243 	 * boot. In the usual case where the core is already unsplit it's a
244 	 * nop, and this just ensures the kernel's notion of the mode is
245 	 * consistent with the hardware.
246 	 */
247 	if (subcores_per_core != 1)
248 		unsplit_core();
249 
250 	if (new_mode != 1)
251 		split_core(new_mode);
252 
253 	mb();
254 	per_cpu(split_state, smp_processor_id()).step = SYNC_STEP_FINISHED;
255 }
256 
257 bool cpu_core_split_required(void)
258 {
259 	smp_rmb();
260 
261 	if (!new_split_mode)
262 		return false;
263 
264 	cpu_do_split(new_split_mode);
265 
266 	return true;
267 }
268 
269 void update_subcore_sibling_mask(void)
270 {
271 	int cpu;
272 	/*
273 	 * sibling mask for the first cpu. Left shift this by required bits
274 	 * to get sibling mask for the rest of the cpus.
275 	 */
276 	int sibling_mask_first_cpu =  (1 << threads_per_subcore) - 1;
277 
278 	for_each_possible_cpu(cpu) {
279 		int tid = cpu_thread_in_core(cpu);
280 		int offset = (tid / threads_per_subcore) * threads_per_subcore;
281 		int mask = sibling_mask_first_cpu << offset;
282 
283 		paca_ptrs[cpu]->subcore_sibling_mask = mask;
284 
285 	}
286 }
287 
288 static int cpu_update_split_mode(void *data)
289 {
290 	int cpu, new_mode = *(int *)data;
291 
292 	if (this_cpu_ptr(&split_state)->master) {
293 		new_split_mode = new_mode;
294 		smp_wmb();
295 
296 		cpumask_andnot(cpu_offline_mask, cpu_present_mask,
297 			       cpu_online_mask);
298 
299 		/* This should work even though the cpu is offline */
300 		for_each_cpu(cpu, cpu_offline_mask)
301 			smp_send_reschedule(cpu);
302 	}
303 
304 	cpu_do_split(new_mode);
305 
306 	if (this_cpu_ptr(&split_state)->master) {
307 		/* Wait for all cpus to finish before we touch subcores_per_core */
308 		for_each_present_cpu(cpu) {
309 			if (cpu >= setup_max_cpus)
310 				break;
311 
312 			while(per_cpu(split_state, cpu).step < SYNC_STEP_FINISHED)
313 				barrier();
314 		}
315 
316 		new_split_mode = 0;
317 
318 		/* Make the new mode public */
319 		subcores_per_core = new_mode;
320 		threads_per_subcore = threads_per_core / subcores_per_core;
321 		update_subcore_sibling_mask();
322 
323 		/* Make sure the new mode is written before we exit */
324 		mb();
325 	}
326 
327 	return 0;
328 }
329 
330 static int set_subcores_per_core(int new_mode)
331 {
332 	struct split_state *state;
333 	int cpu;
334 
335 	if (kvm_hv_mode_active()) {
336 		pr_err("Unable to change split core mode while KVM active.\n");
337 		return -EBUSY;
338 	}
339 
340 	/*
341 	 * We are only called at boot, or from the sysfs write. If that ever
342 	 * changes we'll need a lock here.
343 	 */
344 	BUG_ON(new_mode < 1 || new_mode > 4 || new_mode == 3);
345 
346 	for_each_present_cpu(cpu) {
347 		state = &per_cpu(split_state, cpu);
348 		state->step = SYNC_STEP_INITIAL;
349 		state->master = 0;
350 	}
351 
352 	cpus_read_lock();
353 
354 	/* This cpu will update the globals before exiting stop machine */
355 	this_cpu_ptr(&split_state)->master = 1;
356 
357 	/* Ensure state is consistent before we call the other cpus */
358 	mb();
359 
360 	stop_machine_cpuslocked(cpu_update_split_mode, &new_mode,
361 				cpu_online_mask);
362 
363 	cpus_read_unlock();
364 
365 	return 0;
366 }
367 
368 static ssize_t __used store_subcores_per_core(struct device *dev,
369 		struct device_attribute *attr, const char *buf,
370 		size_t count)
371 {
372 	unsigned long val;
373 	int rc;
374 
375 	/* We are serialised by the attribute lock */
376 
377 	rc = sscanf(buf, "%lx", &val);
378 	if (rc != 1)
379 		return -EINVAL;
380 
381 	switch (val) {
382 	case 1:
383 	case 2:
384 	case 4:
385 		if (subcores_per_core == val)
386 			/* Nothing to do */
387 			goto out;
388 		break;
389 	default:
390 		return -EINVAL;
391 	}
392 
393 	rc = set_subcores_per_core(val);
394 	if (rc)
395 		return rc;
396 
397 out:
398 	return count;
399 }
400 
401 static ssize_t show_subcores_per_core(struct device *dev,
402 		struct device_attribute *attr, char *buf)
403 {
404 	return sprintf(buf, "%x\n", subcores_per_core);
405 }
406 
407 static DEVICE_ATTR(subcores_per_core, 0644,
408 		show_subcores_per_core, store_subcores_per_core);
409 
410 static int subcore_init(void)
411 {
412 	unsigned pvr_ver;
413 
414 	pvr_ver = PVR_VER(mfspr(SPRN_PVR));
415 
416 	if (pvr_ver != PVR_POWER8 &&
417 	    pvr_ver != PVR_POWER8E &&
418 	    pvr_ver != PVR_POWER8NVL)
419 		return 0;
420 
421 	/*
422 	 * We need all threads in a core to be present to split/unsplit so
423          * continue only if max_cpus are aligned to threads_per_core.
424 	 */
425 	if (setup_max_cpus % threads_per_core)
426 		return 0;
427 
428 	BUG_ON(!alloc_cpumask_var(&cpu_offline_mask, GFP_KERNEL));
429 
430 	set_subcores_per_core(1);
431 
432 	return device_create_file(cpu_subsys.dev_root,
433 				  &dev_attr_subcores_per_core);
434 }
435 machine_device_initcall(powernv, subcore_init);
436