1 /* 2 * Copyright 2013, Michael (Ellerman|Neuling), IBM Corporation. 3 * 4 * This program is free software; you can redistribute it and/or 5 * modify it under the terms of the GNU General Public License 6 * as published by the Free Software Foundation; either version 7 * 2 of the License, or (at your option) any later version. 8 */ 9 10 #define pr_fmt(fmt) "powernv: " fmt 11 12 #include <linux/kernel.h> 13 #include <linux/cpu.h> 14 #include <linux/cpumask.h> 15 #include <linux/device.h> 16 #include <linux/gfp.h> 17 #include <linux/smp.h> 18 #include <linux/stop_machine.h> 19 20 #include <asm/cputhreads.h> 21 #include <asm/kvm_ppc.h> 22 #include <asm/machdep.h> 23 #include <asm/opal.h> 24 #include <asm/smp.h> 25 26 #include "subcore.h" 27 28 29 /* 30 * Split/unsplit procedure: 31 * 32 * A core can be in one of three states, unsplit, 2-way split, and 4-way split. 33 * 34 * The mapping to subcores_per_core is simple: 35 * 36 * State | subcores_per_core 37 * ------------|------------------ 38 * Unsplit | 1 39 * 2-way split | 2 40 * 4-way split | 4 41 * 42 * The core is split along thread boundaries, the mapping between subcores and 43 * threads is as follows: 44 * 45 * Unsplit: 46 * ---------------------------- 47 * Subcore | 0 | 48 * ---------------------------- 49 * Thread | 0 1 2 3 4 5 6 7 | 50 * ---------------------------- 51 * 52 * 2-way split: 53 * ------------------------------------- 54 * Subcore | 0 | 1 | 55 * ------------------------------------- 56 * Thread | 0 1 2 3 | 4 5 6 7 | 57 * ------------------------------------- 58 * 59 * 4-way split: 60 * ----------------------------------------- 61 * Subcore | 0 | 1 | 2 | 3 | 62 * ----------------------------------------- 63 * Thread | 0 1 | 2 3 | 4 5 | 6 7 | 64 * ----------------------------------------- 65 * 66 * 67 * Transitions 68 * ----------- 69 * 70 * It is not possible to transition between either of the split states, the 71 * core must first be unsplit. The legal transitions are: 72 * 73 * ----------- --------------- 74 * | | <----> | 2-way split | 75 * | | --------------- 76 * | Unsplit | 77 * | | --------------- 78 * | | <----> | 4-way split | 79 * ----------- --------------- 80 * 81 * Unsplitting 82 * ----------- 83 * 84 * Unsplitting is the simpler procedure. It requires thread 0 to request the 85 * unsplit while all other threads NAP. 86 * 87 * Thread 0 clears HID0_POWER8_DYNLPARDIS (Dynamic LPAR Disable). This tells 88 * the hardware that if all threads except 0 are napping, the hardware should 89 * unsplit the core. 90 * 91 * Non-zero threads are sent to a NAP loop, they don't exit the loop until they 92 * see the core unsplit. 93 * 94 * Core 0 spins waiting for the hardware to see all the other threads napping 95 * and perform the unsplit. 96 * 97 * Once thread 0 sees the unsplit, it IPIs the secondary threads to wake them 98 * out of NAP. They will then see the core unsplit and exit the NAP loop. 99 * 100 * Splitting 101 * --------- 102 * 103 * The basic splitting procedure is fairly straight forward. However it is 104 * complicated by the fact that after the split occurs, the newly created 105 * subcores are not in a fully initialised state. 106 * 107 * Most notably the subcores do not have the correct value for SDR1, which 108 * means they must not be running in virtual mode when the split occurs. The 109 * subcores have separate timebases SPRs but these are pre-synchronised by 110 * opal. 111 * 112 * To begin with secondary threads are sent to an assembly routine. There they 113 * switch to real mode, so they are immune to the uninitialised SDR1 value. 114 * Once in real mode they indicate that they are in real mode, and spin waiting 115 * to see the core split. 116 * 117 * Thread 0 waits to see that all secondaries are in real mode, and then begins 118 * the splitting procedure. It firstly sets HID0_POWER8_DYNLPARDIS, which 119 * prevents the hardware from unsplitting. Then it sets the appropriate HID bit 120 * to request the split, and spins waiting to see that the split has happened. 121 * 122 * Concurrently the secondaries will notice the split. When they do they set up 123 * their SPRs, notably SDR1, and then they can return to virtual mode and exit 124 * the procedure. 125 */ 126 127 /* Initialised at boot by subcore_init() */ 128 static int subcores_per_core; 129 130 /* 131 * Used to communicate to offline cpus that we want them to pop out of the 132 * offline loop and do a split or unsplit. 133 * 134 * 0 - no split happening 135 * 1 - unsplit in progress 136 * 2 - split to 2 in progress 137 * 4 - split to 4 in progress 138 */ 139 static int new_split_mode; 140 141 static cpumask_var_t cpu_offline_mask; 142 143 struct split_state { 144 u8 step; 145 u8 master; 146 }; 147 148 static DEFINE_PER_CPU(struct split_state, split_state); 149 150 static void wait_for_sync_step(int step) 151 { 152 int i, cpu = smp_processor_id(); 153 154 for (i = cpu + 1; i < cpu + threads_per_core; i++) 155 while(per_cpu(split_state, i).step < step) 156 barrier(); 157 158 /* Order the wait loop vs any subsequent loads/stores. */ 159 mb(); 160 } 161 162 static void unsplit_core(void) 163 { 164 u64 hid0, mask; 165 int i, cpu; 166 167 mask = HID0_POWER8_2LPARMODE | HID0_POWER8_4LPARMODE; 168 169 cpu = smp_processor_id(); 170 if (cpu_thread_in_core(cpu) != 0) { 171 while (mfspr(SPRN_HID0) & mask) 172 power7_nap(0); 173 174 per_cpu(split_state, cpu).step = SYNC_STEP_UNSPLIT; 175 return; 176 } 177 178 hid0 = mfspr(SPRN_HID0); 179 hid0 &= ~HID0_POWER8_DYNLPARDIS; 180 mtspr(SPRN_HID0, hid0); 181 182 while (mfspr(SPRN_HID0) & mask) 183 cpu_relax(); 184 185 /* Wake secondaries out of NAP */ 186 for (i = cpu + 1; i < cpu + threads_per_core; i++) 187 smp_send_reschedule(i); 188 189 wait_for_sync_step(SYNC_STEP_UNSPLIT); 190 } 191 192 static void split_core(int new_mode) 193 { 194 struct { u64 value; u64 mask; } split_parms[2] = { 195 { HID0_POWER8_1TO2LPAR, HID0_POWER8_2LPARMODE }, 196 { HID0_POWER8_1TO4LPAR, HID0_POWER8_4LPARMODE } 197 }; 198 int i, cpu; 199 u64 hid0; 200 201 /* Convert new_mode (2 or 4) into an index into our parms array */ 202 i = (new_mode >> 1) - 1; 203 BUG_ON(i < 0 || i > 1); 204 205 cpu = smp_processor_id(); 206 if (cpu_thread_in_core(cpu) != 0) { 207 split_core_secondary_loop(&per_cpu(split_state, cpu).step); 208 return; 209 } 210 211 wait_for_sync_step(SYNC_STEP_REAL_MODE); 212 213 /* Write new mode */ 214 hid0 = mfspr(SPRN_HID0); 215 hid0 |= HID0_POWER8_DYNLPARDIS | split_parms[i].value; 216 mtspr(SPRN_HID0, hid0); 217 218 /* Wait for it to happen */ 219 while (!(mfspr(SPRN_HID0) & split_parms[i].mask)) 220 cpu_relax(); 221 } 222 223 static void cpu_do_split(int new_mode) 224 { 225 /* 226 * At boot subcores_per_core will be 0, so we will always unsplit at 227 * boot. In the usual case where the core is already unsplit it's a 228 * nop, and this just ensures the kernel's notion of the mode is 229 * consistent with the hardware. 230 */ 231 if (subcores_per_core != 1) 232 unsplit_core(); 233 234 if (new_mode != 1) 235 split_core(new_mode); 236 237 mb(); 238 per_cpu(split_state, smp_processor_id()).step = SYNC_STEP_FINISHED; 239 } 240 241 bool cpu_core_split_required(void) 242 { 243 smp_rmb(); 244 245 if (!new_split_mode) 246 return false; 247 248 cpu_do_split(new_split_mode); 249 250 return true; 251 } 252 253 static int cpu_update_split_mode(void *data) 254 { 255 int cpu, new_mode = *(int *)data; 256 257 if (this_cpu_ptr(&split_state)->master) { 258 new_split_mode = new_mode; 259 smp_wmb(); 260 261 cpumask_andnot(cpu_offline_mask, cpu_present_mask, 262 cpu_online_mask); 263 264 /* This should work even though the cpu is offline */ 265 for_each_cpu(cpu, cpu_offline_mask) 266 smp_send_reschedule(cpu); 267 } 268 269 cpu_do_split(new_mode); 270 271 if (this_cpu_ptr(&split_state)->master) { 272 /* Wait for all cpus to finish before we touch subcores_per_core */ 273 for_each_present_cpu(cpu) { 274 if (cpu >= setup_max_cpus) 275 break; 276 277 while(per_cpu(split_state, cpu).step < SYNC_STEP_FINISHED) 278 barrier(); 279 } 280 281 new_split_mode = 0; 282 283 /* Make the new mode public */ 284 subcores_per_core = new_mode; 285 threads_per_subcore = threads_per_core / subcores_per_core; 286 287 /* Make sure the new mode is written before we exit */ 288 mb(); 289 } 290 291 return 0; 292 } 293 294 static int set_subcores_per_core(int new_mode) 295 { 296 struct split_state *state; 297 int cpu; 298 299 if (kvm_hv_mode_active()) { 300 pr_err("Unable to change split core mode while KVM active.\n"); 301 return -EBUSY; 302 } 303 304 /* 305 * We are only called at boot, or from the sysfs write. If that ever 306 * changes we'll need a lock here. 307 */ 308 BUG_ON(new_mode < 1 || new_mode > 4 || new_mode == 3); 309 310 for_each_present_cpu(cpu) { 311 state = &per_cpu(split_state, cpu); 312 state->step = SYNC_STEP_INITIAL; 313 state->master = 0; 314 } 315 316 get_online_cpus(); 317 318 /* This cpu will update the globals before exiting stop machine */ 319 this_cpu_ptr(&split_state)->master = 1; 320 321 /* Ensure state is consistent before we call the other cpus */ 322 mb(); 323 324 stop_machine(cpu_update_split_mode, &new_mode, cpu_online_mask); 325 326 put_online_cpus(); 327 328 return 0; 329 } 330 331 static ssize_t __used store_subcores_per_core(struct device *dev, 332 struct device_attribute *attr, const char *buf, 333 size_t count) 334 { 335 unsigned long val; 336 int rc; 337 338 /* We are serialised by the attribute lock */ 339 340 rc = sscanf(buf, "%lx", &val); 341 if (rc != 1) 342 return -EINVAL; 343 344 switch (val) { 345 case 1: 346 case 2: 347 case 4: 348 if (subcores_per_core == val) 349 /* Nothing to do */ 350 goto out; 351 break; 352 default: 353 return -EINVAL; 354 } 355 356 rc = set_subcores_per_core(val); 357 if (rc) 358 return rc; 359 360 out: 361 return count; 362 } 363 364 static ssize_t show_subcores_per_core(struct device *dev, 365 struct device_attribute *attr, char *buf) 366 { 367 return sprintf(buf, "%x\n", subcores_per_core); 368 } 369 370 static DEVICE_ATTR(subcores_per_core, 0644, 371 show_subcores_per_core, store_subcores_per_core); 372 373 static int subcore_init(void) 374 { 375 if (!cpu_has_feature(CPU_FTR_ARCH_207S)) 376 return 0; 377 378 /* 379 * We need all threads in a core to be present to split/unsplit so 380 * continue only if max_cpus are aligned to threads_per_core. 381 */ 382 if (setup_max_cpus % threads_per_core) 383 return 0; 384 385 BUG_ON(!alloc_cpumask_var(&cpu_offline_mask, GFP_KERNEL)); 386 387 set_subcores_per_core(1); 388 389 return device_create_file(cpu_subsys.dev_root, 390 &dev_attr_subcores_per_core); 391 } 392 machine_device_initcall(powernv, subcore_init); 393