1e2186023SMichael Ellerman /* 2e2186023SMichael Ellerman * Copyright 2013, Michael (Ellerman|Neuling), IBM Corporation. 3e2186023SMichael Ellerman * 4e2186023SMichael Ellerman * This program is free software; you can redistribute it and/or 5e2186023SMichael Ellerman * modify it under the terms of the GNU General Public License 6e2186023SMichael Ellerman * as published by the Free Software Foundation; either version 7e2186023SMichael Ellerman * 2 of the License, or (at your option) any later version. 8e2186023SMichael Ellerman */ 9e2186023SMichael Ellerman 10e2186023SMichael Ellerman #define pr_fmt(fmt) "powernv: " fmt 11e2186023SMichael Ellerman 12e2186023SMichael Ellerman #include <linux/kernel.h> 13e2186023SMichael Ellerman #include <linux/cpu.h> 14e2186023SMichael Ellerman #include <linux/cpumask.h> 15e2186023SMichael Ellerman #include <linux/device.h> 16e2186023SMichael Ellerman #include <linux/gfp.h> 17e2186023SMichael Ellerman #include <linux/smp.h> 18e2186023SMichael Ellerman #include <linux/stop_machine.h> 19e2186023SMichael Ellerman 20e2186023SMichael Ellerman #include <asm/cputhreads.h> 21e2186023SMichael Ellerman #include <asm/kvm_ppc.h> 22e2186023SMichael Ellerman #include <asm/machdep.h> 23e2186023SMichael Ellerman #include <asm/opal.h> 24e2186023SMichael Ellerman #include <asm/smp.h> 25e2186023SMichael Ellerman 26e2186023SMichael Ellerman #include "subcore.h" 27e2186023SMichael Ellerman 28e2186023SMichael Ellerman 29e2186023SMichael Ellerman /* 30e2186023SMichael Ellerman * Split/unsplit procedure: 31e2186023SMichael Ellerman * 32e2186023SMichael Ellerman * A core can be in one of three states, unsplit, 2-way split, and 4-way split. 33e2186023SMichael Ellerman * 34e2186023SMichael Ellerman * The mapping to subcores_per_core is simple: 35e2186023SMichael Ellerman * 36e2186023SMichael Ellerman * State | subcores_per_core 37e2186023SMichael Ellerman * ------------|------------------ 38e2186023SMichael Ellerman * Unsplit | 1 39e2186023SMichael Ellerman * 2-way split | 2 40e2186023SMichael Ellerman * 4-way split | 4 41e2186023SMichael Ellerman * 42e2186023SMichael Ellerman * The core is split along thread boundaries, the mapping between subcores and 43e2186023SMichael Ellerman * threads is as follows: 44e2186023SMichael Ellerman * 45e2186023SMichael Ellerman * Unsplit: 46e2186023SMichael Ellerman * ---------------------------- 47e2186023SMichael Ellerman * Subcore | 0 | 48e2186023SMichael Ellerman * ---------------------------- 49e2186023SMichael Ellerman * Thread | 0 1 2 3 4 5 6 7 | 50e2186023SMichael Ellerman * ---------------------------- 51e2186023SMichael Ellerman * 52e2186023SMichael Ellerman * 2-way split: 53e2186023SMichael Ellerman * ------------------------------------- 54e2186023SMichael Ellerman * Subcore | 0 | 1 | 55e2186023SMichael Ellerman * ------------------------------------- 56e2186023SMichael Ellerman * Thread | 0 1 2 3 | 4 5 6 7 | 57e2186023SMichael Ellerman * ------------------------------------- 58e2186023SMichael Ellerman * 59e2186023SMichael Ellerman * 4-way split: 60e2186023SMichael Ellerman * ----------------------------------------- 61e2186023SMichael Ellerman * Subcore | 0 | 1 | 2 | 3 | 62e2186023SMichael Ellerman * ----------------------------------------- 63e2186023SMichael Ellerman * Thread | 0 1 | 2 3 | 4 5 | 6 7 | 64e2186023SMichael Ellerman * ----------------------------------------- 65e2186023SMichael Ellerman * 66e2186023SMichael Ellerman * 67e2186023SMichael Ellerman * Transitions 68e2186023SMichael Ellerman * ----------- 69e2186023SMichael Ellerman * 70e2186023SMichael Ellerman * It is not possible to transition between either of the split states, the 71e2186023SMichael Ellerman * core must first be unsplit. The legal transitions are: 72e2186023SMichael Ellerman * 73e2186023SMichael Ellerman * ----------- --------------- 74e2186023SMichael Ellerman * | | <----> | 2-way split | 75e2186023SMichael Ellerman * | | --------------- 76e2186023SMichael Ellerman * | Unsplit | 77e2186023SMichael Ellerman * | | --------------- 78e2186023SMichael Ellerman * | | <----> | 4-way split | 79e2186023SMichael Ellerman * ----------- --------------- 80e2186023SMichael Ellerman * 81e2186023SMichael Ellerman * Unsplitting 82e2186023SMichael Ellerman * ----------- 83e2186023SMichael Ellerman * 84e2186023SMichael Ellerman * Unsplitting is the simpler procedure. It requires thread 0 to request the 85e2186023SMichael Ellerman * unsplit while all other threads NAP. 86e2186023SMichael Ellerman * 87e2186023SMichael Ellerman * Thread 0 clears HID0_POWER8_DYNLPARDIS (Dynamic LPAR Disable). This tells 88e2186023SMichael Ellerman * the hardware that if all threads except 0 are napping, the hardware should 89e2186023SMichael Ellerman * unsplit the core. 90e2186023SMichael Ellerman * 91e2186023SMichael Ellerman * Non-zero threads are sent to a NAP loop, they don't exit the loop until they 92e2186023SMichael Ellerman * see the core unsplit. 93e2186023SMichael Ellerman * 94e2186023SMichael Ellerman * Core 0 spins waiting for the hardware to see all the other threads napping 95e2186023SMichael Ellerman * and perform the unsplit. 96e2186023SMichael Ellerman * 97e2186023SMichael Ellerman * Once thread 0 sees the unsplit, it IPIs the secondary threads to wake them 98e2186023SMichael Ellerman * out of NAP. They will then see the core unsplit and exit the NAP loop. 99e2186023SMichael Ellerman * 100e2186023SMichael Ellerman * Splitting 101e2186023SMichael Ellerman * --------- 102e2186023SMichael Ellerman * 103e2186023SMichael Ellerman * The basic splitting procedure is fairly straight forward. However it is 104e2186023SMichael Ellerman * complicated by the fact that after the split occurs, the newly created 105e2186023SMichael Ellerman * subcores are not in a fully initialised state. 106e2186023SMichael Ellerman * 107e2186023SMichael Ellerman * Most notably the subcores do not have the correct value for SDR1, which 108e2186023SMichael Ellerman * means they must not be running in virtual mode when the split occurs. The 109e2186023SMichael Ellerman * subcores have separate timebases SPRs but these are pre-synchronised by 110e2186023SMichael Ellerman * opal. 111e2186023SMichael Ellerman * 112e2186023SMichael Ellerman * To begin with secondary threads are sent to an assembly routine. There they 113e2186023SMichael Ellerman * switch to real mode, so they are immune to the uninitialised SDR1 value. 114e2186023SMichael Ellerman * Once in real mode they indicate that they are in real mode, and spin waiting 115e2186023SMichael Ellerman * to see the core split. 116e2186023SMichael Ellerman * 117e2186023SMichael Ellerman * Thread 0 waits to see that all secondaries are in real mode, and then begins 118e2186023SMichael Ellerman * the splitting procedure. It firstly sets HID0_POWER8_DYNLPARDIS, which 119e2186023SMichael Ellerman * prevents the hardware from unsplitting. Then it sets the appropriate HID bit 120e2186023SMichael Ellerman * to request the split, and spins waiting to see that the split has happened. 121e2186023SMichael Ellerman * 122e2186023SMichael Ellerman * Concurrently the secondaries will notice the split. When they do they set up 123e2186023SMichael Ellerman * their SPRs, notably SDR1, and then they can return to virtual mode and exit 124e2186023SMichael Ellerman * the procedure. 125e2186023SMichael Ellerman */ 126e2186023SMichael Ellerman 127e2186023SMichael Ellerman /* Initialised at boot by subcore_init() */ 128e2186023SMichael Ellerman static int subcores_per_core; 129e2186023SMichael Ellerman 130e2186023SMichael Ellerman /* 131e2186023SMichael Ellerman * Used to communicate to offline cpus that we want them to pop out of the 132e2186023SMichael Ellerman * offline loop and do a split or unsplit. 133e2186023SMichael Ellerman * 134e2186023SMichael Ellerman * 0 - no split happening 135e2186023SMichael Ellerman * 1 - unsplit in progress 136e2186023SMichael Ellerman * 2 - split to 2 in progress 137e2186023SMichael Ellerman * 4 - split to 4 in progress 138e2186023SMichael Ellerman */ 139e2186023SMichael Ellerman static int new_split_mode; 140e2186023SMichael Ellerman 141e2186023SMichael Ellerman static cpumask_var_t cpu_offline_mask; 142e2186023SMichael Ellerman 143e2186023SMichael Ellerman struct split_state { 144e2186023SMichael Ellerman u8 step; 145e2186023SMichael Ellerman u8 master; 146e2186023SMichael Ellerman }; 147e2186023SMichael Ellerman 148e2186023SMichael Ellerman static DEFINE_PER_CPU(struct split_state, split_state); 149e2186023SMichael Ellerman 150e2186023SMichael Ellerman static void wait_for_sync_step(int step) 151e2186023SMichael Ellerman { 152e2186023SMichael Ellerman int i, cpu = smp_processor_id(); 153e2186023SMichael Ellerman 154e2186023SMichael Ellerman for (i = cpu + 1; i < cpu + threads_per_core; i++) 155e2186023SMichael Ellerman while(per_cpu(split_state, i).step < step) 156e2186023SMichael Ellerman barrier(); 157e2186023SMichael Ellerman 158e2186023SMichael Ellerman /* Order the wait loop vs any subsequent loads/stores. */ 159e2186023SMichael Ellerman mb(); 160e2186023SMichael Ellerman } 161e2186023SMichael Ellerman 162e2186023SMichael Ellerman static void unsplit_core(void) 163e2186023SMichael Ellerman { 164e2186023SMichael Ellerman u64 hid0, mask; 165e2186023SMichael Ellerman int i, cpu; 166e2186023SMichael Ellerman 167e2186023SMichael Ellerman mask = HID0_POWER8_2LPARMODE | HID0_POWER8_4LPARMODE; 168e2186023SMichael Ellerman 169e2186023SMichael Ellerman cpu = smp_processor_id(); 170e2186023SMichael Ellerman if (cpu_thread_in_core(cpu) != 0) { 171e2186023SMichael Ellerman while (mfspr(SPRN_HID0) & mask) 172e2186023SMichael Ellerman power7_nap(0); 173e2186023SMichael Ellerman 174e2186023SMichael Ellerman per_cpu(split_state, cpu).step = SYNC_STEP_UNSPLIT; 175e2186023SMichael Ellerman return; 176e2186023SMichael Ellerman } 177e2186023SMichael Ellerman 178e2186023SMichael Ellerman hid0 = mfspr(SPRN_HID0); 179e2186023SMichael Ellerman hid0 &= ~HID0_POWER8_DYNLPARDIS; 180e2186023SMichael Ellerman mtspr(SPRN_HID0, hid0); 181e2186023SMichael Ellerman 182e2186023SMichael Ellerman while (mfspr(SPRN_HID0) & mask) 183e2186023SMichael Ellerman cpu_relax(); 184e2186023SMichael Ellerman 185e2186023SMichael Ellerman /* Wake secondaries out of NAP */ 186e2186023SMichael Ellerman for (i = cpu + 1; i < cpu + threads_per_core; i++) 187e2186023SMichael Ellerman smp_send_reschedule(i); 188e2186023SMichael Ellerman 189e2186023SMichael Ellerman wait_for_sync_step(SYNC_STEP_UNSPLIT); 190e2186023SMichael Ellerman } 191e2186023SMichael Ellerman 192e2186023SMichael Ellerman static void split_core(int new_mode) 193e2186023SMichael Ellerman { 194e2186023SMichael Ellerman struct { u64 value; u64 mask; } split_parms[2] = { 195e2186023SMichael Ellerman { HID0_POWER8_1TO2LPAR, HID0_POWER8_2LPARMODE }, 196e2186023SMichael Ellerman { HID0_POWER8_1TO4LPAR, HID0_POWER8_4LPARMODE } 197e2186023SMichael Ellerman }; 198e2186023SMichael Ellerman int i, cpu; 199e2186023SMichael Ellerman u64 hid0; 200e2186023SMichael Ellerman 201e2186023SMichael Ellerman /* Convert new_mode (2 or 4) into an index into our parms array */ 202e2186023SMichael Ellerman i = (new_mode >> 1) - 1; 203e2186023SMichael Ellerman BUG_ON(i < 0 || i > 1); 204e2186023SMichael Ellerman 205e2186023SMichael Ellerman cpu = smp_processor_id(); 206e2186023SMichael Ellerman if (cpu_thread_in_core(cpu) != 0) { 207e2186023SMichael Ellerman split_core_secondary_loop(&per_cpu(split_state, cpu).step); 208e2186023SMichael Ellerman return; 209e2186023SMichael Ellerman } 210e2186023SMichael Ellerman 211e2186023SMichael Ellerman wait_for_sync_step(SYNC_STEP_REAL_MODE); 212e2186023SMichael Ellerman 213e2186023SMichael Ellerman /* Write new mode */ 214e2186023SMichael Ellerman hid0 = mfspr(SPRN_HID0); 215e2186023SMichael Ellerman hid0 |= HID0_POWER8_DYNLPARDIS | split_parms[i].value; 216e2186023SMichael Ellerman mtspr(SPRN_HID0, hid0); 217e2186023SMichael Ellerman 218e2186023SMichael Ellerman /* Wait for it to happen */ 219e2186023SMichael Ellerman while (!(mfspr(SPRN_HID0) & split_parms[i].mask)) 220e2186023SMichael Ellerman cpu_relax(); 221e2186023SMichael Ellerman } 222e2186023SMichael Ellerman 223e2186023SMichael Ellerman static void cpu_do_split(int new_mode) 224e2186023SMichael Ellerman { 225e2186023SMichael Ellerman /* 226e2186023SMichael Ellerman * At boot subcores_per_core will be 0, so we will always unsplit at 227e2186023SMichael Ellerman * boot. In the usual case where the core is already unsplit it's a 228e2186023SMichael Ellerman * nop, and this just ensures the kernel's notion of the mode is 229e2186023SMichael Ellerman * consistent with the hardware. 230e2186023SMichael Ellerman */ 231e2186023SMichael Ellerman if (subcores_per_core != 1) 232e2186023SMichael Ellerman unsplit_core(); 233e2186023SMichael Ellerman 234e2186023SMichael Ellerman if (new_mode != 1) 235e2186023SMichael Ellerman split_core(new_mode); 236e2186023SMichael Ellerman 237e2186023SMichael Ellerman mb(); 238e2186023SMichael Ellerman per_cpu(split_state, smp_processor_id()).step = SYNC_STEP_FINISHED; 239e2186023SMichael Ellerman } 240e2186023SMichael Ellerman 241e2186023SMichael Ellerman bool cpu_core_split_required(void) 242e2186023SMichael Ellerman { 243e2186023SMichael Ellerman smp_rmb(); 244e2186023SMichael Ellerman 245e2186023SMichael Ellerman if (!new_split_mode) 246e2186023SMichael Ellerman return false; 247e2186023SMichael Ellerman 248e2186023SMichael Ellerman cpu_do_split(new_split_mode); 249e2186023SMichael Ellerman 250e2186023SMichael Ellerman return true; 251e2186023SMichael Ellerman } 252e2186023SMichael Ellerman 253e2186023SMichael Ellerman static int cpu_update_split_mode(void *data) 254e2186023SMichael Ellerman { 255e2186023SMichael Ellerman int cpu, new_mode = *(int *)data; 256e2186023SMichael Ellerman 257e2186023SMichael Ellerman if (this_cpu_ptr(&split_state)->master) { 258e2186023SMichael Ellerman new_split_mode = new_mode; 259e2186023SMichael Ellerman smp_wmb(); 260e2186023SMichael Ellerman 261e2186023SMichael Ellerman cpumask_andnot(cpu_offline_mask, cpu_present_mask, 262e2186023SMichael Ellerman cpu_online_mask); 263e2186023SMichael Ellerman 264e2186023SMichael Ellerman /* This should work even though the cpu is offline */ 265e2186023SMichael Ellerman for_each_cpu(cpu, cpu_offline_mask) 266e2186023SMichael Ellerman smp_send_reschedule(cpu); 267e2186023SMichael Ellerman } 268e2186023SMichael Ellerman 269e2186023SMichael Ellerman cpu_do_split(new_mode); 270e2186023SMichael Ellerman 271e2186023SMichael Ellerman if (this_cpu_ptr(&split_state)->master) { 272e2186023SMichael Ellerman /* Wait for all cpus to finish before we touch subcores_per_core */ 273e2186023SMichael Ellerman for_each_present_cpu(cpu) { 274e2186023SMichael Ellerman if (cpu >= setup_max_cpus) 275e2186023SMichael Ellerman break; 276e2186023SMichael Ellerman 277e2186023SMichael Ellerman while(per_cpu(split_state, cpu).step < SYNC_STEP_FINISHED) 278e2186023SMichael Ellerman barrier(); 279e2186023SMichael Ellerman } 280e2186023SMichael Ellerman 281e2186023SMichael Ellerman new_split_mode = 0; 282e2186023SMichael Ellerman 283e2186023SMichael Ellerman /* Make the new mode public */ 284e2186023SMichael Ellerman subcores_per_core = new_mode; 285e2186023SMichael Ellerman threads_per_subcore = threads_per_core / subcores_per_core; 286e2186023SMichael Ellerman 287e2186023SMichael Ellerman /* Make sure the new mode is written before we exit */ 288e2186023SMichael Ellerman mb(); 289e2186023SMichael Ellerman } 290e2186023SMichael Ellerman 291e2186023SMichael Ellerman return 0; 292e2186023SMichael Ellerman } 293e2186023SMichael Ellerman 294e2186023SMichael Ellerman static int set_subcores_per_core(int new_mode) 295e2186023SMichael Ellerman { 296e2186023SMichael Ellerman struct split_state *state; 297e2186023SMichael Ellerman int cpu; 298e2186023SMichael Ellerman 299e2186023SMichael Ellerman if (kvm_hv_mode_active()) { 300e2186023SMichael Ellerman pr_err("Unable to change split core mode while KVM active.\n"); 301e2186023SMichael Ellerman return -EBUSY; 302e2186023SMichael Ellerman } 303e2186023SMichael Ellerman 304e2186023SMichael Ellerman /* 305e2186023SMichael Ellerman * We are only called at boot, or from the sysfs write. If that ever 306e2186023SMichael Ellerman * changes we'll need a lock here. 307e2186023SMichael Ellerman */ 308e2186023SMichael Ellerman BUG_ON(new_mode < 1 || new_mode > 4 || new_mode == 3); 309e2186023SMichael Ellerman 310e2186023SMichael Ellerman for_each_present_cpu(cpu) { 311e2186023SMichael Ellerman state = &per_cpu(split_state, cpu); 312e2186023SMichael Ellerman state->step = SYNC_STEP_INITIAL; 313e2186023SMichael Ellerman state->master = 0; 314e2186023SMichael Ellerman } 315e2186023SMichael Ellerman 316e2186023SMichael Ellerman get_online_cpus(); 317e2186023SMichael Ellerman 318e2186023SMichael Ellerman /* This cpu will update the globals before exiting stop machine */ 319e2186023SMichael Ellerman this_cpu_ptr(&split_state)->master = 1; 320e2186023SMichael Ellerman 321e2186023SMichael Ellerman /* Ensure state is consistent before we call the other cpus */ 322e2186023SMichael Ellerman mb(); 323e2186023SMichael Ellerman 324e2186023SMichael Ellerman stop_machine(cpu_update_split_mode, &new_mode, cpu_online_mask); 325e2186023SMichael Ellerman 326e2186023SMichael Ellerman put_online_cpus(); 327e2186023SMichael Ellerman 328e2186023SMichael Ellerman return 0; 329e2186023SMichael Ellerman } 330e2186023SMichael Ellerman 331e2186023SMichael Ellerman static ssize_t __used store_subcores_per_core(struct device *dev, 332e2186023SMichael Ellerman struct device_attribute *attr, const char *buf, 333e2186023SMichael Ellerman size_t count) 334e2186023SMichael Ellerman { 335e2186023SMichael Ellerman unsigned long val; 336e2186023SMichael Ellerman int rc; 337e2186023SMichael Ellerman 338e2186023SMichael Ellerman /* We are serialised by the attribute lock */ 339e2186023SMichael Ellerman 340e2186023SMichael Ellerman rc = sscanf(buf, "%lx", &val); 341e2186023SMichael Ellerman if (rc != 1) 342e2186023SMichael Ellerman return -EINVAL; 343e2186023SMichael Ellerman 344e2186023SMichael Ellerman switch (val) { 345e2186023SMichael Ellerman case 1: 346e2186023SMichael Ellerman case 2: 347e2186023SMichael Ellerman case 4: 348e2186023SMichael Ellerman if (subcores_per_core == val) 349e2186023SMichael Ellerman /* Nothing to do */ 350e2186023SMichael Ellerman goto out; 351e2186023SMichael Ellerman break; 352e2186023SMichael Ellerman default: 353e2186023SMichael Ellerman return -EINVAL; 354e2186023SMichael Ellerman } 355e2186023SMichael Ellerman 356e2186023SMichael Ellerman rc = set_subcores_per_core(val); 357e2186023SMichael Ellerman if (rc) 358e2186023SMichael Ellerman return rc; 359e2186023SMichael Ellerman 360e2186023SMichael Ellerman out: 361e2186023SMichael Ellerman return count; 362e2186023SMichael Ellerman } 363e2186023SMichael Ellerman 364e2186023SMichael Ellerman static ssize_t show_subcores_per_core(struct device *dev, 365e2186023SMichael Ellerman struct device_attribute *attr, char *buf) 366e2186023SMichael Ellerman { 367e2186023SMichael Ellerman return sprintf(buf, "%x\n", subcores_per_core); 368e2186023SMichael Ellerman } 369e2186023SMichael Ellerman 370e2186023SMichael Ellerman static DEVICE_ATTR(subcores_per_core, 0644, 371e2186023SMichael Ellerman show_subcores_per_core, store_subcores_per_core); 372e2186023SMichael Ellerman 373e2186023SMichael Ellerman static int subcore_init(void) 374e2186023SMichael Ellerman { 375e2186023SMichael Ellerman if (!cpu_has_feature(CPU_FTR_ARCH_207S)) 376e2186023SMichael Ellerman return 0; 377e2186023SMichael Ellerman 378e2186023SMichael Ellerman /* 379e2186023SMichael Ellerman * We need all threads in a core to be present to split/unsplit so 380e2186023SMichael Ellerman * continue only if max_cpus are aligned to threads_per_core. 381e2186023SMichael Ellerman */ 382e2186023SMichael Ellerman if (setup_max_cpus % threads_per_core) 383e2186023SMichael Ellerman return 0; 384e2186023SMichael Ellerman 385e2186023SMichael Ellerman BUG_ON(!alloc_cpumask_var(&cpu_offline_mask, GFP_KERNEL)); 386e2186023SMichael Ellerman 387e2186023SMichael Ellerman set_subcores_per_core(1); 388e2186023SMichael Ellerman 389e2186023SMichael Ellerman return device_create_file(cpu_subsys.dev_root, 390e2186023SMichael Ellerman &dev_attr_subcores_per_core); 391e2186023SMichael Ellerman } 392e2186023SMichael Ellerman machine_device_initcall(powernv, subcore_init); 393