1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * cpuidle-pseries - idle state cpuidle driver. 4 * Adapted from drivers/idle/intel_idle.c and 5 * drivers/acpi/processor_idle.c 6 * 7 */ 8 9 #include <linux/kernel.h> 10 #include <linux/module.h> 11 #include <linux/init.h> 12 #include <linux/moduleparam.h> 13 #include <linux/cpuidle.h> 14 #include <linux/cpu.h> 15 #include <linux/notifier.h> 16 17 #include <asm/paca.h> 18 #include <asm/reg.h> 19 #include <asm/machdep.h> 20 #include <asm/firmware.h> 21 #include <asm/runlatch.h> 22 #include <asm/plpar_wrappers.h> 23 24 struct cpuidle_driver pseries_idle_driver = { 25 .name = "pseries_idle", 26 .owner = THIS_MODULE, 27 }; 28 29 static int max_idle_state __read_mostly; 30 static struct cpuidle_state *cpuidle_state_table __read_mostly; 31 static u64 snooze_timeout __read_mostly; 32 static bool snooze_timeout_en __read_mostly; 33 34 static inline void idle_loop_prolog(unsigned long *in_purr) 35 { 36 ppc64_runlatch_off(); 37 *in_purr = mfspr(SPRN_PURR); 38 /* 39 * Indicate to the HV that we are idle. Now would be 40 * a good time to find other work to dispatch. 41 */ 42 get_lppaca()->idle = 1; 43 } 44 45 static inline void idle_loop_epilog(unsigned long in_purr) 46 { 47 u64 wait_cycles; 48 49 wait_cycles = be64_to_cpu(get_lppaca()->wait_state_cycles); 50 wait_cycles += mfspr(SPRN_PURR) - in_purr; 51 get_lppaca()->wait_state_cycles = cpu_to_be64(wait_cycles); 52 get_lppaca()->idle = 0; 53 54 ppc64_runlatch_on(); 55 } 56 57 static int snooze_loop(struct cpuidle_device *dev, 58 struct cpuidle_driver *drv, 59 int index) 60 { 61 unsigned long in_purr; 62 u64 snooze_exit_time; 63 64 set_thread_flag(TIF_POLLING_NRFLAG); 65 66 idle_loop_prolog(&in_purr); 67 local_irq_enable(); 68 snooze_exit_time = get_tb() + snooze_timeout; 69 70 while (!need_resched()) { 71 HMT_low(); 72 HMT_very_low(); 73 if (likely(snooze_timeout_en) && get_tb() > snooze_exit_time) { 74 /* 75 * Task has not woken up but we are exiting the polling 76 * loop anyway. Require a barrier after polling is 77 * cleared to order subsequent test of need_resched(). 78 */ 79 clear_thread_flag(TIF_POLLING_NRFLAG); 80 smp_mb(); 81 break; 82 } 83 } 84 85 HMT_medium(); 86 clear_thread_flag(TIF_POLLING_NRFLAG); 87 88 local_irq_disable(); 89 90 idle_loop_epilog(in_purr); 91 92 return index; 93 } 94 95 static void check_and_cede_processor(void) 96 { 97 /* 98 * Ensure our interrupt state is properly tracked, 99 * also checks if no interrupt has occurred while we 100 * were soft-disabled 101 */ 102 if (prep_irq_for_idle()) { 103 cede_processor(); 104 #ifdef CONFIG_TRACE_IRQFLAGS 105 /* Ensure that H_CEDE returns with IRQs on */ 106 if (WARN_ON(!(mfmsr() & MSR_EE))) 107 __hard_irq_enable(); 108 #endif 109 } 110 } 111 112 static int dedicated_cede_loop(struct cpuidle_device *dev, 113 struct cpuidle_driver *drv, 114 int index) 115 { 116 unsigned long in_purr; 117 118 idle_loop_prolog(&in_purr); 119 get_lppaca()->donate_dedicated_cpu = 1; 120 121 HMT_medium(); 122 check_and_cede_processor(); 123 124 local_irq_disable(); 125 get_lppaca()->donate_dedicated_cpu = 0; 126 127 idle_loop_epilog(in_purr); 128 129 return index; 130 } 131 132 static int shared_cede_loop(struct cpuidle_device *dev, 133 struct cpuidle_driver *drv, 134 int index) 135 { 136 unsigned long in_purr; 137 138 idle_loop_prolog(&in_purr); 139 140 /* 141 * Yield the processor to the hypervisor. We return if 142 * an external interrupt occurs (which are driven prior 143 * to returning here) or if a prod occurs from another 144 * processor. When returning here, external interrupts 145 * are enabled. 146 */ 147 check_and_cede_processor(); 148 149 local_irq_disable(); 150 idle_loop_epilog(in_purr); 151 152 return index; 153 } 154 155 /* 156 * States for dedicated partition case. 157 */ 158 static struct cpuidle_state dedicated_states[] = { 159 { /* Snooze */ 160 .name = "snooze", 161 .desc = "snooze", 162 .exit_latency = 0, 163 .target_residency = 0, 164 .enter = &snooze_loop }, 165 { /* CEDE */ 166 .name = "CEDE", 167 .desc = "CEDE", 168 .exit_latency = 10, 169 .target_residency = 100, 170 .enter = &dedicated_cede_loop }, 171 }; 172 173 /* 174 * States for shared partition case. 175 */ 176 static struct cpuidle_state shared_states[] = { 177 { /* Snooze */ 178 .name = "snooze", 179 .desc = "snooze", 180 .exit_latency = 0, 181 .target_residency = 0, 182 .enter = &snooze_loop }, 183 { /* Shared Cede */ 184 .name = "Shared Cede", 185 .desc = "Shared Cede", 186 .exit_latency = 10, 187 .target_residency = 100, 188 .enter = &shared_cede_loop }, 189 }; 190 191 static int pseries_cpuidle_cpu_online(unsigned int cpu) 192 { 193 struct cpuidle_device *dev = per_cpu(cpuidle_devices, cpu); 194 195 if (dev && cpuidle_get_driver()) { 196 cpuidle_pause_and_lock(); 197 cpuidle_enable_device(dev); 198 cpuidle_resume_and_unlock(); 199 } 200 return 0; 201 } 202 203 static int pseries_cpuidle_cpu_dead(unsigned int cpu) 204 { 205 struct cpuidle_device *dev = per_cpu(cpuidle_devices, cpu); 206 207 if (dev && cpuidle_get_driver()) { 208 cpuidle_pause_and_lock(); 209 cpuidle_disable_device(dev); 210 cpuidle_resume_and_unlock(); 211 } 212 return 0; 213 } 214 215 /* 216 * pseries_cpuidle_driver_init() 217 */ 218 static int pseries_cpuidle_driver_init(void) 219 { 220 int idle_state; 221 struct cpuidle_driver *drv = &pseries_idle_driver; 222 223 drv->state_count = 0; 224 225 for (idle_state = 0; idle_state < max_idle_state; ++idle_state) { 226 /* Is the state not enabled? */ 227 if (cpuidle_state_table[idle_state].enter == NULL) 228 continue; 229 230 drv->states[drv->state_count] = /* structure copy */ 231 cpuidle_state_table[idle_state]; 232 233 drv->state_count += 1; 234 } 235 236 return 0; 237 } 238 239 /* 240 * pseries_idle_probe() 241 * Choose state table for shared versus dedicated partition 242 */ 243 static int pseries_idle_probe(void) 244 { 245 246 if (cpuidle_disable != IDLE_NO_OVERRIDE) 247 return -ENODEV; 248 249 if (firmware_has_feature(FW_FEATURE_SPLPAR)) { 250 if (lppaca_shared_proc(get_lppaca())) { 251 cpuidle_state_table = shared_states; 252 max_idle_state = ARRAY_SIZE(shared_states); 253 } else { 254 cpuidle_state_table = dedicated_states; 255 max_idle_state = ARRAY_SIZE(dedicated_states); 256 } 257 } else 258 return -ENODEV; 259 260 if (max_idle_state > 1) { 261 snooze_timeout_en = true; 262 snooze_timeout = cpuidle_state_table[1].target_residency * 263 tb_ticks_per_usec; 264 } 265 return 0; 266 } 267 268 static int __init pseries_processor_idle_init(void) 269 { 270 int retval; 271 272 retval = pseries_idle_probe(); 273 if (retval) 274 return retval; 275 276 pseries_cpuidle_driver_init(); 277 retval = cpuidle_register(&pseries_idle_driver, NULL); 278 if (retval) { 279 printk(KERN_DEBUG "Registration of pseries driver failed.\n"); 280 return retval; 281 } 282 283 retval = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, 284 "cpuidle/pseries:online", 285 pseries_cpuidle_cpu_online, NULL); 286 WARN_ON(retval < 0); 287 retval = cpuhp_setup_state_nocalls(CPUHP_CPUIDLE_DEAD, 288 "cpuidle/pseries:DEAD", NULL, 289 pseries_cpuidle_cpu_dead); 290 WARN_ON(retval < 0); 291 printk(KERN_DEBUG "pseries_idle_driver registered\n"); 292 return 0; 293 } 294 295 device_initcall(pseries_processor_idle_init); 296