1 /* 2 * driver.c - driver support 3 * 4 * (C) 2006-2007 Venkatesh Pallipadi <venkatesh.pallipadi@intel.com> 5 * Shaohua Li <shaohua.li@intel.com> 6 * Adam Belay <abelay@novell.com> 7 * 8 * This code is licenced under the GPL. 9 */ 10 11 #include <linux/mutex.h> 12 #include <linux/module.h> 13 #include <linux/sched.h> 14 #include <linux/sched/idle.h> 15 #include <linux/cpuidle.h> 16 #include <linux/cpumask.h> 17 #include <linux/tick.h> 18 #include <linux/cpu.h> 19 20 #include "cpuidle.h" 21 22 DEFINE_SPINLOCK(cpuidle_driver_lock); 23 24 #ifdef CONFIG_CPU_IDLE_MULTIPLE_DRIVERS 25 26 static DEFINE_PER_CPU(struct cpuidle_driver *, cpuidle_drivers); 27 28 /** 29 * __cpuidle_get_cpu_driver - return the cpuidle driver tied to a CPU. 30 * @cpu: the CPU handled by the driver 31 * 32 * Returns a pointer to struct cpuidle_driver or NULL if no driver has been 33 * registered for @cpu. 34 */ 35 static struct cpuidle_driver *__cpuidle_get_cpu_driver(int cpu) 36 { 37 return per_cpu(cpuidle_drivers, cpu); 38 } 39 40 /** 41 * __cpuidle_unset_driver - unset per CPU driver variables. 42 * @drv: a valid pointer to a struct cpuidle_driver 43 * 44 * For each CPU in the driver's CPU mask, unset the registered driver per CPU 45 * variable. If @drv is different from the registered driver, the corresponding 46 * variable is not cleared. 47 */ 48 static inline void __cpuidle_unset_driver(struct cpuidle_driver *drv) 49 { 50 int cpu; 51 52 for_each_cpu(cpu, drv->cpumask) { 53 54 if (drv != __cpuidle_get_cpu_driver(cpu)) 55 continue; 56 57 per_cpu(cpuidle_drivers, cpu) = NULL; 58 } 59 } 60 61 /** 62 * __cpuidle_set_driver - set per CPU driver variables for the given driver. 63 * @drv: a valid pointer to a struct cpuidle_driver 64 * 65 * Returns 0 on success, -EBUSY if any CPU in the cpumask have a driver 66 * different from drv already. 67 */ 68 static inline int __cpuidle_set_driver(struct cpuidle_driver *drv) 69 { 70 int cpu; 71 72 for_each_cpu(cpu, drv->cpumask) { 73 struct cpuidle_driver *old_drv; 74 75 old_drv = __cpuidle_get_cpu_driver(cpu); 76 if (old_drv && old_drv != drv) 77 return -EBUSY; 78 } 79 80 for_each_cpu(cpu, drv->cpumask) 81 per_cpu(cpuidle_drivers, cpu) = drv; 82 83 return 0; 84 } 85 86 #else 87 88 static struct cpuidle_driver *cpuidle_curr_driver; 89 90 /** 91 * __cpuidle_get_cpu_driver - return the global cpuidle driver pointer. 92 * @cpu: ignored without the multiple driver support 93 * 94 * Return a pointer to a struct cpuidle_driver object or NULL if no driver was 95 * previously registered. 96 */ 97 static inline struct cpuidle_driver *__cpuidle_get_cpu_driver(int cpu) 98 { 99 return cpuidle_curr_driver; 100 } 101 102 /** 103 * __cpuidle_set_driver - assign the global cpuidle driver variable. 104 * @drv: pointer to a struct cpuidle_driver object 105 * 106 * Returns 0 on success, -EBUSY if the driver is already registered. 107 */ 108 static inline int __cpuidle_set_driver(struct cpuidle_driver *drv) 109 { 110 if (cpuidle_curr_driver) 111 return -EBUSY; 112 113 cpuidle_curr_driver = drv; 114 115 return 0; 116 } 117 118 /** 119 * __cpuidle_unset_driver - unset the global cpuidle driver variable. 120 * @drv: a pointer to a struct cpuidle_driver 121 * 122 * Reset the global cpuidle variable to NULL. If @drv does not match the 123 * registered driver, do nothing. 124 */ 125 static inline void __cpuidle_unset_driver(struct cpuidle_driver *drv) 126 { 127 if (drv == cpuidle_curr_driver) 128 cpuidle_curr_driver = NULL; 129 } 130 131 #endif 132 133 /** 134 * cpuidle_setup_broadcast_timer - enable/disable the broadcast timer on a cpu 135 * @arg: a void pointer used to match the SMP cross call API 136 * 137 * If @arg is NULL broadcast is disabled otherwise enabled 138 * 139 * This function is executed per CPU by an SMP cross call. It's not 140 * supposed to be called directly. 141 */ 142 static void cpuidle_setup_broadcast_timer(void *arg) 143 { 144 if (arg) 145 tick_broadcast_enable(); 146 else 147 tick_broadcast_disable(); 148 } 149 150 /** 151 * __cpuidle_driver_init - initialize the driver's internal data 152 * @drv: a valid pointer to a struct cpuidle_driver 153 */ 154 static void __cpuidle_driver_init(struct cpuidle_driver *drv) 155 { 156 int i; 157 158 drv->refcnt = 0; 159 160 /* 161 * Use all possible CPUs as the default, because if the kernel boots 162 * with some CPUs offline and then we online one of them, the CPU 163 * notifier has to know which driver to assign. 164 */ 165 if (!drv->cpumask) 166 drv->cpumask = (struct cpumask *)cpu_possible_mask; 167 168 for (i = 0; i < drv->state_count; i++) { 169 struct cpuidle_state *s = &drv->states[i]; 170 171 /* 172 * Look for the timer stop flag in the different states and if 173 * it is found, indicate that the broadcast timer has to be set 174 * up. 175 */ 176 if (s->flags & CPUIDLE_FLAG_TIMER_STOP) 177 drv->bctimer = 1; 178 179 /* 180 * The core will use the target residency and exit latency 181 * values in nanoseconds, but allow drivers to provide them in 182 * microseconds too. 183 */ 184 if (s->target_residency > 0) 185 s->target_residency_ns = s->target_residency * NSEC_PER_USEC; 186 187 if (s->exit_latency > 0) 188 s->exit_latency_ns = s->exit_latency * NSEC_PER_USEC; 189 } 190 } 191 192 /** 193 * __cpuidle_register_driver: register the driver 194 * @drv: a valid pointer to a struct cpuidle_driver 195 * 196 * Do some sanity checks, initialize the driver, assign the driver to the 197 * global cpuidle driver variable(s) and set up the broadcast timer if the 198 * cpuidle driver has some states that shut down the local timer. 199 * 200 * Returns 0 on success, a negative error code otherwise: 201 * * -EINVAL if the driver pointer is NULL or no idle states are available 202 * * -ENODEV if the cpuidle framework is disabled 203 * * -EBUSY if the driver is already assigned to the global variable(s) 204 */ 205 static int __cpuidle_register_driver(struct cpuidle_driver *drv) 206 { 207 int ret; 208 209 if (!drv || !drv->state_count) 210 return -EINVAL; 211 212 ret = cpuidle_coupled_state_verify(drv); 213 if (ret) 214 return ret; 215 216 if (cpuidle_disabled()) 217 return -ENODEV; 218 219 __cpuidle_driver_init(drv); 220 221 ret = __cpuidle_set_driver(drv); 222 if (ret) 223 return ret; 224 225 if (drv->bctimer) 226 on_each_cpu_mask(drv->cpumask, cpuidle_setup_broadcast_timer, 227 (void *)1, 1); 228 229 return 0; 230 } 231 232 /** 233 * __cpuidle_unregister_driver - unregister the driver 234 * @drv: a valid pointer to a struct cpuidle_driver 235 * 236 * Check if the driver is no longer in use, reset the global cpuidle driver 237 * variable(s) and disable the timer broadcast notification mechanism if it was 238 * in use. 239 * 240 */ 241 static void __cpuidle_unregister_driver(struct cpuidle_driver *drv) 242 { 243 if (WARN_ON(drv->refcnt > 0)) 244 return; 245 246 if (drv->bctimer) { 247 drv->bctimer = 0; 248 on_each_cpu_mask(drv->cpumask, cpuidle_setup_broadcast_timer, 249 NULL, 1); 250 } 251 252 __cpuidle_unset_driver(drv); 253 } 254 255 /** 256 * cpuidle_register_driver - registers a driver 257 * @drv: a pointer to a valid struct cpuidle_driver 258 * 259 * Register the driver under a lock to prevent concurrent attempts to 260 * [un]register the driver from occuring at the same time. 261 * 262 * Returns 0 on success, a negative error code (returned by 263 * __cpuidle_register_driver()) otherwise. 264 */ 265 int cpuidle_register_driver(struct cpuidle_driver *drv) 266 { 267 struct cpuidle_governor *gov; 268 int ret; 269 270 spin_lock(&cpuidle_driver_lock); 271 ret = __cpuidle_register_driver(drv); 272 spin_unlock(&cpuidle_driver_lock); 273 274 if (!ret && !strlen(param_governor) && drv->governor && 275 (cpuidle_get_driver() == drv)) { 276 mutex_lock(&cpuidle_lock); 277 gov = cpuidle_find_governor(drv->governor); 278 if (gov) { 279 cpuidle_prev_governor = cpuidle_curr_governor; 280 if (cpuidle_switch_governor(gov) < 0) 281 cpuidle_prev_governor = NULL; 282 } 283 mutex_unlock(&cpuidle_lock); 284 } 285 286 return ret; 287 } 288 EXPORT_SYMBOL_GPL(cpuidle_register_driver); 289 290 /** 291 * cpuidle_unregister_driver - unregisters a driver 292 * @drv: a pointer to a valid struct cpuidle_driver 293 * 294 * Unregisters the cpuidle driver under a lock to prevent concurrent attempts 295 * to [un]register the driver from occuring at the same time. @drv has to 296 * match the currently registered driver. 297 */ 298 void cpuidle_unregister_driver(struct cpuidle_driver *drv) 299 { 300 bool enabled = (cpuidle_get_driver() == drv); 301 302 spin_lock(&cpuidle_driver_lock); 303 __cpuidle_unregister_driver(drv); 304 spin_unlock(&cpuidle_driver_lock); 305 306 if (!enabled) 307 return; 308 309 mutex_lock(&cpuidle_lock); 310 if (cpuidle_prev_governor) { 311 if (!cpuidle_switch_governor(cpuidle_prev_governor)) 312 cpuidle_prev_governor = NULL; 313 } 314 mutex_unlock(&cpuidle_lock); 315 } 316 EXPORT_SYMBOL_GPL(cpuidle_unregister_driver); 317 318 /** 319 * cpuidle_get_driver - return the driver tied to the current CPU. 320 * 321 * Returns a struct cpuidle_driver pointer, or NULL if no driver is registered. 322 */ 323 struct cpuidle_driver *cpuidle_get_driver(void) 324 { 325 struct cpuidle_driver *drv; 326 int cpu; 327 328 cpu = get_cpu(); 329 drv = __cpuidle_get_cpu_driver(cpu); 330 put_cpu(); 331 332 return drv; 333 } 334 EXPORT_SYMBOL_GPL(cpuidle_get_driver); 335 336 /** 337 * cpuidle_get_cpu_driver - return the driver registered for a CPU. 338 * @dev: a valid pointer to a struct cpuidle_device 339 * 340 * Returns a struct cpuidle_driver pointer, or NULL if no driver is registered 341 * for the CPU associated with @dev. 342 */ 343 struct cpuidle_driver *cpuidle_get_cpu_driver(struct cpuidle_device *dev) 344 { 345 if (!dev) 346 return NULL; 347 348 return __cpuidle_get_cpu_driver(dev->cpu); 349 } 350 EXPORT_SYMBOL_GPL(cpuidle_get_cpu_driver); 351 352 /** 353 * cpuidle_driver_ref - get a reference to the driver. 354 * 355 * Increment the reference counter of the cpuidle driver associated with 356 * the current CPU. 357 * 358 * Returns a pointer to the driver, or NULL if the current CPU has no driver. 359 */ 360 struct cpuidle_driver *cpuidle_driver_ref(void) 361 { 362 struct cpuidle_driver *drv; 363 364 spin_lock(&cpuidle_driver_lock); 365 366 drv = cpuidle_get_driver(); 367 if (drv) 368 drv->refcnt++; 369 370 spin_unlock(&cpuidle_driver_lock); 371 return drv; 372 } 373 374 /** 375 * cpuidle_driver_unref - puts down the refcount for the driver 376 * 377 * Decrement the reference counter of the cpuidle driver associated with 378 * the current CPU. 379 */ 380 void cpuidle_driver_unref(void) 381 { 382 struct cpuidle_driver *drv; 383 384 spin_lock(&cpuidle_driver_lock); 385 386 drv = cpuidle_get_driver(); 387 if (drv && !WARN_ON(drv->refcnt <= 0)) 388 drv->refcnt--; 389 390 spin_unlock(&cpuidle_driver_lock); 391 } 392 393 /** 394 * cpuidle_driver_state_disabled - Disable or enable an idle state 395 * @drv: cpuidle driver owning the state 396 * @idx: State index 397 * @disable: Whether or not to disable the state 398 */ 399 void cpuidle_driver_state_disabled(struct cpuidle_driver *drv, int idx, 400 bool disable) 401 { 402 unsigned int cpu; 403 404 mutex_lock(&cpuidle_lock); 405 406 for_each_cpu(cpu, drv->cpumask) { 407 struct cpuidle_device *dev = per_cpu(cpuidle_devices, cpu); 408 409 if (!dev) 410 continue; 411 412 if (disable) 413 dev->states_usage[idx].disable |= CPUIDLE_STATE_DISABLED_BY_DRIVER; 414 else 415 dev->states_usage[idx].disable &= ~CPUIDLE_STATE_DISABLED_BY_DRIVER; 416 } 417 418 mutex_unlock(&cpuidle_lock); 419 } 420