53 			   struct cpuidle_device *dev)  in cpuidle_not_available()  argument
55 	return off || !initialized || !drv || !dev || !dev->enabled;  in cpuidle_not_available()
65 	struct cpuidle_device *dev = __this_cpu_read(cpuidle_devices);  in cpuidle_play_dead()  local
66 	struct cpuidle_driver *drv = cpuidle_get_cpu_driver(dev);  in cpuidle_play_dead()
75 			return drv->states[i].enter_dead(dev, i);  in cpuidle_play_dead()
81 			      struct cpuidle_device *dev,  in find_deepest_state()  argument
92 		if (dev->states_usage[i].disable ||  in find_deepest_state()
115 	struct cpuidle_device *dev;  in cpuidle_use_deepest_state()  local
118 	dev = cpuidle_get_device();  in cpuidle_use_deepest_state()
119 	if (dev)  in cpuidle_use_deepest_state()
120 		dev->forced_idle_latency_limit_ns = latency_limit_ns;  in cpuidle_use_deepest_state()
133 			       struct cpuidle_device *dev,  in cpuidle_find_deepest_state()  argument
136 	return find_deepest_state(drv, dev, latency_limit_ns, 0, false);  in cpuidle_find_deepest_state()
141 					 struct cpuidle_device *dev, int index)  in enter_s2idle_proper()  argument
162 	target_state->enter_s2idle(dev, drv, index);  in enter_s2idle_proper()
174 	dev->states_usage[index].s2idle_time += ktime_us_delta(time_end, time_start);  in enter_s2idle_proper()
175 	dev->states_usage[index].s2idle_usage++;  in enter_s2idle_proper()
187 int cpuidle_enter_s2idle(struct cpuidle_driver *drv, struct cpuidle_device *dev)  in cpuidle_enter_s2idle()  argument
196 	index = find_deepest_state(drv, dev, U64_MAX, 0, true);  in cpuidle_enter_s2idle()
198 		enter_s2idle_proper(drv, dev, index);  in cpuidle_enter_s2idle()
211 noinstr int cpuidle_enter_state(struct cpuidle_device *dev,  in cpuidle_enter_state()  argument
229 		index = find_deepest_state(drv, dev, target_state->exit_latency_ns,  in cpuidle_enter_state()
240 		leave_mm(dev->cpu);  in cpuidle_enter_state()
245 	trace_cpu_idle(index, dev->cpu);  in cpuidle_enter_state()
267 	entered_state = target_state->enter(dev, drv, index);  in cpuidle_enter_state()
280 	trace_cpu_idle(PWR_EVENT_EXIT, dev->cpu);  in cpuidle_enter_state()
302 		dev->last_residency_ns = diff;  in cpuidle_enter_state()
303 		dev->states_usage[entered_state].time_ns += diff;  in cpuidle_enter_state()
304 		dev->states_usage[entered_state].usage++;  in cpuidle_enter_state()
308 				if (dev->states_usage[i].disable)  in cpuidle_enter_state()
312 				dev->states_usage[entered_state].above++;  in cpuidle_enter_state()
313 				trace_cpu_idle_miss(dev->cpu, entered_state, false);  in cpuidle_enter_state()
318 				if (dev->states_usage[i].disable)  in cpuidle_enter_state()
326 					dev->states_usage[entered_state].below++;  in cpuidle_enter_state()
327 					trace_cpu_idle_miss(dev->cpu, entered_state, true);  in cpuidle_enter_state()
334 		dev->last_residency_ns = 0;  in cpuidle_enter_state()
335 		dev->states_usage[index].rejected++;  in cpuidle_enter_state()
356 int cpuidle_select(struct cpuidle_driver *drv, struct cpuidle_device *dev,  in cpuidle_select()  argument
359 	return cpuidle_curr_governor->select(drv, dev, stop_tick);  in cpuidle_select()
372 int cpuidle_enter(struct cpuidle_driver *drv, struct cpuidle_device *dev,  in cpuidle_enter()  argument
383 	WRITE_ONCE(dev->next_hrtimer, tick_nohz_get_next_hrtimer());  in cpuidle_enter()
386 		ret = cpuidle_enter_state_coupled(dev, drv, index);  in cpuidle_enter()
388 		ret = cpuidle_enter_state(dev, drv, index);  in cpuidle_enter()
390 	WRITE_ONCE(dev->next_hrtimer, 0);  in cpuidle_enter()
402 void cpuidle_reflect(struct cpuidle_device *dev, int index)  in cpuidle_reflect()  argument
405 		cpuidle_curr_governor->reflect(dev, index);  in cpuidle_reflect()
430 		      struct cpuidle_device *dev)  in cpuidle_poll_time()  argument
437 	if (dev->poll_limit_ns)  in cpuidle_poll_time()
438 		return dev->poll_limit_ns;  in cpuidle_poll_time()
444 		if (dev->states_usage[i].disable)  in cpuidle_poll_time()
455 	dev->poll_limit_ns = limit_ns;  in cpuidle_poll_time()
457 	return dev->poll_limit_ns;  in cpuidle_poll_time()
534 int cpuidle_enable_device(struct cpuidle_device *dev)  in cpuidle_enable_device()  argument
539 	if (!dev)  in cpuidle_enable_device()
542 	if (dev->enabled)  in cpuidle_enable_device()
548 	drv = cpuidle_get_cpu_driver(dev);  in cpuidle_enable_device()
553 	if (!dev->registered)  in cpuidle_enable_device()
556 	ret = cpuidle_add_device_sysfs(dev);  in cpuidle_enable_device()
561 		ret = cpuidle_curr_governor->enable(drv, dev);  in cpuidle_enable_device()
568 	dev->enabled = 1;  in cpuidle_enable_device()
574 	cpuidle_remove_device_sysfs(dev);  in cpuidle_enable_device()
588 void cpuidle_disable_device(struct cpuidle_device *dev)  in cpuidle_disable_device()  argument
590 	struct cpuidle_driver *drv = cpuidle_get_cpu_driver(dev);  in cpuidle_disable_device()
592 	if (!dev || !dev->enabled)  in cpuidle_disable_device()
598 	dev->enabled = 0;  in cpuidle_disable_device()
601 		cpuidle_curr_governor->disable(drv, dev);  in cpuidle_disable_device()
603 	cpuidle_remove_device_sysfs(dev);  in cpuidle_disable_device()
609 static void __cpuidle_unregister_device(struct cpuidle_device *dev)  in __cpuidle_unregister_device()  argument
611 	struct cpuidle_driver *drv = cpuidle_get_cpu_driver(dev);  in __cpuidle_unregister_device()
613 	list_del(&dev->device_list);  in __cpuidle_unregister_device()
614 	per_cpu(cpuidle_devices, dev->cpu) = NULL;  in __cpuidle_unregister_device()
617 	dev->registered = 0;  in __cpuidle_unregister_device()
620 static void __cpuidle_device_init(struct cpuidle_device *dev)  in __cpuidle_device_init()  argument
622 	memset(dev->states_usage, 0, sizeof(dev->states_usage));  in __cpuidle_device_init()
623 	dev->last_residency_ns = 0;  in __cpuidle_device_init()
624 	dev->next_hrtimer = 0;  in __cpuidle_device_init()
634 static int __cpuidle_register_device(struct cpuidle_device *dev)  in __cpuidle_register_device()  argument
636 	struct cpuidle_driver *drv = cpuidle_get_cpu_driver(dev);  in __cpuidle_register_device()
644 			dev->states_usage[i].disable |= CPUIDLE_STATE_DISABLED_BY_DRIVER;  in __cpuidle_register_device()
647 			dev->states_usage[i].disable |= CPUIDLE_STATE_DISABLED_BY_USER;  in __cpuidle_register_device()
650 	per_cpu(cpuidle_devices, dev->cpu) = dev;  in __cpuidle_register_device()
651 	list_add(&dev->device_list, &cpuidle_detected_devices);  in __cpuidle_register_device()
653 	ret = cpuidle_coupled_register_device(dev);  in __cpuidle_register_device()
655 		__cpuidle_unregister_device(dev);  in __cpuidle_register_device()
657 		dev->registered = 1;  in __cpuidle_register_device()
666 int cpuidle_register_device(struct cpuidle_device *dev)  in cpuidle_register_device()  argument
670 	if (!dev)  in cpuidle_register_device()
675 	if (dev->registered)  in cpuidle_register_device()
678 	__cpuidle_device_init(dev);  in cpuidle_register_device()
680 	ret = __cpuidle_register_device(dev);  in cpuidle_register_device()
684 	ret = cpuidle_add_sysfs(dev);  in cpuidle_register_device()
688 	ret = cpuidle_enable_device(dev);  in cpuidle_register_device()
700 	cpuidle_remove_sysfs(dev);  in cpuidle_register_device()
702 	__cpuidle_unregister_device(dev);  in cpuidle_register_device()
712 void cpuidle_unregister_device(struct cpuidle_device *dev)  in cpuidle_unregister_device()  argument
714 	if (!dev || dev->registered == 0)  in cpuidle_unregister_device()
719 	cpuidle_disable_device(dev);  in cpuidle_unregister_device()
721 	cpuidle_remove_sysfs(dev);  in cpuidle_unregister_device()
723 	__cpuidle_unregister_device(dev);  in cpuidle_unregister_device()
725 	cpuidle_coupled_unregister_device(dev);  in cpuidle_unregister_device()