42 	struct sugov_policy	*sg_policy;  member
62 static bool sugov_should_update_freq(struct sugov_policy *sg_policy, u64 time)  in sugov_should_update_freq()  argument
81 	if (!cpufreq_this_cpu_can_update(sg_policy->policy))  in sugov_should_update_freq()
84 	if (unlikely(sg_policy->limits_changed)) {  in sugov_should_update_freq()
85 		sg_policy->limits_changed = false;  in sugov_should_update_freq()
86 		sg_policy->need_freq_update = true;  in sugov_should_update_freq()
90 	delta_ns = time - sg_policy->last_freq_update_time;  in sugov_should_update_freq()
92 	return delta_ns >= sg_policy->freq_update_delay_ns;  in sugov_should_update_freq()
95 static bool sugov_update_next_freq(struct sugov_policy *sg_policy, u64 time,  in sugov_update_next_freq()  argument
98 	if (sg_policy->need_freq_update)  in sugov_update_next_freq()
99 		sg_policy->need_freq_update = cpufreq_driver_test_flags(CPUFREQ_NEED_UPDATE_LIMITS);  in sugov_update_next_freq()
100 	else if (sg_policy->next_freq == next_freq)  in sugov_update_next_freq()
103 	sg_policy->next_freq = next_freq;  in sugov_update_next_freq()
104 	sg_policy->last_freq_update_time = time;  in sugov_update_next_freq()
109 static void sugov_deferred_update(struct sugov_policy *sg_policy)  in sugov_deferred_update()  argument
111 	if (!sg_policy->work_in_progress) {  in sugov_deferred_update()
112 		sg_policy->work_in_progress = true;  in sugov_deferred_update()
113 		irq_work_queue(&sg_policy->irq_work);  in sugov_deferred_update()
139 static unsigned int get_next_freq(struct sugov_policy *sg_policy,  in get_next_freq()  argument
142 	struct cpufreq_policy *policy = sg_policy->policy;  in get_next_freq()
149 	if (freq == sg_policy->cached_raw_freq && !sg_policy->need_freq_update)  in get_next_freq()
150 		return sg_policy->next_freq;  in get_next_freq()
152 	sg_policy->cached_raw_freq = freq;  in get_next_freq()
310 		sg_cpu->sg_policy->limits_changed = true;  in ignore_dl_rate_limit()
322 	if (!sugov_should_update_freq(sg_cpu->sg_policy, time))  in sugov_update_single_common()
335 	struct sugov_policy *sg_policy = sg_cpu->sg_policy;  in sugov_update_single_freq()  local
336 	unsigned int cached_freq = sg_policy->cached_raw_freq;  in sugov_update_single_freq()
345 	next_f = get_next_freq(sg_policy, sg_cpu->util, max_cap);  in sugov_update_single_freq()
353 	    sugov_cpu_is_busy(sg_cpu) && next_f < sg_policy->next_freq &&  in sugov_update_single_freq()
354 	    !sg_policy->need_freq_update) {  in sugov_update_single_freq()
355 		next_f = sg_policy->next_freq;  in sugov_update_single_freq()
358 		sg_policy->cached_raw_freq = cached_freq;  in sugov_update_single_freq()
361 	if (!sugov_update_next_freq(sg_policy, time, next_f))  in sugov_update_single_freq()
369 	if (sg_policy->policy->fast_switch_enabled) {  in sugov_update_single_freq()
370 		cpufreq_driver_fast_switch(sg_policy->policy, next_f);  in sugov_update_single_freq()
372 		raw_spin_lock(&sg_policy->update_lock);  in sugov_update_single_freq()
373 		sugov_deferred_update(sg_policy);  in sugov_update_single_freq()
374 		raw_spin_unlock(&sg_policy->update_lock);  in sugov_update_single_freq()
413 	sg_cpu->sg_policy->last_freq_update_time = time;  in sugov_update_single_perf()
418 	struct sugov_policy *sg_policy = sg_cpu->sg_policy;  in sugov_next_freq_shared()  local
419 	struct cpufreq_policy *policy = sg_policy->policy;  in sugov_next_freq_shared()
434 	return get_next_freq(sg_policy, util, max_cap);  in sugov_next_freq_shared()
441 	struct sugov_policy *sg_policy = sg_cpu->sg_policy;  in sugov_update_shared()  local
444 	raw_spin_lock(&sg_policy->update_lock);  in sugov_update_shared()
451 	if (sugov_should_update_freq(sg_policy, time)) {  in sugov_update_shared()
454 		if (!sugov_update_next_freq(sg_policy, time, next_f))  in sugov_update_shared()
457 		if (sg_policy->policy->fast_switch_enabled)  in sugov_update_shared()
458 			cpufreq_driver_fast_switch(sg_policy->policy, next_f);  in sugov_update_shared()
460 			sugov_deferred_update(sg_policy);  in sugov_update_shared()
463 	raw_spin_unlock(&sg_policy->update_lock);  in sugov_update_shared()
468 	struct sugov_policy *sg_policy = container_of(work, struct sugov_policy, work);  in sugov_work()  local
482 	raw_spin_lock_irqsave(&sg_policy->update_lock, flags);  in sugov_work()
483 	freq = sg_policy->next_freq;  in sugov_work()
484 	sg_policy->work_in_progress = false;  in sugov_work()
485 	raw_spin_unlock_irqrestore(&sg_policy->update_lock, flags);  in sugov_work()
487 	mutex_lock(&sg_policy->work_lock);  in sugov_work()
488 	__cpufreq_driver_target(sg_policy->policy, freq, CPUFREQ_RELATION_L);  in sugov_work()
489 	mutex_unlock(&sg_policy->work_lock);  in sugov_work()
494 	struct sugov_policy *sg_policy;  in sugov_irq_work()  local
496 	sg_policy = container_of(irq_work, struct sugov_policy, irq_work);  in sugov_irq_work()
498 	kthread_queue_work(&sg_policy->worker, &sg_policy->work);  in sugov_irq_work()
522 	struct sugov_policy *sg_policy;  in rate_limit_us_store()  local
530 	list_for_each_entry(sg_policy, &attr_set->policy_list, tunables_hook)  in rate_limit_us_store()
531 		sg_policy->freq_update_delay_ns = rate_limit_us * NSEC_PER_USEC;  in rate_limit_us_store()
563 	struct sugov_policy *sg_policy;  in sugov_policy_alloc()  local
565 	sg_policy = kzalloc(sizeof(*sg_policy), GFP_KERNEL);  in sugov_policy_alloc()
566 	if (!sg_policy)  in sugov_policy_alloc()
569 	sg_policy->policy = policy;  in sugov_policy_alloc()
570 	raw_spin_lock_init(&sg_policy->update_lock);  in sugov_policy_alloc()
571 	return sg_policy;  in sugov_policy_alloc()
574 static void sugov_policy_free(struct sugov_policy *sg_policy)  in sugov_policy_free()  argument
576 	kfree(sg_policy);  in sugov_policy_free()
579 static int sugov_kthread_create(struct sugov_policy *sg_policy)  in sugov_kthread_create()  argument
596 	struct cpufreq_policy *policy = sg_policy->policy;  in sugov_kthread_create()
603 	kthread_init_work(&sg_policy->work, sugov_work);  in sugov_kthread_create()
604 	kthread_init_worker(&sg_policy->worker);  in sugov_kthread_create()
605 	thread = kthread_create(kthread_worker_fn, &sg_policy->worker,  in sugov_kthread_create()
620 	sg_policy->thread = thread;  in sugov_kthread_create()
622 	init_irq_work(&sg_policy->irq_work, sugov_irq_work);  in sugov_kthread_create()
623 	mutex_init(&sg_policy->work_lock);  in sugov_kthread_create()
630 static void sugov_kthread_stop(struct sugov_policy *sg_policy)  in sugov_kthread_stop()  argument
633 	if (sg_policy->policy->fast_switch_enabled)  in sugov_kthread_stop()
636 	kthread_flush_worker(&sg_policy->worker);  in sugov_kthread_stop()
637 	kthread_stop(sg_policy->thread);  in sugov_kthread_stop()
638 	mutex_destroy(&sg_policy->work_lock);  in sugov_kthread_stop()
641 static struct sugov_tunables *sugov_tunables_alloc(struct sugov_policy *sg_policy)  in sugov_tunables_alloc()  argument
647 		gov_attr_set_init(&tunables->attr_set, &sg_policy->tunables_hook);  in sugov_tunables_alloc()
662 	struct sugov_policy *sg_policy;  in sugov_init()  local
672 	sg_policy = sugov_policy_alloc(policy);  in sugov_init()
673 	if (!sg_policy) {  in sugov_init()
678 	ret = sugov_kthread_create(sg_policy);  in sugov_init()
689 		policy->governor_data = sg_policy;  in sugov_init()
690 		sg_policy->tunables = global_tunables;  in sugov_init()
692 		gov_attr_set_get(&global_tunables->attr_set, &sg_policy->tunables_hook);  in sugov_init()
696 	tunables = sugov_tunables_alloc(sg_policy);  in sugov_init()
704 	policy->governor_data = sg_policy;  in sugov_init()
705 	sg_policy->tunables = tunables;  in sugov_init()
723 	sugov_kthread_stop(sg_policy);  in sugov_init()
727 	sugov_policy_free(sg_policy);  in sugov_init()
738 	struct sugov_policy *sg_policy = policy->governor_data;  in sugov_exit()  local
739 	struct sugov_tunables *tunables = sg_policy->tunables;  in sugov_exit()
744 	count = gov_attr_set_put(&tunables->attr_set, &sg_policy->tunables_hook);  in sugov_exit()
751 	sugov_kthread_stop(sg_policy);  in sugov_exit()
752 	sugov_policy_free(sg_policy);  in sugov_exit()
758 	struct sugov_policy *sg_policy = policy->governor_data;  in sugov_start()  local
762 	sg_policy->freq_update_delay_ns	= sg_policy->tunables->rate_limit_us * NSEC_PER_USEC;  in sugov_start()
763 	sg_policy->last_freq_update_time	= 0;  in sugov_start()
764 	sg_policy->next_freq			= 0;  in sugov_start()
765 	sg_policy->work_in_progress		= false;  in sugov_start()
766 	sg_policy->limits_changed		= false;  in sugov_start()
767 	sg_policy->cached_raw_freq		= 0;  in sugov_start()
769 	sg_policy->need_freq_update = cpufreq_driver_test_flags(CPUFREQ_NEED_UPDATE_LIMITS);  in sugov_start()
776 		sg_cpu->sg_policy		= sg_policy;  in sugov_start()
796 	struct sugov_policy *sg_policy = policy->governor_data;  in sugov_stop()  local
805 		irq_work_sync(&sg_policy->irq_work);  in sugov_stop()
806 		kthread_cancel_work_sync(&sg_policy->work);  in sugov_stop()
812 	struct sugov_policy *sg_policy = policy->governor_data;  in sugov_limits()  local
815 		mutex_lock(&sg_policy->work_lock);  in sugov_limits()
817 		mutex_unlock(&sg_policy->work_lock);  in sugov_limits()
820 	sg_policy->limits_changed = true;  in sugov_limits()