136 static int set_boost(struct cpufreq_policy *policy, int val)  in set_boost()  argument
138 	on_each_cpu_mask(policy->cpus, boost_set_msr_each,  in set_boost()
141 		 cpumask_pr_args(policy->cpus), str_enabled_disabled(val));  in set_boost()
146 static ssize_t show_freqdomain_cpus(struct cpufreq_policy *policy, char *buf)  in show_freqdomain_cpus()  argument
148 	struct acpi_cpufreq_data *data = policy->driver_data;  in show_freqdomain_cpus()
159 static ssize_t store_cpb(struct cpufreq_policy *policy, const char *buf,  in store_cpb()  argument
173 	set_boost(policy, val);  in store_cpb()
179 static ssize_t show_cpb(struct cpufreq_policy *policy, char *buf)  in show_cpb()  argument
201 static unsigned extract_io(struct cpufreq_policy *policy, u32 value)  in extract_io()  argument
203 	struct acpi_cpufreq_data *data = policy->driver_data;  in extract_io()
211 			return policy->freq_table[i].frequency;  in extract_io()
216 static unsigned extract_msr(struct cpufreq_policy *policy, u32 msr)  in extract_msr()  argument
218 	struct acpi_cpufreq_data *data = policy->driver_data;  in extract_msr()
231 	cpufreq_for_each_entry(pos, policy->freq_table)  in extract_msr()
234 	return policy->freq_table[0].frequency;  in extract_msr()
237 static unsigned extract_freq(struct cpufreq_policy *policy, u32 val)  in extract_freq()  argument
239 	struct acpi_cpufreq_data *data = policy->driver_data;  in extract_freq()
244 		return extract_msr(policy, val);  in extract_freq()
246 		return extract_io(policy, val);  in extract_freq()
370 	struct cpufreq_policy *policy;  in get_cur_freq_on_cpu()  local
376 	policy = cpufreq_cpu_get_raw(cpu);  in get_cur_freq_on_cpu()
377 	if (unlikely(!policy))  in get_cur_freq_on_cpu()
380 	data = policy->driver_data;  in get_cur_freq_on_cpu()
381 	if (unlikely(!data || !policy->freq_table))  in get_cur_freq_on_cpu()
384 	cached_freq = policy->freq_table[to_perf_data(data)->state].frequency;  in get_cur_freq_on_cpu()
385 	freq = extract_freq(policy, get_cur_val(cpumask_of(cpu), data));  in get_cur_freq_on_cpu()
399 static unsigned int check_freqs(struct cpufreq_policy *policy,  in check_freqs()  argument
402 	struct acpi_cpufreq_data *data = policy->driver_data;  in check_freqs()
407 		cur_freq = extract_freq(policy, get_cur_val(mask, data));  in check_freqs()
415 static int acpi_cpufreq_target(struct cpufreq_policy *policy,  in acpi_cpufreq_target()  argument
418 	struct acpi_cpufreq_data *data = policy->driver_data;  in acpi_cpufreq_target()
429 	next_perf_state = policy->freq_table[index].driver_data;  in acpi_cpufreq_target()
444 	 * stopped, so we can rely on the stability of policy->cpus.  in acpi_cpufreq_target()
446 	mask = policy->shared_type == CPUFREQ_SHARED_TYPE_ANY ?  in acpi_cpufreq_target()
447 		cpumask_of(policy->cpu) : policy->cpus;  in acpi_cpufreq_target()
452 		if (!check_freqs(policy, mask,  in acpi_cpufreq_target()
453 				 policy->freq_table[index].frequency)) {  in acpi_cpufreq_target()
454 			pr_debug("%s (%d)\n", __func__, policy->cpu);  in acpi_cpufreq_target()
465 static unsigned int acpi_cpufreq_fast_switch(struct cpufreq_policy *policy,  in acpi_cpufreq_fast_switch()  argument
468 	struct acpi_cpufreq_data *data = policy->driver_data;  in acpi_cpufreq_fast_switch()
476 	if (policy->cached_target_freq == target_freq)  in acpi_cpufreq_fast_switch()
477 		index = policy->cached_resolved_idx;  in acpi_cpufreq_fast_switch()
479 		index = cpufreq_table_find_index_dl(policy, target_freq,  in acpi_cpufreq_fast_switch()
482 	entry = &policy->freq_table[index];  in acpi_cpufreq_fast_switch()
670 static int acpi_cpufreq_cpu_init(struct cpufreq_policy *policy)  in acpi_cpufreq_cpu_init()  argument
675 	unsigned int cpu = policy->cpu;  in acpi_cpufreq_cpu_init()
706 	policy->driver_data = data;  in acpi_cpufreq_cpu_init()
715 	policy->shared_type = perf->shared_type;  in acpi_cpufreq_cpu_init()
718 	 * Will let policy->cpus know about dependency only when software  in acpi_cpufreq_cpu_init()
721 	if (policy->shared_type == CPUFREQ_SHARED_TYPE_ALL ||  in acpi_cpufreq_cpu_init()
722 	    policy->shared_type == CPUFREQ_SHARED_TYPE_ANY) {  in acpi_cpufreq_cpu_init()
723 		cpumask_copy(policy->cpus, perf->shared_cpu_map);  in acpi_cpufreq_cpu_init()
729 	if (bios_with_sw_any_bug && !policy_is_shared(policy)) {  in acpi_cpufreq_cpu_init()
730 		policy->shared_type = CPUFREQ_SHARED_TYPE_ALL;  in acpi_cpufreq_cpu_init()
731 		cpumask_copy(policy->cpus, topology_core_cpumask(cpu));  in acpi_cpufreq_cpu_init()
736 		cpumask_clear(policy->cpus);  in acpi_cpufreq_cpu_init()
737 		cpumask_set_cpu(cpu, policy->cpus);  in acpi_cpufreq_cpu_init()
740 		policy->shared_type = CPUFREQ_SHARED_TYPE_HW;  in acpi_cpufreq_cpu_init()
801 	policy->cpuinfo.transition_latency = 0;  in acpi_cpufreq_cpu_init()
804 		    policy->cpuinfo.transition_latency)  in acpi_cpufreq_cpu_init()
805 			policy->cpuinfo.transition_latency =  in acpi_cpufreq_cpu_init()
811 	    policy->cpuinfo.transition_latency > 20 * 1000) {  in acpi_cpufreq_cpu_init()
812 		policy->cpuinfo.transition_latency = 20 * 1000;  in acpi_cpufreq_cpu_init()
839 		policy->cpuinfo.max_freq = freq * max_boost_ratio >> SCHED_CAPACITY_SHIFT;  in acpi_cpufreq_cpu_init()
850 	policy->freq_table = freq_table;  in acpi_cpufreq_cpu_init()
856 		 * The core will not set policy->cur, because  in acpi_cpufreq_cpu_init()
861 		policy->cur = acpi_cpufreq_guess_freq(data, policy->cpu);  in acpi_cpufreq_cpu_init()
887 	policy->fast_switch_possible = !acpi_pstate_strict &&  in acpi_cpufreq_cpu_init()
888 		!(policy_is_shared(policy) && policy->shared_type != CPUFREQ_SHARED_TYPE_ANY);  in acpi_cpufreq_cpu_init()
894 		set_boost(policy, acpi_cpufreq_driver.boost_enabled);  in acpi_cpufreq_cpu_init()
895 		policy->boost_enabled = acpi_cpufreq_driver.boost_enabled;  in acpi_cpufreq_cpu_init()
906 	policy->driver_data = NULL;  in acpi_cpufreq_cpu_init()
911 static int acpi_cpufreq_cpu_exit(struct cpufreq_policy *policy)  in acpi_cpufreq_cpu_exit()  argument
913 	struct acpi_cpufreq_data *data = policy->driver_data;  in acpi_cpufreq_cpu_exit()
917 	cpufreq_boost_down_prep(policy->cpu);  in acpi_cpufreq_cpu_exit()
918 	policy->fast_switch_possible = false;  in acpi_cpufreq_cpu_exit()
919 	policy->driver_data = NULL;  in acpi_cpufreq_cpu_exit()
922 	kfree(policy->freq_table);  in acpi_cpufreq_cpu_exit()
928 static int acpi_cpufreq_resume(struct cpufreq_policy *policy)  in acpi_cpufreq_resume()  argument
930 	struct acpi_cpufreq_data *data = policy->driver_data;  in acpi_cpufreq_resume()