1 // SPDX-License-Identifier: GPL-2.0
7 #include <linux/clk-provider.h>
49 	 * Mutex to synchronize between de-init sequence and re-starting LMh
79 	dev = get_cpu_device(policy->cpu);  in qcom_cpufreq_set_bw()
81 		return -ENODEV;  in qcom_cpufreq_set_bw()
115 	struct qcom_cpufreq_data *data = policy->driver_data;  in qcom_cpufreq_hw_target_index()
117 	unsigned long freq = policy->freq_table[index].frequency;  in qcom_cpufreq_hw_target_index()
120 	writel_relaxed(index, data->base + soc_data->reg_perf_state);  in qcom_cpufreq_hw_target_index()
122 	if (data->per_core_dcvs)  in qcom_cpufreq_hw_target_index()
123 		for (i = 1; i < cpumask_weight(policy->related_cpus); i++)  in qcom_cpufreq_hw_target_index()
124 			writel_relaxed(index, data->base + soc_data->reg_perf_state + i * 4);  in qcom_cpufreq_hw_target_index()
136 	if (qcom_cpufreq.soc_data->reg_current_vote)  in qcom_lmh_get_throttle_freq()
137 		lval = readl_relaxed(data->base + qcom_cpufreq.soc_data->reg_current_vote) & 0x3ff;  in qcom_lmh_get_throttle_freq()
139 		lval = readl_relaxed(data->base + qcom_cpufreq.soc_data->reg_domain_state) & 0xff;  in qcom_lmh_get_throttle_freq()
144 /* Get the frequency requested by the cpufreq core for the CPU */
154 	data = policy->driver_data;  in qcom_cpufreq_get_freq()
157 	index = readl_relaxed(data->base + soc_data->reg_perf_state);  in qcom_cpufreq_get_freq()
158 	index = min(index, LUT_MAX_ENTRIES - 1);  in qcom_cpufreq_get_freq()
160 	return policy->freq_table[index].frequency;  in qcom_cpufreq_get_freq()
170 	data = policy->driver_data;  in __qcom_cpufreq_hw_get()
172 	if (data->throttle_irq >= 0)  in __qcom_cpufreq_hw_get()
186 	struct qcom_cpufreq_data *data = policy->driver_data;  in qcom_cpufreq_hw_fast_switch()
191 	index = policy->cached_resolved_idx;  in qcom_cpufreq_hw_fast_switch()
192 	writel_relaxed(index, data->base + soc_data->reg_perf_state);  in qcom_cpufreq_hw_fast_switch()
194 	if (data->per_core_dcvs)  in qcom_cpufreq_hw_fast_switch()
195 		for (i = 1; i < cpumask_weight(policy->related_cpus); i++)  in qcom_cpufreq_hw_fast_switch()
196 			writel_relaxed(index, data->base + soc_data->reg_perf_state + i * 4);  in qcom_cpufreq_hw_fast_switch()
198 	return policy->freq_table[index].frequency;  in qcom_cpufreq_hw_fast_switch()
210 	struct qcom_cpufreq_data *drv_data = policy->driver_data;  in qcom_cpufreq_hw_read_lut()
215 		return -ENOMEM;  in qcom_cpufreq_hw_read_lut()
219 		/* Disable all opps and cross-validate against LUT later */  in qcom_cpufreq_hw_read_lut()
229 	} else if (ret != -ENODEV) {  in qcom_cpufreq_hw_read_lut()
234 		policy->fast_switch_possible = true;  in qcom_cpufreq_hw_read_lut()
239 		data = readl_relaxed(drv_data->base + soc_data->reg_freq_lut +  in qcom_cpufreq_hw_read_lut()
240 				      i * soc_data->lut_row_size);  in qcom_cpufreq_hw_read_lut()
245 		data = readl_relaxed(drv_data->base + soc_data->reg_volt_lut +  in qcom_cpufreq_hw_read_lut()
246 				      i * soc_data->lut_row_size);  in qcom_cpufreq_hw_read_lut()
256 				table[i].frequency = freq;  in qcom_cpufreq_hw_read_lut()
261 				table[i].frequency = CPUFREQ_ENTRY_INVALID;  in qcom_cpufreq_hw_read_lut()
265 			table[i].frequency = CPUFREQ_ENTRY_INVALID;  in qcom_cpufreq_hw_read_lut()
273 			struct cpufreq_frequency_table *prev = &table[i - 1];  in qcom_cpufreq_hw_read_lut()
276 			 * Only treat the last frequency that might be a boost  in qcom_cpufreq_hw_read_lut()
277 			 * as the boost frequency  in qcom_cpufreq_hw_read_lut()
279 			if (prev->frequency == CPUFREQ_ENTRY_INVALID) {  in qcom_cpufreq_hw_read_lut()
281 					prev->frequency = prev_freq;  in qcom_cpufreq_hw_read_lut()
282 					prev->flags = CPUFREQ_BOOST_FREQ;  in qcom_cpufreq_hw_read_lut()
295 	table[i].frequency = CPUFREQ_TABLE_END;  in qcom_cpufreq_hw_read_lut()
296 	policy->freq_table = table;  in qcom_cpufreq_hw_read_lut()
297 	dev_pm_opp_set_sharing_cpus(cpu_dev, policy->cpus);  in qcom_cpufreq_hw_read_lut()
313 		ret = of_parse_phandle_with_args(cpu_np, "qcom,freq-domain",  in qcom_get_related_cpus()
314 						 "#freq-domain-cells", 0,  in qcom_get_related_cpus()
327 	struct cpufreq_policy *policy = data->policy;  in qcom_lmh_dcvs_notify()
328 	int cpu = cpumask_first(policy->related_cpus);  in qcom_lmh_dcvs_notify()
334 	 * Get the h/w throttled frequency, normalize it using the  in qcom_lmh_dcvs_notify()
340 	if (IS_ERR(opp) && PTR_ERR(opp) == -ERANGE)  in qcom_lmh_dcvs_notify()
352 	arch_update_thermal_pressure(policy->related_cpus, throttled_freq);  in qcom_lmh_dcvs_notify()
358 	mutex_lock(&data->throttle_lock);  in qcom_lmh_dcvs_notify()
359 	if (data->cancel_throttle)  in qcom_lmh_dcvs_notify()
360 		goto out;  in qcom_lmh_dcvs_notify()
363 	 * If h/w throttled frequency is higher than what cpufreq has requested  in qcom_lmh_dcvs_notify()
367 		enable_irq(data->throttle_irq);  in qcom_lmh_dcvs_notify()
369 		mod_delayed_work(system_highpri_wq, &data->throttle_work,  in qcom_lmh_dcvs_notify()
372 out:  in qcom_lmh_dcvs_notify()
373 	mutex_unlock(&data->throttle_lock);  in qcom_lmh_dcvs_notify()
389 	disable_irq_nosync(c_data->throttle_irq);  in qcom_lmh_dcvs_handle_irq()
390 	schedule_delayed_work(&c_data->throttle_work, 0);  in qcom_lmh_dcvs_handle_irq()
392 	if (qcom_cpufreq.soc_data->reg_intr_clr)  in qcom_lmh_dcvs_handle_irq()
394 			       c_data->base + qcom_cpufreq.soc_data->reg_intr_clr);  in qcom_lmh_dcvs_handle_irq()
421 	{ .compatible = "qcom,cpufreq-hw", .data = &qcom_soc_data },
422 	{ .compatible = "qcom,cpufreq-epss", .data = &epss_soc_data },
429 	struct qcom_cpufreq_data *data = policy->driver_data;  in qcom_cpufreq_hw_lmh_init()
437 	data->throttle_irq = platform_get_irq_optional(pdev, index);  in qcom_cpufreq_hw_lmh_init()
438 	if (data->throttle_irq == -ENXIO)  in qcom_cpufreq_hw_lmh_init()
440 	if (data->throttle_irq < 0)  in qcom_cpufreq_hw_lmh_init()
441 		return data->throttle_irq;  in qcom_cpufreq_hw_lmh_init()
443 	data->cancel_throttle = false;  in qcom_cpufreq_hw_lmh_init()
445 	mutex_init(&data->throttle_lock);  in qcom_cpufreq_hw_lmh_init()
446 	INIT_DEFERRABLE_WORK(&data->throttle_work, qcom_lmh_dcvs_poll);  in qcom_cpufreq_hw_lmh_init()
448 	snprintf(data->irq_name, sizeof(data->irq_name), "dcvsh-irq-%u", policy->cpu);  in qcom_cpufreq_hw_lmh_init()
449 	ret = request_threaded_irq(data->throttle_irq, NULL, qcom_lmh_dcvs_handle_irq,  in qcom_cpufreq_hw_lmh_init()
450 				   IRQF_ONESHOT | IRQF_NO_AUTOEN, data->irq_name, data);  in qcom_cpufreq_hw_lmh_init()
452 		dev_err(&pdev->dev, "Error registering %s: %d\n", data->irq_name, ret);  in qcom_cpufreq_hw_lmh_init()
456 	ret = irq_set_affinity_and_hint(data->throttle_irq, policy->cpus);  in qcom_cpufreq_hw_lmh_init()
458 		dev_err(&pdev->dev, "Failed to set CPU affinity of %s[%d]\n",  in qcom_cpufreq_hw_lmh_init()
459 			data->irq_name, data->throttle_irq);  in qcom_cpufreq_hw_lmh_init()
466 	struct qcom_cpufreq_data *data = policy->driver_data;  in qcom_cpufreq_hw_cpu_online()
470 	if (data->throttle_irq <= 0)  in qcom_cpufreq_hw_cpu_online()
473 	mutex_lock(&data->throttle_lock);  in qcom_cpufreq_hw_cpu_online()
474 	data->cancel_throttle = false;  in qcom_cpufreq_hw_cpu_online()
475 	mutex_unlock(&data->throttle_lock);  in qcom_cpufreq_hw_cpu_online()
477 	ret = irq_set_affinity_and_hint(data->throttle_irq, policy->cpus);  in qcom_cpufreq_hw_cpu_online()
479 		dev_err(&pdev->dev, "Failed to set CPU affinity of %s[%d]\n",  in qcom_cpufreq_hw_cpu_online()
480 			data->irq_name, data->throttle_irq);  in qcom_cpufreq_hw_cpu_online()
487 	struct qcom_cpufreq_data *data = policy->driver_data;  in qcom_cpufreq_hw_cpu_offline()
489 	if (data->throttle_irq <= 0)  in qcom_cpufreq_hw_cpu_offline()
492 	mutex_lock(&data->throttle_lock);  in qcom_cpufreq_hw_cpu_offline()
493 	data->cancel_throttle = true;  in qcom_cpufreq_hw_cpu_offline()
494 	mutex_unlock(&data->throttle_lock);  in qcom_cpufreq_hw_cpu_offline()
496 	cancel_delayed_work_sync(&data->throttle_work);  in qcom_cpufreq_hw_cpu_offline()
497 	irq_set_affinity_and_hint(data->throttle_irq, NULL);  in qcom_cpufreq_hw_cpu_offline()
498 	disable_irq_nosync(data->throttle_irq);  in qcom_cpufreq_hw_cpu_offline()
505 	if (data->throttle_irq <= 0)  in qcom_cpufreq_hw_lmh_exit()
508 	free_irq(data->throttle_irq, data);  in qcom_cpufreq_hw_lmh_exit()
514 	struct device *dev = &pdev->dev;  in qcom_cpufreq_hw_cpu_init()
521 	cpu_dev = get_cpu_device(policy->cpu);  in qcom_cpufreq_hw_cpu_init()
524 		       policy->cpu);  in qcom_cpufreq_hw_cpu_init()
525 		return -ENODEV;  in qcom_cpufreq_hw_cpu_init()
528 	cpu_np = of_cpu_device_node_get(policy->cpu);  in qcom_cpufreq_hw_cpu_init()
530 		return -EINVAL;  in qcom_cpufreq_hw_cpu_init()
532 	ret = of_parse_phandle_with_args(cpu_np, "qcom,freq-domain",  in qcom_cpufreq_hw_cpu_init()
533 					 "#freq-domain-cells", 0, &args);  in qcom_cpufreq_hw_cpu_init()
542 	if (!(readl_relaxed(data->base + qcom_cpufreq.soc_data->reg_enable) & 0x1)) {  in qcom_cpufreq_hw_cpu_init()
543 		dev_err(dev, "Domain-%d cpufreq hardware not enabled\n", index);  in qcom_cpufreq_hw_cpu_init()
544 		return -ENODEV;  in qcom_cpufreq_hw_cpu_init()
547 	if (readl_relaxed(data->base + qcom_cpufreq.soc_data->reg_dcvs_ctrl) & 0x1)  in qcom_cpufreq_hw_cpu_init()
548 		data->per_core_dcvs = true;  in qcom_cpufreq_hw_cpu_init()
550 	qcom_get_related_cpus(index, policy->cpus);  in qcom_cpufreq_hw_cpu_init()
552 	policy->driver_data = data;  in qcom_cpufreq_hw_cpu_init()
553 	policy->dvfs_possible_from_any_cpu = true;  in qcom_cpufreq_hw_cpu_init()
554 	data->policy = policy;  in qcom_cpufreq_hw_cpu_init()
558 		dev_err(dev, "Domain-%d failed to read LUT\n", index);  in qcom_cpufreq_hw_cpu_init()
565 		return -ENODEV;  in qcom_cpufreq_hw_cpu_init()
579 	struct device *cpu_dev = get_cpu_device(policy->cpu);  in qcom_cpufreq_hw_cpu_exit()
580 	struct qcom_cpufreq_data *data = policy->driver_data;  in qcom_cpufreq_hw_cpu_exit()
583 	dev_pm_opp_of_cpumask_remove_table(policy->related_cpus);  in qcom_cpufreq_hw_cpu_exit()
585 	kfree(policy->freq_table);  in qcom_cpufreq_hw_cpu_exit()
593 	struct qcom_cpufreq_data *data = policy->driver_data;  in qcom_cpufreq_ready()
595 	if (data->throttle_irq >= 0)  in qcom_cpufreq_ready()
596 		enable_irq(data->throttle_irq);  in qcom_cpufreq_ready()
618 	.name		= "qcom-cpufreq-hw",
627 	return __qcom_cpufreq_hw_get(data->policy) * HZ_PER_KHZ;  in qcom_cpufreq_hw_recalc_rate()
637 	req->rate = qcom_cpufreq_hw_recalc_rate(hw, 0);  in qcom_cpufreq_hw_determine_rate()
650 	struct device *dev = &pdev->dev;  in qcom_cpufreq_hw_driver_probe()
652 	struct clk *clk;  in qcom_cpufreq_hw_driver_probe()  local
655 	clk = clk_get(dev, "xo");  in qcom_cpufreq_hw_driver_probe()
656 	if (IS_ERR(clk))  in qcom_cpufreq_hw_driver_probe()
657 		return PTR_ERR(clk);  in qcom_cpufreq_hw_driver_probe()
659 	xo_rate = clk_get_rate(clk);  in qcom_cpufreq_hw_driver_probe()
660 	clk_put(clk);  in qcom_cpufreq_hw_driver_probe()
662 	clk = clk_get(dev, "alternate");  in qcom_cpufreq_hw_driver_probe()
663 	if (IS_ERR(clk))  in qcom_cpufreq_hw_driver_probe()
664 		return PTR_ERR(clk);  in qcom_cpufreq_hw_driver_probe()
666 	cpu_hw_rate = clk_get_rate(clk) / CLK_HW_DIV;  in qcom_cpufreq_hw_driver_probe()
667 	clk_put(clk);  in qcom_cpufreq_hw_driver_probe()
674 		return -EPROBE_DEFER;  in qcom_cpufreq_hw_driver_probe()
687 		return -ENOMEM;  in qcom_cpufreq_hw_driver_probe()
691 		return -ENODEV;  in qcom_cpufreq_hw_driver_probe()
695 		return -ENOMEM;  in qcom_cpufreq_hw_driver_probe()
697 	clk_data->num = num_domains;  in qcom_cpufreq_hw_driver_probe()
710 		data->base = base;  in qcom_cpufreq_hw_driver_probe()
712 		/* Register CPU clock for each frequency domain */  in qcom_cpufreq_hw_driver_probe()
715 			return -ENOMEM;  in qcom_cpufreq_hw_driver_probe()
719 		data->cpu_clk.init = &clk_init;  in qcom_cpufreq_hw_driver_probe()
721 		ret = devm_clk_hw_register(dev, &data->cpu_clk);  in qcom_cpufreq_hw_driver_probe()
728 		clk_data->hws[i] = &data->cpu_clk;  in qcom_cpufreq_hw_driver_probe()
756 		.name = "qcom-cpufreq-hw",