12849dd8bSTaniya Das // SPDX-License-Identifier: GPL-2.0 22849dd8bSTaniya Das /* 32849dd8bSTaniya Das * Copyright (c) 2018, The Linux Foundation. All rights reserved. 42849dd8bSTaniya Das */ 52849dd8bSTaniya Das 62849dd8bSTaniya Das #include <linux/bitfield.h> 72849dd8bSTaniya Das #include <linux/cpufreq.h> 82849dd8bSTaniya Das #include <linux/init.h> 951c843cfSSibi Sankar #include <linux/interconnect.h> 10275157b3SThara Gopinath #include <linux/interrupt.h> 112849dd8bSTaniya Das #include <linux/kernel.h> 122849dd8bSTaniya Das #include <linux/module.h> 132849dd8bSTaniya Das #include <linux/of_address.h> 142849dd8bSTaniya Das #include <linux/of_platform.h> 1555538fbcSTaniya Das #include <linux/pm_opp.h> 162849dd8bSTaniya Das #include <linux/slab.h> 17275157b3SThara Gopinath #include <linux/spinlock.h> 182849dd8bSTaniya Das 192849dd8bSTaniya Das #define LUT_MAX_ENTRIES 40U 202849dd8bSTaniya Das #define LUT_SRC GENMASK(31, 30) 212849dd8bSTaniya Das #define LUT_L_VAL GENMASK(7, 0) 222849dd8bSTaniya Das #define LUT_CORE_COUNT GENMASK(18, 16) 2355538fbcSTaniya Das #define LUT_VOLT GENMASK(11, 0) 242849dd8bSTaniya Das #define CLK_HW_DIV 2 250eae1e37SSibi Sankar #define LUT_TURBO_IND 1 262849dd8bSTaniya Das 27275157b3SThara Gopinath #define HZ_PER_KHZ 1000 28275157b3SThara Gopinath 29dcd1fd72SManivannan Sadhasivam struct qcom_cpufreq_soc_data { 30dcd1fd72SManivannan Sadhasivam u32 reg_enable; 31c377d4baSBjorn Andersson u32 reg_dcvs_ctrl; 32dcd1fd72SManivannan Sadhasivam u32 reg_freq_lut; 33dcd1fd72SManivannan Sadhasivam u32 reg_volt_lut; 34275157b3SThara Gopinath u32 reg_current_vote; 35dcd1fd72SManivannan Sadhasivam u32 reg_perf_state; 36dcd1fd72SManivannan Sadhasivam u8 lut_row_size; 37dcd1fd72SManivannan Sadhasivam }; 38dcd1fd72SManivannan Sadhasivam 39dcd1fd72SManivannan Sadhasivam struct qcom_cpufreq_data { 40dcd1fd72SManivannan Sadhasivam void __iomem *base; 4167fc209bSShawn Guo struct resource *res; 42dcd1fd72SManivannan Sadhasivam const struct qcom_cpufreq_soc_data *soc_data; 43275157b3SThara Gopinath 44275157b3SThara Gopinath /* 45275157b3SThara Gopinath * Mutex to synchronize between de-init sequence and re-starting LMh 46275157b3SThara Gopinath * polling/interrupts 47275157b3SThara Gopinath */ 48275157b3SThara Gopinath struct mutex throttle_lock; 49275157b3SThara Gopinath int throttle_irq; 50be6592edSArd Biesheuvel char irq_name[15]; 51275157b3SThara Gopinath bool cancel_throttle; 52275157b3SThara Gopinath struct delayed_work throttle_work; 53275157b3SThara Gopinath struct cpufreq_policy *policy; 54c377d4baSBjorn Andersson 55c377d4baSBjorn Andersson bool per_core_dcvs; 56dcd1fd72SManivannan Sadhasivam }; 572849dd8bSTaniya Das 582849dd8bSTaniya Das static unsigned long cpu_hw_rate, xo_rate; 5951c843cfSSibi Sankar static bool icc_scaling_enabled; 6051c843cfSSibi Sankar 6151c843cfSSibi Sankar static int qcom_cpufreq_set_bw(struct cpufreq_policy *policy, 6251c843cfSSibi Sankar unsigned long freq_khz) 6351c843cfSSibi Sankar { 6451c843cfSSibi Sankar unsigned long freq_hz = freq_khz * 1000; 6551c843cfSSibi Sankar struct dev_pm_opp *opp; 6651c843cfSSibi Sankar struct device *dev; 6751c843cfSSibi Sankar int ret; 6851c843cfSSibi Sankar 6951c843cfSSibi Sankar dev = get_cpu_device(policy->cpu); 7051c843cfSSibi Sankar if (!dev) 7151c843cfSSibi Sankar return -ENODEV; 7251c843cfSSibi Sankar 7351c843cfSSibi Sankar opp = dev_pm_opp_find_freq_exact(dev, freq_hz, true); 7451c843cfSSibi Sankar if (IS_ERR(opp)) 7551c843cfSSibi Sankar return PTR_ERR(opp); 7651c843cfSSibi Sankar 778d25157fSViresh Kumar ret = dev_pm_opp_set_opp(dev, opp); 7851c843cfSSibi Sankar dev_pm_opp_put(opp); 7951c843cfSSibi Sankar return ret; 8051c843cfSSibi Sankar } 8151c843cfSSibi Sankar 8251c843cfSSibi Sankar static int qcom_cpufreq_update_opp(struct device *cpu_dev, 8351c843cfSSibi Sankar unsigned long freq_khz, 8451c843cfSSibi Sankar unsigned long volt) 8551c843cfSSibi Sankar { 8651c843cfSSibi Sankar unsigned long freq_hz = freq_khz * 1000; 8751c843cfSSibi Sankar int ret; 8851c843cfSSibi Sankar 8951c843cfSSibi Sankar /* Skip voltage update if the opp table is not available */ 9051c843cfSSibi Sankar if (!icc_scaling_enabled) 9151c843cfSSibi Sankar return dev_pm_opp_add(cpu_dev, freq_hz, volt); 9251c843cfSSibi Sankar 9351c843cfSSibi Sankar ret = dev_pm_opp_adjust_voltage(cpu_dev, freq_hz, volt, volt, volt); 9451c843cfSSibi Sankar if (ret) { 9551c843cfSSibi Sankar dev_err(cpu_dev, "Voltage update failed freq=%ld\n", freq_khz); 9651c843cfSSibi Sankar return ret; 9751c843cfSSibi Sankar } 9851c843cfSSibi Sankar 9951c843cfSSibi Sankar return dev_pm_opp_enable(cpu_dev, freq_hz); 10051c843cfSSibi Sankar } 1012849dd8bSTaniya Das 1022849dd8bSTaniya Das static int qcom_cpufreq_hw_target_index(struct cpufreq_policy *policy, 1032849dd8bSTaniya Das unsigned int index) 1042849dd8bSTaniya Das { 105dcd1fd72SManivannan Sadhasivam struct qcom_cpufreq_data *data = policy->driver_data; 106dcd1fd72SManivannan Sadhasivam const struct qcom_cpufreq_soc_data *soc_data = data->soc_data; 107ada54f35SDouglas RAILLARD unsigned long freq = policy->freq_table[index].frequency; 108c377d4baSBjorn Andersson unsigned int i; 1092849dd8bSTaniya Das 110dcd1fd72SManivannan Sadhasivam writel_relaxed(index, data->base + soc_data->reg_perf_state); 1112849dd8bSTaniya Das 112c377d4baSBjorn Andersson if (data->per_core_dcvs) 113c377d4baSBjorn Andersson for (i = 1; i < cpumask_weight(policy->related_cpus); i++) 114c377d4baSBjorn Andersson writel_relaxed(index, data->base + soc_data->reg_perf_state + i * 4); 115c377d4baSBjorn Andersson 11651c843cfSSibi Sankar if (icc_scaling_enabled) 11751c843cfSSibi Sankar qcom_cpufreq_set_bw(policy, freq); 11851c843cfSSibi Sankar 1192849dd8bSTaniya Das return 0; 1202849dd8bSTaniya Das } 1212849dd8bSTaniya Das 1222849dd8bSTaniya Das static unsigned int qcom_cpufreq_hw_get(unsigned int cpu) 1232849dd8bSTaniya Das { 124dcd1fd72SManivannan Sadhasivam struct qcom_cpufreq_data *data; 125dcd1fd72SManivannan Sadhasivam const struct qcom_cpufreq_soc_data *soc_data; 1262849dd8bSTaniya Das struct cpufreq_policy *policy; 1272849dd8bSTaniya Das unsigned int index; 1282849dd8bSTaniya Das 1292849dd8bSTaniya Das policy = cpufreq_cpu_get_raw(cpu); 1302849dd8bSTaniya Das if (!policy) 1312849dd8bSTaniya Das return 0; 1322849dd8bSTaniya Das 133dcd1fd72SManivannan Sadhasivam data = policy->driver_data; 134dcd1fd72SManivannan Sadhasivam soc_data = data->soc_data; 1352849dd8bSTaniya Das 136dcd1fd72SManivannan Sadhasivam index = readl_relaxed(data->base + soc_data->reg_perf_state); 1372849dd8bSTaniya Das index = min(index, LUT_MAX_ENTRIES - 1); 1382849dd8bSTaniya Das 1392849dd8bSTaniya Das return policy->freq_table[index].frequency; 1402849dd8bSTaniya Das } 1412849dd8bSTaniya Das 1422849dd8bSTaniya Das static unsigned int qcom_cpufreq_hw_fast_switch(struct cpufreq_policy *policy, 1432849dd8bSTaniya Das unsigned int target_freq) 1442849dd8bSTaniya Das { 145dcd1fd72SManivannan Sadhasivam struct qcom_cpufreq_data *data = policy->driver_data; 146dcd1fd72SManivannan Sadhasivam const struct qcom_cpufreq_soc_data *soc_data = data->soc_data; 147292072c3SViresh Kumar unsigned int index; 148c377d4baSBjorn Andersson unsigned int i; 1492849dd8bSTaniya Das 1502849dd8bSTaniya Das index = policy->cached_resolved_idx; 151dcd1fd72SManivannan Sadhasivam writel_relaxed(index, data->base + soc_data->reg_perf_state); 1522849dd8bSTaniya Das 153c377d4baSBjorn Andersson if (data->per_core_dcvs) 154c377d4baSBjorn Andersson for (i = 1; i < cpumask_weight(policy->related_cpus); i++) 155c377d4baSBjorn Andersson writel_relaxed(index, data->base + soc_data->reg_perf_state + i * 4); 156c377d4baSBjorn Andersson 1571a0419b0SIonela Voinescu return policy->freq_table[index].frequency; 1582849dd8bSTaniya Das } 1592849dd8bSTaniya Das 16055538fbcSTaniya Das static int qcom_cpufreq_hw_read_lut(struct device *cpu_dev, 161dcd1fd72SManivannan Sadhasivam struct cpufreq_policy *policy) 1622849dd8bSTaniya Das { 1630eae1e37SSibi Sankar u32 data, src, lval, i, core_count, prev_freq = 0, freq; 16455538fbcSTaniya Das u32 volt; 1652849dd8bSTaniya Das struct cpufreq_frequency_table *table; 16651c843cfSSibi Sankar struct dev_pm_opp *opp; 16751c843cfSSibi Sankar unsigned long rate; 16851c843cfSSibi Sankar int ret; 169dcd1fd72SManivannan Sadhasivam struct qcom_cpufreq_data *drv_data = policy->driver_data; 170dcd1fd72SManivannan Sadhasivam const struct qcom_cpufreq_soc_data *soc_data = drv_data->soc_data; 1712849dd8bSTaniya Das 1722849dd8bSTaniya Das table = kcalloc(LUT_MAX_ENTRIES + 1, sizeof(*table), GFP_KERNEL); 1732849dd8bSTaniya Das if (!table) 1742849dd8bSTaniya Das return -ENOMEM; 1752849dd8bSTaniya Das 17651c843cfSSibi Sankar ret = dev_pm_opp_of_add_table(cpu_dev); 17751c843cfSSibi Sankar if (!ret) { 17851c843cfSSibi Sankar /* Disable all opps and cross-validate against LUT later */ 17951c843cfSSibi Sankar icc_scaling_enabled = true; 18051c843cfSSibi Sankar for (rate = 0; ; rate++) { 18151c843cfSSibi Sankar opp = dev_pm_opp_find_freq_ceil(cpu_dev, &rate); 18251c843cfSSibi Sankar if (IS_ERR(opp)) 18351c843cfSSibi Sankar break; 18451c843cfSSibi Sankar 18551c843cfSSibi Sankar dev_pm_opp_put(opp); 18651c843cfSSibi Sankar dev_pm_opp_disable(cpu_dev, rate); 18751c843cfSSibi Sankar } 18851c843cfSSibi Sankar } else if (ret != -ENODEV) { 18951c843cfSSibi Sankar dev_err(cpu_dev, "Invalid opp table in device tree\n"); 19051c843cfSSibi Sankar return ret; 19151c843cfSSibi Sankar } else { 192afdb219bSSibi Sankar policy->fast_switch_possible = true; 19351c843cfSSibi Sankar icc_scaling_enabled = false; 19451c843cfSSibi Sankar } 19551c843cfSSibi Sankar 1962849dd8bSTaniya Das for (i = 0; i < LUT_MAX_ENTRIES; i++) { 197dcd1fd72SManivannan Sadhasivam data = readl_relaxed(drv_data->base + soc_data->reg_freq_lut + 198dcd1fd72SManivannan Sadhasivam i * soc_data->lut_row_size); 1992849dd8bSTaniya Das src = FIELD_GET(LUT_SRC, data); 2002849dd8bSTaniya Das lval = FIELD_GET(LUT_L_VAL, data); 2012849dd8bSTaniya Das core_count = FIELD_GET(LUT_CORE_COUNT, data); 2022849dd8bSTaniya Das 203dcd1fd72SManivannan Sadhasivam data = readl_relaxed(drv_data->base + soc_data->reg_volt_lut + 204dcd1fd72SManivannan Sadhasivam i * soc_data->lut_row_size); 20555538fbcSTaniya Das volt = FIELD_GET(LUT_VOLT, data) * 1000; 20655538fbcSTaniya Das 2072849dd8bSTaniya Das if (src) 2082849dd8bSTaniya Das freq = xo_rate * lval / 1000; 2092849dd8bSTaniya Das else 2102849dd8bSTaniya Das freq = cpu_hw_rate / 1000; 2112849dd8bSTaniya Das 2120eae1e37SSibi Sankar if (freq != prev_freq && core_count != LUT_TURBO_IND) { 213bc9b9c5aSMatthias Kaehlcke if (!qcom_cpufreq_update_opp(cpu_dev, freq, volt)) { 2142849dd8bSTaniya Das table[i].frequency = freq; 21555538fbcSTaniya Das dev_dbg(cpu_dev, "index=%d freq=%d, core_count %d\n", i, 2162849dd8bSTaniya Das freq, core_count); 217bc9b9c5aSMatthias Kaehlcke } else { 218bc9b9c5aSMatthias Kaehlcke dev_warn(cpu_dev, "failed to update OPP for freq=%d\n", freq); 219bc9b9c5aSMatthias Kaehlcke table[i].frequency = CPUFREQ_ENTRY_INVALID; 220bc9b9c5aSMatthias Kaehlcke } 221bc9b9c5aSMatthias Kaehlcke 2220eae1e37SSibi Sankar } else if (core_count == LUT_TURBO_IND) { 22355538fbcSTaniya Das table[i].frequency = CPUFREQ_ENTRY_INVALID; 2242849dd8bSTaniya Das } 2252849dd8bSTaniya Das 2262849dd8bSTaniya Das /* 2272849dd8bSTaniya Das * Two of the same frequencies with the same core counts means 2282849dd8bSTaniya Das * end of table 2292849dd8bSTaniya Das */ 2300eae1e37SSibi Sankar if (i > 0 && prev_freq == freq) { 2312849dd8bSTaniya Das struct cpufreq_frequency_table *prev = &table[i - 1]; 2322849dd8bSTaniya Das 2332849dd8bSTaniya Das /* 2342849dd8bSTaniya Das * Only treat the last frequency that might be a boost 2352849dd8bSTaniya Das * as the boost frequency 2362849dd8bSTaniya Das */ 2370eae1e37SSibi Sankar if (prev->frequency == CPUFREQ_ENTRY_INVALID) { 238bc9b9c5aSMatthias Kaehlcke if (!qcom_cpufreq_update_opp(cpu_dev, prev_freq, volt)) { 2392849dd8bSTaniya Das prev->frequency = prev_freq; 2402849dd8bSTaniya Das prev->flags = CPUFREQ_BOOST_FREQ; 241bc9b9c5aSMatthias Kaehlcke } else { 242bc9b9c5aSMatthias Kaehlcke dev_warn(cpu_dev, "failed to update OPP for freq=%d\n", 243bc9b9c5aSMatthias Kaehlcke freq); 244bc9b9c5aSMatthias Kaehlcke } 2452849dd8bSTaniya Das } 2462849dd8bSTaniya Das 2472849dd8bSTaniya Das break; 2482849dd8bSTaniya Das } 2492849dd8bSTaniya Das 2502849dd8bSTaniya Das prev_freq = freq; 2512849dd8bSTaniya Das } 2522849dd8bSTaniya Das 2532849dd8bSTaniya Das table[i].frequency = CPUFREQ_TABLE_END; 2542849dd8bSTaniya Das policy->freq_table = table; 25555538fbcSTaniya Das dev_pm_opp_set_sharing_cpus(cpu_dev, policy->cpus); 2562849dd8bSTaniya Das 2572849dd8bSTaniya Das return 0; 2582849dd8bSTaniya Das } 2592849dd8bSTaniya Das 2602849dd8bSTaniya Das static void qcom_get_related_cpus(int index, struct cpumask *m) 2612849dd8bSTaniya Das { 2622849dd8bSTaniya Das struct device_node *cpu_np; 2632849dd8bSTaniya Das struct of_phandle_args args; 2642849dd8bSTaniya Das int cpu, ret; 2652849dd8bSTaniya Das 2662849dd8bSTaniya Das for_each_possible_cpu(cpu) { 2672849dd8bSTaniya Das cpu_np = of_cpu_device_node_get(cpu); 2682849dd8bSTaniya Das if (!cpu_np) 2692849dd8bSTaniya Das continue; 2702849dd8bSTaniya Das 2712849dd8bSTaniya Das ret = of_parse_phandle_with_args(cpu_np, "qcom,freq-domain", 2722849dd8bSTaniya Das "#freq-domain-cells", 0, 2732849dd8bSTaniya Das &args); 2742849dd8bSTaniya Das of_node_put(cpu_np); 2752849dd8bSTaniya Das if (ret < 0) 2762849dd8bSTaniya Das continue; 2772849dd8bSTaniya Das 2782849dd8bSTaniya Das if (index == args.args[0]) 2792849dd8bSTaniya Das cpumask_set_cpu(cpu, m); 2802849dd8bSTaniya Das } 2812849dd8bSTaniya Das } 2822849dd8bSTaniya Das 283275157b3SThara Gopinath static unsigned int qcom_lmh_get_throttle_freq(struct qcom_cpufreq_data *data) 284275157b3SThara Gopinath { 285275157b3SThara Gopinath unsigned int val = readl_relaxed(data->base + data->soc_data->reg_current_vote); 286275157b3SThara Gopinath 287275157b3SThara Gopinath return (val & 0x3FF) * 19200; 288275157b3SThara Gopinath } 289275157b3SThara Gopinath 290275157b3SThara Gopinath static void qcom_lmh_dcvs_notify(struct qcom_cpufreq_data *data) 291275157b3SThara Gopinath { 292275157b3SThara Gopinath struct cpufreq_policy *policy = data->policy; 293*5e4f009dSDmitry Baryshkov int cpu = cpumask_first(policy->related_cpus); 294275157b3SThara Gopinath struct device *dev = get_cpu_device(cpu); 2950258cb19SLukasz Luba unsigned long freq_hz, throttled_freq; 296275157b3SThara Gopinath struct dev_pm_opp *opp; 297275157b3SThara Gopinath unsigned int freq; 298275157b3SThara Gopinath 299275157b3SThara Gopinath /* 300275157b3SThara Gopinath * Get the h/w throttled frequency, normalize it using the 301275157b3SThara Gopinath * registered opp table and use it to calculate thermal pressure. 302275157b3SThara Gopinath */ 303275157b3SThara Gopinath freq = qcom_lmh_get_throttle_freq(data); 304275157b3SThara Gopinath freq_hz = freq * HZ_PER_KHZ; 305275157b3SThara Gopinath 306275157b3SThara Gopinath opp = dev_pm_opp_find_freq_floor(dev, &freq_hz); 307275157b3SThara Gopinath if (IS_ERR(opp) && PTR_ERR(opp) == -ERANGE) 308275157b3SThara Gopinath dev_pm_opp_find_freq_ceil(dev, &freq_hz); 309275157b3SThara Gopinath 310275157b3SThara Gopinath throttled_freq = freq_hz / HZ_PER_KHZ; 311275157b3SThara Gopinath 3120258cb19SLukasz Luba /* Update thermal pressure (the boost frequencies are accepted) */ 3130258cb19SLukasz Luba arch_update_thermal_pressure(policy->related_cpus, throttled_freq); 314275157b3SThara Gopinath 315275157b3SThara Gopinath /* 316275157b3SThara Gopinath * In the unlikely case policy is unregistered do not enable 317275157b3SThara Gopinath * polling or h/w interrupt 318275157b3SThara Gopinath */ 319275157b3SThara Gopinath mutex_lock(&data->throttle_lock); 320275157b3SThara Gopinath if (data->cancel_throttle) 321275157b3SThara Gopinath goto out; 322275157b3SThara Gopinath 323275157b3SThara Gopinath /* 324275157b3SThara Gopinath * If h/w throttled frequency is higher than what cpufreq has requested 325275157b3SThara Gopinath * for, then stop polling and switch back to interrupt mechanism. 326275157b3SThara Gopinath */ 327275157b3SThara Gopinath if (throttled_freq >= qcom_cpufreq_hw_get(cpu)) 328275157b3SThara Gopinath enable_irq(data->throttle_irq); 329275157b3SThara Gopinath else 330275157b3SThara Gopinath mod_delayed_work(system_highpri_wq, &data->throttle_work, 331275157b3SThara Gopinath msecs_to_jiffies(10)); 332275157b3SThara Gopinath 333275157b3SThara Gopinath out: 334275157b3SThara Gopinath mutex_unlock(&data->throttle_lock); 335275157b3SThara Gopinath } 336275157b3SThara Gopinath 337275157b3SThara Gopinath static void qcom_lmh_dcvs_poll(struct work_struct *work) 338275157b3SThara Gopinath { 339275157b3SThara Gopinath struct qcom_cpufreq_data *data; 340275157b3SThara Gopinath 341275157b3SThara Gopinath data = container_of(work, struct qcom_cpufreq_data, throttle_work.work); 342275157b3SThara Gopinath qcom_lmh_dcvs_notify(data); 343275157b3SThara Gopinath } 344275157b3SThara Gopinath 345275157b3SThara Gopinath static irqreturn_t qcom_lmh_dcvs_handle_irq(int irq, void *data) 346275157b3SThara Gopinath { 347275157b3SThara Gopinath struct qcom_cpufreq_data *c_data = data; 348275157b3SThara Gopinath 349275157b3SThara Gopinath /* Disable interrupt and enable polling */ 350275157b3SThara Gopinath disable_irq_nosync(c_data->throttle_irq); 351e0e27c3dSVladimir Zapolskiy schedule_delayed_work(&c_data->throttle_work, 0); 352275157b3SThara Gopinath 353e0e27c3dSVladimir Zapolskiy return IRQ_HANDLED; 354275157b3SThara Gopinath } 355275157b3SThara Gopinath 356dcd1fd72SManivannan Sadhasivam static const struct qcom_cpufreq_soc_data qcom_soc_data = { 357dcd1fd72SManivannan Sadhasivam .reg_enable = 0x0, 358c377d4baSBjorn Andersson .reg_dcvs_ctrl = 0xbc, 359dcd1fd72SManivannan Sadhasivam .reg_freq_lut = 0x110, 360dcd1fd72SManivannan Sadhasivam .reg_volt_lut = 0x114, 361275157b3SThara Gopinath .reg_current_vote = 0x704, 362dcd1fd72SManivannan Sadhasivam .reg_perf_state = 0x920, 363dcd1fd72SManivannan Sadhasivam .lut_row_size = 32, 364dcd1fd72SManivannan Sadhasivam }; 365dcd1fd72SManivannan Sadhasivam 36649b59f4cSManivannan Sadhasivam static const struct qcom_cpufreq_soc_data epss_soc_data = { 36749b59f4cSManivannan Sadhasivam .reg_enable = 0x0, 368c377d4baSBjorn Andersson .reg_dcvs_ctrl = 0xb0, 36949b59f4cSManivannan Sadhasivam .reg_freq_lut = 0x100, 37049b59f4cSManivannan Sadhasivam .reg_volt_lut = 0x200, 37149b59f4cSManivannan Sadhasivam .reg_perf_state = 0x320, 37249b59f4cSManivannan Sadhasivam .lut_row_size = 4, 37349b59f4cSManivannan Sadhasivam }; 37449b59f4cSManivannan Sadhasivam 375dcd1fd72SManivannan Sadhasivam static const struct of_device_id qcom_cpufreq_hw_match[] = { 376dcd1fd72SManivannan Sadhasivam { .compatible = "qcom,cpufreq-hw", .data = &qcom_soc_data }, 37749b59f4cSManivannan Sadhasivam { .compatible = "qcom,cpufreq-epss", .data = &epss_soc_data }, 378dcd1fd72SManivannan Sadhasivam {} 379dcd1fd72SManivannan Sadhasivam }; 380dcd1fd72SManivannan Sadhasivam MODULE_DEVICE_TABLE(of, qcom_cpufreq_hw_match); 381dcd1fd72SManivannan Sadhasivam 382275157b3SThara Gopinath static int qcom_cpufreq_hw_lmh_init(struct cpufreq_policy *policy, int index) 383275157b3SThara Gopinath { 384275157b3SThara Gopinath struct qcom_cpufreq_data *data = policy->driver_data; 385275157b3SThara Gopinath struct platform_device *pdev = cpufreq_get_driver_data(); 386275157b3SThara Gopinath int ret; 387275157b3SThara Gopinath 388275157b3SThara Gopinath /* 389275157b3SThara Gopinath * Look for LMh interrupt. If no interrupt line is specified / 390275157b3SThara Gopinath * if there is an error, allow cpufreq to be enabled as usual. 391275157b3SThara Gopinath */ 3928f5783adSStephen Boyd data->throttle_irq = platform_get_irq_optional(pdev, index); 3938f5783adSStephen Boyd if (data->throttle_irq == -ENXIO) 3948f5783adSStephen Boyd return 0; 3958f5783adSStephen Boyd if (data->throttle_irq < 0) 3968f5783adSStephen Boyd return data->throttle_irq; 397275157b3SThara Gopinath 398275157b3SThara Gopinath data->cancel_throttle = false; 399275157b3SThara Gopinath data->policy = policy; 400275157b3SThara Gopinath 401275157b3SThara Gopinath mutex_init(&data->throttle_lock); 402275157b3SThara Gopinath INIT_DEFERRABLE_WORK(&data->throttle_work, qcom_lmh_dcvs_poll); 403275157b3SThara Gopinath 404be6592edSArd Biesheuvel snprintf(data->irq_name, sizeof(data->irq_name), "dcvsh-irq-%u", policy->cpu); 405275157b3SThara Gopinath ret = request_threaded_irq(data->throttle_irq, NULL, qcom_lmh_dcvs_handle_irq, 406ef8ee1cbSBjorn Andersson IRQF_ONESHOT | IRQF_NO_AUTOEN, data->irq_name, data); 407275157b3SThara Gopinath if (ret) { 408be6592edSArd Biesheuvel dev_err(&pdev->dev, "Error registering %s: %d\n", data->irq_name, ret); 409275157b3SThara Gopinath return 0; 410275157b3SThara Gopinath } 411275157b3SThara Gopinath 4123ed6dfbdSVladimir Zapolskiy ret = irq_set_affinity_hint(data->throttle_irq, policy->cpus); 4133ed6dfbdSVladimir Zapolskiy if (ret) 4143ed6dfbdSVladimir Zapolskiy dev_err(&pdev->dev, "Failed to set CPU affinity of %s[%d]\n", 4153ed6dfbdSVladimir Zapolskiy data->irq_name, data->throttle_irq); 4163ed6dfbdSVladimir Zapolskiy 417275157b3SThara Gopinath return 0; 418275157b3SThara Gopinath } 419275157b3SThara Gopinath 420275157b3SThara Gopinath static void qcom_cpufreq_hw_lmh_exit(struct qcom_cpufreq_data *data) 421275157b3SThara Gopinath { 422275157b3SThara Gopinath if (data->throttle_irq <= 0) 423275157b3SThara Gopinath return; 424275157b3SThara Gopinath 425275157b3SThara Gopinath mutex_lock(&data->throttle_lock); 426275157b3SThara Gopinath data->cancel_throttle = true; 427275157b3SThara Gopinath mutex_unlock(&data->throttle_lock); 428275157b3SThara Gopinath 429275157b3SThara Gopinath cancel_delayed_work_sync(&data->throttle_work); 430be5985b3SDmitry Baryshkov irq_set_affinity_hint(data->throttle_irq, NULL); 431275157b3SThara Gopinath free_irq(data->throttle_irq, data); 432275157b3SThara Gopinath } 433275157b3SThara Gopinath 4342849dd8bSTaniya Das static int qcom_cpufreq_hw_cpu_init(struct cpufreq_policy *policy) 4352849dd8bSTaniya Das { 436bd74e286SManivannan Sadhasivam struct platform_device *pdev = cpufreq_get_driver_data(); 437bd74e286SManivannan Sadhasivam struct device *dev = &pdev->dev; 4382849dd8bSTaniya Das struct of_phandle_args args; 4392849dd8bSTaniya Das struct device_node *cpu_np; 44055538fbcSTaniya Das struct device *cpu_dev; 44167fc209bSShawn Guo struct resource *res; 4422849dd8bSTaniya Das void __iomem *base; 443dcd1fd72SManivannan Sadhasivam struct qcom_cpufreq_data *data; 4442849dd8bSTaniya Das int ret, index; 4452849dd8bSTaniya Das 44655538fbcSTaniya Das cpu_dev = get_cpu_device(policy->cpu); 44755538fbcSTaniya Das if (!cpu_dev) { 44855538fbcSTaniya Das pr_err("%s: failed to get cpu%d device\n", __func__, 44955538fbcSTaniya Das policy->cpu); 45055538fbcSTaniya Das return -ENODEV; 45155538fbcSTaniya Das } 45255538fbcSTaniya Das 4532849dd8bSTaniya Das cpu_np = of_cpu_device_node_get(policy->cpu); 4542849dd8bSTaniya Das if (!cpu_np) 4552849dd8bSTaniya Das return -EINVAL; 4562849dd8bSTaniya Das 4572849dd8bSTaniya Das ret = of_parse_phandle_with_args(cpu_np, "qcom,freq-domain", 4582849dd8bSTaniya Das "#freq-domain-cells", 0, &args); 4592849dd8bSTaniya Das of_node_put(cpu_np); 4602849dd8bSTaniya Das if (ret) 4612849dd8bSTaniya Das return ret; 4622849dd8bSTaniya Das 4632849dd8bSTaniya Das index = args.args[0]; 4642849dd8bSTaniya Das 46567fc209bSShawn Guo res = platform_get_resource(pdev, IORESOURCE_MEM, index); 46667fc209bSShawn Guo if (!res) { 46767fc209bSShawn Guo dev_err(dev, "failed to get mem resource %d\n", index); 46867fc209bSShawn Guo return -ENODEV; 46967fc209bSShawn Guo } 4702849dd8bSTaniya Das 47167fc209bSShawn Guo if (!request_mem_region(res->start, resource_size(res), res->name)) { 47267fc209bSShawn Guo dev_err(dev, "failed to request resource %pR\n", res); 47367fc209bSShawn Guo return -EBUSY; 47467fc209bSShawn Guo } 47567fc209bSShawn Guo 47667fc209bSShawn Guo base = ioremap(res->start, resource_size(res)); 477536eb97aSWei Yongjun if (!base) { 47867fc209bSShawn Guo dev_err(dev, "failed to map resource %pR\n", res); 479536eb97aSWei Yongjun ret = -ENOMEM; 48067fc209bSShawn Guo goto release_region; 48167fc209bSShawn Guo } 48267fc209bSShawn Guo 48367fc209bSShawn Guo data = kzalloc(sizeof(*data), GFP_KERNEL); 484dcd1fd72SManivannan Sadhasivam if (!data) { 485dcd1fd72SManivannan Sadhasivam ret = -ENOMEM; 48667fc209bSShawn Guo goto unmap_base; 487dcd1fd72SManivannan Sadhasivam } 488dcd1fd72SManivannan Sadhasivam 489dcd1fd72SManivannan Sadhasivam data->soc_data = of_device_get_match_data(&pdev->dev); 490dcd1fd72SManivannan Sadhasivam data->base = base; 49167fc209bSShawn Guo data->res = res; 4922849dd8bSTaniya Das 4932849dd8bSTaniya Das /* HW should be in enabled state to proceed */ 494dcd1fd72SManivannan Sadhasivam if (!(readl_relaxed(base + data->soc_data->reg_enable) & 0x1)) { 4952849dd8bSTaniya Das dev_err(dev, "Domain-%d cpufreq hardware not enabled\n", index); 4962849dd8bSTaniya Das ret = -ENODEV; 4972849dd8bSTaniya Das goto error; 4982849dd8bSTaniya Das } 4992849dd8bSTaniya Das 500c377d4baSBjorn Andersson if (readl_relaxed(base + data->soc_data->reg_dcvs_ctrl) & 0x1) 501c377d4baSBjorn Andersson data->per_core_dcvs = true; 502c377d4baSBjorn Andersson 5032849dd8bSTaniya Das qcom_get_related_cpus(index, policy->cpus); 504b48cd0d1SYury Norov if (cpumask_empty(policy->cpus)) { 5052849dd8bSTaniya Das dev_err(dev, "Domain-%d failed to get related CPUs\n", index); 5062849dd8bSTaniya Das ret = -ENOENT; 5072849dd8bSTaniya Das goto error; 5082849dd8bSTaniya Das } 5092849dd8bSTaniya Das 510dcd1fd72SManivannan Sadhasivam policy->driver_data = data; 511f0712aceSTaniya Das policy->dvfs_possible_from_any_cpu = true; 5122849dd8bSTaniya Das 513dcd1fd72SManivannan Sadhasivam ret = qcom_cpufreq_hw_read_lut(cpu_dev, policy); 5142849dd8bSTaniya Das if (ret) { 5152849dd8bSTaniya Das dev_err(dev, "Domain-%d failed to read LUT\n", index); 5162849dd8bSTaniya Das goto error; 5172849dd8bSTaniya Das } 5182849dd8bSTaniya Das 51955538fbcSTaniya Das ret = dev_pm_opp_get_opp_count(cpu_dev); 52055538fbcSTaniya Das if (ret <= 0) { 52155538fbcSTaniya Das dev_err(cpu_dev, "Failed to add OPPs\n"); 52255538fbcSTaniya Das ret = -ENODEV; 52355538fbcSTaniya Das goto error; 52455538fbcSTaniya Das } 52555538fbcSTaniya Das 52626699172SShawn Guo if (policy_has_boost_freq(policy)) { 52726699172SShawn Guo ret = cpufreq_enable_boost_support(); 52826699172SShawn Guo if (ret) 52926699172SShawn Guo dev_warn(cpu_dev, "failed to enable boost: %d\n", ret); 53026699172SShawn Guo } 53126699172SShawn Guo 532275157b3SThara Gopinath ret = qcom_cpufreq_hw_lmh_init(policy, index); 533275157b3SThara Gopinath if (ret) 534275157b3SThara Gopinath goto error; 535275157b3SThara Gopinath 5362849dd8bSTaniya Das return 0; 5372849dd8bSTaniya Das error: 53867fc209bSShawn Guo kfree(data); 53967fc209bSShawn Guo unmap_base: 54002fc4095SShawn Guo iounmap(base); 54167fc209bSShawn Guo release_region: 54267fc209bSShawn Guo release_mem_region(res->start, resource_size(res)); 5432849dd8bSTaniya Das return ret; 5442849dd8bSTaniya Das } 5452849dd8bSTaniya Das 5462849dd8bSTaniya Das static int qcom_cpufreq_hw_cpu_exit(struct cpufreq_policy *policy) 5472849dd8bSTaniya Das { 54855538fbcSTaniya Das struct device *cpu_dev = get_cpu_device(policy->cpu); 549dcd1fd72SManivannan Sadhasivam struct qcom_cpufreq_data *data = policy->driver_data; 55067fc209bSShawn Guo struct resource *res = data->res; 55167fc209bSShawn Guo void __iomem *base = data->base; 5522849dd8bSTaniya Das 55355538fbcSTaniya Das dev_pm_opp_remove_all_dynamic(cpu_dev); 55451c843cfSSibi Sankar dev_pm_opp_of_cpumask_remove_table(policy->related_cpus); 555275157b3SThara Gopinath qcom_cpufreq_hw_lmh_exit(data); 5562849dd8bSTaniya Das kfree(policy->freq_table); 55767fc209bSShawn Guo kfree(data); 55867fc209bSShawn Guo iounmap(base); 55967fc209bSShawn Guo release_mem_region(res->start, resource_size(res)); 5602849dd8bSTaniya Das 5612849dd8bSTaniya Das return 0; 5622849dd8bSTaniya Das } 5632849dd8bSTaniya Das 564ef8ee1cbSBjorn Andersson static void qcom_cpufreq_ready(struct cpufreq_policy *policy) 565ef8ee1cbSBjorn Andersson { 566ef8ee1cbSBjorn Andersson struct qcom_cpufreq_data *data = policy->driver_data; 567ef8ee1cbSBjorn Andersson 568ef8ee1cbSBjorn Andersson if (data->throttle_irq >= 0) 569ef8ee1cbSBjorn Andersson enable_irq(data->throttle_irq); 570ef8ee1cbSBjorn Andersson } 571ef8ee1cbSBjorn Andersson 5722849dd8bSTaniya Das static struct freq_attr *qcom_cpufreq_hw_attr[] = { 5732849dd8bSTaniya Das &cpufreq_freq_attr_scaling_available_freqs, 5742849dd8bSTaniya Das &cpufreq_freq_attr_scaling_boost_freqs, 5752849dd8bSTaniya Das NULL 5762849dd8bSTaniya Das }; 5772849dd8bSTaniya Das 5782849dd8bSTaniya Das static struct cpufreq_driver cpufreq_qcom_hw_driver = { 5795ae4a4b4SViresh Kumar .flags = CPUFREQ_NEED_INITIAL_FREQ_CHECK | 5804c5ff1c8SAmit Kucheria CPUFREQ_HAVE_GOVERNOR_PER_POLICY | 5814c5ff1c8SAmit Kucheria CPUFREQ_IS_COOLING_DEV, 5822849dd8bSTaniya Das .verify = cpufreq_generic_frequency_table_verify, 5832849dd8bSTaniya Das .target_index = qcom_cpufreq_hw_target_index, 5842849dd8bSTaniya Das .get = qcom_cpufreq_hw_get, 5852849dd8bSTaniya Das .init = qcom_cpufreq_hw_cpu_init, 5862849dd8bSTaniya Das .exit = qcom_cpufreq_hw_cpu_exit, 587e96c2153SViresh Kumar .register_em = cpufreq_register_em_with_opp, 5882849dd8bSTaniya Das .fast_switch = qcom_cpufreq_hw_fast_switch, 5892849dd8bSTaniya Das .name = "qcom-cpufreq-hw", 5902849dd8bSTaniya Das .attr = qcom_cpufreq_hw_attr, 591ef8ee1cbSBjorn Andersson .ready = qcom_cpufreq_ready, 5922849dd8bSTaniya Das }; 5932849dd8bSTaniya Das 5942849dd8bSTaniya Das static int qcom_cpufreq_hw_driver_probe(struct platform_device *pdev) 5952849dd8bSTaniya Das { 59651c843cfSSibi Sankar struct device *cpu_dev; 5972849dd8bSTaniya Das struct clk *clk; 5982849dd8bSTaniya Das int ret; 5992849dd8bSTaniya Das 6002849dd8bSTaniya Das clk = clk_get(&pdev->dev, "xo"); 6012849dd8bSTaniya Das if (IS_ERR(clk)) 6022849dd8bSTaniya Das return PTR_ERR(clk); 6032849dd8bSTaniya Das 6042849dd8bSTaniya Das xo_rate = clk_get_rate(clk); 6052849dd8bSTaniya Das clk_put(clk); 6062849dd8bSTaniya Das 6072849dd8bSTaniya Das clk = clk_get(&pdev->dev, "alternate"); 6082849dd8bSTaniya Das if (IS_ERR(clk)) 6092849dd8bSTaniya Das return PTR_ERR(clk); 6102849dd8bSTaniya Das 6112849dd8bSTaniya Das cpu_hw_rate = clk_get_rate(clk) / CLK_HW_DIV; 6122849dd8bSTaniya Das clk_put(clk); 6132849dd8bSTaniya Das 614bd74e286SManivannan Sadhasivam cpufreq_qcom_hw_driver.driver_data = pdev; 6152849dd8bSTaniya Das 61651c843cfSSibi Sankar /* Check for optional interconnect paths on CPU0 */ 61751c843cfSSibi Sankar cpu_dev = get_cpu_device(0); 61851c843cfSSibi Sankar if (!cpu_dev) 61951c843cfSSibi Sankar return -EPROBE_DEFER; 62051c843cfSSibi Sankar 62151c843cfSSibi Sankar ret = dev_pm_opp_of_find_icc_paths(cpu_dev, NULL); 62251c843cfSSibi Sankar if (ret) 62351c843cfSSibi Sankar return ret; 62451c843cfSSibi Sankar 6252849dd8bSTaniya Das ret = cpufreq_register_driver(&cpufreq_qcom_hw_driver); 6262849dd8bSTaniya Das if (ret) 6272849dd8bSTaniya Das dev_err(&pdev->dev, "CPUFreq HW driver failed to register\n"); 6282849dd8bSTaniya Das else 6292849dd8bSTaniya Das dev_dbg(&pdev->dev, "QCOM CPUFreq HW driver initialized\n"); 6302849dd8bSTaniya Das 6312849dd8bSTaniya Das return ret; 6322849dd8bSTaniya Das } 6332849dd8bSTaniya Das 6342849dd8bSTaniya Das static int qcom_cpufreq_hw_driver_remove(struct platform_device *pdev) 6352849dd8bSTaniya Das { 6362849dd8bSTaniya Das return cpufreq_unregister_driver(&cpufreq_qcom_hw_driver); 6372849dd8bSTaniya Das } 6382849dd8bSTaniya Das 6392849dd8bSTaniya Das static struct platform_driver qcom_cpufreq_hw_driver = { 6402849dd8bSTaniya Das .probe = qcom_cpufreq_hw_driver_probe, 6412849dd8bSTaniya Das .remove = qcom_cpufreq_hw_driver_remove, 6422849dd8bSTaniya Das .driver = { 6432849dd8bSTaniya Das .name = "qcom-cpufreq-hw", 6442849dd8bSTaniya Das .of_match_table = qcom_cpufreq_hw_match, 6452849dd8bSTaniya Das }, 6462849dd8bSTaniya Das }; 6472849dd8bSTaniya Das 6482849dd8bSTaniya Das static int __init qcom_cpufreq_hw_init(void) 6492849dd8bSTaniya Das { 6502849dd8bSTaniya Das return platform_driver_register(&qcom_cpufreq_hw_driver); 6512849dd8bSTaniya Das } 65211ff4bddSAmit Kucheria postcore_initcall(qcom_cpufreq_hw_init); 6532849dd8bSTaniya Das 6542849dd8bSTaniya Das static void __exit qcom_cpufreq_hw_exit(void) 6552849dd8bSTaniya Das { 6562849dd8bSTaniya Das platform_driver_unregister(&qcom_cpufreq_hw_driver); 6572849dd8bSTaniya Das } 6582849dd8bSTaniya Das module_exit(qcom_cpufreq_hw_exit); 6592849dd8bSTaniya Das 6602849dd8bSTaniya Das MODULE_DESCRIPTION("QCOM CPUFREQ HW Driver"); 6612849dd8bSTaniya Das MODULE_LICENSE("GPL v2"); 662