12849dd8bSTaniya Das // SPDX-License-Identifier: GPL-2.0 22849dd8bSTaniya Das /* 32849dd8bSTaniya Das * Copyright (c) 2018, The Linux Foundation. All rights reserved. 42849dd8bSTaniya Das */ 52849dd8bSTaniya Das 62849dd8bSTaniya Das #include <linux/bitfield.h> 72849dd8bSTaniya Das #include <linux/cpufreq.h> 82849dd8bSTaniya Das #include <linux/init.h> 951c843cfSSibi Sankar #include <linux/interconnect.h> 10275157b3SThara Gopinath #include <linux/interrupt.h> 112849dd8bSTaniya Das #include <linux/kernel.h> 122849dd8bSTaniya Das #include <linux/module.h> 132849dd8bSTaniya Das #include <linux/of_address.h> 142849dd8bSTaniya Das #include <linux/of_platform.h> 1555538fbcSTaniya Das #include <linux/pm_opp.h> 162849dd8bSTaniya Das #include <linux/slab.h> 17275157b3SThara Gopinath #include <linux/spinlock.h> 182849dd8bSTaniya Das 192849dd8bSTaniya Das #define LUT_MAX_ENTRIES 40U 202849dd8bSTaniya Das #define LUT_SRC GENMASK(31, 30) 212849dd8bSTaniya Das #define LUT_L_VAL GENMASK(7, 0) 222849dd8bSTaniya Das #define LUT_CORE_COUNT GENMASK(18, 16) 2355538fbcSTaniya Das #define LUT_VOLT GENMASK(11, 0) 242849dd8bSTaniya Das #define CLK_HW_DIV 2 250eae1e37SSibi Sankar #define LUT_TURBO_IND 1 262849dd8bSTaniya Das 27275157b3SThara Gopinath #define HZ_PER_KHZ 1000 28275157b3SThara Gopinath 29dcd1fd72SManivannan Sadhasivam struct qcom_cpufreq_soc_data { 30dcd1fd72SManivannan Sadhasivam u32 reg_enable; 31c377d4baSBjorn Andersson u32 reg_dcvs_ctrl; 32dcd1fd72SManivannan Sadhasivam u32 reg_freq_lut; 33dcd1fd72SManivannan Sadhasivam u32 reg_volt_lut; 34275157b3SThara Gopinath u32 reg_current_vote; 35dcd1fd72SManivannan Sadhasivam u32 reg_perf_state; 36dcd1fd72SManivannan Sadhasivam u8 lut_row_size; 37dcd1fd72SManivannan Sadhasivam }; 38dcd1fd72SManivannan Sadhasivam 39dcd1fd72SManivannan Sadhasivam struct qcom_cpufreq_data { 40dcd1fd72SManivannan Sadhasivam void __iomem *base; 4167fc209bSShawn Guo struct resource *res; 42dcd1fd72SManivannan Sadhasivam const struct qcom_cpufreq_soc_data *soc_data; 43275157b3SThara Gopinath 44275157b3SThara Gopinath /* 45275157b3SThara Gopinath * Mutex to synchronize between de-init sequence and re-starting LMh 46275157b3SThara Gopinath * polling/interrupts 47275157b3SThara Gopinath */ 48275157b3SThara Gopinath struct mutex throttle_lock; 49275157b3SThara Gopinath int throttle_irq; 50be6592edSArd Biesheuvel char irq_name[15]; 51275157b3SThara Gopinath bool cancel_throttle; 52275157b3SThara Gopinath struct delayed_work throttle_work; 53275157b3SThara Gopinath struct cpufreq_policy *policy; 54c377d4baSBjorn Andersson 55c377d4baSBjorn Andersson bool per_core_dcvs; 56dcd1fd72SManivannan Sadhasivam }; 572849dd8bSTaniya Das 582849dd8bSTaniya Das static unsigned long cpu_hw_rate, xo_rate; 5951c843cfSSibi Sankar static bool icc_scaling_enabled; 6051c843cfSSibi Sankar 6151c843cfSSibi Sankar static int qcom_cpufreq_set_bw(struct cpufreq_policy *policy, 6251c843cfSSibi Sankar unsigned long freq_khz) 6351c843cfSSibi Sankar { 6451c843cfSSibi Sankar unsigned long freq_hz = freq_khz * 1000; 6551c843cfSSibi Sankar struct dev_pm_opp *opp; 6651c843cfSSibi Sankar struct device *dev; 6751c843cfSSibi Sankar int ret; 6851c843cfSSibi Sankar 6951c843cfSSibi Sankar dev = get_cpu_device(policy->cpu); 7051c843cfSSibi Sankar if (!dev) 7151c843cfSSibi Sankar return -ENODEV; 7251c843cfSSibi Sankar 7351c843cfSSibi Sankar opp = dev_pm_opp_find_freq_exact(dev, freq_hz, true); 7451c843cfSSibi Sankar if (IS_ERR(opp)) 7551c843cfSSibi Sankar return PTR_ERR(opp); 7651c843cfSSibi Sankar 778d25157fSViresh Kumar ret = dev_pm_opp_set_opp(dev, opp); 7851c843cfSSibi Sankar dev_pm_opp_put(opp); 7951c843cfSSibi Sankar return ret; 8051c843cfSSibi Sankar } 8151c843cfSSibi Sankar 8251c843cfSSibi Sankar static int qcom_cpufreq_update_opp(struct device *cpu_dev, 8351c843cfSSibi Sankar unsigned long freq_khz, 8451c843cfSSibi Sankar unsigned long volt) 8551c843cfSSibi Sankar { 8651c843cfSSibi Sankar unsigned long freq_hz = freq_khz * 1000; 8751c843cfSSibi Sankar int ret; 8851c843cfSSibi Sankar 8951c843cfSSibi Sankar /* Skip voltage update if the opp table is not available */ 9051c843cfSSibi Sankar if (!icc_scaling_enabled) 9151c843cfSSibi Sankar return dev_pm_opp_add(cpu_dev, freq_hz, volt); 9251c843cfSSibi Sankar 9351c843cfSSibi Sankar ret = dev_pm_opp_adjust_voltage(cpu_dev, freq_hz, volt, volt, volt); 9451c843cfSSibi Sankar if (ret) { 9551c843cfSSibi Sankar dev_err(cpu_dev, "Voltage update failed freq=%ld\n", freq_khz); 9651c843cfSSibi Sankar return ret; 9751c843cfSSibi Sankar } 9851c843cfSSibi Sankar 9951c843cfSSibi Sankar return dev_pm_opp_enable(cpu_dev, freq_hz); 10051c843cfSSibi Sankar } 1012849dd8bSTaniya Das 1022849dd8bSTaniya Das static int qcom_cpufreq_hw_target_index(struct cpufreq_policy *policy, 1032849dd8bSTaniya Das unsigned int index) 1042849dd8bSTaniya Das { 105dcd1fd72SManivannan Sadhasivam struct qcom_cpufreq_data *data = policy->driver_data; 106dcd1fd72SManivannan Sadhasivam const struct qcom_cpufreq_soc_data *soc_data = data->soc_data; 107ada54f35SDouglas RAILLARD unsigned long freq = policy->freq_table[index].frequency; 108c377d4baSBjorn Andersson unsigned int i; 1092849dd8bSTaniya Das 110dcd1fd72SManivannan Sadhasivam writel_relaxed(index, data->base + soc_data->reg_perf_state); 1112849dd8bSTaniya Das 112c377d4baSBjorn Andersson if (data->per_core_dcvs) 113c377d4baSBjorn Andersson for (i = 1; i < cpumask_weight(policy->related_cpus); i++) 114c377d4baSBjorn Andersson writel_relaxed(index, data->base + soc_data->reg_perf_state + i * 4); 115c377d4baSBjorn Andersson 11651c843cfSSibi Sankar if (icc_scaling_enabled) 11751c843cfSSibi Sankar qcom_cpufreq_set_bw(policy, freq); 11851c843cfSSibi Sankar 1192849dd8bSTaniya Das return 0; 1202849dd8bSTaniya Das } 1212849dd8bSTaniya Das 1222849dd8bSTaniya Das static unsigned int qcom_cpufreq_hw_get(unsigned int cpu) 1232849dd8bSTaniya Das { 124dcd1fd72SManivannan Sadhasivam struct qcom_cpufreq_data *data; 125dcd1fd72SManivannan Sadhasivam const struct qcom_cpufreq_soc_data *soc_data; 1262849dd8bSTaniya Das struct cpufreq_policy *policy; 1272849dd8bSTaniya Das unsigned int index; 1282849dd8bSTaniya Das 1292849dd8bSTaniya Das policy = cpufreq_cpu_get_raw(cpu); 1302849dd8bSTaniya Das if (!policy) 1312849dd8bSTaniya Das return 0; 1322849dd8bSTaniya Das 133dcd1fd72SManivannan Sadhasivam data = policy->driver_data; 134dcd1fd72SManivannan Sadhasivam soc_data = data->soc_data; 1352849dd8bSTaniya Das 136dcd1fd72SManivannan Sadhasivam index = readl_relaxed(data->base + soc_data->reg_perf_state); 1372849dd8bSTaniya Das index = min(index, LUT_MAX_ENTRIES - 1); 1382849dd8bSTaniya Das 1392849dd8bSTaniya Das return policy->freq_table[index].frequency; 1402849dd8bSTaniya Das } 1412849dd8bSTaniya Das 1422849dd8bSTaniya Das static unsigned int qcom_cpufreq_hw_fast_switch(struct cpufreq_policy *policy, 1432849dd8bSTaniya Das unsigned int target_freq) 1442849dd8bSTaniya Das { 145dcd1fd72SManivannan Sadhasivam struct qcom_cpufreq_data *data = policy->driver_data; 146dcd1fd72SManivannan Sadhasivam const struct qcom_cpufreq_soc_data *soc_data = data->soc_data; 147292072c3SViresh Kumar unsigned int index; 148c377d4baSBjorn Andersson unsigned int i; 1492849dd8bSTaniya Das 1502849dd8bSTaniya Das index = policy->cached_resolved_idx; 151dcd1fd72SManivannan Sadhasivam writel_relaxed(index, data->base + soc_data->reg_perf_state); 1522849dd8bSTaniya Das 153c377d4baSBjorn Andersson if (data->per_core_dcvs) 154c377d4baSBjorn Andersson for (i = 1; i < cpumask_weight(policy->related_cpus); i++) 155c377d4baSBjorn Andersson writel_relaxed(index, data->base + soc_data->reg_perf_state + i * 4); 156c377d4baSBjorn Andersson 1571a0419b0SIonela Voinescu return policy->freq_table[index].frequency; 1582849dd8bSTaniya Das } 1592849dd8bSTaniya Das 16055538fbcSTaniya Das static int qcom_cpufreq_hw_read_lut(struct device *cpu_dev, 161dcd1fd72SManivannan Sadhasivam struct cpufreq_policy *policy) 1622849dd8bSTaniya Das { 1630eae1e37SSibi Sankar u32 data, src, lval, i, core_count, prev_freq = 0, freq; 16455538fbcSTaniya Das u32 volt; 1652849dd8bSTaniya Das struct cpufreq_frequency_table *table; 16651c843cfSSibi Sankar struct dev_pm_opp *opp; 16751c843cfSSibi Sankar unsigned long rate; 16851c843cfSSibi Sankar int ret; 169dcd1fd72SManivannan Sadhasivam struct qcom_cpufreq_data *drv_data = policy->driver_data; 170dcd1fd72SManivannan Sadhasivam const struct qcom_cpufreq_soc_data *soc_data = drv_data->soc_data; 1712849dd8bSTaniya Das 1722849dd8bSTaniya Das table = kcalloc(LUT_MAX_ENTRIES + 1, sizeof(*table), GFP_KERNEL); 1732849dd8bSTaniya Das if (!table) 1742849dd8bSTaniya Das return -ENOMEM; 1752849dd8bSTaniya Das 17651c843cfSSibi Sankar ret = dev_pm_opp_of_add_table(cpu_dev); 17751c843cfSSibi Sankar if (!ret) { 17851c843cfSSibi Sankar /* Disable all opps and cross-validate against LUT later */ 17951c843cfSSibi Sankar icc_scaling_enabled = true; 18051c843cfSSibi Sankar for (rate = 0; ; rate++) { 18151c843cfSSibi Sankar opp = dev_pm_opp_find_freq_ceil(cpu_dev, &rate); 18251c843cfSSibi Sankar if (IS_ERR(opp)) 18351c843cfSSibi Sankar break; 18451c843cfSSibi Sankar 18551c843cfSSibi Sankar dev_pm_opp_put(opp); 18651c843cfSSibi Sankar dev_pm_opp_disable(cpu_dev, rate); 18751c843cfSSibi Sankar } 18851c843cfSSibi Sankar } else if (ret != -ENODEV) { 18951c843cfSSibi Sankar dev_err(cpu_dev, "Invalid opp table in device tree\n"); 19051c843cfSSibi Sankar return ret; 19151c843cfSSibi Sankar } else { 192afdb219bSSibi Sankar policy->fast_switch_possible = true; 19351c843cfSSibi Sankar icc_scaling_enabled = false; 19451c843cfSSibi Sankar } 19551c843cfSSibi Sankar 1962849dd8bSTaniya Das for (i = 0; i < LUT_MAX_ENTRIES; i++) { 197dcd1fd72SManivannan Sadhasivam data = readl_relaxed(drv_data->base + soc_data->reg_freq_lut + 198dcd1fd72SManivannan Sadhasivam i * soc_data->lut_row_size); 1992849dd8bSTaniya Das src = FIELD_GET(LUT_SRC, data); 2002849dd8bSTaniya Das lval = FIELD_GET(LUT_L_VAL, data); 2012849dd8bSTaniya Das core_count = FIELD_GET(LUT_CORE_COUNT, data); 2022849dd8bSTaniya Das 203dcd1fd72SManivannan Sadhasivam data = readl_relaxed(drv_data->base + soc_data->reg_volt_lut + 204dcd1fd72SManivannan Sadhasivam i * soc_data->lut_row_size); 20555538fbcSTaniya Das volt = FIELD_GET(LUT_VOLT, data) * 1000; 20655538fbcSTaniya Das 2072849dd8bSTaniya Das if (src) 2082849dd8bSTaniya Das freq = xo_rate * lval / 1000; 2092849dd8bSTaniya Das else 2102849dd8bSTaniya Das freq = cpu_hw_rate / 1000; 2112849dd8bSTaniya Das 2120eae1e37SSibi Sankar if (freq != prev_freq && core_count != LUT_TURBO_IND) { 213bc9b9c5aSMatthias Kaehlcke if (!qcom_cpufreq_update_opp(cpu_dev, freq, volt)) { 2142849dd8bSTaniya Das table[i].frequency = freq; 21555538fbcSTaniya Das dev_dbg(cpu_dev, "index=%d freq=%d, core_count %d\n", i, 2162849dd8bSTaniya Das freq, core_count); 217bc9b9c5aSMatthias Kaehlcke } else { 218bc9b9c5aSMatthias Kaehlcke dev_warn(cpu_dev, "failed to update OPP for freq=%d\n", freq); 219bc9b9c5aSMatthias Kaehlcke table[i].frequency = CPUFREQ_ENTRY_INVALID; 220bc9b9c5aSMatthias Kaehlcke } 221bc9b9c5aSMatthias Kaehlcke 2220eae1e37SSibi Sankar } else if (core_count == LUT_TURBO_IND) { 22355538fbcSTaniya Das table[i].frequency = CPUFREQ_ENTRY_INVALID; 2242849dd8bSTaniya Das } 2252849dd8bSTaniya Das 2262849dd8bSTaniya Das /* 2272849dd8bSTaniya Das * Two of the same frequencies with the same core counts means 2282849dd8bSTaniya Das * end of table 2292849dd8bSTaniya Das */ 2300eae1e37SSibi Sankar if (i > 0 && prev_freq == freq) { 2312849dd8bSTaniya Das struct cpufreq_frequency_table *prev = &table[i - 1]; 2322849dd8bSTaniya Das 2332849dd8bSTaniya Das /* 2342849dd8bSTaniya Das * Only treat the last frequency that might be a boost 2352849dd8bSTaniya Das * as the boost frequency 2362849dd8bSTaniya Das */ 2370eae1e37SSibi Sankar if (prev->frequency == CPUFREQ_ENTRY_INVALID) { 238bc9b9c5aSMatthias Kaehlcke if (!qcom_cpufreq_update_opp(cpu_dev, prev_freq, volt)) { 2392849dd8bSTaniya Das prev->frequency = prev_freq; 2402849dd8bSTaniya Das prev->flags = CPUFREQ_BOOST_FREQ; 241bc9b9c5aSMatthias Kaehlcke } else { 242bc9b9c5aSMatthias Kaehlcke dev_warn(cpu_dev, "failed to update OPP for freq=%d\n", 243bc9b9c5aSMatthias Kaehlcke freq); 244bc9b9c5aSMatthias Kaehlcke } 2452849dd8bSTaniya Das } 2462849dd8bSTaniya Das 2472849dd8bSTaniya Das break; 2482849dd8bSTaniya Das } 2492849dd8bSTaniya Das 2502849dd8bSTaniya Das prev_freq = freq; 2512849dd8bSTaniya Das } 2522849dd8bSTaniya Das 2532849dd8bSTaniya Das table[i].frequency = CPUFREQ_TABLE_END; 2542849dd8bSTaniya Das policy->freq_table = table; 25555538fbcSTaniya Das dev_pm_opp_set_sharing_cpus(cpu_dev, policy->cpus); 2562849dd8bSTaniya Das 2572849dd8bSTaniya Das return 0; 2582849dd8bSTaniya Das } 2592849dd8bSTaniya Das 2602849dd8bSTaniya Das static void qcom_get_related_cpus(int index, struct cpumask *m) 2612849dd8bSTaniya Das { 2622849dd8bSTaniya Das struct device_node *cpu_np; 2632849dd8bSTaniya Das struct of_phandle_args args; 2642849dd8bSTaniya Das int cpu, ret; 2652849dd8bSTaniya Das 2662849dd8bSTaniya Das for_each_possible_cpu(cpu) { 2672849dd8bSTaniya Das cpu_np = of_cpu_device_node_get(cpu); 2682849dd8bSTaniya Das if (!cpu_np) 2692849dd8bSTaniya Das continue; 2702849dd8bSTaniya Das 2712849dd8bSTaniya Das ret = of_parse_phandle_with_args(cpu_np, "qcom,freq-domain", 2722849dd8bSTaniya Das "#freq-domain-cells", 0, 2732849dd8bSTaniya Das &args); 2742849dd8bSTaniya Das of_node_put(cpu_np); 2752849dd8bSTaniya Das if (ret < 0) 2762849dd8bSTaniya Das continue; 2772849dd8bSTaniya Das 2782849dd8bSTaniya Das if (index == args.args[0]) 2792849dd8bSTaniya Das cpumask_set_cpu(cpu, m); 2802849dd8bSTaniya Das } 2812849dd8bSTaniya Das } 2822849dd8bSTaniya Das 283275157b3SThara Gopinath static unsigned int qcom_lmh_get_throttle_freq(struct qcom_cpufreq_data *data) 284275157b3SThara Gopinath { 285275157b3SThara Gopinath unsigned int val = readl_relaxed(data->base + data->soc_data->reg_current_vote); 286275157b3SThara Gopinath 287275157b3SThara Gopinath return (val & 0x3FF) * 19200; 288275157b3SThara Gopinath } 289275157b3SThara Gopinath 290275157b3SThara Gopinath static void qcom_lmh_dcvs_notify(struct qcom_cpufreq_data *data) 291275157b3SThara Gopinath { 292275157b3SThara Gopinath struct cpufreq_policy *policy = data->policy; 2935e4f009dSDmitry Baryshkov int cpu = cpumask_first(policy->related_cpus); 294275157b3SThara Gopinath struct device *dev = get_cpu_device(cpu); 2950258cb19SLukasz Luba unsigned long freq_hz, throttled_freq; 296275157b3SThara Gopinath struct dev_pm_opp *opp; 297275157b3SThara Gopinath unsigned int freq; 298275157b3SThara Gopinath 299275157b3SThara Gopinath /* 300275157b3SThara Gopinath * Get the h/w throttled frequency, normalize it using the 301275157b3SThara Gopinath * registered opp table and use it to calculate thermal pressure. 302275157b3SThara Gopinath */ 303275157b3SThara Gopinath freq = qcom_lmh_get_throttle_freq(data); 304275157b3SThara Gopinath freq_hz = freq * HZ_PER_KHZ; 305275157b3SThara Gopinath 306275157b3SThara Gopinath opp = dev_pm_opp_find_freq_floor(dev, &freq_hz); 307275157b3SThara Gopinath if (IS_ERR(opp) && PTR_ERR(opp) == -ERANGE) 308*6240aaadSDmitry Baryshkov opp = dev_pm_opp_find_freq_ceil(dev, &freq_hz); 309275157b3SThara Gopinath 310*6240aaadSDmitry Baryshkov if (IS_ERR(opp)) { 311*6240aaadSDmitry Baryshkov dev_warn(dev, "Can't find the OPP for throttling: %pe!\n", opp); 312*6240aaadSDmitry Baryshkov } else { 313275157b3SThara Gopinath throttled_freq = freq_hz / HZ_PER_KHZ; 314275157b3SThara Gopinath 3150258cb19SLukasz Luba /* Update thermal pressure (the boost frequencies are accepted) */ 3160258cb19SLukasz Luba arch_update_thermal_pressure(policy->related_cpus, throttled_freq); 317275157b3SThara Gopinath 318*6240aaadSDmitry Baryshkov dev_pm_opp_put(opp); 319*6240aaadSDmitry Baryshkov } 320*6240aaadSDmitry Baryshkov 321275157b3SThara Gopinath /* 322275157b3SThara Gopinath * In the unlikely case policy is unregistered do not enable 323275157b3SThara Gopinath * polling or h/w interrupt 324275157b3SThara Gopinath */ 325275157b3SThara Gopinath mutex_lock(&data->throttle_lock); 326275157b3SThara Gopinath if (data->cancel_throttle) 327275157b3SThara Gopinath goto out; 328275157b3SThara Gopinath 329275157b3SThara Gopinath /* 330275157b3SThara Gopinath * If h/w throttled frequency is higher than what cpufreq has requested 331275157b3SThara Gopinath * for, then stop polling and switch back to interrupt mechanism. 332275157b3SThara Gopinath */ 333275157b3SThara Gopinath if (throttled_freq >= qcom_cpufreq_hw_get(cpu)) 334275157b3SThara Gopinath enable_irq(data->throttle_irq); 335275157b3SThara Gopinath else 336275157b3SThara Gopinath mod_delayed_work(system_highpri_wq, &data->throttle_work, 337275157b3SThara Gopinath msecs_to_jiffies(10)); 338275157b3SThara Gopinath 339275157b3SThara Gopinath out: 340275157b3SThara Gopinath mutex_unlock(&data->throttle_lock); 341275157b3SThara Gopinath } 342275157b3SThara Gopinath 343275157b3SThara Gopinath static void qcom_lmh_dcvs_poll(struct work_struct *work) 344275157b3SThara Gopinath { 345275157b3SThara Gopinath struct qcom_cpufreq_data *data; 346275157b3SThara Gopinath 347275157b3SThara Gopinath data = container_of(work, struct qcom_cpufreq_data, throttle_work.work); 348275157b3SThara Gopinath qcom_lmh_dcvs_notify(data); 349275157b3SThara Gopinath } 350275157b3SThara Gopinath 351275157b3SThara Gopinath static irqreturn_t qcom_lmh_dcvs_handle_irq(int irq, void *data) 352275157b3SThara Gopinath { 353275157b3SThara Gopinath struct qcom_cpufreq_data *c_data = data; 354275157b3SThara Gopinath 355275157b3SThara Gopinath /* Disable interrupt and enable polling */ 356275157b3SThara Gopinath disable_irq_nosync(c_data->throttle_irq); 357e0e27c3dSVladimir Zapolskiy schedule_delayed_work(&c_data->throttle_work, 0); 358275157b3SThara Gopinath 359e0e27c3dSVladimir Zapolskiy return IRQ_HANDLED; 360275157b3SThara Gopinath } 361275157b3SThara Gopinath 362dcd1fd72SManivannan Sadhasivam static const struct qcom_cpufreq_soc_data qcom_soc_data = { 363dcd1fd72SManivannan Sadhasivam .reg_enable = 0x0, 364c377d4baSBjorn Andersson .reg_dcvs_ctrl = 0xbc, 365dcd1fd72SManivannan Sadhasivam .reg_freq_lut = 0x110, 366dcd1fd72SManivannan Sadhasivam .reg_volt_lut = 0x114, 367275157b3SThara Gopinath .reg_current_vote = 0x704, 368dcd1fd72SManivannan Sadhasivam .reg_perf_state = 0x920, 369dcd1fd72SManivannan Sadhasivam .lut_row_size = 32, 370dcd1fd72SManivannan Sadhasivam }; 371dcd1fd72SManivannan Sadhasivam 37249b59f4cSManivannan Sadhasivam static const struct qcom_cpufreq_soc_data epss_soc_data = { 37349b59f4cSManivannan Sadhasivam .reg_enable = 0x0, 374c377d4baSBjorn Andersson .reg_dcvs_ctrl = 0xb0, 37549b59f4cSManivannan Sadhasivam .reg_freq_lut = 0x100, 37649b59f4cSManivannan Sadhasivam .reg_volt_lut = 0x200, 37749b59f4cSManivannan Sadhasivam .reg_perf_state = 0x320, 37849b59f4cSManivannan Sadhasivam .lut_row_size = 4, 37949b59f4cSManivannan Sadhasivam }; 38049b59f4cSManivannan Sadhasivam 381dcd1fd72SManivannan Sadhasivam static const struct of_device_id qcom_cpufreq_hw_match[] = { 382dcd1fd72SManivannan Sadhasivam { .compatible = "qcom,cpufreq-hw", .data = &qcom_soc_data }, 38349b59f4cSManivannan Sadhasivam { .compatible = "qcom,cpufreq-epss", .data = &epss_soc_data }, 384dcd1fd72SManivannan Sadhasivam {} 385dcd1fd72SManivannan Sadhasivam }; 386dcd1fd72SManivannan Sadhasivam MODULE_DEVICE_TABLE(of, qcom_cpufreq_hw_match); 387dcd1fd72SManivannan Sadhasivam 388275157b3SThara Gopinath static int qcom_cpufreq_hw_lmh_init(struct cpufreq_policy *policy, int index) 389275157b3SThara Gopinath { 390275157b3SThara Gopinath struct qcom_cpufreq_data *data = policy->driver_data; 391275157b3SThara Gopinath struct platform_device *pdev = cpufreq_get_driver_data(); 392275157b3SThara Gopinath int ret; 393275157b3SThara Gopinath 394275157b3SThara Gopinath /* 395275157b3SThara Gopinath * Look for LMh interrupt. If no interrupt line is specified / 396275157b3SThara Gopinath * if there is an error, allow cpufreq to be enabled as usual. 397275157b3SThara Gopinath */ 3988f5783adSStephen Boyd data->throttle_irq = platform_get_irq_optional(pdev, index); 3998f5783adSStephen Boyd if (data->throttle_irq == -ENXIO) 4008f5783adSStephen Boyd return 0; 4018f5783adSStephen Boyd if (data->throttle_irq < 0) 4028f5783adSStephen Boyd return data->throttle_irq; 403275157b3SThara Gopinath 404275157b3SThara Gopinath data->cancel_throttle = false; 405275157b3SThara Gopinath data->policy = policy; 406275157b3SThara Gopinath 407275157b3SThara Gopinath mutex_init(&data->throttle_lock); 408275157b3SThara Gopinath INIT_DEFERRABLE_WORK(&data->throttle_work, qcom_lmh_dcvs_poll); 409275157b3SThara Gopinath 410be6592edSArd Biesheuvel snprintf(data->irq_name, sizeof(data->irq_name), "dcvsh-irq-%u", policy->cpu); 411275157b3SThara Gopinath ret = request_threaded_irq(data->throttle_irq, NULL, qcom_lmh_dcvs_handle_irq, 412ef8ee1cbSBjorn Andersson IRQF_ONESHOT | IRQF_NO_AUTOEN, data->irq_name, data); 413275157b3SThara Gopinath if (ret) { 414be6592edSArd Biesheuvel dev_err(&pdev->dev, "Error registering %s: %d\n", data->irq_name, ret); 415275157b3SThara Gopinath return 0; 416275157b3SThara Gopinath } 417275157b3SThara Gopinath 4183ed6dfbdSVladimir Zapolskiy ret = irq_set_affinity_hint(data->throttle_irq, policy->cpus); 4193ed6dfbdSVladimir Zapolskiy if (ret) 4203ed6dfbdSVladimir Zapolskiy dev_err(&pdev->dev, "Failed to set CPU affinity of %s[%d]\n", 4213ed6dfbdSVladimir Zapolskiy data->irq_name, data->throttle_irq); 4223ed6dfbdSVladimir Zapolskiy 423275157b3SThara Gopinath return 0; 424275157b3SThara Gopinath } 425275157b3SThara Gopinath 426275157b3SThara Gopinath static void qcom_cpufreq_hw_lmh_exit(struct qcom_cpufreq_data *data) 427275157b3SThara Gopinath { 428275157b3SThara Gopinath if (data->throttle_irq <= 0) 429275157b3SThara Gopinath return; 430275157b3SThara Gopinath 431275157b3SThara Gopinath mutex_lock(&data->throttle_lock); 432275157b3SThara Gopinath data->cancel_throttle = true; 433275157b3SThara Gopinath mutex_unlock(&data->throttle_lock); 434275157b3SThara Gopinath 435275157b3SThara Gopinath cancel_delayed_work_sync(&data->throttle_work); 436be5985b3SDmitry Baryshkov irq_set_affinity_hint(data->throttle_irq, NULL); 437275157b3SThara Gopinath free_irq(data->throttle_irq, data); 438275157b3SThara Gopinath } 439275157b3SThara Gopinath 4402849dd8bSTaniya Das static int qcom_cpufreq_hw_cpu_init(struct cpufreq_policy *policy) 4412849dd8bSTaniya Das { 442bd74e286SManivannan Sadhasivam struct platform_device *pdev = cpufreq_get_driver_data(); 443bd74e286SManivannan Sadhasivam struct device *dev = &pdev->dev; 4442849dd8bSTaniya Das struct of_phandle_args args; 4452849dd8bSTaniya Das struct device_node *cpu_np; 44655538fbcSTaniya Das struct device *cpu_dev; 44767fc209bSShawn Guo struct resource *res; 4482849dd8bSTaniya Das void __iomem *base; 449dcd1fd72SManivannan Sadhasivam struct qcom_cpufreq_data *data; 4502849dd8bSTaniya Das int ret, index; 4512849dd8bSTaniya Das 45255538fbcSTaniya Das cpu_dev = get_cpu_device(policy->cpu); 45355538fbcSTaniya Das if (!cpu_dev) { 45455538fbcSTaniya Das pr_err("%s: failed to get cpu%d device\n", __func__, 45555538fbcSTaniya Das policy->cpu); 45655538fbcSTaniya Das return -ENODEV; 45755538fbcSTaniya Das } 45855538fbcSTaniya Das 4592849dd8bSTaniya Das cpu_np = of_cpu_device_node_get(policy->cpu); 4602849dd8bSTaniya Das if (!cpu_np) 4612849dd8bSTaniya Das return -EINVAL; 4622849dd8bSTaniya Das 4632849dd8bSTaniya Das ret = of_parse_phandle_with_args(cpu_np, "qcom,freq-domain", 4642849dd8bSTaniya Das "#freq-domain-cells", 0, &args); 4652849dd8bSTaniya Das of_node_put(cpu_np); 4662849dd8bSTaniya Das if (ret) 4672849dd8bSTaniya Das return ret; 4682849dd8bSTaniya Das 4692849dd8bSTaniya Das index = args.args[0]; 4702849dd8bSTaniya Das 47167fc209bSShawn Guo res = platform_get_resource(pdev, IORESOURCE_MEM, index); 47267fc209bSShawn Guo if (!res) { 47367fc209bSShawn Guo dev_err(dev, "failed to get mem resource %d\n", index); 47467fc209bSShawn Guo return -ENODEV; 47567fc209bSShawn Guo } 4762849dd8bSTaniya Das 47767fc209bSShawn Guo if (!request_mem_region(res->start, resource_size(res), res->name)) { 47867fc209bSShawn Guo dev_err(dev, "failed to request resource %pR\n", res); 47967fc209bSShawn Guo return -EBUSY; 48067fc209bSShawn Guo } 48167fc209bSShawn Guo 48267fc209bSShawn Guo base = ioremap(res->start, resource_size(res)); 483536eb97aSWei Yongjun if (!base) { 48467fc209bSShawn Guo dev_err(dev, "failed to map resource %pR\n", res); 485536eb97aSWei Yongjun ret = -ENOMEM; 48667fc209bSShawn Guo goto release_region; 48767fc209bSShawn Guo } 48867fc209bSShawn Guo 48967fc209bSShawn Guo data = kzalloc(sizeof(*data), GFP_KERNEL); 490dcd1fd72SManivannan Sadhasivam if (!data) { 491dcd1fd72SManivannan Sadhasivam ret = -ENOMEM; 49267fc209bSShawn Guo goto unmap_base; 493dcd1fd72SManivannan Sadhasivam } 494dcd1fd72SManivannan Sadhasivam 495dcd1fd72SManivannan Sadhasivam data->soc_data = of_device_get_match_data(&pdev->dev); 496dcd1fd72SManivannan Sadhasivam data->base = base; 49767fc209bSShawn Guo data->res = res; 4982849dd8bSTaniya Das 4992849dd8bSTaniya Das /* HW should be in enabled state to proceed */ 500dcd1fd72SManivannan Sadhasivam if (!(readl_relaxed(base + data->soc_data->reg_enable) & 0x1)) { 5012849dd8bSTaniya Das dev_err(dev, "Domain-%d cpufreq hardware not enabled\n", index); 5022849dd8bSTaniya Das ret = -ENODEV; 5032849dd8bSTaniya Das goto error; 5042849dd8bSTaniya Das } 5052849dd8bSTaniya Das 506c377d4baSBjorn Andersson if (readl_relaxed(base + data->soc_data->reg_dcvs_ctrl) & 0x1) 507c377d4baSBjorn Andersson data->per_core_dcvs = true; 508c377d4baSBjorn Andersson 5092849dd8bSTaniya Das qcom_get_related_cpus(index, policy->cpus); 510b48cd0d1SYury Norov if (cpumask_empty(policy->cpus)) { 5112849dd8bSTaniya Das dev_err(dev, "Domain-%d failed to get related CPUs\n", index); 5122849dd8bSTaniya Das ret = -ENOENT; 5132849dd8bSTaniya Das goto error; 5142849dd8bSTaniya Das } 5152849dd8bSTaniya Das 516dcd1fd72SManivannan Sadhasivam policy->driver_data = data; 517f0712aceSTaniya Das policy->dvfs_possible_from_any_cpu = true; 5182849dd8bSTaniya Das 519dcd1fd72SManivannan Sadhasivam ret = qcom_cpufreq_hw_read_lut(cpu_dev, policy); 5202849dd8bSTaniya Das if (ret) { 5212849dd8bSTaniya Das dev_err(dev, "Domain-%d failed to read LUT\n", index); 5222849dd8bSTaniya Das goto error; 5232849dd8bSTaniya Das } 5242849dd8bSTaniya Das 52555538fbcSTaniya Das ret = dev_pm_opp_get_opp_count(cpu_dev); 52655538fbcSTaniya Das if (ret <= 0) { 52755538fbcSTaniya Das dev_err(cpu_dev, "Failed to add OPPs\n"); 52855538fbcSTaniya Das ret = -ENODEV; 52955538fbcSTaniya Das goto error; 53055538fbcSTaniya Das } 53155538fbcSTaniya Das 53226699172SShawn Guo if (policy_has_boost_freq(policy)) { 53326699172SShawn Guo ret = cpufreq_enable_boost_support(); 53426699172SShawn Guo if (ret) 53526699172SShawn Guo dev_warn(cpu_dev, "failed to enable boost: %d\n", ret); 53626699172SShawn Guo } 53726699172SShawn Guo 538275157b3SThara Gopinath ret = qcom_cpufreq_hw_lmh_init(policy, index); 539275157b3SThara Gopinath if (ret) 540275157b3SThara Gopinath goto error; 541275157b3SThara Gopinath 5422849dd8bSTaniya Das return 0; 5432849dd8bSTaniya Das error: 54467fc209bSShawn Guo kfree(data); 54567fc209bSShawn Guo unmap_base: 54602fc4095SShawn Guo iounmap(base); 54767fc209bSShawn Guo release_region: 54867fc209bSShawn Guo release_mem_region(res->start, resource_size(res)); 5492849dd8bSTaniya Das return ret; 5502849dd8bSTaniya Das } 5512849dd8bSTaniya Das 5522849dd8bSTaniya Das static int qcom_cpufreq_hw_cpu_exit(struct cpufreq_policy *policy) 5532849dd8bSTaniya Das { 55455538fbcSTaniya Das struct device *cpu_dev = get_cpu_device(policy->cpu); 555dcd1fd72SManivannan Sadhasivam struct qcom_cpufreq_data *data = policy->driver_data; 55667fc209bSShawn Guo struct resource *res = data->res; 55767fc209bSShawn Guo void __iomem *base = data->base; 5582849dd8bSTaniya Das 55955538fbcSTaniya Das dev_pm_opp_remove_all_dynamic(cpu_dev); 56051c843cfSSibi Sankar dev_pm_opp_of_cpumask_remove_table(policy->related_cpus); 561275157b3SThara Gopinath qcom_cpufreq_hw_lmh_exit(data); 5622849dd8bSTaniya Das kfree(policy->freq_table); 56367fc209bSShawn Guo kfree(data); 56467fc209bSShawn Guo iounmap(base); 56567fc209bSShawn Guo release_mem_region(res->start, resource_size(res)); 5662849dd8bSTaniya Das 5672849dd8bSTaniya Das return 0; 5682849dd8bSTaniya Das } 5692849dd8bSTaniya Das 570ef8ee1cbSBjorn Andersson static void qcom_cpufreq_ready(struct cpufreq_policy *policy) 571ef8ee1cbSBjorn Andersson { 572ef8ee1cbSBjorn Andersson struct qcom_cpufreq_data *data = policy->driver_data; 573ef8ee1cbSBjorn Andersson 574ef8ee1cbSBjorn Andersson if (data->throttle_irq >= 0) 575ef8ee1cbSBjorn Andersson enable_irq(data->throttle_irq); 576ef8ee1cbSBjorn Andersson } 577ef8ee1cbSBjorn Andersson 5782849dd8bSTaniya Das static struct freq_attr *qcom_cpufreq_hw_attr[] = { 5792849dd8bSTaniya Das &cpufreq_freq_attr_scaling_available_freqs, 5802849dd8bSTaniya Das &cpufreq_freq_attr_scaling_boost_freqs, 5812849dd8bSTaniya Das NULL 5822849dd8bSTaniya Das }; 5832849dd8bSTaniya Das 5842849dd8bSTaniya Das static struct cpufreq_driver cpufreq_qcom_hw_driver = { 5855ae4a4b4SViresh Kumar .flags = CPUFREQ_NEED_INITIAL_FREQ_CHECK | 5864c5ff1c8SAmit Kucheria CPUFREQ_HAVE_GOVERNOR_PER_POLICY | 5874c5ff1c8SAmit Kucheria CPUFREQ_IS_COOLING_DEV, 5882849dd8bSTaniya Das .verify = cpufreq_generic_frequency_table_verify, 5892849dd8bSTaniya Das .target_index = qcom_cpufreq_hw_target_index, 5902849dd8bSTaniya Das .get = qcom_cpufreq_hw_get, 5912849dd8bSTaniya Das .init = qcom_cpufreq_hw_cpu_init, 5922849dd8bSTaniya Das .exit = qcom_cpufreq_hw_cpu_exit, 593e96c2153SViresh Kumar .register_em = cpufreq_register_em_with_opp, 5942849dd8bSTaniya Das .fast_switch = qcom_cpufreq_hw_fast_switch, 5952849dd8bSTaniya Das .name = "qcom-cpufreq-hw", 5962849dd8bSTaniya Das .attr = qcom_cpufreq_hw_attr, 597ef8ee1cbSBjorn Andersson .ready = qcom_cpufreq_ready, 5982849dd8bSTaniya Das }; 5992849dd8bSTaniya Das 6002849dd8bSTaniya Das static int qcom_cpufreq_hw_driver_probe(struct platform_device *pdev) 6012849dd8bSTaniya Das { 60251c843cfSSibi Sankar struct device *cpu_dev; 6032849dd8bSTaniya Das struct clk *clk; 6042849dd8bSTaniya Das int ret; 6052849dd8bSTaniya Das 6062849dd8bSTaniya Das clk = clk_get(&pdev->dev, "xo"); 6072849dd8bSTaniya Das if (IS_ERR(clk)) 6082849dd8bSTaniya Das return PTR_ERR(clk); 6092849dd8bSTaniya Das 6102849dd8bSTaniya Das xo_rate = clk_get_rate(clk); 6112849dd8bSTaniya Das clk_put(clk); 6122849dd8bSTaniya Das 6132849dd8bSTaniya Das clk = clk_get(&pdev->dev, "alternate"); 6142849dd8bSTaniya Das if (IS_ERR(clk)) 6152849dd8bSTaniya Das return PTR_ERR(clk); 6162849dd8bSTaniya Das 6172849dd8bSTaniya Das cpu_hw_rate = clk_get_rate(clk) / CLK_HW_DIV; 6182849dd8bSTaniya Das clk_put(clk); 6192849dd8bSTaniya Das 620bd74e286SManivannan Sadhasivam cpufreq_qcom_hw_driver.driver_data = pdev; 6212849dd8bSTaniya Das 62251c843cfSSibi Sankar /* Check for optional interconnect paths on CPU0 */ 62351c843cfSSibi Sankar cpu_dev = get_cpu_device(0); 62451c843cfSSibi Sankar if (!cpu_dev) 62551c843cfSSibi Sankar return -EPROBE_DEFER; 62651c843cfSSibi Sankar 62751c843cfSSibi Sankar ret = dev_pm_opp_of_find_icc_paths(cpu_dev, NULL); 62851c843cfSSibi Sankar if (ret) 62951c843cfSSibi Sankar return ret; 63051c843cfSSibi Sankar 6312849dd8bSTaniya Das ret = cpufreq_register_driver(&cpufreq_qcom_hw_driver); 6322849dd8bSTaniya Das if (ret) 6332849dd8bSTaniya Das dev_err(&pdev->dev, "CPUFreq HW driver failed to register\n"); 6342849dd8bSTaniya Das else 6352849dd8bSTaniya Das dev_dbg(&pdev->dev, "QCOM CPUFreq HW driver initialized\n"); 6362849dd8bSTaniya Das 6372849dd8bSTaniya Das return ret; 6382849dd8bSTaniya Das } 6392849dd8bSTaniya Das 6402849dd8bSTaniya Das static int qcom_cpufreq_hw_driver_remove(struct platform_device *pdev) 6412849dd8bSTaniya Das { 6422849dd8bSTaniya Das return cpufreq_unregister_driver(&cpufreq_qcom_hw_driver); 6432849dd8bSTaniya Das } 6442849dd8bSTaniya Das 6452849dd8bSTaniya Das static struct platform_driver qcom_cpufreq_hw_driver = { 6462849dd8bSTaniya Das .probe = qcom_cpufreq_hw_driver_probe, 6472849dd8bSTaniya Das .remove = qcom_cpufreq_hw_driver_remove, 6482849dd8bSTaniya Das .driver = { 6492849dd8bSTaniya Das .name = "qcom-cpufreq-hw", 6502849dd8bSTaniya Das .of_match_table = qcom_cpufreq_hw_match, 6512849dd8bSTaniya Das }, 6522849dd8bSTaniya Das }; 6532849dd8bSTaniya Das 6542849dd8bSTaniya Das static int __init qcom_cpufreq_hw_init(void) 6552849dd8bSTaniya Das { 6562849dd8bSTaniya Das return platform_driver_register(&qcom_cpufreq_hw_driver); 6572849dd8bSTaniya Das } 65811ff4bddSAmit Kucheria postcore_initcall(qcom_cpufreq_hw_init); 6592849dd8bSTaniya Das 6602849dd8bSTaniya Das static void __exit qcom_cpufreq_hw_exit(void) 6612849dd8bSTaniya Das { 6622849dd8bSTaniya Das platform_driver_unregister(&qcom_cpufreq_hw_driver); 6632849dd8bSTaniya Das } 6642849dd8bSTaniya Das module_exit(qcom_cpufreq_hw_exit); 6652849dd8bSTaniya Das 6662849dd8bSTaniya Das MODULE_DESCRIPTION("QCOM CPUFREQ HW Driver"); 6672849dd8bSTaniya Das MODULE_LICENSE("GPL v2"); 668