12849dd8bSTaniya Das // SPDX-License-Identifier: GPL-2.0 22849dd8bSTaniya Das /* 32849dd8bSTaniya Das * Copyright (c) 2018, The Linux Foundation. All rights reserved. 42849dd8bSTaniya Das */ 52849dd8bSTaniya Das 62849dd8bSTaniya Das #include <linux/bitfield.h> 72849dd8bSTaniya Das #include <linux/cpufreq.h> 82849dd8bSTaniya Das #include <linux/init.h> 951c843cfSSibi Sankar #include <linux/interconnect.h> 10275157b3SThara Gopinath #include <linux/interrupt.h> 112849dd8bSTaniya Das #include <linux/kernel.h> 122849dd8bSTaniya Das #include <linux/module.h> 132849dd8bSTaniya Das #include <linux/of_address.h> 142849dd8bSTaniya Das #include <linux/of_platform.h> 1555538fbcSTaniya Das #include <linux/pm_opp.h> 162849dd8bSTaniya Das #include <linux/slab.h> 17275157b3SThara Gopinath #include <linux/spinlock.h> 189de0d75bSYicong Yang #include <linux/units.h> 192849dd8bSTaniya Das 202849dd8bSTaniya Das #define LUT_MAX_ENTRIES 40U 212849dd8bSTaniya Das #define LUT_SRC GENMASK(31, 30) 222849dd8bSTaniya Das #define LUT_L_VAL GENMASK(7, 0) 232849dd8bSTaniya Das #define LUT_CORE_COUNT GENMASK(18, 16) 2455538fbcSTaniya Das #define LUT_VOLT GENMASK(11, 0) 252849dd8bSTaniya Das #define CLK_HW_DIV 2 260eae1e37SSibi Sankar #define LUT_TURBO_IND 1 272849dd8bSTaniya Das 28e4e64486SVladimir Zapolskiy #define GT_IRQ_STATUS BIT(2) 29e4e64486SVladimir Zapolskiy 30dcd1fd72SManivannan Sadhasivam struct qcom_cpufreq_soc_data { 31dcd1fd72SManivannan Sadhasivam u32 reg_enable; 32f84ccad5SVladimir Zapolskiy u32 reg_domain_state; 33c377d4baSBjorn Andersson u32 reg_dcvs_ctrl; 34dcd1fd72SManivannan Sadhasivam u32 reg_freq_lut; 35dcd1fd72SManivannan Sadhasivam u32 reg_volt_lut; 36e4e64486SVladimir Zapolskiy u32 reg_intr_clr; 37275157b3SThara Gopinath u32 reg_current_vote; 38dcd1fd72SManivannan Sadhasivam u32 reg_perf_state; 39dcd1fd72SManivannan Sadhasivam u8 lut_row_size; 40dcd1fd72SManivannan Sadhasivam }; 41dcd1fd72SManivannan Sadhasivam 42dcd1fd72SManivannan Sadhasivam struct qcom_cpufreq_data { 43dcd1fd72SManivannan Sadhasivam void __iomem *base; 4467fc209bSShawn Guo struct resource *res; 45dcd1fd72SManivannan Sadhasivam const struct qcom_cpufreq_soc_data *soc_data; 46275157b3SThara Gopinath 47275157b3SThara Gopinath /* 48275157b3SThara Gopinath * Mutex to synchronize between de-init sequence and re-starting LMh 49275157b3SThara Gopinath * polling/interrupts 50275157b3SThara Gopinath */ 51275157b3SThara Gopinath struct mutex throttle_lock; 52275157b3SThara Gopinath int throttle_irq; 53be6592edSArd Biesheuvel char irq_name[15]; 54275157b3SThara Gopinath bool cancel_throttle; 55275157b3SThara Gopinath struct delayed_work throttle_work; 56275157b3SThara Gopinath struct cpufreq_policy *policy; 57c377d4baSBjorn Andersson 58c377d4baSBjorn Andersson bool per_core_dcvs; 59dcd1fd72SManivannan Sadhasivam }; 602849dd8bSTaniya Das 612849dd8bSTaniya Das static unsigned long cpu_hw_rate, xo_rate; 6251c843cfSSibi Sankar static bool icc_scaling_enabled; 6351c843cfSSibi Sankar 6451c843cfSSibi Sankar static int qcom_cpufreq_set_bw(struct cpufreq_policy *policy, 6551c843cfSSibi Sankar unsigned long freq_khz) 6651c843cfSSibi Sankar { 6751c843cfSSibi Sankar unsigned long freq_hz = freq_khz * 1000; 6851c843cfSSibi Sankar struct dev_pm_opp *opp; 6951c843cfSSibi Sankar struct device *dev; 7051c843cfSSibi Sankar int ret; 7151c843cfSSibi Sankar 7251c843cfSSibi Sankar dev = get_cpu_device(policy->cpu); 7351c843cfSSibi Sankar if (!dev) 7451c843cfSSibi Sankar return -ENODEV; 7551c843cfSSibi Sankar 7651c843cfSSibi Sankar opp = dev_pm_opp_find_freq_exact(dev, freq_hz, true); 7751c843cfSSibi Sankar if (IS_ERR(opp)) 7851c843cfSSibi Sankar return PTR_ERR(opp); 7951c843cfSSibi Sankar 808d25157fSViresh Kumar ret = dev_pm_opp_set_opp(dev, opp); 8151c843cfSSibi Sankar dev_pm_opp_put(opp); 8251c843cfSSibi Sankar return ret; 8351c843cfSSibi Sankar } 8451c843cfSSibi Sankar 8551c843cfSSibi Sankar static int qcom_cpufreq_update_opp(struct device *cpu_dev, 8651c843cfSSibi Sankar unsigned long freq_khz, 8751c843cfSSibi Sankar unsigned long volt) 8851c843cfSSibi Sankar { 8951c843cfSSibi Sankar unsigned long freq_hz = freq_khz * 1000; 9051c843cfSSibi Sankar int ret; 9151c843cfSSibi Sankar 9251c843cfSSibi Sankar /* Skip voltage update if the opp table is not available */ 9351c843cfSSibi Sankar if (!icc_scaling_enabled) 9451c843cfSSibi Sankar return dev_pm_opp_add(cpu_dev, freq_hz, volt); 9551c843cfSSibi Sankar 9651c843cfSSibi Sankar ret = dev_pm_opp_adjust_voltage(cpu_dev, freq_hz, volt, volt, volt); 9751c843cfSSibi Sankar if (ret) { 9851c843cfSSibi Sankar dev_err(cpu_dev, "Voltage update failed freq=%ld\n", freq_khz); 9951c843cfSSibi Sankar return ret; 10051c843cfSSibi Sankar } 10151c843cfSSibi Sankar 10251c843cfSSibi Sankar return dev_pm_opp_enable(cpu_dev, freq_hz); 10351c843cfSSibi Sankar } 1042849dd8bSTaniya Das 1052849dd8bSTaniya Das static int qcom_cpufreq_hw_target_index(struct cpufreq_policy *policy, 1062849dd8bSTaniya Das unsigned int index) 1072849dd8bSTaniya Das { 108dcd1fd72SManivannan Sadhasivam struct qcom_cpufreq_data *data = policy->driver_data; 109dcd1fd72SManivannan Sadhasivam const struct qcom_cpufreq_soc_data *soc_data = data->soc_data; 110ada54f35SDouglas RAILLARD unsigned long freq = policy->freq_table[index].frequency; 111c377d4baSBjorn Andersson unsigned int i; 1122849dd8bSTaniya Das 113dcd1fd72SManivannan Sadhasivam writel_relaxed(index, data->base + soc_data->reg_perf_state); 1142849dd8bSTaniya Das 115c377d4baSBjorn Andersson if (data->per_core_dcvs) 116c377d4baSBjorn Andersson for (i = 1; i < cpumask_weight(policy->related_cpus); i++) 117c377d4baSBjorn Andersson writel_relaxed(index, data->base + soc_data->reg_perf_state + i * 4); 118c377d4baSBjorn Andersson 11951c843cfSSibi Sankar if (icc_scaling_enabled) 12051c843cfSSibi Sankar qcom_cpufreq_set_bw(policy, freq); 12151c843cfSSibi Sankar 1222849dd8bSTaniya Das return 0; 1232849dd8bSTaniya Das } 1242849dd8bSTaniya Das 1252849dd8bSTaniya Das static unsigned int qcom_cpufreq_hw_get(unsigned int cpu) 1262849dd8bSTaniya Das { 127dcd1fd72SManivannan Sadhasivam struct qcom_cpufreq_data *data; 128dcd1fd72SManivannan Sadhasivam const struct qcom_cpufreq_soc_data *soc_data; 1292849dd8bSTaniya Das struct cpufreq_policy *policy; 1302849dd8bSTaniya Das unsigned int index; 1312849dd8bSTaniya Das 1322849dd8bSTaniya Das policy = cpufreq_cpu_get_raw(cpu); 1332849dd8bSTaniya Das if (!policy) 1342849dd8bSTaniya Das return 0; 1352849dd8bSTaniya Das 136dcd1fd72SManivannan Sadhasivam data = policy->driver_data; 137dcd1fd72SManivannan Sadhasivam soc_data = data->soc_data; 1382849dd8bSTaniya Das 139dcd1fd72SManivannan Sadhasivam index = readl_relaxed(data->base + soc_data->reg_perf_state); 1402849dd8bSTaniya Das index = min(index, LUT_MAX_ENTRIES - 1); 1412849dd8bSTaniya Das 1422849dd8bSTaniya Das return policy->freq_table[index].frequency; 1432849dd8bSTaniya Das } 1442849dd8bSTaniya Das 1452849dd8bSTaniya Das static unsigned int qcom_cpufreq_hw_fast_switch(struct cpufreq_policy *policy, 1462849dd8bSTaniya Das unsigned int target_freq) 1472849dd8bSTaniya Das { 148dcd1fd72SManivannan Sadhasivam struct qcom_cpufreq_data *data = policy->driver_data; 149dcd1fd72SManivannan Sadhasivam const struct qcom_cpufreq_soc_data *soc_data = data->soc_data; 150292072c3SViresh Kumar unsigned int index; 151c377d4baSBjorn Andersson unsigned int i; 1522849dd8bSTaniya Das 1532849dd8bSTaniya Das index = policy->cached_resolved_idx; 154dcd1fd72SManivannan Sadhasivam writel_relaxed(index, data->base + soc_data->reg_perf_state); 1552849dd8bSTaniya Das 156c377d4baSBjorn Andersson if (data->per_core_dcvs) 157c377d4baSBjorn Andersson for (i = 1; i < cpumask_weight(policy->related_cpus); i++) 158c377d4baSBjorn Andersson writel_relaxed(index, data->base + soc_data->reg_perf_state + i * 4); 159c377d4baSBjorn Andersson 1601a0419b0SIonela Voinescu return policy->freq_table[index].frequency; 1612849dd8bSTaniya Das } 1622849dd8bSTaniya Das 16355538fbcSTaniya Das static int qcom_cpufreq_hw_read_lut(struct device *cpu_dev, 164dcd1fd72SManivannan Sadhasivam struct cpufreq_policy *policy) 1652849dd8bSTaniya Das { 1660eae1e37SSibi Sankar u32 data, src, lval, i, core_count, prev_freq = 0, freq; 16755538fbcSTaniya Das u32 volt; 1682849dd8bSTaniya Das struct cpufreq_frequency_table *table; 16951c843cfSSibi Sankar struct dev_pm_opp *opp; 17051c843cfSSibi Sankar unsigned long rate; 17151c843cfSSibi Sankar int ret; 172dcd1fd72SManivannan Sadhasivam struct qcom_cpufreq_data *drv_data = policy->driver_data; 173dcd1fd72SManivannan Sadhasivam const struct qcom_cpufreq_soc_data *soc_data = drv_data->soc_data; 1742849dd8bSTaniya Das 1752849dd8bSTaniya Das table = kcalloc(LUT_MAX_ENTRIES + 1, sizeof(*table), GFP_KERNEL); 1762849dd8bSTaniya Das if (!table) 1772849dd8bSTaniya Das return -ENOMEM; 1782849dd8bSTaniya Das 17951c843cfSSibi Sankar ret = dev_pm_opp_of_add_table(cpu_dev); 18051c843cfSSibi Sankar if (!ret) { 18151c843cfSSibi Sankar /* Disable all opps and cross-validate against LUT later */ 18251c843cfSSibi Sankar icc_scaling_enabled = true; 18351c843cfSSibi Sankar for (rate = 0; ; rate++) { 18451c843cfSSibi Sankar opp = dev_pm_opp_find_freq_ceil(cpu_dev, &rate); 18551c843cfSSibi Sankar if (IS_ERR(opp)) 18651c843cfSSibi Sankar break; 18751c843cfSSibi Sankar 18851c843cfSSibi Sankar dev_pm_opp_put(opp); 18951c843cfSSibi Sankar dev_pm_opp_disable(cpu_dev, rate); 19051c843cfSSibi Sankar } 19151c843cfSSibi Sankar } else if (ret != -ENODEV) { 19251c843cfSSibi Sankar dev_err(cpu_dev, "Invalid opp table in device tree\n"); 19351c843cfSSibi Sankar return ret; 19451c843cfSSibi Sankar } else { 195afdb219bSSibi Sankar policy->fast_switch_possible = true; 19651c843cfSSibi Sankar icc_scaling_enabled = false; 19751c843cfSSibi Sankar } 19851c843cfSSibi Sankar 1992849dd8bSTaniya Das for (i = 0; i < LUT_MAX_ENTRIES; i++) { 200dcd1fd72SManivannan Sadhasivam data = readl_relaxed(drv_data->base + soc_data->reg_freq_lut + 201dcd1fd72SManivannan Sadhasivam i * soc_data->lut_row_size); 2022849dd8bSTaniya Das src = FIELD_GET(LUT_SRC, data); 2032849dd8bSTaniya Das lval = FIELD_GET(LUT_L_VAL, data); 2042849dd8bSTaniya Das core_count = FIELD_GET(LUT_CORE_COUNT, data); 2052849dd8bSTaniya Das 206dcd1fd72SManivannan Sadhasivam data = readl_relaxed(drv_data->base + soc_data->reg_volt_lut + 207dcd1fd72SManivannan Sadhasivam i * soc_data->lut_row_size); 20855538fbcSTaniya Das volt = FIELD_GET(LUT_VOLT, data) * 1000; 20955538fbcSTaniya Das 2102849dd8bSTaniya Das if (src) 2112849dd8bSTaniya Das freq = xo_rate * lval / 1000; 2122849dd8bSTaniya Das else 2132849dd8bSTaniya Das freq = cpu_hw_rate / 1000; 2142849dd8bSTaniya Das 2150eae1e37SSibi Sankar if (freq != prev_freq && core_count != LUT_TURBO_IND) { 216bc9b9c5aSMatthias Kaehlcke if (!qcom_cpufreq_update_opp(cpu_dev, freq, volt)) { 2172849dd8bSTaniya Das table[i].frequency = freq; 21855538fbcSTaniya Das dev_dbg(cpu_dev, "index=%d freq=%d, core_count %d\n", i, 2192849dd8bSTaniya Das freq, core_count); 220bc9b9c5aSMatthias Kaehlcke } else { 221bc9b9c5aSMatthias Kaehlcke dev_warn(cpu_dev, "failed to update OPP for freq=%d\n", freq); 222bc9b9c5aSMatthias Kaehlcke table[i].frequency = CPUFREQ_ENTRY_INVALID; 223bc9b9c5aSMatthias Kaehlcke } 224bc9b9c5aSMatthias Kaehlcke 2250eae1e37SSibi Sankar } else if (core_count == LUT_TURBO_IND) { 22655538fbcSTaniya Das table[i].frequency = CPUFREQ_ENTRY_INVALID; 2272849dd8bSTaniya Das } 2282849dd8bSTaniya Das 2292849dd8bSTaniya Das /* 2302849dd8bSTaniya Das * Two of the same frequencies with the same core counts means 2312849dd8bSTaniya Das * end of table 2322849dd8bSTaniya Das */ 2330eae1e37SSibi Sankar if (i > 0 && prev_freq == freq) { 2342849dd8bSTaniya Das struct cpufreq_frequency_table *prev = &table[i - 1]; 2352849dd8bSTaniya Das 2362849dd8bSTaniya Das /* 2372849dd8bSTaniya Das * Only treat the last frequency that might be a boost 2382849dd8bSTaniya Das * as the boost frequency 2392849dd8bSTaniya Das */ 2400eae1e37SSibi Sankar if (prev->frequency == CPUFREQ_ENTRY_INVALID) { 241bc9b9c5aSMatthias Kaehlcke if (!qcom_cpufreq_update_opp(cpu_dev, prev_freq, volt)) { 2422849dd8bSTaniya Das prev->frequency = prev_freq; 2432849dd8bSTaniya Das prev->flags = CPUFREQ_BOOST_FREQ; 244bc9b9c5aSMatthias Kaehlcke } else { 245bc9b9c5aSMatthias Kaehlcke dev_warn(cpu_dev, "failed to update OPP for freq=%d\n", 246bc9b9c5aSMatthias Kaehlcke freq); 247bc9b9c5aSMatthias Kaehlcke } 2482849dd8bSTaniya Das } 2492849dd8bSTaniya Das 2502849dd8bSTaniya Das break; 2512849dd8bSTaniya Das } 2522849dd8bSTaniya Das 2532849dd8bSTaniya Das prev_freq = freq; 2542849dd8bSTaniya Das } 2552849dd8bSTaniya Das 2562849dd8bSTaniya Das table[i].frequency = CPUFREQ_TABLE_END; 2572849dd8bSTaniya Das policy->freq_table = table; 25855538fbcSTaniya Das dev_pm_opp_set_sharing_cpus(cpu_dev, policy->cpus); 2592849dd8bSTaniya Das 2602849dd8bSTaniya Das return 0; 2612849dd8bSTaniya Das } 2622849dd8bSTaniya Das 2632849dd8bSTaniya Das static void qcom_get_related_cpus(int index, struct cpumask *m) 2642849dd8bSTaniya Das { 2652849dd8bSTaniya Das struct device_node *cpu_np; 2662849dd8bSTaniya Das struct of_phandle_args args; 2672849dd8bSTaniya Das int cpu, ret; 2682849dd8bSTaniya Das 2692849dd8bSTaniya Das for_each_possible_cpu(cpu) { 2702849dd8bSTaniya Das cpu_np = of_cpu_device_node_get(cpu); 2712849dd8bSTaniya Das if (!cpu_np) 2722849dd8bSTaniya Das continue; 2732849dd8bSTaniya Das 2742849dd8bSTaniya Das ret = of_parse_phandle_with_args(cpu_np, "qcom,freq-domain", 2752849dd8bSTaniya Das "#freq-domain-cells", 0, 2762849dd8bSTaniya Das &args); 2772849dd8bSTaniya Das of_node_put(cpu_np); 2782849dd8bSTaniya Das if (ret < 0) 2792849dd8bSTaniya Das continue; 2802849dd8bSTaniya Das 2812849dd8bSTaniya Das if (index == args.args[0]) 2822849dd8bSTaniya Das cpumask_set_cpu(cpu, m); 2832849dd8bSTaniya Das } 2842849dd8bSTaniya Das } 2852849dd8bSTaniya Das 286f84ccad5SVladimir Zapolskiy static unsigned long qcom_lmh_get_throttle_freq(struct qcom_cpufreq_data *data) 287275157b3SThara Gopinath { 288f84ccad5SVladimir Zapolskiy unsigned int lval; 289275157b3SThara Gopinath 290f84ccad5SVladimir Zapolskiy if (data->soc_data->reg_current_vote) 291f84ccad5SVladimir Zapolskiy lval = readl_relaxed(data->base + data->soc_data->reg_current_vote) & 0x3ff; 292f84ccad5SVladimir Zapolskiy else 293f84ccad5SVladimir Zapolskiy lval = readl_relaxed(data->base + data->soc_data->reg_domain_state) & 0xff; 294f84ccad5SVladimir Zapolskiy 295f84ccad5SVladimir Zapolskiy return lval * xo_rate; 296275157b3SThara Gopinath } 297275157b3SThara Gopinath 298275157b3SThara Gopinath static void qcom_lmh_dcvs_notify(struct qcom_cpufreq_data *data) 299275157b3SThara Gopinath { 300275157b3SThara Gopinath struct cpufreq_policy *policy = data->policy; 3015e4f009dSDmitry Baryshkov int cpu = cpumask_first(policy->related_cpus); 302275157b3SThara Gopinath struct device *dev = get_cpu_device(cpu); 3030258cb19SLukasz Luba unsigned long freq_hz, throttled_freq; 304275157b3SThara Gopinath struct dev_pm_opp *opp; 305275157b3SThara Gopinath 306275157b3SThara Gopinath /* 307275157b3SThara Gopinath * Get the h/w throttled frequency, normalize it using the 308275157b3SThara Gopinath * registered opp table and use it to calculate thermal pressure. 309275157b3SThara Gopinath */ 310f84ccad5SVladimir Zapolskiy freq_hz = qcom_lmh_get_throttle_freq(data); 311275157b3SThara Gopinath 312275157b3SThara Gopinath opp = dev_pm_opp_find_freq_floor(dev, &freq_hz); 313275157b3SThara Gopinath if (IS_ERR(opp) && PTR_ERR(opp) == -ERANGE) 3146240aaadSDmitry Baryshkov opp = dev_pm_opp_find_freq_ceil(dev, &freq_hz); 315275157b3SThara Gopinath 3166240aaadSDmitry Baryshkov if (IS_ERR(opp)) { 3176240aaadSDmitry Baryshkov dev_warn(dev, "Can't find the OPP for throttling: %pe!\n", opp); 3186240aaadSDmitry Baryshkov } else { 319275157b3SThara Gopinath throttled_freq = freq_hz / HZ_PER_KHZ; 320275157b3SThara Gopinath 3210258cb19SLukasz Luba /* Update thermal pressure (the boost frequencies are accepted) */ 3220258cb19SLukasz Luba arch_update_thermal_pressure(policy->related_cpus, throttled_freq); 323275157b3SThara Gopinath 3246240aaadSDmitry Baryshkov dev_pm_opp_put(opp); 3256240aaadSDmitry Baryshkov } 3266240aaadSDmitry Baryshkov 327275157b3SThara Gopinath /* 328275157b3SThara Gopinath * In the unlikely case policy is unregistered do not enable 329275157b3SThara Gopinath * polling or h/w interrupt 330275157b3SThara Gopinath */ 331275157b3SThara Gopinath mutex_lock(&data->throttle_lock); 332275157b3SThara Gopinath if (data->cancel_throttle) 333275157b3SThara Gopinath goto out; 334275157b3SThara Gopinath 335275157b3SThara Gopinath /* 336275157b3SThara Gopinath * If h/w throttled frequency is higher than what cpufreq has requested 337275157b3SThara Gopinath * for, then stop polling and switch back to interrupt mechanism. 338275157b3SThara Gopinath */ 339275157b3SThara Gopinath if (throttled_freq >= qcom_cpufreq_hw_get(cpu)) 340275157b3SThara Gopinath enable_irq(data->throttle_irq); 341275157b3SThara Gopinath else 342275157b3SThara Gopinath mod_delayed_work(system_highpri_wq, &data->throttle_work, 343275157b3SThara Gopinath msecs_to_jiffies(10)); 344275157b3SThara Gopinath 345275157b3SThara Gopinath out: 346275157b3SThara Gopinath mutex_unlock(&data->throttle_lock); 347275157b3SThara Gopinath } 348275157b3SThara Gopinath 349275157b3SThara Gopinath static void qcom_lmh_dcvs_poll(struct work_struct *work) 350275157b3SThara Gopinath { 351275157b3SThara Gopinath struct qcom_cpufreq_data *data; 352275157b3SThara Gopinath 353275157b3SThara Gopinath data = container_of(work, struct qcom_cpufreq_data, throttle_work.work); 354275157b3SThara Gopinath qcom_lmh_dcvs_notify(data); 355275157b3SThara Gopinath } 356275157b3SThara Gopinath 357275157b3SThara Gopinath static irqreturn_t qcom_lmh_dcvs_handle_irq(int irq, void *data) 358275157b3SThara Gopinath { 359275157b3SThara Gopinath struct qcom_cpufreq_data *c_data = data; 360275157b3SThara Gopinath 361275157b3SThara Gopinath /* Disable interrupt and enable polling */ 362275157b3SThara Gopinath disable_irq_nosync(c_data->throttle_irq); 363e0e27c3dSVladimir Zapolskiy schedule_delayed_work(&c_data->throttle_work, 0); 364275157b3SThara Gopinath 365e4e64486SVladimir Zapolskiy if (c_data->soc_data->reg_intr_clr) 366e4e64486SVladimir Zapolskiy writel_relaxed(GT_IRQ_STATUS, 367e4e64486SVladimir Zapolskiy c_data->base + c_data->soc_data->reg_intr_clr); 368e4e64486SVladimir Zapolskiy 369e0e27c3dSVladimir Zapolskiy return IRQ_HANDLED; 370275157b3SThara Gopinath } 371275157b3SThara Gopinath 372dcd1fd72SManivannan Sadhasivam static const struct qcom_cpufreq_soc_data qcom_soc_data = { 373dcd1fd72SManivannan Sadhasivam .reg_enable = 0x0, 374c377d4baSBjorn Andersson .reg_dcvs_ctrl = 0xbc, 375dcd1fd72SManivannan Sadhasivam .reg_freq_lut = 0x110, 376dcd1fd72SManivannan Sadhasivam .reg_volt_lut = 0x114, 377275157b3SThara Gopinath .reg_current_vote = 0x704, 378dcd1fd72SManivannan Sadhasivam .reg_perf_state = 0x920, 379dcd1fd72SManivannan Sadhasivam .lut_row_size = 32, 380dcd1fd72SManivannan Sadhasivam }; 381dcd1fd72SManivannan Sadhasivam 38249b59f4cSManivannan Sadhasivam static const struct qcom_cpufreq_soc_data epss_soc_data = { 38349b59f4cSManivannan Sadhasivam .reg_enable = 0x0, 384f84ccad5SVladimir Zapolskiy .reg_domain_state = 0x20, 385c377d4baSBjorn Andersson .reg_dcvs_ctrl = 0xb0, 38649b59f4cSManivannan Sadhasivam .reg_freq_lut = 0x100, 38749b59f4cSManivannan Sadhasivam .reg_volt_lut = 0x200, 388e4e64486SVladimir Zapolskiy .reg_intr_clr = 0x308, 38949b59f4cSManivannan Sadhasivam .reg_perf_state = 0x320, 39049b59f4cSManivannan Sadhasivam .lut_row_size = 4, 39149b59f4cSManivannan Sadhasivam }; 39249b59f4cSManivannan Sadhasivam 393dcd1fd72SManivannan Sadhasivam static const struct of_device_id qcom_cpufreq_hw_match[] = { 394dcd1fd72SManivannan Sadhasivam { .compatible = "qcom,cpufreq-hw", .data = &qcom_soc_data }, 39549b59f4cSManivannan Sadhasivam { .compatible = "qcom,cpufreq-epss", .data = &epss_soc_data }, 396dcd1fd72SManivannan Sadhasivam {} 397dcd1fd72SManivannan Sadhasivam }; 398dcd1fd72SManivannan Sadhasivam MODULE_DEVICE_TABLE(of, qcom_cpufreq_hw_match); 399dcd1fd72SManivannan Sadhasivam 400275157b3SThara Gopinath static int qcom_cpufreq_hw_lmh_init(struct cpufreq_policy *policy, int index) 401275157b3SThara Gopinath { 402275157b3SThara Gopinath struct qcom_cpufreq_data *data = policy->driver_data; 403275157b3SThara Gopinath struct platform_device *pdev = cpufreq_get_driver_data(); 404275157b3SThara Gopinath int ret; 405275157b3SThara Gopinath 406275157b3SThara Gopinath /* 407275157b3SThara Gopinath * Look for LMh interrupt. If no interrupt line is specified / 408275157b3SThara Gopinath * if there is an error, allow cpufreq to be enabled as usual. 409275157b3SThara Gopinath */ 4108f5783adSStephen Boyd data->throttle_irq = platform_get_irq_optional(pdev, index); 4118f5783adSStephen Boyd if (data->throttle_irq == -ENXIO) 4128f5783adSStephen Boyd return 0; 4138f5783adSStephen Boyd if (data->throttle_irq < 0) 4148f5783adSStephen Boyd return data->throttle_irq; 415275157b3SThara Gopinath 416275157b3SThara Gopinath data->cancel_throttle = false; 417275157b3SThara Gopinath data->policy = policy; 418275157b3SThara Gopinath 419275157b3SThara Gopinath mutex_init(&data->throttle_lock); 420275157b3SThara Gopinath INIT_DEFERRABLE_WORK(&data->throttle_work, qcom_lmh_dcvs_poll); 421275157b3SThara Gopinath 422be6592edSArd Biesheuvel snprintf(data->irq_name, sizeof(data->irq_name), "dcvsh-irq-%u", policy->cpu); 423275157b3SThara Gopinath ret = request_threaded_irq(data->throttle_irq, NULL, qcom_lmh_dcvs_handle_irq, 424ef8ee1cbSBjorn Andersson IRQF_ONESHOT | IRQF_NO_AUTOEN, data->irq_name, data); 425275157b3SThara Gopinath if (ret) { 426be6592edSArd Biesheuvel dev_err(&pdev->dev, "Error registering %s: %d\n", data->irq_name, ret); 427275157b3SThara Gopinath return 0; 428275157b3SThara Gopinath } 429275157b3SThara Gopinath 4303ed6dfbdSVladimir Zapolskiy ret = irq_set_affinity_hint(data->throttle_irq, policy->cpus); 4313ed6dfbdSVladimir Zapolskiy if (ret) 4323ed6dfbdSVladimir Zapolskiy dev_err(&pdev->dev, "Failed to set CPU affinity of %s[%d]\n", 4333ed6dfbdSVladimir Zapolskiy data->irq_name, data->throttle_irq); 4343ed6dfbdSVladimir Zapolskiy 435275157b3SThara Gopinath return 0; 436275157b3SThara Gopinath } 437275157b3SThara Gopinath 438a1eb080aSDmitry Baryshkov static int qcom_cpufreq_hw_cpu_online(struct cpufreq_policy *policy) 439275157b3SThara Gopinath { 440a1eb080aSDmitry Baryshkov struct qcom_cpufreq_data *data = policy->driver_data; 441a1eb080aSDmitry Baryshkov struct platform_device *pdev = cpufreq_get_driver_data(); 442a1eb080aSDmitry Baryshkov int ret; 443a1eb080aSDmitry Baryshkov 444668a7a12SStephen Boyd if (data->throttle_irq <= 0) 445668a7a12SStephen Boyd return 0; 446668a7a12SStephen Boyd 447*cdcf8eb3SPierre Gondois mutex_lock(&data->throttle_lock); 448*cdcf8eb3SPierre Gondois data->cancel_throttle = false; 449*cdcf8eb3SPierre Gondois mutex_unlock(&data->throttle_lock); 450*cdcf8eb3SPierre Gondois 451a1eb080aSDmitry Baryshkov ret = irq_set_affinity_hint(data->throttle_irq, policy->cpus); 452a1eb080aSDmitry Baryshkov if (ret) 453a1eb080aSDmitry Baryshkov dev_err(&pdev->dev, "Failed to set CPU affinity of %s[%d]\n", 454a1eb080aSDmitry Baryshkov data->irq_name, data->throttle_irq); 455a1eb080aSDmitry Baryshkov 456a1eb080aSDmitry Baryshkov return ret; 457a1eb080aSDmitry Baryshkov } 458a1eb080aSDmitry Baryshkov 459a1eb080aSDmitry Baryshkov static int qcom_cpufreq_hw_cpu_offline(struct cpufreq_policy *policy) 460a1eb080aSDmitry Baryshkov { 461a1eb080aSDmitry Baryshkov struct qcom_cpufreq_data *data = policy->driver_data; 462a1eb080aSDmitry Baryshkov 463275157b3SThara Gopinath if (data->throttle_irq <= 0) 464a1eb080aSDmitry Baryshkov return 0; 465275157b3SThara Gopinath 466275157b3SThara Gopinath mutex_lock(&data->throttle_lock); 467275157b3SThara Gopinath data->cancel_throttle = true; 468275157b3SThara Gopinath mutex_unlock(&data->throttle_lock); 469275157b3SThara Gopinath 470275157b3SThara Gopinath cancel_delayed_work_sync(&data->throttle_work); 471be5985b3SDmitry Baryshkov irq_set_affinity_hint(data->throttle_irq, NULL); 472a1eb080aSDmitry Baryshkov 473a1eb080aSDmitry Baryshkov return 0; 474a1eb080aSDmitry Baryshkov } 475a1eb080aSDmitry Baryshkov 476a1eb080aSDmitry Baryshkov static void qcom_cpufreq_hw_lmh_exit(struct qcom_cpufreq_data *data) 477a1eb080aSDmitry Baryshkov { 478668a7a12SStephen Boyd if (data->throttle_irq <= 0) 479668a7a12SStephen Boyd return; 480668a7a12SStephen Boyd 481275157b3SThara Gopinath free_irq(data->throttle_irq, data); 482275157b3SThara Gopinath } 483275157b3SThara Gopinath 4842849dd8bSTaniya Das static int qcom_cpufreq_hw_cpu_init(struct cpufreq_policy *policy) 4852849dd8bSTaniya Das { 486bd74e286SManivannan Sadhasivam struct platform_device *pdev = cpufreq_get_driver_data(); 487bd74e286SManivannan Sadhasivam struct device *dev = &pdev->dev; 4882849dd8bSTaniya Das struct of_phandle_args args; 4892849dd8bSTaniya Das struct device_node *cpu_np; 49055538fbcSTaniya Das struct device *cpu_dev; 49167fc209bSShawn Guo struct resource *res; 4922849dd8bSTaniya Das void __iomem *base; 493dcd1fd72SManivannan Sadhasivam struct qcom_cpufreq_data *data; 4942849dd8bSTaniya Das int ret, index; 4952849dd8bSTaniya Das 49655538fbcSTaniya Das cpu_dev = get_cpu_device(policy->cpu); 49755538fbcSTaniya Das if (!cpu_dev) { 49855538fbcSTaniya Das pr_err("%s: failed to get cpu%d device\n", __func__, 49955538fbcSTaniya Das policy->cpu); 50055538fbcSTaniya Das return -ENODEV; 50155538fbcSTaniya Das } 50255538fbcSTaniya Das 5032849dd8bSTaniya Das cpu_np = of_cpu_device_node_get(policy->cpu); 5042849dd8bSTaniya Das if (!cpu_np) 5052849dd8bSTaniya Das return -EINVAL; 5062849dd8bSTaniya Das 5072849dd8bSTaniya Das ret = of_parse_phandle_with_args(cpu_np, "qcom,freq-domain", 5082849dd8bSTaniya Das "#freq-domain-cells", 0, &args); 5092849dd8bSTaniya Das of_node_put(cpu_np); 5102849dd8bSTaniya Das if (ret) 5112849dd8bSTaniya Das return ret; 5122849dd8bSTaniya Das 5132849dd8bSTaniya Das index = args.args[0]; 5142849dd8bSTaniya Das 51567fc209bSShawn Guo res = platform_get_resource(pdev, IORESOURCE_MEM, index); 51667fc209bSShawn Guo if (!res) { 51767fc209bSShawn Guo dev_err(dev, "failed to get mem resource %d\n", index); 51867fc209bSShawn Guo return -ENODEV; 51967fc209bSShawn Guo } 5202849dd8bSTaniya Das 52167fc209bSShawn Guo if (!request_mem_region(res->start, resource_size(res), res->name)) { 52267fc209bSShawn Guo dev_err(dev, "failed to request resource %pR\n", res); 52367fc209bSShawn Guo return -EBUSY; 52467fc209bSShawn Guo } 52567fc209bSShawn Guo 52667fc209bSShawn Guo base = ioremap(res->start, resource_size(res)); 527536eb97aSWei Yongjun if (!base) { 52867fc209bSShawn Guo dev_err(dev, "failed to map resource %pR\n", res); 529536eb97aSWei Yongjun ret = -ENOMEM; 53067fc209bSShawn Guo goto release_region; 53167fc209bSShawn Guo } 53267fc209bSShawn Guo 53367fc209bSShawn Guo data = kzalloc(sizeof(*data), GFP_KERNEL); 534dcd1fd72SManivannan Sadhasivam if (!data) { 535dcd1fd72SManivannan Sadhasivam ret = -ENOMEM; 53667fc209bSShawn Guo goto unmap_base; 537dcd1fd72SManivannan Sadhasivam } 538dcd1fd72SManivannan Sadhasivam 539dcd1fd72SManivannan Sadhasivam data->soc_data = of_device_get_match_data(&pdev->dev); 540dcd1fd72SManivannan Sadhasivam data->base = base; 54167fc209bSShawn Guo data->res = res; 5422849dd8bSTaniya Das 5432849dd8bSTaniya Das /* HW should be in enabled state to proceed */ 544dcd1fd72SManivannan Sadhasivam if (!(readl_relaxed(base + data->soc_data->reg_enable) & 0x1)) { 5452849dd8bSTaniya Das dev_err(dev, "Domain-%d cpufreq hardware not enabled\n", index); 5462849dd8bSTaniya Das ret = -ENODEV; 5472849dd8bSTaniya Das goto error; 5482849dd8bSTaniya Das } 5492849dd8bSTaniya Das 550c377d4baSBjorn Andersson if (readl_relaxed(base + data->soc_data->reg_dcvs_ctrl) & 0x1) 551c377d4baSBjorn Andersson data->per_core_dcvs = true; 552c377d4baSBjorn Andersson 5532849dd8bSTaniya Das qcom_get_related_cpus(index, policy->cpus); 554b48cd0d1SYury Norov if (cpumask_empty(policy->cpus)) { 5552849dd8bSTaniya Das dev_err(dev, "Domain-%d failed to get related CPUs\n", index); 5562849dd8bSTaniya Das ret = -ENOENT; 5572849dd8bSTaniya Das goto error; 5582849dd8bSTaniya Das } 5592849dd8bSTaniya Das 560dcd1fd72SManivannan Sadhasivam policy->driver_data = data; 561f0712aceSTaniya Das policy->dvfs_possible_from_any_cpu = true; 5622849dd8bSTaniya Das 563dcd1fd72SManivannan Sadhasivam ret = qcom_cpufreq_hw_read_lut(cpu_dev, policy); 5642849dd8bSTaniya Das if (ret) { 5652849dd8bSTaniya Das dev_err(dev, "Domain-%d failed to read LUT\n", index); 5662849dd8bSTaniya Das goto error; 5672849dd8bSTaniya Das } 5682849dd8bSTaniya Das 56955538fbcSTaniya Das ret = dev_pm_opp_get_opp_count(cpu_dev); 57055538fbcSTaniya Das if (ret <= 0) { 57155538fbcSTaniya Das dev_err(cpu_dev, "Failed to add OPPs\n"); 57255538fbcSTaniya Das ret = -ENODEV; 57355538fbcSTaniya Das goto error; 57455538fbcSTaniya Das } 57555538fbcSTaniya Das 57626699172SShawn Guo if (policy_has_boost_freq(policy)) { 57726699172SShawn Guo ret = cpufreq_enable_boost_support(); 57826699172SShawn Guo if (ret) 57926699172SShawn Guo dev_warn(cpu_dev, "failed to enable boost: %d\n", ret); 58026699172SShawn Guo } 58126699172SShawn Guo 582275157b3SThara Gopinath ret = qcom_cpufreq_hw_lmh_init(policy, index); 583275157b3SThara Gopinath if (ret) 584275157b3SThara Gopinath goto error; 585275157b3SThara Gopinath 5862849dd8bSTaniya Das return 0; 5872849dd8bSTaniya Das error: 58867fc209bSShawn Guo kfree(data); 58967fc209bSShawn Guo unmap_base: 59002fc4095SShawn Guo iounmap(base); 59167fc209bSShawn Guo release_region: 59267fc209bSShawn Guo release_mem_region(res->start, resource_size(res)); 5932849dd8bSTaniya Das return ret; 5942849dd8bSTaniya Das } 5952849dd8bSTaniya Das 5962849dd8bSTaniya Das static int qcom_cpufreq_hw_cpu_exit(struct cpufreq_policy *policy) 5972849dd8bSTaniya Das { 59855538fbcSTaniya Das struct device *cpu_dev = get_cpu_device(policy->cpu); 599dcd1fd72SManivannan Sadhasivam struct qcom_cpufreq_data *data = policy->driver_data; 60067fc209bSShawn Guo struct resource *res = data->res; 60167fc209bSShawn Guo void __iomem *base = data->base; 6022849dd8bSTaniya Das 60355538fbcSTaniya Das dev_pm_opp_remove_all_dynamic(cpu_dev); 60451c843cfSSibi Sankar dev_pm_opp_of_cpumask_remove_table(policy->related_cpus); 605275157b3SThara Gopinath qcom_cpufreq_hw_lmh_exit(data); 6062849dd8bSTaniya Das kfree(policy->freq_table); 60767fc209bSShawn Guo kfree(data); 60867fc209bSShawn Guo iounmap(base); 60967fc209bSShawn Guo release_mem_region(res->start, resource_size(res)); 6102849dd8bSTaniya Das 6112849dd8bSTaniya Das return 0; 6122849dd8bSTaniya Das } 6132849dd8bSTaniya Das 614ef8ee1cbSBjorn Andersson static void qcom_cpufreq_ready(struct cpufreq_policy *policy) 615ef8ee1cbSBjorn Andersson { 616ef8ee1cbSBjorn Andersson struct qcom_cpufreq_data *data = policy->driver_data; 617ef8ee1cbSBjorn Andersson 618ef8ee1cbSBjorn Andersson if (data->throttle_irq >= 0) 619ef8ee1cbSBjorn Andersson enable_irq(data->throttle_irq); 620ef8ee1cbSBjorn Andersson } 621ef8ee1cbSBjorn Andersson 6222849dd8bSTaniya Das static struct freq_attr *qcom_cpufreq_hw_attr[] = { 6232849dd8bSTaniya Das &cpufreq_freq_attr_scaling_available_freqs, 6242849dd8bSTaniya Das &cpufreq_freq_attr_scaling_boost_freqs, 6252849dd8bSTaniya Das NULL 6262849dd8bSTaniya Das }; 6272849dd8bSTaniya Das 6282849dd8bSTaniya Das static struct cpufreq_driver cpufreq_qcom_hw_driver = { 6295ae4a4b4SViresh Kumar .flags = CPUFREQ_NEED_INITIAL_FREQ_CHECK | 6304c5ff1c8SAmit Kucheria CPUFREQ_HAVE_GOVERNOR_PER_POLICY | 6314c5ff1c8SAmit Kucheria CPUFREQ_IS_COOLING_DEV, 6322849dd8bSTaniya Das .verify = cpufreq_generic_frequency_table_verify, 6332849dd8bSTaniya Das .target_index = qcom_cpufreq_hw_target_index, 6342849dd8bSTaniya Das .get = qcom_cpufreq_hw_get, 6352849dd8bSTaniya Das .init = qcom_cpufreq_hw_cpu_init, 6362849dd8bSTaniya Das .exit = qcom_cpufreq_hw_cpu_exit, 637a1eb080aSDmitry Baryshkov .online = qcom_cpufreq_hw_cpu_online, 638a1eb080aSDmitry Baryshkov .offline = qcom_cpufreq_hw_cpu_offline, 639e96c2153SViresh Kumar .register_em = cpufreq_register_em_with_opp, 6402849dd8bSTaniya Das .fast_switch = qcom_cpufreq_hw_fast_switch, 6412849dd8bSTaniya Das .name = "qcom-cpufreq-hw", 6422849dd8bSTaniya Das .attr = qcom_cpufreq_hw_attr, 643ef8ee1cbSBjorn Andersson .ready = qcom_cpufreq_ready, 6442849dd8bSTaniya Das }; 6452849dd8bSTaniya Das 6462849dd8bSTaniya Das static int qcom_cpufreq_hw_driver_probe(struct platform_device *pdev) 6472849dd8bSTaniya Das { 64851c843cfSSibi Sankar struct device *cpu_dev; 6492849dd8bSTaniya Das struct clk *clk; 6502849dd8bSTaniya Das int ret; 6512849dd8bSTaniya Das 6522849dd8bSTaniya Das clk = clk_get(&pdev->dev, "xo"); 6532849dd8bSTaniya Das if (IS_ERR(clk)) 6542849dd8bSTaniya Das return PTR_ERR(clk); 6552849dd8bSTaniya Das 6562849dd8bSTaniya Das xo_rate = clk_get_rate(clk); 6572849dd8bSTaniya Das clk_put(clk); 6582849dd8bSTaniya Das 6592849dd8bSTaniya Das clk = clk_get(&pdev->dev, "alternate"); 6602849dd8bSTaniya Das if (IS_ERR(clk)) 6612849dd8bSTaniya Das return PTR_ERR(clk); 6622849dd8bSTaniya Das 6632849dd8bSTaniya Das cpu_hw_rate = clk_get_rate(clk) / CLK_HW_DIV; 6642849dd8bSTaniya Das clk_put(clk); 6652849dd8bSTaniya Das 666bd74e286SManivannan Sadhasivam cpufreq_qcom_hw_driver.driver_data = pdev; 6672849dd8bSTaniya Das 66851c843cfSSibi Sankar /* Check for optional interconnect paths on CPU0 */ 66951c843cfSSibi Sankar cpu_dev = get_cpu_device(0); 67051c843cfSSibi Sankar if (!cpu_dev) 67151c843cfSSibi Sankar return -EPROBE_DEFER; 67251c843cfSSibi Sankar 67351c843cfSSibi Sankar ret = dev_pm_opp_of_find_icc_paths(cpu_dev, NULL); 67451c843cfSSibi Sankar if (ret) 67551c843cfSSibi Sankar return ret; 67651c843cfSSibi Sankar 6772849dd8bSTaniya Das ret = cpufreq_register_driver(&cpufreq_qcom_hw_driver); 6782849dd8bSTaniya Das if (ret) 6792849dd8bSTaniya Das dev_err(&pdev->dev, "CPUFreq HW driver failed to register\n"); 6802849dd8bSTaniya Das else 6812849dd8bSTaniya Das dev_dbg(&pdev->dev, "QCOM CPUFreq HW driver initialized\n"); 6822849dd8bSTaniya Das 6832849dd8bSTaniya Das return ret; 6842849dd8bSTaniya Das } 6852849dd8bSTaniya Das 6862849dd8bSTaniya Das static int qcom_cpufreq_hw_driver_remove(struct platform_device *pdev) 6872849dd8bSTaniya Das { 6882849dd8bSTaniya Das return cpufreq_unregister_driver(&cpufreq_qcom_hw_driver); 6892849dd8bSTaniya Das } 6902849dd8bSTaniya Das 6912849dd8bSTaniya Das static struct platform_driver qcom_cpufreq_hw_driver = { 6922849dd8bSTaniya Das .probe = qcom_cpufreq_hw_driver_probe, 6932849dd8bSTaniya Das .remove = qcom_cpufreq_hw_driver_remove, 6942849dd8bSTaniya Das .driver = { 6952849dd8bSTaniya Das .name = "qcom-cpufreq-hw", 6962849dd8bSTaniya Das .of_match_table = qcom_cpufreq_hw_match, 6972849dd8bSTaniya Das }, 6982849dd8bSTaniya Das }; 6992849dd8bSTaniya Das 7002849dd8bSTaniya Das static int __init qcom_cpufreq_hw_init(void) 7012849dd8bSTaniya Das { 7022849dd8bSTaniya Das return platform_driver_register(&qcom_cpufreq_hw_driver); 7032849dd8bSTaniya Das } 70411ff4bddSAmit Kucheria postcore_initcall(qcom_cpufreq_hw_init); 7052849dd8bSTaniya Das 7062849dd8bSTaniya Das static void __exit qcom_cpufreq_hw_exit(void) 7072849dd8bSTaniya Das { 7082849dd8bSTaniya Das platform_driver_unregister(&qcom_cpufreq_hw_driver); 7092849dd8bSTaniya Das } 7102849dd8bSTaniya Das module_exit(qcom_cpufreq_hw_exit); 7112849dd8bSTaniya Das 7122849dd8bSTaniya Das MODULE_DESCRIPTION("QCOM CPUFREQ HW Driver"); 7132849dd8bSTaniya Das MODULE_LICENSE("GPL v2"); 714