1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2020 Linaro Ltd 4 */ 5 6 #include <linux/clk.h> 7 #include <linux/device.h> 8 #include <linux/interconnect-provider.h> 9 #include <linux/io.h> 10 #include <linux/module.h> 11 #include <linux/of_device.h> 12 #include <linux/of_platform.h> 13 #include <linux/platform_device.h> 14 #include <linux/pm_domain.h> 15 #include <linux/regmap.h> 16 #include <linux/slab.h> 17 18 #include "smd-rpm.h" 19 #include "icc-common.h" 20 #include "icc-rpm.h" 21 22 /* QNOC QoS */ 23 #define QNOC_QOS_MCTL_LOWn_ADDR(n) (0x8 + (n * 0x1000)) 24 #define QNOC_QOS_MCTL_DFLT_PRIO_MASK 0x70 25 #define QNOC_QOS_MCTL_DFLT_PRIO_SHIFT 4 26 #define QNOC_QOS_MCTL_URGFWD_EN_MASK 0x8 27 #define QNOC_QOS_MCTL_URGFWD_EN_SHIFT 3 28 29 /* BIMC QoS */ 30 #define M_BKE_REG_BASE(n) (0x300 + (0x4000 * n)) 31 #define M_BKE_EN_ADDR(n) (M_BKE_REG_BASE(n)) 32 #define M_BKE_HEALTH_CFG_ADDR(i, n) (M_BKE_REG_BASE(n) + 0x40 + (0x4 * i)) 33 34 #define M_BKE_HEALTH_CFG_LIMITCMDS_MASK 0x80000000 35 #define M_BKE_HEALTH_CFG_AREQPRIO_MASK 0x300 36 #define M_BKE_HEALTH_CFG_PRIOLVL_MASK 0x3 37 #define M_BKE_HEALTH_CFG_AREQPRIO_SHIFT 0x8 38 #define M_BKE_HEALTH_CFG_LIMITCMDS_SHIFT 0x1f 39 40 #define M_BKE_EN_EN_BMASK 0x1 41 42 /* NoC QoS */ 43 #define NOC_QOS_PRIORITYn_ADDR(n) (0x8 + (n * 0x1000)) 44 #define NOC_QOS_PRIORITY_P1_MASK 0xc 45 #define NOC_QOS_PRIORITY_P0_MASK 0x3 46 #define NOC_QOS_PRIORITY_P1_SHIFT 0x2 47 48 #define NOC_QOS_MODEn_ADDR(n) (0xc + (n * 0x1000)) 49 #define NOC_QOS_MODEn_MASK 0x3 50 51 static int qcom_icc_set_qnoc_qos(struct icc_node *src, u64 max_bw) 52 { 53 struct icc_provider *provider = src->provider; 54 struct qcom_icc_provider *qp = to_qcom_provider(provider); 55 struct qcom_icc_node *qn = src->data; 56 struct qcom_icc_qos *qos = &qn->qos; 57 int rc; 58 59 rc = regmap_update_bits(qp->regmap, 60 qp->qos_offset + QNOC_QOS_MCTL_LOWn_ADDR(qos->qos_port), 61 QNOC_QOS_MCTL_DFLT_PRIO_MASK, 62 qos->areq_prio << QNOC_QOS_MCTL_DFLT_PRIO_SHIFT); 63 if (rc) 64 return rc; 65 66 return regmap_update_bits(qp->regmap, 67 qp->qos_offset + QNOC_QOS_MCTL_LOWn_ADDR(qos->qos_port), 68 QNOC_QOS_MCTL_URGFWD_EN_MASK, 69 !!qos->urg_fwd_en << QNOC_QOS_MCTL_URGFWD_EN_SHIFT); 70 } 71 72 static int qcom_icc_bimc_set_qos_health(struct qcom_icc_provider *qp, 73 struct qcom_icc_qos *qos, 74 int regnum) 75 { 76 u32 val; 77 u32 mask; 78 79 val = qos->prio_level; 80 mask = M_BKE_HEALTH_CFG_PRIOLVL_MASK; 81 82 val |= qos->areq_prio << M_BKE_HEALTH_CFG_AREQPRIO_SHIFT; 83 mask |= M_BKE_HEALTH_CFG_AREQPRIO_MASK; 84 85 /* LIMITCMDS is not present on M_BKE_HEALTH_3 */ 86 if (regnum != 3) { 87 val |= qos->limit_commands << M_BKE_HEALTH_CFG_LIMITCMDS_SHIFT; 88 mask |= M_BKE_HEALTH_CFG_LIMITCMDS_MASK; 89 } 90 91 return regmap_update_bits(qp->regmap, 92 qp->qos_offset + M_BKE_HEALTH_CFG_ADDR(regnum, qos->qos_port), 93 mask, val); 94 } 95 96 static int qcom_icc_set_bimc_qos(struct icc_node *src, u64 max_bw) 97 { 98 struct qcom_icc_provider *qp; 99 struct qcom_icc_node *qn; 100 struct icc_provider *provider; 101 u32 mode = NOC_QOS_MODE_BYPASS; 102 u32 val = 0; 103 int i, rc = 0; 104 105 qn = src->data; 106 provider = src->provider; 107 qp = to_qcom_provider(provider); 108 109 if (qn->qos.qos_mode != NOC_QOS_MODE_INVALID) 110 mode = qn->qos.qos_mode; 111 112 /* QoS Priority: The QoS Health parameters are getting considered 113 * only if we are NOT in Bypass Mode. 114 */ 115 if (mode != NOC_QOS_MODE_BYPASS) { 116 for (i = 3; i >= 0; i--) { 117 rc = qcom_icc_bimc_set_qos_health(qp, 118 &qn->qos, i); 119 if (rc) 120 return rc; 121 } 122 123 /* Set BKE_EN to 1 when Fixed, Regulator or Limiter Mode */ 124 val = 1; 125 } 126 127 return regmap_update_bits(qp->regmap, 128 qp->qos_offset + M_BKE_EN_ADDR(qn->qos.qos_port), 129 M_BKE_EN_EN_BMASK, val); 130 } 131 132 static int qcom_icc_noc_set_qos_priority(struct qcom_icc_provider *qp, 133 struct qcom_icc_qos *qos) 134 { 135 u32 val; 136 int rc; 137 138 /* Must be updated one at a time, P1 first, P0 last */ 139 val = qos->areq_prio << NOC_QOS_PRIORITY_P1_SHIFT; 140 rc = regmap_update_bits(qp->regmap, 141 qp->qos_offset + NOC_QOS_PRIORITYn_ADDR(qos->qos_port), 142 NOC_QOS_PRIORITY_P1_MASK, val); 143 if (rc) 144 return rc; 145 146 return regmap_update_bits(qp->regmap, 147 qp->qos_offset + NOC_QOS_PRIORITYn_ADDR(qos->qos_port), 148 NOC_QOS_PRIORITY_P0_MASK, qos->prio_level); 149 } 150 151 static int qcom_icc_set_noc_qos(struct icc_node *src, u64 max_bw) 152 { 153 struct qcom_icc_provider *qp; 154 struct qcom_icc_node *qn; 155 struct icc_provider *provider; 156 u32 mode = NOC_QOS_MODE_BYPASS; 157 int rc = 0; 158 159 qn = src->data; 160 provider = src->provider; 161 qp = to_qcom_provider(provider); 162 163 if (qn->qos.qos_port < 0) { 164 dev_dbg(src->provider->dev, 165 "NoC QoS: Skipping %s: vote aggregated on parent.\n", 166 qn->name); 167 return 0; 168 } 169 170 if (qn->qos.qos_mode != NOC_QOS_MODE_INVALID) 171 mode = qn->qos.qos_mode; 172 173 if (mode == NOC_QOS_MODE_FIXED) { 174 dev_dbg(src->provider->dev, "NoC QoS: %s: Set Fixed mode\n", 175 qn->name); 176 rc = qcom_icc_noc_set_qos_priority(qp, &qn->qos); 177 if (rc) 178 return rc; 179 } else if (mode == NOC_QOS_MODE_BYPASS) { 180 dev_dbg(src->provider->dev, "NoC QoS: %s: Set Bypass mode\n", 181 qn->name); 182 } 183 184 return regmap_update_bits(qp->regmap, 185 qp->qos_offset + NOC_QOS_MODEn_ADDR(qn->qos.qos_port), 186 NOC_QOS_MODEn_MASK, mode); 187 } 188 189 static int qcom_icc_qos_set(struct icc_node *node, u64 sum_bw) 190 { 191 struct qcom_icc_provider *qp = to_qcom_provider(node->provider); 192 struct qcom_icc_node *qn = node->data; 193 194 dev_dbg(node->provider->dev, "Setting QoS for %s\n", qn->name); 195 196 switch (qp->type) { 197 case QCOM_ICC_BIMC: 198 return qcom_icc_set_bimc_qos(node, sum_bw); 199 case QCOM_ICC_QNOC: 200 return qcom_icc_set_qnoc_qos(node, sum_bw); 201 default: 202 return qcom_icc_set_noc_qos(node, sum_bw); 203 } 204 } 205 206 static int qcom_icc_rpm_set(int mas_rpm_id, int slv_rpm_id, u64 sum_bw) 207 { 208 int ret = 0; 209 210 if (mas_rpm_id != -1) { 211 ret = qcom_icc_rpm_smd_send(QCOM_SMD_RPM_ACTIVE_STATE, 212 RPM_BUS_MASTER_REQ, 213 mas_rpm_id, 214 sum_bw); 215 if (ret) { 216 pr_err("qcom_icc_rpm_smd_send mas %d error %d\n", 217 mas_rpm_id, ret); 218 return ret; 219 } 220 } 221 222 if (slv_rpm_id != -1) { 223 ret = qcom_icc_rpm_smd_send(QCOM_SMD_RPM_ACTIVE_STATE, 224 RPM_BUS_SLAVE_REQ, 225 slv_rpm_id, 226 sum_bw); 227 if (ret) { 228 pr_err("qcom_icc_rpm_smd_send slv %d error %d\n", 229 slv_rpm_id, ret); 230 return ret; 231 } 232 } 233 234 return ret; 235 } 236 237 static int __qcom_icc_set(struct icc_node *n, struct qcom_icc_node *qn, 238 u64 sum_bw) 239 { 240 int ret; 241 242 if (!qn->qos.ap_owned) { 243 /* send bandwidth request message to the RPM processor */ 244 ret = qcom_icc_rpm_set(qn->mas_rpm_id, qn->slv_rpm_id, sum_bw); 245 if (ret) 246 return ret; 247 } else if (qn->qos.qos_mode != -1) { 248 /* set bandwidth directly from the AP */ 249 ret = qcom_icc_qos_set(n, sum_bw); 250 if (ret) 251 return ret; 252 } 253 254 return 0; 255 } 256 257 /** 258 * qcom_icc_pre_bw_aggregate - cleans up values before re-aggregate requests 259 * @node: icc node to operate on 260 */ 261 static void qcom_icc_pre_bw_aggregate(struct icc_node *node) 262 { 263 struct qcom_icc_node *qn; 264 size_t i; 265 266 qn = node->data; 267 for (i = 0; i < QCOM_ICC_NUM_BUCKETS; i++) { 268 qn->sum_avg[i] = 0; 269 qn->max_peak[i] = 0; 270 } 271 } 272 273 /** 274 * qcom_icc_bw_aggregate - aggregate bw for buckets indicated by tag 275 * @node: node to aggregate 276 * @tag: tag to indicate which buckets to aggregate 277 * @avg_bw: new bw to sum aggregate 278 * @peak_bw: new bw to max aggregate 279 * @agg_avg: existing aggregate avg bw val 280 * @agg_peak: existing aggregate peak bw val 281 */ 282 static int qcom_icc_bw_aggregate(struct icc_node *node, u32 tag, u32 avg_bw, 283 u32 peak_bw, u32 *agg_avg, u32 *agg_peak) 284 { 285 size_t i; 286 struct qcom_icc_node *qn; 287 288 qn = node->data; 289 290 if (!tag) 291 tag = QCOM_ICC_TAG_ALWAYS; 292 293 for (i = 0; i < QCOM_ICC_NUM_BUCKETS; i++) { 294 if (tag & BIT(i)) { 295 qn->sum_avg[i] += avg_bw; 296 qn->max_peak[i] = max_t(u32, qn->max_peak[i], peak_bw); 297 } 298 } 299 300 *agg_avg += avg_bw; 301 *agg_peak = max_t(u32, *agg_peak, peak_bw); 302 return 0; 303 } 304 305 /** 306 * qcom_icc_bus_aggregate - aggregate bandwidth by traversing all nodes 307 * @provider: generic interconnect provider 308 * @agg_avg: an array for aggregated average bandwidth of buckets 309 * @agg_peak: an array for aggregated peak bandwidth of buckets 310 * @max_agg_avg: pointer to max value of aggregated average bandwidth 311 */ 312 static void qcom_icc_bus_aggregate(struct icc_provider *provider, 313 u64 *agg_avg, u64 *agg_peak, 314 u64 *max_agg_avg) 315 { 316 struct icc_node *node; 317 struct qcom_icc_node *qn; 318 int i; 319 320 /* Initialise aggregate values */ 321 for (i = 0; i < QCOM_ICC_NUM_BUCKETS; i++) { 322 agg_avg[i] = 0; 323 agg_peak[i] = 0; 324 } 325 326 *max_agg_avg = 0; 327 328 /* 329 * Iterate nodes on the interconnect and aggregate bandwidth 330 * requests for every bucket. 331 */ 332 list_for_each_entry(node, &provider->nodes, node_list) { 333 qn = node->data; 334 for (i = 0; i < QCOM_ICC_NUM_BUCKETS; i++) { 335 agg_avg[i] += qn->sum_avg[i]; 336 agg_peak[i] = max_t(u64, agg_peak[i], qn->max_peak[i]); 337 } 338 } 339 340 /* Find maximum values across all buckets */ 341 for (i = 0; i < QCOM_ICC_NUM_BUCKETS; i++) 342 *max_agg_avg = max_t(u64, *max_agg_avg, agg_avg[i]); 343 } 344 345 static int qcom_icc_set(struct icc_node *src, struct icc_node *dst) 346 { 347 struct qcom_icc_provider *qp; 348 struct qcom_icc_node *src_qn = NULL, *dst_qn = NULL; 349 struct icc_provider *provider; 350 u64 sum_bw; 351 u64 rate; 352 u64 agg_avg[QCOM_ICC_NUM_BUCKETS], agg_peak[QCOM_ICC_NUM_BUCKETS]; 353 u64 max_agg_avg; 354 int ret, i; 355 int bucket; 356 357 src_qn = src->data; 358 if (dst) 359 dst_qn = dst->data; 360 provider = src->provider; 361 qp = to_qcom_provider(provider); 362 363 qcom_icc_bus_aggregate(provider, agg_avg, agg_peak, &max_agg_avg); 364 365 sum_bw = icc_units_to_bps(max_agg_avg); 366 367 ret = __qcom_icc_set(src, src_qn, sum_bw); 368 if (ret) 369 return ret; 370 if (dst_qn) { 371 ret = __qcom_icc_set(dst, dst_qn, sum_bw); 372 if (ret) 373 return ret; 374 } 375 376 for (i = 0; i < qp->num_clks; i++) { 377 /* 378 * Use WAKE bucket for active clock, otherwise, use SLEEP bucket 379 * for other clocks. If a platform doesn't set interconnect 380 * path tags, by default use sleep bucket for all clocks. 381 * 382 * Note, AMC bucket is not supported yet. 383 */ 384 if (!strcmp(qp->bus_clks[i].id, "bus_a")) 385 bucket = QCOM_ICC_BUCKET_WAKE; 386 else 387 bucket = QCOM_ICC_BUCKET_SLEEP; 388 389 rate = icc_units_to_bps(max(agg_avg[bucket], agg_peak[bucket])); 390 do_div(rate, src_qn->buswidth); 391 rate = min_t(u64, rate, LONG_MAX); 392 393 if (qp->bus_clk_rate[i] == rate) 394 continue; 395 396 ret = clk_set_rate(qp->bus_clks[i].clk, rate); 397 if (ret) { 398 pr_err("%s clk_set_rate error: %d\n", 399 qp->bus_clks[i].id, ret); 400 return ret; 401 } 402 qp->bus_clk_rate[i] = rate; 403 } 404 405 return 0; 406 } 407 408 static const char * const bus_clocks[] = { 409 "bus", "bus_a", 410 }; 411 412 int qnoc_probe(struct platform_device *pdev) 413 { 414 struct device *dev = &pdev->dev; 415 const struct qcom_icc_desc *desc; 416 struct icc_onecell_data *data; 417 struct icc_provider *provider; 418 struct qcom_icc_node * const *qnodes; 419 struct qcom_icc_provider *qp; 420 struct icc_node *node; 421 size_t num_nodes, i; 422 const char * const *cds; 423 int cd_num; 424 int ret; 425 426 /* wait for the RPM proxy */ 427 if (!qcom_icc_rpm_smd_available()) 428 return -EPROBE_DEFER; 429 430 desc = of_device_get_match_data(dev); 431 if (!desc) 432 return -EINVAL; 433 434 qnodes = desc->nodes; 435 num_nodes = desc->num_nodes; 436 437 if (desc->num_clocks) { 438 cds = desc->clocks; 439 cd_num = desc->num_clocks; 440 } else { 441 cds = bus_clocks; 442 cd_num = ARRAY_SIZE(bus_clocks); 443 } 444 445 qp = devm_kzalloc(dev, struct_size(qp, bus_clks, cd_num), GFP_KERNEL); 446 if (!qp) 447 return -ENOMEM; 448 449 qp->bus_clk_rate = devm_kcalloc(dev, cd_num, sizeof(*qp->bus_clk_rate), 450 GFP_KERNEL); 451 if (!qp->bus_clk_rate) 452 return -ENOMEM; 453 454 data = devm_kzalloc(dev, struct_size(data, nodes, num_nodes), 455 GFP_KERNEL); 456 if (!data) 457 return -ENOMEM; 458 459 for (i = 0; i < cd_num; i++) 460 qp->bus_clks[i].id = cds[i]; 461 qp->num_clks = cd_num; 462 463 qp->type = desc->type; 464 qp->qos_offset = desc->qos_offset; 465 466 if (desc->regmap_cfg) { 467 struct resource *res; 468 void __iomem *mmio; 469 470 res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 471 if (!res) { 472 /* Try parent's regmap */ 473 qp->regmap = dev_get_regmap(dev->parent, NULL); 474 if (qp->regmap) 475 goto regmap_done; 476 return -ENODEV; 477 } 478 479 mmio = devm_ioremap_resource(dev, res); 480 481 if (IS_ERR(mmio)) { 482 dev_err(dev, "Cannot ioremap interconnect bus resource\n"); 483 return PTR_ERR(mmio); 484 } 485 486 qp->regmap = devm_regmap_init_mmio(dev, mmio, desc->regmap_cfg); 487 if (IS_ERR(qp->regmap)) { 488 dev_err(dev, "Cannot regmap interconnect bus resource\n"); 489 return PTR_ERR(qp->regmap); 490 } 491 } 492 493 regmap_done: 494 ret = devm_clk_bulk_get(dev, qp->num_clks, qp->bus_clks); 495 if (ret) 496 return ret; 497 498 ret = clk_bulk_prepare_enable(qp->num_clks, qp->bus_clks); 499 if (ret) 500 return ret; 501 502 if (desc->has_bus_pd) { 503 ret = dev_pm_domain_attach(dev, true); 504 if (ret) 505 return ret; 506 } 507 508 provider = &qp->provider; 509 INIT_LIST_HEAD(&provider->nodes); 510 provider->dev = dev; 511 provider->set = qcom_icc_set; 512 provider->pre_aggregate = qcom_icc_pre_bw_aggregate; 513 provider->aggregate = qcom_icc_bw_aggregate; 514 provider->xlate_extended = qcom_icc_xlate_extended; 515 provider->data = data; 516 517 ret = icc_provider_add(provider); 518 if (ret) { 519 dev_err(dev, "error adding interconnect provider: %d\n", ret); 520 clk_bulk_disable_unprepare(qp->num_clks, qp->bus_clks); 521 return ret; 522 } 523 524 for (i = 0; i < num_nodes; i++) { 525 size_t j; 526 527 node = icc_node_create(qnodes[i]->id); 528 if (IS_ERR(node)) { 529 ret = PTR_ERR(node); 530 goto err; 531 } 532 533 node->name = qnodes[i]->name; 534 node->data = qnodes[i]; 535 icc_node_add(node, provider); 536 537 for (j = 0; j < qnodes[i]->num_links; j++) 538 icc_link_create(node, qnodes[i]->links[j]); 539 540 data->nodes[i] = node; 541 } 542 data->num_nodes = num_nodes; 543 544 platform_set_drvdata(pdev, qp); 545 546 /* Populate child NoC devices if any */ 547 if (of_get_child_count(dev->of_node) > 0) 548 return of_platform_populate(dev->of_node, NULL, NULL, dev); 549 550 return 0; 551 err: 552 icc_nodes_remove(provider); 553 clk_bulk_disable_unprepare(qp->num_clks, qp->bus_clks); 554 icc_provider_del(provider); 555 556 return ret; 557 } 558 EXPORT_SYMBOL(qnoc_probe); 559 560 int qnoc_remove(struct platform_device *pdev) 561 { 562 struct qcom_icc_provider *qp = platform_get_drvdata(pdev); 563 564 icc_nodes_remove(&qp->provider); 565 clk_bulk_disable_unprepare(qp->num_clks, qp->bus_clks); 566 return icc_provider_del(&qp->provider); 567 } 568 EXPORT_SYMBOL(qnoc_remove); 569