1658628e7SLina Iyer // SPDX-License-Identifier: GPL-2.0 2658628e7SLina Iyer /* 3658628e7SLina Iyer * Copyright (c) 2016-2018, The Linux Foundation. All rights reserved. 4658628e7SLina Iyer */ 5658628e7SLina Iyer 6658628e7SLina Iyer #define pr_fmt(fmt) "%s " fmt, KBUILD_MODNAME 7658628e7SLina Iyer 8658628e7SLina Iyer #include <linux/atomic.h> 9658628e7SLina Iyer #include <linux/delay.h> 10658628e7SLina Iyer #include <linux/interrupt.h> 11658628e7SLina Iyer #include <linux/io.h> 12658628e7SLina Iyer #include <linux/kernel.h> 13658628e7SLina Iyer #include <linux/list.h> 14658628e7SLina Iyer #include <linux/of.h> 15658628e7SLina Iyer #include <linux/of_irq.h> 16658628e7SLina Iyer #include <linux/of_platform.h> 17658628e7SLina Iyer #include <linux/platform_device.h> 18658628e7SLina Iyer #include <linux/slab.h> 19658628e7SLina Iyer #include <linux/spinlock.h> 20658628e7SLina Iyer 21fdd102b5SDouglas Anderson #include <soc/qcom/cmd-db.h> 22658628e7SLina Iyer #include <soc/qcom/tcs.h> 23658628e7SLina Iyer #include <dt-bindings/soc/qcom,rpmh-rsc.h> 24658628e7SLina Iyer 25658628e7SLina Iyer #include "rpmh-internal.h" 26658628e7SLina Iyer 27fc087fe5SLina Iyer #define CREATE_TRACE_POINTS 28fc087fe5SLina Iyer #include "trace-rpmh.h" 29fc087fe5SLina Iyer 30658628e7SLina Iyer #define RSC_DRV_TCS_OFFSET 672 31658628e7SLina Iyer #define RSC_DRV_CMD_OFFSET 20 32658628e7SLina Iyer 33658628e7SLina Iyer /* DRV Configuration Information Register */ 34658628e7SLina Iyer #define DRV_PRNT_CHLD_CONFIG 0x0C 35658628e7SLina Iyer #define DRV_NUM_TCS_MASK 0x3F 36658628e7SLina Iyer #define DRV_NUM_TCS_SHIFT 6 37658628e7SLina Iyer #define DRV_NCPT_MASK 0x1F 38658628e7SLina Iyer #define DRV_NCPT_SHIFT 27 39658628e7SLina Iyer 40658628e7SLina Iyer /* Register offsets */ 41658628e7SLina Iyer #define RSC_DRV_IRQ_ENABLE 0x00 42658628e7SLina Iyer #define RSC_DRV_IRQ_STATUS 0x04 43658628e7SLina Iyer #define RSC_DRV_IRQ_CLEAR 0x08 44658628e7SLina Iyer #define RSC_DRV_CMD_WAIT_FOR_CMPL 0x10 45658628e7SLina Iyer #define RSC_DRV_CONTROL 0x14 46658628e7SLina Iyer #define RSC_DRV_STATUS 0x18 47658628e7SLina Iyer #define RSC_DRV_CMD_ENABLE 0x1C 48658628e7SLina Iyer #define RSC_DRV_CMD_MSGID 0x30 49658628e7SLina Iyer #define RSC_DRV_CMD_ADDR 0x34 50658628e7SLina Iyer #define RSC_DRV_CMD_DATA 0x38 51658628e7SLina Iyer #define RSC_DRV_CMD_STATUS 0x3C 52658628e7SLina Iyer #define RSC_DRV_CMD_RESP_DATA 0x40 53658628e7SLina Iyer 54658628e7SLina Iyer #define TCS_AMC_MODE_ENABLE BIT(16) 55658628e7SLina Iyer #define TCS_AMC_MODE_TRIGGER BIT(24) 56658628e7SLina Iyer 57658628e7SLina Iyer /* TCS CMD register bit mask */ 58658628e7SLina Iyer #define CMD_MSGID_LEN 8 59658628e7SLina Iyer #define CMD_MSGID_RESP_REQ BIT(8) 60658628e7SLina Iyer #define CMD_MSGID_WRITE BIT(16) 61658628e7SLina Iyer #define CMD_STATUS_ISSUED BIT(8) 62658628e7SLina Iyer #define CMD_STATUS_COMPL BIT(16) 63658628e7SLina Iyer 64658628e7SLina Iyer static u32 read_tcs_reg(struct rsc_drv *drv, int reg, int tcs_id, int cmd_id) 65658628e7SLina Iyer { 66658628e7SLina Iyer return readl_relaxed(drv->tcs_base + reg + RSC_DRV_TCS_OFFSET * tcs_id + 67658628e7SLina Iyer RSC_DRV_CMD_OFFSET * cmd_id); 68658628e7SLina Iyer } 69658628e7SLina Iyer 70658628e7SLina Iyer static void write_tcs_cmd(struct rsc_drv *drv, int reg, int tcs_id, int cmd_id, 71658628e7SLina Iyer u32 data) 72658628e7SLina Iyer { 73658628e7SLina Iyer writel_relaxed(data, drv->tcs_base + reg + RSC_DRV_TCS_OFFSET * tcs_id + 74658628e7SLina Iyer RSC_DRV_CMD_OFFSET * cmd_id); 75658628e7SLina Iyer } 76658628e7SLina Iyer 77658628e7SLina Iyer static void write_tcs_reg(struct rsc_drv *drv, int reg, int tcs_id, u32 data) 78658628e7SLina Iyer { 79658628e7SLina Iyer writel_relaxed(data, drv->tcs_base + reg + RSC_DRV_TCS_OFFSET * tcs_id); 80658628e7SLina Iyer } 81658628e7SLina Iyer 82658628e7SLina Iyer static void write_tcs_reg_sync(struct rsc_drv *drv, int reg, int tcs_id, 83658628e7SLina Iyer u32 data) 84658628e7SLina Iyer { 85658628e7SLina Iyer writel(data, drv->tcs_base + reg + RSC_DRV_TCS_OFFSET * tcs_id); 86658628e7SLina Iyer for (;;) { 87658628e7SLina Iyer if (data == readl(drv->tcs_base + reg + 88658628e7SLina Iyer RSC_DRV_TCS_OFFSET * tcs_id)) 89658628e7SLina Iyer break; 90658628e7SLina Iyer udelay(1); 91658628e7SLina Iyer } 92658628e7SLina Iyer } 93658628e7SLina Iyer 94658628e7SLina Iyer static bool tcs_is_free(struct rsc_drv *drv, int tcs_id) 95658628e7SLina Iyer { 96658628e7SLina Iyer return !test_bit(tcs_id, drv->tcs_in_use) && 97658628e7SLina Iyer read_tcs_reg(drv, RSC_DRV_STATUS, tcs_id, 0); 98658628e7SLina Iyer } 99658628e7SLina Iyer 100658628e7SLina Iyer static struct tcs_group *get_tcs_of_type(struct rsc_drv *drv, int type) 101658628e7SLina Iyer { 102658628e7SLina Iyer return &drv->tcs[type]; 103658628e7SLina Iyer } 104658628e7SLina Iyer 1059a3afcfbSLina Iyer static int tcs_invalidate(struct rsc_drv *drv, int type) 1069a3afcfbSLina Iyer { 1079a3afcfbSLina Iyer int m; 1089a3afcfbSLina Iyer struct tcs_group *tcs; 1099a3afcfbSLina Iyer 1109a3afcfbSLina Iyer tcs = get_tcs_of_type(drv, type); 1119a3afcfbSLina Iyer 1129a3afcfbSLina Iyer spin_lock(&tcs->lock); 1139a3afcfbSLina Iyer if (bitmap_empty(tcs->slots, MAX_TCS_SLOTS)) { 1149a3afcfbSLina Iyer spin_unlock(&tcs->lock); 1159a3afcfbSLina Iyer return 0; 1169a3afcfbSLina Iyer } 1179a3afcfbSLina Iyer 1189a3afcfbSLina Iyer for (m = tcs->offset; m < tcs->offset + tcs->num_tcs; m++) { 1199a3afcfbSLina Iyer if (!tcs_is_free(drv, m)) { 1209a3afcfbSLina Iyer spin_unlock(&tcs->lock); 1219a3afcfbSLina Iyer return -EAGAIN; 1229a3afcfbSLina Iyer } 1239a3afcfbSLina Iyer write_tcs_reg_sync(drv, RSC_DRV_CMD_ENABLE, m, 0); 1249a3afcfbSLina Iyer } 1259a3afcfbSLina Iyer bitmap_zero(tcs->slots, MAX_TCS_SLOTS); 1269a3afcfbSLina Iyer spin_unlock(&tcs->lock); 1279a3afcfbSLina Iyer 1289a3afcfbSLina Iyer return 0; 1299a3afcfbSLina Iyer } 1309a3afcfbSLina Iyer 1319a3afcfbSLina Iyer /** 1329a3afcfbSLina Iyer * rpmh_rsc_invalidate - Invalidate sleep and wake TCSes 1339a3afcfbSLina Iyer * 1349a3afcfbSLina Iyer * @drv: the RSC controller 1359a3afcfbSLina Iyer */ 1369a3afcfbSLina Iyer int rpmh_rsc_invalidate(struct rsc_drv *drv) 1379a3afcfbSLina Iyer { 1389a3afcfbSLina Iyer int ret; 1399a3afcfbSLina Iyer 1409a3afcfbSLina Iyer ret = tcs_invalidate(drv, SLEEP_TCS); 1419a3afcfbSLina Iyer if (!ret) 1429a3afcfbSLina Iyer ret = tcs_invalidate(drv, WAKE_TCS); 1439a3afcfbSLina Iyer 1449a3afcfbSLina Iyer return ret; 1459a3afcfbSLina Iyer } 1469a3afcfbSLina Iyer 147658628e7SLina Iyer static struct tcs_group *get_tcs_for_msg(struct rsc_drv *drv, 148658628e7SLina Iyer const struct tcs_request *msg) 149658628e7SLina Iyer { 1502de4b8d3SLina Iyer int type, ret; 1512de4b8d3SLina Iyer struct tcs_group *tcs; 152658628e7SLina Iyer 153658628e7SLina Iyer switch (msg->state) { 154658628e7SLina Iyer case RPMH_ACTIVE_ONLY_STATE: 155658628e7SLina Iyer type = ACTIVE_TCS; 156658628e7SLina Iyer break; 157fa460e45SLina Iyer case RPMH_WAKE_ONLY_STATE: 158fa460e45SLina Iyer type = WAKE_TCS; 159fa460e45SLina Iyer break; 160fa460e45SLina Iyer case RPMH_SLEEP_STATE: 161fa460e45SLina Iyer type = SLEEP_TCS; 162fa460e45SLina Iyer break; 163658628e7SLina Iyer default: 164658628e7SLina Iyer return ERR_PTR(-EINVAL); 165658628e7SLina Iyer } 166658628e7SLina Iyer 1672de4b8d3SLina Iyer /* 1682de4b8d3SLina Iyer * If we are making an active request on a RSC that does not have a 1692de4b8d3SLina Iyer * dedicated TCS for active state use, then re-purpose a wake TCS to 1702de4b8d3SLina Iyer * send active votes. 1712de4b8d3SLina Iyer * NOTE: The driver must be aware that this RSC does not have a 1722de4b8d3SLina Iyer * dedicated AMC, and therefore would invalidate the sleep and wake 1732de4b8d3SLina Iyer * TCSes before making an active state request. 1742de4b8d3SLina Iyer */ 1752de4b8d3SLina Iyer tcs = get_tcs_of_type(drv, type); 1766c805adfSRaju P.L.S.S.S.N if (msg->state == RPMH_ACTIVE_ONLY_STATE && !tcs->num_tcs) { 1772de4b8d3SLina Iyer tcs = get_tcs_of_type(drv, WAKE_TCS); 1786c805adfSRaju P.L.S.S.S.N if (tcs->num_tcs) { 1792de4b8d3SLina Iyer ret = rpmh_rsc_invalidate(drv); 1802de4b8d3SLina Iyer if (ret) 1812de4b8d3SLina Iyer return ERR_PTR(ret); 1822de4b8d3SLina Iyer } 1832de4b8d3SLina Iyer } 1842de4b8d3SLina Iyer 1852de4b8d3SLina Iyer return tcs; 186658628e7SLina Iyer } 187658628e7SLina Iyer 188658628e7SLina Iyer static const struct tcs_request *get_req_from_tcs(struct rsc_drv *drv, 189658628e7SLina Iyer int tcs_id) 190658628e7SLina Iyer { 191658628e7SLina Iyer struct tcs_group *tcs; 192658628e7SLina Iyer int i; 193658628e7SLina Iyer 194efa1c257SRaju P.L.S.S.S.N for (i = 0; i < TCS_TYPE_NR; i++) { 195658628e7SLina Iyer tcs = &drv->tcs[i]; 196658628e7SLina Iyer if (tcs->mask & BIT(tcs_id)) 197658628e7SLina Iyer return tcs->req[tcs_id - tcs->offset]; 198658628e7SLina Iyer } 199658628e7SLina Iyer 200658628e7SLina Iyer return NULL; 201658628e7SLina Iyer } 202658628e7SLina Iyer 203658628e7SLina Iyer /** 204658628e7SLina Iyer * tcs_tx_done: TX Done interrupt handler 205658628e7SLina Iyer */ 206658628e7SLina Iyer static irqreturn_t tcs_tx_done(int irq, void *p) 207658628e7SLina Iyer { 208658628e7SLina Iyer struct rsc_drv *drv = p; 209c1038456SLina Iyer int i, j, err = 0; 210658628e7SLina Iyer unsigned long irq_status; 211658628e7SLina Iyer const struct tcs_request *req; 212658628e7SLina Iyer struct tcs_cmd *cmd; 213658628e7SLina Iyer 214658628e7SLina Iyer irq_status = read_tcs_reg(drv, RSC_DRV_IRQ_STATUS, 0, 0); 215658628e7SLina Iyer 216658628e7SLina Iyer for_each_set_bit(i, &irq_status, BITS_PER_LONG) { 217658628e7SLina Iyer req = get_req_from_tcs(drv, i); 218658628e7SLina Iyer if (!req) { 219658628e7SLina Iyer WARN_ON(1); 220658628e7SLina Iyer goto skip; 221658628e7SLina Iyer } 222658628e7SLina Iyer 223fc087fe5SLina Iyer err = 0; 224658628e7SLina Iyer for (j = 0; j < req->num_cmds; j++) { 225658628e7SLina Iyer u32 sts; 226658628e7SLina Iyer 227658628e7SLina Iyer cmd = &req->cmds[j]; 228658628e7SLina Iyer sts = read_tcs_reg(drv, RSC_DRV_CMD_STATUS, i, j); 229658628e7SLina Iyer if (!(sts & CMD_STATUS_ISSUED) || 230658628e7SLina Iyer ((req->wait_for_compl || cmd->wait) && 231658628e7SLina Iyer !(sts & CMD_STATUS_COMPL))) { 232658628e7SLina Iyer pr_err("Incomplete request: %s: addr=%#x data=%#x", 233658628e7SLina Iyer drv->name, cmd->addr, cmd->data); 234fc087fe5SLina Iyer err = -EIO; 235658628e7SLina Iyer } 236658628e7SLina Iyer } 237fc087fe5SLina Iyer 238fc087fe5SLina Iyer trace_rpmh_tx_done(drv, i, req, err); 239658628e7SLina Iyer skip: 240658628e7SLina Iyer /* Reclaim the TCS */ 241658628e7SLina Iyer write_tcs_reg(drv, RSC_DRV_CMD_ENABLE, i, 0); 242658628e7SLina Iyer write_tcs_reg(drv, RSC_DRV_IRQ_CLEAR, 0, BIT(i)); 243658628e7SLina Iyer spin_lock(&drv->lock); 244658628e7SLina Iyer clear_bit(i, drv->tcs_in_use); 245658628e7SLina Iyer spin_unlock(&drv->lock); 246c1038456SLina Iyer if (req) 247c1038456SLina Iyer rpmh_tx_done(req, err); 248658628e7SLina Iyer } 249658628e7SLina Iyer 250658628e7SLina Iyer return IRQ_HANDLED; 251658628e7SLina Iyer } 252658628e7SLina Iyer 253658628e7SLina Iyer static void __tcs_buffer_write(struct rsc_drv *drv, int tcs_id, int cmd_id, 254658628e7SLina Iyer const struct tcs_request *msg) 255658628e7SLina Iyer { 256658628e7SLina Iyer u32 msgid, cmd_msgid; 257658628e7SLina Iyer u32 cmd_enable = 0; 258658628e7SLina Iyer u32 cmd_complete; 259658628e7SLina Iyer struct tcs_cmd *cmd; 260658628e7SLina Iyer int i, j; 261658628e7SLina Iyer 262658628e7SLina Iyer cmd_msgid = CMD_MSGID_LEN; 263658628e7SLina Iyer cmd_msgid |= msg->wait_for_compl ? CMD_MSGID_RESP_REQ : 0; 264658628e7SLina Iyer cmd_msgid |= CMD_MSGID_WRITE; 265658628e7SLina Iyer 266658628e7SLina Iyer cmd_complete = read_tcs_reg(drv, RSC_DRV_CMD_WAIT_FOR_CMPL, tcs_id, 0); 267658628e7SLina Iyer 268658628e7SLina Iyer for (i = 0, j = cmd_id; i < msg->num_cmds; i++, j++) { 269658628e7SLina Iyer cmd = &msg->cmds[i]; 270658628e7SLina Iyer cmd_enable |= BIT(j); 271658628e7SLina Iyer cmd_complete |= cmd->wait << j; 272658628e7SLina Iyer msgid = cmd_msgid; 273658628e7SLina Iyer msgid |= cmd->wait ? CMD_MSGID_RESP_REQ : 0; 274fc087fe5SLina Iyer 275658628e7SLina Iyer write_tcs_cmd(drv, RSC_DRV_CMD_MSGID, tcs_id, j, msgid); 276658628e7SLina Iyer write_tcs_cmd(drv, RSC_DRV_CMD_ADDR, tcs_id, j, cmd->addr); 277658628e7SLina Iyer write_tcs_cmd(drv, RSC_DRV_CMD_DATA, tcs_id, j, cmd->data); 278fc087fe5SLina Iyer trace_rpmh_send_msg(drv, tcs_id, j, msgid, cmd); 279658628e7SLina Iyer } 280658628e7SLina Iyer 281658628e7SLina Iyer write_tcs_reg(drv, RSC_DRV_CMD_WAIT_FOR_CMPL, tcs_id, cmd_complete); 282658628e7SLina Iyer cmd_enable |= read_tcs_reg(drv, RSC_DRV_CMD_ENABLE, tcs_id, 0); 283658628e7SLina Iyer write_tcs_reg(drv, RSC_DRV_CMD_ENABLE, tcs_id, cmd_enable); 284658628e7SLina Iyer } 285658628e7SLina Iyer 286658628e7SLina Iyer static void __tcs_trigger(struct rsc_drv *drv, int tcs_id) 287658628e7SLina Iyer { 288658628e7SLina Iyer u32 enable; 289658628e7SLina Iyer 290658628e7SLina Iyer /* 291658628e7SLina Iyer * HW req: Clear the DRV_CONTROL and enable TCS again 292658628e7SLina Iyer * While clearing ensure that the AMC mode trigger is cleared 293658628e7SLina Iyer * and then the mode enable is cleared. 294658628e7SLina Iyer */ 295658628e7SLina Iyer enable = read_tcs_reg(drv, RSC_DRV_CONTROL, tcs_id, 0); 296658628e7SLina Iyer enable &= ~TCS_AMC_MODE_TRIGGER; 297658628e7SLina Iyer write_tcs_reg_sync(drv, RSC_DRV_CONTROL, tcs_id, enable); 298658628e7SLina Iyer enable &= ~TCS_AMC_MODE_ENABLE; 299658628e7SLina Iyer write_tcs_reg_sync(drv, RSC_DRV_CONTROL, tcs_id, enable); 300658628e7SLina Iyer 301658628e7SLina Iyer /* Enable the AMC mode on the TCS and then trigger the TCS */ 302658628e7SLina Iyer enable = TCS_AMC_MODE_ENABLE; 303658628e7SLina Iyer write_tcs_reg_sync(drv, RSC_DRV_CONTROL, tcs_id, enable); 304658628e7SLina Iyer enable |= TCS_AMC_MODE_TRIGGER; 305658628e7SLina Iyer write_tcs_reg_sync(drv, RSC_DRV_CONTROL, tcs_id, enable); 306658628e7SLina Iyer } 307658628e7SLina Iyer 308658628e7SLina Iyer static int check_for_req_inflight(struct rsc_drv *drv, struct tcs_group *tcs, 309658628e7SLina Iyer const struct tcs_request *msg) 310658628e7SLina Iyer { 311658628e7SLina Iyer unsigned long curr_enabled; 312658628e7SLina Iyer u32 addr; 313658628e7SLina Iyer int i, j, k; 314658628e7SLina Iyer int tcs_id = tcs->offset; 315658628e7SLina Iyer 316658628e7SLina Iyer for (i = 0; i < tcs->num_tcs; i++, tcs_id++) { 317658628e7SLina Iyer if (tcs_is_free(drv, tcs_id)) 318658628e7SLina Iyer continue; 319658628e7SLina Iyer 320658628e7SLina Iyer curr_enabled = read_tcs_reg(drv, RSC_DRV_CMD_ENABLE, tcs_id, 0); 321658628e7SLina Iyer 322658628e7SLina Iyer for_each_set_bit(j, &curr_enabled, MAX_CMDS_PER_TCS) { 323658628e7SLina Iyer addr = read_tcs_reg(drv, RSC_DRV_CMD_ADDR, tcs_id, j); 324658628e7SLina Iyer for (k = 0; k < msg->num_cmds; k++) { 325658628e7SLina Iyer if (addr == msg->cmds[k].addr) 326658628e7SLina Iyer return -EBUSY; 327658628e7SLina Iyer } 328658628e7SLina Iyer } 329658628e7SLina Iyer } 330658628e7SLina Iyer 331658628e7SLina Iyer return 0; 332658628e7SLina Iyer } 333658628e7SLina Iyer 334658628e7SLina Iyer static int find_free_tcs(struct tcs_group *tcs) 335658628e7SLina Iyer { 336658628e7SLina Iyer int i; 337658628e7SLina Iyer 338658628e7SLina Iyer for (i = 0; i < tcs->num_tcs; i++) { 339658628e7SLina Iyer if (tcs_is_free(tcs->drv, tcs->offset + i)) 340658628e7SLina Iyer return tcs->offset + i; 341658628e7SLina Iyer } 342658628e7SLina Iyer 343658628e7SLina Iyer return -EBUSY; 344658628e7SLina Iyer } 345658628e7SLina Iyer 346658628e7SLina Iyer static int tcs_write(struct rsc_drv *drv, const struct tcs_request *msg) 347658628e7SLina Iyer { 348658628e7SLina Iyer struct tcs_group *tcs; 349658628e7SLina Iyer int tcs_id; 350658628e7SLina Iyer unsigned long flags; 351658628e7SLina Iyer int ret; 352658628e7SLina Iyer 353658628e7SLina Iyer tcs = get_tcs_for_msg(drv, msg); 354658628e7SLina Iyer if (IS_ERR(tcs)) 355658628e7SLina Iyer return PTR_ERR(tcs); 356658628e7SLina Iyer 357658628e7SLina Iyer spin_lock_irqsave(&tcs->lock, flags); 358658628e7SLina Iyer spin_lock(&drv->lock); 359658628e7SLina Iyer /* 360658628e7SLina Iyer * The h/w does not like if we send a request to the same address, 361658628e7SLina Iyer * when one is already in-flight or being processed. 362658628e7SLina Iyer */ 363658628e7SLina Iyer ret = check_for_req_inflight(drv, tcs, msg); 364658628e7SLina Iyer if (ret) { 365658628e7SLina Iyer spin_unlock(&drv->lock); 366658628e7SLina Iyer goto done_write; 367658628e7SLina Iyer } 368658628e7SLina Iyer 369658628e7SLina Iyer tcs_id = find_free_tcs(tcs); 370658628e7SLina Iyer if (tcs_id < 0) { 371658628e7SLina Iyer ret = tcs_id; 372658628e7SLina Iyer spin_unlock(&drv->lock); 373658628e7SLina Iyer goto done_write; 374658628e7SLina Iyer } 375658628e7SLina Iyer 376658628e7SLina Iyer tcs->req[tcs_id - tcs->offset] = msg; 377658628e7SLina Iyer set_bit(tcs_id, drv->tcs_in_use); 378658628e7SLina Iyer spin_unlock(&drv->lock); 379658628e7SLina Iyer 380658628e7SLina Iyer __tcs_buffer_write(drv, tcs_id, 0, msg); 381658628e7SLina Iyer __tcs_trigger(drv, tcs_id); 382658628e7SLina Iyer 383658628e7SLina Iyer done_write: 384658628e7SLina Iyer spin_unlock_irqrestore(&tcs->lock, flags); 385658628e7SLina Iyer return ret; 386658628e7SLina Iyer } 387658628e7SLina Iyer 388658628e7SLina Iyer /** 389658628e7SLina Iyer * rpmh_rsc_send_data: Validate the incoming message and write to the 390658628e7SLina Iyer * appropriate TCS block. 391658628e7SLina Iyer * 392658628e7SLina Iyer * @drv: the controller 393658628e7SLina Iyer * @msg: the data to be sent 394658628e7SLina Iyer * 395658628e7SLina Iyer * Return: 0 on success, -EINVAL on error. 396658628e7SLina Iyer * Note: This call blocks until a valid data is written to the TCS. 397658628e7SLina Iyer */ 398658628e7SLina Iyer int rpmh_rsc_send_data(struct rsc_drv *drv, const struct tcs_request *msg) 399658628e7SLina Iyer { 400658628e7SLina Iyer int ret; 401658628e7SLina Iyer 402658628e7SLina Iyer if (!msg || !msg->cmds || !msg->num_cmds || 403658628e7SLina Iyer msg->num_cmds > MAX_RPMH_PAYLOAD) { 404658628e7SLina Iyer WARN_ON(1); 405658628e7SLina Iyer return -EINVAL; 406658628e7SLina Iyer } 407658628e7SLina Iyer 408658628e7SLina Iyer do { 409658628e7SLina Iyer ret = tcs_write(drv, msg); 410658628e7SLina Iyer if (ret == -EBUSY) { 411658628e7SLina Iyer pr_info_ratelimited("TCS Busy, retrying RPMH message send: addr=%#x\n", 412658628e7SLina Iyer msg->cmds[0].addr); 413658628e7SLina Iyer udelay(10); 414658628e7SLina Iyer } 415658628e7SLina Iyer } while (ret == -EBUSY); 416658628e7SLina Iyer 417658628e7SLina Iyer return ret; 418658628e7SLina Iyer } 419658628e7SLina Iyer 420fa460e45SLina Iyer static int find_match(const struct tcs_group *tcs, const struct tcs_cmd *cmd, 421fa460e45SLina Iyer int len) 422fa460e45SLina Iyer { 423fa460e45SLina Iyer int i, j; 424fa460e45SLina Iyer 425fa460e45SLina Iyer /* Check for already cached commands */ 426fa460e45SLina Iyer for_each_set_bit(i, tcs->slots, MAX_TCS_SLOTS) { 427fa460e45SLina Iyer if (tcs->cmd_cache[i] != cmd[0].addr) 428fa460e45SLina Iyer continue; 429fa460e45SLina Iyer if (i + len >= tcs->num_tcs * tcs->ncpt) 430fa460e45SLina Iyer goto seq_err; 431fa460e45SLina Iyer for (j = 0; j < len; j++) { 432fa460e45SLina Iyer if (tcs->cmd_cache[i + j] != cmd[j].addr) 433fa460e45SLina Iyer goto seq_err; 434fa460e45SLina Iyer } 435fa460e45SLina Iyer return i; 436fa460e45SLina Iyer } 437fa460e45SLina Iyer 438fa460e45SLina Iyer return -ENODATA; 439fa460e45SLina Iyer 440fa460e45SLina Iyer seq_err: 441fa460e45SLina Iyer WARN(1, "Message does not match previous sequence.\n"); 442fa460e45SLina Iyer return -EINVAL; 443fa460e45SLina Iyer } 444fa460e45SLina Iyer 445fa460e45SLina Iyer static int find_slots(struct tcs_group *tcs, const struct tcs_request *msg, 446fa460e45SLina Iyer int *tcs_id, int *cmd_id) 447fa460e45SLina Iyer { 448fa460e45SLina Iyer int slot, offset; 449fa460e45SLina Iyer int i = 0; 450fa460e45SLina Iyer 451fa460e45SLina Iyer /* Find if we already have the msg in our TCS */ 452fa460e45SLina Iyer slot = find_match(tcs, msg->cmds, msg->num_cmds); 453fa460e45SLina Iyer if (slot >= 0) 454fa460e45SLina Iyer goto copy_data; 455fa460e45SLina Iyer 456fa460e45SLina Iyer /* Do over, until we can fit the full payload in a TCS */ 457fa460e45SLina Iyer do { 458fa460e45SLina Iyer slot = bitmap_find_next_zero_area(tcs->slots, MAX_TCS_SLOTS, 459fa460e45SLina Iyer i, msg->num_cmds, 0); 460fa460e45SLina Iyer if (slot == tcs->num_tcs * tcs->ncpt) 461fa460e45SLina Iyer return -ENOMEM; 462fa460e45SLina Iyer i += tcs->ncpt; 463fa460e45SLina Iyer } while (slot + msg->num_cmds - 1 >= i); 464fa460e45SLina Iyer 465fa460e45SLina Iyer copy_data: 466fa460e45SLina Iyer bitmap_set(tcs->slots, slot, msg->num_cmds); 467fa460e45SLina Iyer /* Copy the addresses of the resources over to the slots */ 468fa460e45SLina Iyer for (i = 0; i < msg->num_cmds; i++) 469fa460e45SLina Iyer tcs->cmd_cache[slot + i] = msg->cmds[i].addr; 470fa460e45SLina Iyer 471fa460e45SLina Iyer offset = slot / tcs->ncpt; 472fa460e45SLina Iyer *tcs_id = offset + tcs->offset; 473fa460e45SLina Iyer *cmd_id = slot % tcs->ncpt; 474fa460e45SLina Iyer 475fa460e45SLina Iyer return 0; 476fa460e45SLina Iyer } 477fa460e45SLina Iyer 478fa460e45SLina Iyer static int tcs_ctrl_write(struct rsc_drv *drv, const struct tcs_request *msg) 479fa460e45SLina Iyer { 480fa460e45SLina Iyer struct tcs_group *tcs; 481fa460e45SLina Iyer int tcs_id = 0, cmd_id = 0; 482fa460e45SLina Iyer unsigned long flags; 483fa460e45SLina Iyer int ret; 484fa460e45SLina Iyer 485fa460e45SLina Iyer tcs = get_tcs_for_msg(drv, msg); 486fa460e45SLina Iyer if (IS_ERR(tcs)) 487fa460e45SLina Iyer return PTR_ERR(tcs); 488fa460e45SLina Iyer 489fa460e45SLina Iyer spin_lock_irqsave(&tcs->lock, flags); 490fa460e45SLina Iyer /* find the TCS id and the command in the TCS to write to */ 491fa460e45SLina Iyer ret = find_slots(tcs, msg, &tcs_id, &cmd_id); 492fa460e45SLina Iyer if (!ret) 493fa460e45SLina Iyer __tcs_buffer_write(drv, tcs_id, cmd_id, msg); 494fa460e45SLina Iyer spin_unlock_irqrestore(&tcs->lock, flags); 495fa460e45SLina Iyer 496fa460e45SLina Iyer return ret; 497fa460e45SLina Iyer } 498fa460e45SLina Iyer 499fa460e45SLina Iyer /** 500fa460e45SLina Iyer * rpmh_rsc_write_ctrl_data: Write request to the controller 501fa460e45SLina Iyer * 502fa460e45SLina Iyer * @drv: the controller 503fa460e45SLina Iyer * @msg: the data to be written to the controller 504fa460e45SLina Iyer * 505fa460e45SLina Iyer * There is no response returned for writing the request to the controller. 506fa460e45SLina Iyer */ 507fa460e45SLina Iyer int rpmh_rsc_write_ctrl_data(struct rsc_drv *drv, const struct tcs_request *msg) 508fa460e45SLina Iyer { 509fa460e45SLina Iyer if (!msg || !msg->cmds || !msg->num_cmds || 510fa460e45SLina Iyer msg->num_cmds > MAX_RPMH_PAYLOAD) { 511fa460e45SLina Iyer pr_err("Payload error\n"); 512fa460e45SLina Iyer return -EINVAL; 513fa460e45SLina Iyer } 514fa460e45SLina Iyer 515fa460e45SLina Iyer /* Data sent to this API will not be sent immediately */ 516fa460e45SLina Iyer if (msg->state == RPMH_ACTIVE_ONLY_STATE) 517fa460e45SLina Iyer return -EINVAL; 518fa460e45SLina Iyer 519fa460e45SLina Iyer return tcs_ctrl_write(drv, msg); 520fa460e45SLina Iyer } 521fa460e45SLina Iyer 522658628e7SLina Iyer static int rpmh_probe_tcs_config(struct platform_device *pdev, 523658628e7SLina Iyer struct rsc_drv *drv) 524658628e7SLina Iyer { 525658628e7SLina Iyer struct tcs_type_config { 526658628e7SLina Iyer u32 type; 527658628e7SLina Iyer u32 n; 528658628e7SLina Iyer } tcs_cfg[TCS_TYPE_NR] = { { 0 } }; 529658628e7SLina Iyer struct device_node *dn = pdev->dev.of_node; 530658628e7SLina Iyer u32 config, max_tcs, ncpt, offset; 531658628e7SLina Iyer int i, ret, n, st = 0; 532658628e7SLina Iyer struct tcs_group *tcs; 533658628e7SLina Iyer struct resource *res; 534658628e7SLina Iyer void __iomem *base; 535658628e7SLina Iyer char drv_id[10] = {0}; 536658628e7SLina Iyer 537658628e7SLina Iyer snprintf(drv_id, ARRAY_SIZE(drv_id), "drv-%d", drv->id); 538658628e7SLina Iyer res = platform_get_resource_byname(pdev, IORESOURCE_MEM, drv_id); 539658628e7SLina Iyer base = devm_ioremap_resource(&pdev->dev, res); 540658628e7SLina Iyer if (IS_ERR(base)) 541658628e7SLina Iyer return PTR_ERR(base); 542658628e7SLina Iyer 543658628e7SLina Iyer ret = of_property_read_u32(dn, "qcom,tcs-offset", &offset); 544658628e7SLina Iyer if (ret) 545658628e7SLina Iyer return ret; 546658628e7SLina Iyer drv->tcs_base = base + offset; 547658628e7SLina Iyer 548658628e7SLina Iyer config = readl_relaxed(base + DRV_PRNT_CHLD_CONFIG); 549658628e7SLina Iyer 550658628e7SLina Iyer max_tcs = config; 551658628e7SLina Iyer max_tcs &= DRV_NUM_TCS_MASK << (DRV_NUM_TCS_SHIFT * drv->id); 552658628e7SLina Iyer max_tcs = max_tcs >> (DRV_NUM_TCS_SHIFT * drv->id); 553658628e7SLina Iyer 554658628e7SLina Iyer ncpt = config & (DRV_NCPT_MASK << DRV_NCPT_SHIFT); 555658628e7SLina Iyer ncpt = ncpt >> DRV_NCPT_SHIFT; 556658628e7SLina Iyer 557658628e7SLina Iyer n = of_property_count_u32_elems(dn, "qcom,tcs-config"); 558658628e7SLina Iyer if (n != 2 * TCS_TYPE_NR) 559658628e7SLina Iyer return -EINVAL; 560658628e7SLina Iyer 561658628e7SLina Iyer for (i = 0; i < TCS_TYPE_NR; i++) { 562658628e7SLina Iyer ret = of_property_read_u32_index(dn, "qcom,tcs-config", 563658628e7SLina Iyer i * 2, &tcs_cfg[i].type); 564658628e7SLina Iyer if (ret) 565658628e7SLina Iyer return ret; 566658628e7SLina Iyer if (tcs_cfg[i].type >= TCS_TYPE_NR) 567658628e7SLina Iyer return -EINVAL; 568658628e7SLina Iyer 569658628e7SLina Iyer ret = of_property_read_u32_index(dn, "qcom,tcs-config", 570658628e7SLina Iyer i * 2 + 1, &tcs_cfg[i].n); 571658628e7SLina Iyer if (ret) 572658628e7SLina Iyer return ret; 573658628e7SLina Iyer if (tcs_cfg[i].n > MAX_TCS_PER_TYPE) 574658628e7SLina Iyer return -EINVAL; 575658628e7SLina Iyer } 576658628e7SLina Iyer 577658628e7SLina Iyer for (i = 0; i < TCS_TYPE_NR; i++) { 578658628e7SLina Iyer tcs = &drv->tcs[tcs_cfg[i].type]; 579658628e7SLina Iyer if (tcs->drv) 580658628e7SLina Iyer return -EINVAL; 581658628e7SLina Iyer tcs->drv = drv; 582658628e7SLina Iyer tcs->type = tcs_cfg[i].type; 583658628e7SLina Iyer tcs->num_tcs = tcs_cfg[i].n; 584658628e7SLina Iyer tcs->ncpt = ncpt; 585658628e7SLina Iyer spin_lock_init(&tcs->lock); 586658628e7SLina Iyer 587658628e7SLina Iyer if (!tcs->num_tcs || tcs->type == CONTROL_TCS) 588658628e7SLina Iyer continue; 589658628e7SLina Iyer 590658628e7SLina Iyer if (st + tcs->num_tcs > max_tcs || 591658628e7SLina Iyer st + tcs->num_tcs >= BITS_PER_BYTE * sizeof(tcs->mask)) 592658628e7SLina Iyer return -EINVAL; 593658628e7SLina Iyer 594658628e7SLina Iyer tcs->mask = ((1 << tcs->num_tcs) - 1) << st; 595658628e7SLina Iyer tcs->offset = st; 596658628e7SLina Iyer st += tcs->num_tcs; 597fa460e45SLina Iyer 598fa460e45SLina Iyer /* 599fa460e45SLina Iyer * Allocate memory to cache sleep and wake requests to 600fa460e45SLina Iyer * avoid reading TCS register memory. 601fa460e45SLina Iyer */ 602fa460e45SLina Iyer if (tcs->type == ACTIVE_TCS) 603fa460e45SLina Iyer continue; 604fa460e45SLina Iyer 605fa460e45SLina Iyer tcs->cmd_cache = devm_kcalloc(&pdev->dev, 606fa460e45SLina Iyer tcs->num_tcs * ncpt, sizeof(u32), 607fa460e45SLina Iyer GFP_KERNEL); 608fa460e45SLina Iyer if (!tcs->cmd_cache) 609fa460e45SLina Iyer return -ENOMEM; 610658628e7SLina Iyer } 611658628e7SLina Iyer 612658628e7SLina Iyer drv->num_tcs = st; 613658628e7SLina Iyer 614658628e7SLina Iyer return 0; 615658628e7SLina Iyer } 616658628e7SLina Iyer 617658628e7SLina Iyer static int rpmh_rsc_probe(struct platform_device *pdev) 618658628e7SLina Iyer { 619658628e7SLina Iyer struct device_node *dn = pdev->dev.of_node; 620658628e7SLina Iyer struct rsc_drv *drv; 621658628e7SLina Iyer int ret, irq; 622658628e7SLina Iyer 623fdd102b5SDouglas Anderson /* 624fdd102b5SDouglas Anderson * Even though RPMh doesn't directly use cmd-db, all of its children 625fdd102b5SDouglas Anderson * do. To avoid adding this check to our children we'll do it now. 626fdd102b5SDouglas Anderson */ 627fdd102b5SDouglas Anderson ret = cmd_db_ready(); 628fdd102b5SDouglas Anderson if (ret) { 629fdd102b5SDouglas Anderson if (ret != -EPROBE_DEFER) 630fdd102b5SDouglas Anderson dev_err(&pdev->dev, "Command DB not available (%d)\n", 631fdd102b5SDouglas Anderson ret); 632fdd102b5SDouglas Anderson return ret; 633fdd102b5SDouglas Anderson } 634fdd102b5SDouglas Anderson 635658628e7SLina Iyer drv = devm_kzalloc(&pdev->dev, sizeof(*drv), GFP_KERNEL); 636658628e7SLina Iyer if (!drv) 637658628e7SLina Iyer return -ENOMEM; 638658628e7SLina Iyer 639658628e7SLina Iyer ret = of_property_read_u32(dn, "qcom,drv-id", &drv->id); 640658628e7SLina Iyer if (ret) 641658628e7SLina Iyer return ret; 642658628e7SLina Iyer 643658628e7SLina Iyer drv->name = of_get_property(dn, "label", NULL); 644658628e7SLina Iyer if (!drv->name) 645658628e7SLina Iyer drv->name = dev_name(&pdev->dev); 646658628e7SLina Iyer 647658628e7SLina Iyer ret = rpmh_probe_tcs_config(pdev, drv); 648658628e7SLina Iyer if (ret) 649658628e7SLina Iyer return ret; 650658628e7SLina Iyer 651658628e7SLina Iyer spin_lock_init(&drv->lock); 652658628e7SLina Iyer bitmap_zero(drv->tcs_in_use, MAX_TCS_NR); 653658628e7SLina Iyer 654658628e7SLina Iyer irq = platform_get_irq(pdev, drv->id); 655658628e7SLina Iyer if (irq < 0) 656658628e7SLina Iyer return irq; 657658628e7SLina Iyer 658658628e7SLina Iyer ret = devm_request_irq(&pdev->dev, irq, tcs_tx_done, 659658628e7SLina Iyer IRQF_TRIGGER_HIGH | IRQF_NO_SUSPEND, 660658628e7SLina Iyer drv->name, drv); 661658628e7SLina Iyer if (ret) 662658628e7SLina Iyer return ret; 663658628e7SLina Iyer 664658628e7SLina Iyer /* Enable the active TCS to send requests immediately */ 665658628e7SLina Iyer write_tcs_reg(drv, RSC_DRV_IRQ_ENABLE, 0, drv->tcs[ACTIVE_TCS].mask); 666658628e7SLina Iyer 667600513dfSLina Iyer spin_lock_init(&drv->client.cache_lock); 668600513dfSLina Iyer INIT_LIST_HEAD(&drv->client.cache); 669c8790cb6SLina Iyer INIT_LIST_HEAD(&drv->client.batch_cache); 670600513dfSLina Iyer 671c1038456SLina Iyer dev_set_drvdata(&pdev->dev, drv); 672c1038456SLina Iyer 673658628e7SLina Iyer return devm_of_platform_populate(&pdev->dev); 674658628e7SLina Iyer } 675658628e7SLina Iyer 676658628e7SLina Iyer static const struct of_device_id rpmh_drv_match[] = { 677658628e7SLina Iyer { .compatible = "qcom,rpmh-rsc", }, 678658628e7SLina Iyer { } 679658628e7SLina Iyer }; 680658628e7SLina Iyer 681658628e7SLina Iyer static struct platform_driver rpmh_driver = { 682658628e7SLina Iyer .probe = rpmh_rsc_probe, 683658628e7SLina Iyer .driver = { 684658628e7SLina Iyer .name = "rpmh", 685658628e7SLina Iyer .of_match_table = rpmh_drv_match, 686658628e7SLina Iyer }, 687658628e7SLina Iyer }; 688658628e7SLina Iyer 689658628e7SLina Iyer static int __init rpmh_driver_init(void) 690658628e7SLina Iyer { 691658628e7SLina Iyer return platform_driver_register(&rpmh_driver); 692658628e7SLina Iyer } 693658628e7SLina Iyer arch_initcall(rpmh_driver_init); 694