1658628e7SLina Iyer // SPDX-License-Identifier: GPL-2.0 2658628e7SLina Iyer /* 3658628e7SLina Iyer * Copyright (c) 2016-2018, The Linux Foundation. All rights reserved. 4658628e7SLina Iyer */ 5658628e7SLina Iyer 6658628e7SLina Iyer #define pr_fmt(fmt) "%s " fmt, KBUILD_MODNAME 7658628e7SLina Iyer 8658628e7SLina Iyer #include <linux/atomic.h> 9985427f9SMaulik Shah #include <linux/cpu_pm.h> 10658628e7SLina Iyer #include <linux/delay.h> 11658628e7SLina Iyer #include <linux/interrupt.h> 12658628e7SLina Iyer #include <linux/io.h> 13658628e7SLina Iyer #include <linux/kernel.h> 14658628e7SLina Iyer #include <linux/list.h> 15658628e7SLina Iyer #include <linux/of.h> 16658628e7SLina Iyer #include <linux/of_irq.h> 17658628e7SLina Iyer #include <linux/of_platform.h> 18658628e7SLina Iyer #include <linux/platform_device.h> 19658628e7SLina Iyer #include <linux/slab.h> 20658628e7SLina Iyer #include <linux/spinlock.h> 21658628e7SLina Iyer 22fdd102b5SDouglas Anderson #include <soc/qcom/cmd-db.h> 23658628e7SLina Iyer #include <soc/qcom/tcs.h> 24658628e7SLina Iyer #include <dt-bindings/soc/qcom,rpmh-rsc.h> 25658628e7SLina Iyer 26658628e7SLina Iyer #include "rpmh-internal.h" 27658628e7SLina Iyer 28fc087fe5SLina Iyer #define CREATE_TRACE_POINTS 29fc087fe5SLina Iyer #include "trace-rpmh.h" 30fc087fe5SLina Iyer 31658628e7SLina Iyer #define RSC_DRV_TCS_OFFSET 672 32658628e7SLina Iyer #define RSC_DRV_CMD_OFFSET 20 33658628e7SLina Iyer 34985427f9SMaulik Shah /* DRV HW Solver Configuration Information Register */ 35985427f9SMaulik Shah #define DRV_SOLVER_CONFIG 0x04 36985427f9SMaulik Shah #define DRV_HW_SOLVER_MASK 1 37985427f9SMaulik Shah #define DRV_HW_SOLVER_SHIFT 24 38985427f9SMaulik Shah 39985427f9SMaulik Shah /* DRV TCS Configuration Information Register */ 40658628e7SLina Iyer #define DRV_PRNT_CHLD_CONFIG 0x0C 41658628e7SLina Iyer #define DRV_NUM_TCS_MASK 0x3F 42658628e7SLina Iyer #define DRV_NUM_TCS_SHIFT 6 43658628e7SLina Iyer #define DRV_NCPT_MASK 0x1F 44658628e7SLina Iyer #define DRV_NCPT_SHIFT 27 45658628e7SLina Iyer 461f7dbeb5SDouglas Anderson /* Offsets for common TCS Registers, one bit per TCS */ 47658628e7SLina Iyer #define RSC_DRV_IRQ_ENABLE 0x00 48658628e7SLina Iyer #define RSC_DRV_IRQ_STATUS 0x04 491f7dbeb5SDouglas Anderson #define RSC_DRV_IRQ_CLEAR 0x08 /* w/o; write 1 to clear */ 501f7dbeb5SDouglas Anderson 511f7dbeb5SDouglas Anderson /* 521f7dbeb5SDouglas Anderson * Offsets for per TCS Registers. 531f7dbeb5SDouglas Anderson * 541f7dbeb5SDouglas Anderson * TCSes start at 0x10 from tcs_base and are stored one after another. 551f7dbeb5SDouglas Anderson * Multiply tcs_id by RSC_DRV_TCS_OFFSET to find a given TCS and add one 561f7dbeb5SDouglas Anderson * of the below to find a register. 571f7dbeb5SDouglas Anderson */ 581f7dbeb5SDouglas Anderson #define RSC_DRV_CMD_WAIT_FOR_CMPL 0x10 /* 1 bit per command */ 59658628e7SLina Iyer #define RSC_DRV_CONTROL 0x14 601f7dbeb5SDouglas Anderson #define RSC_DRV_STATUS 0x18 /* zero if tcs is busy */ 611f7dbeb5SDouglas Anderson #define RSC_DRV_CMD_ENABLE 0x1C /* 1 bit per command */ 621f7dbeb5SDouglas Anderson 631f7dbeb5SDouglas Anderson /* 641f7dbeb5SDouglas Anderson * Offsets for per command in a TCS. 651f7dbeb5SDouglas Anderson * 661f7dbeb5SDouglas Anderson * Commands (up to 16) start at 0x30 in a TCS; multiply command index 671f7dbeb5SDouglas Anderson * by RSC_DRV_CMD_OFFSET and add one of the below to find a register. 681f7dbeb5SDouglas Anderson */ 69658628e7SLina Iyer #define RSC_DRV_CMD_MSGID 0x30 70658628e7SLina Iyer #define RSC_DRV_CMD_ADDR 0x34 71658628e7SLina Iyer #define RSC_DRV_CMD_DATA 0x38 72658628e7SLina Iyer #define RSC_DRV_CMD_STATUS 0x3C 73658628e7SLina Iyer #define RSC_DRV_CMD_RESP_DATA 0x40 74658628e7SLina Iyer 75658628e7SLina Iyer #define TCS_AMC_MODE_ENABLE BIT(16) 76658628e7SLina Iyer #define TCS_AMC_MODE_TRIGGER BIT(24) 77658628e7SLina Iyer 78658628e7SLina Iyer /* TCS CMD register bit mask */ 79658628e7SLina Iyer #define CMD_MSGID_LEN 8 80658628e7SLina Iyer #define CMD_MSGID_RESP_REQ BIT(8) 81658628e7SLina Iyer #define CMD_MSGID_WRITE BIT(16) 82658628e7SLina Iyer #define CMD_STATUS_ISSUED BIT(8) 83658628e7SLina Iyer #define CMD_STATUS_COMPL BIT(16) 84658628e7SLina Iyer 851f7dbeb5SDouglas Anderson /* 861f7dbeb5SDouglas Anderson * Here's a high level overview of how all the registers in RPMH work 871f7dbeb5SDouglas Anderson * together: 881f7dbeb5SDouglas Anderson * 891f7dbeb5SDouglas Anderson * - The main rpmh-rsc address is the base of a register space that can 901f7dbeb5SDouglas Anderson * be used to find overall configuration of the hardware 911f7dbeb5SDouglas Anderson * (DRV_PRNT_CHLD_CONFIG). Also found within the rpmh-rsc register 921f7dbeb5SDouglas Anderson * space are all the TCS blocks. The offset of the TCS blocks is 931f7dbeb5SDouglas Anderson * specified in the device tree by "qcom,tcs-offset" and used to 941f7dbeb5SDouglas Anderson * compute tcs_base. 951f7dbeb5SDouglas Anderson * - TCS blocks come one after another. Type, count, and order are 961f7dbeb5SDouglas Anderson * specified by the device tree as "qcom,tcs-config". 971f7dbeb5SDouglas Anderson * - Each TCS block has some registers, then space for up to 16 commands. 981f7dbeb5SDouglas Anderson * Note that though address space is reserved for 16 commands, fewer 991f7dbeb5SDouglas Anderson * might be present. See ncpt (num cmds per TCS). 1001f7dbeb5SDouglas Anderson * 1011f7dbeb5SDouglas Anderson * Here's a picture: 1021f7dbeb5SDouglas Anderson * 1031f7dbeb5SDouglas Anderson * +---------------------------------------------------+ 1041f7dbeb5SDouglas Anderson * |RSC | 1051f7dbeb5SDouglas Anderson * | ctrl | 1061f7dbeb5SDouglas Anderson * | | 1071f7dbeb5SDouglas Anderson * | Drvs: | 1081f7dbeb5SDouglas Anderson * | +-----------------------------------------------+ | 1091f7dbeb5SDouglas Anderson * | |DRV0 | | 1101f7dbeb5SDouglas Anderson * | | ctrl/config | | 1111f7dbeb5SDouglas Anderson * | | IRQ | | 1121f7dbeb5SDouglas Anderson * | | | | 1131f7dbeb5SDouglas Anderson * | | TCSes: | | 1141f7dbeb5SDouglas Anderson * | | +------------------------------------------+ | | 1151f7dbeb5SDouglas Anderson * | | |TCS0 | | | | | | | | | | | | | | | 1161f7dbeb5SDouglas Anderson * | | | ctrl | 0| 1| 2| 3| 4| 5| .| .| .| .|14|15| | | 1171f7dbeb5SDouglas Anderson * | | | | | | | | | | | | | | | | | | 1181f7dbeb5SDouglas Anderson * | | +------------------------------------------+ | | 1191f7dbeb5SDouglas Anderson * | | +------------------------------------------+ | | 1201f7dbeb5SDouglas Anderson * | | |TCS1 | | | | | | | | | | | | | | | 1211f7dbeb5SDouglas Anderson * | | | ctrl | 0| 1| 2| 3| 4| 5| .| .| .| .|14|15| | | 1221f7dbeb5SDouglas Anderson * | | | | | | | | | | | | | | | | | | 1231f7dbeb5SDouglas Anderson * | | +------------------------------------------+ | | 1241f7dbeb5SDouglas Anderson * | | +------------------------------------------+ | | 1251f7dbeb5SDouglas Anderson * | | |TCS2 | | | | | | | | | | | | | | | 1261f7dbeb5SDouglas Anderson * | | | ctrl | 0| 1| 2| 3| 4| 5| .| .| .| .|14|15| | | 1271f7dbeb5SDouglas Anderson * | | | | | | | | | | | | | | | | | | 1281f7dbeb5SDouglas Anderson * | | +------------------------------------------+ | | 1291f7dbeb5SDouglas Anderson * | | ...... | | 1301f7dbeb5SDouglas Anderson * | +-----------------------------------------------+ | 1311f7dbeb5SDouglas Anderson * | +-----------------------------------------------+ | 1321f7dbeb5SDouglas Anderson * | |DRV1 | | 1331f7dbeb5SDouglas Anderson * | | (same as DRV0) | | 1341f7dbeb5SDouglas Anderson * | +-----------------------------------------------+ | 1351f7dbeb5SDouglas Anderson * | ...... | 1361f7dbeb5SDouglas Anderson * +---------------------------------------------------+ 1371f7dbeb5SDouglas Anderson */ 1381f7dbeb5SDouglas Anderson 1393b5e3d50SDouglas Anderson static u32 read_tcs_cmd(struct rsc_drv *drv, int reg, int tcs_id, int cmd_id) 140658628e7SLina Iyer { 1413b5e3d50SDouglas Anderson return readl_relaxed(drv->tcs_base + RSC_DRV_TCS_OFFSET * tcs_id + reg + 142658628e7SLina Iyer RSC_DRV_CMD_OFFSET * cmd_id); 143658628e7SLina Iyer } 144658628e7SLina Iyer 1453b5e3d50SDouglas Anderson static u32 read_tcs_reg(struct rsc_drv *drv, int reg, int tcs_id) 1463b5e3d50SDouglas Anderson { 1473b5e3d50SDouglas Anderson return readl_relaxed(drv->tcs_base + RSC_DRV_TCS_OFFSET * tcs_id + reg); 1483b5e3d50SDouglas Anderson } 1493b5e3d50SDouglas Anderson 150658628e7SLina Iyer static void write_tcs_cmd(struct rsc_drv *drv, int reg, int tcs_id, int cmd_id, 151658628e7SLina Iyer u32 data) 152658628e7SLina Iyer { 1533b5e3d50SDouglas Anderson writel_relaxed(data, drv->tcs_base + RSC_DRV_TCS_OFFSET * tcs_id + reg + 154658628e7SLina Iyer RSC_DRV_CMD_OFFSET * cmd_id); 155658628e7SLina Iyer } 156658628e7SLina Iyer 157658628e7SLina Iyer static void write_tcs_reg(struct rsc_drv *drv, int reg, int tcs_id, u32 data) 158658628e7SLina Iyer { 1593b5e3d50SDouglas Anderson writel_relaxed(data, drv->tcs_base + RSC_DRV_TCS_OFFSET * tcs_id + reg); 160658628e7SLina Iyer } 161658628e7SLina Iyer 162658628e7SLina Iyer static void write_tcs_reg_sync(struct rsc_drv *drv, int reg, int tcs_id, 163658628e7SLina Iyer u32 data) 164658628e7SLina Iyer { 1653b5e3d50SDouglas Anderson writel(data, drv->tcs_base + RSC_DRV_TCS_OFFSET * tcs_id + reg); 166658628e7SLina Iyer for (;;) { 167658628e7SLina Iyer if (data == readl(drv->tcs_base + reg + 168658628e7SLina Iyer RSC_DRV_TCS_OFFSET * tcs_id)) 169658628e7SLina Iyer break; 170658628e7SLina Iyer udelay(1); 171658628e7SLina Iyer } 172658628e7SLina Iyer } 173658628e7SLina Iyer 174e40b0c16SDouglas Anderson /** 175e40b0c16SDouglas Anderson * tcs_is_free() - Return if a TCS is totally free. 176e40b0c16SDouglas Anderson * @drv: The RSC controller. 177e40b0c16SDouglas Anderson * @tcs_id: The global ID of this TCS. 178e40b0c16SDouglas Anderson * 179e40b0c16SDouglas Anderson * Returns true if nobody has claimed this TCS (by setting tcs_in_use). 180e40b0c16SDouglas Anderson * 181e40b0c16SDouglas Anderson * Context: Must be called with the drv->lock held or the tcs_lock for the TCS 182e40b0c16SDouglas Anderson * being tested. If only the tcs_lock is held then it is possible that 183e40b0c16SDouglas Anderson * this function will return that a tcs is still busy when it has been 184e40b0c16SDouglas Anderson * recently been freed but it will never return free when a TCS is 185e40b0c16SDouglas Anderson * actually in use. 186e40b0c16SDouglas Anderson * 187e40b0c16SDouglas Anderson * Return: true if the given TCS is free. 188e40b0c16SDouglas Anderson */ 189658628e7SLina Iyer static bool tcs_is_free(struct rsc_drv *drv, int tcs_id) 190658628e7SLina Iyer { 191ff304ea3SDouglas Anderson return !test_bit(tcs_id, drv->tcs_in_use); 192658628e7SLina Iyer } 193658628e7SLina Iyer 194e40b0c16SDouglas Anderson /** 195e40b0c16SDouglas Anderson * tcs_invalidate() - Invalidate all TCSes of the given type (sleep or wake). 196e40b0c16SDouglas Anderson * @drv: The RSC controller. 197e40b0c16SDouglas Anderson * @type: SLEEP_TCS or WAKE_TCS 198e40b0c16SDouglas Anderson * 199e40b0c16SDouglas Anderson * This will clear the "slots" variable of the given tcs_group and also 200e40b0c16SDouglas Anderson * tell the hardware to forget about all entries. 201e40b0c16SDouglas Anderson * 202e40b0c16SDouglas Anderson * Return: 0 if no problem, or -EAGAIN if the caller should try again in a 203e40b0c16SDouglas Anderson * bit. Caller should make sure to enable interrupts between tries. 204e40b0c16SDouglas Anderson */ 2059a3afcfbSLina Iyer static int tcs_invalidate(struct rsc_drv *drv, int type) 2069a3afcfbSLina Iyer { 2079a3afcfbSLina Iyer int m; 20853d49fe1SDouglas Anderson struct tcs_group *tcs = &drv->tcs[type]; 2099a3afcfbSLina Iyer 2109a3afcfbSLina Iyer spin_lock(&tcs->lock); 2119a3afcfbSLina Iyer if (bitmap_empty(tcs->slots, MAX_TCS_SLOTS)) { 2129a3afcfbSLina Iyer spin_unlock(&tcs->lock); 2139a3afcfbSLina Iyer return 0; 2149a3afcfbSLina Iyer } 2159a3afcfbSLina Iyer 2169a3afcfbSLina Iyer for (m = tcs->offset; m < tcs->offset + tcs->num_tcs; m++) { 2179a3afcfbSLina Iyer if (!tcs_is_free(drv, m)) { 2189a3afcfbSLina Iyer spin_unlock(&tcs->lock); 2199a3afcfbSLina Iyer return -EAGAIN; 2209a3afcfbSLina Iyer } 2219a3afcfbSLina Iyer write_tcs_reg_sync(drv, RSC_DRV_CMD_ENABLE, m, 0); 22209e97b6cSLina Iyer write_tcs_reg_sync(drv, RSC_DRV_CMD_WAIT_FOR_CMPL, m, 0); 2239a3afcfbSLina Iyer } 2249a3afcfbSLina Iyer bitmap_zero(tcs->slots, MAX_TCS_SLOTS); 2259a3afcfbSLina Iyer spin_unlock(&tcs->lock); 2269a3afcfbSLina Iyer 2279a3afcfbSLina Iyer return 0; 2289a3afcfbSLina Iyer } 2299a3afcfbSLina Iyer 2309a3afcfbSLina Iyer /** 231e40b0c16SDouglas Anderson * rpmh_rsc_invalidate() - Invalidate sleep and wake TCSes. 232e40b0c16SDouglas Anderson * @drv: The RSC controller. 2339a3afcfbSLina Iyer * 234e40b0c16SDouglas Anderson * Return: 0 if no problem, or -EAGAIN if the caller should try again in a 235e40b0c16SDouglas Anderson * bit. Caller should make sure to enable interrupts between tries. 2369a3afcfbSLina Iyer */ 2379a3afcfbSLina Iyer int rpmh_rsc_invalidate(struct rsc_drv *drv) 2389a3afcfbSLina Iyer { 2399a3afcfbSLina Iyer int ret; 2409a3afcfbSLina Iyer 2419a3afcfbSLina Iyer ret = tcs_invalidate(drv, SLEEP_TCS); 2429a3afcfbSLina Iyer if (!ret) 2439a3afcfbSLina Iyer ret = tcs_invalidate(drv, WAKE_TCS); 2449a3afcfbSLina Iyer 2459a3afcfbSLina Iyer return ret; 2469a3afcfbSLina Iyer } 2479a3afcfbSLina Iyer 248e40b0c16SDouglas Anderson /** 249e40b0c16SDouglas Anderson * get_tcs_for_msg() - Get the tcs_group used to send the given message. 250e40b0c16SDouglas Anderson * @drv: The RSC controller. 251e40b0c16SDouglas Anderson * @msg: The message we want to send. 252e40b0c16SDouglas Anderson * 253e40b0c16SDouglas Anderson * This is normally pretty straightforward except if we are trying to send 254e40b0c16SDouglas Anderson * an ACTIVE_ONLY message but don't have any active_only TCSes. 255e40b0c16SDouglas Anderson * 256e40b0c16SDouglas Anderson * Called without drv->lock held and with no tcs_lock locks held. 257e40b0c16SDouglas Anderson * 258e40b0c16SDouglas Anderson * Return: A pointer to a tcs_group or an ERR_PTR. 259e40b0c16SDouglas Anderson */ 260658628e7SLina Iyer static struct tcs_group *get_tcs_for_msg(struct rsc_drv *drv, 261658628e7SLina Iyer const struct tcs_request *msg) 262658628e7SLina Iyer { 26338427e5aSMaulik Shah int type; 2642de4b8d3SLina Iyer struct tcs_group *tcs; 265658628e7SLina Iyer 266658628e7SLina Iyer switch (msg->state) { 267658628e7SLina Iyer case RPMH_ACTIVE_ONLY_STATE: 268658628e7SLina Iyer type = ACTIVE_TCS; 269658628e7SLina Iyer break; 270fa460e45SLina Iyer case RPMH_WAKE_ONLY_STATE: 271fa460e45SLina Iyer type = WAKE_TCS; 272fa460e45SLina Iyer break; 273fa460e45SLina Iyer case RPMH_SLEEP_STATE: 274fa460e45SLina Iyer type = SLEEP_TCS; 275fa460e45SLina Iyer break; 276658628e7SLina Iyer default: 277658628e7SLina Iyer return ERR_PTR(-EINVAL); 278658628e7SLina Iyer } 279658628e7SLina Iyer 2802de4b8d3SLina Iyer /* 2812de4b8d3SLina Iyer * If we are making an active request on a RSC that does not have a 2822de4b8d3SLina Iyer * dedicated TCS for active state use, then re-purpose a wake TCS to 283e40b0c16SDouglas Anderson * send active votes. This is safe because we ensure any active-only 284e40b0c16SDouglas Anderson * transfers have finished before we use it (maybe by running from 285e40b0c16SDouglas Anderson * the last CPU in PM code). 2862de4b8d3SLina Iyer */ 28753d49fe1SDouglas Anderson tcs = &drv->tcs[type]; 28838427e5aSMaulik Shah if (msg->state == RPMH_ACTIVE_ONLY_STATE && !tcs->num_tcs) 28953d49fe1SDouglas Anderson tcs = &drv->tcs[WAKE_TCS]; 2902de4b8d3SLina Iyer 2912de4b8d3SLina Iyer return tcs; 292658628e7SLina Iyer } 293658628e7SLina Iyer 294e40b0c16SDouglas Anderson /** 295e40b0c16SDouglas Anderson * get_req_from_tcs() - Get a stashed request that was xfering on the given TCS. 296e40b0c16SDouglas Anderson * @drv: The RSC controller. 297e40b0c16SDouglas Anderson * @tcs_id: The global ID of this TCS. 298e40b0c16SDouglas Anderson * 299e40b0c16SDouglas Anderson * For ACTIVE_ONLY transfers we want to call back into the client when the 300e40b0c16SDouglas Anderson * transfer finishes. To do this we need the "request" that the client 301e40b0c16SDouglas Anderson * originally provided us. This function grabs the request that we stashed 302e40b0c16SDouglas Anderson * when we started the transfer. 303e40b0c16SDouglas Anderson * 304e40b0c16SDouglas Anderson * This only makes sense for ACTIVE_ONLY transfers since those are the only 305e40b0c16SDouglas Anderson * ones we track sending (the only ones we enable interrupts for and the only 306e40b0c16SDouglas Anderson * ones we call back to the client for). 307e40b0c16SDouglas Anderson * 308e40b0c16SDouglas Anderson * Return: The stashed request. 309e40b0c16SDouglas Anderson */ 310658628e7SLina Iyer static const struct tcs_request *get_req_from_tcs(struct rsc_drv *drv, 311658628e7SLina Iyer int tcs_id) 312658628e7SLina Iyer { 313658628e7SLina Iyer struct tcs_group *tcs; 314658628e7SLina Iyer int i; 315658628e7SLina Iyer 316efa1c257SRaju P.L.S.S.S.N for (i = 0; i < TCS_TYPE_NR; i++) { 317658628e7SLina Iyer tcs = &drv->tcs[i]; 318658628e7SLina Iyer if (tcs->mask & BIT(tcs_id)) 319658628e7SLina Iyer return tcs->req[tcs_id - tcs->offset]; 320658628e7SLina Iyer } 321658628e7SLina Iyer 322658628e7SLina Iyer return NULL; 323658628e7SLina Iyer } 324658628e7SLina Iyer 325e40b0c16SDouglas Anderson /** 326e40b0c16SDouglas Anderson * __tcs_set_trigger() - Start xfer on a TCS or unset trigger on a borrowed TCS 327e40b0c16SDouglas Anderson * @drv: The controller. 328e40b0c16SDouglas Anderson * @tcs_id: The global ID of this TCS. 329e40b0c16SDouglas Anderson * @trigger: If true then untrigger/retrigger. If false then just untrigger. 330e40b0c16SDouglas Anderson * 331e40b0c16SDouglas Anderson * In the normal case we only ever call with "trigger=true" to start a 332e40b0c16SDouglas Anderson * transfer. That will un-trigger/disable the TCS from the last transfer 333e40b0c16SDouglas Anderson * then trigger/enable for this transfer. 334e40b0c16SDouglas Anderson * 335e40b0c16SDouglas Anderson * If we borrowed a wake TCS for an active-only transfer we'll also call 336e40b0c16SDouglas Anderson * this function with "trigger=false" to just do the un-trigger/disable 337e40b0c16SDouglas Anderson * before using the TCS for wake purposes again. 338e40b0c16SDouglas Anderson * 339e40b0c16SDouglas Anderson * Note that the AP is only in charge of triggering active-only transfers. 340e40b0c16SDouglas Anderson * The AP never triggers sleep/wake values using this function. 341e40b0c16SDouglas Anderson */ 34215b3bf61SRaju P.L.S.S.S.N static void __tcs_set_trigger(struct rsc_drv *drv, int tcs_id, bool trigger) 34315b3bf61SRaju P.L.S.S.S.N { 34415b3bf61SRaju P.L.S.S.S.N u32 enable; 34515b3bf61SRaju P.L.S.S.S.N 34615b3bf61SRaju P.L.S.S.S.N /* 34715b3bf61SRaju P.L.S.S.S.N * HW req: Clear the DRV_CONTROL and enable TCS again 34815b3bf61SRaju P.L.S.S.S.N * While clearing ensure that the AMC mode trigger is cleared 34915b3bf61SRaju P.L.S.S.S.N * and then the mode enable is cleared. 35015b3bf61SRaju P.L.S.S.S.N */ 3513b5e3d50SDouglas Anderson enable = read_tcs_reg(drv, RSC_DRV_CONTROL, tcs_id); 35215b3bf61SRaju P.L.S.S.S.N enable &= ~TCS_AMC_MODE_TRIGGER; 35315b3bf61SRaju P.L.S.S.S.N write_tcs_reg_sync(drv, RSC_DRV_CONTROL, tcs_id, enable); 35415b3bf61SRaju P.L.S.S.S.N enable &= ~TCS_AMC_MODE_ENABLE; 35515b3bf61SRaju P.L.S.S.S.N write_tcs_reg_sync(drv, RSC_DRV_CONTROL, tcs_id, enable); 35615b3bf61SRaju P.L.S.S.S.N 35715b3bf61SRaju P.L.S.S.S.N if (trigger) { 35815b3bf61SRaju P.L.S.S.S.N /* Enable the AMC mode on the TCS and then trigger the TCS */ 35915b3bf61SRaju P.L.S.S.S.N enable = TCS_AMC_MODE_ENABLE; 36015b3bf61SRaju P.L.S.S.S.N write_tcs_reg_sync(drv, RSC_DRV_CONTROL, tcs_id, enable); 36115b3bf61SRaju P.L.S.S.S.N enable |= TCS_AMC_MODE_TRIGGER; 36215b3bf61SRaju P.L.S.S.S.N write_tcs_reg_sync(drv, RSC_DRV_CONTROL, tcs_id, enable); 36315b3bf61SRaju P.L.S.S.S.N } 36415b3bf61SRaju P.L.S.S.S.N } 36515b3bf61SRaju P.L.S.S.S.N 366e40b0c16SDouglas Anderson /** 367e40b0c16SDouglas Anderson * enable_tcs_irq() - Enable or disable interrupts on the given TCS. 368e40b0c16SDouglas Anderson * @drv: The controller. 369e40b0c16SDouglas Anderson * @tcs_id: The global ID of this TCS. 370e40b0c16SDouglas Anderson * @enable: If true then enable; if false then disable 371e40b0c16SDouglas Anderson * 372e40b0c16SDouglas Anderson * We only ever call this when we borrow a wake TCS for an active-only 373e40b0c16SDouglas Anderson * transfer. For active-only TCSes interrupts are always left enabled. 374e40b0c16SDouglas Anderson */ 37515b3bf61SRaju P.L.S.S.S.N static void enable_tcs_irq(struct rsc_drv *drv, int tcs_id, bool enable) 37615b3bf61SRaju P.L.S.S.S.N { 37715b3bf61SRaju P.L.S.S.S.N u32 data; 37815b3bf61SRaju P.L.S.S.S.N 3793b5e3d50SDouglas Anderson data = read_tcs_reg(drv, RSC_DRV_IRQ_ENABLE, 0); 38015b3bf61SRaju P.L.S.S.S.N if (enable) 38115b3bf61SRaju P.L.S.S.S.N data |= BIT(tcs_id); 38215b3bf61SRaju P.L.S.S.S.N else 38315b3bf61SRaju P.L.S.S.S.N data &= ~BIT(tcs_id); 38415b3bf61SRaju P.L.S.S.S.N write_tcs_reg(drv, RSC_DRV_IRQ_ENABLE, 0, data); 38515b3bf61SRaju P.L.S.S.S.N } 38615b3bf61SRaju P.L.S.S.S.N 387658628e7SLina Iyer /** 388e40b0c16SDouglas Anderson * tcs_tx_done() - TX Done interrupt handler. 389e40b0c16SDouglas Anderson * @irq: The IRQ number (ignored). 390e40b0c16SDouglas Anderson * @p: Pointer to "struct rsc_drv". 391e40b0c16SDouglas Anderson * 392e40b0c16SDouglas Anderson * Called for ACTIVE_ONLY transfers (those are the only ones we enable the 393e40b0c16SDouglas Anderson * IRQ for) when a transfer is done. 394e40b0c16SDouglas Anderson * 395e40b0c16SDouglas Anderson * Return: IRQ_HANDLED 396658628e7SLina Iyer */ 397658628e7SLina Iyer static irqreturn_t tcs_tx_done(int irq, void *p) 398658628e7SLina Iyer { 399658628e7SLina Iyer struct rsc_drv *drv = p; 400c1038456SLina Iyer int i, j, err = 0; 401658628e7SLina Iyer unsigned long irq_status; 402658628e7SLina Iyer const struct tcs_request *req; 403658628e7SLina Iyer struct tcs_cmd *cmd; 404658628e7SLina Iyer 4053b5e3d50SDouglas Anderson irq_status = read_tcs_reg(drv, RSC_DRV_IRQ_STATUS, 0); 406658628e7SLina Iyer 407658628e7SLina Iyer for_each_set_bit(i, &irq_status, BITS_PER_LONG) { 408658628e7SLina Iyer req = get_req_from_tcs(drv, i); 409658628e7SLina Iyer if (!req) { 410658628e7SLina Iyer WARN_ON(1); 411658628e7SLina Iyer goto skip; 412658628e7SLina Iyer } 413658628e7SLina Iyer 414fc087fe5SLina Iyer err = 0; 415658628e7SLina Iyer for (j = 0; j < req->num_cmds; j++) { 416658628e7SLina Iyer u32 sts; 417658628e7SLina Iyer 418658628e7SLina Iyer cmd = &req->cmds[j]; 4193b5e3d50SDouglas Anderson sts = read_tcs_cmd(drv, RSC_DRV_CMD_STATUS, i, j); 420658628e7SLina Iyer if (!(sts & CMD_STATUS_ISSUED) || 421658628e7SLina Iyer ((req->wait_for_compl || cmd->wait) && 422658628e7SLina Iyer !(sts & CMD_STATUS_COMPL))) { 423658628e7SLina Iyer pr_err("Incomplete request: %s: addr=%#x data=%#x", 424658628e7SLina Iyer drv->name, cmd->addr, cmd->data); 425fc087fe5SLina Iyer err = -EIO; 426658628e7SLina Iyer } 427658628e7SLina Iyer } 428fc087fe5SLina Iyer 429fc087fe5SLina Iyer trace_rpmh_tx_done(drv, i, req, err); 43015b3bf61SRaju P.L.S.S.S.N 43115b3bf61SRaju P.L.S.S.S.N /* 43215b3bf61SRaju P.L.S.S.S.N * If wake tcs was re-purposed for sending active 43315b3bf61SRaju P.L.S.S.S.N * votes, clear AMC trigger & enable modes and 43415b3bf61SRaju P.L.S.S.S.N * disable interrupt for this TCS 43515b3bf61SRaju P.L.S.S.S.N */ 43615b3bf61SRaju P.L.S.S.S.N if (!drv->tcs[ACTIVE_TCS].num_tcs) 43715b3bf61SRaju P.L.S.S.S.N __tcs_set_trigger(drv, i, false); 438658628e7SLina Iyer skip: 439658628e7SLina Iyer /* Reclaim the TCS */ 440658628e7SLina Iyer write_tcs_reg(drv, RSC_DRV_CMD_ENABLE, i, 0); 44109e97b6cSLina Iyer write_tcs_reg(drv, RSC_DRV_CMD_WAIT_FOR_CMPL, i, 0); 442658628e7SLina Iyer write_tcs_reg(drv, RSC_DRV_IRQ_CLEAR, 0, BIT(i)); 443658628e7SLina Iyer spin_lock(&drv->lock); 444658628e7SLina Iyer clear_bit(i, drv->tcs_in_use); 44515b3bf61SRaju P.L.S.S.S.N /* 44615b3bf61SRaju P.L.S.S.S.N * Disable interrupt for WAKE TCS to avoid being 44715b3bf61SRaju P.L.S.S.S.N * spammed with interrupts coming when the solver 44815b3bf61SRaju P.L.S.S.S.N * sends its wake votes. 44915b3bf61SRaju P.L.S.S.S.N */ 45015b3bf61SRaju P.L.S.S.S.N if (!drv->tcs[ACTIVE_TCS].num_tcs) 45115b3bf61SRaju P.L.S.S.S.N enable_tcs_irq(drv, i, false); 452658628e7SLina Iyer spin_unlock(&drv->lock); 453c1038456SLina Iyer if (req) 454c1038456SLina Iyer rpmh_tx_done(req, err); 455658628e7SLina Iyer } 456658628e7SLina Iyer 457658628e7SLina Iyer return IRQ_HANDLED; 458658628e7SLina Iyer } 459658628e7SLina Iyer 460e40b0c16SDouglas Anderson /** 461e40b0c16SDouglas Anderson * __tcs_buffer_write() - Write to TCS hardware from a request; don't trigger. 462e40b0c16SDouglas Anderson * @drv: The controller. 463e40b0c16SDouglas Anderson * @tcs_id: The global ID of this TCS. 464e40b0c16SDouglas Anderson * @cmd_id: The index within the TCS to start writing. 465e40b0c16SDouglas Anderson * @msg: The message we want to send, which will contain several addr/data 466e40b0c16SDouglas Anderson * pairs to program (but few enough that they all fit in one TCS). 467e40b0c16SDouglas Anderson * 468e40b0c16SDouglas Anderson * This is used for all types of transfers (active, sleep, and wake). 469e40b0c16SDouglas Anderson */ 470658628e7SLina Iyer static void __tcs_buffer_write(struct rsc_drv *drv, int tcs_id, int cmd_id, 471658628e7SLina Iyer const struct tcs_request *msg) 472658628e7SLina Iyer { 473658628e7SLina Iyer u32 msgid, cmd_msgid; 474658628e7SLina Iyer u32 cmd_enable = 0; 475658628e7SLina Iyer u32 cmd_complete; 476658628e7SLina Iyer struct tcs_cmd *cmd; 477658628e7SLina Iyer int i, j; 478658628e7SLina Iyer 479658628e7SLina Iyer cmd_msgid = CMD_MSGID_LEN; 480658628e7SLina Iyer cmd_msgid |= msg->wait_for_compl ? CMD_MSGID_RESP_REQ : 0; 481658628e7SLina Iyer cmd_msgid |= CMD_MSGID_WRITE; 482658628e7SLina Iyer 4833b5e3d50SDouglas Anderson cmd_complete = read_tcs_reg(drv, RSC_DRV_CMD_WAIT_FOR_CMPL, tcs_id); 484658628e7SLina Iyer 485658628e7SLina Iyer for (i = 0, j = cmd_id; i < msg->num_cmds; i++, j++) { 486658628e7SLina Iyer cmd = &msg->cmds[i]; 487658628e7SLina Iyer cmd_enable |= BIT(j); 488658628e7SLina Iyer cmd_complete |= cmd->wait << j; 489658628e7SLina Iyer msgid = cmd_msgid; 490658628e7SLina Iyer msgid |= cmd->wait ? CMD_MSGID_RESP_REQ : 0; 491fc087fe5SLina Iyer 492658628e7SLina Iyer write_tcs_cmd(drv, RSC_DRV_CMD_MSGID, tcs_id, j, msgid); 493658628e7SLina Iyer write_tcs_cmd(drv, RSC_DRV_CMD_ADDR, tcs_id, j, cmd->addr); 494658628e7SLina Iyer write_tcs_cmd(drv, RSC_DRV_CMD_DATA, tcs_id, j, cmd->data); 495efde2659SStephen Boyd trace_rpmh_send_msg_rcuidle(drv, tcs_id, j, msgid, cmd); 496658628e7SLina Iyer } 497658628e7SLina Iyer 498658628e7SLina Iyer write_tcs_reg(drv, RSC_DRV_CMD_WAIT_FOR_CMPL, tcs_id, cmd_complete); 4993b5e3d50SDouglas Anderson cmd_enable |= read_tcs_reg(drv, RSC_DRV_CMD_ENABLE, tcs_id); 500658628e7SLina Iyer write_tcs_reg(drv, RSC_DRV_CMD_ENABLE, tcs_id, cmd_enable); 501658628e7SLina Iyer } 502658628e7SLina Iyer 503e40b0c16SDouglas Anderson /** 504e40b0c16SDouglas Anderson * check_for_req_inflight() - Look to see if conflicting cmds are in flight. 505e40b0c16SDouglas Anderson * @drv: The controller. 506e40b0c16SDouglas Anderson * @tcs: A pointer to the tcs_group used for ACTIVE_ONLY transfers. 507e40b0c16SDouglas Anderson * @msg: The message we want to send, which will contain several addr/data 508e40b0c16SDouglas Anderson * pairs to program (but few enough that they all fit in one TCS). 509e40b0c16SDouglas Anderson * 510e40b0c16SDouglas Anderson * This will walk through the TCSes in the group and check if any of them 511e40b0c16SDouglas Anderson * appear to be sending to addresses referenced in the message. If it finds 512e40b0c16SDouglas Anderson * one it'll return -EBUSY. 513e40b0c16SDouglas Anderson * 514e40b0c16SDouglas Anderson * Only for use for active-only transfers. 515e40b0c16SDouglas Anderson * 516e40b0c16SDouglas Anderson * Must be called with the drv->lock held since that protects tcs_in_use. 517e40b0c16SDouglas Anderson * 518e40b0c16SDouglas Anderson * Return: 0 if nothing in flight or -EBUSY if we should try again later. 519e40b0c16SDouglas Anderson * The caller must re-enable interrupts between tries since that's 520e40b0c16SDouglas Anderson * the only way tcs_is_free() will ever return true and the only way 521e40b0c16SDouglas Anderson * RSC_DRV_CMD_ENABLE will ever be cleared. 522e40b0c16SDouglas Anderson */ 523658628e7SLina Iyer static int check_for_req_inflight(struct rsc_drv *drv, struct tcs_group *tcs, 524658628e7SLina Iyer const struct tcs_request *msg) 525658628e7SLina Iyer { 526658628e7SLina Iyer unsigned long curr_enabled; 527658628e7SLina Iyer u32 addr; 528658628e7SLina Iyer int i, j, k; 529658628e7SLina Iyer int tcs_id = tcs->offset; 530658628e7SLina Iyer 531658628e7SLina Iyer for (i = 0; i < tcs->num_tcs; i++, tcs_id++) { 532658628e7SLina Iyer if (tcs_is_free(drv, tcs_id)) 533658628e7SLina Iyer continue; 534658628e7SLina Iyer 5353b5e3d50SDouglas Anderson curr_enabled = read_tcs_reg(drv, RSC_DRV_CMD_ENABLE, tcs_id); 536658628e7SLina Iyer 537658628e7SLina Iyer for_each_set_bit(j, &curr_enabled, MAX_CMDS_PER_TCS) { 5383b5e3d50SDouglas Anderson addr = read_tcs_cmd(drv, RSC_DRV_CMD_ADDR, tcs_id, j); 539658628e7SLina Iyer for (k = 0; k < msg->num_cmds; k++) { 540658628e7SLina Iyer if (addr == msg->cmds[k].addr) 541658628e7SLina Iyer return -EBUSY; 542658628e7SLina Iyer } 543658628e7SLina Iyer } 544658628e7SLina Iyer } 545658628e7SLina Iyer 546658628e7SLina Iyer return 0; 547658628e7SLina Iyer } 548658628e7SLina Iyer 549e40b0c16SDouglas Anderson /** 550e40b0c16SDouglas Anderson * find_free_tcs() - Find free tcs in the given tcs_group; only for active. 551e40b0c16SDouglas Anderson * @tcs: A pointer to the active-only tcs_group (or the wake tcs_group if 552e40b0c16SDouglas Anderson * we borrowed it because there are zero active-only ones). 553e40b0c16SDouglas Anderson * 554e40b0c16SDouglas Anderson * Must be called with the drv->lock held since that protects tcs_in_use. 555e40b0c16SDouglas Anderson * 556e40b0c16SDouglas Anderson * Return: The first tcs that's free. 557e40b0c16SDouglas Anderson */ 558658628e7SLina Iyer static int find_free_tcs(struct tcs_group *tcs) 559658628e7SLina Iyer { 560658628e7SLina Iyer int i; 561658628e7SLina Iyer 562658628e7SLina Iyer for (i = 0; i < tcs->num_tcs; i++) { 563658628e7SLina Iyer if (tcs_is_free(tcs->drv, tcs->offset + i)) 564658628e7SLina Iyer return tcs->offset + i; 565658628e7SLina Iyer } 566658628e7SLina Iyer 567658628e7SLina Iyer return -EBUSY; 568658628e7SLina Iyer } 569658628e7SLina Iyer 570e40b0c16SDouglas Anderson /** 571e40b0c16SDouglas Anderson * tcs_write() - Store messages into a TCS right now, or return -EBUSY. 572e40b0c16SDouglas Anderson * @drv: The controller. 573e40b0c16SDouglas Anderson * @msg: The data to be sent. 574e40b0c16SDouglas Anderson * 575e40b0c16SDouglas Anderson * Grabs a TCS for ACTIVE_ONLY transfers and writes the messages to it. 576e40b0c16SDouglas Anderson * 577e40b0c16SDouglas Anderson * If there are no free TCSes for ACTIVE_ONLY transfers or if a command for 578e40b0c16SDouglas Anderson * the same address is already transferring returns -EBUSY which means the 579e40b0c16SDouglas Anderson * client should retry shortly. 580e40b0c16SDouglas Anderson * 581e40b0c16SDouglas Anderson * Return: 0 on success, -EBUSY if client should retry, or an error. 582e40b0c16SDouglas Anderson * Client should have interrupts enabled for a bit before retrying. 583e40b0c16SDouglas Anderson */ 584658628e7SLina Iyer static int tcs_write(struct rsc_drv *drv, const struct tcs_request *msg) 585658628e7SLina Iyer { 586658628e7SLina Iyer struct tcs_group *tcs; 587658628e7SLina Iyer int tcs_id; 588658628e7SLina Iyer unsigned long flags; 589658628e7SLina Iyer int ret; 590658628e7SLina Iyer 591658628e7SLina Iyer tcs = get_tcs_for_msg(drv, msg); 592658628e7SLina Iyer if (IS_ERR(tcs)) 593658628e7SLina Iyer return PTR_ERR(tcs); 594658628e7SLina Iyer 595658628e7SLina Iyer spin_lock_irqsave(&tcs->lock, flags); 596658628e7SLina Iyer spin_lock(&drv->lock); 597658628e7SLina Iyer /* 598658628e7SLina Iyer * The h/w does not like if we send a request to the same address, 599658628e7SLina Iyer * when one is already in-flight or being processed. 600658628e7SLina Iyer */ 601658628e7SLina Iyer ret = check_for_req_inflight(drv, tcs, msg); 602658628e7SLina Iyer if (ret) { 603658628e7SLina Iyer spin_unlock(&drv->lock); 604658628e7SLina Iyer goto done_write; 605658628e7SLina Iyer } 606658628e7SLina Iyer 607658628e7SLina Iyer tcs_id = find_free_tcs(tcs); 608658628e7SLina Iyer if (tcs_id < 0) { 609658628e7SLina Iyer ret = tcs_id; 610658628e7SLina Iyer spin_unlock(&drv->lock); 611658628e7SLina Iyer goto done_write; 612658628e7SLina Iyer } 613658628e7SLina Iyer 614658628e7SLina Iyer tcs->req[tcs_id - tcs->offset] = msg; 615658628e7SLina Iyer set_bit(tcs_id, drv->tcs_in_use); 61638427e5aSMaulik Shah if (msg->state == RPMH_ACTIVE_ONLY_STATE && tcs->type != ACTIVE_TCS) { 61738427e5aSMaulik Shah /* 61838427e5aSMaulik Shah * Clear previously programmed WAKE commands in selected 61938427e5aSMaulik Shah * repurposed TCS to avoid triggering them. tcs->slots will be 62038427e5aSMaulik Shah * cleaned from rpmh_flush() by invoking rpmh_rsc_invalidate() 62138427e5aSMaulik Shah */ 62238427e5aSMaulik Shah write_tcs_reg_sync(drv, RSC_DRV_CMD_ENABLE, tcs_id, 0); 62338427e5aSMaulik Shah write_tcs_reg_sync(drv, RSC_DRV_CMD_WAIT_FOR_CMPL, tcs_id, 0); 62415b3bf61SRaju P.L.S.S.S.N enable_tcs_irq(drv, tcs_id, true); 62538427e5aSMaulik Shah } 626658628e7SLina Iyer spin_unlock(&drv->lock); 627658628e7SLina Iyer 628658628e7SLina Iyer __tcs_buffer_write(drv, tcs_id, 0, msg); 62915b3bf61SRaju P.L.S.S.S.N __tcs_set_trigger(drv, tcs_id, true); 630658628e7SLina Iyer 631658628e7SLina Iyer done_write: 632658628e7SLina Iyer spin_unlock_irqrestore(&tcs->lock, flags); 633658628e7SLina Iyer return ret; 634658628e7SLina Iyer } 635658628e7SLina Iyer 636658628e7SLina Iyer /** 637e40b0c16SDouglas Anderson * rpmh_rsc_send_data() - Validate the incoming message + write to TCS block. 638e40b0c16SDouglas Anderson * @drv: The controller. 639e40b0c16SDouglas Anderson * @msg: The data to be sent. 640658628e7SLina Iyer * 641e40b0c16SDouglas Anderson * NOTES: 642e40b0c16SDouglas Anderson * - This is only used for "ACTIVE_ONLY" since the limitations of this 643e40b0c16SDouglas Anderson * function don't make sense for sleep/wake cases. 644e40b0c16SDouglas Anderson * - To do the transfer, we will grab a whole TCS for ourselves--we don't 645e40b0c16SDouglas Anderson * try to share. If there are none available we'll wait indefinitely 646e40b0c16SDouglas Anderson * for a free one. 647e40b0c16SDouglas Anderson * - This function will not wait for the commands to be finished, only for 648e40b0c16SDouglas Anderson * data to be programmed into the RPMh. See rpmh_tx_done() which will 649e40b0c16SDouglas Anderson * be called when the transfer is fully complete. 650e40b0c16SDouglas Anderson * - This function must be called with interrupts enabled. If the hardware 651e40b0c16SDouglas Anderson * is busy doing someone else's transfer we need that transfer to fully 652e40b0c16SDouglas Anderson * finish so that we can have the hardware, and to fully finish it needs 653e40b0c16SDouglas Anderson * the interrupt handler to run. If the interrupts is set to run on the 654e40b0c16SDouglas Anderson * active CPU this can never happen if interrupts are disabled. 655658628e7SLina Iyer * 656658628e7SLina Iyer * Return: 0 on success, -EINVAL on error. 657658628e7SLina Iyer */ 658658628e7SLina Iyer int rpmh_rsc_send_data(struct rsc_drv *drv, const struct tcs_request *msg) 659658628e7SLina Iyer { 660658628e7SLina Iyer int ret; 661658628e7SLina Iyer 662658628e7SLina Iyer if (!msg || !msg->cmds || !msg->num_cmds || 663658628e7SLina Iyer msg->num_cmds > MAX_RPMH_PAYLOAD) { 664658628e7SLina Iyer WARN_ON(1); 665658628e7SLina Iyer return -EINVAL; 666658628e7SLina Iyer } 667658628e7SLina Iyer 668658628e7SLina Iyer do { 669658628e7SLina Iyer ret = tcs_write(drv, msg); 670658628e7SLina Iyer if (ret == -EBUSY) { 671658628e7SLina Iyer pr_info_ratelimited("TCS Busy, retrying RPMH message send: addr=%#x\n", 672658628e7SLina Iyer msg->cmds[0].addr); 673658628e7SLina Iyer udelay(10); 674658628e7SLina Iyer } 675658628e7SLina Iyer } while (ret == -EBUSY); 676658628e7SLina Iyer 677658628e7SLina Iyer return ret; 678658628e7SLina Iyer } 679658628e7SLina Iyer 680e40b0c16SDouglas Anderson /** 681e40b0c16SDouglas Anderson * find_slots() - Find a place to write the given message. 682e40b0c16SDouglas Anderson * @tcs: The tcs group to search. 683e40b0c16SDouglas Anderson * @msg: The message we want to find room for. 684e40b0c16SDouglas Anderson * @tcs_id: If we return 0 from the function, we return the global ID of the 685e40b0c16SDouglas Anderson * TCS to write to here. 686e40b0c16SDouglas Anderson * @cmd_id: If we return 0 from the function, we return the index of 687e40b0c16SDouglas Anderson * the command array of the returned TCS where the client should 688e40b0c16SDouglas Anderson * start writing the message. 689e40b0c16SDouglas Anderson * 690e40b0c16SDouglas Anderson * Only for use on sleep/wake TCSes since those are the only ones we maintain 691e40b0c16SDouglas Anderson * tcs->slots for. 692e40b0c16SDouglas Anderson * 693e40b0c16SDouglas Anderson * Must be called with the tcs_lock for the group held. 694e40b0c16SDouglas Anderson * 695e40b0c16SDouglas Anderson * Return: -ENOMEM if there was no room, else 0. 696e40b0c16SDouglas Anderson */ 697fa460e45SLina Iyer static int find_slots(struct tcs_group *tcs, const struct tcs_request *msg, 698fa460e45SLina Iyer int *tcs_id, int *cmd_id) 699fa460e45SLina Iyer { 700fa460e45SLina Iyer int slot, offset; 701fa460e45SLina Iyer int i = 0; 702fa460e45SLina Iyer 703e40b0c16SDouglas Anderson /* Do over, until we can fit the full payload in a single TCS */ 704fa460e45SLina Iyer do { 705fa460e45SLina Iyer slot = bitmap_find_next_zero_area(tcs->slots, MAX_TCS_SLOTS, 706fa460e45SLina Iyer i, msg->num_cmds, 0); 707bbeac60fSMaulik Shah if (slot >= tcs->num_tcs * tcs->ncpt) 708fa460e45SLina Iyer return -ENOMEM; 709fa460e45SLina Iyer i += tcs->ncpt; 710fa460e45SLina Iyer } while (slot + msg->num_cmds - 1 >= i); 711fa460e45SLina Iyer 712fa460e45SLina Iyer bitmap_set(tcs->slots, slot, msg->num_cmds); 713fa460e45SLina Iyer 714fa460e45SLina Iyer offset = slot / tcs->ncpt; 715fa460e45SLina Iyer *tcs_id = offset + tcs->offset; 716fa460e45SLina Iyer *cmd_id = slot % tcs->ncpt; 717fa460e45SLina Iyer 718fa460e45SLina Iyer return 0; 719fa460e45SLina Iyer } 720fa460e45SLina Iyer 721427ef4f7SDouglas Anderson /** 722e40b0c16SDouglas Anderson * rpmh_rsc_write_ctrl_data() - Write request to controller but don't trigger. 723e40b0c16SDouglas Anderson * @drv: The controller. 724e40b0c16SDouglas Anderson * @msg: The data to be written to the controller. 725427ef4f7SDouglas Anderson * 726e40b0c16SDouglas Anderson * This should only be called for for sleep/wake state, never active-only 727e40b0c16SDouglas Anderson * state. 728427ef4f7SDouglas Anderson * 729e40b0c16SDouglas Anderson * Return: 0 if no error; else -error. 730427ef4f7SDouglas Anderson */ 731427ef4f7SDouglas Anderson int rpmh_rsc_write_ctrl_data(struct rsc_drv *drv, const struct tcs_request *msg) 732fa460e45SLina Iyer { 733fa460e45SLina Iyer struct tcs_group *tcs; 734fa460e45SLina Iyer int tcs_id = 0, cmd_id = 0; 735fa460e45SLina Iyer unsigned long flags; 736fa460e45SLina Iyer int ret; 737fa460e45SLina Iyer 738427ef4f7SDouglas Anderson if (!msg || !msg->cmds || !msg->num_cmds || 739427ef4f7SDouglas Anderson msg->num_cmds > MAX_RPMH_PAYLOAD) { 740427ef4f7SDouglas Anderson pr_err("Payload error\n"); 741427ef4f7SDouglas Anderson return -EINVAL; 742427ef4f7SDouglas Anderson } 743427ef4f7SDouglas Anderson 744427ef4f7SDouglas Anderson /* Data sent to this API will not be sent immediately */ 745427ef4f7SDouglas Anderson if (msg->state == RPMH_ACTIVE_ONLY_STATE) 746427ef4f7SDouglas Anderson return -EINVAL; 747427ef4f7SDouglas Anderson 748fa460e45SLina Iyer tcs = get_tcs_for_msg(drv, msg); 749fa460e45SLina Iyer if (IS_ERR(tcs)) 750fa460e45SLina Iyer return PTR_ERR(tcs); 751fa460e45SLina Iyer 752fa460e45SLina Iyer spin_lock_irqsave(&tcs->lock, flags); 753fa460e45SLina Iyer /* find the TCS id and the command in the TCS to write to */ 754fa460e45SLina Iyer ret = find_slots(tcs, msg, &tcs_id, &cmd_id); 755fa460e45SLina Iyer if (!ret) 756fa460e45SLina Iyer __tcs_buffer_write(drv, tcs_id, cmd_id, msg); 757fa460e45SLina Iyer spin_unlock_irqrestore(&tcs->lock, flags); 758fa460e45SLina Iyer 759fa460e45SLina Iyer return ret; 760fa460e45SLina Iyer } 761fa460e45SLina Iyer 762fa460e45SLina Iyer /** 763985427f9SMaulik Shah * rpmh_rsc_ctrlr_is_busy() - Check if any of the AMCs are busy. 764985427f9SMaulik Shah * @drv: The controller 765985427f9SMaulik Shah * 766985427f9SMaulik Shah * Checks if any of the AMCs are busy in handling ACTIVE sets. 767985427f9SMaulik Shah * This is called from the last cpu powering down before flushing 768985427f9SMaulik Shah * SLEEP and WAKE sets. If AMCs are busy, controller can not enter 769985427f9SMaulik Shah * power collapse, so deny from the last cpu's pm notification. 770985427f9SMaulik Shah * 771985427f9SMaulik Shah * Return: 772985427f9SMaulik Shah * * False - AMCs are idle 773985427f9SMaulik Shah * * True - AMCs are busy 774985427f9SMaulik Shah */ 775985427f9SMaulik Shah static bool rpmh_rsc_ctrlr_is_busy(struct rsc_drv *drv) 776985427f9SMaulik Shah { 777985427f9SMaulik Shah int m; 77853d49fe1SDouglas Anderson struct tcs_group *tcs = &drv->tcs[ACTIVE_TCS]; 779985427f9SMaulik Shah 780985427f9SMaulik Shah /* 781985427f9SMaulik Shah * If we made an active request on a RSC that does not have a 782985427f9SMaulik Shah * dedicated TCS for active state use, then re-purposed wake TCSes 783985427f9SMaulik Shah * should be checked for not busy, because we used wake TCSes for 784985427f9SMaulik Shah * active requests in this case. 785985427f9SMaulik Shah * 786985427f9SMaulik Shah * Since this is called from the last cpu, need not take drv or tcs 787985427f9SMaulik Shah * lock before checking tcs_is_free(). 788985427f9SMaulik Shah */ 789985427f9SMaulik Shah if (!tcs->num_tcs) 79053d49fe1SDouglas Anderson tcs = &drv->tcs[WAKE_TCS]; 791985427f9SMaulik Shah 792985427f9SMaulik Shah for (m = tcs->offset; m < tcs->offset + tcs->num_tcs; m++) { 793985427f9SMaulik Shah if (!tcs_is_free(drv, m)) 794985427f9SMaulik Shah return true; 795985427f9SMaulik Shah } 796985427f9SMaulik Shah 797985427f9SMaulik Shah return false; 798985427f9SMaulik Shah } 799985427f9SMaulik Shah 800e40b0c16SDouglas Anderson /** 801e40b0c16SDouglas Anderson * rpmh_rsc_cpu_pm_callback() - Check if any of the AMCs are busy. 802e40b0c16SDouglas Anderson * @nfb: Pointer to the notifier block in struct rsc_drv. 803e40b0c16SDouglas Anderson * @action: CPU_PM_ENTER, CPU_PM_ENTER_FAILED, or CPU_PM_EXIT. 804e40b0c16SDouglas Anderson * @v: Unused 805e40b0c16SDouglas Anderson * 806e40b0c16SDouglas Anderson * This function is given to cpu_pm_register_notifier so we can be informed 807e40b0c16SDouglas Anderson * about when CPUs go down. When all CPUs go down we know no more active 808e40b0c16SDouglas Anderson * transfers will be started so we write sleep/wake sets. This function gets 809e40b0c16SDouglas Anderson * called from cpuidle code paths and also at system suspend time. 810e40b0c16SDouglas Anderson * 811e40b0c16SDouglas Anderson * If its last CPU going down and AMCs are not busy then writes cached sleep 812e40b0c16SDouglas Anderson * and wake messages to TCSes. The firmware then takes care of triggering 813e40b0c16SDouglas Anderson * them when entering deepest low power modes. 814e40b0c16SDouglas Anderson * 815e40b0c16SDouglas Anderson * Return: See cpu_pm_register_notifier() 816e40b0c16SDouglas Anderson */ 817985427f9SMaulik Shah static int rpmh_rsc_cpu_pm_callback(struct notifier_block *nfb, 818985427f9SMaulik Shah unsigned long action, void *v) 819985427f9SMaulik Shah { 820985427f9SMaulik Shah struct rsc_drv *drv = container_of(nfb, struct rsc_drv, rsc_pm); 821985427f9SMaulik Shah int ret = NOTIFY_OK; 822985427f9SMaulik Shah 823985427f9SMaulik Shah spin_lock(&drv->pm_lock); 824985427f9SMaulik Shah 825985427f9SMaulik Shah switch (action) { 826985427f9SMaulik Shah case CPU_PM_ENTER: 827985427f9SMaulik Shah cpumask_set_cpu(smp_processor_id(), &drv->cpus_entered_pm); 828985427f9SMaulik Shah 829985427f9SMaulik Shah if (!cpumask_equal(&drv->cpus_entered_pm, cpu_online_mask)) 830985427f9SMaulik Shah goto exit; 831985427f9SMaulik Shah break; 832985427f9SMaulik Shah case CPU_PM_ENTER_FAILED: 833985427f9SMaulik Shah case CPU_PM_EXIT: 834985427f9SMaulik Shah cpumask_clear_cpu(smp_processor_id(), &drv->cpus_entered_pm); 835985427f9SMaulik Shah goto exit; 836985427f9SMaulik Shah } 837985427f9SMaulik Shah 838985427f9SMaulik Shah ret = rpmh_rsc_ctrlr_is_busy(drv); 839985427f9SMaulik Shah if (ret) { 840985427f9SMaulik Shah ret = NOTIFY_BAD; 841985427f9SMaulik Shah goto exit; 842985427f9SMaulik Shah } 843985427f9SMaulik Shah 844985427f9SMaulik Shah ret = rpmh_flush(&drv->client); 845985427f9SMaulik Shah if (ret) 846985427f9SMaulik Shah ret = NOTIFY_BAD; 847985427f9SMaulik Shah else 848985427f9SMaulik Shah ret = NOTIFY_OK; 849985427f9SMaulik Shah 850985427f9SMaulik Shah exit: 851985427f9SMaulik Shah spin_unlock(&drv->pm_lock); 852985427f9SMaulik Shah return ret; 853985427f9SMaulik Shah } 854985427f9SMaulik Shah 855658628e7SLina Iyer static int rpmh_probe_tcs_config(struct platform_device *pdev, 856985427f9SMaulik Shah struct rsc_drv *drv, void __iomem *base) 857658628e7SLina Iyer { 858658628e7SLina Iyer struct tcs_type_config { 859658628e7SLina Iyer u32 type; 860658628e7SLina Iyer u32 n; 861658628e7SLina Iyer } tcs_cfg[TCS_TYPE_NR] = { { 0 } }; 862658628e7SLina Iyer struct device_node *dn = pdev->dev.of_node; 863658628e7SLina Iyer u32 config, max_tcs, ncpt, offset; 864658628e7SLina Iyer int i, ret, n, st = 0; 865658628e7SLina Iyer struct tcs_group *tcs; 866658628e7SLina Iyer 867658628e7SLina Iyer ret = of_property_read_u32(dn, "qcom,tcs-offset", &offset); 868658628e7SLina Iyer if (ret) 869658628e7SLina Iyer return ret; 870658628e7SLina Iyer drv->tcs_base = base + offset; 871658628e7SLina Iyer 872658628e7SLina Iyer config = readl_relaxed(base + DRV_PRNT_CHLD_CONFIG); 873658628e7SLina Iyer 874658628e7SLina Iyer max_tcs = config; 875658628e7SLina Iyer max_tcs &= DRV_NUM_TCS_MASK << (DRV_NUM_TCS_SHIFT * drv->id); 876658628e7SLina Iyer max_tcs = max_tcs >> (DRV_NUM_TCS_SHIFT * drv->id); 877658628e7SLina Iyer 878658628e7SLina Iyer ncpt = config & (DRV_NCPT_MASK << DRV_NCPT_SHIFT); 879658628e7SLina Iyer ncpt = ncpt >> DRV_NCPT_SHIFT; 880658628e7SLina Iyer 881658628e7SLina Iyer n = of_property_count_u32_elems(dn, "qcom,tcs-config"); 882658628e7SLina Iyer if (n != 2 * TCS_TYPE_NR) 883658628e7SLina Iyer return -EINVAL; 884658628e7SLina Iyer 885658628e7SLina Iyer for (i = 0; i < TCS_TYPE_NR; i++) { 886658628e7SLina Iyer ret = of_property_read_u32_index(dn, "qcom,tcs-config", 887658628e7SLina Iyer i * 2, &tcs_cfg[i].type); 888658628e7SLina Iyer if (ret) 889658628e7SLina Iyer return ret; 890658628e7SLina Iyer if (tcs_cfg[i].type >= TCS_TYPE_NR) 891658628e7SLina Iyer return -EINVAL; 892658628e7SLina Iyer 893658628e7SLina Iyer ret = of_property_read_u32_index(dn, "qcom,tcs-config", 894658628e7SLina Iyer i * 2 + 1, &tcs_cfg[i].n); 895658628e7SLina Iyer if (ret) 896658628e7SLina Iyer return ret; 897658628e7SLina Iyer if (tcs_cfg[i].n > MAX_TCS_PER_TYPE) 898658628e7SLina Iyer return -EINVAL; 899658628e7SLina Iyer } 900658628e7SLina Iyer 901658628e7SLina Iyer for (i = 0; i < TCS_TYPE_NR; i++) { 902658628e7SLina Iyer tcs = &drv->tcs[tcs_cfg[i].type]; 903658628e7SLina Iyer if (tcs->drv) 904658628e7SLina Iyer return -EINVAL; 905658628e7SLina Iyer tcs->drv = drv; 906658628e7SLina Iyer tcs->type = tcs_cfg[i].type; 907658628e7SLina Iyer tcs->num_tcs = tcs_cfg[i].n; 908658628e7SLina Iyer tcs->ncpt = ncpt; 909658628e7SLina Iyer spin_lock_init(&tcs->lock); 910658628e7SLina Iyer 911658628e7SLina Iyer if (!tcs->num_tcs || tcs->type == CONTROL_TCS) 912658628e7SLina Iyer continue; 913658628e7SLina Iyer 914658628e7SLina Iyer if (st + tcs->num_tcs > max_tcs || 915658628e7SLina Iyer st + tcs->num_tcs >= BITS_PER_BYTE * sizeof(tcs->mask)) 916658628e7SLina Iyer return -EINVAL; 917658628e7SLina Iyer 918658628e7SLina Iyer tcs->mask = ((1 << tcs->num_tcs) - 1) << st; 919658628e7SLina Iyer tcs->offset = st; 920658628e7SLina Iyer st += tcs->num_tcs; 921658628e7SLina Iyer } 922658628e7SLina Iyer 923658628e7SLina Iyer drv->num_tcs = st; 924658628e7SLina Iyer 925658628e7SLina Iyer return 0; 926658628e7SLina Iyer } 927658628e7SLina Iyer 928658628e7SLina Iyer static int rpmh_rsc_probe(struct platform_device *pdev) 929658628e7SLina Iyer { 930658628e7SLina Iyer struct device_node *dn = pdev->dev.of_node; 931658628e7SLina Iyer struct rsc_drv *drv; 932985427f9SMaulik Shah struct resource *res; 933985427f9SMaulik Shah char drv_id[10] = {0}; 934658628e7SLina Iyer int ret, irq; 935985427f9SMaulik Shah u32 solver_config; 936985427f9SMaulik Shah void __iomem *base; 937658628e7SLina Iyer 938fdd102b5SDouglas Anderson /* 939fdd102b5SDouglas Anderson * Even though RPMh doesn't directly use cmd-db, all of its children 940fdd102b5SDouglas Anderson * do. To avoid adding this check to our children we'll do it now. 941fdd102b5SDouglas Anderson */ 942fdd102b5SDouglas Anderson ret = cmd_db_ready(); 943fdd102b5SDouglas Anderson if (ret) { 944fdd102b5SDouglas Anderson if (ret != -EPROBE_DEFER) 945fdd102b5SDouglas Anderson dev_err(&pdev->dev, "Command DB not available (%d)\n", 946fdd102b5SDouglas Anderson ret); 947fdd102b5SDouglas Anderson return ret; 948fdd102b5SDouglas Anderson } 949fdd102b5SDouglas Anderson 950658628e7SLina Iyer drv = devm_kzalloc(&pdev->dev, sizeof(*drv), GFP_KERNEL); 951658628e7SLina Iyer if (!drv) 952658628e7SLina Iyer return -ENOMEM; 953658628e7SLina Iyer 954658628e7SLina Iyer ret = of_property_read_u32(dn, "qcom,drv-id", &drv->id); 955658628e7SLina Iyer if (ret) 956658628e7SLina Iyer return ret; 957658628e7SLina Iyer 958658628e7SLina Iyer drv->name = of_get_property(dn, "label", NULL); 959658628e7SLina Iyer if (!drv->name) 960658628e7SLina Iyer drv->name = dev_name(&pdev->dev); 961658628e7SLina Iyer 962985427f9SMaulik Shah snprintf(drv_id, ARRAY_SIZE(drv_id), "drv-%d", drv->id); 963985427f9SMaulik Shah res = platform_get_resource_byname(pdev, IORESOURCE_MEM, drv_id); 964985427f9SMaulik Shah base = devm_ioremap_resource(&pdev->dev, res); 965985427f9SMaulik Shah if (IS_ERR(base)) 966985427f9SMaulik Shah return PTR_ERR(base); 967985427f9SMaulik Shah 968985427f9SMaulik Shah ret = rpmh_probe_tcs_config(pdev, drv, base); 969658628e7SLina Iyer if (ret) 970658628e7SLina Iyer return ret; 971658628e7SLina Iyer 972658628e7SLina Iyer spin_lock_init(&drv->lock); 973658628e7SLina Iyer bitmap_zero(drv->tcs_in_use, MAX_TCS_NR); 974658628e7SLina Iyer 975658628e7SLina Iyer irq = platform_get_irq(pdev, drv->id); 976658628e7SLina Iyer if (irq < 0) 977658628e7SLina Iyer return irq; 978658628e7SLina Iyer 979658628e7SLina Iyer ret = devm_request_irq(&pdev->dev, irq, tcs_tx_done, 980658628e7SLina Iyer IRQF_TRIGGER_HIGH | IRQF_NO_SUSPEND, 981658628e7SLina Iyer drv->name, drv); 982658628e7SLina Iyer if (ret) 983658628e7SLina Iyer return ret; 984658628e7SLina Iyer 985985427f9SMaulik Shah /* 986985427f9SMaulik Shah * CPU PM notification are not required for controllers that support 987985427f9SMaulik Shah * 'HW solver' mode where they can be in autonomous mode executing low 988985427f9SMaulik Shah * power mode to power down. 989985427f9SMaulik Shah */ 990985427f9SMaulik Shah solver_config = readl_relaxed(base + DRV_SOLVER_CONFIG); 991985427f9SMaulik Shah solver_config &= DRV_HW_SOLVER_MASK << DRV_HW_SOLVER_SHIFT; 992985427f9SMaulik Shah solver_config = solver_config >> DRV_HW_SOLVER_SHIFT; 993985427f9SMaulik Shah if (!solver_config) { 994985427f9SMaulik Shah drv->rsc_pm.notifier_call = rpmh_rsc_cpu_pm_callback; 995985427f9SMaulik Shah spin_lock_init(&drv->pm_lock); 996985427f9SMaulik Shah cpu_pm_register_notifier(&drv->rsc_pm); 997985427f9SMaulik Shah } 998985427f9SMaulik Shah 999658628e7SLina Iyer /* Enable the active TCS to send requests immediately */ 1000658628e7SLina Iyer write_tcs_reg(drv, RSC_DRV_IRQ_ENABLE, 0, drv->tcs[ACTIVE_TCS].mask); 1001658628e7SLina Iyer 1002600513dfSLina Iyer spin_lock_init(&drv->client.cache_lock); 1003600513dfSLina Iyer INIT_LIST_HEAD(&drv->client.cache); 1004c8790cb6SLina Iyer INIT_LIST_HEAD(&drv->client.batch_cache); 1005600513dfSLina Iyer 1006c1038456SLina Iyer dev_set_drvdata(&pdev->dev, drv); 1007c1038456SLina Iyer 1008658628e7SLina Iyer return devm_of_platform_populate(&pdev->dev); 1009658628e7SLina Iyer } 1010658628e7SLina Iyer 1011658628e7SLina Iyer static const struct of_device_id rpmh_drv_match[] = { 1012658628e7SLina Iyer { .compatible = "qcom,rpmh-rsc", }, 1013658628e7SLina Iyer { } 1014658628e7SLina Iyer }; 1015658628e7SLina Iyer 1016658628e7SLina Iyer static struct platform_driver rpmh_driver = { 1017658628e7SLina Iyer .probe = rpmh_rsc_probe, 1018658628e7SLina Iyer .driver = { 1019658628e7SLina Iyer .name = "rpmh", 1020658628e7SLina Iyer .of_match_table = rpmh_drv_match, 1021658628e7SLina Iyer }, 1022658628e7SLina Iyer }; 1023658628e7SLina Iyer 1024658628e7SLina Iyer static int __init rpmh_driver_init(void) 1025658628e7SLina Iyer { 1026658628e7SLina Iyer return platform_driver_register(&rpmh_driver); 1027658628e7SLina Iyer } 1028658628e7SLina Iyer arch_initcall(rpmh_driver_init); 1029