1658628e7SLina Iyer // SPDX-License-Identifier: GPL-2.0 2658628e7SLina Iyer /* 3658628e7SLina Iyer * Copyright (c) 2016-2018, The Linux Foundation. All rights reserved. 4658628e7SLina Iyer */ 5658628e7SLina Iyer 6658628e7SLina Iyer #define pr_fmt(fmt) "%s " fmt, KBUILD_MODNAME 7658628e7SLina Iyer 8658628e7SLina Iyer #include <linux/atomic.h> 9985427f9SMaulik Shah #include <linux/cpu_pm.h> 10658628e7SLina Iyer #include <linux/delay.h> 11658628e7SLina Iyer #include <linux/interrupt.h> 12658628e7SLina Iyer #include <linux/io.h> 1391160150SDouglas Anderson #include <linux/iopoll.h> 14658628e7SLina Iyer #include <linux/kernel.h> 15658628e7SLina Iyer #include <linux/list.h> 16658628e7SLina Iyer #include <linux/of.h> 17658628e7SLina Iyer #include <linux/of_irq.h> 18658628e7SLina Iyer #include <linux/of_platform.h> 19658628e7SLina Iyer #include <linux/platform_device.h> 20658628e7SLina Iyer #include <linux/slab.h> 21658628e7SLina Iyer #include <linux/spinlock.h> 222bc20f3cSStephen Boyd #include <linux/wait.h> 23658628e7SLina Iyer 24fdd102b5SDouglas Anderson #include <soc/qcom/cmd-db.h> 25658628e7SLina Iyer #include <soc/qcom/tcs.h> 26658628e7SLina Iyer #include <dt-bindings/soc/qcom,rpmh-rsc.h> 27658628e7SLina Iyer 28658628e7SLina Iyer #include "rpmh-internal.h" 29658628e7SLina Iyer 30fc087fe5SLina Iyer #define CREATE_TRACE_POINTS 31fc087fe5SLina Iyer #include "trace-rpmh.h" 32fc087fe5SLina Iyer 33658628e7SLina Iyer #define RSC_DRV_TCS_OFFSET 672 34658628e7SLina Iyer #define RSC_DRV_CMD_OFFSET 20 35658628e7SLina Iyer 36985427f9SMaulik Shah /* DRV HW Solver Configuration Information Register */ 37985427f9SMaulik Shah #define DRV_SOLVER_CONFIG 0x04 38985427f9SMaulik Shah #define DRV_HW_SOLVER_MASK 1 39985427f9SMaulik Shah #define DRV_HW_SOLVER_SHIFT 24 40985427f9SMaulik Shah 41985427f9SMaulik Shah /* DRV TCS Configuration Information Register */ 42658628e7SLina Iyer #define DRV_PRNT_CHLD_CONFIG 0x0C 43658628e7SLina Iyer #define DRV_NUM_TCS_MASK 0x3F 44658628e7SLina Iyer #define DRV_NUM_TCS_SHIFT 6 45658628e7SLina Iyer #define DRV_NCPT_MASK 0x1F 46658628e7SLina Iyer #define DRV_NCPT_SHIFT 27 47658628e7SLina Iyer 481f7dbeb5SDouglas Anderson /* Offsets for common TCS Registers, one bit per TCS */ 49658628e7SLina Iyer #define RSC_DRV_IRQ_ENABLE 0x00 50658628e7SLina Iyer #define RSC_DRV_IRQ_STATUS 0x04 511f7dbeb5SDouglas Anderson #define RSC_DRV_IRQ_CLEAR 0x08 /* w/o; write 1 to clear */ 521f7dbeb5SDouglas Anderson 531f7dbeb5SDouglas Anderson /* 541f7dbeb5SDouglas Anderson * Offsets for per TCS Registers. 551f7dbeb5SDouglas Anderson * 561f7dbeb5SDouglas Anderson * TCSes start at 0x10 from tcs_base and are stored one after another. 571f7dbeb5SDouglas Anderson * Multiply tcs_id by RSC_DRV_TCS_OFFSET to find a given TCS and add one 581f7dbeb5SDouglas Anderson * of the below to find a register. 591f7dbeb5SDouglas Anderson */ 601f7dbeb5SDouglas Anderson #define RSC_DRV_CMD_WAIT_FOR_CMPL 0x10 /* 1 bit per command */ 61658628e7SLina Iyer #define RSC_DRV_CONTROL 0x14 621f7dbeb5SDouglas Anderson #define RSC_DRV_STATUS 0x18 /* zero if tcs is busy */ 631f7dbeb5SDouglas Anderson #define RSC_DRV_CMD_ENABLE 0x1C /* 1 bit per command */ 641f7dbeb5SDouglas Anderson 651f7dbeb5SDouglas Anderson /* 661f7dbeb5SDouglas Anderson * Offsets for per command in a TCS. 671f7dbeb5SDouglas Anderson * 681f7dbeb5SDouglas Anderson * Commands (up to 16) start at 0x30 in a TCS; multiply command index 691f7dbeb5SDouglas Anderson * by RSC_DRV_CMD_OFFSET and add one of the below to find a register. 701f7dbeb5SDouglas Anderson */ 71658628e7SLina Iyer #define RSC_DRV_CMD_MSGID 0x30 72658628e7SLina Iyer #define RSC_DRV_CMD_ADDR 0x34 73658628e7SLina Iyer #define RSC_DRV_CMD_DATA 0x38 74658628e7SLina Iyer #define RSC_DRV_CMD_STATUS 0x3C 75658628e7SLina Iyer #define RSC_DRV_CMD_RESP_DATA 0x40 76658628e7SLina Iyer 77658628e7SLina Iyer #define TCS_AMC_MODE_ENABLE BIT(16) 78658628e7SLina Iyer #define TCS_AMC_MODE_TRIGGER BIT(24) 79658628e7SLina Iyer 80658628e7SLina Iyer /* TCS CMD register bit mask */ 81658628e7SLina Iyer #define CMD_MSGID_LEN 8 82658628e7SLina Iyer #define CMD_MSGID_RESP_REQ BIT(8) 83658628e7SLina Iyer #define CMD_MSGID_WRITE BIT(16) 84658628e7SLina Iyer #define CMD_STATUS_ISSUED BIT(8) 85658628e7SLina Iyer #define CMD_STATUS_COMPL BIT(16) 86658628e7SLina Iyer 871f7dbeb5SDouglas Anderson /* 881f7dbeb5SDouglas Anderson * Here's a high level overview of how all the registers in RPMH work 891f7dbeb5SDouglas Anderson * together: 901f7dbeb5SDouglas Anderson * 911f7dbeb5SDouglas Anderson * - The main rpmh-rsc address is the base of a register space that can 921f7dbeb5SDouglas Anderson * be used to find overall configuration of the hardware 931f7dbeb5SDouglas Anderson * (DRV_PRNT_CHLD_CONFIG). Also found within the rpmh-rsc register 941f7dbeb5SDouglas Anderson * space are all the TCS blocks. The offset of the TCS blocks is 951f7dbeb5SDouglas Anderson * specified in the device tree by "qcom,tcs-offset" and used to 961f7dbeb5SDouglas Anderson * compute tcs_base. 971f7dbeb5SDouglas Anderson * - TCS blocks come one after another. Type, count, and order are 981f7dbeb5SDouglas Anderson * specified by the device tree as "qcom,tcs-config". 991f7dbeb5SDouglas Anderson * - Each TCS block has some registers, then space for up to 16 commands. 1001f7dbeb5SDouglas Anderson * Note that though address space is reserved for 16 commands, fewer 1011f7dbeb5SDouglas Anderson * might be present. See ncpt (num cmds per TCS). 1021f7dbeb5SDouglas Anderson * 1031f7dbeb5SDouglas Anderson * Here's a picture: 1041f7dbeb5SDouglas Anderson * 1051f7dbeb5SDouglas Anderson * +---------------------------------------------------+ 1061f7dbeb5SDouglas Anderson * |RSC | 1071f7dbeb5SDouglas Anderson * | ctrl | 1081f7dbeb5SDouglas Anderson * | | 1091f7dbeb5SDouglas Anderson * | Drvs: | 1101f7dbeb5SDouglas Anderson * | +-----------------------------------------------+ | 1111f7dbeb5SDouglas Anderson * | |DRV0 | | 1121f7dbeb5SDouglas Anderson * | | ctrl/config | | 1131f7dbeb5SDouglas Anderson * | | IRQ | | 1141f7dbeb5SDouglas Anderson * | | | | 1151f7dbeb5SDouglas Anderson * | | TCSes: | | 1161f7dbeb5SDouglas Anderson * | | +------------------------------------------+ | | 1171f7dbeb5SDouglas Anderson * | | |TCS0 | | | | | | | | | | | | | | | 1181f7dbeb5SDouglas Anderson * | | | ctrl | 0| 1| 2| 3| 4| 5| .| .| .| .|14|15| | | 1191f7dbeb5SDouglas Anderson * | | | | | | | | | | | | | | | | | | 1201f7dbeb5SDouglas Anderson * | | +------------------------------------------+ | | 1211f7dbeb5SDouglas Anderson * | | +------------------------------------------+ | | 1221f7dbeb5SDouglas Anderson * | | |TCS1 | | | | | | | | | | | | | | | 1231f7dbeb5SDouglas Anderson * | | | ctrl | 0| 1| 2| 3| 4| 5| .| .| .| .|14|15| | | 1241f7dbeb5SDouglas Anderson * | | | | | | | | | | | | | | | | | | 1251f7dbeb5SDouglas Anderson * | | +------------------------------------------+ | | 1261f7dbeb5SDouglas Anderson * | | +------------------------------------------+ | | 1271f7dbeb5SDouglas Anderson * | | |TCS2 | | | | | | | | | | | | | | | 1281f7dbeb5SDouglas Anderson * | | | ctrl | 0| 1| 2| 3| 4| 5| .| .| .| .|14|15| | | 1291f7dbeb5SDouglas Anderson * | | | | | | | | | | | | | | | | | | 1301f7dbeb5SDouglas Anderson * | | +------------------------------------------+ | | 1311f7dbeb5SDouglas Anderson * | | ...... | | 1321f7dbeb5SDouglas Anderson * | +-----------------------------------------------+ | 1331f7dbeb5SDouglas Anderson * | +-----------------------------------------------+ | 1341f7dbeb5SDouglas Anderson * | |DRV1 | | 1351f7dbeb5SDouglas Anderson * | | (same as DRV0) | | 1361f7dbeb5SDouglas Anderson * | +-----------------------------------------------+ | 1371f7dbeb5SDouglas Anderson * | ...... | 1381f7dbeb5SDouglas Anderson * +---------------------------------------------------+ 1391f7dbeb5SDouglas Anderson */ 1401f7dbeb5SDouglas Anderson 141faa0c1f1SDouglas Anderson static inline void __iomem * 142faa0c1f1SDouglas Anderson tcs_reg_addr(const struct rsc_drv *drv, int reg, int tcs_id) 143658628e7SLina Iyer { 144faa0c1f1SDouglas Anderson return drv->tcs_base + RSC_DRV_TCS_OFFSET * tcs_id + reg; 145658628e7SLina Iyer } 146658628e7SLina Iyer 147faa0c1f1SDouglas Anderson static inline void __iomem * 148faa0c1f1SDouglas Anderson tcs_cmd_addr(const struct rsc_drv *drv, int reg, int tcs_id, int cmd_id) 1493b5e3d50SDouglas Anderson { 150faa0c1f1SDouglas Anderson return tcs_reg_addr(drv, reg, tcs_id) + RSC_DRV_CMD_OFFSET * cmd_id; 1513b5e3d50SDouglas Anderson } 1523b5e3d50SDouglas Anderson 153faa0c1f1SDouglas Anderson static u32 read_tcs_cmd(const struct rsc_drv *drv, int reg, int tcs_id, 154faa0c1f1SDouglas Anderson int cmd_id) 155faa0c1f1SDouglas Anderson { 156faa0c1f1SDouglas Anderson return readl_relaxed(tcs_cmd_addr(drv, reg, tcs_id, cmd_id)); 157faa0c1f1SDouglas Anderson } 158faa0c1f1SDouglas Anderson 159faa0c1f1SDouglas Anderson static u32 read_tcs_reg(const struct rsc_drv *drv, int reg, int tcs_id) 160faa0c1f1SDouglas Anderson { 161faa0c1f1SDouglas Anderson return readl_relaxed(tcs_reg_addr(drv, reg, tcs_id)); 162faa0c1f1SDouglas Anderson } 163faa0c1f1SDouglas Anderson 164faa0c1f1SDouglas Anderson static void write_tcs_cmd(const struct rsc_drv *drv, int reg, int tcs_id, 165faa0c1f1SDouglas Anderson int cmd_id, u32 data) 166faa0c1f1SDouglas Anderson { 167faa0c1f1SDouglas Anderson writel_relaxed(data, tcs_cmd_addr(drv, reg, tcs_id, cmd_id)); 168faa0c1f1SDouglas Anderson } 169faa0c1f1SDouglas Anderson 170faa0c1f1SDouglas Anderson static void write_tcs_reg(const struct rsc_drv *drv, int reg, int tcs_id, 171658628e7SLina Iyer u32 data) 172658628e7SLina Iyer { 173faa0c1f1SDouglas Anderson writel_relaxed(data, tcs_reg_addr(drv, reg, tcs_id)); 174658628e7SLina Iyer } 175658628e7SLina Iyer 176faa0c1f1SDouglas Anderson static void write_tcs_reg_sync(const struct rsc_drv *drv, int reg, int tcs_id, 177658628e7SLina Iyer u32 data) 178658628e7SLina Iyer { 179be24c6a7SDouglas Anderson int i; 18091160150SDouglas Anderson 181faa0c1f1SDouglas Anderson writel(data, tcs_reg_addr(drv, reg, tcs_id)); 182be24c6a7SDouglas Anderson 183be24c6a7SDouglas Anderson /* 184be24c6a7SDouglas Anderson * Wait until we read back the same value. Use a counter rather than 185be24c6a7SDouglas Anderson * ktime for timeout since this may be called after timekeeping stops. 186be24c6a7SDouglas Anderson */ 187be24c6a7SDouglas Anderson for (i = 0; i < USEC_PER_SEC; i++) { 188be24c6a7SDouglas Anderson if (readl(tcs_reg_addr(drv, reg, tcs_id)) == data) 189be24c6a7SDouglas Anderson return; 190be24c6a7SDouglas Anderson udelay(1); 191be24c6a7SDouglas Anderson } 19291160150SDouglas Anderson pr_err("%s: error writing %#x to %d:%#x\n", drv->name, 19391160150SDouglas Anderson data, tcs_id, reg); 194658628e7SLina Iyer } 195658628e7SLina Iyer 196e40b0c16SDouglas Anderson /** 197e40b0c16SDouglas Anderson * tcs_is_free() - Return if a TCS is totally free. 198e40b0c16SDouglas Anderson * @drv: The RSC controller. 199e40b0c16SDouglas Anderson * @tcs_id: The global ID of this TCS. 200e40b0c16SDouglas Anderson * 201e40b0c16SDouglas Anderson * Returns true if nobody has claimed this TCS (by setting tcs_in_use). 202e40b0c16SDouglas Anderson * 203555701a4SDouglas Anderson * Context: Must be called with the drv->lock held. 204e40b0c16SDouglas Anderson * 205e40b0c16SDouglas Anderson * Return: true if the given TCS is free. 206e40b0c16SDouglas Anderson */ 207658628e7SLina Iyer static bool tcs_is_free(struct rsc_drv *drv, int tcs_id) 208658628e7SLina Iyer { 209ff304ea3SDouglas Anderson return !test_bit(tcs_id, drv->tcs_in_use); 210658628e7SLina Iyer } 211658628e7SLina Iyer 212e40b0c16SDouglas Anderson /** 213e40b0c16SDouglas Anderson * tcs_invalidate() - Invalidate all TCSes of the given type (sleep or wake). 214e40b0c16SDouglas Anderson * @drv: The RSC controller. 215e40b0c16SDouglas Anderson * @type: SLEEP_TCS or WAKE_TCS 216e40b0c16SDouglas Anderson * 217e40b0c16SDouglas Anderson * This will clear the "slots" variable of the given tcs_group and also 218e40b0c16SDouglas Anderson * tell the hardware to forget about all entries. 219e40b0c16SDouglas Anderson * 220881808d0SDouglas Anderson * The caller must ensure that no other RPMH actions are happening when this 221881808d0SDouglas Anderson * function is called, since otherwise the device may immediately become 222881808d0SDouglas Anderson * used again even before this function exits. 223e40b0c16SDouglas Anderson */ 224881808d0SDouglas Anderson static void tcs_invalidate(struct rsc_drv *drv, int type) 2259a3afcfbSLina Iyer { 2269a3afcfbSLina Iyer int m; 22753d49fe1SDouglas Anderson struct tcs_group *tcs = &drv->tcs[type]; 2289a3afcfbSLina Iyer 229881808d0SDouglas Anderson /* Caller ensures nobody else is running so no lock */ 230881808d0SDouglas Anderson if (bitmap_empty(tcs->slots, MAX_TCS_SLOTS)) 231881808d0SDouglas Anderson return; 2329a3afcfbSLina Iyer 2339a3afcfbSLina Iyer for (m = tcs->offset; m < tcs->offset + tcs->num_tcs; m++) { 2349a3afcfbSLina Iyer write_tcs_reg_sync(drv, RSC_DRV_CMD_ENABLE, m, 0); 23509e97b6cSLina Iyer write_tcs_reg_sync(drv, RSC_DRV_CMD_WAIT_FOR_CMPL, m, 0); 2369a3afcfbSLina Iyer } 2379a3afcfbSLina Iyer bitmap_zero(tcs->slots, MAX_TCS_SLOTS); 2389a3afcfbSLina Iyer } 2399a3afcfbSLina Iyer 2409a3afcfbSLina Iyer /** 241e40b0c16SDouglas Anderson * rpmh_rsc_invalidate() - Invalidate sleep and wake TCSes. 242e40b0c16SDouglas Anderson * @drv: The RSC controller. 2439a3afcfbSLina Iyer * 244881808d0SDouglas Anderson * The caller must ensure that no other RPMH actions are happening when this 245881808d0SDouglas Anderson * function is called, since otherwise the device may immediately become 246881808d0SDouglas Anderson * used again even before this function exits. 2479a3afcfbSLina Iyer */ 248881808d0SDouglas Anderson void rpmh_rsc_invalidate(struct rsc_drv *drv) 2499a3afcfbSLina Iyer { 250881808d0SDouglas Anderson tcs_invalidate(drv, SLEEP_TCS); 251881808d0SDouglas Anderson tcs_invalidate(drv, WAKE_TCS); 2529a3afcfbSLina Iyer } 2539a3afcfbSLina Iyer 254e40b0c16SDouglas Anderson /** 255e40b0c16SDouglas Anderson * get_tcs_for_msg() - Get the tcs_group used to send the given message. 256e40b0c16SDouglas Anderson * @drv: The RSC controller. 257e40b0c16SDouglas Anderson * @msg: The message we want to send. 258e40b0c16SDouglas Anderson * 259e40b0c16SDouglas Anderson * This is normally pretty straightforward except if we are trying to send 260e40b0c16SDouglas Anderson * an ACTIVE_ONLY message but don't have any active_only TCSes. 261e40b0c16SDouglas Anderson * 262e40b0c16SDouglas Anderson * Return: A pointer to a tcs_group or an ERR_PTR. 263e40b0c16SDouglas Anderson */ 264658628e7SLina Iyer static struct tcs_group *get_tcs_for_msg(struct rsc_drv *drv, 265658628e7SLina Iyer const struct tcs_request *msg) 266658628e7SLina Iyer { 26738427e5aSMaulik Shah int type; 2682de4b8d3SLina Iyer struct tcs_group *tcs; 269658628e7SLina Iyer 270658628e7SLina Iyer switch (msg->state) { 271658628e7SLina Iyer case RPMH_ACTIVE_ONLY_STATE: 272658628e7SLina Iyer type = ACTIVE_TCS; 273658628e7SLina Iyer break; 274fa460e45SLina Iyer case RPMH_WAKE_ONLY_STATE: 275fa460e45SLina Iyer type = WAKE_TCS; 276fa460e45SLina Iyer break; 277fa460e45SLina Iyer case RPMH_SLEEP_STATE: 278fa460e45SLina Iyer type = SLEEP_TCS; 279fa460e45SLina Iyer break; 280658628e7SLina Iyer default: 281658628e7SLina Iyer return ERR_PTR(-EINVAL); 282658628e7SLina Iyer } 283658628e7SLina Iyer 2842de4b8d3SLina Iyer /* 2852de4b8d3SLina Iyer * If we are making an active request on a RSC that does not have a 2862de4b8d3SLina Iyer * dedicated TCS for active state use, then re-purpose a wake TCS to 287e40b0c16SDouglas Anderson * send active votes. This is safe because we ensure any active-only 288e40b0c16SDouglas Anderson * transfers have finished before we use it (maybe by running from 289e40b0c16SDouglas Anderson * the last CPU in PM code). 2902de4b8d3SLina Iyer */ 29153d49fe1SDouglas Anderson tcs = &drv->tcs[type]; 29238427e5aSMaulik Shah if (msg->state == RPMH_ACTIVE_ONLY_STATE && !tcs->num_tcs) 29353d49fe1SDouglas Anderson tcs = &drv->tcs[WAKE_TCS]; 2942de4b8d3SLina Iyer 2952de4b8d3SLina Iyer return tcs; 296658628e7SLina Iyer } 297658628e7SLina Iyer 298e40b0c16SDouglas Anderson /** 299e40b0c16SDouglas Anderson * get_req_from_tcs() - Get a stashed request that was xfering on the given TCS. 300e40b0c16SDouglas Anderson * @drv: The RSC controller. 301e40b0c16SDouglas Anderson * @tcs_id: The global ID of this TCS. 302e40b0c16SDouglas Anderson * 303e40b0c16SDouglas Anderson * For ACTIVE_ONLY transfers we want to call back into the client when the 304e40b0c16SDouglas Anderson * transfer finishes. To do this we need the "request" that the client 305e40b0c16SDouglas Anderson * originally provided us. This function grabs the request that we stashed 306e40b0c16SDouglas Anderson * when we started the transfer. 307e40b0c16SDouglas Anderson * 308e40b0c16SDouglas Anderson * This only makes sense for ACTIVE_ONLY transfers since those are the only 309e40b0c16SDouglas Anderson * ones we track sending (the only ones we enable interrupts for and the only 310e40b0c16SDouglas Anderson * ones we call back to the client for). 311e40b0c16SDouglas Anderson * 312e40b0c16SDouglas Anderson * Return: The stashed request. 313e40b0c16SDouglas Anderson */ 314658628e7SLina Iyer static const struct tcs_request *get_req_from_tcs(struct rsc_drv *drv, 315658628e7SLina Iyer int tcs_id) 316658628e7SLina Iyer { 317658628e7SLina Iyer struct tcs_group *tcs; 318658628e7SLina Iyer int i; 319658628e7SLina Iyer 320efa1c257SRaju P.L.S.S.S.N for (i = 0; i < TCS_TYPE_NR; i++) { 321658628e7SLina Iyer tcs = &drv->tcs[i]; 322658628e7SLina Iyer if (tcs->mask & BIT(tcs_id)) 323658628e7SLina Iyer return tcs->req[tcs_id - tcs->offset]; 324658628e7SLina Iyer } 325658628e7SLina Iyer 326658628e7SLina Iyer return NULL; 327658628e7SLina Iyer } 328658628e7SLina Iyer 329e40b0c16SDouglas Anderson /** 330e40b0c16SDouglas Anderson * __tcs_set_trigger() - Start xfer on a TCS or unset trigger on a borrowed TCS 331e40b0c16SDouglas Anderson * @drv: The controller. 332e40b0c16SDouglas Anderson * @tcs_id: The global ID of this TCS. 333e40b0c16SDouglas Anderson * @trigger: If true then untrigger/retrigger. If false then just untrigger. 334e40b0c16SDouglas Anderson * 335e40b0c16SDouglas Anderson * In the normal case we only ever call with "trigger=true" to start a 336e40b0c16SDouglas Anderson * transfer. That will un-trigger/disable the TCS from the last transfer 337e40b0c16SDouglas Anderson * then trigger/enable for this transfer. 338e40b0c16SDouglas Anderson * 339e40b0c16SDouglas Anderson * If we borrowed a wake TCS for an active-only transfer we'll also call 340e40b0c16SDouglas Anderson * this function with "trigger=false" to just do the un-trigger/disable 341e40b0c16SDouglas Anderson * before using the TCS for wake purposes again. 342e40b0c16SDouglas Anderson * 343e40b0c16SDouglas Anderson * Note that the AP is only in charge of triggering active-only transfers. 344e40b0c16SDouglas Anderson * The AP never triggers sleep/wake values using this function. 345e40b0c16SDouglas Anderson */ 34615b3bf61SRaju P.L.S.S.S.N static void __tcs_set_trigger(struct rsc_drv *drv, int tcs_id, bool trigger) 34715b3bf61SRaju P.L.S.S.S.N { 34815b3bf61SRaju P.L.S.S.S.N u32 enable; 34915b3bf61SRaju P.L.S.S.S.N 35015b3bf61SRaju P.L.S.S.S.N /* 35115b3bf61SRaju P.L.S.S.S.N * HW req: Clear the DRV_CONTROL and enable TCS again 35215b3bf61SRaju P.L.S.S.S.N * While clearing ensure that the AMC mode trigger is cleared 35315b3bf61SRaju P.L.S.S.S.N * and then the mode enable is cleared. 35415b3bf61SRaju P.L.S.S.S.N */ 3553b5e3d50SDouglas Anderson enable = read_tcs_reg(drv, RSC_DRV_CONTROL, tcs_id); 35615b3bf61SRaju P.L.S.S.S.N enable &= ~TCS_AMC_MODE_TRIGGER; 35715b3bf61SRaju P.L.S.S.S.N write_tcs_reg_sync(drv, RSC_DRV_CONTROL, tcs_id, enable); 35815b3bf61SRaju P.L.S.S.S.N enable &= ~TCS_AMC_MODE_ENABLE; 35915b3bf61SRaju P.L.S.S.S.N write_tcs_reg_sync(drv, RSC_DRV_CONTROL, tcs_id, enable); 36015b3bf61SRaju P.L.S.S.S.N 36115b3bf61SRaju P.L.S.S.S.N if (trigger) { 36215b3bf61SRaju P.L.S.S.S.N /* Enable the AMC mode on the TCS and then trigger the TCS */ 36315b3bf61SRaju P.L.S.S.S.N enable = TCS_AMC_MODE_ENABLE; 36415b3bf61SRaju P.L.S.S.S.N write_tcs_reg_sync(drv, RSC_DRV_CONTROL, tcs_id, enable); 36515b3bf61SRaju P.L.S.S.S.N enable |= TCS_AMC_MODE_TRIGGER; 36615b3bf61SRaju P.L.S.S.S.N write_tcs_reg_sync(drv, RSC_DRV_CONTROL, tcs_id, enable); 36715b3bf61SRaju P.L.S.S.S.N } 36815b3bf61SRaju P.L.S.S.S.N } 36915b3bf61SRaju P.L.S.S.S.N 370e40b0c16SDouglas Anderson /** 371e40b0c16SDouglas Anderson * enable_tcs_irq() - Enable or disable interrupts on the given TCS. 372e40b0c16SDouglas Anderson * @drv: The controller. 373e40b0c16SDouglas Anderson * @tcs_id: The global ID of this TCS. 374e40b0c16SDouglas Anderson * @enable: If true then enable; if false then disable 375e40b0c16SDouglas Anderson * 376e40b0c16SDouglas Anderson * We only ever call this when we borrow a wake TCS for an active-only 377e40b0c16SDouglas Anderson * transfer. For active-only TCSes interrupts are always left enabled. 378e40b0c16SDouglas Anderson */ 37915b3bf61SRaju P.L.S.S.S.N static void enable_tcs_irq(struct rsc_drv *drv, int tcs_id, bool enable) 38015b3bf61SRaju P.L.S.S.S.N { 38115b3bf61SRaju P.L.S.S.S.N u32 data; 38215b3bf61SRaju P.L.S.S.S.N 383032c692aSDouglas Anderson data = readl_relaxed(drv->tcs_base + RSC_DRV_IRQ_ENABLE); 38415b3bf61SRaju P.L.S.S.S.N if (enable) 38515b3bf61SRaju P.L.S.S.S.N data |= BIT(tcs_id); 38615b3bf61SRaju P.L.S.S.S.N else 38715b3bf61SRaju P.L.S.S.S.N data &= ~BIT(tcs_id); 388032c692aSDouglas Anderson writel_relaxed(data, drv->tcs_base + RSC_DRV_IRQ_ENABLE); 38915b3bf61SRaju P.L.S.S.S.N } 39015b3bf61SRaju P.L.S.S.S.N 391658628e7SLina Iyer /** 392e40b0c16SDouglas Anderson * tcs_tx_done() - TX Done interrupt handler. 393e40b0c16SDouglas Anderson * @irq: The IRQ number (ignored). 394e40b0c16SDouglas Anderson * @p: Pointer to "struct rsc_drv". 395e40b0c16SDouglas Anderson * 396e40b0c16SDouglas Anderson * Called for ACTIVE_ONLY transfers (those are the only ones we enable the 397e40b0c16SDouglas Anderson * IRQ for) when a transfer is done. 398e40b0c16SDouglas Anderson * 399e40b0c16SDouglas Anderson * Return: IRQ_HANDLED 400658628e7SLina Iyer */ 401658628e7SLina Iyer static irqreturn_t tcs_tx_done(int irq, void *p) 402658628e7SLina Iyer { 403658628e7SLina Iyer struct rsc_drv *drv = p; 404c1038456SLina Iyer int i, j, err = 0; 405658628e7SLina Iyer unsigned long irq_status; 406658628e7SLina Iyer const struct tcs_request *req; 407658628e7SLina Iyer struct tcs_cmd *cmd; 408658628e7SLina Iyer 409032c692aSDouglas Anderson irq_status = readl_relaxed(drv->tcs_base + RSC_DRV_IRQ_STATUS); 410658628e7SLina Iyer 411658628e7SLina Iyer for_each_set_bit(i, &irq_status, BITS_PER_LONG) { 412658628e7SLina Iyer req = get_req_from_tcs(drv, i); 413658628e7SLina Iyer if (!req) { 414658628e7SLina Iyer WARN_ON(1); 415658628e7SLina Iyer goto skip; 416658628e7SLina Iyer } 417658628e7SLina Iyer 418fc087fe5SLina Iyer err = 0; 419658628e7SLina Iyer for (j = 0; j < req->num_cmds; j++) { 420658628e7SLina Iyer u32 sts; 421658628e7SLina Iyer 422658628e7SLina Iyer cmd = &req->cmds[j]; 4233b5e3d50SDouglas Anderson sts = read_tcs_cmd(drv, RSC_DRV_CMD_STATUS, i, j); 424658628e7SLina Iyer if (!(sts & CMD_STATUS_ISSUED) || 425658628e7SLina Iyer ((req->wait_for_compl || cmd->wait) && 426658628e7SLina Iyer !(sts & CMD_STATUS_COMPL))) { 427658628e7SLina Iyer pr_err("Incomplete request: %s: addr=%#x data=%#x", 428658628e7SLina Iyer drv->name, cmd->addr, cmd->data); 429fc087fe5SLina Iyer err = -EIO; 430658628e7SLina Iyer } 431658628e7SLina Iyer } 432fc087fe5SLina Iyer 433fc087fe5SLina Iyer trace_rpmh_tx_done(drv, i, req, err); 43415b3bf61SRaju P.L.S.S.S.N 43515b3bf61SRaju P.L.S.S.S.N /* 43615b3bf61SRaju P.L.S.S.S.N * If wake tcs was re-purposed for sending active 43715b3bf61SRaju P.L.S.S.S.N * votes, clear AMC trigger & enable modes and 43815b3bf61SRaju P.L.S.S.S.N * disable interrupt for this TCS 43915b3bf61SRaju P.L.S.S.S.N */ 44015b3bf61SRaju P.L.S.S.S.N if (!drv->tcs[ACTIVE_TCS].num_tcs) 44115b3bf61SRaju P.L.S.S.S.N __tcs_set_trigger(drv, i, false); 442658628e7SLina Iyer skip: 443658628e7SLina Iyer /* Reclaim the TCS */ 444658628e7SLina Iyer write_tcs_reg(drv, RSC_DRV_CMD_ENABLE, i, 0); 44509e97b6cSLina Iyer write_tcs_reg(drv, RSC_DRV_CMD_WAIT_FOR_CMPL, i, 0); 446032c692aSDouglas Anderson writel_relaxed(BIT(i), drv->tcs_base + RSC_DRV_IRQ_CLEAR); 447658628e7SLina Iyer spin_lock(&drv->lock); 448658628e7SLina Iyer clear_bit(i, drv->tcs_in_use); 44915b3bf61SRaju P.L.S.S.S.N /* 45015b3bf61SRaju P.L.S.S.S.N * Disable interrupt for WAKE TCS to avoid being 45115b3bf61SRaju P.L.S.S.S.N * spammed with interrupts coming when the solver 45215b3bf61SRaju P.L.S.S.S.N * sends its wake votes. 45315b3bf61SRaju P.L.S.S.S.N */ 45415b3bf61SRaju P.L.S.S.S.N if (!drv->tcs[ACTIVE_TCS].num_tcs) 45515b3bf61SRaju P.L.S.S.S.N enable_tcs_irq(drv, i, false); 456658628e7SLina Iyer spin_unlock(&drv->lock); 4572bc20f3cSStephen Boyd wake_up(&drv->tcs_wait); 458c1038456SLina Iyer if (req) 459c1038456SLina Iyer rpmh_tx_done(req, err); 460658628e7SLina Iyer } 461658628e7SLina Iyer 462658628e7SLina Iyer return IRQ_HANDLED; 463658628e7SLina Iyer } 464658628e7SLina Iyer 465e40b0c16SDouglas Anderson /** 466e40b0c16SDouglas Anderson * __tcs_buffer_write() - Write to TCS hardware from a request; don't trigger. 467e40b0c16SDouglas Anderson * @drv: The controller. 468e40b0c16SDouglas Anderson * @tcs_id: The global ID of this TCS. 469e40b0c16SDouglas Anderson * @cmd_id: The index within the TCS to start writing. 470e40b0c16SDouglas Anderson * @msg: The message we want to send, which will contain several addr/data 471e40b0c16SDouglas Anderson * pairs to program (but few enough that they all fit in one TCS). 472e40b0c16SDouglas Anderson * 473e40b0c16SDouglas Anderson * This is used for all types of transfers (active, sleep, and wake). 474e40b0c16SDouglas Anderson */ 475658628e7SLina Iyer static void __tcs_buffer_write(struct rsc_drv *drv, int tcs_id, int cmd_id, 476658628e7SLina Iyer const struct tcs_request *msg) 477658628e7SLina Iyer { 478658628e7SLina Iyer u32 msgid, cmd_msgid; 479658628e7SLina Iyer u32 cmd_enable = 0; 480658628e7SLina Iyer u32 cmd_complete; 481658628e7SLina Iyer struct tcs_cmd *cmd; 482658628e7SLina Iyer int i, j; 483658628e7SLina Iyer 484658628e7SLina Iyer cmd_msgid = CMD_MSGID_LEN; 485658628e7SLina Iyer cmd_msgid |= msg->wait_for_compl ? CMD_MSGID_RESP_REQ : 0; 486658628e7SLina Iyer cmd_msgid |= CMD_MSGID_WRITE; 487658628e7SLina Iyer 4883b5e3d50SDouglas Anderson cmd_complete = read_tcs_reg(drv, RSC_DRV_CMD_WAIT_FOR_CMPL, tcs_id); 489658628e7SLina Iyer 490658628e7SLina Iyer for (i = 0, j = cmd_id; i < msg->num_cmds; i++, j++) { 491658628e7SLina Iyer cmd = &msg->cmds[i]; 492658628e7SLina Iyer cmd_enable |= BIT(j); 493658628e7SLina Iyer cmd_complete |= cmd->wait << j; 494658628e7SLina Iyer msgid = cmd_msgid; 495658628e7SLina Iyer msgid |= cmd->wait ? CMD_MSGID_RESP_REQ : 0; 496fc087fe5SLina Iyer 497658628e7SLina Iyer write_tcs_cmd(drv, RSC_DRV_CMD_MSGID, tcs_id, j, msgid); 498658628e7SLina Iyer write_tcs_cmd(drv, RSC_DRV_CMD_ADDR, tcs_id, j, cmd->addr); 499658628e7SLina Iyer write_tcs_cmd(drv, RSC_DRV_CMD_DATA, tcs_id, j, cmd->data); 500*7bb7a83fSMaulik Shah trace_rpmh_send_msg(drv, tcs_id, j, msgid, cmd); 501658628e7SLina Iyer } 502658628e7SLina Iyer 503658628e7SLina Iyer write_tcs_reg(drv, RSC_DRV_CMD_WAIT_FOR_CMPL, tcs_id, cmd_complete); 5043b5e3d50SDouglas Anderson cmd_enable |= read_tcs_reg(drv, RSC_DRV_CMD_ENABLE, tcs_id); 505658628e7SLina Iyer write_tcs_reg(drv, RSC_DRV_CMD_ENABLE, tcs_id, cmd_enable); 506658628e7SLina Iyer } 507658628e7SLina Iyer 508e40b0c16SDouglas Anderson /** 509e40b0c16SDouglas Anderson * check_for_req_inflight() - Look to see if conflicting cmds are in flight. 510e40b0c16SDouglas Anderson * @drv: The controller. 511e40b0c16SDouglas Anderson * @tcs: A pointer to the tcs_group used for ACTIVE_ONLY transfers. 512e40b0c16SDouglas Anderson * @msg: The message we want to send, which will contain several addr/data 513e40b0c16SDouglas Anderson * pairs to program (but few enough that they all fit in one TCS). 514e40b0c16SDouglas Anderson * 515e40b0c16SDouglas Anderson * This will walk through the TCSes in the group and check if any of them 516e40b0c16SDouglas Anderson * appear to be sending to addresses referenced in the message. If it finds 517e40b0c16SDouglas Anderson * one it'll return -EBUSY. 518e40b0c16SDouglas Anderson * 519e40b0c16SDouglas Anderson * Only for use for active-only transfers. 520e40b0c16SDouglas Anderson * 521e40b0c16SDouglas Anderson * Must be called with the drv->lock held since that protects tcs_in_use. 522e40b0c16SDouglas Anderson * 523e40b0c16SDouglas Anderson * Return: 0 if nothing in flight or -EBUSY if we should try again later. 524e40b0c16SDouglas Anderson * The caller must re-enable interrupts between tries since that's 525e40b0c16SDouglas Anderson * the only way tcs_is_free() will ever return true and the only way 526e40b0c16SDouglas Anderson * RSC_DRV_CMD_ENABLE will ever be cleared. 527e40b0c16SDouglas Anderson */ 528658628e7SLina Iyer static int check_for_req_inflight(struct rsc_drv *drv, struct tcs_group *tcs, 529658628e7SLina Iyer const struct tcs_request *msg) 530658628e7SLina Iyer { 531658628e7SLina Iyer unsigned long curr_enabled; 532658628e7SLina Iyer u32 addr; 533658628e7SLina Iyer int i, j, k; 534658628e7SLina Iyer int tcs_id = tcs->offset; 535658628e7SLina Iyer 536658628e7SLina Iyer for (i = 0; i < tcs->num_tcs; i++, tcs_id++) { 537658628e7SLina Iyer if (tcs_is_free(drv, tcs_id)) 538658628e7SLina Iyer continue; 539658628e7SLina Iyer 5403b5e3d50SDouglas Anderson curr_enabled = read_tcs_reg(drv, RSC_DRV_CMD_ENABLE, tcs_id); 541658628e7SLina Iyer 542658628e7SLina Iyer for_each_set_bit(j, &curr_enabled, MAX_CMDS_PER_TCS) { 5433b5e3d50SDouglas Anderson addr = read_tcs_cmd(drv, RSC_DRV_CMD_ADDR, tcs_id, j); 544658628e7SLina Iyer for (k = 0; k < msg->num_cmds; k++) { 545658628e7SLina Iyer if (addr == msg->cmds[k].addr) 546658628e7SLina Iyer return -EBUSY; 547658628e7SLina Iyer } 548658628e7SLina Iyer } 549658628e7SLina Iyer } 550658628e7SLina Iyer 551658628e7SLina Iyer return 0; 552658628e7SLina Iyer } 553658628e7SLina Iyer 554e40b0c16SDouglas Anderson /** 555e40b0c16SDouglas Anderson * find_free_tcs() - Find free tcs in the given tcs_group; only for active. 556e40b0c16SDouglas Anderson * @tcs: A pointer to the active-only tcs_group (or the wake tcs_group if 557e40b0c16SDouglas Anderson * we borrowed it because there are zero active-only ones). 558e40b0c16SDouglas Anderson * 559e40b0c16SDouglas Anderson * Must be called with the drv->lock held since that protects tcs_in_use. 560e40b0c16SDouglas Anderson * 561e40b0c16SDouglas Anderson * Return: The first tcs that's free. 562e40b0c16SDouglas Anderson */ 563658628e7SLina Iyer static int find_free_tcs(struct tcs_group *tcs) 564658628e7SLina Iyer { 565658628e7SLina Iyer int i; 566658628e7SLina Iyer 567658628e7SLina Iyer for (i = 0; i < tcs->num_tcs; i++) { 568658628e7SLina Iyer if (tcs_is_free(tcs->drv, tcs->offset + i)) 569658628e7SLina Iyer return tcs->offset + i; 570658628e7SLina Iyer } 571658628e7SLina Iyer 572658628e7SLina Iyer return -EBUSY; 573658628e7SLina Iyer } 574658628e7SLina Iyer 575e40b0c16SDouglas Anderson /** 5762bc20f3cSStephen Boyd * claim_tcs_for_req() - Claim a tcs in the given tcs_group; only for active. 577e40b0c16SDouglas Anderson * @drv: The controller. 5782bc20f3cSStephen Boyd * @tcs: The tcs_group used for ACTIVE_ONLY transfers. 579e40b0c16SDouglas Anderson * @msg: The data to be sent. 580e40b0c16SDouglas Anderson * 5812bc20f3cSStephen Boyd * Claims a tcs in the given tcs_group while making sure that no existing cmd 5822bc20f3cSStephen Boyd * is in flight that would conflict with the one in @msg. 583e40b0c16SDouglas Anderson * 5842bc20f3cSStephen Boyd * Context: Must be called with the drv->lock held since that protects 5852bc20f3cSStephen Boyd * tcs_in_use. 586e40b0c16SDouglas Anderson * 5872bc20f3cSStephen Boyd * Return: The id of the claimed tcs or -EBUSY if a matching msg is in flight 5882bc20f3cSStephen Boyd * or the tcs_group is full. 589e40b0c16SDouglas Anderson */ 5902bc20f3cSStephen Boyd static int claim_tcs_for_req(struct rsc_drv *drv, struct tcs_group *tcs, 5912bc20f3cSStephen Boyd const struct tcs_request *msg) 592658628e7SLina Iyer { 593658628e7SLina Iyer int ret; 594658628e7SLina Iyer 595658628e7SLina Iyer /* 596658628e7SLina Iyer * The h/w does not like if we send a request to the same address, 597658628e7SLina Iyer * when one is already in-flight or being processed. 598658628e7SLina Iyer */ 599658628e7SLina Iyer ret = check_for_req_inflight(drv, tcs, msg); 600555701a4SDouglas Anderson if (ret) 6012bc20f3cSStephen Boyd return ret; 602658628e7SLina Iyer 6032bc20f3cSStephen Boyd return find_free_tcs(tcs); 6042bc20f3cSStephen Boyd } 6052bc20f3cSStephen Boyd 6062bc20f3cSStephen Boyd /** 6072bc20f3cSStephen Boyd * rpmh_rsc_send_data() - Write / trigger active-only message. 6082bc20f3cSStephen Boyd * @drv: The controller. 6092bc20f3cSStephen Boyd * @msg: The data to be sent. 6102bc20f3cSStephen Boyd * 6112bc20f3cSStephen Boyd * NOTES: 6122bc20f3cSStephen Boyd * - This is only used for "ACTIVE_ONLY" since the limitations of this 6132bc20f3cSStephen Boyd * function don't make sense for sleep/wake cases. 6142bc20f3cSStephen Boyd * - To do the transfer, we will grab a whole TCS for ourselves--we don't 6152bc20f3cSStephen Boyd * try to share. If there are none available we'll wait indefinitely 6162bc20f3cSStephen Boyd * for a free one. 6172bc20f3cSStephen Boyd * - This function will not wait for the commands to be finished, only for 6182bc20f3cSStephen Boyd * data to be programmed into the RPMh. See rpmh_tx_done() which will 6192bc20f3cSStephen Boyd * be called when the transfer is fully complete. 6202bc20f3cSStephen Boyd * - This function must be called with interrupts enabled. If the hardware 6212bc20f3cSStephen Boyd * is busy doing someone else's transfer we need that transfer to fully 6222bc20f3cSStephen Boyd * finish so that we can have the hardware, and to fully finish it needs 6232bc20f3cSStephen Boyd * the interrupt handler to run. If the interrupts is set to run on the 6242bc20f3cSStephen Boyd * active CPU this can never happen if interrupts are disabled. 6252bc20f3cSStephen Boyd * 6262bc20f3cSStephen Boyd * Return: 0 on success, -EINVAL on error. 6272bc20f3cSStephen Boyd */ 6282bc20f3cSStephen Boyd int rpmh_rsc_send_data(struct rsc_drv *drv, const struct tcs_request *msg) 6292bc20f3cSStephen Boyd { 6302bc20f3cSStephen Boyd struct tcs_group *tcs; 6312bc20f3cSStephen Boyd int tcs_id; 6322bc20f3cSStephen Boyd unsigned long flags; 6332bc20f3cSStephen Boyd 6342bc20f3cSStephen Boyd tcs = get_tcs_for_msg(drv, msg); 6352bc20f3cSStephen Boyd if (IS_ERR(tcs)) 6362bc20f3cSStephen Boyd return PTR_ERR(tcs); 6372bc20f3cSStephen Boyd 6382bc20f3cSStephen Boyd spin_lock_irqsave(&drv->lock, flags); 6392bc20f3cSStephen Boyd 6402bc20f3cSStephen Boyd /* Wait forever for a free tcs. It better be there eventually! */ 6412bc20f3cSStephen Boyd wait_event_lock_irq(drv->tcs_wait, 6422bc20f3cSStephen Boyd (tcs_id = claim_tcs_for_req(drv, tcs, msg)) >= 0, 6432bc20f3cSStephen Boyd drv->lock); 644658628e7SLina Iyer 645658628e7SLina Iyer tcs->req[tcs_id - tcs->offset] = msg; 646658628e7SLina Iyer set_bit(tcs_id, drv->tcs_in_use); 64738427e5aSMaulik Shah if (msg->state == RPMH_ACTIVE_ONLY_STATE && tcs->type != ACTIVE_TCS) { 64838427e5aSMaulik Shah /* 64938427e5aSMaulik Shah * Clear previously programmed WAKE commands in selected 65038427e5aSMaulik Shah * repurposed TCS to avoid triggering them. tcs->slots will be 65138427e5aSMaulik Shah * cleaned from rpmh_flush() by invoking rpmh_rsc_invalidate() 65238427e5aSMaulik Shah */ 65338427e5aSMaulik Shah write_tcs_reg_sync(drv, RSC_DRV_CMD_ENABLE, tcs_id, 0); 65438427e5aSMaulik Shah write_tcs_reg_sync(drv, RSC_DRV_CMD_WAIT_FOR_CMPL, tcs_id, 0); 65515b3bf61SRaju P.L.S.S.S.N enable_tcs_irq(drv, tcs_id, true); 65638427e5aSMaulik Shah } 657555701a4SDouglas Anderson spin_unlock_irqrestore(&drv->lock, flags); 658658628e7SLina Iyer 659555701a4SDouglas Anderson /* 660555701a4SDouglas Anderson * These two can be done after the lock is released because: 661555701a4SDouglas Anderson * - We marked "tcs_in_use" under lock. 662555701a4SDouglas Anderson * - Once "tcs_in_use" has been marked nobody else could be writing 663555701a4SDouglas Anderson * to these registers until the interrupt goes off. 664555701a4SDouglas Anderson * - The interrupt can't go off until we trigger w/ the last line 665555701a4SDouglas Anderson * of __tcs_set_trigger() below. 666555701a4SDouglas Anderson */ 667658628e7SLina Iyer __tcs_buffer_write(drv, tcs_id, 0, msg); 66815b3bf61SRaju P.L.S.S.S.N __tcs_set_trigger(drv, tcs_id, true); 669658628e7SLina Iyer 670555701a4SDouglas Anderson return 0; 671658628e7SLina Iyer } 672658628e7SLina Iyer 673e40b0c16SDouglas Anderson /** 674e40b0c16SDouglas Anderson * find_slots() - Find a place to write the given message. 675e40b0c16SDouglas Anderson * @tcs: The tcs group to search. 676e40b0c16SDouglas Anderson * @msg: The message we want to find room for. 677e40b0c16SDouglas Anderson * @tcs_id: If we return 0 from the function, we return the global ID of the 678e40b0c16SDouglas Anderson * TCS to write to here. 679e40b0c16SDouglas Anderson * @cmd_id: If we return 0 from the function, we return the index of 680e40b0c16SDouglas Anderson * the command array of the returned TCS where the client should 681e40b0c16SDouglas Anderson * start writing the message. 682e40b0c16SDouglas Anderson * 683e40b0c16SDouglas Anderson * Only for use on sleep/wake TCSes since those are the only ones we maintain 684e40b0c16SDouglas Anderson * tcs->slots for. 685e40b0c16SDouglas Anderson * 686e40b0c16SDouglas Anderson * Return: -ENOMEM if there was no room, else 0. 687e40b0c16SDouglas Anderson */ 688fa460e45SLina Iyer static int find_slots(struct tcs_group *tcs, const struct tcs_request *msg, 689fa460e45SLina Iyer int *tcs_id, int *cmd_id) 690fa460e45SLina Iyer { 691fa460e45SLina Iyer int slot, offset; 692fa460e45SLina Iyer int i = 0; 693fa460e45SLina Iyer 694e40b0c16SDouglas Anderson /* Do over, until we can fit the full payload in a single TCS */ 695fa460e45SLina Iyer do { 696fa460e45SLina Iyer slot = bitmap_find_next_zero_area(tcs->slots, MAX_TCS_SLOTS, 697fa460e45SLina Iyer i, msg->num_cmds, 0); 698bbeac60fSMaulik Shah if (slot >= tcs->num_tcs * tcs->ncpt) 699fa460e45SLina Iyer return -ENOMEM; 700fa460e45SLina Iyer i += tcs->ncpt; 701fa460e45SLina Iyer } while (slot + msg->num_cmds - 1 >= i); 702fa460e45SLina Iyer 703fa460e45SLina Iyer bitmap_set(tcs->slots, slot, msg->num_cmds); 704fa460e45SLina Iyer 705fa460e45SLina Iyer offset = slot / tcs->ncpt; 706fa460e45SLina Iyer *tcs_id = offset + tcs->offset; 707fa460e45SLina Iyer *cmd_id = slot % tcs->ncpt; 708fa460e45SLina Iyer 709fa460e45SLina Iyer return 0; 710fa460e45SLina Iyer } 711fa460e45SLina Iyer 712427ef4f7SDouglas Anderson /** 713e40b0c16SDouglas Anderson * rpmh_rsc_write_ctrl_data() - Write request to controller but don't trigger. 714e40b0c16SDouglas Anderson * @drv: The controller. 715e40b0c16SDouglas Anderson * @msg: The data to be written to the controller. 716427ef4f7SDouglas Anderson * 717e40b0c16SDouglas Anderson * This should only be called for for sleep/wake state, never active-only 718e40b0c16SDouglas Anderson * state. 719427ef4f7SDouglas Anderson * 720555701a4SDouglas Anderson * The caller must ensure that no other RPMH actions are happening and the 721555701a4SDouglas Anderson * controller is idle when this function is called since it runs lockless. 722555701a4SDouglas Anderson * 723e40b0c16SDouglas Anderson * Return: 0 if no error; else -error. 724427ef4f7SDouglas Anderson */ 725427ef4f7SDouglas Anderson int rpmh_rsc_write_ctrl_data(struct rsc_drv *drv, const struct tcs_request *msg) 726fa460e45SLina Iyer { 727fa460e45SLina Iyer struct tcs_group *tcs; 728fa460e45SLina Iyer int tcs_id = 0, cmd_id = 0; 729fa460e45SLina Iyer int ret; 730fa460e45SLina Iyer 731fa460e45SLina Iyer tcs = get_tcs_for_msg(drv, msg); 732fa460e45SLina Iyer if (IS_ERR(tcs)) 733fa460e45SLina Iyer return PTR_ERR(tcs); 734fa460e45SLina Iyer 735fa460e45SLina Iyer /* find the TCS id and the command in the TCS to write to */ 736fa460e45SLina Iyer ret = find_slots(tcs, msg, &tcs_id, &cmd_id); 737fa460e45SLina Iyer if (!ret) 738fa460e45SLina Iyer __tcs_buffer_write(drv, tcs_id, cmd_id, msg); 739fa460e45SLina Iyer 740fa460e45SLina Iyer return ret; 741fa460e45SLina Iyer } 742fa460e45SLina Iyer 743fa460e45SLina Iyer /** 744985427f9SMaulik Shah * rpmh_rsc_ctrlr_is_busy() - Check if any of the AMCs are busy. 745985427f9SMaulik Shah * @drv: The controller 746985427f9SMaulik Shah * 747985427f9SMaulik Shah * Checks if any of the AMCs are busy in handling ACTIVE sets. 748985427f9SMaulik Shah * This is called from the last cpu powering down before flushing 749985427f9SMaulik Shah * SLEEP and WAKE sets. If AMCs are busy, controller can not enter 750985427f9SMaulik Shah * power collapse, so deny from the last cpu's pm notification. 751985427f9SMaulik Shah * 752d2a8cfc6SDouglas Anderson * Context: Must be called with the drv->lock held. 753d2a8cfc6SDouglas Anderson * 754985427f9SMaulik Shah * Return: 755985427f9SMaulik Shah * * False - AMCs are idle 756985427f9SMaulik Shah * * True - AMCs are busy 757985427f9SMaulik Shah */ 758985427f9SMaulik Shah static bool rpmh_rsc_ctrlr_is_busy(struct rsc_drv *drv) 759985427f9SMaulik Shah { 760985427f9SMaulik Shah int m; 76153d49fe1SDouglas Anderson struct tcs_group *tcs = &drv->tcs[ACTIVE_TCS]; 762985427f9SMaulik Shah 763985427f9SMaulik Shah /* 764985427f9SMaulik Shah * If we made an active request on a RSC that does not have a 765985427f9SMaulik Shah * dedicated TCS for active state use, then re-purposed wake TCSes 766985427f9SMaulik Shah * should be checked for not busy, because we used wake TCSes for 767985427f9SMaulik Shah * active requests in this case. 768985427f9SMaulik Shah */ 769985427f9SMaulik Shah if (!tcs->num_tcs) 77053d49fe1SDouglas Anderson tcs = &drv->tcs[WAKE_TCS]; 771985427f9SMaulik Shah 772985427f9SMaulik Shah for (m = tcs->offset; m < tcs->offset + tcs->num_tcs; m++) { 773985427f9SMaulik Shah if (!tcs_is_free(drv, m)) 774985427f9SMaulik Shah return true; 775985427f9SMaulik Shah } 776985427f9SMaulik Shah 777985427f9SMaulik Shah return false; 778985427f9SMaulik Shah } 779985427f9SMaulik Shah 780e40b0c16SDouglas Anderson /** 781e40b0c16SDouglas Anderson * rpmh_rsc_cpu_pm_callback() - Check if any of the AMCs are busy. 782e40b0c16SDouglas Anderson * @nfb: Pointer to the notifier block in struct rsc_drv. 783e40b0c16SDouglas Anderson * @action: CPU_PM_ENTER, CPU_PM_ENTER_FAILED, or CPU_PM_EXIT. 784e40b0c16SDouglas Anderson * @v: Unused 785e40b0c16SDouglas Anderson * 786e40b0c16SDouglas Anderson * This function is given to cpu_pm_register_notifier so we can be informed 787e40b0c16SDouglas Anderson * about when CPUs go down. When all CPUs go down we know no more active 788e40b0c16SDouglas Anderson * transfers will be started so we write sleep/wake sets. This function gets 789e40b0c16SDouglas Anderson * called from cpuidle code paths and also at system suspend time. 790e40b0c16SDouglas Anderson * 791e40b0c16SDouglas Anderson * If its last CPU going down and AMCs are not busy then writes cached sleep 792e40b0c16SDouglas Anderson * and wake messages to TCSes. The firmware then takes care of triggering 793e40b0c16SDouglas Anderson * them when entering deepest low power modes. 794e40b0c16SDouglas Anderson * 795e40b0c16SDouglas Anderson * Return: See cpu_pm_register_notifier() 796e40b0c16SDouglas Anderson */ 797985427f9SMaulik Shah static int rpmh_rsc_cpu_pm_callback(struct notifier_block *nfb, 798985427f9SMaulik Shah unsigned long action, void *v) 799985427f9SMaulik Shah { 800985427f9SMaulik Shah struct rsc_drv *drv = container_of(nfb, struct rsc_drv, rsc_pm); 801985427f9SMaulik Shah int ret = NOTIFY_OK; 802d2a8cfc6SDouglas Anderson int cpus_in_pm; 803985427f9SMaulik Shah 804985427f9SMaulik Shah switch (action) { 805985427f9SMaulik Shah case CPU_PM_ENTER: 806d2a8cfc6SDouglas Anderson cpus_in_pm = atomic_inc_return(&drv->cpus_in_pm); 807d2a8cfc6SDouglas Anderson /* 808d2a8cfc6SDouglas Anderson * NOTE: comments for num_online_cpus() point out that it's 809d2a8cfc6SDouglas Anderson * only a snapshot so we need to be careful. It should be OK 810d2a8cfc6SDouglas Anderson * for us to use, though. It's important for us not to miss 811d2a8cfc6SDouglas Anderson * if we're the last CPU going down so it would only be a 812d2a8cfc6SDouglas Anderson * problem if a CPU went offline right after we did the check 813d2a8cfc6SDouglas Anderson * AND that CPU was not idle AND that CPU was the last non-idle 814d2a8cfc6SDouglas Anderson * CPU. That can't happen. CPUs would have to come out of idle 815d2a8cfc6SDouglas Anderson * before the CPU could go offline. 816d2a8cfc6SDouglas Anderson */ 817d2a8cfc6SDouglas Anderson if (cpus_in_pm < num_online_cpus()) 818d2a8cfc6SDouglas Anderson return NOTIFY_OK; 819985427f9SMaulik Shah break; 820985427f9SMaulik Shah case CPU_PM_ENTER_FAILED: 821985427f9SMaulik Shah case CPU_PM_EXIT: 822d2a8cfc6SDouglas Anderson atomic_dec(&drv->cpus_in_pm); 823d2a8cfc6SDouglas Anderson return NOTIFY_OK; 8241143c366SDouglas Anderson default: 825d2a8cfc6SDouglas Anderson return NOTIFY_DONE; 826985427f9SMaulik Shah } 827985427f9SMaulik Shah 828d2a8cfc6SDouglas Anderson /* 829d2a8cfc6SDouglas Anderson * It's likely we're on the last CPU. Grab the drv->lock and write 830d2a8cfc6SDouglas Anderson * out the sleep/wake commands to RPMH hardware. Grabbing the lock 831d2a8cfc6SDouglas Anderson * means that if we race with another CPU coming up we are still 832d2a8cfc6SDouglas Anderson * guaranteed to be safe. If another CPU came up just after we checked 833d2a8cfc6SDouglas Anderson * and has grabbed the lock or started an active transfer then we'll 834d2a8cfc6SDouglas Anderson * notice we're busy and abort. If another CPU comes up after we start 835d2a8cfc6SDouglas Anderson * flushing it will be blocked from starting an active transfer until 836d2a8cfc6SDouglas Anderson * we're done flushing. If another CPU starts an active transfer after 837d2a8cfc6SDouglas Anderson * we release the lock we're still OK because we're no longer the last 838d2a8cfc6SDouglas Anderson * CPU. 839d2a8cfc6SDouglas Anderson */ 840d2a8cfc6SDouglas Anderson if (spin_trylock(&drv->lock)) { 841d2a8cfc6SDouglas Anderson if (rpmh_rsc_ctrlr_is_busy(drv) || rpmh_flush(&drv->client)) 842985427f9SMaulik Shah ret = NOTIFY_BAD; 843d2a8cfc6SDouglas Anderson spin_unlock(&drv->lock); 844d2a8cfc6SDouglas Anderson } else { 845d2a8cfc6SDouglas Anderson /* Another CPU must be up */ 846d2a8cfc6SDouglas Anderson return NOTIFY_OK; 847985427f9SMaulik Shah } 848985427f9SMaulik Shah 849d2a8cfc6SDouglas Anderson if (ret == NOTIFY_BAD) { 850d2a8cfc6SDouglas Anderson /* Double-check if we're here because someone else is up */ 851d2a8cfc6SDouglas Anderson if (cpus_in_pm < num_online_cpus()) 852985427f9SMaulik Shah ret = NOTIFY_OK; 853d2a8cfc6SDouglas Anderson else 854c45def5dSDouglas Anderson /* We won't be called w/ CPU_PM_ENTER_FAILED */ 855d2a8cfc6SDouglas Anderson atomic_dec(&drv->cpus_in_pm); 856d2a8cfc6SDouglas Anderson } 857c45def5dSDouglas Anderson 858985427f9SMaulik Shah return ret; 859985427f9SMaulik Shah } 860985427f9SMaulik Shah 861658628e7SLina Iyer static int rpmh_probe_tcs_config(struct platform_device *pdev, 862985427f9SMaulik Shah struct rsc_drv *drv, void __iomem *base) 863658628e7SLina Iyer { 864658628e7SLina Iyer struct tcs_type_config { 865658628e7SLina Iyer u32 type; 866658628e7SLina Iyer u32 n; 867658628e7SLina Iyer } tcs_cfg[TCS_TYPE_NR] = { { 0 } }; 868658628e7SLina Iyer struct device_node *dn = pdev->dev.of_node; 869658628e7SLina Iyer u32 config, max_tcs, ncpt, offset; 870658628e7SLina Iyer int i, ret, n, st = 0; 871658628e7SLina Iyer struct tcs_group *tcs; 872658628e7SLina Iyer 873658628e7SLina Iyer ret = of_property_read_u32(dn, "qcom,tcs-offset", &offset); 874658628e7SLina Iyer if (ret) 875658628e7SLina Iyer return ret; 876658628e7SLina Iyer drv->tcs_base = base + offset; 877658628e7SLina Iyer 878658628e7SLina Iyer config = readl_relaxed(base + DRV_PRNT_CHLD_CONFIG); 879658628e7SLina Iyer 880658628e7SLina Iyer max_tcs = config; 881658628e7SLina Iyer max_tcs &= DRV_NUM_TCS_MASK << (DRV_NUM_TCS_SHIFT * drv->id); 882658628e7SLina Iyer max_tcs = max_tcs >> (DRV_NUM_TCS_SHIFT * drv->id); 883658628e7SLina Iyer 884658628e7SLina Iyer ncpt = config & (DRV_NCPT_MASK << DRV_NCPT_SHIFT); 885658628e7SLina Iyer ncpt = ncpt >> DRV_NCPT_SHIFT; 886658628e7SLina Iyer 887658628e7SLina Iyer n = of_property_count_u32_elems(dn, "qcom,tcs-config"); 888658628e7SLina Iyer if (n != 2 * TCS_TYPE_NR) 889658628e7SLina Iyer return -EINVAL; 890658628e7SLina Iyer 891658628e7SLina Iyer for (i = 0; i < TCS_TYPE_NR; i++) { 892658628e7SLina Iyer ret = of_property_read_u32_index(dn, "qcom,tcs-config", 893658628e7SLina Iyer i * 2, &tcs_cfg[i].type); 894658628e7SLina Iyer if (ret) 895658628e7SLina Iyer return ret; 896658628e7SLina Iyer if (tcs_cfg[i].type >= TCS_TYPE_NR) 897658628e7SLina Iyer return -EINVAL; 898658628e7SLina Iyer 899658628e7SLina Iyer ret = of_property_read_u32_index(dn, "qcom,tcs-config", 900658628e7SLina Iyer i * 2 + 1, &tcs_cfg[i].n); 901658628e7SLina Iyer if (ret) 902658628e7SLina Iyer return ret; 903658628e7SLina Iyer if (tcs_cfg[i].n > MAX_TCS_PER_TYPE) 904658628e7SLina Iyer return -EINVAL; 905658628e7SLina Iyer } 906658628e7SLina Iyer 907658628e7SLina Iyer for (i = 0; i < TCS_TYPE_NR; i++) { 908658628e7SLina Iyer tcs = &drv->tcs[tcs_cfg[i].type]; 909658628e7SLina Iyer if (tcs->drv) 910658628e7SLina Iyer return -EINVAL; 911658628e7SLina Iyer tcs->drv = drv; 912658628e7SLina Iyer tcs->type = tcs_cfg[i].type; 913658628e7SLina Iyer tcs->num_tcs = tcs_cfg[i].n; 914658628e7SLina Iyer tcs->ncpt = ncpt; 915658628e7SLina Iyer 916658628e7SLina Iyer if (!tcs->num_tcs || tcs->type == CONTROL_TCS) 917658628e7SLina Iyer continue; 918658628e7SLina Iyer 919658628e7SLina Iyer if (st + tcs->num_tcs > max_tcs || 920658628e7SLina Iyer st + tcs->num_tcs >= BITS_PER_BYTE * sizeof(tcs->mask)) 921658628e7SLina Iyer return -EINVAL; 922658628e7SLina Iyer 923658628e7SLina Iyer tcs->mask = ((1 << tcs->num_tcs) - 1) << st; 924658628e7SLina Iyer tcs->offset = st; 925658628e7SLina Iyer st += tcs->num_tcs; 926658628e7SLina Iyer } 927658628e7SLina Iyer 928658628e7SLina Iyer drv->num_tcs = st; 929658628e7SLina Iyer 930658628e7SLina Iyer return 0; 931658628e7SLina Iyer } 932658628e7SLina Iyer 933658628e7SLina Iyer static int rpmh_rsc_probe(struct platform_device *pdev) 934658628e7SLina Iyer { 935658628e7SLina Iyer struct device_node *dn = pdev->dev.of_node; 936658628e7SLina Iyer struct rsc_drv *drv; 937985427f9SMaulik Shah struct resource *res; 938985427f9SMaulik Shah char drv_id[10] = {0}; 939658628e7SLina Iyer int ret, irq; 940985427f9SMaulik Shah u32 solver_config; 941985427f9SMaulik Shah void __iomem *base; 942658628e7SLina Iyer 943fdd102b5SDouglas Anderson /* 944fdd102b5SDouglas Anderson * Even though RPMh doesn't directly use cmd-db, all of its children 945fdd102b5SDouglas Anderson * do. To avoid adding this check to our children we'll do it now. 946fdd102b5SDouglas Anderson */ 947fdd102b5SDouglas Anderson ret = cmd_db_ready(); 948fdd102b5SDouglas Anderson if (ret) { 949fdd102b5SDouglas Anderson if (ret != -EPROBE_DEFER) 950fdd102b5SDouglas Anderson dev_err(&pdev->dev, "Command DB not available (%d)\n", 951fdd102b5SDouglas Anderson ret); 952fdd102b5SDouglas Anderson return ret; 953fdd102b5SDouglas Anderson } 954fdd102b5SDouglas Anderson 955658628e7SLina Iyer drv = devm_kzalloc(&pdev->dev, sizeof(*drv), GFP_KERNEL); 956658628e7SLina Iyer if (!drv) 957658628e7SLina Iyer return -ENOMEM; 958658628e7SLina Iyer 959658628e7SLina Iyer ret = of_property_read_u32(dn, "qcom,drv-id", &drv->id); 960658628e7SLina Iyer if (ret) 961658628e7SLina Iyer return ret; 962658628e7SLina Iyer 963658628e7SLina Iyer drv->name = of_get_property(dn, "label", NULL); 964658628e7SLina Iyer if (!drv->name) 965658628e7SLina Iyer drv->name = dev_name(&pdev->dev); 966658628e7SLina Iyer 967985427f9SMaulik Shah snprintf(drv_id, ARRAY_SIZE(drv_id), "drv-%d", drv->id); 968985427f9SMaulik Shah res = platform_get_resource_byname(pdev, IORESOURCE_MEM, drv_id); 969985427f9SMaulik Shah base = devm_ioremap_resource(&pdev->dev, res); 970985427f9SMaulik Shah if (IS_ERR(base)) 971985427f9SMaulik Shah return PTR_ERR(base); 972985427f9SMaulik Shah 973985427f9SMaulik Shah ret = rpmh_probe_tcs_config(pdev, drv, base); 974658628e7SLina Iyer if (ret) 975658628e7SLina Iyer return ret; 976658628e7SLina Iyer 977658628e7SLina Iyer spin_lock_init(&drv->lock); 9782bc20f3cSStephen Boyd init_waitqueue_head(&drv->tcs_wait); 979658628e7SLina Iyer bitmap_zero(drv->tcs_in_use, MAX_TCS_NR); 980658628e7SLina Iyer 981658628e7SLina Iyer irq = platform_get_irq(pdev, drv->id); 982658628e7SLina Iyer if (irq < 0) 983658628e7SLina Iyer return irq; 984658628e7SLina Iyer 985658628e7SLina Iyer ret = devm_request_irq(&pdev->dev, irq, tcs_tx_done, 986658628e7SLina Iyer IRQF_TRIGGER_HIGH | IRQF_NO_SUSPEND, 987658628e7SLina Iyer drv->name, drv); 988658628e7SLina Iyer if (ret) 989658628e7SLina Iyer return ret; 990658628e7SLina Iyer 991985427f9SMaulik Shah /* 992985427f9SMaulik Shah * CPU PM notification are not required for controllers that support 993985427f9SMaulik Shah * 'HW solver' mode where they can be in autonomous mode executing low 994985427f9SMaulik Shah * power mode to power down. 995985427f9SMaulik Shah */ 996985427f9SMaulik Shah solver_config = readl_relaxed(base + DRV_SOLVER_CONFIG); 997985427f9SMaulik Shah solver_config &= DRV_HW_SOLVER_MASK << DRV_HW_SOLVER_SHIFT; 998985427f9SMaulik Shah solver_config = solver_config >> DRV_HW_SOLVER_SHIFT; 999985427f9SMaulik Shah if (!solver_config) { 1000985427f9SMaulik Shah drv->rsc_pm.notifier_call = rpmh_rsc_cpu_pm_callback; 1001985427f9SMaulik Shah cpu_pm_register_notifier(&drv->rsc_pm); 1002985427f9SMaulik Shah } 1003985427f9SMaulik Shah 1004658628e7SLina Iyer /* Enable the active TCS to send requests immediately */ 1005032c692aSDouglas Anderson writel_relaxed(drv->tcs[ACTIVE_TCS].mask, 1006032c692aSDouglas Anderson drv->tcs_base + RSC_DRV_IRQ_ENABLE); 1007658628e7SLina Iyer 1008600513dfSLina Iyer spin_lock_init(&drv->client.cache_lock); 1009600513dfSLina Iyer INIT_LIST_HEAD(&drv->client.cache); 1010c8790cb6SLina Iyer INIT_LIST_HEAD(&drv->client.batch_cache); 1011600513dfSLina Iyer 1012c1038456SLina Iyer dev_set_drvdata(&pdev->dev, drv); 1013c1038456SLina Iyer 1014658628e7SLina Iyer return devm_of_platform_populate(&pdev->dev); 1015658628e7SLina Iyer } 1016658628e7SLina Iyer 1017658628e7SLina Iyer static const struct of_device_id rpmh_drv_match[] = { 1018658628e7SLina Iyer { .compatible = "qcom,rpmh-rsc", }, 1019658628e7SLina Iyer { } 1020658628e7SLina Iyer }; 1021658628e7SLina Iyer 1022658628e7SLina Iyer static struct platform_driver rpmh_driver = { 1023658628e7SLina Iyer .probe = rpmh_rsc_probe, 1024658628e7SLina Iyer .driver = { 1025658628e7SLina Iyer .name = "rpmh", 1026658628e7SLina Iyer .of_match_table = rpmh_drv_match, 10271a53ce9aSMaulik Shah .suppress_bind_attrs = true, 1028658628e7SLina Iyer }, 1029658628e7SLina Iyer }; 1030658628e7SLina Iyer 1031658628e7SLina Iyer static int __init rpmh_driver_init(void) 1032658628e7SLina Iyer { 1033658628e7SLina Iyer return platform_driver_register(&rpmh_driver); 1034658628e7SLina Iyer } 1035658628e7SLina Iyer arch_initcall(rpmh_driver_init); 1036