1658628e7SLina Iyer // SPDX-License-Identifier: GPL-2.0 2658628e7SLina Iyer /* 3658628e7SLina Iyer * Copyright (c) 2016-2018, The Linux Foundation. All rights reserved. 4658628e7SLina Iyer */ 5658628e7SLina Iyer 6658628e7SLina Iyer #define pr_fmt(fmt) "%s " fmt, KBUILD_MODNAME 7658628e7SLina Iyer 8658628e7SLina Iyer #include <linux/atomic.h> 9985427f9SMaulik Shah #include <linux/cpu_pm.h> 10658628e7SLina Iyer #include <linux/delay.h> 11658628e7SLina Iyer #include <linux/interrupt.h> 12658628e7SLina Iyer #include <linux/io.h> 1391160150SDouglas Anderson #include <linux/iopoll.h> 14658628e7SLina Iyer #include <linux/kernel.h> 15658628e7SLina Iyer #include <linux/list.h> 16cb365926SJohn Stultz #include <linux/module.h> 17658628e7SLina Iyer #include <linux/of.h> 18658628e7SLina Iyer #include <linux/of_irq.h> 19658628e7SLina Iyer #include <linux/of_platform.h> 20658628e7SLina Iyer #include <linux/platform_device.h> 21658628e7SLina Iyer #include <linux/slab.h> 22658628e7SLina Iyer #include <linux/spinlock.h> 232bc20f3cSStephen Boyd #include <linux/wait.h> 24658628e7SLina Iyer 25fdd102b5SDouglas Anderson #include <soc/qcom/cmd-db.h> 26658628e7SLina Iyer #include <soc/qcom/tcs.h> 27658628e7SLina Iyer #include <dt-bindings/soc/qcom,rpmh-rsc.h> 28658628e7SLina Iyer 29658628e7SLina Iyer #include "rpmh-internal.h" 30658628e7SLina Iyer 31fc087fe5SLina Iyer #define CREATE_TRACE_POINTS 32fc087fe5SLina Iyer #include "trace-rpmh.h" 33fc087fe5SLina Iyer 34658628e7SLina Iyer #define RSC_DRV_TCS_OFFSET 672 35658628e7SLina Iyer #define RSC_DRV_CMD_OFFSET 20 36658628e7SLina Iyer 37985427f9SMaulik Shah /* DRV HW Solver Configuration Information Register */ 38985427f9SMaulik Shah #define DRV_SOLVER_CONFIG 0x04 39985427f9SMaulik Shah #define DRV_HW_SOLVER_MASK 1 40985427f9SMaulik Shah #define DRV_HW_SOLVER_SHIFT 24 41985427f9SMaulik Shah 42985427f9SMaulik Shah /* DRV TCS Configuration Information Register */ 43658628e7SLina Iyer #define DRV_PRNT_CHLD_CONFIG 0x0C 44658628e7SLina Iyer #define DRV_NUM_TCS_MASK 0x3F 45658628e7SLina Iyer #define DRV_NUM_TCS_SHIFT 6 46658628e7SLina Iyer #define DRV_NCPT_MASK 0x1F 47658628e7SLina Iyer #define DRV_NCPT_SHIFT 27 48658628e7SLina Iyer 491f7dbeb5SDouglas Anderson /* Offsets for common TCS Registers, one bit per TCS */ 50658628e7SLina Iyer #define RSC_DRV_IRQ_ENABLE 0x00 51658628e7SLina Iyer #define RSC_DRV_IRQ_STATUS 0x04 521f7dbeb5SDouglas Anderson #define RSC_DRV_IRQ_CLEAR 0x08 /* w/o; write 1 to clear */ 531f7dbeb5SDouglas Anderson 541f7dbeb5SDouglas Anderson /* 551f7dbeb5SDouglas Anderson * Offsets for per TCS Registers. 561f7dbeb5SDouglas Anderson * 571f7dbeb5SDouglas Anderson * TCSes start at 0x10 from tcs_base and are stored one after another. 581f7dbeb5SDouglas Anderson * Multiply tcs_id by RSC_DRV_TCS_OFFSET to find a given TCS and add one 591f7dbeb5SDouglas Anderson * of the below to find a register. 601f7dbeb5SDouglas Anderson */ 611f7dbeb5SDouglas Anderson #define RSC_DRV_CMD_WAIT_FOR_CMPL 0x10 /* 1 bit per command */ 62658628e7SLina Iyer #define RSC_DRV_CONTROL 0x14 631f7dbeb5SDouglas Anderson #define RSC_DRV_STATUS 0x18 /* zero if tcs is busy */ 641f7dbeb5SDouglas Anderson #define RSC_DRV_CMD_ENABLE 0x1C /* 1 bit per command */ 651f7dbeb5SDouglas Anderson 661f7dbeb5SDouglas Anderson /* 671f7dbeb5SDouglas Anderson * Offsets for per command in a TCS. 681f7dbeb5SDouglas Anderson * 691f7dbeb5SDouglas Anderson * Commands (up to 16) start at 0x30 in a TCS; multiply command index 701f7dbeb5SDouglas Anderson * by RSC_DRV_CMD_OFFSET and add one of the below to find a register. 711f7dbeb5SDouglas Anderson */ 72658628e7SLina Iyer #define RSC_DRV_CMD_MSGID 0x30 73658628e7SLina Iyer #define RSC_DRV_CMD_ADDR 0x34 74658628e7SLina Iyer #define RSC_DRV_CMD_DATA 0x38 75658628e7SLina Iyer #define RSC_DRV_CMD_STATUS 0x3C 76658628e7SLina Iyer #define RSC_DRV_CMD_RESP_DATA 0x40 77658628e7SLina Iyer 78658628e7SLina Iyer #define TCS_AMC_MODE_ENABLE BIT(16) 79658628e7SLina Iyer #define TCS_AMC_MODE_TRIGGER BIT(24) 80658628e7SLina Iyer 81658628e7SLina Iyer /* TCS CMD register bit mask */ 82658628e7SLina Iyer #define CMD_MSGID_LEN 8 83658628e7SLina Iyer #define CMD_MSGID_RESP_REQ BIT(8) 84658628e7SLina Iyer #define CMD_MSGID_WRITE BIT(16) 85658628e7SLina Iyer #define CMD_STATUS_ISSUED BIT(8) 86658628e7SLina Iyer #define CMD_STATUS_COMPL BIT(16) 87658628e7SLina Iyer 881f7dbeb5SDouglas Anderson /* 891f7dbeb5SDouglas Anderson * Here's a high level overview of how all the registers in RPMH work 901f7dbeb5SDouglas Anderson * together: 911f7dbeb5SDouglas Anderson * 921f7dbeb5SDouglas Anderson * - The main rpmh-rsc address is the base of a register space that can 931f7dbeb5SDouglas Anderson * be used to find overall configuration of the hardware 941f7dbeb5SDouglas Anderson * (DRV_PRNT_CHLD_CONFIG). Also found within the rpmh-rsc register 951f7dbeb5SDouglas Anderson * space are all the TCS blocks. The offset of the TCS blocks is 961f7dbeb5SDouglas Anderson * specified in the device tree by "qcom,tcs-offset" and used to 971f7dbeb5SDouglas Anderson * compute tcs_base. 981f7dbeb5SDouglas Anderson * - TCS blocks come one after another. Type, count, and order are 991f7dbeb5SDouglas Anderson * specified by the device tree as "qcom,tcs-config". 1001f7dbeb5SDouglas Anderson * - Each TCS block has some registers, then space for up to 16 commands. 1011f7dbeb5SDouglas Anderson * Note that though address space is reserved for 16 commands, fewer 1021f7dbeb5SDouglas Anderson * might be present. See ncpt (num cmds per TCS). 1031f7dbeb5SDouglas Anderson * 1041f7dbeb5SDouglas Anderson * Here's a picture: 1051f7dbeb5SDouglas Anderson * 1061f7dbeb5SDouglas Anderson * +---------------------------------------------------+ 1071f7dbeb5SDouglas Anderson * |RSC | 1081f7dbeb5SDouglas Anderson * | ctrl | 1091f7dbeb5SDouglas Anderson * | | 1101f7dbeb5SDouglas Anderson * | Drvs: | 1111f7dbeb5SDouglas Anderson * | +-----------------------------------------------+ | 1121f7dbeb5SDouglas Anderson * | |DRV0 | | 1131f7dbeb5SDouglas Anderson * | | ctrl/config | | 1141f7dbeb5SDouglas Anderson * | | IRQ | | 1151f7dbeb5SDouglas Anderson * | | | | 1161f7dbeb5SDouglas Anderson * | | TCSes: | | 1171f7dbeb5SDouglas Anderson * | | +------------------------------------------+ | | 1181f7dbeb5SDouglas Anderson * | | |TCS0 | | | | | | | | | | | | | | | 1191f7dbeb5SDouglas Anderson * | | | ctrl | 0| 1| 2| 3| 4| 5| .| .| .| .|14|15| | | 1201f7dbeb5SDouglas Anderson * | | | | | | | | | | | | | | | | | | 1211f7dbeb5SDouglas Anderson * | | +------------------------------------------+ | | 1221f7dbeb5SDouglas Anderson * | | +------------------------------------------+ | | 1231f7dbeb5SDouglas Anderson * | | |TCS1 | | | | | | | | | | | | | | | 1241f7dbeb5SDouglas Anderson * | | | ctrl | 0| 1| 2| 3| 4| 5| .| .| .| .|14|15| | | 1251f7dbeb5SDouglas Anderson * | | | | | | | | | | | | | | | | | | 1261f7dbeb5SDouglas Anderson * | | +------------------------------------------+ | | 1271f7dbeb5SDouglas Anderson * | | +------------------------------------------+ | | 1281f7dbeb5SDouglas Anderson * | | |TCS2 | | | | | | | | | | | | | | | 1291f7dbeb5SDouglas Anderson * | | | ctrl | 0| 1| 2| 3| 4| 5| .| .| .| .|14|15| | | 1301f7dbeb5SDouglas Anderson * | | | | | | | | | | | | | | | | | | 1311f7dbeb5SDouglas Anderson * | | +------------------------------------------+ | | 1321f7dbeb5SDouglas Anderson * | | ...... | | 1331f7dbeb5SDouglas Anderson * | +-----------------------------------------------+ | 1341f7dbeb5SDouglas Anderson * | +-----------------------------------------------+ | 1351f7dbeb5SDouglas Anderson * | |DRV1 | | 1361f7dbeb5SDouglas Anderson * | | (same as DRV0) | | 1371f7dbeb5SDouglas Anderson * | +-----------------------------------------------+ | 1381f7dbeb5SDouglas Anderson * | ...... | 1391f7dbeb5SDouglas Anderson * +---------------------------------------------------+ 1401f7dbeb5SDouglas Anderson */ 1411f7dbeb5SDouglas Anderson 142faa0c1f1SDouglas Anderson static inline void __iomem * 143faa0c1f1SDouglas Anderson tcs_reg_addr(const struct rsc_drv *drv, int reg, int tcs_id) 144658628e7SLina Iyer { 145faa0c1f1SDouglas Anderson return drv->tcs_base + RSC_DRV_TCS_OFFSET * tcs_id + reg; 146658628e7SLina Iyer } 147658628e7SLina Iyer 148faa0c1f1SDouglas Anderson static inline void __iomem * 149faa0c1f1SDouglas Anderson tcs_cmd_addr(const struct rsc_drv *drv, int reg, int tcs_id, int cmd_id) 1503b5e3d50SDouglas Anderson { 151faa0c1f1SDouglas Anderson return tcs_reg_addr(drv, reg, tcs_id) + RSC_DRV_CMD_OFFSET * cmd_id; 1523b5e3d50SDouglas Anderson } 1533b5e3d50SDouglas Anderson 154faa0c1f1SDouglas Anderson static u32 read_tcs_cmd(const struct rsc_drv *drv, int reg, int tcs_id, 155faa0c1f1SDouglas Anderson int cmd_id) 156faa0c1f1SDouglas Anderson { 157faa0c1f1SDouglas Anderson return readl_relaxed(tcs_cmd_addr(drv, reg, tcs_id, cmd_id)); 158faa0c1f1SDouglas Anderson } 159faa0c1f1SDouglas Anderson 160faa0c1f1SDouglas Anderson static u32 read_tcs_reg(const struct rsc_drv *drv, int reg, int tcs_id) 161faa0c1f1SDouglas Anderson { 162faa0c1f1SDouglas Anderson return readl_relaxed(tcs_reg_addr(drv, reg, tcs_id)); 163faa0c1f1SDouglas Anderson } 164faa0c1f1SDouglas Anderson 165faa0c1f1SDouglas Anderson static void write_tcs_cmd(const struct rsc_drv *drv, int reg, int tcs_id, 166faa0c1f1SDouglas Anderson int cmd_id, u32 data) 167faa0c1f1SDouglas Anderson { 168faa0c1f1SDouglas Anderson writel_relaxed(data, tcs_cmd_addr(drv, reg, tcs_id, cmd_id)); 169faa0c1f1SDouglas Anderson } 170faa0c1f1SDouglas Anderson 171faa0c1f1SDouglas Anderson static void write_tcs_reg(const struct rsc_drv *drv, int reg, int tcs_id, 172658628e7SLina Iyer u32 data) 173658628e7SLina Iyer { 174faa0c1f1SDouglas Anderson writel_relaxed(data, tcs_reg_addr(drv, reg, tcs_id)); 175658628e7SLina Iyer } 176658628e7SLina Iyer 177faa0c1f1SDouglas Anderson static void write_tcs_reg_sync(const struct rsc_drv *drv, int reg, int tcs_id, 178658628e7SLina Iyer u32 data) 179658628e7SLina Iyer { 180be24c6a7SDouglas Anderson int i; 18191160150SDouglas Anderson 182faa0c1f1SDouglas Anderson writel(data, tcs_reg_addr(drv, reg, tcs_id)); 183be24c6a7SDouglas Anderson 184be24c6a7SDouglas Anderson /* 185be24c6a7SDouglas Anderson * Wait until we read back the same value. Use a counter rather than 186be24c6a7SDouglas Anderson * ktime for timeout since this may be called after timekeeping stops. 187be24c6a7SDouglas Anderson */ 188be24c6a7SDouglas Anderson for (i = 0; i < USEC_PER_SEC; i++) { 189be24c6a7SDouglas Anderson if (readl(tcs_reg_addr(drv, reg, tcs_id)) == data) 190be24c6a7SDouglas Anderson return; 191be24c6a7SDouglas Anderson udelay(1); 192be24c6a7SDouglas Anderson } 19391160150SDouglas Anderson pr_err("%s: error writing %#x to %d:%#x\n", drv->name, 19491160150SDouglas Anderson data, tcs_id, reg); 195658628e7SLina Iyer } 196658628e7SLina Iyer 197e40b0c16SDouglas Anderson /** 198e40b0c16SDouglas Anderson * tcs_invalidate() - Invalidate all TCSes of the given type (sleep or wake). 199e40b0c16SDouglas Anderson * @drv: The RSC controller. 200e40b0c16SDouglas Anderson * @type: SLEEP_TCS or WAKE_TCS 201e40b0c16SDouglas Anderson * 202e40b0c16SDouglas Anderson * This will clear the "slots" variable of the given tcs_group and also 203e40b0c16SDouglas Anderson * tell the hardware to forget about all entries. 204e40b0c16SDouglas Anderson * 205881808d0SDouglas Anderson * The caller must ensure that no other RPMH actions are happening when this 206881808d0SDouglas Anderson * function is called, since otherwise the device may immediately become 207881808d0SDouglas Anderson * used again even before this function exits. 208e40b0c16SDouglas Anderson */ 209881808d0SDouglas Anderson static void tcs_invalidate(struct rsc_drv *drv, int type) 2109a3afcfbSLina Iyer { 2119a3afcfbSLina Iyer int m; 21253d49fe1SDouglas Anderson struct tcs_group *tcs = &drv->tcs[type]; 2139a3afcfbSLina Iyer 214881808d0SDouglas Anderson /* Caller ensures nobody else is running so no lock */ 215881808d0SDouglas Anderson if (bitmap_empty(tcs->slots, MAX_TCS_SLOTS)) 216881808d0SDouglas Anderson return; 2179a3afcfbSLina Iyer 218fef419c4SLina Iyer for (m = tcs->offset; m < tcs->offset + tcs->num_tcs; m++) 2199a3afcfbSLina Iyer write_tcs_reg_sync(drv, RSC_DRV_CMD_ENABLE, m, 0); 220fef419c4SLina Iyer 2219a3afcfbSLina Iyer bitmap_zero(tcs->slots, MAX_TCS_SLOTS); 2229a3afcfbSLina Iyer } 2239a3afcfbSLina Iyer 2249a3afcfbSLina Iyer /** 225e40b0c16SDouglas Anderson * rpmh_rsc_invalidate() - Invalidate sleep and wake TCSes. 226e40b0c16SDouglas Anderson * @drv: The RSC controller. 2279a3afcfbSLina Iyer * 228881808d0SDouglas Anderson * The caller must ensure that no other RPMH actions are happening when this 229881808d0SDouglas Anderson * function is called, since otherwise the device may immediately become 230881808d0SDouglas Anderson * used again even before this function exits. 2319a3afcfbSLina Iyer */ 232881808d0SDouglas Anderson void rpmh_rsc_invalidate(struct rsc_drv *drv) 2339a3afcfbSLina Iyer { 234881808d0SDouglas Anderson tcs_invalidate(drv, SLEEP_TCS); 235881808d0SDouglas Anderson tcs_invalidate(drv, WAKE_TCS); 2369a3afcfbSLina Iyer } 2379a3afcfbSLina Iyer 238e40b0c16SDouglas Anderson /** 239e40b0c16SDouglas Anderson * get_tcs_for_msg() - Get the tcs_group used to send the given message. 240e40b0c16SDouglas Anderson * @drv: The RSC controller. 241e40b0c16SDouglas Anderson * @msg: The message we want to send. 242e40b0c16SDouglas Anderson * 243e40b0c16SDouglas Anderson * This is normally pretty straightforward except if we are trying to send 244e40b0c16SDouglas Anderson * an ACTIVE_ONLY message but don't have any active_only TCSes. 245e40b0c16SDouglas Anderson * 246e40b0c16SDouglas Anderson * Return: A pointer to a tcs_group or an ERR_PTR. 247e40b0c16SDouglas Anderson */ 248658628e7SLina Iyer static struct tcs_group *get_tcs_for_msg(struct rsc_drv *drv, 249658628e7SLina Iyer const struct tcs_request *msg) 250658628e7SLina Iyer { 25138427e5aSMaulik Shah int type; 2522de4b8d3SLina Iyer struct tcs_group *tcs; 253658628e7SLina Iyer 254658628e7SLina Iyer switch (msg->state) { 255658628e7SLina Iyer case RPMH_ACTIVE_ONLY_STATE: 256658628e7SLina Iyer type = ACTIVE_TCS; 257658628e7SLina Iyer break; 258fa460e45SLina Iyer case RPMH_WAKE_ONLY_STATE: 259fa460e45SLina Iyer type = WAKE_TCS; 260fa460e45SLina Iyer break; 261fa460e45SLina Iyer case RPMH_SLEEP_STATE: 262fa460e45SLina Iyer type = SLEEP_TCS; 263fa460e45SLina Iyer break; 264658628e7SLina Iyer default: 265658628e7SLina Iyer return ERR_PTR(-EINVAL); 266658628e7SLina Iyer } 267658628e7SLina Iyer 2682de4b8d3SLina Iyer /* 2692de4b8d3SLina Iyer * If we are making an active request on a RSC that does not have a 2702de4b8d3SLina Iyer * dedicated TCS for active state use, then re-purpose a wake TCS to 271e40b0c16SDouglas Anderson * send active votes. This is safe because we ensure any active-only 272e40b0c16SDouglas Anderson * transfers have finished before we use it (maybe by running from 273e40b0c16SDouglas Anderson * the last CPU in PM code). 2742de4b8d3SLina Iyer */ 27553d49fe1SDouglas Anderson tcs = &drv->tcs[type]; 27638427e5aSMaulik Shah if (msg->state == RPMH_ACTIVE_ONLY_STATE && !tcs->num_tcs) 27753d49fe1SDouglas Anderson tcs = &drv->tcs[WAKE_TCS]; 2782de4b8d3SLina Iyer 2792de4b8d3SLina Iyer return tcs; 280658628e7SLina Iyer } 281658628e7SLina Iyer 282e40b0c16SDouglas Anderson /** 283e40b0c16SDouglas Anderson * get_req_from_tcs() - Get a stashed request that was xfering on the given TCS. 284e40b0c16SDouglas Anderson * @drv: The RSC controller. 285e40b0c16SDouglas Anderson * @tcs_id: The global ID of this TCS. 286e40b0c16SDouglas Anderson * 287e40b0c16SDouglas Anderson * For ACTIVE_ONLY transfers we want to call back into the client when the 288e40b0c16SDouglas Anderson * transfer finishes. To do this we need the "request" that the client 289e40b0c16SDouglas Anderson * originally provided us. This function grabs the request that we stashed 290e40b0c16SDouglas Anderson * when we started the transfer. 291e40b0c16SDouglas Anderson * 292e40b0c16SDouglas Anderson * This only makes sense for ACTIVE_ONLY transfers since those are the only 293e40b0c16SDouglas Anderson * ones we track sending (the only ones we enable interrupts for and the only 294e40b0c16SDouglas Anderson * ones we call back to the client for). 295e40b0c16SDouglas Anderson * 296e40b0c16SDouglas Anderson * Return: The stashed request. 297e40b0c16SDouglas Anderson */ 298658628e7SLina Iyer static const struct tcs_request *get_req_from_tcs(struct rsc_drv *drv, 299658628e7SLina Iyer int tcs_id) 300658628e7SLina Iyer { 301658628e7SLina Iyer struct tcs_group *tcs; 302658628e7SLina Iyer int i; 303658628e7SLina Iyer 304efa1c257SRaju P.L.S.S.S.N for (i = 0; i < TCS_TYPE_NR; i++) { 305658628e7SLina Iyer tcs = &drv->tcs[i]; 306658628e7SLina Iyer if (tcs->mask & BIT(tcs_id)) 307658628e7SLina Iyer return tcs->req[tcs_id - tcs->offset]; 308658628e7SLina Iyer } 309658628e7SLina Iyer 310658628e7SLina Iyer return NULL; 311658628e7SLina Iyer } 312658628e7SLina Iyer 313e40b0c16SDouglas Anderson /** 314e40b0c16SDouglas Anderson * __tcs_set_trigger() - Start xfer on a TCS or unset trigger on a borrowed TCS 315e40b0c16SDouglas Anderson * @drv: The controller. 316e40b0c16SDouglas Anderson * @tcs_id: The global ID of this TCS. 317e40b0c16SDouglas Anderson * @trigger: If true then untrigger/retrigger. If false then just untrigger. 318e40b0c16SDouglas Anderson * 319e40b0c16SDouglas Anderson * In the normal case we only ever call with "trigger=true" to start a 320e40b0c16SDouglas Anderson * transfer. That will un-trigger/disable the TCS from the last transfer 321e40b0c16SDouglas Anderson * then trigger/enable for this transfer. 322e40b0c16SDouglas Anderson * 323e40b0c16SDouglas Anderson * If we borrowed a wake TCS for an active-only transfer we'll also call 324e40b0c16SDouglas Anderson * this function with "trigger=false" to just do the un-trigger/disable 325e40b0c16SDouglas Anderson * before using the TCS for wake purposes again. 326e40b0c16SDouglas Anderson * 327e40b0c16SDouglas Anderson * Note that the AP is only in charge of triggering active-only transfers. 328e40b0c16SDouglas Anderson * The AP never triggers sleep/wake values using this function. 329e40b0c16SDouglas Anderson */ 33015b3bf61SRaju P.L.S.S.S.N static void __tcs_set_trigger(struct rsc_drv *drv, int tcs_id, bool trigger) 33115b3bf61SRaju P.L.S.S.S.N { 33215b3bf61SRaju P.L.S.S.S.N u32 enable; 33315b3bf61SRaju P.L.S.S.S.N 33415b3bf61SRaju P.L.S.S.S.N /* 33515b3bf61SRaju P.L.S.S.S.N * HW req: Clear the DRV_CONTROL and enable TCS again 33615b3bf61SRaju P.L.S.S.S.N * While clearing ensure that the AMC mode trigger is cleared 33715b3bf61SRaju P.L.S.S.S.N * and then the mode enable is cleared. 33815b3bf61SRaju P.L.S.S.S.N */ 3393b5e3d50SDouglas Anderson enable = read_tcs_reg(drv, RSC_DRV_CONTROL, tcs_id); 34015b3bf61SRaju P.L.S.S.S.N enable &= ~TCS_AMC_MODE_TRIGGER; 34115b3bf61SRaju P.L.S.S.S.N write_tcs_reg_sync(drv, RSC_DRV_CONTROL, tcs_id, enable); 34215b3bf61SRaju P.L.S.S.S.N enable &= ~TCS_AMC_MODE_ENABLE; 34315b3bf61SRaju P.L.S.S.S.N write_tcs_reg_sync(drv, RSC_DRV_CONTROL, tcs_id, enable); 34415b3bf61SRaju P.L.S.S.S.N 34515b3bf61SRaju P.L.S.S.S.N if (trigger) { 34615b3bf61SRaju P.L.S.S.S.N /* Enable the AMC mode on the TCS and then trigger the TCS */ 34715b3bf61SRaju P.L.S.S.S.N enable = TCS_AMC_MODE_ENABLE; 34815b3bf61SRaju P.L.S.S.S.N write_tcs_reg_sync(drv, RSC_DRV_CONTROL, tcs_id, enable); 34915b3bf61SRaju P.L.S.S.S.N enable |= TCS_AMC_MODE_TRIGGER; 350e1d80081SLina Iyer write_tcs_reg(drv, RSC_DRV_CONTROL, tcs_id, enable); 35115b3bf61SRaju P.L.S.S.S.N } 35215b3bf61SRaju P.L.S.S.S.N } 35315b3bf61SRaju P.L.S.S.S.N 354e40b0c16SDouglas Anderson /** 355e40b0c16SDouglas Anderson * enable_tcs_irq() - Enable or disable interrupts on the given TCS. 356e40b0c16SDouglas Anderson * @drv: The controller. 357e40b0c16SDouglas Anderson * @tcs_id: The global ID of this TCS. 358e40b0c16SDouglas Anderson * @enable: If true then enable; if false then disable 359e40b0c16SDouglas Anderson * 360e40b0c16SDouglas Anderson * We only ever call this when we borrow a wake TCS for an active-only 361e40b0c16SDouglas Anderson * transfer. For active-only TCSes interrupts are always left enabled. 362e40b0c16SDouglas Anderson */ 36315b3bf61SRaju P.L.S.S.S.N static void enable_tcs_irq(struct rsc_drv *drv, int tcs_id, bool enable) 36415b3bf61SRaju P.L.S.S.S.N { 36515b3bf61SRaju P.L.S.S.S.N u32 data; 36615b3bf61SRaju P.L.S.S.S.N 367032c692aSDouglas Anderson data = readl_relaxed(drv->tcs_base + RSC_DRV_IRQ_ENABLE); 36815b3bf61SRaju P.L.S.S.S.N if (enable) 36915b3bf61SRaju P.L.S.S.S.N data |= BIT(tcs_id); 37015b3bf61SRaju P.L.S.S.S.N else 37115b3bf61SRaju P.L.S.S.S.N data &= ~BIT(tcs_id); 372032c692aSDouglas Anderson writel_relaxed(data, drv->tcs_base + RSC_DRV_IRQ_ENABLE); 37315b3bf61SRaju P.L.S.S.S.N } 37415b3bf61SRaju P.L.S.S.S.N 375658628e7SLina Iyer /** 376e40b0c16SDouglas Anderson * tcs_tx_done() - TX Done interrupt handler. 377e40b0c16SDouglas Anderson * @irq: The IRQ number (ignored). 378e40b0c16SDouglas Anderson * @p: Pointer to "struct rsc_drv". 379e40b0c16SDouglas Anderson * 380e40b0c16SDouglas Anderson * Called for ACTIVE_ONLY transfers (those are the only ones we enable the 381e40b0c16SDouglas Anderson * IRQ for) when a transfer is done. 382e40b0c16SDouglas Anderson * 383e40b0c16SDouglas Anderson * Return: IRQ_HANDLED 384658628e7SLina Iyer */ 385658628e7SLina Iyer static irqreturn_t tcs_tx_done(int irq, void *p) 386658628e7SLina Iyer { 387658628e7SLina Iyer struct rsc_drv *drv = p; 388c1038456SLina Iyer int i, j, err = 0; 389658628e7SLina Iyer unsigned long irq_status; 390658628e7SLina Iyer const struct tcs_request *req; 391658628e7SLina Iyer struct tcs_cmd *cmd; 392658628e7SLina Iyer 393032c692aSDouglas Anderson irq_status = readl_relaxed(drv->tcs_base + RSC_DRV_IRQ_STATUS); 394658628e7SLina Iyer 395*a0776620SStephen Boyd for_each_set_bit(i, &irq_status, BITS_PER_TYPE(u32)) { 396658628e7SLina Iyer req = get_req_from_tcs(drv, i); 397658628e7SLina Iyer if (!req) { 398658628e7SLina Iyer WARN_ON(1); 399658628e7SLina Iyer goto skip; 400658628e7SLina Iyer } 401658628e7SLina Iyer 402fc087fe5SLina Iyer err = 0; 403658628e7SLina Iyer for (j = 0; j < req->num_cmds; j++) { 404658628e7SLina Iyer u32 sts; 405658628e7SLina Iyer 406658628e7SLina Iyer cmd = &req->cmds[j]; 4073b5e3d50SDouglas Anderson sts = read_tcs_cmd(drv, RSC_DRV_CMD_STATUS, i, j); 408658628e7SLina Iyer if (!(sts & CMD_STATUS_ISSUED) || 409658628e7SLina Iyer ((req->wait_for_compl || cmd->wait) && 410658628e7SLina Iyer !(sts & CMD_STATUS_COMPL))) { 411658628e7SLina Iyer pr_err("Incomplete request: %s: addr=%#x data=%#x", 412658628e7SLina Iyer drv->name, cmd->addr, cmd->data); 413fc087fe5SLina Iyer err = -EIO; 414658628e7SLina Iyer } 415658628e7SLina Iyer } 416fc087fe5SLina Iyer 417fc087fe5SLina Iyer trace_rpmh_tx_done(drv, i, req, err); 41815b3bf61SRaju P.L.S.S.S.N 41915b3bf61SRaju P.L.S.S.S.N /* 42015b3bf61SRaju P.L.S.S.S.N * If wake tcs was re-purposed for sending active 42115b3bf61SRaju P.L.S.S.S.N * votes, clear AMC trigger & enable modes and 42215b3bf61SRaju P.L.S.S.S.N * disable interrupt for this TCS 42315b3bf61SRaju P.L.S.S.S.N */ 42415b3bf61SRaju P.L.S.S.S.N if (!drv->tcs[ACTIVE_TCS].num_tcs) 42515b3bf61SRaju P.L.S.S.S.N __tcs_set_trigger(drv, i, false); 426658628e7SLina Iyer skip: 427658628e7SLina Iyer /* Reclaim the TCS */ 428658628e7SLina Iyer write_tcs_reg(drv, RSC_DRV_CMD_ENABLE, i, 0); 429032c692aSDouglas Anderson writel_relaxed(BIT(i), drv->tcs_base + RSC_DRV_IRQ_CLEAR); 430658628e7SLina Iyer spin_lock(&drv->lock); 431658628e7SLina Iyer clear_bit(i, drv->tcs_in_use); 43215b3bf61SRaju P.L.S.S.S.N /* 43315b3bf61SRaju P.L.S.S.S.N * Disable interrupt for WAKE TCS to avoid being 43415b3bf61SRaju P.L.S.S.S.N * spammed with interrupts coming when the solver 43515b3bf61SRaju P.L.S.S.S.N * sends its wake votes. 43615b3bf61SRaju P.L.S.S.S.N */ 43715b3bf61SRaju P.L.S.S.S.N if (!drv->tcs[ACTIVE_TCS].num_tcs) 43815b3bf61SRaju P.L.S.S.S.N enable_tcs_irq(drv, i, false); 439658628e7SLina Iyer spin_unlock(&drv->lock); 4402bc20f3cSStephen Boyd wake_up(&drv->tcs_wait); 441c1038456SLina Iyer if (req) 442c1038456SLina Iyer rpmh_tx_done(req, err); 443658628e7SLina Iyer } 444658628e7SLina Iyer 445658628e7SLina Iyer return IRQ_HANDLED; 446658628e7SLina Iyer } 447658628e7SLina Iyer 448e40b0c16SDouglas Anderson /** 449e40b0c16SDouglas Anderson * __tcs_buffer_write() - Write to TCS hardware from a request; don't trigger. 450e40b0c16SDouglas Anderson * @drv: The controller. 451e40b0c16SDouglas Anderson * @tcs_id: The global ID of this TCS. 452e40b0c16SDouglas Anderson * @cmd_id: The index within the TCS to start writing. 453e40b0c16SDouglas Anderson * @msg: The message we want to send, which will contain several addr/data 454e40b0c16SDouglas Anderson * pairs to program (but few enough that they all fit in one TCS). 455e40b0c16SDouglas Anderson * 456e40b0c16SDouglas Anderson * This is used for all types of transfers (active, sleep, and wake). 457e40b0c16SDouglas Anderson */ 458658628e7SLina Iyer static void __tcs_buffer_write(struct rsc_drv *drv, int tcs_id, int cmd_id, 459658628e7SLina Iyer const struct tcs_request *msg) 460658628e7SLina Iyer { 461fef419c4SLina Iyer u32 msgid; 462fef419c4SLina Iyer u32 cmd_msgid = CMD_MSGID_LEN | CMD_MSGID_WRITE; 463658628e7SLina Iyer u32 cmd_enable = 0; 464658628e7SLina Iyer struct tcs_cmd *cmd; 465658628e7SLina Iyer int i, j; 466658628e7SLina Iyer 467fef419c4SLina Iyer /* Convert all commands to RR when the request has wait_for_compl set */ 468658628e7SLina Iyer cmd_msgid |= msg->wait_for_compl ? CMD_MSGID_RESP_REQ : 0; 469658628e7SLina Iyer 470658628e7SLina Iyer for (i = 0, j = cmd_id; i < msg->num_cmds; i++, j++) { 471658628e7SLina Iyer cmd = &msg->cmds[i]; 472658628e7SLina Iyer cmd_enable |= BIT(j); 473658628e7SLina Iyer msgid = cmd_msgid; 474fef419c4SLina Iyer /* 475fef419c4SLina Iyer * Additionally, if the cmd->wait is set, make the command 476fef419c4SLina Iyer * response reqd even if the overall request was fire-n-forget. 477fef419c4SLina Iyer */ 478658628e7SLina Iyer msgid |= cmd->wait ? CMD_MSGID_RESP_REQ : 0; 479fc087fe5SLina Iyer 480658628e7SLina Iyer write_tcs_cmd(drv, RSC_DRV_CMD_MSGID, tcs_id, j, msgid); 481658628e7SLina Iyer write_tcs_cmd(drv, RSC_DRV_CMD_ADDR, tcs_id, j, cmd->addr); 482658628e7SLina Iyer write_tcs_cmd(drv, RSC_DRV_CMD_DATA, tcs_id, j, cmd->data); 4837bb7a83fSMaulik Shah trace_rpmh_send_msg(drv, tcs_id, j, msgid, cmd); 484658628e7SLina Iyer } 485658628e7SLina Iyer 4863b5e3d50SDouglas Anderson cmd_enable |= read_tcs_reg(drv, RSC_DRV_CMD_ENABLE, tcs_id); 487658628e7SLina Iyer write_tcs_reg(drv, RSC_DRV_CMD_ENABLE, tcs_id, cmd_enable); 488658628e7SLina Iyer } 489658628e7SLina Iyer 490e40b0c16SDouglas Anderson /** 491e40b0c16SDouglas Anderson * check_for_req_inflight() - Look to see if conflicting cmds are in flight. 492e40b0c16SDouglas Anderson * @drv: The controller. 493e40b0c16SDouglas Anderson * @tcs: A pointer to the tcs_group used for ACTIVE_ONLY transfers. 494e40b0c16SDouglas Anderson * @msg: The message we want to send, which will contain several addr/data 495e40b0c16SDouglas Anderson * pairs to program (but few enough that they all fit in one TCS). 496e40b0c16SDouglas Anderson * 497e40b0c16SDouglas Anderson * This will walk through the TCSes in the group and check if any of them 498e40b0c16SDouglas Anderson * appear to be sending to addresses referenced in the message. If it finds 499e40b0c16SDouglas Anderson * one it'll return -EBUSY. 500e40b0c16SDouglas Anderson * 501e40b0c16SDouglas Anderson * Only for use for active-only transfers. 502e40b0c16SDouglas Anderson * 503e40b0c16SDouglas Anderson * Must be called with the drv->lock held since that protects tcs_in_use. 504e40b0c16SDouglas Anderson * 505e40b0c16SDouglas Anderson * Return: 0 if nothing in flight or -EBUSY if we should try again later. 506e40b0c16SDouglas Anderson * The caller must re-enable interrupts between tries since that's 507814a0d46SStephen Boyd * the only way tcs_in_use will ever be updated and the only way 508e40b0c16SDouglas Anderson * RSC_DRV_CMD_ENABLE will ever be cleared. 509e40b0c16SDouglas Anderson */ 510658628e7SLina Iyer static int check_for_req_inflight(struct rsc_drv *drv, struct tcs_group *tcs, 511658628e7SLina Iyer const struct tcs_request *msg) 512658628e7SLina Iyer { 513658628e7SLina Iyer unsigned long curr_enabled; 514658628e7SLina Iyer u32 addr; 515814a0d46SStephen Boyd int j, k; 516814a0d46SStephen Boyd int i = tcs->offset; 517658628e7SLina Iyer 518814a0d46SStephen Boyd for_each_set_bit_from(i, drv->tcs_in_use, tcs->offset + tcs->num_tcs) { 519814a0d46SStephen Boyd curr_enabled = read_tcs_reg(drv, RSC_DRV_CMD_ENABLE, i); 520658628e7SLina Iyer 521658628e7SLina Iyer for_each_set_bit(j, &curr_enabled, MAX_CMDS_PER_TCS) { 522814a0d46SStephen Boyd addr = read_tcs_cmd(drv, RSC_DRV_CMD_ADDR, i, j); 523658628e7SLina Iyer for (k = 0; k < msg->num_cmds; k++) { 524658628e7SLina Iyer if (addr == msg->cmds[k].addr) 525658628e7SLina Iyer return -EBUSY; 526658628e7SLina Iyer } 527658628e7SLina Iyer } 528658628e7SLina Iyer } 529658628e7SLina Iyer 530658628e7SLina Iyer return 0; 531658628e7SLina Iyer } 532658628e7SLina Iyer 533e40b0c16SDouglas Anderson /** 534e40b0c16SDouglas Anderson * find_free_tcs() - Find free tcs in the given tcs_group; only for active. 535e40b0c16SDouglas Anderson * @tcs: A pointer to the active-only tcs_group (or the wake tcs_group if 536e40b0c16SDouglas Anderson * we borrowed it because there are zero active-only ones). 537e40b0c16SDouglas Anderson * 538e40b0c16SDouglas Anderson * Must be called with the drv->lock held since that protects tcs_in_use. 539e40b0c16SDouglas Anderson * 540814a0d46SStephen Boyd * Return: The first tcs that's free or -EBUSY if all in use. 541e40b0c16SDouglas Anderson */ 542658628e7SLina Iyer static int find_free_tcs(struct tcs_group *tcs) 543658628e7SLina Iyer { 544814a0d46SStephen Boyd const struct rsc_drv *drv = tcs->drv; 545814a0d46SStephen Boyd unsigned long i; 546814a0d46SStephen Boyd unsigned long max = tcs->offset + tcs->num_tcs; 547658628e7SLina Iyer 548814a0d46SStephen Boyd i = find_next_zero_bit(drv->tcs_in_use, max, tcs->offset); 549814a0d46SStephen Boyd if (i >= max) 550658628e7SLina Iyer return -EBUSY; 551814a0d46SStephen Boyd 552814a0d46SStephen Boyd return i; 553658628e7SLina Iyer } 554658628e7SLina Iyer 555e40b0c16SDouglas Anderson /** 5562bc20f3cSStephen Boyd * claim_tcs_for_req() - Claim a tcs in the given tcs_group; only for active. 557e40b0c16SDouglas Anderson * @drv: The controller. 5582bc20f3cSStephen Boyd * @tcs: The tcs_group used for ACTIVE_ONLY transfers. 559e40b0c16SDouglas Anderson * @msg: The data to be sent. 560e40b0c16SDouglas Anderson * 5612bc20f3cSStephen Boyd * Claims a tcs in the given tcs_group while making sure that no existing cmd 5622bc20f3cSStephen Boyd * is in flight that would conflict with the one in @msg. 563e40b0c16SDouglas Anderson * 5642bc20f3cSStephen Boyd * Context: Must be called with the drv->lock held since that protects 5652bc20f3cSStephen Boyd * tcs_in_use. 566e40b0c16SDouglas Anderson * 5672bc20f3cSStephen Boyd * Return: The id of the claimed tcs or -EBUSY if a matching msg is in flight 5682bc20f3cSStephen Boyd * or the tcs_group is full. 569e40b0c16SDouglas Anderson */ 5702bc20f3cSStephen Boyd static int claim_tcs_for_req(struct rsc_drv *drv, struct tcs_group *tcs, 5712bc20f3cSStephen Boyd const struct tcs_request *msg) 572658628e7SLina Iyer { 573658628e7SLina Iyer int ret; 574658628e7SLina Iyer 575658628e7SLina Iyer /* 576658628e7SLina Iyer * The h/w does not like if we send a request to the same address, 577658628e7SLina Iyer * when one is already in-flight or being processed. 578658628e7SLina Iyer */ 579658628e7SLina Iyer ret = check_for_req_inflight(drv, tcs, msg); 580555701a4SDouglas Anderson if (ret) 5812bc20f3cSStephen Boyd return ret; 582658628e7SLina Iyer 5832bc20f3cSStephen Boyd return find_free_tcs(tcs); 5842bc20f3cSStephen Boyd } 5852bc20f3cSStephen Boyd 5862bc20f3cSStephen Boyd /** 5872bc20f3cSStephen Boyd * rpmh_rsc_send_data() - Write / trigger active-only message. 5882bc20f3cSStephen Boyd * @drv: The controller. 5892bc20f3cSStephen Boyd * @msg: The data to be sent. 5902bc20f3cSStephen Boyd * 5912bc20f3cSStephen Boyd * NOTES: 5922bc20f3cSStephen Boyd * - This is only used for "ACTIVE_ONLY" since the limitations of this 5932bc20f3cSStephen Boyd * function don't make sense for sleep/wake cases. 5942bc20f3cSStephen Boyd * - To do the transfer, we will grab a whole TCS for ourselves--we don't 5952bc20f3cSStephen Boyd * try to share. If there are none available we'll wait indefinitely 5962bc20f3cSStephen Boyd * for a free one. 5972bc20f3cSStephen Boyd * - This function will not wait for the commands to be finished, only for 5982bc20f3cSStephen Boyd * data to be programmed into the RPMh. See rpmh_tx_done() which will 5992bc20f3cSStephen Boyd * be called when the transfer is fully complete. 6002bc20f3cSStephen Boyd * - This function must be called with interrupts enabled. If the hardware 6012bc20f3cSStephen Boyd * is busy doing someone else's transfer we need that transfer to fully 6022bc20f3cSStephen Boyd * finish so that we can have the hardware, and to fully finish it needs 6032bc20f3cSStephen Boyd * the interrupt handler to run. If the interrupts is set to run on the 6042bc20f3cSStephen Boyd * active CPU this can never happen if interrupts are disabled. 6052bc20f3cSStephen Boyd * 6062bc20f3cSStephen Boyd * Return: 0 on success, -EINVAL on error. 6072bc20f3cSStephen Boyd */ 6082bc20f3cSStephen Boyd int rpmh_rsc_send_data(struct rsc_drv *drv, const struct tcs_request *msg) 6092bc20f3cSStephen Boyd { 6102bc20f3cSStephen Boyd struct tcs_group *tcs; 6112bc20f3cSStephen Boyd int tcs_id; 6122bc20f3cSStephen Boyd unsigned long flags; 6132bc20f3cSStephen Boyd 6142bc20f3cSStephen Boyd tcs = get_tcs_for_msg(drv, msg); 6152bc20f3cSStephen Boyd if (IS_ERR(tcs)) 6162bc20f3cSStephen Boyd return PTR_ERR(tcs); 6172bc20f3cSStephen Boyd 6182bc20f3cSStephen Boyd spin_lock_irqsave(&drv->lock, flags); 6192bc20f3cSStephen Boyd 6202bc20f3cSStephen Boyd /* Wait forever for a free tcs. It better be there eventually! */ 6212bc20f3cSStephen Boyd wait_event_lock_irq(drv->tcs_wait, 6222bc20f3cSStephen Boyd (tcs_id = claim_tcs_for_req(drv, tcs, msg)) >= 0, 6232bc20f3cSStephen Boyd drv->lock); 624658628e7SLina Iyer 625658628e7SLina Iyer tcs->req[tcs_id - tcs->offset] = msg; 626658628e7SLina Iyer set_bit(tcs_id, drv->tcs_in_use); 62738427e5aSMaulik Shah if (msg->state == RPMH_ACTIVE_ONLY_STATE && tcs->type != ACTIVE_TCS) { 62838427e5aSMaulik Shah /* 62938427e5aSMaulik Shah * Clear previously programmed WAKE commands in selected 63038427e5aSMaulik Shah * repurposed TCS to avoid triggering them. tcs->slots will be 63138427e5aSMaulik Shah * cleaned from rpmh_flush() by invoking rpmh_rsc_invalidate() 63238427e5aSMaulik Shah */ 63338427e5aSMaulik Shah write_tcs_reg_sync(drv, RSC_DRV_CMD_ENABLE, tcs_id, 0); 63415b3bf61SRaju P.L.S.S.S.N enable_tcs_irq(drv, tcs_id, true); 63538427e5aSMaulik Shah } 636555701a4SDouglas Anderson spin_unlock_irqrestore(&drv->lock, flags); 637658628e7SLina Iyer 638555701a4SDouglas Anderson /* 639555701a4SDouglas Anderson * These two can be done after the lock is released because: 640555701a4SDouglas Anderson * - We marked "tcs_in_use" under lock. 641555701a4SDouglas Anderson * - Once "tcs_in_use" has been marked nobody else could be writing 642555701a4SDouglas Anderson * to these registers until the interrupt goes off. 643555701a4SDouglas Anderson * - The interrupt can't go off until we trigger w/ the last line 644555701a4SDouglas Anderson * of __tcs_set_trigger() below. 645555701a4SDouglas Anderson */ 646658628e7SLina Iyer __tcs_buffer_write(drv, tcs_id, 0, msg); 64715b3bf61SRaju P.L.S.S.S.N __tcs_set_trigger(drv, tcs_id, true); 648658628e7SLina Iyer 649555701a4SDouglas Anderson return 0; 650658628e7SLina Iyer } 651658628e7SLina Iyer 652e40b0c16SDouglas Anderson /** 653e40b0c16SDouglas Anderson * find_slots() - Find a place to write the given message. 654e40b0c16SDouglas Anderson * @tcs: The tcs group to search. 655e40b0c16SDouglas Anderson * @msg: The message we want to find room for. 656e40b0c16SDouglas Anderson * @tcs_id: If we return 0 from the function, we return the global ID of the 657e40b0c16SDouglas Anderson * TCS to write to here. 658e40b0c16SDouglas Anderson * @cmd_id: If we return 0 from the function, we return the index of 659e40b0c16SDouglas Anderson * the command array of the returned TCS where the client should 660e40b0c16SDouglas Anderson * start writing the message. 661e40b0c16SDouglas Anderson * 662e40b0c16SDouglas Anderson * Only for use on sleep/wake TCSes since those are the only ones we maintain 663e40b0c16SDouglas Anderson * tcs->slots for. 664e40b0c16SDouglas Anderson * 665e40b0c16SDouglas Anderson * Return: -ENOMEM if there was no room, else 0. 666e40b0c16SDouglas Anderson */ 667fa460e45SLina Iyer static int find_slots(struct tcs_group *tcs, const struct tcs_request *msg, 668fa460e45SLina Iyer int *tcs_id, int *cmd_id) 669fa460e45SLina Iyer { 670fa460e45SLina Iyer int slot, offset; 671fa460e45SLina Iyer int i = 0; 672fa460e45SLina Iyer 673e40b0c16SDouglas Anderson /* Do over, until we can fit the full payload in a single TCS */ 674fa460e45SLina Iyer do { 675fa460e45SLina Iyer slot = bitmap_find_next_zero_area(tcs->slots, MAX_TCS_SLOTS, 676fa460e45SLina Iyer i, msg->num_cmds, 0); 677bbeac60fSMaulik Shah if (slot >= tcs->num_tcs * tcs->ncpt) 678fa460e45SLina Iyer return -ENOMEM; 679fa460e45SLina Iyer i += tcs->ncpt; 680fa460e45SLina Iyer } while (slot + msg->num_cmds - 1 >= i); 681fa460e45SLina Iyer 682fa460e45SLina Iyer bitmap_set(tcs->slots, slot, msg->num_cmds); 683fa460e45SLina Iyer 684fa460e45SLina Iyer offset = slot / tcs->ncpt; 685fa460e45SLina Iyer *tcs_id = offset + tcs->offset; 686fa460e45SLina Iyer *cmd_id = slot % tcs->ncpt; 687fa460e45SLina Iyer 688fa460e45SLina Iyer return 0; 689fa460e45SLina Iyer } 690fa460e45SLina Iyer 691427ef4f7SDouglas Anderson /** 692e40b0c16SDouglas Anderson * rpmh_rsc_write_ctrl_data() - Write request to controller but don't trigger. 693e40b0c16SDouglas Anderson * @drv: The controller. 694e40b0c16SDouglas Anderson * @msg: The data to be written to the controller. 695427ef4f7SDouglas Anderson * 696e40b0c16SDouglas Anderson * This should only be called for for sleep/wake state, never active-only 697e40b0c16SDouglas Anderson * state. 698427ef4f7SDouglas Anderson * 699555701a4SDouglas Anderson * The caller must ensure that no other RPMH actions are happening and the 700555701a4SDouglas Anderson * controller is idle when this function is called since it runs lockless. 701555701a4SDouglas Anderson * 702e40b0c16SDouglas Anderson * Return: 0 if no error; else -error. 703427ef4f7SDouglas Anderson */ 704427ef4f7SDouglas Anderson int rpmh_rsc_write_ctrl_data(struct rsc_drv *drv, const struct tcs_request *msg) 705fa460e45SLina Iyer { 706fa460e45SLina Iyer struct tcs_group *tcs; 707fa460e45SLina Iyer int tcs_id = 0, cmd_id = 0; 708fa460e45SLina Iyer int ret; 709fa460e45SLina Iyer 710fa460e45SLina Iyer tcs = get_tcs_for_msg(drv, msg); 711fa460e45SLina Iyer if (IS_ERR(tcs)) 712fa460e45SLina Iyer return PTR_ERR(tcs); 713fa460e45SLina Iyer 714fa460e45SLina Iyer /* find the TCS id and the command in the TCS to write to */ 715fa460e45SLina Iyer ret = find_slots(tcs, msg, &tcs_id, &cmd_id); 716fa460e45SLina Iyer if (!ret) 717fa460e45SLina Iyer __tcs_buffer_write(drv, tcs_id, cmd_id, msg); 718fa460e45SLina Iyer 719fa460e45SLina Iyer return ret; 720fa460e45SLina Iyer } 721fa460e45SLina Iyer 722fa460e45SLina Iyer /** 723985427f9SMaulik Shah * rpmh_rsc_ctrlr_is_busy() - Check if any of the AMCs are busy. 724985427f9SMaulik Shah * @drv: The controller 725985427f9SMaulik Shah * 726985427f9SMaulik Shah * Checks if any of the AMCs are busy in handling ACTIVE sets. 727985427f9SMaulik Shah * This is called from the last cpu powering down before flushing 728985427f9SMaulik Shah * SLEEP and WAKE sets. If AMCs are busy, controller can not enter 729985427f9SMaulik Shah * power collapse, so deny from the last cpu's pm notification. 730985427f9SMaulik Shah * 731d2a8cfc6SDouglas Anderson * Context: Must be called with the drv->lock held. 732d2a8cfc6SDouglas Anderson * 733985427f9SMaulik Shah * Return: 734985427f9SMaulik Shah * * False - AMCs are idle 735985427f9SMaulik Shah * * True - AMCs are busy 736985427f9SMaulik Shah */ 737985427f9SMaulik Shah static bool rpmh_rsc_ctrlr_is_busy(struct rsc_drv *drv) 738985427f9SMaulik Shah { 739814a0d46SStephen Boyd unsigned long set; 740814a0d46SStephen Boyd const struct tcs_group *tcs = &drv->tcs[ACTIVE_TCS]; 741814a0d46SStephen Boyd unsigned long max; 742985427f9SMaulik Shah 743985427f9SMaulik Shah /* 744985427f9SMaulik Shah * If we made an active request on a RSC that does not have a 745985427f9SMaulik Shah * dedicated TCS for active state use, then re-purposed wake TCSes 746985427f9SMaulik Shah * should be checked for not busy, because we used wake TCSes for 747985427f9SMaulik Shah * active requests in this case. 748985427f9SMaulik Shah */ 749985427f9SMaulik Shah if (!tcs->num_tcs) 75053d49fe1SDouglas Anderson tcs = &drv->tcs[WAKE_TCS]; 751985427f9SMaulik Shah 752814a0d46SStephen Boyd max = tcs->offset + tcs->num_tcs; 753814a0d46SStephen Boyd set = find_next_bit(drv->tcs_in_use, max, tcs->offset); 754985427f9SMaulik Shah 755814a0d46SStephen Boyd return set < max; 756985427f9SMaulik Shah } 757985427f9SMaulik Shah 758e40b0c16SDouglas Anderson /** 759e40b0c16SDouglas Anderson * rpmh_rsc_cpu_pm_callback() - Check if any of the AMCs are busy. 760e40b0c16SDouglas Anderson * @nfb: Pointer to the notifier block in struct rsc_drv. 761e40b0c16SDouglas Anderson * @action: CPU_PM_ENTER, CPU_PM_ENTER_FAILED, or CPU_PM_EXIT. 762e40b0c16SDouglas Anderson * @v: Unused 763e40b0c16SDouglas Anderson * 764e40b0c16SDouglas Anderson * This function is given to cpu_pm_register_notifier so we can be informed 765e40b0c16SDouglas Anderson * about when CPUs go down. When all CPUs go down we know no more active 766e40b0c16SDouglas Anderson * transfers will be started so we write sleep/wake sets. This function gets 767e40b0c16SDouglas Anderson * called from cpuidle code paths and also at system suspend time. 768e40b0c16SDouglas Anderson * 769e40b0c16SDouglas Anderson * If its last CPU going down and AMCs are not busy then writes cached sleep 770e40b0c16SDouglas Anderson * and wake messages to TCSes. The firmware then takes care of triggering 771e40b0c16SDouglas Anderson * them when entering deepest low power modes. 772e40b0c16SDouglas Anderson * 773e40b0c16SDouglas Anderson * Return: See cpu_pm_register_notifier() 774e40b0c16SDouglas Anderson */ 775985427f9SMaulik Shah static int rpmh_rsc_cpu_pm_callback(struct notifier_block *nfb, 776985427f9SMaulik Shah unsigned long action, void *v) 777985427f9SMaulik Shah { 778985427f9SMaulik Shah struct rsc_drv *drv = container_of(nfb, struct rsc_drv, rsc_pm); 779985427f9SMaulik Shah int ret = NOTIFY_OK; 780d2a8cfc6SDouglas Anderson int cpus_in_pm; 781985427f9SMaulik Shah 782985427f9SMaulik Shah switch (action) { 783985427f9SMaulik Shah case CPU_PM_ENTER: 784d2a8cfc6SDouglas Anderson cpus_in_pm = atomic_inc_return(&drv->cpus_in_pm); 785d2a8cfc6SDouglas Anderson /* 786d2a8cfc6SDouglas Anderson * NOTE: comments for num_online_cpus() point out that it's 787d2a8cfc6SDouglas Anderson * only a snapshot so we need to be careful. It should be OK 788d2a8cfc6SDouglas Anderson * for us to use, though. It's important for us not to miss 789d2a8cfc6SDouglas Anderson * if we're the last CPU going down so it would only be a 790d2a8cfc6SDouglas Anderson * problem if a CPU went offline right after we did the check 791d2a8cfc6SDouglas Anderson * AND that CPU was not idle AND that CPU was the last non-idle 792d2a8cfc6SDouglas Anderson * CPU. That can't happen. CPUs would have to come out of idle 793d2a8cfc6SDouglas Anderson * before the CPU could go offline. 794d2a8cfc6SDouglas Anderson */ 795d2a8cfc6SDouglas Anderson if (cpus_in_pm < num_online_cpus()) 796d2a8cfc6SDouglas Anderson return NOTIFY_OK; 797985427f9SMaulik Shah break; 798985427f9SMaulik Shah case CPU_PM_ENTER_FAILED: 799985427f9SMaulik Shah case CPU_PM_EXIT: 800d2a8cfc6SDouglas Anderson atomic_dec(&drv->cpus_in_pm); 801d2a8cfc6SDouglas Anderson return NOTIFY_OK; 8021143c366SDouglas Anderson default: 803d2a8cfc6SDouglas Anderson return NOTIFY_DONE; 804985427f9SMaulik Shah } 805985427f9SMaulik Shah 806d2a8cfc6SDouglas Anderson /* 807d2a8cfc6SDouglas Anderson * It's likely we're on the last CPU. Grab the drv->lock and write 808d2a8cfc6SDouglas Anderson * out the sleep/wake commands to RPMH hardware. Grabbing the lock 809d2a8cfc6SDouglas Anderson * means that if we race with another CPU coming up we are still 810d2a8cfc6SDouglas Anderson * guaranteed to be safe. If another CPU came up just after we checked 811d2a8cfc6SDouglas Anderson * and has grabbed the lock or started an active transfer then we'll 812d2a8cfc6SDouglas Anderson * notice we're busy and abort. If another CPU comes up after we start 813d2a8cfc6SDouglas Anderson * flushing it will be blocked from starting an active transfer until 814d2a8cfc6SDouglas Anderson * we're done flushing. If another CPU starts an active transfer after 815d2a8cfc6SDouglas Anderson * we release the lock we're still OK because we're no longer the last 816d2a8cfc6SDouglas Anderson * CPU. 817d2a8cfc6SDouglas Anderson */ 818d2a8cfc6SDouglas Anderson if (spin_trylock(&drv->lock)) { 819d2a8cfc6SDouglas Anderson if (rpmh_rsc_ctrlr_is_busy(drv) || rpmh_flush(&drv->client)) 820985427f9SMaulik Shah ret = NOTIFY_BAD; 821d2a8cfc6SDouglas Anderson spin_unlock(&drv->lock); 822d2a8cfc6SDouglas Anderson } else { 823d2a8cfc6SDouglas Anderson /* Another CPU must be up */ 824d2a8cfc6SDouglas Anderson return NOTIFY_OK; 825985427f9SMaulik Shah } 826985427f9SMaulik Shah 827d2a8cfc6SDouglas Anderson if (ret == NOTIFY_BAD) { 828d2a8cfc6SDouglas Anderson /* Double-check if we're here because someone else is up */ 829d2a8cfc6SDouglas Anderson if (cpus_in_pm < num_online_cpus()) 830985427f9SMaulik Shah ret = NOTIFY_OK; 831d2a8cfc6SDouglas Anderson else 832c45def5dSDouglas Anderson /* We won't be called w/ CPU_PM_ENTER_FAILED */ 833d2a8cfc6SDouglas Anderson atomic_dec(&drv->cpus_in_pm); 834d2a8cfc6SDouglas Anderson } 835c45def5dSDouglas Anderson 836985427f9SMaulik Shah return ret; 837985427f9SMaulik Shah } 838985427f9SMaulik Shah 839658628e7SLina Iyer static int rpmh_probe_tcs_config(struct platform_device *pdev, 840985427f9SMaulik Shah struct rsc_drv *drv, void __iomem *base) 841658628e7SLina Iyer { 842658628e7SLina Iyer struct tcs_type_config { 843658628e7SLina Iyer u32 type; 844658628e7SLina Iyer u32 n; 845658628e7SLina Iyer } tcs_cfg[TCS_TYPE_NR] = { { 0 } }; 846658628e7SLina Iyer struct device_node *dn = pdev->dev.of_node; 847658628e7SLina Iyer u32 config, max_tcs, ncpt, offset; 848658628e7SLina Iyer int i, ret, n, st = 0; 849658628e7SLina Iyer struct tcs_group *tcs; 850658628e7SLina Iyer 851658628e7SLina Iyer ret = of_property_read_u32(dn, "qcom,tcs-offset", &offset); 852658628e7SLina Iyer if (ret) 853658628e7SLina Iyer return ret; 854658628e7SLina Iyer drv->tcs_base = base + offset; 855658628e7SLina Iyer 856658628e7SLina Iyer config = readl_relaxed(base + DRV_PRNT_CHLD_CONFIG); 857658628e7SLina Iyer 858658628e7SLina Iyer max_tcs = config; 859658628e7SLina Iyer max_tcs &= DRV_NUM_TCS_MASK << (DRV_NUM_TCS_SHIFT * drv->id); 860658628e7SLina Iyer max_tcs = max_tcs >> (DRV_NUM_TCS_SHIFT * drv->id); 861658628e7SLina Iyer 862658628e7SLina Iyer ncpt = config & (DRV_NCPT_MASK << DRV_NCPT_SHIFT); 863658628e7SLina Iyer ncpt = ncpt >> DRV_NCPT_SHIFT; 864658628e7SLina Iyer 865658628e7SLina Iyer n = of_property_count_u32_elems(dn, "qcom,tcs-config"); 866658628e7SLina Iyer if (n != 2 * TCS_TYPE_NR) 867658628e7SLina Iyer return -EINVAL; 868658628e7SLina Iyer 869658628e7SLina Iyer for (i = 0; i < TCS_TYPE_NR; i++) { 870658628e7SLina Iyer ret = of_property_read_u32_index(dn, "qcom,tcs-config", 871658628e7SLina Iyer i * 2, &tcs_cfg[i].type); 872658628e7SLina Iyer if (ret) 873658628e7SLina Iyer return ret; 874658628e7SLina Iyer if (tcs_cfg[i].type >= TCS_TYPE_NR) 875658628e7SLina Iyer return -EINVAL; 876658628e7SLina Iyer 877658628e7SLina Iyer ret = of_property_read_u32_index(dn, "qcom,tcs-config", 878658628e7SLina Iyer i * 2 + 1, &tcs_cfg[i].n); 879658628e7SLina Iyer if (ret) 880658628e7SLina Iyer return ret; 881658628e7SLina Iyer if (tcs_cfg[i].n > MAX_TCS_PER_TYPE) 882658628e7SLina Iyer return -EINVAL; 883658628e7SLina Iyer } 884658628e7SLina Iyer 885658628e7SLina Iyer for (i = 0; i < TCS_TYPE_NR; i++) { 886658628e7SLina Iyer tcs = &drv->tcs[tcs_cfg[i].type]; 887658628e7SLina Iyer if (tcs->drv) 888658628e7SLina Iyer return -EINVAL; 889658628e7SLina Iyer tcs->drv = drv; 890658628e7SLina Iyer tcs->type = tcs_cfg[i].type; 891658628e7SLina Iyer tcs->num_tcs = tcs_cfg[i].n; 892658628e7SLina Iyer tcs->ncpt = ncpt; 893658628e7SLina Iyer 894658628e7SLina Iyer if (!tcs->num_tcs || tcs->type == CONTROL_TCS) 895658628e7SLina Iyer continue; 896658628e7SLina Iyer 897658628e7SLina Iyer if (st + tcs->num_tcs > max_tcs || 898658628e7SLina Iyer st + tcs->num_tcs >= BITS_PER_BYTE * sizeof(tcs->mask)) 899658628e7SLina Iyer return -EINVAL; 900658628e7SLina Iyer 901658628e7SLina Iyer tcs->mask = ((1 << tcs->num_tcs) - 1) << st; 902658628e7SLina Iyer tcs->offset = st; 903658628e7SLina Iyer st += tcs->num_tcs; 904658628e7SLina Iyer } 905658628e7SLina Iyer 906658628e7SLina Iyer drv->num_tcs = st; 907658628e7SLina Iyer 908658628e7SLina Iyer return 0; 909658628e7SLina Iyer } 910658628e7SLina Iyer 911658628e7SLina Iyer static int rpmh_rsc_probe(struct platform_device *pdev) 912658628e7SLina Iyer { 913658628e7SLina Iyer struct device_node *dn = pdev->dev.of_node; 914658628e7SLina Iyer struct rsc_drv *drv; 915985427f9SMaulik Shah struct resource *res; 916985427f9SMaulik Shah char drv_id[10] = {0}; 917658628e7SLina Iyer int ret, irq; 918985427f9SMaulik Shah u32 solver_config; 919985427f9SMaulik Shah void __iomem *base; 920658628e7SLina Iyer 921fdd102b5SDouglas Anderson /* 922fdd102b5SDouglas Anderson * Even though RPMh doesn't directly use cmd-db, all of its children 923fdd102b5SDouglas Anderson * do. To avoid adding this check to our children we'll do it now. 924fdd102b5SDouglas Anderson */ 925fdd102b5SDouglas Anderson ret = cmd_db_ready(); 926fdd102b5SDouglas Anderson if (ret) { 927fdd102b5SDouglas Anderson if (ret != -EPROBE_DEFER) 928fdd102b5SDouglas Anderson dev_err(&pdev->dev, "Command DB not available (%d)\n", 929fdd102b5SDouglas Anderson ret); 930fdd102b5SDouglas Anderson return ret; 931fdd102b5SDouglas Anderson } 932fdd102b5SDouglas Anderson 933658628e7SLina Iyer drv = devm_kzalloc(&pdev->dev, sizeof(*drv), GFP_KERNEL); 934658628e7SLina Iyer if (!drv) 935658628e7SLina Iyer return -ENOMEM; 936658628e7SLina Iyer 937658628e7SLina Iyer ret = of_property_read_u32(dn, "qcom,drv-id", &drv->id); 938658628e7SLina Iyer if (ret) 939658628e7SLina Iyer return ret; 940658628e7SLina Iyer 941658628e7SLina Iyer drv->name = of_get_property(dn, "label", NULL); 942658628e7SLina Iyer if (!drv->name) 943658628e7SLina Iyer drv->name = dev_name(&pdev->dev); 944658628e7SLina Iyer 945985427f9SMaulik Shah snprintf(drv_id, ARRAY_SIZE(drv_id), "drv-%d", drv->id); 946985427f9SMaulik Shah res = platform_get_resource_byname(pdev, IORESOURCE_MEM, drv_id); 947985427f9SMaulik Shah base = devm_ioremap_resource(&pdev->dev, res); 948985427f9SMaulik Shah if (IS_ERR(base)) 949985427f9SMaulik Shah return PTR_ERR(base); 950985427f9SMaulik Shah 951985427f9SMaulik Shah ret = rpmh_probe_tcs_config(pdev, drv, base); 952658628e7SLina Iyer if (ret) 953658628e7SLina Iyer return ret; 954658628e7SLina Iyer 955658628e7SLina Iyer spin_lock_init(&drv->lock); 9562bc20f3cSStephen Boyd init_waitqueue_head(&drv->tcs_wait); 957658628e7SLina Iyer bitmap_zero(drv->tcs_in_use, MAX_TCS_NR); 958658628e7SLina Iyer 959658628e7SLina Iyer irq = platform_get_irq(pdev, drv->id); 960658628e7SLina Iyer if (irq < 0) 961658628e7SLina Iyer return irq; 962658628e7SLina Iyer 963658628e7SLina Iyer ret = devm_request_irq(&pdev->dev, irq, tcs_tx_done, 964658628e7SLina Iyer IRQF_TRIGGER_HIGH | IRQF_NO_SUSPEND, 965658628e7SLina Iyer drv->name, drv); 966658628e7SLina Iyer if (ret) 967658628e7SLina Iyer return ret; 968658628e7SLina Iyer 969985427f9SMaulik Shah /* 970985427f9SMaulik Shah * CPU PM notification are not required for controllers that support 971985427f9SMaulik Shah * 'HW solver' mode where they can be in autonomous mode executing low 972985427f9SMaulik Shah * power mode to power down. 973985427f9SMaulik Shah */ 974985427f9SMaulik Shah solver_config = readl_relaxed(base + DRV_SOLVER_CONFIG); 975985427f9SMaulik Shah solver_config &= DRV_HW_SOLVER_MASK << DRV_HW_SOLVER_SHIFT; 976985427f9SMaulik Shah solver_config = solver_config >> DRV_HW_SOLVER_SHIFT; 977985427f9SMaulik Shah if (!solver_config) { 978985427f9SMaulik Shah drv->rsc_pm.notifier_call = rpmh_rsc_cpu_pm_callback; 979985427f9SMaulik Shah cpu_pm_register_notifier(&drv->rsc_pm); 980985427f9SMaulik Shah } 981985427f9SMaulik Shah 982658628e7SLina Iyer /* Enable the active TCS to send requests immediately */ 983032c692aSDouglas Anderson writel_relaxed(drv->tcs[ACTIVE_TCS].mask, 984032c692aSDouglas Anderson drv->tcs_base + RSC_DRV_IRQ_ENABLE); 985658628e7SLina Iyer 986600513dfSLina Iyer spin_lock_init(&drv->client.cache_lock); 987600513dfSLina Iyer INIT_LIST_HEAD(&drv->client.cache); 988c8790cb6SLina Iyer INIT_LIST_HEAD(&drv->client.batch_cache); 989600513dfSLina Iyer 990c1038456SLina Iyer dev_set_drvdata(&pdev->dev, drv); 991c1038456SLina Iyer 992658628e7SLina Iyer return devm_of_platform_populate(&pdev->dev); 993658628e7SLina Iyer } 994658628e7SLina Iyer 995658628e7SLina Iyer static const struct of_device_id rpmh_drv_match[] = { 996658628e7SLina Iyer { .compatible = "qcom,rpmh-rsc", }, 997658628e7SLina Iyer { } 998658628e7SLina Iyer }; 999cb365926SJohn Stultz MODULE_DEVICE_TABLE(of, rpmh_drv_match); 1000658628e7SLina Iyer 1001658628e7SLina Iyer static struct platform_driver rpmh_driver = { 1002658628e7SLina Iyer .probe = rpmh_rsc_probe, 1003658628e7SLina Iyer .driver = { 1004658628e7SLina Iyer .name = "rpmh", 1005658628e7SLina Iyer .of_match_table = rpmh_drv_match, 10061a53ce9aSMaulik Shah .suppress_bind_attrs = true, 1007658628e7SLina Iyer }, 1008658628e7SLina Iyer }; 1009658628e7SLina Iyer 1010658628e7SLina Iyer static int __init rpmh_driver_init(void) 1011658628e7SLina Iyer { 1012658628e7SLina Iyer return platform_driver_register(&rpmh_driver); 1013658628e7SLina Iyer } 1014658628e7SLina Iyer arch_initcall(rpmh_driver_init); 1015cb365926SJohn Stultz 1016cb365926SJohn Stultz MODULE_DESCRIPTION("Qualcomm Technologies, Inc. RPMh Driver"); 1017cb365926SJohn Stultz MODULE_LICENSE("GPL v2"); 1018