1658628e7SLina Iyer // SPDX-License-Identifier: GPL-2.0 2658628e7SLina Iyer /* 3658628e7SLina Iyer * Copyright (c) 2016-2018, The Linux Foundation. All rights reserved. 4*6a283d64SMaulik Shah * Copyright (c) 2023-2024, Qualcomm Innovation Center, Inc. All rights reserved. 5658628e7SLina Iyer */ 6658628e7SLina Iyer 7658628e7SLina Iyer #define pr_fmt(fmt) "%s " fmt, KBUILD_MODNAME 8658628e7SLina Iyer 9658628e7SLina Iyer #include <linux/atomic.h> 10985427f9SMaulik Shah #include <linux/cpu_pm.h> 11658628e7SLina Iyer #include <linux/delay.h> 12658628e7SLina Iyer #include <linux/interrupt.h> 13658628e7SLina Iyer #include <linux/io.h> 1491160150SDouglas Anderson #include <linux/iopoll.h> 15658628e7SLina Iyer #include <linux/kernel.h> 16cccbe3e5SMaulik Shah #include <linux/ktime.h> 17658628e7SLina Iyer #include <linux/list.h> 18cb365926SJohn Stultz #include <linux/module.h> 1925092e61SLina Iyer #include <linux/notifier.h> 20658628e7SLina Iyer #include <linux/of.h> 21658628e7SLina Iyer #include <linux/of_irq.h> 22658628e7SLina Iyer #include <linux/of_platform.h> 23658628e7SLina Iyer #include <linux/platform_device.h> 2425092e61SLina Iyer #include <linux/pm_domain.h> 2525092e61SLina Iyer #include <linux/pm_runtime.h> 26658628e7SLina Iyer #include <linux/slab.h> 27658628e7SLina Iyer #include <linux/spinlock.h> 282bc20f3cSStephen Boyd #include <linux/wait.h> 29658628e7SLina Iyer 30cccbe3e5SMaulik Shah #include <clocksource/arm_arch_timer.h> 31fdd102b5SDouglas Anderson #include <soc/qcom/cmd-db.h> 32658628e7SLina Iyer #include <soc/qcom/tcs.h> 33658628e7SLina Iyer #include <dt-bindings/soc/qcom,rpmh-rsc.h> 34658628e7SLina Iyer 35658628e7SLina Iyer #include "rpmh-internal.h" 36658628e7SLina Iyer 37fc087fe5SLina Iyer #define CREATE_TRACE_POINTS 38fc087fe5SLina Iyer #include "trace-rpmh.h" 39fc087fe5SLina Iyer 4040482e4fSAbel Vesa 4140482e4fSAbel Vesa #define RSC_DRV_ID 0 4240482e4fSAbel Vesa 4340482e4fSAbel Vesa #define MAJOR_VER_MASK 0xFF 4440482e4fSAbel Vesa #define MAJOR_VER_SHIFT 16 4540482e4fSAbel Vesa #define MINOR_VER_MASK 0xFF 4640482e4fSAbel Vesa #define MINOR_VER_SHIFT 8 4740482e4fSAbel Vesa 4840482e4fSAbel Vesa enum { 4940482e4fSAbel Vesa RSC_DRV_TCS_OFFSET, 5040482e4fSAbel Vesa RSC_DRV_CMD_OFFSET, 5140482e4fSAbel Vesa DRV_SOLVER_CONFIG, 5240482e4fSAbel Vesa DRV_PRNT_CHLD_CONFIG, 5340482e4fSAbel Vesa RSC_DRV_IRQ_ENABLE, 5440482e4fSAbel Vesa RSC_DRV_IRQ_STATUS, 5540482e4fSAbel Vesa RSC_DRV_IRQ_CLEAR, 5640482e4fSAbel Vesa RSC_DRV_CMD_WAIT_FOR_CMPL, 5740482e4fSAbel Vesa RSC_DRV_CONTROL, 5840482e4fSAbel Vesa RSC_DRV_STATUS, 5940482e4fSAbel Vesa RSC_DRV_CMD_ENABLE, 6040482e4fSAbel Vesa RSC_DRV_CMD_MSGID, 6140482e4fSAbel Vesa RSC_DRV_CMD_ADDR, 6240482e4fSAbel Vesa RSC_DRV_CMD_DATA, 6340482e4fSAbel Vesa RSC_DRV_CMD_STATUS, 6440482e4fSAbel Vesa RSC_DRV_CMD_RESP_DATA, 6540482e4fSAbel Vesa }; 66658628e7SLina Iyer 67985427f9SMaulik Shah /* DRV HW Solver Configuration Information Register */ 68985427f9SMaulik Shah #define DRV_HW_SOLVER_MASK 1 69985427f9SMaulik Shah #define DRV_HW_SOLVER_SHIFT 24 70985427f9SMaulik Shah 71985427f9SMaulik Shah /* DRV TCS Configuration Information Register */ 72658628e7SLina Iyer #define DRV_NUM_TCS_MASK 0x3F 73658628e7SLina Iyer #define DRV_NUM_TCS_SHIFT 6 74658628e7SLina Iyer #define DRV_NCPT_MASK 0x1F 75658628e7SLina Iyer #define DRV_NCPT_SHIFT 27 76658628e7SLina Iyer 77cccbe3e5SMaulik Shah /* Offsets for CONTROL TCS Registers */ 78cccbe3e5SMaulik Shah #define RSC_DRV_CTL_TCS_DATA_HI 0x38 79cccbe3e5SMaulik Shah #define RSC_DRV_CTL_TCS_DATA_HI_MASK 0xFFFFFF 80cccbe3e5SMaulik Shah #define RSC_DRV_CTL_TCS_DATA_HI_VALID BIT(31) 81cccbe3e5SMaulik Shah #define RSC_DRV_CTL_TCS_DATA_LO 0x40 82cccbe3e5SMaulik Shah #define RSC_DRV_CTL_TCS_DATA_LO_MASK 0xFFFFFFFF 83cccbe3e5SMaulik Shah #define RSC_DRV_CTL_TCS_DATA_SIZE 32 84cccbe3e5SMaulik Shah 85658628e7SLina Iyer #define TCS_AMC_MODE_ENABLE BIT(16) 86658628e7SLina Iyer #define TCS_AMC_MODE_TRIGGER BIT(24) 87658628e7SLina Iyer 88658628e7SLina Iyer /* TCS CMD register bit mask */ 89658628e7SLina Iyer #define CMD_MSGID_LEN 8 90658628e7SLina Iyer #define CMD_MSGID_RESP_REQ BIT(8) 91658628e7SLina Iyer #define CMD_MSGID_WRITE BIT(16) 92658628e7SLina Iyer #define CMD_STATUS_ISSUED BIT(8) 93658628e7SLina Iyer #define CMD_STATUS_COMPL BIT(16) 94658628e7SLina Iyer 951f7dbeb5SDouglas Anderson /* 961f7dbeb5SDouglas Anderson * Here's a high level overview of how all the registers in RPMH work 971f7dbeb5SDouglas Anderson * together: 981f7dbeb5SDouglas Anderson * 991f7dbeb5SDouglas Anderson * - The main rpmh-rsc address is the base of a register space that can 1001f7dbeb5SDouglas Anderson * be used to find overall configuration of the hardware 1011f7dbeb5SDouglas Anderson * (DRV_PRNT_CHLD_CONFIG). Also found within the rpmh-rsc register 1021f7dbeb5SDouglas Anderson * space are all the TCS blocks. The offset of the TCS blocks is 1031f7dbeb5SDouglas Anderson * specified in the device tree by "qcom,tcs-offset" and used to 1041f7dbeb5SDouglas Anderson * compute tcs_base. 1051f7dbeb5SDouglas Anderson * - TCS blocks come one after another. Type, count, and order are 1061f7dbeb5SDouglas Anderson * specified by the device tree as "qcom,tcs-config". 1071f7dbeb5SDouglas Anderson * - Each TCS block has some registers, then space for up to 16 commands. 1081f7dbeb5SDouglas Anderson * Note that though address space is reserved for 16 commands, fewer 1091f7dbeb5SDouglas Anderson * might be present. See ncpt (num cmds per TCS). 1101f7dbeb5SDouglas Anderson * 1111f7dbeb5SDouglas Anderson * Here's a picture: 1121f7dbeb5SDouglas Anderson * 1131f7dbeb5SDouglas Anderson * +---------------------------------------------------+ 1141f7dbeb5SDouglas Anderson * |RSC | 1151f7dbeb5SDouglas Anderson * | ctrl | 1161f7dbeb5SDouglas Anderson * | | 1171f7dbeb5SDouglas Anderson * | Drvs: | 1181f7dbeb5SDouglas Anderson * | +-----------------------------------------------+ | 1191f7dbeb5SDouglas Anderson * | |DRV0 | | 1201f7dbeb5SDouglas Anderson * | | ctrl/config | | 1211f7dbeb5SDouglas Anderson * | | IRQ | | 1221f7dbeb5SDouglas Anderson * | | | | 1231f7dbeb5SDouglas Anderson * | | TCSes: | | 1241f7dbeb5SDouglas Anderson * | | +------------------------------------------+ | | 1251f7dbeb5SDouglas Anderson * | | |TCS0 | | | | | | | | | | | | | | | 1261f7dbeb5SDouglas Anderson * | | | ctrl | 0| 1| 2| 3| 4| 5| .| .| .| .|14|15| | | 1271f7dbeb5SDouglas Anderson * | | | | | | | | | | | | | | | | | | 1281f7dbeb5SDouglas Anderson * | | +------------------------------------------+ | | 1291f7dbeb5SDouglas Anderson * | | +------------------------------------------+ | | 1301f7dbeb5SDouglas Anderson * | | |TCS1 | | | | | | | | | | | | | | | 1311f7dbeb5SDouglas Anderson * | | | ctrl | 0| 1| 2| 3| 4| 5| .| .| .| .|14|15| | | 1321f7dbeb5SDouglas Anderson * | | | | | | | | | | | | | | | | | | 1331f7dbeb5SDouglas Anderson * | | +------------------------------------------+ | | 1341f7dbeb5SDouglas Anderson * | | +------------------------------------------+ | | 1351f7dbeb5SDouglas Anderson * | | |TCS2 | | | | | | | | | | | | | | | 1361f7dbeb5SDouglas Anderson * | | | ctrl | 0| 1| 2| 3| 4| 5| .| .| .| .|14|15| | | 1371f7dbeb5SDouglas Anderson * | | | | | | | | | | | | | | | | | | 1381f7dbeb5SDouglas Anderson * | | +------------------------------------------+ | | 1391f7dbeb5SDouglas Anderson * | | ...... | | 1401f7dbeb5SDouglas Anderson * | +-----------------------------------------------+ | 1411f7dbeb5SDouglas Anderson * | +-----------------------------------------------+ | 1421f7dbeb5SDouglas Anderson * | |DRV1 | | 1431f7dbeb5SDouglas Anderson * | | (same as DRV0) | | 1441f7dbeb5SDouglas Anderson * | +-----------------------------------------------+ | 1451f7dbeb5SDouglas Anderson * | ...... | 1461f7dbeb5SDouglas Anderson * +---------------------------------------------------+ 1471f7dbeb5SDouglas Anderson */ 1481f7dbeb5SDouglas Anderson 149cccbe3e5SMaulik Shah #define USECS_TO_CYCLES(time_usecs) \ 150cccbe3e5SMaulik Shah xloops_to_cycles((time_usecs) * 0x10C7UL) 151cccbe3e5SMaulik Shah 152cccbe3e5SMaulik Shah static inline unsigned long xloops_to_cycles(u64 xloops) 153cccbe3e5SMaulik Shah { 154cccbe3e5SMaulik Shah return (xloops * loops_per_jiffy * HZ) >> 32; 155cccbe3e5SMaulik Shah } 156cccbe3e5SMaulik Shah 15740482e4fSAbel Vesa static u32 rpmh_rsc_reg_offset_ver_2_7[] = { 15840482e4fSAbel Vesa [RSC_DRV_TCS_OFFSET] = 672, 15940482e4fSAbel Vesa [RSC_DRV_CMD_OFFSET] = 20, 16040482e4fSAbel Vesa [DRV_SOLVER_CONFIG] = 0x04, 16140482e4fSAbel Vesa [DRV_PRNT_CHLD_CONFIG] = 0x0C, 16240482e4fSAbel Vesa [RSC_DRV_IRQ_ENABLE] = 0x00, 16340482e4fSAbel Vesa [RSC_DRV_IRQ_STATUS] = 0x04, 16440482e4fSAbel Vesa [RSC_DRV_IRQ_CLEAR] = 0x08, 16540482e4fSAbel Vesa [RSC_DRV_CMD_WAIT_FOR_CMPL] = 0x10, 16640482e4fSAbel Vesa [RSC_DRV_CONTROL] = 0x14, 16740482e4fSAbel Vesa [RSC_DRV_STATUS] = 0x18, 16840482e4fSAbel Vesa [RSC_DRV_CMD_ENABLE] = 0x1C, 16940482e4fSAbel Vesa [RSC_DRV_CMD_MSGID] = 0x30, 17040482e4fSAbel Vesa [RSC_DRV_CMD_ADDR] = 0x34, 17140482e4fSAbel Vesa [RSC_DRV_CMD_DATA] = 0x38, 17240482e4fSAbel Vesa [RSC_DRV_CMD_STATUS] = 0x3C, 17340482e4fSAbel Vesa [RSC_DRV_CMD_RESP_DATA] = 0x40, 17440482e4fSAbel Vesa }; 17540482e4fSAbel Vesa 17640482e4fSAbel Vesa static u32 rpmh_rsc_reg_offset_ver_3_0[] = { 17740482e4fSAbel Vesa [RSC_DRV_TCS_OFFSET] = 672, 17840482e4fSAbel Vesa [RSC_DRV_CMD_OFFSET] = 24, 17940482e4fSAbel Vesa [DRV_SOLVER_CONFIG] = 0x04, 18040482e4fSAbel Vesa [DRV_PRNT_CHLD_CONFIG] = 0x0C, 18140482e4fSAbel Vesa [RSC_DRV_IRQ_ENABLE] = 0x00, 18240482e4fSAbel Vesa [RSC_DRV_IRQ_STATUS] = 0x04, 18340482e4fSAbel Vesa [RSC_DRV_IRQ_CLEAR] = 0x08, 18440482e4fSAbel Vesa [RSC_DRV_CMD_WAIT_FOR_CMPL] = 0x20, 18540482e4fSAbel Vesa [RSC_DRV_CONTROL] = 0x24, 18640482e4fSAbel Vesa [RSC_DRV_STATUS] = 0x28, 18740482e4fSAbel Vesa [RSC_DRV_CMD_ENABLE] = 0x2C, 18840482e4fSAbel Vesa [RSC_DRV_CMD_MSGID] = 0x34, 18940482e4fSAbel Vesa [RSC_DRV_CMD_ADDR] = 0x38, 19040482e4fSAbel Vesa [RSC_DRV_CMD_DATA] = 0x3C, 19140482e4fSAbel Vesa [RSC_DRV_CMD_STATUS] = 0x40, 19240482e4fSAbel Vesa [RSC_DRV_CMD_RESP_DATA] = 0x44, 19340482e4fSAbel Vesa }; 19440482e4fSAbel Vesa 195faa0c1f1SDouglas Anderson static inline void __iomem * 196faa0c1f1SDouglas Anderson tcs_reg_addr(const struct rsc_drv *drv, int reg, int tcs_id) 197658628e7SLina Iyer { 19840482e4fSAbel Vesa return drv->tcs_base + drv->regs[RSC_DRV_TCS_OFFSET] * tcs_id + reg; 199658628e7SLina Iyer } 200658628e7SLina Iyer 201faa0c1f1SDouglas Anderson static inline void __iomem * 202faa0c1f1SDouglas Anderson tcs_cmd_addr(const struct rsc_drv *drv, int reg, int tcs_id, int cmd_id) 2033b5e3d50SDouglas Anderson { 20440482e4fSAbel Vesa return tcs_reg_addr(drv, reg, tcs_id) + drv->regs[RSC_DRV_CMD_OFFSET] * cmd_id; 2053b5e3d50SDouglas Anderson } 2063b5e3d50SDouglas Anderson 207faa0c1f1SDouglas Anderson static u32 read_tcs_cmd(const struct rsc_drv *drv, int reg, int tcs_id, 208faa0c1f1SDouglas Anderson int cmd_id) 209faa0c1f1SDouglas Anderson { 210faa0c1f1SDouglas Anderson return readl_relaxed(tcs_cmd_addr(drv, reg, tcs_id, cmd_id)); 211faa0c1f1SDouglas Anderson } 212faa0c1f1SDouglas Anderson 213faa0c1f1SDouglas Anderson static u32 read_tcs_reg(const struct rsc_drv *drv, int reg, int tcs_id) 214faa0c1f1SDouglas Anderson { 215faa0c1f1SDouglas Anderson return readl_relaxed(tcs_reg_addr(drv, reg, tcs_id)); 216faa0c1f1SDouglas Anderson } 217faa0c1f1SDouglas Anderson 218faa0c1f1SDouglas Anderson static void write_tcs_cmd(const struct rsc_drv *drv, int reg, int tcs_id, 219faa0c1f1SDouglas Anderson int cmd_id, u32 data) 220faa0c1f1SDouglas Anderson { 221faa0c1f1SDouglas Anderson writel_relaxed(data, tcs_cmd_addr(drv, reg, tcs_id, cmd_id)); 222faa0c1f1SDouglas Anderson } 223faa0c1f1SDouglas Anderson 224faa0c1f1SDouglas Anderson static void write_tcs_reg(const struct rsc_drv *drv, int reg, int tcs_id, 225658628e7SLina Iyer u32 data) 226658628e7SLina Iyer { 227faa0c1f1SDouglas Anderson writel_relaxed(data, tcs_reg_addr(drv, reg, tcs_id)); 228658628e7SLina Iyer } 229658628e7SLina Iyer 230faa0c1f1SDouglas Anderson static void write_tcs_reg_sync(const struct rsc_drv *drv, int reg, int tcs_id, 231658628e7SLina Iyer u32 data) 232658628e7SLina Iyer { 233be24c6a7SDouglas Anderson int i; 23491160150SDouglas Anderson 235faa0c1f1SDouglas Anderson writel(data, tcs_reg_addr(drv, reg, tcs_id)); 236be24c6a7SDouglas Anderson 237be24c6a7SDouglas Anderson /* 238be24c6a7SDouglas Anderson * Wait until we read back the same value. Use a counter rather than 239be24c6a7SDouglas Anderson * ktime for timeout since this may be called after timekeeping stops. 240be24c6a7SDouglas Anderson */ 241be24c6a7SDouglas Anderson for (i = 0; i < USEC_PER_SEC; i++) { 242be24c6a7SDouglas Anderson if (readl(tcs_reg_addr(drv, reg, tcs_id)) == data) 243be24c6a7SDouglas Anderson return; 244be24c6a7SDouglas Anderson udelay(1); 245be24c6a7SDouglas Anderson } 24691160150SDouglas Anderson pr_err("%s: error writing %#x to %d:%#x\n", drv->name, 24791160150SDouglas Anderson data, tcs_id, reg); 248658628e7SLina Iyer } 249658628e7SLina Iyer 250e40b0c16SDouglas Anderson /** 251e40b0c16SDouglas Anderson * tcs_invalidate() - Invalidate all TCSes of the given type (sleep or wake). 252e40b0c16SDouglas Anderson * @drv: The RSC controller. 253e40b0c16SDouglas Anderson * @type: SLEEP_TCS or WAKE_TCS 254e40b0c16SDouglas Anderson * 255e40b0c16SDouglas Anderson * This will clear the "slots" variable of the given tcs_group and also 256e40b0c16SDouglas Anderson * tell the hardware to forget about all entries. 257e40b0c16SDouglas Anderson * 258881808d0SDouglas Anderson * The caller must ensure that no other RPMH actions are happening when this 259881808d0SDouglas Anderson * function is called, since otherwise the device may immediately become 260881808d0SDouglas Anderson * used again even before this function exits. 261e40b0c16SDouglas Anderson */ 262881808d0SDouglas Anderson static void tcs_invalidate(struct rsc_drv *drv, int type) 2639a3afcfbSLina Iyer { 2649a3afcfbSLina Iyer int m; 26553d49fe1SDouglas Anderson struct tcs_group *tcs = &drv->tcs[type]; 2669a3afcfbSLina Iyer 267881808d0SDouglas Anderson /* Caller ensures nobody else is running so no lock */ 268881808d0SDouglas Anderson if (bitmap_empty(tcs->slots, MAX_TCS_SLOTS)) 269881808d0SDouglas Anderson return; 2709a3afcfbSLina Iyer 271fef419c4SLina Iyer for (m = tcs->offset; m < tcs->offset + tcs->num_tcs; m++) 27240482e4fSAbel Vesa write_tcs_reg_sync(drv, drv->regs[RSC_DRV_CMD_ENABLE], m, 0); 273fef419c4SLina Iyer 2749a3afcfbSLina Iyer bitmap_zero(tcs->slots, MAX_TCS_SLOTS); 2759a3afcfbSLina Iyer } 2769a3afcfbSLina Iyer 2779a3afcfbSLina Iyer /** 278e40b0c16SDouglas Anderson * rpmh_rsc_invalidate() - Invalidate sleep and wake TCSes. 279e40b0c16SDouglas Anderson * @drv: The RSC controller. 2809a3afcfbSLina Iyer * 281881808d0SDouglas Anderson * The caller must ensure that no other RPMH actions are happening when this 282881808d0SDouglas Anderson * function is called, since otherwise the device may immediately become 283881808d0SDouglas Anderson * used again even before this function exits. 2849a3afcfbSLina Iyer */ 285881808d0SDouglas Anderson void rpmh_rsc_invalidate(struct rsc_drv *drv) 2869a3afcfbSLina Iyer { 287881808d0SDouglas Anderson tcs_invalidate(drv, SLEEP_TCS); 288881808d0SDouglas Anderson tcs_invalidate(drv, WAKE_TCS); 2899a3afcfbSLina Iyer } 2909a3afcfbSLina Iyer 291e40b0c16SDouglas Anderson /** 292e40b0c16SDouglas Anderson * get_tcs_for_msg() - Get the tcs_group used to send the given message. 293e40b0c16SDouglas Anderson * @drv: The RSC controller. 294e40b0c16SDouglas Anderson * @msg: The message we want to send. 295e40b0c16SDouglas Anderson * 296e40b0c16SDouglas Anderson * This is normally pretty straightforward except if we are trying to send 297e40b0c16SDouglas Anderson * an ACTIVE_ONLY message but don't have any active_only TCSes. 298e40b0c16SDouglas Anderson * 299e40b0c16SDouglas Anderson * Return: A pointer to a tcs_group or an ERR_PTR. 300e40b0c16SDouglas Anderson */ 301658628e7SLina Iyer static struct tcs_group *get_tcs_for_msg(struct rsc_drv *drv, 302658628e7SLina Iyer const struct tcs_request *msg) 303658628e7SLina Iyer { 30438427e5aSMaulik Shah int type; 3052de4b8d3SLina Iyer struct tcs_group *tcs; 306658628e7SLina Iyer 307658628e7SLina Iyer switch (msg->state) { 308658628e7SLina Iyer case RPMH_ACTIVE_ONLY_STATE: 309658628e7SLina Iyer type = ACTIVE_TCS; 310658628e7SLina Iyer break; 311fa460e45SLina Iyer case RPMH_WAKE_ONLY_STATE: 312fa460e45SLina Iyer type = WAKE_TCS; 313fa460e45SLina Iyer break; 314fa460e45SLina Iyer case RPMH_SLEEP_STATE: 315fa460e45SLina Iyer type = SLEEP_TCS; 316fa460e45SLina Iyer break; 317658628e7SLina Iyer default: 318658628e7SLina Iyer return ERR_PTR(-EINVAL); 319658628e7SLina Iyer } 320658628e7SLina Iyer 3212de4b8d3SLina Iyer /* 3222de4b8d3SLina Iyer * If we are making an active request on a RSC that does not have a 3232de4b8d3SLina Iyer * dedicated TCS for active state use, then re-purpose a wake TCS to 324e40b0c16SDouglas Anderson * send active votes. This is safe because we ensure any active-only 325e40b0c16SDouglas Anderson * transfers have finished before we use it (maybe by running from 326e40b0c16SDouglas Anderson * the last CPU in PM code). 3272de4b8d3SLina Iyer */ 32853d49fe1SDouglas Anderson tcs = &drv->tcs[type]; 32938427e5aSMaulik Shah if (msg->state == RPMH_ACTIVE_ONLY_STATE && !tcs->num_tcs) 33053d49fe1SDouglas Anderson tcs = &drv->tcs[WAKE_TCS]; 3312de4b8d3SLina Iyer 3322de4b8d3SLina Iyer return tcs; 333658628e7SLina Iyer } 334658628e7SLina Iyer 335e40b0c16SDouglas Anderson /** 336e40b0c16SDouglas Anderson * get_req_from_tcs() - Get a stashed request that was xfering on the given TCS. 337e40b0c16SDouglas Anderson * @drv: The RSC controller. 338e40b0c16SDouglas Anderson * @tcs_id: The global ID of this TCS. 339e40b0c16SDouglas Anderson * 340e40b0c16SDouglas Anderson * For ACTIVE_ONLY transfers we want to call back into the client when the 341e40b0c16SDouglas Anderson * transfer finishes. To do this we need the "request" that the client 342e40b0c16SDouglas Anderson * originally provided us. This function grabs the request that we stashed 343e40b0c16SDouglas Anderson * when we started the transfer. 344e40b0c16SDouglas Anderson * 345e40b0c16SDouglas Anderson * This only makes sense for ACTIVE_ONLY transfers since those are the only 346e40b0c16SDouglas Anderson * ones we track sending (the only ones we enable interrupts for and the only 347e40b0c16SDouglas Anderson * ones we call back to the client for). 348e40b0c16SDouglas Anderson * 349e40b0c16SDouglas Anderson * Return: The stashed request. 350e40b0c16SDouglas Anderson */ 351658628e7SLina Iyer static const struct tcs_request *get_req_from_tcs(struct rsc_drv *drv, 352658628e7SLina Iyer int tcs_id) 353658628e7SLina Iyer { 354658628e7SLina Iyer struct tcs_group *tcs; 355658628e7SLina Iyer int i; 356658628e7SLina Iyer 357efa1c257SRaju P.L.S.S.S.N for (i = 0; i < TCS_TYPE_NR; i++) { 358658628e7SLina Iyer tcs = &drv->tcs[i]; 359658628e7SLina Iyer if (tcs->mask & BIT(tcs_id)) 360658628e7SLina Iyer return tcs->req[tcs_id - tcs->offset]; 361658628e7SLina Iyer } 362658628e7SLina Iyer 363658628e7SLina Iyer return NULL; 364658628e7SLina Iyer } 365658628e7SLina Iyer 366e40b0c16SDouglas Anderson /** 367e40b0c16SDouglas Anderson * __tcs_set_trigger() - Start xfer on a TCS or unset trigger on a borrowed TCS 368e40b0c16SDouglas Anderson * @drv: The controller. 369e40b0c16SDouglas Anderson * @tcs_id: The global ID of this TCS. 370e40b0c16SDouglas Anderson * @trigger: If true then untrigger/retrigger. If false then just untrigger. 371e40b0c16SDouglas Anderson * 372e40b0c16SDouglas Anderson * In the normal case we only ever call with "trigger=true" to start a 373e40b0c16SDouglas Anderson * transfer. That will un-trigger/disable the TCS from the last transfer 374e40b0c16SDouglas Anderson * then trigger/enable for this transfer. 375e40b0c16SDouglas Anderson * 376e40b0c16SDouglas Anderson * If we borrowed a wake TCS for an active-only transfer we'll also call 377e40b0c16SDouglas Anderson * this function with "trigger=false" to just do the un-trigger/disable 378e40b0c16SDouglas Anderson * before using the TCS for wake purposes again. 379e40b0c16SDouglas Anderson * 380e40b0c16SDouglas Anderson * Note that the AP is only in charge of triggering active-only transfers. 381e40b0c16SDouglas Anderson * The AP never triggers sleep/wake values using this function. 382e40b0c16SDouglas Anderson */ 38315b3bf61SRaju P.L.S.S.S.N static void __tcs_set_trigger(struct rsc_drv *drv, int tcs_id, bool trigger) 38415b3bf61SRaju P.L.S.S.S.N { 38515b3bf61SRaju P.L.S.S.S.N u32 enable; 38640482e4fSAbel Vesa u32 reg = drv->regs[RSC_DRV_CONTROL]; 38715b3bf61SRaju P.L.S.S.S.N 38815b3bf61SRaju P.L.S.S.S.N /* 38915b3bf61SRaju P.L.S.S.S.N * HW req: Clear the DRV_CONTROL and enable TCS again 39015b3bf61SRaju P.L.S.S.S.N * While clearing ensure that the AMC mode trigger is cleared 39115b3bf61SRaju P.L.S.S.S.N * and then the mode enable is cleared. 39215b3bf61SRaju P.L.S.S.S.N */ 39340482e4fSAbel Vesa enable = read_tcs_reg(drv, reg, tcs_id); 39415b3bf61SRaju P.L.S.S.S.N enable &= ~TCS_AMC_MODE_TRIGGER; 39540482e4fSAbel Vesa write_tcs_reg_sync(drv, reg, tcs_id, enable); 39615b3bf61SRaju P.L.S.S.S.N enable &= ~TCS_AMC_MODE_ENABLE; 39740482e4fSAbel Vesa write_tcs_reg_sync(drv, reg, tcs_id, enable); 39815b3bf61SRaju P.L.S.S.S.N 39915b3bf61SRaju P.L.S.S.S.N if (trigger) { 40015b3bf61SRaju P.L.S.S.S.N /* Enable the AMC mode on the TCS and then trigger the TCS */ 40115b3bf61SRaju P.L.S.S.S.N enable = TCS_AMC_MODE_ENABLE; 40240482e4fSAbel Vesa write_tcs_reg_sync(drv, reg, tcs_id, enable); 40315b3bf61SRaju P.L.S.S.S.N enable |= TCS_AMC_MODE_TRIGGER; 40440482e4fSAbel Vesa write_tcs_reg(drv, reg, tcs_id, enable); 40515b3bf61SRaju P.L.S.S.S.N } 40615b3bf61SRaju P.L.S.S.S.N } 40715b3bf61SRaju P.L.S.S.S.N 408e40b0c16SDouglas Anderson /** 409e40b0c16SDouglas Anderson * enable_tcs_irq() - Enable or disable interrupts on the given TCS. 410e40b0c16SDouglas Anderson * @drv: The controller. 411e40b0c16SDouglas Anderson * @tcs_id: The global ID of this TCS. 412e40b0c16SDouglas Anderson * @enable: If true then enable; if false then disable 413e40b0c16SDouglas Anderson * 414e40b0c16SDouglas Anderson * We only ever call this when we borrow a wake TCS for an active-only 415e40b0c16SDouglas Anderson * transfer. For active-only TCSes interrupts are always left enabled. 416e40b0c16SDouglas Anderson */ 41715b3bf61SRaju P.L.S.S.S.N static void enable_tcs_irq(struct rsc_drv *drv, int tcs_id, bool enable) 41815b3bf61SRaju P.L.S.S.S.N { 41915b3bf61SRaju P.L.S.S.S.N u32 data; 42040482e4fSAbel Vesa u32 reg = drv->regs[RSC_DRV_IRQ_ENABLE]; 42115b3bf61SRaju P.L.S.S.S.N 42240482e4fSAbel Vesa data = readl_relaxed(drv->tcs_base + reg); 42315b3bf61SRaju P.L.S.S.S.N if (enable) 42415b3bf61SRaju P.L.S.S.S.N data |= BIT(tcs_id); 42515b3bf61SRaju P.L.S.S.S.N else 42615b3bf61SRaju P.L.S.S.S.N data &= ~BIT(tcs_id); 42740482e4fSAbel Vesa writel_relaxed(data, drv->tcs_base + reg); 42815b3bf61SRaju P.L.S.S.S.N } 42915b3bf61SRaju P.L.S.S.S.N 430658628e7SLina Iyer /** 431e40b0c16SDouglas Anderson * tcs_tx_done() - TX Done interrupt handler. 432e40b0c16SDouglas Anderson * @irq: The IRQ number (ignored). 433e40b0c16SDouglas Anderson * @p: Pointer to "struct rsc_drv". 434e40b0c16SDouglas Anderson * 435e40b0c16SDouglas Anderson * Called for ACTIVE_ONLY transfers (those are the only ones we enable the 436e40b0c16SDouglas Anderson * IRQ for) when a transfer is done. 437e40b0c16SDouglas Anderson * 438e40b0c16SDouglas Anderson * Return: IRQ_HANDLED 439658628e7SLina Iyer */ 440658628e7SLina Iyer static irqreturn_t tcs_tx_done(int irq, void *p) 441658628e7SLina Iyer { 442658628e7SLina Iyer struct rsc_drv *drv = p; 443323dc2dcSAbel Vesa int i; 444658628e7SLina Iyer unsigned long irq_status; 445658628e7SLina Iyer const struct tcs_request *req; 446658628e7SLina Iyer 44740482e4fSAbel Vesa irq_status = readl_relaxed(drv->tcs_base + drv->regs[RSC_DRV_IRQ_STATUS]); 448658628e7SLina Iyer 449a0776620SStephen Boyd for_each_set_bit(i, &irq_status, BITS_PER_TYPE(u32)) { 450658628e7SLina Iyer req = get_req_from_tcs(drv, i); 451f2590e4bSStephen Boyd if (WARN_ON(!req)) 452658628e7SLina Iyer goto skip; 453658628e7SLina Iyer 454323dc2dcSAbel Vesa trace_rpmh_tx_done(drv, i, req); 45515b3bf61SRaju P.L.S.S.S.N 45615b3bf61SRaju P.L.S.S.S.N /* 45715b3bf61SRaju P.L.S.S.S.N * If wake tcs was re-purposed for sending active 45815b3bf61SRaju P.L.S.S.S.N * votes, clear AMC trigger & enable modes and 45915b3bf61SRaju P.L.S.S.S.N * disable interrupt for this TCS 46015b3bf61SRaju P.L.S.S.S.N */ 46115b3bf61SRaju P.L.S.S.S.N if (!drv->tcs[ACTIVE_TCS].num_tcs) 46215b3bf61SRaju P.L.S.S.S.N __tcs_set_trigger(drv, i, false); 463658628e7SLina Iyer skip: 464658628e7SLina Iyer /* Reclaim the TCS */ 46540482e4fSAbel Vesa write_tcs_reg(drv, drv->regs[RSC_DRV_CMD_ENABLE], i, 0); 46640482e4fSAbel Vesa writel_relaxed(BIT(i), drv->tcs_base + drv->regs[RSC_DRV_IRQ_CLEAR]); 467658628e7SLina Iyer spin_lock(&drv->lock); 468658628e7SLina Iyer clear_bit(i, drv->tcs_in_use); 46915b3bf61SRaju P.L.S.S.S.N /* 47015b3bf61SRaju P.L.S.S.S.N * Disable interrupt for WAKE TCS to avoid being 47115b3bf61SRaju P.L.S.S.S.N * spammed with interrupts coming when the solver 47215b3bf61SRaju P.L.S.S.S.N * sends its wake votes. 47315b3bf61SRaju P.L.S.S.S.N */ 47415b3bf61SRaju P.L.S.S.S.N if (!drv->tcs[ACTIVE_TCS].num_tcs) 47515b3bf61SRaju P.L.S.S.S.N enable_tcs_irq(drv, i, false); 476658628e7SLina Iyer spin_unlock(&drv->lock); 4772bc20f3cSStephen Boyd wake_up(&drv->tcs_wait); 478c1038456SLina Iyer if (req) 479323dc2dcSAbel Vesa rpmh_tx_done(req); 480658628e7SLina Iyer } 481658628e7SLina Iyer 482658628e7SLina Iyer return IRQ_HANDLED; 483658628e7SLina Iyer } 484658628e7SLina Iyer 485e40b0c16SDouglas Anderson /** 486e40b0c16SDouglas Anderson * __tcs_buffer_write() - Write to TCS hardware from a request; don't trigger. 487e40b0c16SDouglas Anderson * @drv: The controller. 488e40b0c16SDouglas Anderson * @tcs_id: The global ID of this TCS. 489e40b0c16SDouglas Anderson * @cmd_id: The index within the TCS to start writing. 490e40b0c16SDouglas Anderson * @msg: The message we want to send, which will contain several addr/data 491e40b0c16SDouglas Anderson * pairs to program (but few enough that they all fit in one TCS). 492e40b0c16SDouglas Anderson * 493e40b0c16SDouglas Anderson * This is used for all types of transfers (active, sleep, and wake). 494e40b0c16SDouglas Anderson */ 495658628e7SLina Iyer static void __tcs_buffer_write(struct rsc_drv *drv, int tcs_id, int cmd_id, 496658628e7SLina Iyer const struct tcs_request *msg) 497658628e7SLina Iyer { 498fef419c4SLina Iyer u32 msgid; 499fef419c4SLina Iyer u32 cmd_msgid = CMD_MSGID_LEN | CMD_MSGID_WRITE; 500658628e7SLina Iyer u32 cmd_enable = 0; 501658628e7SLina Iyer struct tcs_cmd *cmd; 502658628e7SLina Iyer int i, j; 503658628e7SLina Iyer 504fef419c4SLina Iyer /* Convert all commands to RR when the request has wait_for_compl set */ 505658628e7SLina Iyer cmd_msgid |= msg->wait_for_compl ? CMD_MSGID_RESP_REQ : 0; 506658628e7SLina Iyer 507658628e7SLina Iyer for (i = 0, j = cmd_id; i < msg->num_cmds; i++, j++) { 508658628e7SLina Iyer cmd = &msg->cmds[i]; 509658628e7SLina Iyer cmd_enable |= BIT(j); 510658628e7SLina Iyer msgid = cmd_msgid; 511fef419c4SLina Iyer /* 512fef419c4SLina Iyer * Additionally, if the cmd->wait is set, make the command 513fef419c4SLina Iyer * response reqd even if the overall request was fire-n-forget. 514fef419c4SLina Iyer */ 515658628e7SLina Iyer msgid |= cmd->wait ? CMD_MSGID_RESP_REQ : 0; 516fc087fe5SLina Iyer 51740482e4fSAbel Vesa write_tcs_cmd(drv, drv->regs[RSC_DRV_CMD_MSGID], tcs_id, j, msgid); 51840482e4fSAbel Vesa write_tcs_cmd(drv, drv->regs[RSC_DRV_CMD_ADDR], tcs_id, j, cmd->addr); 51940482e4fSAbel Vesa write_tcs_cmd(drv, drv->regs[RSC_DRV_CMD_DATA], tcs_id, j, cmd->data); 5201b06d8caSBjorn Andersson trace_rpmh_send_msg(drv, tcs_id, msg->state, j, msgid, cmd); 521658628e7SLina Iyer } 522658628e7SLina Iyer 52340482e4fSAbel Vesa cmd_enable |= read_tcs_reg(drv, drv->regs[RSC_DRV_CMD_ENABLE], tcs_id); 52440482e4fSAbel Vesa write_tcs_reg(drv, drv->regs[RSC_DRV_CMD_ENABLE], tcs_id, cmd_enable); 525658628e7SLina Iyer } 526658628e7SLina Iyer 527e40b0c16SDouglas Anderson /** 528e40b0c16SDouglas Anderson * check_for_req_inflight() - Look to see if conflicting cmds are in flight. 529e40b0c16SDouglas Anderson * @drv: The controller. 530e40b0c16SDouglas Anderson * @tcs: A pointer to the tcs_group used for ACTIVE_ONLY transfers. 531e40b0c16SDouglas Anderson * @msg: The message we want to send, which will contain several addr/data 532e40b0c16SDouglas Anderson * pairs to program (but few enough that they all fit in one TCS). 533e40b0c16SDouglas Anderson * 534e40b0c16SDouglas Anderson * This will walk through the TCSes in the group and check if any of them 535e40b0c16SDouglas Anderson * appear to be sending to addresses referenced in the message. If it finds 536e40b0c16SDouglas Anderson * one it'll return -EBUSY. 537e40b0c16SDouglas Anderson * 538e40b0c16SDouglas Anderson * Only for use for active-only transfers. 539e40b0c16SDouglas Anderson * 540e40b0c16SDouglas Anderson * Must be called with the drv->lock held since that protects tcs_in_use. 541e40b0c16SDouglas Anderson * 542e40b0c16SDouglas Anderson * Return: 0 if nothing in flight or -EBUSY if we should try again later. 543e40b0c16SDouglas Anderson * The caller must re-enable interrupts between tries since that's 544814a0d46SStephen Boyd * the only way tcs_in_use will ever be updated and the only way 545e40b0c16SDouglas Anderson * RSC_DRV_CMD_ENABLE will ever be cleared. 546e40b0c16SDouglas Anderson */ 547658628e7SLina Iyer static int check_for_req_inflight(struct rsc_drv *drv, struct tcs_group *tcs, 548658628e7SLina Iyer const struct tcs_request *msg) 549658628e7SLina Iyer { 550658628e7SLina Iyer unsigned long curr_enabled; 551658628e7SLina Iyer u32 addr; 552814a0d46SStephen Boyd int j, k; 553814a0d46SStephen Boyd int i = tcs->offset; 554658628e7SLina Iyer 555814a0d46SStephen Boyd for_each_set_bit_from(i, drv->tcs_in_use, tcs->offset + tcs->num_tcs) { 55640482e4fSAbel Vesa curr_enabled = read_tcs_reg(drv, drv->regs[RSC_DRV_CMD_ENABLE], i); 557658628e7SLina Iyer 558658628e7SLina Iyer for_each_set_bit(j, &curr_enabled, MAX_CMDS_PER_TCS) { 55940482e4fSAbel Vesa addr = read_tcs_cmd(drv, drv->regs[RSC_DRV_CMD_ADDR], i, j); 560658628e7SLina Iyer for (k = 0; k < msg->num_cmds; k++) { 561*6a283d64SMaulik Shah if (cmd_db_match_resource_addr(msg->cmds[k].addr, addr)) 562658628e7SLina Iyer return -EBUSY; 563658628e7SLina Iyer } 564658628e7SLina Iyer } 565658628e7SLina Iyer } 566658628e7SLina Iyer 567658628e7SLina Iyer return 0; 568658628e7SLina Iyer } 569658628e7SLina Iyer 570e40b0c16SDouglas Anderson /** 571e40b0c16SDouglas Anderson * find_free_tcs() - Find free tcs in the given tcs_group; only for active. 572e40b0c16SDouglas Anderson * @tcs: A pointer to the active-only tcs_group (or the wake tcs_group if 573e40b0c16SDouglas Anderson * we borrowed it because there are zero active-only ones). 574e40b0c16SDouglas Anderson * 575e40b0c16SDouglas Anderson * Must be called with the drv->lock held since that protects tcs_in_use. 576e40b0c16SDouglas Anderson * 577814a0d46SStephen Boyd * Return: The first tcs that's free or -EBUSY if all in use. 578e40b0c16SDouglas Anderson */ 579658628e7SLina Iyer static int find_free_tcs(struct tcs_group *tcs) 580658628e7SLina Iyer { 581814a0d46SStephen Boyd const struct rsc_drv *drv = tcs->drv; 582814a0d46SStephen Boyd unsigned long i; 583814a0d46SStephen Boyd unsigned long max = tcs->offset + tcs->num_tcs; 584658628e7SLina Iyer 585814a0d46SStephen Boyd i = find_next_zero_bit(drv->tcs_in_use, max, tcs->offset); 586814a0d46SStephen Boyd if (i >= max) 587658628e7SLina Iyer return -EBUSY; 588814a0d46SStephen Boyd 589814a0d46SStephen Boyd return i; 590658628e7SLina Iyer } 591658628e7SLina Iyer 592e40b0c16SDouglas Anderson /** 5932bc20f3cSStephen Boyd * claim_tcs_for_req() - Claim a tcs in the given tcs_group; only for active. 594e40b0c16SDouglas Anderson * @drv: The controller. 5952bc20f3cSStephen Boyd * @tcs: The tcs_group used for ACTIVE_ONLY transfers. 596e40b0c16SDouglas Anderson * @msg: The data to be sent. 597e40b0c16SDouglas Anderson * 5982bc20f3cSStephen Boyd * Claims a tcs in the given tcs_group while making sure that no existing cmd 5992bc20f3cSStephen Boyd * is in flight that would conflict with the one in @msg. 600e40b0c16SDouglas Anderson * 6012bc20f3cSStephen Boyd * Context: Must be called with the drv->lock held since that protects 6022bc20f3cSStephen Boyd * tcs_in_use. 603e40b0c16SDouglas Anderson * 6042bc20f3cSStephen Boyd * Return: The id of the claimed tcs or -EBUSY if a matching msg is in flight 6052bc20f3cSStephen Boyd * or the tcs_group is full. 606e40b0c16SDouglas Anderson */ 6072bc20f3cSStephen Boyd static int claim_tcs_for_req(struct rsc_drv *drv, struct tcs_group *tcs, 6082bc20f3cSStephen Boyd const struct tcs_request *msg) 609658628e7SLina Iyer { 610658628e7SLina Iyer int ret; 611658628e7SLina Iyer 612658628e7SLina Iyer /* 613658628e7SLina Iyer * The h/w does not like if we send a request to the same address, 614658628e7SLina Iyer * when one is already in-flight or being processed. 615658628e7SLina Iyer */ 616658628e7SLina Iyer ret = check_for_req_inflight(drv, tcs, msg); 617555701a4SDouglas Anderson if (ret) 6182bc20f3cSStephen Boyd return ret; 619658628e7SLina Iyer 6202bc20f3cSStephen Boyd return find_free_tcs(tcs); 6212bc20f3cSStephen Boyd } 6222bc20f3cSStephen Boyd 6232bc20f3cSStephen Boyd /** 6242bc20f3cSStephen Boyd * rpmh_rsc_send_data() - Write / trigger active-only message. 6252bc20f3cSStephen Boyd * @drv: The controller. 6262bc20f3cSStephen Boyd * @msg: The data to be sent. 6272bc20f3cSStephen Boyd * 6282bc20f3cSStephen Boyd * NOTES: 6292bc20f3cSStephen Boyd * - This is only used for "ACTIVE_ONLY" since the limitations of this 6302bc20f3cSStephen Boyd * function don't make sense for sleep/wake cases. 6312bc20f3cSStephen Boyd * - To do the transfer, we will grab a whole TCS for ourselves--we don't 6322bc20f3cSStephen Boyd * try to share. If there are none available we'll wait indefinitely 6332bc20f3cSStephen Boyd * for a free one. 6342bc20f3cSStephen Boyd * - This function will not wait for the commands to be finished, only for 6352bc20f3cSStephen Boyd * data to be programmed into the RPMh. See rpmh_tx_done() which will 6362bc20f3cSStephen Boyd * be called when the transfer is fully complete. 6372bc20f3cSStephen Boyd * - This function must be called with interrupts enabled. If the hardware 6382bc20f3cSStephen Boyd * is busy doing someone else's transfer we need that transfer to fully 6392bc20f3cSStephen Boyd * finish so that we can have the hardware, and to fully finish it needs 6402bc20f3cSStephen Boyd * the interrupt handler to run. If the interrupts is set to run on the 6412bc20f3cSStephen Boyd * active CPU this can never happen if interrupts are disabled. 6422bc20f3cSStephen Boyd * 6432bc20f3cSStephen Boyd * Return: 0 on success, -EINVAL on error. 6442bc20f3cSStephen Boyd */ 6452bc20f3cSStephen Boyd int rpmh_rsc_send_data(struct rsc_drv *drv, const struct tcs_request *msg) 6462bc20f3cSStephen Boyd { 6472bc20f3cSStephen Boyd struct tcs_group *tcs; 6482bc20f3cSStephen Boyd int tcs_id; 6492bc20f3cSStephen Boyd unsigned long flags; 6502bc20f3cSStephen Boyd 6512bc20f3cSStephen Boyd tcs = get_tcs_for_msg(drv, msg); 6522bc20f3cSStephen Boyd if (IS_ERR(tcs)) 6532bc20f3cSStephen Boyd return PTR_ERR(tcs); 6542bc20f3cSStephen Boyd 6552bc20f3cSStephen Boyd spin_lock_irqsave(&drv->lock, flags); 6562bc20f3cSStephen Boyd 6572bc20f3cSStephen Boyd /* Wait forever for a free tcs. It better be there eventually! */ 6582bc20f3cSStephen Boyd wait_event_lock_irq(drv->tcs_wait, 6592bc20f3cSStephen Boyd (tcs_id = claim_tcs_for_req(drv, tcs, msg)) >= 0, 6602bc20f3cSStephen Boyd drv->lock); 661658628e7SLina Iyer 662658628e7SLina Iyer tcs->req[tcs_id - tcs->offset] = msg; 663658628e7SLina Iyer set_bit(tcs_id, drv->tcs_in_use); 66438427e5aSMaulik Shah if (msg->state == RPMH_ACTIVE_ONLY_STATE && tcs->type != ACTIVE_TCS) { 66538427e5aSMaulik Shah /* 66638427e5aSMaulik Shah * Clear previously programmed WAKE commands in selected 66738427e5aSMaulik Shah * repurposed TCS to avoid triggering them. tcs->slots will be 66838427e5aSMaulik Shah * cleaned from rpmh_flush() by invoking rpmh_rsc_invalidate() 66938427e5aSMaulik Shah */ 67040482e4fSAbel Vesa write_tcs_reg_sync(drv, drv->regs[RSC_DRV_CMD_ENABLE], tcs_id, 0); 67115b3bf61SRaju P.L.S.S.S.N enable_tcs_irq(drv, tcs_id, true); 67238427e5aSMaulik Shah } 673555701a4SDouglas Anderson spin_unlock_irqrestore(&drv->lock, flags); 674658628e7SLina Iyer 675555701a4SDouglas Anderson /* 676555701a4SDouglas Anderson * These two can be done after the lock is released because: 677555701a4SDouglas Anderson * - We marked "tcs_in_use" under lock. 678555701a4SDouglas Anderson * - Once "tcs_in_use" has been marked nobody else could be writing 679555701a4SDouglas Anderson * to these registers until the interrupt goes off. 680555701a4SDouglas Anderson * - The interrupt can't go off until we trigger w/ the last line 681555701a4SDouglas Anderson * of __tcs_set_trigger() below. 682555701a4SDouglas Anderson */ 683658628e7SLina Iyer __tcs_buffer_write(drv, tcs_id, 0, msg); 68415b3bf61SRaju P.L.S.S.S.N __tcs_set_trigger(drv, tcs_id, true); 685658628e7SLina Iyer 686555701a4SDouglas Anderson return 0; 687658628e7SLina Iyer } 688658628e7SLina Iyer 689e40b0c16SDouglas Anderson /** 690e40b0c16SDouglas Anderson * find_slots() - Find a place to write the given message. 691e40b0c16SDouglas Anderson * @tcs: The tcs group to search. 692e40b0c16SDouglas Anderson * @msg: The message we want to find room for. 693e40b0c16SDouglas Anderson * @tcs_id: If we return 0 from the function, we return the global ID of the 694e40b0c16SDouglas Anderson * TCS to write to here. 695e40b0c16SDouglas Anderson * @cmd_id: If we return 0 from the function, we return the index of 696e40b0c16SDouglas Anderson * the command array of the returned TCS where the client should 697e40b0c16SDouglas Anderson * start writing the message. 698e40b0c16SDouglas Anderson * 699e40b0c16SDouglas Anderson * Only for use on sleep/wake TCSes since those are the only ones we maintain 700e40b0c16SDouglas Anderson * tcs->slots for. 701e40b0c16SDouglas Anderson * 702e40b0c16SDouglas Anderson * Return: -ENOMEM if there was no room, else 0. 703e40b0c16SDouglas Anderson */ 704fa460e45SLina Iyer static int find_slots(struct tcs_group *tcs, const struct tcs_request *msg, 705fa460e45SLina Iyer int *tcs_id, int *cmd_id) 706fa460e45SLina Iyer { 707fa460e45SLina Iyer int slot, offset; 708fa460e45SLina Iyer int i = 0; 709fa460e45SLina Iyer 710e40b0c16SDouglas Anderson /* Do over, until we can fit the full payload in a single TCS */ 711fa460e45SLina Iyer do { 712fa460e45SLina Iyer slot = bitmap_find_next_zero_area(tcs->slots, MAX_TCS_SLOTS, 713fa460e45SLina Iyer i, msg->num_cmds, 0); 714bbeac60fSMaulik Shah if (slot >= tcs->num_tcs * tcs->ncpt) 715fa460e45SLina Iyer return -ENOMEM; 716fa460e45SLina Iyer i += tcs->ncpt; 717fa460e45SLina Iyer } while (slot + msg->num_cmds - 1 >= i); 718fa460e45SLina Iyer 719fa460e45SLina Iyer bitmap_set(tcs->slots, slot, msg->num_cmds); 720fa460e45SLina Iyer 721fa460e45SLina Iyer offset = slot / tcs->ncpt; 722fa460e45SLina Iyer *tcs_id = offset + tcs->offset; 723fa460e45SLina Iyer *cmd_id = slot % tcs->ncpt; 724fa460e45SLina Iyer 725fa460e45SLina Iyer return 0; 726fa460e45SLina Iyer } 727fa460e45SLina Iyer 728427ef4f7SDouglas Anderson /** 729e40b0c16SDouglas Anderson * rpmh_rsc_write_ctrl_data() - Write request to controller but don't trigger. 730e40b0c16SDouglas Anderson * @drv: The controller. 731e40b0c16SDouglas Anderson * @msg: The data to be written to the controller. 732427ef4f7SDouglas Anderson * 733e395f021SJason Wang * This should only be called for sleep/wake state, never active-only 734e40b0c16SDouglas Anderson * state. 735427ef4f7SDouglas Anderson * 736555701a4SDouglas Anderson * The caller must ensure that no other RPMH actions are happening and the 737555701a4SDouglas Anderson * controller is idle when this function is called since it runs lockless. 738555701a4SDouglas Anderson * 739e40b0c16SDouglas Anderson * Return: 0 if no error; else -error. 740427ef4f7SDouglas Anderson */ 741427ef4f7SDouglas Anderson int rpmh_rsc_write_ctrl_data(struct rsc_drv *drv, const struct tcs_request *msg) 742fa460e45SLina Iyer { 743fa460e45SLina Iyer struct tcs_group *tcs; 744fa460e45SLina Iyer int tcs_id = 0, cmd_id = 0; 745fa460e45SLina Iyer int ret; 746fa460e45SLina Iyer 747fa460e45SLina Iyer tcs = get_tcs_for_msg(drv, msg); 748fa460e45SLina Iyer if (IS_ERR(tcs)) 749fa460e45SLina Iyer return PTR_ERR(tcs); 750fa460e45SLina Iyer 751fa460e45SLina Iyer /* find the TCS id and the command in the TCS to write to */ 752fa460e45SLina Iyer ret = find_slots(tcs, msg, &tcs_id, &cmd_id); 753fa460e45SLina Iyer if (!ret) 754fa460e45SLina Iyer __tcs_buffer_write(drv, tcs_id, cmd_id, msg); 755fa460e45SLina Iyer 756fa460e45SLina Iyer return ret; 757fa460e45SLina Iyer } 758fa460e45SLina Iyer 759fa460e45SLina Iyer /** 760985427f9SMaulik Shah * rpmh_rsc_ctrlr_is_busy() - Check if any of the AMCs are busy. 761985427f9SMaulik Shah * @drv: The controller 762985427f9SMaulik Shah * 763985427f9SMaulik Shah * Checks if any of the AMCs are busy in handling ACTIVE sets. 764985427f9SMaulik Shah * This is called from the last cpu powering down before flushing 765985427f9SMaulik Shah * SLEEP and WAKE sets. If AMCs are busy, controller can not enter 766985427f9SMaulik Shah * power collapse, so deny from the last cpu's pm notification. 767985427f9SMaulik Shah * 768d2a8cfc6SDouglas Anderson * Context: Must be called with the drv->lock held. 769d2a8cfc6SDouglas Anderson * 770985427f9SMaulik Shah * Return: 771985427f9SMaulik Shah * * False - AMCs are idle 772985427f9SMaulik Shah * * True - AMCs are busy 773985427f9SMaulik Shah */ 774985427f9SMaulik Shah static bool rpmh_rsc_ctrlr_is_busy(struct rsc_drv *drv) 775985427f9SMaulik Shah { 776814a0d46SStephen Boyd unsigned long set; 777814a0d46SStephen Boyd const struct tcs_group *tcs = &drv->tcs[ACTIVE_TCS]; 778814a0d46SStephen Boyd unsigned long max; 779985427f9SMaulik Shah 780985427f9SMaulik Shah /* 781985427f9SMaulik Shah * If we made an active request on a RSC that does not have a 782985427f9SMaulik Shah * dedicated TCS for active state use, then re-purposed wake TCSes 783985427f9SMaulik Shah * should be checked for not busy, because we used wake TCSes for 784985427f9SMaulik Shah * active requests in this case. 785985427f9SMaulik Shah */ 786985427f9SMaulik Shah if (!tcs->num_tcs) 78753d49fe1SDouglas Anderson tcs = &drv->tcs[WAKE_TCS]; 788985427f9SMaulik Shah 789814a0d46SStephen Boyd max = tcs->offset + tcs->num_tcs; 790814a0d46SStephen Boyd set = find_next_bit(drv->tcs_in_use, max, tcs->offset); 791985427f9SMaulik Shah 792814a0d46SStephen Boyd return set < max; 793985427f9SMaulik Shah } 794985427f9SMaulik Shah 795e40b0c16SDouglas Anderson /** 796cccbe3e5SMaulik Shah * rpmh_rsc_write_next_wakeup() - Write next wakeup in CONTROL_TCS. 797cccbe3e5SMaulik Shah * @drv: The controller 798cccbe3e5SMaulik Shah * 799cccbe3e5SMaulik Shah * Writes maximum wakeup cycles when called from suspend. 800cccbe3e5SMaulik Shah * Writes earliest hrtimer wakeup when called from idle. 801cccbe3e5SMaulik Shah */ 802cccbe3e5SMaulik Shah void rpmh_rsc_write_next_wakeup(struct rsc_drv *drv) 803cccbe3e5SMaulik Shah { 804cccbe3e5SMaulik Shah ktime_t now, wakeup; 805cccbe3e5SMaulik Shah u64 wakeup_us, wakeup_cycles = ~0; 806cccbe3e5SMaulik Shah u32 lo, hi; 807cccbe3e5SMaulik Shah 808cccbe3e5SMaulik Shah if (!drv->tcs[CONTROL_TCS].num_tcs || !drv->genpd_nb.notifier_call) 809cccbe3e5SMaulik Shah return; 810cccbe3e5SMaulik Shah 811cccbe3e5SMaulik Shah /* Set highest time when system (timekeeping) is suspended */ 812cccbe3e5SMaulik Shah if (system_state == SYSTEM_SUSPEND) 813cccbe3e5SMaulik Shah goto exit; 814cccbe3e5SMaulik Shah 815cccbe3e5SMaulik Shah /* Find the earliest hrtimer wakeup from online cpus */ 816cccbe3e5SMaulik Shah wakeup = dev_pm_genpd_get_next_hrtimer(drv->dev); 817cccbe3e5SMaulik Shah 818cccbe3e5SMaulik Shah /* Find the relative wakeup in kernel time scale */ 819cccbe3e5SMaulik Shah now = ktime_get(); 820cccbe3e5SMaulik Shah wakeup = ktime_sub(wakeup, now); 821cccbe3e5SMaulik Shah wakeup_us = ktime_to_us(wakeup); 822cccbe3e5SMaulik Shah 823cccbe3e5SMaulik Shah /* Convert the wakeup to arch timer scale */ 824cccbe3e5SMaulik Shah wakeup_cycles = USECS_TO_CYCLES(wakeup_us); 825cccbe3e5SMaulik Shah wakeup_cycles += arch_timer_read_counter(); 826cccbe3e5SMaulik Shah 827cccbe3e5SMaulik Shah exit: 828cccbe3e5SMaulik Shah lo = wakeup_cycles & RSC_DRV_CTL_TCS_DATA_LO_MASK; 829cccbe3e5SMaulik Shah hi = wakeup_cycles >> RSC_DRV_CTL_TCS_DATA_SIZE; 830cccbe3e5SMaulik Shah hi &= RSC_DRV_CTL_TCS_DATA_HI_MASK; 831cccbe3e5SMaulik Shah hi |= RSC_DRV_CTL_TCS_DATA_HI_VALID; 832cccbe3e5SMaulik Shah 833cccbe3e5SMaulik Shah writel_relaxed(lo, drv->base + RSC_DRV_CTL_TCS_DATA_LO); 834cccbe3e5SMaulik Shah writel_relaxed(hi, drv->base + RSC_DRV_CTL_TCS_DATA_HI); 835cccbe3e5SMaulik Shah } 836cccbe3e5SMaulik Shah 837cccbe3e5SMaulik Shah /** 838e40b0c16SDouglas Anderson * rpmh_rsc_cpu_pm_callback() - Check if any of the AMCs are busy. 839e40b0c16SDouglas Anderson * @nfb: Pointer to the notifier block in struct rsc_drv. 840e40b0c16SDouglas Anderson * @action: CPU_PM_ENTER, CPU_PM_ENTER_FAILED, or CPU_PM_EXIT. 841e40b0c16SDouglas Anderson * @v: Unused 842e40b0c16SDouglas Anderson * 843e40b0c16SDouglas Anderson * This function is given to cpu_pm_register_notifier so we can be informed 844e40b0c16SDouglas Anderson * about when CPUs go down. When all CPUs go down we know no more active 845e40b0c16SDouglas Anderson * transfers will be started so we write sleep/wake sets. This function gets 846e40b0c16SDouglas Anderson * called from cpuidle code paths and also at system suspend time. 847e40b0c16SDouglas Anderson * 848e40b0c16SDouglas Anderson * If its last CPU going down and AMCs are not busy then writes cached sleep 849e40b0c16SDouglas Anderson * and wake messages to TCSes. The firmware then takes care of triggering 850e40b0c16SDouglas Anderson * them when entering deepest low power modes. 851e40b0c16SDouglas Anderson * 852e40b0c16SDouglas Anderson * Return: See cpu_pm_register_notifier() 853e40b0c16SDouglas Anderson */ 854985427f9SMaulik Shah static int rpmh_rsc_cpu_pm_callback(struct notifier_block *nfb, 855985427f9SMaulik Shah unsigned long action, void *v) 856985427f9SMaulik Shah { 857985427f9SMaulik Shah struct rsc_drv *drv = container_of(nfb, struct rsc_drv, rsc_pm); 858985427f9SMaulik Shah int ret = NOTIFY_OK; 859d2a8cfc6SDouglas Anderson int cpus_in_pm; 860985427f9SMaulik Shah 861985427f9SMaulik Shah switch (action) { 862985427f9SMaulik Shah case CPU_PM_ENTER: 863d2a8cfc6SDouglas Anderson cpus_in_pm = atomic_inc_return(&drv->cpus_in_pm); 864d2a8cfc6SDouglas Anderson /* 865d2a8cfc6SDouglas Anderson * NOTE: comments for num_online_cpus() point out that it's 866d2a8cfc6SDouglas Anderson * only a snapshot so we need to be careful. It should be OK 867d2a8cfc6SDouglas Anderson * for us to use, though. It's important for us not to miss 868d2a8cfc6SDouglas Anderson * if we're the last CPU going down so it would only be a 869d2a8cfc6SDouglas Anderson * problem if a CPU went offline right after we did the check 870d2a8cfc6SDouglas Anderson * AND that CPU was not idle AND that CPU was the last non-idle 871d2a8cfc6SDouglas Anderson * CPU. That can't happen. CPUs would have to come out of idle 872d2a8cfc6SDouglas Anderson * before the CPU could go offline. 873d2a8cfc6SDouglas Anderson */ 874d2a8cfc6SDouglas Anderson if (cpus_in_pm < num_online_cpus()) 875d2a8cfc6SDouglas Anderson return NOTIFY_OK; 876985427f9SMaulik Shah break; 877985427f9SMaulik Shah case CPU_PM_ENTER_FAILED: 878985427f9SMaulik Shah case CPU_PM_EXIT: 879d2a8cfc6SDouglas Anderson atomic_dec(&drv->cpus_in_pm); 880d2a8cfc6SDouglas Anderson return NOTIFY_OK; 8811143c366SDouglas Anderson default: 882d2a8cfc6SDouglas Anderson return NOTIFY_DONE; 883985427f9SMaulik Shah } 884985427f9SMaulik Shah 885d2a8cfc6SDouglas Anderson /* 886d2a8cfc6SDouglas Anderson * It's likely we're on the last CPU. Grab the drv->lock and write 887d2a8cfc6SDouglas Anderson * out the sleep/wake commands to RPMH hardware. Grabbing the lock 888d2a8cfc6SDouglas Anderson * means that if we race with another CPU coming up we are still 889d2a8cfc6SDouglas Anderson * guaranteed to be safe. If another CPU came up just after we checked 890d2a8cfc6SDouglas Anderson * and has grabbed the lock or started an active transfer then we'll 891d2a8cfc6SDouglas Anderson * notice we're busy and abort. If another CPU comes up after we start 892d2a8cfc6SDouglas Anderson * flushing it will be blocked from starting an active transfer until 893d2a8cfc6SDouglas Anderson * we're done flushing. If another CPU starts an active transfer after 894d2a8cfc6SDouglas Anderson * we release the lock we're still OK because we're no longer the last 895d2a8cfc6SDouglas Anderson * CPU. 896d2a8cfc6SDouglas Anderson */ 897d2a8cfc6SDouglas Anderson if (spin_trylock(&drv->lock)) { 898d2a8cfc6SDouglas Anderson if (rpmh_rsc_ctrlr_is_busy(drv) || rpmh_flush(&drv->client)) 899985427f9SMaulik Shah ret = NOTIFY_BAD; 900d2a8cfc6SDouglas Anderson spin_unlock(&drv->lock); 901d2a8cfc6SDouglas Anderson } else { 902d2a8cfc6SDouglas Anderson /* Another CPU must be up */ 903d2a8cfc6SDouglas Anderson return NOTIFY_OK; 904985427f9SMaulik Shah } 905985427f9SMaulik Shah 906d2a8cfc6SDouglas Anderson if (ret == NOTIFY_BAD) { 907d2a8cfc6SDouglas Anderson /* Double-check if we're here because someone else is up */ 908d2a8cfc6SDouglas Anderson if (cpus_in_pm < num_online_cpus()) 909985427f9SMaulik Shah ret = NOTIFY_OK; 910d2a8cfc6SDouglas Anderson else 911c45def5dSDouglas Anderson /* We won't be called w/ CPU_PM_ENTER_FAILED */ 912d2a8cfc6SDouglas Anderson atomic_dec(&drv->cpus_in_pm); 913d2a8cfc6SDouglas Anderson } 914c45def5dSDouglas Anderson 915985427f9SMaulik Shah return ret; 916985427f9SMaulik Shah } 917985427f9SMaulik Shah 91825092e61SLina Iyer /** 91925092e61SLina Iyer * rpmh_rsc_pd_callback() - Check if any of the AMCs are busy. 92025092e61SLina Iyer * @nfb: Pointer to the genpd notifier block in struct rsc_drv. 92125092e61SLina Iyer * @action: GENPD_NOTIFY_PRE_OFF, GENPD_NOTIFY_OFF, GENPD_NOTIFY_PRE_ON or GENPD_NOTIFY_ON. 92225092e61SLina Iyer * @v: Unused 92325092e61SLina Iyer * 92425092e61SLina Iyer * This function is given to dev_pm_genpd_add_notifier() so we can be informed 92525092e61SLina Iyer * about when cluster-pd is going down. When cluster go down we know no more active 92625092e61SLina Iyer * transfers will be started so we write sleep/wake sets. This function gets 92725092e61SLina Iyer * called from cpuidle code paths and also at system suspend time. 92825092e61SLina Iyer * 92925092e61SLina Iyer * If AMCs are not busy then writes cached sleep and wake messages to TCSes. 93025092e61SLina Iyer * The firmware then takes care of triggering them when entering deepest low power modes. 93125092e61SLina Iyer * 93225092e61SLina Iyer * Return: 93325092e61SLina Iyer * * NOTIFY_OK - success 93425092e61SLina Iyer * * NOTIFY_BAD - failure 93525092e61SLina Iyer */ 93625092e61SLina Iyer static int rpmh_rsc_pd_callback(struct notifier_block *nfb, 93725092e61SLina Iyer unsigned long action, void *v) 93825092e61SLina Iyer { 93925092e61SLina Iyer struct rsc_drv *drv = container_of(nfb, struct rsc_drv, genpd_nb); 94025092e61SLina Iyer 94125092e61SLina Iyer /* We don't need to lock as genpd on/off are serialized */ 94225092e61SLina Iyer if ((action == GENPD_NOTIFY_PRE_OFF) && 94325092e61SLina Iyer (rpmh_rsc_ctrlr_is_busy(drv) || rpmh_flush(&drv->client))) 94425092e61SLina Iyer return NOTIFY_BAD; 94525092e61SLina Iyer 94625092e61SLina Iyer return NOTIFY_OK; 94725092e61SLina Iyer } 94825092e61SLina Iyer 94925092e61SLina Iyer static int rpmh_rsc_pd_attach(struct rsc_drv *drv, struct device *dev) 95025092e61SLina Iyer { 95125092e61SLina Iyer int ret; 95225092e61SLina Iyer 95325092e61SLina Iyer pm_runtime_enable(dev); 95425092e61SLina Iyer drv->genpd_nb.notifier_call = rpmh_rsc_pd_callback; 95525092e61SLina Iyer ret = dev_pm_genpd_add_notifier(dev, &drv->genpd_nb); 95625092e61SLina Iyer if (ret) 95725092e61SLina Iyer pm_runtime_disable(dev); 95825092e61SLina Iyer 95925092e61SLina Iyer return ret; 96025092e61SLina Iyer } 96125092e61SLina Iyer 962ab33c8f3SMaulik Shah static int rpmh_probe_tcs_config(struct platform_device *pdev, struct rsc_drv *drv) 963658628e7SLina Iyer { 964658628e7SLina Iyer struct tcs_type_config { 965658628e7SLina Iyer u32 type; 966658628e7SLina Iyer u32 n; 967658628e7SLina Iyer } tcs_cfg[TCS_TYPE_NR] = { { 0 } }; 968658628e7SLina Iyer struct device_node *dn = pdev->dev.of_node; 969658628e7SLina Iyer u32 config, max_tcs, ncpt, offset; 970658628e7SLina Iyer int i, ret, n, st = 0; 971658628e7SLina Iyer struct tcs_group *tcs; 972658628e7SLina Iyer 973658628e7SLina Iyer ret = of_property_read_u32(dn, "qcom,tcs-offset", &offset); 974658628e7SLina Iyer if (ret) 975658628e7SLina Iyer return ret; 976ab33c8f3SMaulik Shah drv->tcs_base = drv->base + offset; 977658628e7SLina Iyer 97840482e4fSAbel Vesa config = readl_relaxed(drv->base + drv->regs[DRV_PRNT_CHLD_CONFIG]); 979658628e7SLina Iyer 980658628e7SLina Iyer max_tcs = config; 981658628e7SLina Iyer max_tcs &= DRV_NUM_TCS_MASK << (DRV_NUM_TCS_SHIFT * drv->id); 982658628e7SLina Iyer max_tcs = max_tcs >> (DRV_NUM_TCS_SHIFT * drv->id); 983658628e7SLina Iyer 984658628e7SLina Iyer ncpt = config & (DRV_NCPT_MASK << DRV_NCPT_SHIFT); 985658628e7SLina Iyer ncpt = ncpt >> DRV_NCPT_SHIFT; 986658628e7SLina Iyer 987658628e7SLina Iyer n = of_property_count_u32_elems(dn, "qcom,tcs-config"); 988658628e7SLina Iyer if (n != 2 * TCS_TYPE_NR) 989658628e7SLina Iyer return -EINVAL; 990658628e7SLina Iyer 991658628e7SLina Iyer for (i = 0; i < TCS_TYPE_NR; i++) { 992658628e7SLina Iyer ret = of_property_read_u32_index(dn, "qcom,tcs-config", 993658628e7SLina Iyer i * 2, &tcs_cfg[i].type); 994658628e7SLina Iyer if (ret) 995658628e7SLina Iyer return ret; 996658628e7SLina Iyer if (tcs_cfg[i].type >= TCS_TYPE_NR) 997658628e7SLina Iyer return -EINVAL; 998658628e7SLina Iyer 999658628e7SLina Iyer ret = of_property_read_u32_index(dn, "qcom,tcs-config", 1000658628e7SLina Iyer i * 2 + 1, &tcs_cfg[i].n); 1001658628e7SLina Iyer if (ret) 1002658628e7SLina Iyer return ret; 1003658628e7SLina Iyer if (tcs_cfg[i].n > MAX_TCS_PER_TYPE) 1004658628e7SLina Iyer return -EINVAL; 1005658628e7SLina Iyer } 1006658628e7SLina Iyer 1007658628e7SLina Iyer for (i = 0; i < TCS_TYPE_NR; i++) { 1008658628e7SLina Iyer tcs = &drv->tcs[tcs_cfg[i].type]; 1009658628e7SLina Iyer if (tcs->drv) 1010658628e7SLina Iyer return -EINVAL; 1011658628e7SLina Iyer tcs->drv = drv; 1012658628e7SLina Iyer tcs->type = tcs_cfg[i].type; 1013658628e7SLina Iyer tcs->num_tcs = tcs_cfg[i].n; 1014658628e7SLina Iyer tcs->ncpt = ncpt; 1015658628e7SLina Iyer 1016658628e7SLina Iyer if (!tcs->num_tcs || tcs->type == CONTROL_TCS) 1017658628e7SLina Iyer continue; 1018658628e7SLina Iyer 1019658628e7SLina Iyer if (st + tcs->num_tcs > max_tcs || 1020658628e7SLina Iyer st + tcs->num_tcs >= BITS_PER_BYTE * sizeof(tcs->mask)) 1021658628e7SLina Iyer return -EINVAL; 1022658628e7SLina Iyer 1023658628e7SLina Iyer tcs->mask = ((1 << tcs->num_tcs) - 1) << st; 1024658628e7SLina Iyer tcs->offset = st; 1025658628e7SLina Iyer st += tcs->num_tcs; 1026658628e7SLina Iyer } 1027658628e7SLina Iyer 1028658628e7SLina Iyer drv->num_tcs = st; 1029658628e7SLina Iyer 1030658628e7SLina Iyer return 0; 1031658628e7SLina Iyer } 1032658628e7SLina Iyer 1033658628e7SLina Iyer static int rpmh_rsc_probe(struct platform_device *pdev) 1034658628e7SLina Iyer { 1035658628e7SLina Iyer struct device_node *dn = pdev->dev.of_node; 1036658628e7SLina Iyer struct rsc_drv *drv; 1037985427f9SMaulik Shah char drv_id[10] = {0}; 1038658628e7SLina Iyer int ret, irq; 1039985427f9SMaulik Shah u32 solver_config; 104040482e4fSAbel Vesa u32 rsc_id; 1041658628e7SLina Iyer 1042fdd102b5SDouglas Anderson /* 1043fdd102b5SDouglas Anderson * Even though RPMh doesn't directly use cmd-db, all of its children 1044fdd102b5SDouglas Anderson * do. To avoid adding this check to our children we'll do it now. 1045fdd102b5SDouglas Anderson */ 1046fdd102b5SDouglas Anderson ret = cmd_db_ready(); 1047fdd102b5SDouglas Anderson if (ret) { 1048fdd102b5SDouglas Anderson if (ret != -EPROBE_DEFER) 1049fdd102b5SDouglas Anderson dev_err(&pdev->dev, "Command DB not available (%d)\n", 1050fdd102b5SDouglas Anderson ret); 1051fdd102b5SDouglas Anderson return ret; 1052fdd102b5SDouglas Anderson } 1053fdd102b5SDouglas Anderson 1054658628e7SLina Iyer drv = devm_kzalloc(&pdev->dev, sizeof(*drv), GFP_KERNEL); 1055658628e7SLina Iyer if (!drv) 1056658628e7SLina Iyer return -ENOMEM; 1057658628e7SLina Iyer 1058658628e7SLina Iyer ret = of_property_read_u32(dn, "qcom,drv-id", &drv->id); 1059658628e7SLina Iyer if (ret) 1060658628e7SLina Iyer return ret; 1061658628e7SLina Iyer 1062658628e7SLina Iyer drv->name = of_get_property(dn, "label", NULL); 1063658628e7SLina Iyer if (!drv->name) 1064658628e7SLina Iyer drv->name = dev_name(&pdev->dev); 1065658628e7SLina Iyer 1066985427f9SMaulik Shah snprintf(drv_id, ARRAY_SIZE(drv_id), "drv-%d", drv->id); 1067ab33c8f3SMaulik Shah drv->base = devm_platform_ioremap_resource_byname(pdev, drv_id); 1068ab33c8f3SMaulik Shah if (IS_ERR(drv->base)) 1069ab33c8f3SMaulik Shah return PTR_ERR(drv->base); 1070985427f9SMaulik Shah 107140482e4fSAbel Vesa rsc_id = readl_relaxed(drv->base + RSC_DRV_ID); 107240482e4fSAbel Vesa drv->ver.major = rsc_id & (MAJOR_VER_MASK << MAJOR_VER_SHIFT); 107340482e4fSAbel Vesa drv->ver.major >>= MAJOR_VER_SHIFT; 107440482e4fSAbel Vesa drv->ver.minor = rsc_id & (MINOR_VER_MASK << MINOR_VER_SHIFT); 107540482e4fSAbel Vesa drv->ver.minor >>= MINOR_VER_SHIFT; 107640482e4fSAbel Vesa 10773395d36eSKrzysztof Kozlowski if (drv->ver.major == 3) 107840482e4fSAbel Vesa drv->regs = rpmh_rsc_reg_offset_ver_3_0; 107940482e4fSAbel Vesa else 108040482e4fSAbel Vesa drv->regs = rpmh_rsc_reg_offset_ver_2_7; 108140482e4fSAbel Vesa 1082ab33c8f3SMaulik Shah ret = rpmh_probe_tcs_config(pdev, drv); 1083658628e7SLina Iyer if (ret) 1084658628e7SLina Iyer return ret; 1085658628e7SLina Iyer 1086658628e7SLina Iyer spin_lock_init(&drv->lock); 10872bc20f3cSStephen Boyd init_waitqueue_head(&drv->tcs_wait); 1088658628e7SLina Iyer bitmap_zero(drv->tcs_in_use, MAX_TCS_NR); 1089658628e7SLina Iyer 1090658628e7SLina Iyer irq = platform_get_irq(pdev, drv->id); 1091658628e7SLina Iyer if (irq < 0) 1092658628e7SLina Iyer return irq; 1093658628e7SLina Iyer 1094658628e7SLina Iyer ret = devm_request_irq(&pdev->dev, irq, tcs_tx_done, 1095658628e7SLina Iyer IRQF_TRIGGER_HIGH | IRQF_NO_SUSPEND, 1096658628e7SLina Iyer drv->name, drv); 1097658628e7SLina Iyer if (ret) 1098658628e7SLina Iyer return ret; 1099658628e7SLina Iyer 1100985427f9SMaulik Shah /* 110125092e61SLina Iyer * CPU PM/genpd notification are not required for controllers that support 1102985427f9SMaulik Shah * 'HW solver' mode where they can be in autonomous mode executing low 1103985427f9SMaulik Shah * power mode to power down. 1104985427f9SMaulik Shah */ 110540482e4fSAbel Vesa solver_config = readl_relaxed(drv->base + drv->regs[DRV_SOLVER_CONFIG]); 1106985427f9SMaulik Shah solver_config &= DRV_HW_SOLVER_MASK << DRV_HW_SOLVER_SHIFT; 1107985427f9SMaulik Shah solver_config = solver_config >> DRV_HW_SOLVER_SHIFT; 1108985427f9SMaulik Shah if (!solver_config) { 110925092e61SLina Iyer if (pdev->dev.pm_domain) { 111025092e61SLina Iyer ret = rpmh_rsc_pd_attach(drv, &pdev->dev); 111125092e61SLina Iyer if (ret) 111225092e61SLina Iyer return ret; 111325092e61SLina Iyer } else { 1114985427f9SMaulik Shah drv->rsc_pm.notifier_call = rpmh_rsc_cpu_pm_callback; 1115985427f9SMaulik Shah cpu_pm_register_notifier(&drv->rsc_pm); 1116985427f9SMaulik Shah } 111725092e61SLina Iyer } 1118985427f9SMaulik Shah 1119658628e7SLina Iyer /* Enable the active TCS to send requests immediately */ 1120032c692aSDouglas Anderson writel_relaxed(drv->tcs[ACTIVE_TCS].mask, 112140482e4fSAbel Vesa drv->tcs_base + drv->regs[RSC_DRV_IRQ_ENABLE]); 1122658628e7SLina Iyer 1123600513dfSLina Iyer spin_lock_init(&drv->client.cache_lock); 1124600513dfSLina Iyer INIT_LIST_HEAD(&drv->client.cache); 1125c8790cb6SLina Iyer INIT_LIST_HEAD(&drv->client.batch_cache); 1126600513dfSLina Iyer 1127c1038456SLina Iyer dev_set_drvdata(&pdev->dev, drv); 1128cccbe3e5SMaulik Shah drv->dev = &pdev->dev; 1129c1038456SLina Iyer 113025092e61SLina Iyer ret = devm_of_platform_populate(&pdev->dev); 113125092e61SLina Iyer if (ret && pdev->dev.pm_domain) { 113225092e61SLina Iyer dev_pm_genpd_remove_notifier(&pdev->dev); 113325092e61SLina Iyer pm_runtime_disable(&pdev->dev); 113425092e61SLina Iyer } 113525092e61SLina Iyer 113625092e61SLina Iyer return ret; 1137658628e7SLina Iyer } 1138658628e7SLina Iyer 1139658628e7SLina Iyer static const struct of_device_id rpmh_drv_match[] = { 1140658628e7SLina Iyer { .compatible = "qcom,rpmh-rsc", }, 1141658628e7SLina Iyer { } 1142658628e7SLina Iyer }; 1143cb365926SJohn Stultz MODULE_DEVICE_TABLE(of, rpmh_drv_match); 1144658628e7SLina Iyer 1145658628e7SLina Iyer static struct platform_driver rpmh_driver = { 1146658628e7SLina Iyer .probe = rpmh_rsc_probe, 1147658628e7SLina Iyer .driver = { 1148658628e7SLina Iyer .name = "rpmh", 1149658628e7SLina Iyer .of_match_table = rpmh_drv_match, 11501a53ce9aSMaulik Shah .suppress_bind_attrs = true, 1151658628e7SLina Iyer }, 1152658628e7SLina Iyer }; 1153658628e7SLina Iyer 1154658628e7SLina Iyer static int __init rpmh_driver_init(void) 1155658628e7SLina Iyer { 1156658628e7SLina Iyer return platform_driver_register(&rpmh_driver); 1157658628e7SLina Iyer } 1158658628e7SLina Iyer arch_initcall(rpmh_driver_init); 1159cb365926SJohn Stultz 1160cb365926SJohn Stultz MODULE_DESCRIPTION("Qualcomm Technologies, Inc. RPMh Driver"); 1161cb365926SJohn Stultz MODULE_LICENSE("GPL v2"); 1162