1658628e7SLina Iyer // SPDX-License-Identifier: GPL-2.0
2658628e7SLina Iyer /*
3658628e7SLina Iyer * Copyright (c) 2016-2018, The Linux Foundation. All rights reserved.
46a283d64SMaulik Shah * Copyright (c) 2023-2024, Qualcomm Innovation Center, Inc. All rights reserved.
5658628e7SLina Iyer */
6658628e7SLina Iyer
7658628e7SLina Iyer #define pr_fmt(fmt) "%s " fmt, KBUILD_MODNAME
8658628e7SLina Iyer
9658628e7SLina Iyer #include <linux/atomic.h>
10985427f9SMaulik Shah #include <linux/cpu_pm.h>
11658628e7SLina Iyer #include <linux/delay.h>
12658628e7SLina Iyer #include <linux/interrupt.h>
13658628e7SLina Iyer #include <linux/io.h>
1491160150SDouglas Anderson #include <linux/iopoll.h>
15658628e7SLina Iyer #include <linux/kernel.h>
16cccbe3e5SMaulik Shah #include <linux/ktime.h>
17658628e7SLina Iyer #include <linux/list.h>
18cb365926SJohn Stultz #include <linux/module.h>
1925092e61SLina Iyer #include <linux/notifier.h>
20658628e7SLina Iyer #include <linux/of.h>
21658628e7SLina Iyer #include <linux/of_irq.h>
22658628e7SLina Iyer #include <linux/of_platform.h>
23658628e7SLina Iyer #include <linux/platform_device.h>
2425092e61SLina Iyer #include <linux/pm_domain.h>
2525092e61SLina Iyer #include <linux/pm_runtime.h>
26658628e7SLina Iyer #include <linux/slab.h>
27658628e7SLina Iyer #include <linux/spinlock.h>
282bc20f3cSStephen Boyd #include <linux/wait.h>
29658628e7SLina Iyer
30cccbe3e5SMaulik Shah #include <clocksource/arm_arch_timer.h>
31fdd102b5SDouglas Anderson #include <soc/qcom/cmd-db.h>
32658628e7SLina Iyer #include <soc/qcom/tcs.h>
33658628e7SLina Iyer #include <dt-bindings/soc/qcom,rpmh-rsc.h>
34658628e7SLina Iyer
35658628e7SLina Iyer #include "rpmh-internal.h"
36658628e7SLina Iyer
37fc087fe5SLina Iyer #define CREATE_TRACE_POINTS
38fc087fe5SLina Iyer #include "trace-rpmh.h"
39fc087fe5SLina Iyer
4040482e4fSAbel Vesa
4140482e4fSAbel Vesa #define RSC_DRV_ID 0
4240482e4fSAbel Vesa
4340482e4fSAbel Vesa #define MAJOR_VER_MASK 0xFF
4440482e4fSAbel Vesa #define MAJOR_VER_SHIFT 16
4540482e4fSAbel Vesa #define MINOR_VER_MASK 0xFF
4640482e4fSAbel Vesa #define MINOR_VER_SHIFT 8
4740482e4fSAbel Vesa
4840482e4fSAbel Vesa enum {
4940482e4fSAbel Vesa RSC_DRV_TCS_OFFSET,
5040482e4fSAbel Vesa RSC_DRV_CMD_OFFSET,
5140482e4fSAbel Vesa DRV_SOLVER_CONFIG,
5240482e4fSAbel Vesa DRV_PRNT_CHLD_CONFIG,
5340482e4fSAbel Vesa RSC_DRV_IRQ_ENABLE,
5440482e4fSAbel Vesa RSC_DRV_IRQ_STATUS,
5540482e4fSAbel Vesa RSC_DRV_IRQ_CLEAR,
5640482e4fSAbel Vesa RSC_DRV_CMD_WAIT_FOR_CMPL,
5740482e4fSAbel Vesa RSC_DRV_CONTROL,
5840482e4fSAbel Vesa RSC_DRV_STATUS,
5940482e4fSAbel Vesa RSC_DRV_CMD_ENABLE,
6040482e4fSAbel Vesa RSC_DRV_CMD_MSGID,
6140482e4fSAbel Vesa RSC_DRV_CMD_ADDR,
6240482e4fSAbel Vesa RSC_DRV_CMD_DATA,
6340482e4fSAbel Vesa RSC_DRV_CMD_STATUS,
6440482e4fSAbel Vesa RSC_DRV_CMD_RESP_DATA,
6540482e4fSAbel Vesa };
66658628e7SLina Iyer
67985427f9SMaulik Shah /* DRV HW Solver Configuration Information Register */
68985427f9SMaulik Shah #define DRV_HW_SOLVER_MASK 1
69985427f9SMaulik Shah #define DRV_HW_SOLVER_SHIFT 24
70985427f9SMaulik Shah
71985427f9SMaulik Shah /* DRV TCS Configuration Information Register */
72658628e7SLina Iyer #define DRV_NUM_TCS_MASK 0x3F
73658628e7SLina Iyer #define DRV_NUM_TCS_SHIFT 6
74658628e7SLina Iyer #define DRV_NCPT_MASK 0x1F
75658628e7SLina Iyer #define DRV_NCPT_SHIFT 27
76658628e7SLina Iyer
77cccbe3e5SMaulik Shah /* Offsets for CONTROL TCS Registers */
78cccbe3e5SMaulik Shah #define RSC_DRV_CTL_TCS_DATA_HI 0x38
79cccbe3e5SMaulik Shah #define RSC_DRV_CTL_TCS_DATA_HI_MASK 0xFFFFFF
80cccbe3e5SMaulik Shah #define RSC_DRV_CTL_TCS_DATA_HI_VALID BIT(31)
81cccbe3e5SMaulik Shah #define RSC_DRV_CTL_TCS_DATA_LO 0x40
82cccbe3e5SMaulik Shah #define RSC_DRV_CTL_TCS_DATA_LO_MASK 0xFFFFFFFF
83cccbe3e5SMaulik Shah #define RSC_DRV_CTL_TCS_DATA_SIZE 32
84cccbe3e5SMaulik Shah
85658628e7SLina Iyer #define TCS_AMC_MODE_ENABLE BIT(16)
86658628e7SLina Iyer #define TCS_AMC_MODE_TRIGGER BIT(24)
87658628e7SLina Iyer
88658628e7SLina Iyer /* TCS CMD register bit mask */
89658628e7SLina Iyer #define CMD_MSGID_LEN 8
90658628e7SLina Iyer #define CMD_MSGID_RESP_REQ BIT(8)
91658628e7SLina Iyer #define CMD_MSGID_WRITE BIT(16)
92658628e7SLina Iyer #define CMD_STATUS_ISSUED BIT(8)
93658628e7SLina Iyer #define CMD_STATUS_COMPL BIT(16)
94658628e7SLina Iyer
951f7dbeb5SDouglas Anderson /*
961f7dbeb5SDouglas Anderson * Here's a high level overview of how all the registers in RPMH work
971f7dbeb5SDouglas Anderson * together:
981f7dbeb5SDouglas Anderson *
991f7dbeb5SDouglas Anderson * - The main rpmh-rsc address is the base of a register space that can
1001f7dbeb5SDouglas Anderson * be used to find overall configuration of the hardware
1011f7dbeb5SDouglas Anderson * (DRV_PRNT_CHLD_CONFIG). Also found within the rpmh-rsc register
1021f7dbeb5SDouglas Anderson * space are all the TCS blocks. The offset of the TCS blocks is
1031f7dbeb5SDouglas Anderson * specified in the device tree by "qcom,tcs-offset" and used to
1041f7dbeb5SDouglas Anderson * compute tcs_base.
1051f7dbeb5SDouglas Anderson * - TCS blocks come one after another. Type, count, and order are
1061f7dbeb5SDouglas Anderson * specified by the device tree as "qcom,tcs-config".
1071f7dbeb5SDouglas Anderson * - Each TCS block has some registers, then space for up to 16 commands.
1081f7dbeb5SDouglas Anderson * Note that though address space is reserved for 16 commands, fewer
1091f7dbeb5SDouglas Anderson * might be present. See ncpt (num cmds per TCS).
1101f7dbeb5SDouglas Anderson *
1111f7dbeb5SDouglas Anderson * Here's a picture:
1121f7dbeb5SDouglas Anderson *
1131f7dbeb5SDouglas Anderson * +---------------------------------------------------+
1141f7dbeb5SDouglas Anderson * |RSC |
1151f7dbeb5SDouglas Anderson * | ctrl |
1161f7dbeb5SDouglas Anderson * | |
1171f7dbeb5SDouglas Anderson * | Drvs: |
1181f7dbeb5SDouglas Anderson * | +-----------------------------------------------+ |
1191f7dbeb5SDouglas Anderson * | |DRV0 | |
1201f7dbeb5SDouglas Anderson * | | ctrl/config | |
1211f7dbeb5SDouglas Anderson * | | IRQ | |
1221f7dbeb5SDouglas Anderson * | | | |
1231f7dbeb5SDouglas Anderson * | | TCSes: | |
1241f7dbeb5SDouglas Anderson * | | +------------------------------------------+ | |
1251f7dbeb5SDouglas Anderson * | | |TCS0 | | | | | | | | | | | | | | |
1261f7dbeb5SDouglas Anderson * | | | ctrl | 0| 1| 2| 3| 4| 5| .| .| .| .|14|15| | |
1271f7dbeb5SDouglas Anderson * | | | | | | | | | | | | | | | | | |
1281f7dbeb5SDouglas Anderson * | | +------------------------------------------+ | |
1291f7dbeb5SDouglas Anderson * | | +------------------------------------------+ | |
1301f7dbeb5SDouglas Anderson * | | |TCS1 | | | | | | | | | | | | | | |
1311f7dbeb5SDouglas Anderson * | | | ctrl | 0| 1| 2| 3| 4| 5| .| .| .| .|14|15| | |
1321f7dbeb5SDouglas Anderson * | | | | | | | | | | | | | | | | | |
1331f7dbeb5SDouglas Anderson * | | +------------------------------------------+ | |
1341f7dbeb5SDouglas Anderson * | | +------------------------------------------+ | |
1351f7dbeb5SDouglas Anderson * | | |TCS2 | | | | | | | | | | | | | | |
1361f7dbeb5SDouglas Anderson * | | | ctrl | 0| 1| 2| 3| 4| 5| .| .| .| .|14|15| | |
1371f7dbeb5SDouglas Anderson * | | | | | | | | | | | | | | | | | |
1381f7dbeb5SDouglas Anderson * | | +------------------------------------------+ | |
1391f7dbeb5SDouglas Anderson * | | ...... | |
1401f7dbeb5SDouglas Anderson * | +-----------------------------------------------+ |
1411f7dbeb5SDouglas Anderson * | +-----------------------------------------------+ |
1421f7dbeb5SDouglas Anderson * | |DRV1 | |
1431f7dbeb5SDouglas Anderson * | | (same as DRV0) | |
1441f7dbeb5SDouglas Anderson * | +-----------------------------------------------+ |
1451f7dbeb5SDouglas Anderson * | ...... |
1461f7dbeb5SDouglas Anderson * +---------------------------------------------------+
1471f7dbeb5SDouglas Anderson */
1481f7dbeb5SDouglas Anderson
149cccbe3e5SMaulik Shah #define USECS_TO_CYCLES(time_usecs) \
150cccbe3e5SMaulik Shah xloops_to_cycles((time_usecs) * 0x10C7UL)
151cccbe3e5SMaulik Shah
xloops_to_cycles(u64 xloops)152cccbe3e5SMaulik Shah static inline unsigned long xloops_to_cycles(u64 xloops)
153cccbe3e5SMaulik Shah {
154cccbe3e5SMaulik Shah return (xloops * loops_per_jiffy * HZ) >> 32;
155cccbe3e5SMaulik Shah }
156cccbe3e5SMaulik Shah
15740482e4fSAbel Vesa static u32 rpmh_rsc_reg_offset_ver_2_7[] = {
15840482e4fSAbel Vesa [RSC_DRV_TCS_OFFSET] = 672,
15940482e4fSAbel Vesa [RSC_DRV_CMD_OFFSET] = 20,
16040482e4fSAbel Vesa [DRV_SOLVER_CONFIG] = 0x04,
16140482e4fSAbel Vesa [DRV_PRNT_CHLD_CONFIG] = 0x0C,
16240482e4fSAbel Vesa [RSC_DRV_IRQ_ENABLE] = 0x00,
16340482e4fSAbel Vesa [RSC_DRV_IRQ_STATUS] = 0x04,
16440482e4fSAbel Vesa [RSC_DRV_IRQ_CLEAR] = 0x08,
16540482e4fSAbel Vesa [RSC_DRV_CMD_WAIT_FOR_CMPL] = 0x10,
16640482e4fSAbel Vesa [RSC_DRV_CONTROL] = 0x14,
16740482e4fSAbel Vesa [RSC_DRV_STATUS] = 0x18,
16840482e4fSAbel Vesa [RSC_DRV_CMD_ENABLE] = 0x1C,
16940482e4fSAbel Vesa [RSC_DRV_CMD_MSGID] = 0x30,
17040482e4fSAbel Vesa [RSC_DRV_CMD_ADDR] = 0x34,
17140482e4fSAbel Vesa [RSC_DRV_CMD_DATA] = 0x38,
17240482e4fSAbel Vesa [RSC_DRV_CMD_STATUS] = 0x3C,
17340482e4fSAbel Vesa [RSC_DRV_CMD_RESP_DATA] = 0x40,
17440482e4fSAbel Vesa };
17540482e4fSAbel Vesa
17640482e4fSAbel Vesa static u32 rpmh_rsc_reg_offset_ver_3_0[] = {
17740482e4fSAbel Vesa [RSC_DRV_TCS_OFFSET] = 672,
17840482e4fSAbel Vesa [RSC_DRV_CMD_OFFSET] = 24,
17940482e4fSAbel Vesa [DRV_SOLVER_CONFIG] = 0x04,
18040482e4fSAbel Vesa [DRV_PRNT_CHLD_CONFIG] = 0x0C,
18140482e4fSAbel Vesa [RSC_DRV_IRQ_ENABLE] = 0x00,
18240482e4fSAbel Vesa [RSC_DRV_IRQ_STATUS] = 0x04,
18340482e4fSAbel Vesa [RSC_DRV_IRQ_CLEAR] = 0x08,
18440482e4fSAbel Vesa [RSC_DRV_CMD_WAIT_FOR_CMPL] = 0x20,
18540482e4fSAbel Vesa [RSC_DRV_CONTROL] = 0x24,
18640482e4fSAbel Vesa [RSC_DRV_STATUS] = 0x28,
18740482e4fSAbel Vesa [RSC_DRV_CMD_ENABLE] = 0x2C,
18840482e4fSAbel Vesa [RSC_DRV_CMD_MSGID] = 0x34,
18940482e4fSAbel Vesa [RSC_DRV_CMD_ADDR] = 0x38,
19040482e4fSAbel Vesa [RSC_DRV_CMD_DATA] = 0x3C,
19140482e4fSAbel Vesa [RSC_DRV_CMD_STATUS] = 0x40,
19240482e4fSAbel Vesa [RSC_DRV_CMD_RESP_DATA] = 0x44,
19340482e4fSAbel Vesa };
19440482e4fSAbel Vesa
195faa0c1f1SDouglas Anderson static inline void __iomem *
tcs_reg_addr(const struct rsc_drv * drv,int reg,int tcs_id)196faa0c1f1SDouglas Anderson tcs_reg_addr(const struct rsc_drv *drv, int reg, int tcs_id)
197658628e7SLina Iyer {
19840482e4fSAbel Vesa return drv->tcs_base + drv->regs[RSC_DRV_TCS_OFFSET] * tcs_id + reg;
199658628e7SLina Iyer }
200658628e7SLina Iyer
201faa0c1f1SDouglas Anderson static inline void __iomem *
tcs_cmd_addr(const struct rsc_drv * drv,int reg,int tcs_id,int cmd_id)202faa0c1f1SDouglas Anderson tcs_cmd_addr(const struct rsc_drv *drv, int reg, int tcs_id, int cmd_id)
2033b5e3d50SDouglas Anderson {
20440482e4fSAbel Vesa return tcs_reg_addr(drv, reg, tcs_id) + drv->regs[RSC_DRV_CMD_OFFSET] * cmd_id;
2053b5e3d50SDouglas Anderson }
2063b5e3d50SDouglas Anderson
read_tcs_cmd(const struct rsc_drv * drv,int reg,int tcs_id,int cmd_id)207faa0c1f1SDouglas Anderson static u32 read_tcs_cmd(const struct rsc_drv *drv, int reg, int tcs_id,
208faa0c1f1SDouglas Anderson int cmd_id)
209faa0c1f1SDouglas Anderson {
210faa0c1f1SDouglas Anderson return readl_relaxed(tcs_cmd_addr(drv, reg, tcs_id, cmd_id));
211faa0c1f1SDouglas Anderson }
212faa0c1f1SDouglas Anderson
read_tcs_reg(const struct rsc_drv * drv,int reg,int tcs_id)213faa0c1f1SDouglas Anderson static u32 read_tcs_reg(const struct rsc_drv *drv, int reg, int tcs_id)
214faa0c1f1SDouglas Anderson {
215faa0c1f1SDouglas Anderson return readl_relaxed(tcs_reg_addr(drv, reg, tcs_id));
216faa0c1f1SDouglas Anderson }
217faa0c1f1SDouglas Anderson
write_tcs_cmd(const struct rsc_drv * drv,int reg,int tcs_id,int cmd_id,u32 data)218faa0c1f1SDouglas Anderson static void write_tcs_cmd(const struct rsc_drv *drv, int reg, int tcs_id,
219faa0c1f1SDouglas Anderson int cmd_id, u32 data)
220faa0c1f1SDouglas Anderson {
221faa0c1f1SDouglas Anderson writel_relaxed(data, tcs_cmd_addr(drv, reg, tcs_id, cmd_id));
222faa0c1f1SDouglas Anderson }
223faa0c1f1SDouglas Anderson
write_tcs_reg(const struct rsc_drv * drv,int reg,int tcs_id,u32 data)224faa0c1f1SDouglas Anderson static void write_tcs_reg(const struct rsc_drv *drv, int reg, int tcs_id,
225658628e7SLina Iyer u32 data)
226658628e7SLina Iyer {
227faa0c1f1SDouglas Anderson writel_relaxed(data, tcs_reg_addr(drv, reg, tcs_id));
228658628e7SLina Iyer }
229658628e7SLina Iyer
write_tcs_reg_sync(const struct rsc_drv * drv,int reg,int tcs_id,u32 data)230faa0c1f1SDouglas Anderson static void write_tcs_reg_sync(const struct rsc_drv *drv, int reg, int tcs_id,
231658628e7SLina Iyer u32 data)
232658628e7SLina Iyer {
233be24c6a7SDouglas Anderson int i;
23491160150SDouglas Anderson
235faa0c1f1SDouglas Anderson writel(data, tcs_reg_addr(drv, reg, tcs_id));
236be24c6a7SDouglas Anderson
237be24c6a7SDouglas Anderson /*
238be24c6a7SDouglas Anderson * Wait until we read back the same value. Use a counter rather than
239be24c6a7SDouglas Anderson * ktime for timeout since this may be called after timekeeping stops.
240be24c6a7SDouglas Anderson */
241be24c6a7SDouglas Anderson for (i = 0; i < USEC_PER_SEC; i++) {
242be24c6a7SDouglas Anderson if (readl(tcs_reg_addr(drv, reg, tcs_id)) == data)
243be24c6a7SDouglas Anderson return;
244be24c6a7SDouglas Anderson udelay(1);
245be24c6a7SDouglas Anderson }
24691160150SDouglas Anderson pr_err("%s: error writing %#x to %d:%#x\n", drv->name,
24791160150SDouglas Anderson data, tcs_id, reg);
248658628e7SLina Iyer }
249658628e7SLina Iyer
250e40b0c16SDouglas Anderson /**
251e40b0c16SDouglas Anderson * tcs_invalidate() - Invalidate all TCSes of the given type (sleep or wake).
252e40b0c16SDouglas Anderson * @drv: The RSC controller.
253e40b0c16SDouglas Anderson * @type: SLEEP_TCS or WAKE_TCS
254e40b0c16SDouglas Anderson *
255e40b0c16SDouglas Anderson * This will clear the "slots" variable of the given tcs_group and also
256e40b0c16SDouglas Anderson * tell the hardware to forget about all entries.
257e40b0c16SDouglas Anderson *
258881808d0SDouglas Anderson * The caller must ensure that no other RPMH actions are happening when this
259881808d0SDouglas Anderson * function is called, since otherwise the device may immediately become
260881808d0SDouglas Anderson * used again even before this function exits.
261e40b0c16SDouglas Anderson */
tcs_invalidate(struct rsc_drv * drv,int type)262881808d0SDouglas Anderson static void tcs_invalidate(struct rsc_drv *drv, int type)
2639a3afcfbSLina Iyer {
2649a3afcfbSLina Iyer int m;
26553d49fe1SDouglas Anderson struct tcs_group *tcs = &drv->tcs[type];
2669a3afcfbSLina Iyer
267881808d0SDouglas Anderson /* Caller ensures nobody else is running so no lock */
268881808d0SDouglas Anderson if (bitmap_empty(tcs->slots, MAX_TCS_SLOTS))
269881808d0SDouglas Anderson return;
2709a3afcfbSLina Iyer
271fef419c4SLina Iyer for (m = tcs->offset; m < tcs->offset + tcs->num_tcs; m++)
27240482e4fSAbel Vesa write_tcs_reg_sync(drv, drv->regs[RSC_DRV_CMD_ENABLE], m, 0);
273fef419c4SLina Iyer
2749a3afcfbSLina Iyer bitmap_zero(tcs->slots, MAX_TCS_SLOTS);
2759a3afcfbSLina Iyer }
2769a3afcfbSLina Iyer
2779a3afcfbSLina Iyer /**
278e40b0c16SDouglas Anderson * rpmh_rsc_invalidate() - Invalidate sleep and wake TCSes.
279e40b0c16SDouglas Anderson * @drv: The RSC controller.
2809a3afcfbSLina Iyer *
281881808d0SDouglas Anderson * The caller must ensure that no other RPMH actions are happening when this
282881808d0SDouglas Anderson * function is called, since otherwise the device may immediately become
283881808d0SDouglas Anderson * used again even before this function exits.
2849a3afcfbSLina Iyer */
rpmh_rsc_invalidate(struct rsc_drv * drv)285881808d0SDouglas Anderson void rpmh_rsc_invalidate(struct rsc_drv *drv)
2869a3afcfbSLina Iyer {
287881808d0SDouglas Anderson tcs_invalidate(drv, SLEEP_TCS);
288881808d0SDouglas Anderson tcs_invalidate(drv, WAKE_TCS);
2899a3afcfbSLina Iyer }
2909a3afcfbSLina Iyer
291e40b0c16SDouglas Anderson /**
292e40b0c16SDouglas Anderson * get_tcs_for_msg() - Get the tcs_group used to send the given message.
293e40b0c16SDouglas Anderson * @drv: The RSC controller.
294e40b0c16SDouglas Anderson * @msg: The message we want to send.
295e40b0c16SDouglas Anderson *
296e40b0c16SDouglas Anderson * This is normally pretty straightforward except if we are trying to send
297e40b0c16SDouglas Anderson * an ACTIVE_ONLY message but don't have any active_only TCSes.
298e40b0c16SDouglas Anderson *
299e40b0c16SDouglas Anderson * Return: A pointer to a tcs_group or an ERR_PTR.
300e40b0c16SDouglas Anderson */
get_tcs_for_msg(struct rsc_drv * drv,const struct tcs_request * msg)301658628e7SLina Iyer static struct tcs_group *get_tcs_for_msg(struct rsc_drv *drv,
302658628e7SLina Iyer const struct tcs_request *msg)
303658628e7SLina Iyer {
30438427e5aSMaulik Shah int type;
3052de4b8d3SLina Iyer struct tcs_group *tcs;
306658628e7SLina Iyer
307658628e7SLina Iyer switch (msg->state) {
308658628e7SLina Iyer case RPMH_ACTIVE_ONLY_STATE:
309658628e7SLina Iyer type = ACTIVE_TCS;
310658628e7SLina Iyer break;
311fa460e45SLina Iyer case RPMH_WAKE_ONLY_STATE:
312fa460e45SLina Iyer type = WAKE_TCS;
313fa460e45SLina Iyer break;
314fa460e45SLina Iyer case RPMH_SLEEP_STATE:
315fa460e45SLina Iyer type = SLEEP_TCS;
316fa460e45SLina Iyer break;
317658628e7SLina Iyer default:
318658628e7SLina Iyer return ERR_PTR(-EINVAL);
319658628e7SLina Iyer }
320658628e7SLina Iyer
3212de4b8d3SLina Iyer /*
3222de4b8d3SLina Iyer * If we are making an active request on a RSC that does not have a
3232de4b8d3SLina Iyer * dedicated TCS for active state use, then re-purpose a wake TCS to
324e40b0c16SDouglas Anderson * send active votes. This is safe because we ensure any active-only
325e40b0c16SDouglas Anderson * transfers have finished before we use it (maybe by running from
326e40b0c16SDouglas Anderson * the last CPU in PM code).
3272de4b8d3SLina Iyer */
32853d49fe1SDouglas Anderson tcs = &drv->tcs[type];
32938427e5aSMaulik Shah if (msg->state == RPMH_ACTIVE_ONLY_STATE && !tcs->num_tcs)
33053d49fe1SDouglas Anderson tcs = &drv->tcs[WAKE_TCS];
3312de4b8d3SLina Iyer
3322de4b8d3SLina Iyer return tcs;
333658628e7SLina Iyer }
334658628e7SLina Iyer
335e40b0c16SDouglas Anderson /**
336e40b0c16SDouglas Anderson * get_req_from_tcs() - Get a stashed request that was xfering on the given TCS.
337e40b0c16SDouglas Anderson * @drv: The RSC controller.
338e40b0c16SDouglas Anderson * @tcs_id: The global ID of this TCS.
339e40b0c16SDouglas Anderson *
340e40b0c16SDouglas Anderson * For ACTIVE_ONLY transfers we want to call back into the client when the
341e40b0c16SDouglas Anderson * transfer finishes. To do this we need the "request" that the client
342e40b0c16SDouglas Anderson * originally provided us. This function grabs the request that we stashed
343e40b0c16SDouglas Anderson * when we started the transfer.
344e40b0c16SDouglas Anderson *
345e40b0c16SDouglas Anderson * This only makes sense for ACTIVE_ONLY transfers since those are the only
346e40b0c16SDouglas Anderson * ones we track sending (the only ones we enable interrupts for and the only
347e40b0c16SDouglas Anderson * ones we call back to the client for).
348e40b0c16SDouglas Anderson *
349e40b0c16SDouglas Anderson * Return: The stashed request.
350e40b0c16SDouglas Anderson */
get_req_from_tcs(struct rsc_drv * drv,int tcs_id)351658628e7SLina Iyer static const struct tcs_request *get_req_from_tcs(struct rsc_drv *drv,
352658628e7SLina Iyer int tcs_id)
353658628e7SLina Iyer {
354658628e7SLina Iyer struct tcs_group *tcs;
355658628e7SLina Iyer int i;
356658628e7SLina Iyer
357efa1c257SRaju P.L.S.S.S.N for (i = 0; i < TCS_TYPE_NR; i++) {
358658628e7SLina Iyer tcs = &drv->tcs[i];
359658628e7SLina Iyer if (tcs->mask & BIT(tcs_id))
360658628e7SLina Iyer return tcs->req[tcs_id - tcs->offset];
361658628e7SLina Iyer }
362658628e7SLina Iyer
363658628e7SLina Iyer return NULL;
364658628e7SLina Iyer }
365658628e7SLina Iyer
366e40b0c16SDouglas Anderson /**
367e40b0c16SDouglas Anderson * __tcs_set_trigger() - Start xfer on a TCS or unset trigger on a borrowed TCS
368e40b0c16SDouglas Anderson * @drv: The controller.
369e40b0c16SDouglas Anderson * @tcs_id: The global ID of this TCS.
370e40b0c16SDouglas Anderson * @trigger: If true then untrigger/retrigger. If false then just untrigger.
371e40b0c16SDouglas Anderson *
372e40b0c16SDouglas Anderson * In the normal case we only ever call with "trigger=true" to start a
373e40b0c16SDouglas Anderson * transfer. That will un-trigger/disable the TCS from the last transfer
374e40b0c16SDouglas Anderson * then trigger/enable for this transfer.
375e40b0c16SDouglas Anderson *
376e40b0c16SDouglas Anderson * If we borrowed a wake TCS for an active-only transfer we'll also call
377e40b0c16SDouglas Anderson * this function with "trigger=false" to just do the un-trigger/disable
378e40b0c16SDouglas Anderson * before using the TCS for wake purposes again.
379e40b0c16SDouglas Anderson *
380e40b0c16SDouglas Anderson * Note that the AP is only in charge of triggering active-only transfers.
381e40b0c16SDouglas Anderson * The AP never triggers sleep/wake values using this function.
382e40b0c16SDouglas Anderson */
__tcs_set_trigger(struct rsc_drv * drv,int tcs_id,bool trigger)38315b3bf61SRaju P.L.S.S.S.N static void __tcs_set_trigger(struct rsc_drv *drv, int tcs_id, bool trigger)
38415b3bf61SRaju P.L.S.S.S.N {
38515b3bf61SRaju P.L.S.S.S.N u32 enable;
38640482e4fSAbel Vesa u32 reg = drv->regs[RSC_DRV_CONTROL];
38715b3bf61SRaju P.L.S.S.S.N
38815b3bf61SRaju P.L.S.S.S.N /*
38915b3bf61SRaju P.L.S.S.S.N * HW req: Clear the DRV_CONTROL and enable TCS again
39015b3bf61SRaju P.L.S.S.S.N * While clearing ensure that the AMC mode trigger is cleared
39115b3bf61SRaju P.L.S.S.S.N * and then the mode enable is cleared.
39215b3bf61SRaju P.L.S.S.S.N */
39340482e4fSAbel Vesa enable = read_tcs_reg(drv, reg, tcs_id);
39415b3bf61SRaju P.L.S.S.S.N enable &= ~TCS_AMC_MODE_TRIGGER;
39540482e4fSAbel Vesa write_tcs_reg_sync(drv, reg, tcs_id, enable);
39615b3bf61SRaju P.L.S.S.S.N enable &= ~TCS_AMC_MODE_ENABLE;
39740482e4fSAbel Vesa write_tcs_reg_sync(drv, reg, tcs_id, enable);
39815b3bf61SRaju P.L.S.S.S.N
39915b3bf61SRaju P.L.S.S.S.N if (trigger) {
40015b3bf61SRaju P.L.S.S.S.N /* Enable the AMC mode on the TCS and then trigger the TCS */
40115b3bf61SRaju P.L.S.S.S.N enable = TCS_AMC_MODE_ENABLE;
40240482e4fSAbel Vesa write_tcs_reg_sync(drv, reg, tcs_id, enable);
40315b3bf61SRaju P.L.S.S.S.N enable |= TCS_AMC_MODE_TRIGGER;
40440482e4fSAbel Vesa write_tcs_reg(drv, reg, tcs_id, enable);
40515b3bf61SRaju P.L.S.S.S.N }
40615b3bf61SRaju P.L.S.S.S.N }
40715b3bf61SRaju P.L.S.S.S.N
408e40b0c16SDouglas Anderson /**
409e40b0c16SDouglas Anderson * enable_tcs_irq() - Enable or disable interrupts on the given TCS.
410e40b0c16SDouglas Anderson * @drv: The controller.
411e40b0c16SDouglas Anderson * @tcs_id: The global ID of this TCS.
412e40b0c16SDouglas Anderson * @enable: If true then enable; if false then disable
413e40b0c16SDouglas Anderson *
414e40b0c16SDouglas Anderson * We only ever call this when we borrow a wake TCS for an active-only
415e40b0c16SDouglas Anderson * transfer. For active-only TCSes interrupts are always left enabled.
416e40b0c16SDouglas Anderson */
enable_tcs_irq(struct rsc_drv * drv,int tcs_id,bool enable)41715b3bf61SRaju P.L.S.S.S.N static void enable_tcs_irq(struct rsc_drv *drv, int tcs_id, bool enable)
41815b3bf61SRaju P.L.S.S.S.N {
41915b3bf61SRaju P.L.S.S.S.N u32 data;
42040482e4fSAbel Vesa u32 reg = drv->regs[RSC_DRV_IRQ_ENABLE];
42115b3bf61SRaju P.L.S.S.S.N
42240482e4fSAbel Vesa data = readl_relaxed(drv->tcs_base + reg);
42315b3bf61SRaju P.L.S.S.S.N if (enable)
42415b3bf61SRaju P.L.S.S.S.N data |= BIT(tcs_id);
42515b3bf61SRaju P.L.S.S.S.N else
42615b3bf61SRaju P.L.S.S.S.N data &= ~BIT(tcs_id);
42740482e4fSAbel Vesa writel_relaxed(data, drv->tcs_base + reg);
42815b3bf61SRaju P.L.S.S.S.N }
42915b3bf61SRaju P.L.S.S.S.N
430658628e7SLina Iyer /**
431e40b0c16SDouglas Anderson * tcs_tx_done() - TX Done interrupt handler.
432e40b0c16SDouglas Anderson * @irq: The IRQ number (ignored).
433e40b0c16SDouglas Anderson * @p: Pointer to "struct rsc_drv".
434e40b0c16SDouglas Anderson *
435e40b0c16SDouglas Anderson * Called for ACTIVE_ONLY transfers (those are the only ones we enable the
436e40b0c16SDouglas Anderson * IRQ for) when a transfer is done.
437e40b0c16SDouglas Anderson *
438e40b0c16SDouglas Anderson * Return: IRQ_HANDLED
439658628e7SLina Iyer */
tcs_tx_done(int irq,void * p)440658628e7SLina Iyer static irqreturn_t tcs_tx_done(int irq, void *p)
441658628e7SLina Iyer {
442658628e7SLina Iyer struct rsc_drv *drv = p;
443323dc2dcSAbel Vesa int i;
444658628e7SLina Iyer unsigned long irq_status;
445658628e7SLina Iyer const struct tcs_request *req;
446658628e7SLina Iyer
44740482e4fSAbel Vesa irq_status = readl_relaxed(drv->tcs_base + drv->regs[RSC_DRV_IRQ_STATUS]);
448658628e7SLina Iyer
449a0776620SStephen Boyd for_each_set_bit(i, &irq_status, BITS_PER_TYPE(u32)) {
450658628e7SLina Iyer req = get_req_from_tcs(drv, i);
451f2590e4bSStephen Boyd if (WARN_ON(!req))
452658628e7SLina Iyer goto skip;
453658628e7SLina Iyer
454323dc2dcSAbel Vesa trace_rpmh_tx_done(drv, i, req);
45515b3bf61SRaju P.L.S.S.S.N
45615b3bf61SRaju P.L.S.S.S.N /*
45715b3bf61SRaju P.L.S.S.S.N * If wake tcs was re-purposed for sending active
45815b3bf61SRaju P.L.S.S.S.N * votes, clear AMC trigger & enable modes and
45915b3bf61SRaju P.L.S.S.S.N * disable interrupt for this TCS
46015b3bf61SRaju P.L.S.S.S.N */
46115b3bf61SRaju P.L.S.S.S.N if (!drv->tcs[ACTIVE_TCS].num_tcs)
46215b3bf61SRaju P.L.S.S.S.N __tcs_set_trigger(drv, i, false);
463658628e7SLina Iyer skip:
464658628e7SLina Iyer /* Reclaim the TCS */
46540482e4fSAbel Vesa write_tcs_reg(drv, drv->regs[RSC_DRV_CMD_ENABLE], i, 0);
46640482e4fSAbel Vesa writel_relaxed(BIT(i), drv->tcs_base + drv->regs[RSC_DRV_IRQ_CLEAR]);
467658628e7SLina Iyer spin_lock(&drv->lock);
468658628e7SLina Iyer clear_bit(i, drv->tcs_in_use);
46915b3bf61SRaju P.L.S.S.S.N /*
47015b3bf61SRaju P.L.S.S.S.N * Disable interrupt for WAKE TCS to avoid being
47115b3bf61SRaju P.L.S.S.S.N * spammed with interrupts coming when the solver
47215b3bf61SRaju P.L.S.S.S.N * sends its wake votes.
47315b3bf61SRaju P.L.S.S.S.N */
47415b3bf61SRaju P.L.S.S.S.N if (!drv->tcs[ACTIVE_TCS].num_tcs)
47515b3bf61SRaju P.L.S.S.S.N enable_tcs_irq(drv, i, false);
476658628e7SLina Iyer spin_unlock(&drv->lock);
4772bc20f3cSStephen Boyd wake_up(&drv->tcs_wait);
478c1038456SLina Iyer if (req)
479323dc2dcSAbel Vesa rpmh_tx_done(req);
480658628e7SLina Iyer }
481658628e7SLina Iyer
482658628e7SLina Iyer return IRQ_HANDLED;
483658628e7SLina Iyer }
484658628e7SLina Iyer
485e40b0c16SDouglas Anderson /**
486e40b0c16SDouglas Anderson * __tcs_buffer_write() - Write to TCS hardware from a request; don't trigger.
487e40b0c16SDouglas Anderson * @drv: The controller.
488e40b0c16SDouglas Anderson * @tcs_id: The global ID of this TCS.
489e40b0c16SDouglas Anderson * @cmd_id: The index within the TCS to start writing.
490e40b0c16SDouglas Anderson * @msg: The message we want to send, which will contain several addr/data
491e40b0c16SDouglas Anderson * pairs to program (but few enough that they all fit in one TCS).
492e40b0c16SDouglas Anderson *
493e40b0c16SDouglas Anderson * This is used for all types of transfers (active, sleep, and wake).
494e40b0c16SDouglas Anderson */
__tcs_buffer_write(struct rsc_drv * drv,int tcs_id,int cmd_id,const struct tcs_request * msg)495658628e7SLina Iyer static void __tcs_buffer_write(struct rsc_drv *drv, int tcs_id, int cmd_id,
496658628e7SLina Iyer const struct tcs_request *msg)
497658628e7SLina Iyer {
498fef419c4SLina Iyer u32 msgid;
499fef419c4SLina Iyer u32 cmd_msgid = CMD_MSGID_LEN | CMD_MSGID_WRITE;
500658628e7SLina Iyer u32 cmd_enable = 0;
501658628e7SLina Iyer struct tcs_cmd *cmd;
502658628e7SLina Iyer int i, j;
503658628e7SLina Iyer
504fef419c4SLina Iyer /* Convert all commands to RR when the request has wait_for_compl set */
505658628e7SLina Iyer cmd_msgid |= msg->wait_for_compl ? CMD_MSGID_RESP_REQ : 0;
506658628e7SLina Iyer
507658628e7SLina Iyer for (i = 0, j = cmd_id; i < msg->num_cmds; i++, j++) {
508658628e7SLina Iyer cmd = &msg->cmds[i];
509658628e7SLina Iyer cmd_enable |= BIT(j);
510658628e7SLina Iyer msgid = cmd_msgid;
511fef419c4SLina Iyer /*
512fef419c4SLina Iyer * Additionally, if the cmd->wait is set, make the command
513fef419c4SLina Iyer * response reqd even if the overall request was fire-n-forget.
514fef419c4SLina Iyer */
515658628e7SLina Iyer msgid |= cmd->wait ? CMD_MSGID_RESP_REQ : 0;
516fc087fe5SLina Iyer
51740482e4fSAbel Vesa write_tcs_cmd(drv, drv->regs[RSC_DRV_CMD_MSGID], tcs_id, j, msgid);
51840482e4fSAbel Vesa write_tcs_cmd(drv, drv->regs[RSC_DRV_CMD_ADDR], tcs_id, j, cmd->addr);
51940482e4fSAbel Vesa write_tcs_cmd(drv, drv->regs[RSC_DRV_CMD_DATA], tcs_id, j, cmd->data);
5201b06d8caSBjorn Andersson trace_rpmh_send_msg(drv, tcs_id, msg->state, j, msgid, cmd);
521658628e7SLina Iyer }
522658628e7SLina Iyer
52340482e4fSAbel Vesa cmd_enable |= read_tcs_reg(drv, drv->regs[RSC_DRV_CMD_ENABLE], tcs_id);
52440482e4fSAbel Vesa write_tcs_reg(drv, drv->regs[RSC_DRV_CMD_ENABLE], tcs_id, cmd_enable);
525658628e7SLina Iyer }
526658628e7SLina Iyer
527e40b0c16SDouglas Anderson /**
528e40b0c16SDouglas Anderson * check_for_req_inflight() - Look to see if conflicting cmds are in flight.
529e40b0c16SDouglas Anderson * @drv: The controller.
530e40b0c16SDouglas Anderson * @tcs: A pointer to the tcs_group used for ACTIVE_ONLY transfers.
531e40b0c16SDouglas Anderson * @msg: The message we want to send, which will contain several addr/data
532e40b0c16SDouglas Anderson * pairs to program (but few enough that they all fit in one TCS).
533e40b0c16SDouglas Anderson *
534e40b0c16SDouglas Anderson * This will walk through the TCSes in the group and check if any of them
535e40b0c16SDouglas Anderson * appear to be sending to addresses referenced in the message. If it finds
536e40b0c16SDouglas Anderson * one it'll return -EBUSY.
537e40b0c16SDouglas Anderson *
538e40b0c16SDouglas Anderson * Only for use for active-only transfers.
539e40b0c16SDouglas Anderson *
540e40b0c16SDouglas Anderson * Must be called with the drv->lock held since that protects tcs_in_use.
541e40b0c16SDouglas Anderson *
542e40b0c16SDouglas Anderson * Return: 0 if nothing in flight or -EBUSY if we should try again later.
543e40b0c16SDouglas Anderson * The caller must re-enable interrupts between tries since that's
544814a0d46SStephen Boyd * the only way tcs_in_use will ever be updated and the only way
545e40b0c16SDouglas Anderson * RSC_DRV_CMD_ENABLE will ever be cleared.
546e40b0c16SDouglas Anderson */
check_for_req_inflight(struct rsc_drv * drv,struct tcs_group * tcs,const struct tcs_request * msg)547658628e7SLina Iyer static int check_for_req_inflight(struct rsc_drv *drv, struct tcs_group *tcs,
548658628e7SLina Iyer const struct tcs_request *msg)
549658628e7SLina Iyer {
550658628e7SLina Iyer unsigned long curr_enabled;
551658628e7SLina Iyer u32 addr;
552814a0d46SStephen Boyd int j, k;
553814a0d46SStephen Boyd int i = tcs->offset;
554658628e7SLina Iyer
555814a0d46SStephen Boyd for_each_set_bit_from(i, drv->tcs_in_use, tcs->offset + tcs->num_tcs) {
55640482e4fSAbel Vesa curr_enabled = read_tcs_reg(drv, drv->regs[RSC_DRV_CMD_ENABLE], i);
557658628e7SLina Iyer
558658628e7SLina Iyer for_each_set_bit(j, &curr_enabled, MAX_CMDS_PER_TCS) {
55940482e4fSAbel Vesa addr = read_tcs_cmd(drv, drv->regs[RSC_DRV_CMD_ADDR], i, j);
560658628e7SLina Iyer for (k = 0; k < msg->num_cmds; k++) {
5616a283d64SMaulik Shah if (cmd_db_match_resource_addr(msg->cmds[k].addr, addr))
562658628e7SLina Iyer return -EBUSY;
563658628e7SLina Iyer }
564658628e7SLina Iyer }
565658628e7SLina Iyer }
566658628e7SLina Iyer
567658628e7SLina Iyer return 0;
568658628e7SLina Iyer }
569658628e7SLina Iyer
570e40b0c16SDouglas Anderson /**
571e40b0c16SDouglas Anderson * find_free_tcs() - Find free tcs in the given tcs_group; only for active.
572e40b0c16SDouglas Anderson * @tcs: A pointer to the active-only tcs_group (or the wake tcs_group if
573e40b0c16SDouglas Anderson * we borrowed it because there are zero active-only ones).
574e40b0c16SDouglas Anderson *
575e40b0c16SDouglas Anderson * Must be called with the drv->lock held since that protects tcs_in_use.
576e40b0c16SDouglas Anderson *
577814a0d46SStephen Boyd * Return: The first tcs that's free or -EBUSY if all in use.
578e40b0c16SDouglas Anderson */
find_free_tcs(struct tcs_group * tcs)579658628e7SLina Iyer static int find_free_tcs(struct tcs_group *tcs)
580658628e7SLina Iyer {
581814a0d46SStephen Boyd const struct rsc_drv *drv = tcs->drv;
582814a0d46SStephen Boyd unsigned long i;
583814a0d46SStephen Boyd unsigned long max = tcs->offset + tcs->num_tcs;
584658628e7SLina Iyer
585814a0d46SStephen Boyd i = find_next_zero_bit(drv->tcs_in_use, max, tcs->offset);
586814a0d46SStephen Boyd if (i >= max)
587658628e7SLina Iyer return -EBUSY;
588814a0d46SStephen Boyd
589814a0d46SStephen Boyd return i;
590658628e7SLina Iyer }
591658628e7SLina Iyer
592e40b0c16SDouglas Anderson /**
5932bc20f3cSStephen Boyd * claim_tcs_for_req() - Claim a tcs in the given tcs_group; only for active.
594e40b0c16SDouglas Anderson * @drv: The controller.
5952bc20f3cSStephen Boyd * @tcs: The tcs_group used for ACTIVE_ONLY transfers.
596e40b0c16SDouglas Anderson * @msg: The data to be sent.
597e40b0c16SDouglas Anderson *
5982bc20f3cSStephen Boyd * Claims a tcs in the given tcs_group while making sure that no existing cmd
5992bc20f3cSStephen Boyd * is in flight that would conflict with the one in @msg.
600e40b0c16SDouglas Anderson *
6012bc20f3cSStephen Boyd * Context: Must be called with the drv->lock held since that protects
6022bc20f3cSStephen Boyd * tcs_in_use.
603e40b0c16SDouglas Anderson *
6042bc20f3cSStephen Boyd * Return: The id of the claimed tcs or -EBUSY if a matching msg is in flight
6052bc20f3cSStephen Boyd * or the tcs_group is full.
606e40b0c16SDouglas Anderson */
claim_tcs_for_req(struct rsc_drv * drv,struct tcs_group * tcs,const struct tcs_request * msg)6072bc20f3cSStephen Boyd static int claim_tcs_for_req(struct rsc_drv *drv, struct tcs_group *tcs,
6082bc20f3cSStephen Boyd const struct tcs_request *msg)
609658628e7SLina Iyer {
610658628e7SLina Iyer int ret;
611658628e7SLina Iyer
612658628e7SLina Iyer /*
613658628e7SLina Iyer * The h/w does not like if we send a request to the same address,
614658628e7SLina Iyer * when one is already in-flight or being processed.
615658628e7SLina Iyer */
616658628e7SLina Iyer ret = check_for_req_inflight(drv, tcs, msg);
617555701a4SDouglas Anderson if (ret)
6182bc20f3cSStephen Boyd return ret;
619658628e7SLina Iyer
6202bc20f3cSStephen Boyd return find_free_tcs(tcs);
6212bc20f3cSStephen Boyd }
6222bc20f3cSStephen Boyd
6232bc20f3cSStephen Boyd /**
6242bc20f3cSStephen Boyd * rpmh_rsc_send_data() - Write / trigger active-only message.
6252bc20f3cSStephen Boyd * @drv: The controller.
6262bc20f3cSStephen Boyd * @msg: The data to be sent.
6272bc20f3cSStephen Boyd *
6282bc20f3cSStephen Boyd * NOTES:
6292bc20f3cSStephen Boyd * - This is only used for "ACTIVE_ONLY" since the limitations of this
6302bc20f3cSStephen Boyd * function don't make sense for sleep/wake cases.
6312bc20f3cSStephen Boyd * - To do the transfer, we will grab a whole TCS for ourselves--we don't
6322bc20f3cSStephen Boyd * try to share. If there are none available we'll wait indefinitely
6332bc20f3cSStephen Boyd * for a free one.
6342bc20f3cSStephen Boyd * - This function will not wait for the commands to be finished, only for
6352bc20f3cSStephen Boyd * data to be programmed into the RPMh. See rpmh_tx_done() which will
6362bc20f3cSStephen Boyd * be called when the transfer is fully complete.
6372bc20f3cSStephen Boyd * - This function must be called with interrupts enabled. If the hardware
6382bc20f3cSStephen Boyd * is busy doing someone else's transfer we need that transfer to fully
6392bc20f3cSStephen Boyd * finish so that we can have the hardware, and to fully finish it needs
6402bc20f3cSStephen Boyd * the interrupt handler to run. If the interrupts is set to run on the
6412bc20f3cSStephen Boyd * active CPU this can never happen if interrupts are disabled.
6422bc20f3cSStephen Boyd *
6432bc20f3cSStephen Boyd * Return: 0 on success, -EINVAL on error.
6442bc20f3cSStephen Boyd */
rpmh_rsc_send_data(struct rsc_drv * drv,const struct tcs_request * msg)6452bc20f3cSStephen Boyd int rpmh_rsc_send_data(struct rsc_drv *drv, const struct tcs_request *msg)
6462bc20f3cSStephen Boyd {
6472bc20f3cSStephen Boyd struct tcs_group *tcs;
6482bc20f3cSStephen Boyd int tcs_id;
6492bc20f3cSStephen Boyd unsigned long flags;
6502bc20f3cSStephen Boyd
6512bc20f3cSStephen Boyd tcs = get_tcs_for_msg(drv, msg);
6522bc20f3cSStephen Boyd if (IS_ERR(tcs))
6532bc20f3cSStephen Boyd return PTR_ERR(tcs);
6542bc20f3cSStephen Boyd
6552bc20f3cSStephen Boyd spin_lock_irqsave(&drv->lock, flags);
6562bc20f3cSStephen Boyd
6572bc20f3cSStephen Boyd /* Wait forever for a free tcs. It better be there eventually! */
6582bc20f3cSStephen Boyd wait_event_lock_irq(drv->tcs_wait,
6592bc20f3cSStephen Boyd (tcs_id = claim_tcs_for_req(drv, tcs, msg)) >= 0,
6602bc20f3cSStephen Boyd drv->lock);
661658628e7SLina Iyer
662658628e7SLina Iyer tcs->req[tcs_id - tcs->offset] = msg;
663658628e7SLina Iyer set_bit(tcs_id, drv->tcs_in_use);
66438427e5aSMaulik Shah if (msg->state == RPMH_ACTIVE_ONLY_STATE && tcs->type != ACTIVE_TCS) {
66538427e5aSMaulik Shah /*
66638427e5aSMaulik Shah * Clear previously programmed WAKE commands in selected
66738427e5aSMaulik Shah * repurposed TCS to avoid triggering them. tcs->slots will be
66838427e5aSMaulik Shah * cleaned from rpmh_flush() by invoking rpmh_rsc_invalidate()
66938427e5aSMaulik Shah */
67040482e4fSAbel Vesa write_tcs_reg_sync(drv, drv->regs[RSC_DRV_CMD_ENABLE], tcs_id, 0);
67115b3bf61SRaju P.L.S.S.S.N enable_tcs_irq(drv, tcs_id, true);
67238427e5aSMaulik Shah }
673555701a4SDouglas Anderson spin_unlock_irqrestore(&drv->lock, flags);
674658628e7SLina Iyer
675555701a4SDouglas Anderson /*
676555701a4SDouglas Anderson * These two can be done after the lock is released because:
677555701a4SDouglas Anderson * - We marked "tcs_in_use" under lock.
678555701a4SDouglas Anderson * - Once "tcs_in_use" has been marked nobody else could be writing
679555701a4SDouglas Anderson * to these registers until the interrupt goes off.
680555701a4SDouglas Anderson * - The interrupt can't go off until we trigger w/ the last line
681555701a4SDouglas Anderson * of __tcs_set_trigger() below.
682555701a4SDouglas Anderson */
683658628e7SLina Iyer __tcs_buffer_write(drv, tcs_id, 0, msg);
68415b3bf61SRaju P.L.S.S.S.N __tcs_set_trigger(drv, tcs_id, true);
685658628e7SLina Iyer
686555701a4SDouglas Anderson return 0;
687658628e7SLina Iyer }
688658628e7SLina Iyer
689e40b0c16SDouglas Anderson /**
690e40b0c16SDouglas Anderson * find_slots() - Find a place to write the given message.
691e40b0c16SDouglas Anderson * @tcs: The tcs group to search.
692e40b0c16SDouglas Anderson * @msg: The message we want to find room for.
693e40b0c16SDouglas Anderson * @tcs_id: If we return 0 from the function, we return the global ID of the
694e40b0c16SDouglas Anderson * TCS to write to here.
695e40b0c16SDouglas Anderson * @cmd_id: If we return 0 from the function, we return the index of
696e40b0c16SDouglas Anderson * the command array of the returned TCS where the client should
697e40b0c16SDouglas Anderson * start writing the message.
698e40b0c16SDouglas Anderson *
699e40b0c16SDouglas Anderson * Only for use on sleep/wake TCSes since those are the only ones we maintain
700e40b0c16SDouglas Anderson * tcs->slots for.
701e40b0c16SDouglas Anderson *
702e40b0c16SDouglas Anderson * Return: -ENOMEM if there was no room, else 0.
703e40b0c16SDouglas Anderson */
find_slots(struct tcs_group * tcs,const struct tcs_request * msg,int * tcs_id,int * cmd_id)704fa460e45SLina Iyer static int find_slots(struct tcs_group *tcs, const struct tcs_request *msg,
705fa460e45SLina Iyer int *tcs_id, int *cmd_id)
706fa460e45SLina Iyer {
707fa460e45SLina Iyer int slot, offset;
708fa460e45SLina Iyer int i = 0;
709fa460e45SLina Iyer
710e40b0c16SDouglas Anderson /* Do over, until we can fit the full payload in a single TCS */
711fa460e45SLina Iyer do {
712fa460e45SLina Iyer slot = bitmap_find_next_zero_area(tcs->slots, MAX_TCS_SLOTS,
713fa460e45SLina Iyer i, msg->num_cmds, 0);
714bbeac60fSMaulik Shah if (slot >= tcs->num_tcs * tcs->ncpt)
715fa460e45SLina Iyer return -ENOMEM;
716fa460e45SLina Iyer i += tcs->ncpt;
717fa460e45SLina Iyer } while (slot + msg->num_cmds - 1 >= i);
718fa460e45SLina Iyer
719fa460e45SLina Iyer bitmap_set(tcs->slots, slot, msg->num_cmds);
720fa460e45SLina Iyer
721fa460e45SLina Iyer offset = slot / tcs->ncpt;
722fa460e45SLina Iyer *tcs_id = offset + tcs->offset;
723fa460e45SLina Iyer *cmd_id = slot % tcs->ncpt;
724fa460e45SLina Iyer
725fa460e45SLina Iyer return 0;
726fa460e45SLina Iyer }
727fa460e45SLina Iyer
728427ef4f7SDouglas Anderson /**
729e40b0c16SDouglas Anderson * rpmh_rsc_write_ctrl_data() - Write request to controller but don't trigger.
730e40b0c16SDouglas Anderson * @drv: The controller.
731e40b0c16SDouglas Anderson * @msg: The data to be written to the controller.
732427ef4f7SDouglas Anderson *
733e395f021SJason Wang * This should only be called for sleep/wake state, never active-only
734e40b0c16SDouglas Anderson * state.
735427ef4f7SDouglas Anderson *
736555701a4SDouglas Anderson * The caller must ensure that no other RPMH actions are happening and the
737555701a4SDouglas Anderson * controller is idle when this function is called since it runs lockless.
738555701a4SDouglas Anderson *
739e40b0c16SDouglas Anderson * Return: 0 if no error; else -error.
740427ef4f7SDouglas Anderson */
rpmh_rsc_write_ctrl_data(struct rsc_drv * drv,const struct tcs_request * msg)741427ef4f7SDouglas Anderson int rpmh_rsc_write_ctrl_data(struct rsc_drv *drv, const struct tcs_request *msg)
742fa460e45SLina Iyer {
743fa460e45SLina Iyer struct tcs_group *tcs;
744fa460e45SLina Iyer int tcs_id = 0, cmd_id = 0;
745fa460e45SLina Iyer int ret;
746fa460e45SLina Iyer
747fa460e45SLina Iyer tcs = get_tcs_for_msg(drv, msg);
748fa460e45SLina Iyer if (IS_ERR(tcs))
749fa460e45SLina Iyer return PTR_ERR(tcs);
750fa460e45SLina Iyer
751fa460e45SLina Iyer /* find the TCS id and the command in the TCS to write to */
752fa460e45SLina Iyer ret = find_slots(tcs, msg, &tcs_id, &cmd_id);
753fa460e45SLina Iyer if (!ret)
754fa460e45SLina Iyer __tcs_buffer_write(drv, tcs_id, cmd_id, msg);
755fa460e45SLina Iyer
756fa460e45SLina Iyer return ret;
757fa460e45SLina Iyer }
758fa460e45SLina Iyer
759fa460e45SLina Iyer /**
760985427f9SMaulik Shah * rpmh_rsc_ctrlr_is_busy() - Check if any of the AMCs are busy.
761985427f9SMaulik Shah * @drv: The controller
762985427f9SMaulik Shah *
763985427f9SMaulik Shah * Checks if any of the AMCs are busy in handling ACTIVE sets.
764985427f9SMaulik Shah * This is called from the last cpu powering down before flushing
765985427f9SMaulik Shah * SLEEP and WAKE sets. If AMCs are busy, controller can not enter
766985427f9SMaulik Shah * power collapse, so deny from the last cpu's pm notification.
767985427f9SMaulik Shah *
768d2a8cfc6SDouglas Anderson * Context: Must be called with the drv->lock held.
769d2a8cfc6SDouglas Anderson *
770985427f9SMaulik Shah * Return:
771985427f9SMaulik Shah * * False - AMCs are idle
772985427f9SMaulik Shah * * True - AMCs are busy
773985427f9SMaulik Shah */
rpmh_rsc_ctrlr_is_busy(struct rsc_drv * drv)774985427f9SMaulik Shah static bool rpmh_rsc_ctrlr_is_busy(struct rsc_drv *drv)
775985427f9SMaulik Shah {
776814a0d46SStephen Boyd unsigned long set;
777814a0d46SStephen Boyd const struct tcs_group *tcs = &drv->tcs[ACTIVE_TCS];
778814a0d46SStephen Boyd unsigned long max;
779985427f9SMaulik Shah
780985427f9SMaulik Shah /*
781985427f9SMaulik Shah * If we made an active request on a RSC that does not have a
782985427f9SMaulik Shah * dedicated TCS for active state use, then re-purposed wake TCSes
783985427f9SMaulik Shah * should be checked for not busy, because we used wake TCSes for
784985427f9SMaulik Shah * active requests in this case.
785985427f9SMaulik Shah */
786985427f9SMaulik Shah if (!tcs->num_tcs)
78753d49fe1SDouglas Anderson tcs = &drv->tcs[WAKE_TCS];
788985427f9SMaulik Shah
789814a0d46SStephen Boyd max = tcs->offset + tcs->num_tcs;
790814a0d46SStephen Boyd set = find_next_bit(drv->tcs_in_use, max, tcs->offset);
791985427f9SMaulik Shah
792814a0d46SStephen Boyd return set < max;
793985427f9SMaulik Shah }
794985427f9SMaulik Shah
795e40b0c16SDouglas Anderson /**
796cccbe3e5SMaulik Shah * rpmh_rsc_write_next_wakeup() - Write next wakeup in CONTROL_TCS.
797cccbe3e5SMaulik Shah * @drv: The controller
798cccbe3e5SMaulik Shah *
799cccbe3e5SMaulik Shah * Writes maximum wakeup cycles when called from suspend.
800cccbe3e5SMaulik Shah * Writes earliest hrtimer wakeup when called from idle.
801cccbe3e5SMaulik Shah */
rpmh_rsc_write_next_wakeup(struct rsc_drv * drv)802cccbe3e5SMaulik Shah void rpmh_rsc_write_next_wakeup(struct rsc_drv *drv)
803cccbe3e5SMaulik Shah {
804cccbe3e5SMaulik Shah ktime_t now, wakeup;
805cccbe3e5SMaulik Shah u64 wakeup_us, wakeup_cycles = ~0;
806cccbe3e5SMaulik Shah u32 lo, hi;
807cccbe3e5SMaulik Shah
808cccbe3e5SMaulik Shah if (!drv->tcs[CONTROL_TCS].num_tcs || !drv->genpd_nb.notifier_call)
809cccbe3e5SMaulik Shah return;
810cccbe3e5SMaulik Shah
811cccbe3e5SMaulik Shah /* Set highest time when system (timekeeping) is suspended */
812cccbe3e5SMaulik Shah if (system_state == SYSTEM_SUSPEND)
813cccbe3e5SMaulik Shah goto exit;
814cccbe3e5SMaulik Shah
815cccbe3e5SMaulik Shah /* Find the earliest hrtimer wakeup from online cpus */
816cccbe3e5SMaulik Shah wakeup = dev_pm_genpd_get_next_hrtimer(drv->dev);
817cccbe3e5SMaulik Shah
818cccbe3e5SMaulik Shah /* Find the relative wakeup in kernel time scale */
819cccbe3e5SMaulik Shah now = ktime_get();
820cccbe3e5SMaulik Shah wakeup = ktime_sub(wakeup, now);
821cccbe3e5SMaulik Shah wakeup_us = ktime_to_us(wakeup);
822cccbe3e5SMaulik Shah
823cccbe3e5SMaulik Shah /* Convert the wakeup to arch timer scale */
824cccbe3e5SMaulik Shah wakeup_cycles = USECS_TO_CYCLES(wakeup_us);
825cccbe3e5SMaulik Shah wakeup_cycles += arch_timer_read_counter();
826cccbe3e5SMaulik Shah
827cccbe3e5SMaulik Shah exit:
828cccbe3e5SMaulik Shah lo = wakeup_cycles & RSC_DRV_CTL_TCS_DATA_LO_MASK;
829cccbe3e5SMaulik Shah hi = wakeup_cycles >> RSC_DRV_CTL_TCS_DATA_SIZE;
830cccbe3e5SMaulik Shah hi &= RSC_DRV_CTL_TCS_DATA_HI_MASK;
831cccbe3e5SMaulik Shah hi |= RSC_DRV_CTL_TCS_DATA_HI_VALID;
832cccbe3e5SMaulik Shah
833cccbe3e5SMaulik Shah writel_relaxed(lo, drv->base + RSC_DRV_CTL_TCS_DATA_LO);
834cccbe3e5SMaulik Shah writel_relaxed(hi, drv->base + RSC_DRV_CTL_TCS_DATA_HI);
835cccbe3e5SMaulik Shah }
836cccbe3e5SMaulik Shah
837cccbe3e5SMaulik Shah /**
838e40b0c16SDouglas Anderson * rpmh_rsc_cpu_pm_callback() - Check if any of the AMCs are busy.
839e40b0c16SDouglas Anderson * @nfb: Pointer to the notifier block in struct rsc_drv.
840e40b0c16SDouglas Anderson * @action: CPU_PM_ENTER, CPU_PM_ENTER_FAILED, or CPU_PM_EXIT.
841e40b0c16SDouglas Anderson * @v: Unused
842e40b0c16SDouglas Anderson *
843e40b0c16SDouglas Anderson * This function is given to cpu_pm_register_notifier so we can be informed
844e40b0c16SDouglas Anderson * about when CPUs go down. When all CPUs go down we know no more active
845e40b0c16SDouglas Anderson * transfers will be started so we write sleep/wake sets. This function gets
846e40b0c16SDouglas Anderson * called from cpuidle code paths and also at system suspend time.
847e40b0c16SDouglas Anderson *
848e40b0c16SDouglas Anderson * If its last CPU going down and AMCs are not busy then writes cached sleep
849e40b0c16SDouglas Anderson * and wake messages to TCSes. The firmware then takes care of triggering
850e40b0c16SDouglas Anderson * them when entering deepest low power modes.
851e40b0c16SDouglas Anderson *
852e40b0c16SDouglas Anderson * Return: See cpu_pm_register_notifier()
853e40b0c16SDouglas Anderson */
rpmh_rsc_cpu_pm_callback(struct notifier_block * nfb,unsigned long action,void * v)854985427f9SMaulik Shah static int rpmh_rsc_cpu_pm_callback(struct notifier_block *nfb,
855985427f9SMaulik Shah unsigned long action, void *v)
856985427f9SMaulik Shah {
857985427f9SMaulik Shah struct rsc_drv *drv = container_of(nfb, struct rsc_drv, rsc_pm);
858985427f9SMaulik Shah int ret = NOTIFY_OK;
859d2a8cfc6SDouglas Anderson int cpus_in_pm;
860985427f9SMaulik Shah
861985427f9SMaulik Shah switch (action) {
862985427f9SMaulik Shah case CPU_PM_ENTER:
863d2a8cfc6SDouglas Anderson cpus_in_pm = atomic_inc_return(&drv->cpus_in_pm);
864d2a8cfc6SDouglas Anderson /*
865d2a8cfc6SDouglas Anderson * NOTE: comments for num_online_cpus() point out that it's
866d2a8cfc6SDouglas Anderson * only a snapshot so we need to be careful. It should be OK
867d2a8cfc6SDouglas Anderson * for us to use, though. It's important for us not to miss
868d2a8cfc6SDouglas Anderson * if we're the last CPU going down so it would only be a
869d2a8cfc6SDouglas Anderson * problem if a CPU went offline right after we did the check
870d2a8cfc6SDouglas Anderson * AND that CPU was not idle AND that CPU was the last non-idle
871d2a8cfc6SDouglas Anderson * CPU. That can't happen. CPUs would have to come out of idle
872d2a8cfc6SDouglas Anderson * before the CPU could go offline.
873d2a8cfc6SDouglas Anderson */
874d2a8cfc6SDouglas Anderson if (cpus_in_pm < num_online_cpus())
875d2a8cfc6SDouglas Anderson return NOTIFY_OK;
876985427f9SMaulik Shah break;
877985427f9SMaulik Shah case CPU_PM_ENTER_FAILED:
878985427f9SMaulik Shah case CPU_PM_EXIT:
879d2a8cfc6SDouglas Anderson atomic_dec(&drv->cpus_in_pm);
880d2a8cfc6SDouglas Anderson return NOTIFY_OK;
8811143c366SDouglas Anderson default:
882d2a8cfc6SDouglas Anderson return NOTIFY_DONE;
883985427f9SMaulik Shah }
884985427f9SMaulik Shah
885d2a8cfc6SDouglas Anderson /*
886d2a8cfc6SDouglas Anderson * It's likely we're on the last CPU. Grab the drv->lock and write
887d2a8cfc6SDouglas Anderson * out the sleep/wake commands to RPMH hardware. Grabbing the lock
888d2a8cfc6SDouglas Anderson * means that if we race with another CPU coming up we are still
889d2a8cfc6SDouglas Anderson * guaranteed to be safe. If another CPU came up just after we checked
890d2a8cfc6SDouglas Anderson * and has grabbed the lock or started an active transfer then we'll
891d2a8cfc6SDouglas Anderson * notice we're busy and abort. If another CPU comes up after we start
892d2a8cfc6SDouglas Anderson * flushing it will be blocked from starting an active transfer until
893d2a8cfc6SDouglas Anderson * we're done flushing. If another CPU starts an active transfer after
894d2a8cfc6SDouglas Anderson * we release the lock we're still OK because we're no longer the last
895d2a8cfc6SDouglas Anderson * CPU.
896d2a8cfc6SDouglas Anderson */
897d2a8cfc6SDouglas Anderson if (spin_trylock(&drv->lock)) {
898d2a8cfc6SDouglas Anderson if (rpmh_rsc_ctrlr_is_busy(drv) || rpmh_flush(&drv->client))
899985427f9SMaulik Shah ret = NOTIFY_BAD;
900d2a8cfc6SDouglas Anderson spin_unlock(&drv->lock);
901d2a8cfc6SDouglas Anderson } else {
902d2a8cfc6SDouglas Anderson /* Another CPU must be up */
903d2a8cfc6SDouglas Anderson return NOTIFY_OK;
904985427f9SMaulik Shah }
905985427f9SMaulik Shah
906d2a8cfc6SDouglas Anderson if (ret == NOTIFY_BAD) {
907d2a8cfc6SDouglas Anderson /* Double-check if we're here because someone else is up */
908d2a8cfc6SDouglas Anderson if (cpus_in_pm < num_online_cpus())
909985427f9SMaulik Shah ret = NOTIFY_OK;
910d2a8cfc6SDouglas Anderson else
911c45def5dSDouglas Anderson /* We won't be called w/ CPU_PM_ENTER_FAILED */
912d2a8cfc6SDouglas Anderson atomic_dec(&drv->cpus_in_pm);
913d2a8cfc6SDouglas Anderson }
914c45def5dSDouglas Anderson
915985427f9SMaulik Shah return ret;
916985427f9SMaulik Shah }
917985427f9SMaulik Shah
91825092e61SLina Iyer /**
91925092e61SLina Iyer * rpmh_rsc_pd_callback() - Check if any of the AMCs are busy.
92025092e61SLina Iyer * @nfb: Pointer to the genpd notifier block in struct rsc_drv.
92125092e61SLina Iyer * @action: GENPD_NOTIFY_PRE_OFF, GENPD_NOTIFY_OFF, GENPD_NOTIFY_PRE_ON or GENPD_NOTIFY_ON.
92225092e61SLina Iyer * @v: Unused
92325092e61SLina Iyer *
92425092e61SLina Iyer * This function is given to dev_pm_genpd_add_notifier() so we can be informed
92525092e61SLina Iyer * about when cluster-pd is going down. When cluster go down we know no more active
92625092e61SLina Iyer * transfers will be started so we write sleep/wake sets. This function gets
92725092e61SLina Iyer * called from cpuidle code paths and also at system suspend time.
92825092e61SLina Iyer *
92925092e61SLina Iyer * If AMCs are not busy then writes cached sleep and wake messages to TCSes.
93025092e61SLina Iyer * The firmware then takes care of triggering them when entering deepest low power modes.
93125092e61SLina Iyer *
93225092e61SLina Iyer * Return:
93325092e61SLina Iyer * * NOTIFY_OK - success
93425092e61SLina Iyer * * NOTIFY_BAD - failure
93525092e61SLina Iyer */
rpmh_rsc_pd_callback(struct notifier_block * nfb,unsigned long action,void * v)93625092e61SLina Iyer static int rpmh_rsc_pd_callback(struct notifier_block *nfb,
93725092e61SLina Iyer unsigned long action, void *v)
93825092e61SLina Iyer {
93925092e61SLina Iyer struct rsc_drv *drv = container_of(nfb, struct rsc_drv, genpd_nb);
94025092e61SLina Iyer
94125092e61SLina Iyer /* We don't need to lock as genpd on/off are serialized */
94225092e61SLina Iyer if ((action == GENPD_NOTIFY_PRE_OFF) &&
94325092e61SLina Iyer (rpmh_rsc_ctrlr_is_busy(drv) || rpmh_flush(&drv->client)))
94425092e61SLina Iyer return NOTIFY_BAD;
94525092e61SLina Iyer
94625092e61SLina Iyer return NOTIFY_OK;
94725092e61SLina Iyer }
94825092e61SLina Iyer
rpmh_rsc_pd_attach(struct rsc_drv * drv,struct device * dev)94925092e61SLina Iyer static int rpmh_rsc_pd_attach(struct rsc_drv *drv, struct device *dev)
95025092e61SLina Iyer {
95125092e61SLina Iyer int ret;
95225092e61SLina Iyer
95325092e61SLina Iyer pm_runtime_enable(dev);
95425092e61SLina Iyer drv->genpd_nb.notifier_call = rpmh_rsc_pd_callback;
95525092e61SLina Iyer ret = dev_pm_genpd_add_notifier(dev, &drv->genpd_nb);
95625092e61SLina Iyer if (ret)
95725092e61SLina Iyer pm_runtime_disable(dev);
95825092e61SLina Iyer
95925092e61SLina Iyer return ret;
96025092e61SLina Iyer }
96125092e61SLina Iyer
rpmh_probe_tcs_config(struct platform_device * pdev,struct rsc_drv * drv)962ab33c8f3SMaulik Shah static int rpmh_probe_tcs_config(struct platform_device *pdev, struct rsc_drv *drv)
963658628e7SLina Iyer {
964658628e7SLina Iyer struct tcs_type_config {
965658628e7SLina Iyer u32 type;
966658628e7SLina Iyer u32 n;
967658628e7SLina Iyer } tcs_cfg[TCS_TYPE_NR] = { { 0 } };
968658628e7SLina Iyer struct device_node *dn = pdev->dev.of_node;
969658628e7SLina Iyer u32 config, max_tcs, ncpt, offset;
970658628e7SLina Iyer int i, ret, n, st = 0;
971658628e7SLina Iyer struct tcs_group *tcs;
972658628e7SLina Iyer
973658628e7SLina Iyer ret = of_property_read_u32(dn, "qcom,tcs-offset", &offset);
974658628e7SLina Iyer if (ret)
975658628e7SLina Iyer return ret;
976ab33c8f3SMaulik Shah drv->tcs_base = drv->base + offset;
977658628e7SLina Iyer
97840482e4fSAbel Vesa config = readl_relaxed(drv->base + drv->regs[DRV_PRNT_CHLD_CONFIG]);
979658628e7SLina Iyer
980658628e7SLina Iyer max_tcs = config;
981658628e7SLina Iyer max_tcs &= DRV_NUM_TCS_MASK << (DRV_NUM_TCS_SHIFT * drv->id);
982658628e7SLina Iyer max_tcs = max_tcs >> (DRV_NUM_TCS_SHIFT * drv->id);
983658628e7SLina Iyer
984658628e7SLina Iyer ncpt = config & (DRV_NCPT_MASK << DRV_NCPT_SHIFT);
985658628e7SLina Iyer ncpt = ncpt >> DRV_NCPT_SHIFT;
986658628e7SLina Iyer
987658628e7SLina Iyer n = of_property_count_u32_elems(dn, "qcom,tcs-config");
988658628e7SLina Iyer if (n != 2 * TCS_TYPE_NR)
989658628e7SLina Iyer return -EINVAL;
990658628e7SLina Iyer
991658628e7SLina Iyer for (i = 0; i < TCS_TYPE_NR; i++) {
992658628e7SLina Iyer ret = of_property_read_u32_index(dn, "qcom,tcs-config",
993658628e7SLina Iyer i * 2, &tcs_cfg[i].type);
994658628e7SLina Iyer if (ret)
995658628e7SLina Iyer return ret;
996658628e7SLina Iyer if (tcs_cfg[i].type >= TCS_TYPE_NR)
997658628e7SLina Iyer return -EINVAL;
998658628e7SLina Iyer
999658628e7SLina Iyer ret = of_property_read_u32_index(dn, "qcom,tcs-config",
1000658628e7SLina Iyer i * 2 + 1, &tcs_cfg[i].n);
1001658628e7SLina Iyer if (ret)
1002658628e7SLina Iyer return ret;
1003658628e7SLina Iyer if (tcs_cfg[i].n > MAX_TCS_PER_TYPE)
1004658628e7SLina Iyer return -EINVAL;
1005658628e7SLina Iyer }
1006658628e7SLina Iyer
1007658628e7SLina Iyer for (i = 0; i < TCS_TYPE_NR; i++) {
1008658628e7SLina Iyer tcs = &drv->tcs[tcs_cfg[i].type];
1009658628e7SLina Iyer if (tcs->drv)
1010658628e7SLina Iyer return -EINVAL;
1011658628e7SLina Iyer tcs->drv = drv;
1012658628e7SLina Iyer tcs->type = tcs_cfg[i].type;
1013658628e7SLina Iyer tcs->num_tcs = tcs_cfg[i].n;
1014658628e7SLina Iyer tcs->ncpt = ncpt;
1015658628e7SLina Iyer
1016658628e7SLina Iyer if (!tcs->num_tcs || tcs->type == CONTROL_TCS)
1017658628e7SLina Iyer continue;
1018658628e7SLina Iyer
1019658628e7SLina Iyer if (st + tcs->num_tcs > max_tcs ||
1020658628e7SLina Iyer st + tcs->num_tcs >= BITS_PER_BYTE * sizeof(tcs->mask))
1021658628e7SLina Iyer return -EINVAL;
1022658628e7SLina Iyer
1023658628e7SLina Iyer tcs->mask = ((1 << tcs->num_tcs) - 1) << st;
1024658628e7SLina Iyer tcs->offset = st;
1025658628e7SLina Iyer st += tcs->num_tcs;
1026658628e7SLina Iyer }
1027658628e7SLina Iyer
1028658628e7SLina Iyer drv->num_tcs = st;
1029658628e7SLina Iyer
1030658628e7SLina Iyer return 0;
1031658628e7SLina Iyer }
1032658628e7SLina Iyer
rpmh_rsc_probe(struct platform_device * pdev)1033658628e7SLina Iyer static int rpmh_rsc_probe(struct platform_device *pdev)
1034658628e7SLina Iyer {
1035658628e7SLina Iyer struct device_node *dn = pdev->dev.of_node;
1036658628e7SLina Iyer struct rsc_drv *drv;
1037985427f9SMaulik Shah char drv_id[10] = {0};
1038658628e7SLina Iyer int ret, irq;
1039985427f9SMaulik Shah u32 solver_config;
104040482e4fSAbel Vesa u32 rsc_id;
1041658628e7SLina Iyer
1042fdd102b5SDouglas Anderson /*
1043fdd102b5SDouglas Anderson * Even though RPMh doesn't directly use cmd-db, all of its children
1044fdd102b5SDouglas Anderson * do. To avoid adding this check to our children we'll do it now.
1045fdd102b5SDouglas Anderson */
1046fdd102b5SDouglas Anderson ret = cmd_db_ready();
1047fdd102b5SDouglas Anderson if (ret) {
1048fdd102b5SDouglas Anderson if (ret != -EPROBE_DEFER)
1049fdd102b5SDouglas Anderson dev_err(&pdev->dev, "Command DB not available (%d)\n",
1050fdd102b5SDouglas Anderson ret);
1051fdd102b5SDouglas Anderson return ret;
1052fdd102b5SDouglas Anderson }
1053fdd102b5SDouglas Anderson
1054658628e7SLina Iyer drv = devm_kzalloc(&pdev->dev, sizeof(*drv), GFP_KERNEL);
1055658628e7SLina Iyer if (!drv)
1056658628e7SLina Iyer return -ENOMEM;
1057658628e7SLina Iyer
1058658628e7SLina Iyer ret = of_property_read_u32(dn, "qcom,drv-id", &drv->id);
1059658628e7SLina Iyer if (ret)
1060658628e7SLina Iyer return ret;
1061658628e7SLina Iyer
1062658628e7SLina Iyer drv->name = of_get_property(dn, "label", NULL);
1063658628e7SLina Iyer if (!drv->name)
1064658628e7SLina Iyer drv->name = dev_name(&pdev->dev);
1065658628e7SLina Iyer
1066985427f9SMaulik Shah snprintf(drv_id, ARRAY_SIZE(drv_id), "drv-%d", drv->id);
1067ab33c8f3SMaulik Shah drv->base = devm_platform_ioremap_resource_byname(pdev, drv_id);
1068ab33c8f3SMaulik Shah if (IS_ERR(drv->base))
1069ab33c8f3SMaulik Shah return PTR_ERR(drv->base);
1070985427f9SMaulik Shah
107140482e4fSAbel Vesa rsc_id = readl_relaxed(drv->base + RSC_DRV_ID);
107240482e4fSAbel Vesa drv->ver.major = rsc_id & (MAJOR_VER_MASK << MAJOR_VER_SHIFT);
107340482e4fSAbel Vesa drv->ver.major >>= MAJOR_VER_SHIFT;
107440482e4fSAbel Vesa drv->ver.minor = rsc_id & (MINOR_VER_MASK << MINOR_VER_SHIFT);
107540482e4fSAbel Vesa drv->ver.minor >>= MINOR_VER_SHIFT;
107640482e4fSAbel Vesa
10773395d36eSKrzysztof Kozlowski if (drv->ver.major == 3)
107840482e4fSAbel Vesa drv->regs = rpmh_rsc_reg_offset_ver_3_0;
107940482e4fSAbel Vesa else
108040482e4fSAbel Vesa drv->regs = rpmh_rsc_reg_offset_ver_2_7;
108140482e4fSAbel Vesa
1082ab33c8f3SMaulik Shah ret = rpmh_probe_tcs_config(pdev, drv);
1083658628e7SLina Iyer if (ret)
1084658628e7SLina Iyer return ret;
1085658628e7SLina Iyer
1086658628e7SLina Iyer spin_lock_init(&drv->lock);
10872bc20f3cSStephen Boyd init_waitqueue_head(&drv->tcs_wait);
1088658628e7SLina Iyer bitmap_zero(drv->tcs_in_use, MAX_TCS_NR);
1089658628e7SLina Iyer
1090658628e7SLina Iyer irq = platform_get_irq(pdev, drv->id);
1091658628e7SLina Iyer if (irq < 0)
1092658628e7SLina Iyer return irq;
1093658628e7SLina Iyer
1094658628e7SLina Iyer ret = devm_request_irq(&pdev->dev, irq, tcs_tx_done,
1095658628e7SLina Iyer IRQF_TRIGGER_HIGH | IRQF_NO_SUSPEND,
1096658628e7SLina Iyer drv->name, drv);
1097658628e7SLina Iyer if (ret)
1098658628e7SLina Iyer return ret;
1099658628e7SLina Iyer
1100985427f9SMaulik Shah /*
110125092e61SLina Iyer * CPU PM/genpd notification are not required for controllers that support
1102985427f9SMaulik Shah * 'HW solver' mode where they can be in autonomous mode executing low
1103985427f9SMaulik Shah * power mode to power down.
1104985427f9SMaulik Shah */
110540482e4fSAbel Vesa solver_config = readl_relaxed(drv->base + drv->regs[DRV_SOLVER_CONFIG]);
1106985427f9SMaulik Shah solver_config &= DRV_HW_SOLVER_MASK << DRV_HW_SOLVER_SHIFT;
1107985427f9SMaulik Shah solver_config = solver_config >> DRV_HW_SOLVER_SHIFT;
1108985427f9SMaulik Shah if (!solver_config) {
110925092e61SLina Iyer if (pdev->dev.pm_domain) {
111025092e61SLina Iyer ret = rpmh_rsc_pd_attach(drv, &pdev->dev);
111125092e61SLina Iyer if (ret)
111225092e61SLina Iyer return ret;
111325092e61SLina Iyer } else {
1114985427f9SMaulik Shah drv->rsc_pm.notifier_call = rpmh_rsc_cpu_pm_callback;
1115985427f9SMaulik Shah cpu_pm_register_notifier(&drv->rsc_pm);
1116985427f9SMaulik Shah }
111725092e61SLina Iyer }
1118985427f9SMaulik Shah
1119658628e7SLina Iyer /* Enable the active TCS to send requests immediately */
1120032c692aSDouglas Anderson writel_relaxed(drv->tcs[ACTIVE_TCS].mask,
112140482e4fSAbel Vesa drv->tcs_base + drv->regs[RSC_DRV_IRQ_ENABLE]);
1122658628e7SLina Iyer
1123600513dfSLina Iyer spin_lock_init(&drv->client.cache_lock);
1124600513dfSLina Iyer INIT_LIST_HEAD(&drv->client.cache);
1125c8790cb6SLina Iyer INIT_LIST_HEAD(&drv->client.batch_cache);
1126600513dfSLina Iyer
1127c1038456SLina Iyer dev_set_drvdata(&pdev->dev, drv);
1128cccbe3e5SMaulik Shah drv->dev = &pdev->dev;
1129c1038456SLina Iyer
113025092e61SLina Iyer ret = devm_of_platform_populate(&pdev->dev);
113125092e61SLina Iyer if (ret && pdev->dev.pm_domain) {
113225092e61SLina Iyer dev_pm_genpd_remove_notifier(&pdev->dev);
113325092e61SLina Iyer pm_runtime_disable(&pdev->dev);
113425092e61SLina Iyer }
113525092e61SLina Iyer
113625092e61SLina Iyer return ret;
1137658628e7SLina Iyer }
1138658628e7SLina Iyer
1139658628e7SLina Iyer static const struct of_device_id rpmh_drv_match[] = {
1140658628e7SLina Iyer { .compatible = "qcom,rpmh-rsc", },
1141658628e7SLina Iyer { }
1142658628e7SLina Iyer };
1143cb365926SJohn Stultz MODULE_DEVICE_TABLE(of, rpmh_drv_match);
1144658628e7SLina Iyer
1145658628e7SLina Iyer static struct platform_driver rpmh_driver = {
1146658628e7SLina Iyer .probe = rpmh_rsc_probe,
1147658628e7SLina Iyer .driver = {
1148658628e7SLina Iyer .name = "rpmh",
1149658628e7SLina Iyer .of_match_table = rpmh_drv_match,
11501a53ce9aSMaulik Shah .suppress_bind_attrs = true,
1151658628e7SLina Iyer },
1152658628e7SLina Iyer };
1153658628e7SLina Iyer
rpmh_driver_init(void)1154658628e7SLina Iyer static int __init rpmh_driver_init(void)
1155658628e7SLina Iyer {
1156658628e7SLina Iyer return platform_driver_register(&rpmh_driver);
1157658628e7SLina Iyer }
1158658628e7SLina Iyer arch_initcall(rpmh_driver_init);
1159cb365926SJohn Stultz
1160cb365926SJohn Stultz MODULE_DESCRIPTION("Qualcomm Technologies, Inc. RPMh Driver");
1161cb365926SJohn Stultz MODULE_LICENSE("GPL v2");
1162