196 tcs_reg_addr(const struct rsc_drv *drv, int reg, int tcs_id)  in tcs_reg_addr()  argument
198 	return drv->tcs_base + drv->regs[RSC_DRV_TCS_OFFSET] * tcs_id + reg;  in tcs_reg_addr()
202 tcs_cmd_addr(const struct rsc_drv *drv, int reg, int tcs_id, int cmd_id)  in tcs_cmd_addr()  argument
204 	return tcs_reg_addr(drv, reg, tcs_id) + drv->regs[RSC_DRV_CMD_OFFSET] * cmd_id;  in tcs_cmd_addr()
207 static u32 read_tcs_cmd(const struct rsc_drv *drv, int reg, int tcs_id,  in read_tcs_cmd()  argument
210 	return readl_relaxed(tcs_cmd_addr(drv, reg, tcs_id, cmd_id));  in read_tcs_cmd()
213 static u32 read_tcs_reg(const struct rsc_drv *drv, int reg, int tcs_id)  in read_tcs_reg()  argument
215 	return readl_relaxed(tcs_reg_addr(drv, reg, tcs_id));  in read_tcs_reg()
218 static void write_tcs_cmd(const struct rsc_drv *drv, int reg, int tcs_id,  in write_tcs_cmd()  argument
221 	writel_relaxed(data, tcs_cmd_addr(drv, reg, tcs_id, cmd_id));  in write_tcs_cmd()
224 static void write_tcs_reg(const struct rsc_drv *drv, int reg, int tcs_id,  in write_tcs_reg()  argument
227 	writel_relaxed(data, tcs_reg_addr(drv, reg, tcs_id));  in write_tcs_reg()
230 static void write_tcs_reg_sync(const struct rsc_drv *drv, int reg, int tcs_id,  in write_tcs_reg_sync()  argument
235 	writel(data, tcs_reg_addr(drv, reg, tcs_id));  in write_tcs_reg_sync()
242 		if (readl(tcs_reg_addr(drv, reg, tcs_id)) == data)  in write_tcs_reg_sync()
246 	pr_err("%s: error writing %#x to %d:%#x\n", drv->name,  in write_tcs_reg_sync()
262 static void tcs_invalidate(struct rsc_drv *drv, int type)  in tcs_invalidate()  argument
265 	struct tcs_group *tcs = &drv->tcs[type];  in tcs_invalidate()
272 		write_tcs_reg_sync(drv, drv->regs[RSC_DRV_CMD_ENABLE], m, 0);  in tcs_invalidate()
285 void rpmh_rsc_invalidate(struct rsc_drv *drv)  in rpmh_rsc_invalidate()  argument
287 	tcs_invalidate(drv, SLEEP_TCS);  in rpmh_rsc_invalidate()
288 	tcs_invalidate(drv, WAKE_TCS);  in rpmh_rsc_invalidate()
301 static struct tcs_group *get_tcs_for_msg(struct rsc_drv *drv,  in get_tcs_for_msg()  argument
328 	tcs = &drv->tcs[type];  in get_tcs_for_msg()
330 		tcs = &drv->tcs[WAKE_TCS];  in get_tcs_for_msg()
351 static const struct tcs_request *get_req_from_tcs(struct rsc_drv *drv,  in get_req_from_tcs()  argument
358 		tcs = &drv->tcs[i];  in get_req_from_tcs()
383 static void __tcs_set_trigger(struct rsc_drv *drv, int tcs_id, bool trigger)  in __tcs_set_trigger()  argument
386 	u32 reg = drv->regs[RSC_DRV_CONTROL];  in __tcs_set_trigger()
393 	enable = read_tcs_reg(drv, reg, tcs_id);  in __tcs_set_trigger()
395 	write_tcs_reg_sync(drv, reg, tcs_id, enable);  in __tcs_set_trigger()
397 	write_tcs_reg_sync(drv, reg, tcs_id, enable);  in __tcs_set_trigger()
402 		write_tcs_reg_sync(drv, reg, tcs_id, enable);  in __tcs_set_trigger()
404 		write_tcs_reg(drv, reg, tcs_id, enable);  in __tcs_set_trigger()
417 static void enable_tcs_irq(struct rsc_drv *drv, int tcs_id, bool enable)  in enable_tcs_irq()  argument
420 	u32 reg = drv->regs[RSC_DRV_IRQ_ENABLE];  in enable_tcs_irq()
422 	data = readl_relaxed(drv->tcs_base + reg);  in enable_tcs_irq()
427 	writel_relaxed(data, drv->tcs_base + reg);  in enable_tcs_irq()
442 	struct rsc_drv *drv = p;  in tcs_tx_done()  local
447 	irq_status = readl_relaxed(drv->tcs_base + drv->regs[RSC_DRV_IRQ_STATUS]);  in tcs_tx_done()
450 		req = get_req_from_tcs(drv, i);  in tcs_tx_done()
454 		trace_rpmh_tx_done(drv, i, req);  in tcs_tx_done()
461 		if (!drv->tcs[ACTIVE_TCS].num_tcs)  in tcs_tx_done()
462 			__tcs_set_trigger(drv, i, false);  in tcs_tx_done()
465 		write_tcs_reg(drv, drv->regs[RSC_DRV_CMD_ENABLE], i, 0);  in tcs_tx_done()
466 		writel_relaxed(BIT(i), drv->tcs_base + drv->regs[RSC_DRV_IRQ_CLEAR]);  in tcs_tx_done()
467 		spin_lock(&drv->lock);  in tcs_tx_done()
468 		clear_bit(i, drv->tcs_in_use);  in tcs_tx_done()
474 		if (!drv->tcs[ACTIVE_TCS].num_tcs)  in tcs_tx_done()
475 			enable_tcs_irq(drv, i, false);  in tcs_tx_done()
476 		spin_unlock(&drv->lock);  in tcs_tx_done()
477 		wake_up(&drv->tcs_wait);  in tcs_tx_done()
495 static void __tcs_buffer_write(struct rsc_drv *drv, int tcs_id, int cmd_id,  in __tcs_buffer_write()  argument
517 		write_tcs_cmd(drv, drv->regs[RSC_DRV_CMD_MSGID], tcs_id, j, msgid);  in __tcs_buffer_write()
518 		write_tcs_cmd(drv, drv->regs[RSC_DRV_CMD_ADDR], tcs_id, j, cmd->addr);  in __tcs_buffer_write()
519 		write_tcs_cmd(drv, drv->regs[RSC_DRV_CMD_DATA], tcs_id, j, cmd->data);  in __tcs_buffer_write()
520 		trace_rpmh_send_msg(drv, tcs_id, msg->state, j, msgid, cmd);  in __tcs_buffer_write()
523 	cmd_enable |= read_tcs_reg(drv, drv->regs[RSC_DRV_CMD_ENABLE], tcs_id);  in __tcs_buffer_write()
524 	write_tcs_reg(drv, drv->regs[RSC_DRV_CMD_ENABLE], tcs_id, cmd_enable);  in __tcs_buffer_write()
547 static int check_for_req_inflight(struct rsc_drv *drv, struct tcs_group *tcs,  in check_for_req_inflight()  argument
555 	for_each_set_bit_from(i, drv->tcs_in_use, tcs->offset + tcs->num_tcs) {  in check_for_req_inflight()
556 		curr_enabled = read_tcs_reg(drv, drv->regs[RSC_DRV_CMD_ENABLE], i);  in check_for_req_inflight()
559 			addr = read_tcs_cmd(drv, drv->regs[RSC_DRV_CMD_ADDR], i, j);  in check_for_req_inflight()
581 	const struct rsc_drv *drv = tcs->drv;  in find_free_tcs()  local
585 	i = find_next_zero_bit(drv->tcs_in_use, max, tcs->offset);  in find_free_tcs()
607 static int claim_tcs_for_req(struct rsc_drv *drv, struct tcs_group *tcs,  in claim_tcs_for_req()  argument
616 	ret = check_for_req_inflight(drv, tcs, msg);  in claim_tcs_for_req()
645 int rpmh_rsc_send_data(struct rsc_drv *drv, const struct tcs_request *msg)  in rpmh_rsc_send_data()  argument
651 	tcs = get_tcs_for_msg(drv, msg);  in rpmh_rsc_send_data()
655 	spin_lock_irqsave(&drv->lock, flags);  in rpmh_rsc_send_data()
658 	wait_event_lock_irq(drv->tcs_wait,  in rpmh_rsc_send_data()
659 			    (tcs_id = claim_tcs_for_req(drv, tcs, msg)) >= 0,  in rpmh_rsc_send_data()
660 			    drv->lock);  in rpmh_rsc_send_data()
663 	set_bit(tcs_id, drv->tcs_in_use);  in rpmh_rsc_send_data()
670 		write_tcs_reg_sync(drv, drv->regs[RSC_DRV_CMD_ENABLE], tcs_id, 0);  in rpmh_rsc_send_data()
671 		enable_tcs_irq(drv, tcs_id, true);  in rpmh_rsc_send_data()
673 	spin_unlock_irqrestore(&drv->lock, flags);  in rpmh_rsc_send_data()
683 	__tcs_buffer_write(drv, tcs_id, 0, msg);  in rpmh_rsc_send_data()
684 	__tcs_set_trigger(drv, tcs_id, true);  in rpmh_rsc_send_data()
741 int rpmh_rsc_write_ctrl_data(struct rsc_drv *drv, const struct tcs_request *msg)  in rpmh_rsc_write_ctrl_data()  argument
747 	tcs = get_tcs_for_msg(drv, msg);  in rpmh_rsc_write_ctrl_data()
754 		__tcs_buffer_write(drv, tcs_id, cmd_id, msg);  in rpmh_rsc_write_ctrl_data()
774 static bool rpmh_rsc_ctrlr_is_busy(struct rsc_drv *drv)  in rpmh_rsc_ctrlr_is_busy()  argument
777 	const struct tcs_group *tcs = &drv->tcs[ACTIVE_TCS];  in rpmh_rsc_ctrlr_is_busy()
787 		tcs = &drv->tcs[WAKE_TCS];  in rpmh_rsc_ctrlr_is_busy()
790 	set = find_next_bit(drv->tcs_in_use, max, tcs->offset);  in rpmh_rsc_ctrlr_is_busy()
802 void rpmh_rsc_write_next_wakeup(struct rsc_drv *drv)  in rpmh_rsc_write_next_wakeup()  argument
808 	if (!drv->tcs[CONTROL_TCS].num_tcs || !drv->genpd_nb.notifier_call)  in rpmh_rsc_write_next_wakeup()
816 	wakeup = dev_pm_genpd_get_next_hrtimer(drv->dev);  in rpmh_rsc_write_next_wakeup()
833 	writel_relaxed(lo, drv->base + RSC_DRV_CTL_TCS_DATA_LO);  in rpmh_rsc_write_next_wakeup()
834 	writel_relaxed(hi, drv->base + RSC_DRV_CTL_TCS_DATA_HI);  in rpmh_rsc_write_next_wakeup()
857 	struct rsc_drv *drv = container_of(nfb, struct rsc_drv, rsc_pm);  in rpmh_rsc_cpu_pm_callback()  local
863 		cpus_in_pm = atomic_inc_return(&drv->cpus_in_pm);  in rpmh_rsc_cpu_pm_callback()
879 		atomic_dec(&drv->cpus_in_pm);  in rpmh_rsc_cpu_pm_callback()
897 	if (spin_trylock(&drv->lock)) {  in rpmh_rsc_cpu_pm_callback()
898 		if (rpmh_rsc_ctrlr_is_busy(drv) || rpmh_flush(&drv->client))  in rpmh_rsc_cpu_pm_callback()
900 		spin_unlock(&drv->lock);  in rpmh_rsc_cpu_pm_callback()
912 			atomic_dec(&drv->cpus_in_pm);  in rpmh_rsc_cpu_pm_callback()
939 	struct rsc_drv *drv = container_of(nfb, struct rsc_drv, genpd_nb);  in rpmh_rsc_pd_callback()  local
943 	    (rpmh_rsc_ctrlr_is_busy(drv) || rpmh_flush(&drv->client)))  in rpmh_rsc_pd_callback()
949 static int rpmh_rsc_pd_attach(struct rsc_drv *drv, struct device *dev)  in rpmh_rsc_pd_attach()  argument
954 	drv->genpd_nb.notifier_call = rpmh_rsc_pd_callback;  in rpmh_rsc_pd_attach()
955 	ret = dev_pm_genpd_add_notifier(dev, &drv->genpd_nb);  in rpmh_rsc_pd_attach()
962 static int rpmh_probe_tcs_config(struct platform_device *pdev, struct rsc_drv *drv)  in rpmh_probe_tcs_config()  argument
976 	drv->tcs_base = drv->base + offset;  in rpmh_probe_tcs_config()
978 	config = readl_relaxed(drv->base + drv->regs[DRV_PRNT_CHLD_CONFIG]);  in rpmh_probe_tcs_config()
981 	max_tcs &= DRV_NUM_TCS_MASK << (DRV_NUM_TCS_SHIFT * drv->id);  in rpmh_probe_tcs_config()
982 	max_tcs = max_tcs >> (DRV_NUM_TCS_SHIFT * drv->id);  in rpmh_probe_tcs_config()
1008 		tcs = &drv->tcs[tcs_cfg[i].type];  in rpmh_probe_tcs_config()
1009 		if (tcs->drv)  in rpmh_probe_tcs_config()
1011 		tcs->drv = drv;  in rpmh_probe_tcs_config()
1028 	drv->num_tcs = st;  in rpmh_probe_tcs_config()
1036 	struct rsc_drv *drv;  in rpmh_rsc_probe()  local
1054 	drv = devm_kzalloc(&pdev->dev, sizeof(*drv), GFP_KERNEL);  in rpmh_rsc_probe()
1055 	if (!drv)  in rpmh_rsc_probe()
1058 	ret = of_property_read_u32(dn, "qcom,drv-id", &drv->id);  in rpmh_rsc_probe()
1062 	drv->name = of_get_property(dn, "label", NULL);  in rpmh_rsc_probe()
1063 	if (!drv->name)  in rpmh_rsc_probe()
1064 		drv->name = dev_name(&pdev->dev);  in rpmh_rsc_probe()
1066 	snprintf(drv_id, ARRAY_SIZE(drv_id), "drv-%d", drv->id);  in rpmh_rsc_probe()
1067 	drv->base = devm_platform_ioremap_resource_byname(pdev, drv_id);  in rpmh_rsc_probe()
1068 	if (IS_ERR(drv->base))  in rpmh_rsc_probe()
1069 		return PTR_ERR(drv->base);  in rpmh_rsc_probe()
1071 	rsc_id = readl_relaxed(drv->base + RSC_DRV_ID);  in rpmh_rsc_probe()
1072 	drv->ver.major = rsc_id & (MAJOR_VER_MASK << MAJOR_VER_SHIFT);  in rpmh_rsc_probe()
1073 	drv->ver.major >>= MAJOR_VER_SHIFT;  in rpmh_rsc_probe()
1074 	drv->ver.minor = rsc_id & (MINOR_VER_MASK << MINOR_VER_SHIFT);  in rpmh_rsc_probe()
1075 	drv->ver.minor >>= MINOR_VER_SHIFT;  in rpmh_rsc_probe()
1077 	if (drv->ver.major == 3)  in rpmh_rsc_probe()
1078 		drv->regs = rpmh_rsc_reg_offset_ver_3_0;  in rpmh_rsc_probe()
1080 		drv->regs = rpmh_rsc_reg_offset_ver_2_7;  in rpmh_rsc_probe()
1082 	ret = rpmh_probe_tcs_config(pdev, drv);  in rpmh_rsc_probe()
1086 	spin_lock_init(&drv->lock);  in rpmh_rsc_probe()
1087 	init_waitqueue_head(&drv->tcs_wait);  in rpmh_rsc_probe()
1088 	bitmap_zero(drv->tcs_in_use, MAX_TCS_NR);  in rpmh_rsc_probe()
1090 	irq = platform_get_irq(pdev, drv->id);  in rpmh_rsc_probe()
1096 			       drv->name, drv);  in rpmh_rsc_probe()
1105 	solver_config = readl_relaxed(drv->base + drv->regs[DRV_SOLVER_CONFIG]);  in rpmh_rsc_probe()
1110 			ret = rpmh_rsc_pd_attach(drv, &pdev->dev);  in rpmh_rsc_probe()
1114 			drv->rsc_pm.notifier_call = rpmh_rsc_cpu_pm_callback;  in rpmh_rsc_probe()
1115 			cpu_pm_register_notifier(&drv->rsc_pm);  in rpmh_rsc_probe()
1120 	writel_relaxed(drv->tcs[ACTIVE_TCS].mask,  in rpmh_rsc_probe()
1121 		       drv->tcs_base + drv->regs[RSC_DRV_IRQ_ENABLE]);  in rpmh_rsc_probe()
1123 	spin_lock_init(&drv->client.cache_lock);  in rpmh_rsc_probe()
1124 	INIT_LIST_HEAD(&drv->client.cache);  in rpmh_rsc_probe()
1125 	INIT_LIST_HEAD(&drv->client.batch_cache);  in rpmh_rsc_probe()
1127 	dev_set_drvdata(&pdev->dev, drv);  in rpmh_rsc_probe()
1128 	drv->dev = &pdev->dev;  in rpmh_rsc_probe()