1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (c) 2016-2018, The Linux Foundation. All rights reserved. 4 */ 5 6 #define pr_fmt(fmt) "%s " fmt, KBUILD_MODNAME 7 8 #include <linux/atomic.h> 9 #include <linux/cpu_pm.h> 10 #include <linux/delay.h> 11 #include <linux/interrupt.h> 12 #include <linux/io.h> 13 #include <linux/iopoll.h> 14 #include <linux/kernel.h> 15 #include <linux/ktime.h> 16 #include <linux/list.h> 17 #include <linux/module.h> 18 #include <linux/notifier.h> 19 #include <linux/of.h> 20 #include <linux/of_irq.h> 21 #include <linux/of_platform.h> 22 #include <linux/platform_device.h> 23 #include <linux/pm_domain.h> 24 #include <linux/pm_runtime.h> 25 #include <linux/slab.h> 26 #include <linux/spinlock.h> 27 #include <linux/wait.h> 28 29 #include <clocksource/arm_arch_timer.h> 30 #include <soc/qcom/cmd-db.h> 31 #include <soc/qcom/tcs.h> 32 #include <dt-bindings/soc/qcom,rpmh-rsc.h> 33 34 #include "rpmh-internal.h" 35 36 #define CREATE_TRACE_POINTS 37 #include "trace-rpmh.h" 38 39 40 #define RSC_DRV_ID 0 41 42 #define MAJOR_VER_MASK 0xFF 43 #define MAJOR_VER_SHIFT 16 44 #define MINOR_VER_MASK 0xFF 45 #define MINOR_VER_SHIFT 8 46 47 enum { 48 RSC_DRV_TCS_OFFSET, 49 RSC_DRV_CMD_OFFSET, 50 DRV_SOLVER_CONFIG, 51 DRV_PRNT_CHLD_CONFIG, 52 RSC_DRV_IRQ_ENABLE, 53 RSC_DRV_IRQ_STATUS, 54 RSC_DRV_IRQ_CLEAR, 55 RSC_DRV_CMD_WAIT_FOR_CMPL, 56 RSC_DRV_CONTROL, 57 RSC_DRV_STATUS, 58 RSC_DRV_CMD_ENABLE, 59 RSC_DRV_CMD_MSGID, 60 RSC_DRV_CMD_ADDR, 61 RSC_DRV_CMD_DATA, 62 RSC_DRV_CMD_STATUS, 63 RSC_DRV_CMD_RESP_DATA, 64 }; 65 66 /* DRV HW Solver Configuration Information Register */ 67 #define DRV_HW_SOLVER_MASK 1 68 #define DRV_HW_SOLVER_SHIFT 24 69 70 /* DRV TCS Configuration Information Register */ 71 #define DRV_NUM_TCS_MASK 0x3F 72 #define DRV_NUM_TCS_SHIFT 6 73 #define DRV_NCPT_MASK 0x1F 74 #define DRV_NCPT_SHIFT 27 75 76 /* Offsets for CONTROL TCS Registers */ 77 #define RSC_DRV_CTL_TCS_DATA_HI 0x38 78 #define RSC_DRV_CTL_TCS_DATA_HI_MASK 0xFFFFFF 79 #define RSC_DRV_CTL_TCS_DATA_HI_VALID BIT(31) 80 #define RSC_DRV_CTL_TCS_DATA_LO 0x40 81 #define RSC_DRV_CTL_TCS_DATA_LO_MASK 0xFFFFFFFF 82 #define RSC_DRV_CTL_TCS_DATA_SIZE 32 83 84 #define TCS_AMC_MODE_ENABLE BIT(16) 85 #define TCS_AMC_MODE_TRIGGER BIT(24) 86 87 /* TCS CMD register bit mask */ 88 #define CMD_MSGID_LEN 8 89 #define CMD_MSGID_RESP_REQ BIT(8) 90 #define CMD_MSGID_WRITE BIT(16) 91 #define CMD_STATUS_ISSUED BIT(8) 92 #define CMD_STATUS_COMPL BIT(16) 93 94 /* 95 * Here's a high level overview of how all the registers in RPMH work 96 * together: 97 * 98 * - The main rpmh-rsc address is the base of a register space that can 99 * be used to find overall configuration of the hardware 100 * (DRV_PRNT_CHLD_CONFIG). Also found within the rpmh-rsc register 101 * space are all the TCS blocks. The offset of the TCS blocks is 102 * specified in the device tree by "qcom,tcs-offset" and used to 103 * compute tcs_base. 104 * - TCS blocks come one after another. Type, count, and order are 105 * specified by the device tree as "qcom,tcs-config". 106 * - Each TCS block has some registers, then space for up to 16 commands. 107 * Note that though address space is reserved for 16 commands, fewer 108 * might be present. See ncpt (num cmds per TCS). 109 * 110 * Here's a picture: 111 * 112 * +---------------------------------------------------+ 113 * |RSC | 114 * | ctrl | 115 * | | 116 * | Drvs: | 117 * | +-----------------------------------------------+ | 118 * | |DRV0 | | 119 * | | ctrl/config | | 120 * | | IRQ | | 121 * | | | | 122 * | | TCSes: | | 123 * | | +------------------------------------------+ | | 124 * | | |TCS0 | | | | | | | | | | | | | | | 125 * | | | ctrl | 0| 1| 2| 3| 4| 5| .| .| .| .|14|15| | | 126 * | | | | | | | | | | | | | | | | | | 127 * | | +------------------------------------------+ | | 128 * | | +------------------------------------------+ | | 129 * | | |TCS1 | | | | | | | | | | | | | | | 130 * | | | ctrl | 0| 1| 2| 3| 4| 5| .| .| .| .|14|15| | | 131 * | | | | | | | | | | | | | | | | | | 132 * | | +------------------------------------------+ | | 133 * | | +------------------------------------------+ | | 134 * | | |TCS2 | | | | | | | | | | | | | | | 135 * | | | ctrl | 0| 1| 2| 3| 4| 5| .| .| .| .|14|15| | | 136 * | | | | | | | | | | | | | | | | | | 137 * | | +------------------------------------------+ | | 138 * | | ...... | | 139 * | +-----------------------------------------------+ | 140 * | +-----------------------------------------------+ | 141 * | |DRV1 | | 142 * | | (same as DRV0) | | 143 * | +-----------------------------------------------+ | 144 * | ...... | 145 * +---------------------------------------------------+ 146 */ 147 148 #define USECS_TO_CYCLES(time_usecs) \ 149 xloops_to_cycles((time_usecs) * 0x10C7UL) 150 151 static inline unsigned long xloops_to_cycles(u64 xloops) 152 { 153 return (xloops * loops_per_jiffy * HZ) >> 32; 154 } 155 156 static u32 rpmh_rsc_reg_offset_ver_2_7[] = { 157 [RSC_DRV_TCS_OFFSET] = 672, 158 [RSC_DRV_CMD_OFFSET] = 20, 159 [DRV_SOLVER_CONFIG] = 0x04, 160 [DRV_PRNT_CHLD_CONFIG] = 0x0C, 161 [RSC_DRV_IRQ_ENABLE] = 0x00, 162 [RSC_DRV_IRQ_STATUS] = 0x04, 163 [RSC_DRV_IRQ_CLEAR] = 0x08, 164 [RSC_DRV_CMD_WAIT_FOR_CMPL] = 0x10, 165 [RSC_DRV_CONTROL] = 0x14, 166 [RSC_DRV_STATUS] = 0x18, 167 [RSC_DRV_CMD_ENABLE] = 0x1C, 168 [RSC_DRV_CMD_MSGID] = 0x30, 169 [RSC_DRV_CMD_ADDR] = 0x34, 170 [RSC_DRV_CMD_DATA] = 0x38, 171 [RSC_DRV_CMD_STATUS] = 0x3C, 172 [RSC_DRV_CMD_RESP_DATA] = 0x40, 173 }; 174 175 static u32 rpmh_rsc_reg_offset_ver_3_0[] = { 176 [RSC_DRV_TCS_OFFSET] = 672, 177 [RSC_DRV_CMD_OFFSET] = 24, 178 [DRV_SOLVER_CONFIG] = 0x04, 179 [DRV_PRNT_CHLD_CONFIG] = 0x0C, 180 [RSC_DRV_IRQ_ENABLE] = 0x00, 181 [RSC_DRV_IRQ_STATUS] = 0x04, 182 [RSC_DRV_IRQ_CLEAR] = 0x08, 183 [RSC_DRV_CMD_WAIT_FOR_CMPL] = 0x20, 184 [RSC_DRV_CONTROL] = 0x24, 185 [RSC_DRV_STATUS] = 0x28, 186 [RSC_DRV_CMD_ENABLE] = 0x2C, 187 [RSC_DRV_CMD_MSGID] = 0x34, 188 [RSC_DRV_CMD_ADDR] = 0x38, 189 [RSC_DRV_CMD_DATA] = 0x3C, 190 [RSC_DRV_CMD_STATUS] = 0x40, 191 [RSC_DRV_CMD_RESP_DATA] = 0x44, 192 }; 193 194 static inline void __iomem * 195 tcs_reg_addr(const struct rsc_drv *drv, int reg, int tcs_id) 196 { 197 return drv->tcs_base + drv->regs[RSC_DRV_TCS_OFFSET] * tcs_id + reg; 198 } 199 200 static inline void __iomem * 201 tcs_cmd_addr(const struct rsc_drv *drv, int reg, int tcs_id, int cmd_id) 202 { 203 return tcs_reg_addr(drv, reg, tcs_id) + drv->regs[RSC_DRV_CMD_OFFSET] * cmd_id; 204 } 205 206 static u32 read_tcs_cmd(const struct rsc_drv *drv, int reg, int tcs_id, 207 int cmd_id) 208 { 209 return readl_relaxed(tcs_cmd_addr(drv, reg, tcs_id, cmd_id)); 210 } 211 212 static u32 read_tcs_reg(const struct rsc_drv *drv, int reg, int tcs_id) 213 { 214 return readl_relaxed(tcs_reg_addr(drv, reg, tcs_id)); 215 } 216 217 static void write_tcs_cmd(const struct rsc_drv *drv, int reg, int tcs_id, 218 int cmd_id, u32 data) 219 { 220 writel_relaxed(data, tcs_cmd_addr(drv, reg, tcs_id, cmd_id)); 221 } 222 223 static void write_tcs_reg(const struct rsc_drv *drv, int reg, int tcs_id, 224 u32 data) 225 { 226 writel_relaxed(data, tcs_reg_addr(drv, reg, tcs_id)); 227 } 228 229 static void write_tcs_reg_sync(const struct rsc_drv *drv, int reg, int tcs_id, 230 u32 data) 231 { 232 int i; 233 234 writel(data, tcs_reg_addr(drv, reg, tcs_id)); 235 236 /* 237 * Wait until we read back the same value. Use a counter rather than 238 * ktime for timeout since this may be called after timekeeping stops. 239 */ 240 for (i = 0; i < USEC_PER_SEC; i++) { 241 if (readl(tcs_reg_addr(drv, reg, tcs_id)) == data) 242 return; 243 udelay(1); 244 } 245 pr_err("%s: error writing %#x to %d:%#x\n", drv->name, 246 data, tcs_id, reg); 247 } 248 249 /** 250 * tcs_invalidate() - Invalidate all TCSes of the given type (sleep or wake). 251 * @drv: The RSC controller. 252 * @type: SLEEP_TCS or WAKE_TCS 253 * 254 * This will clear the "slots" variable of the given tcs_group and also 255 * tell the hardware to forget about all entries. 256 * 257 * The caller must ensure that no other RPMH actions are happening when this 258 * function is called, since otherwise the device may immediately become 259 * used again even before this function exits. 260 */ 261 static void tcs_invalidate(struct rsc_drv *drv, int type) 262 { 263 int m; 264 struct tcs_group *tcs = &drv->tcs[type]; 265 266 /* Caller ensures nobody else is running so no lock */ 267 if (bitmap_empty(tcs->slots, MAX_TCS_SLOTS)) 268 return; 269 270 for (m = tcs->offset; m < tcs->offset + tcs->num_tcs; m++) 271 write_tcs_reg_sync(drv, drv->regs[RSC_DRV_CMD_ENABLE], m, 0); 272 273 bitmap_zero(tcs->slots, MAX_TCS_SLOTS); 274 } 275 276 /** 277 * rpmh_rsc_invalidate() - Invalidate sleep and wake TCSes. 278 * @drv: The RSC controller. 279 * 280 * The caller must ensure that no other RPMH actions are happening when this 281 * function is called, since otherwise the device may immediately become 282 * used again even before this function exits. 283 */ 284 void rpmh_rsc_invalidate(struct rsc_drv *drv) 285 { 286 tcs_invalidate(drv, SLEEP_TCS); 287 tcs_invalidate(drv, WAKE_TCS); 288 } 289 290 /** 291 * get_tcs_for_msg() - Get the tcs_group used to send the given message. 292 * @drv: The RSC controller. 293 * @msg: The message we want to send. 294 * 295 * This is normally pretty straightforward except if we are trying to send 296 * an ACTIVE_ONLY message but don't have any active_only TCSes. 297 * 298 * Return: A pointer to a tcs_group or an ERR_PTR. 299 */ 300 static struct tcs_group *get_tcs_for_msg(struct rsc_drv *drv, 301 const struct tcs_request *msg) 302 { 303 int type; 304 struct tcs_group *tcs; 305 306 switch (msg->state) { 307 case RPMH_ACTIVE_ONLY_STATE: 308 type = ACTIVE_TCS; 309 break; 310 case RPMH_WAKE_ONLY_STATE: 311 type = WAKE_TCS; 312 break; 313 case RPMH_SLEEP_STATE: 314 type = SLEEP_TCS; 315 break; 316 default: 317 return ERR_PTR(-EINVAL); 318 } 319 320 /* 321 * If we are making an active request on a RSC that does not have a 322 * dedicated TCS for active state use, then re-purpose a wake TCS to 323 * send active votes. This is safe because we ensure any active-only 324 * transfers have finished before we use it (maybe by running from 325 * the last CPU in PM code). 326 */ 327 tcs = &drv->tcs[type]; 328 if (msg->state == RPMH_ACTIVE_ONLY_STATE && !tcs->num_tcs) 329 tcs = &drv->tcs[WAKE_TCS]; 330 331 return tcs; 332 } 333 334 /** 335 * get_req_from_tcs() - Get a stashed request that was xfering on the given TCS. 336 * @drv: The RSC controller. 337 * @tcs_id: The global ID of this TCS. 338 * 339 * For ACTIVE_ONLY transfers we want to call back into the client when the 340 * transfer finishes. To do this we need the "request" that the client 341 * originally provided us. This function grabs the request that we stashed 342 * when we started the transfer. 343 * 344 * This only makes sense for ACTIVE_ONLY transfers since those are the only 345 * ones we track sending (the only ones we enable interrupts for and the only 346 * ones we call back to the client for). 347 * 348 * Return: The stashed request. 349 */ 350 static const struct tcs_request *get_req_from_tcs(struct rsc_drv *drv, 351 int tcs_id) 352 { 353 struct tcs_group *tcs; 354 int i; 355 356 for (i = 0; i < TCS_TYPE_NR; i++) { 357 tcs = &drv->tcs[i]; 358 if (tcs->mask & BIT(tcs_id)) 359 return tcs->req[tcs_id - tcs->offset]; 360 } 361 362 return NULL; 363 } 364 365 /** 366 * __tcs_set_trigger() - Start xfer on a TCS or unset trigger on a borrowed TCS 367 * @drv: The controller. 368 * @tcs_id: The global ID of this TCS. 369 * @trigger: If true then untrigger/retrigger. If false then just untrigger. 370 * 371 * In the normal case we only ever call with "trigger=true" to start a 372 * transfer. That will un-trigger/disable the TCS from the last transfer 373 * then trigger/enable for this transfer. 374 * 375 * If we borrowed a wake TCS for an active-only transfer we'll also call 376 * this function with "trigger=false" to just do the un-trigger/disable 377 * before using the TCS for wake purposes again. 378 * 379 * Note that the AP is only in charge of triggering active-only transfers. 380 * The AP never triggers sleep/wake values using this function. 381 */ 382 static void __tcs_set_trigger(struct rsc_drv *drv, int tcs_id, bool trigger) 383 { 384 u32 enable; 385 u32 reg = drv->regs[RSC_DRV_CONTROL]; 386 387 /* 388 * HW req: Clear the DRV_CONTROL and enable TCS again 389 * While clearing ensure that the AMC mode trigger is cleared 390 * and then the mode enable is cleared. 391 */ 392 enable = read_tcs_reg(drv, reg, tcs_id); 393 enable &= ~TCS_AMC_MODE_TRIGGER; 394 write_tcs_reg_sync(drv, reg, tcs_id, enable); 395 enable &= ~TCS_AMC_MODE_ENABLE; 396 write_tcs_reg_sync(drv, reg, tcs_id, enable); 397 398 if (trigger) { 399 /* Enable the AMC mode on the TCS and then trigger the TCS */ 400 enable = TCS_AMC_MODE_ENABLE; 401 write_tcs_reg_sync(drv, reg, tcs_id, enable); 402 enable |= TCS_AMC_MODE_TRIGGER; 403 write_tcs_reg(drv, reg, tcs_id, enable); 404 } 405 } 406 407 /** 408 * enable_tcs_irq() - Enable or disable interrupts on the given TCS. 409 * @drv: The controller. 410 * @tcs_id: The global ID of this TCS. 411 * @enable: If true then enable; if false then disable 412 * 413 * We only ever call this when we borrow a wake TCS for an active-only 414 * transfer. For active-only TCSes interrupts are always left enabled. 415 */ 416 static void enable_tcs_irq(struct rsc_drv *drv, int tcs_id, bool enable) 417 { 418 u32 data; 419 u32 reg = drv->regs[RSC_DRV_IRQ_ENABLE]; 420 421 data = readl_relaxed(drv->tcs_base + reg); 422 if (enable) 423 data |= BIT(tcs_id); 424 else 425 data &= ~BIT(tcs_id); 426 writel_relaxed(data, drv->tcs_base + reg); 427 } 428 429 /** 430 * tcs_tx_done() - TX Done interrupt handler. 431 * @irq: The IRQ number (ignored). 432 * @p: Pointer to "struct rsc_drv". 433 * 434 * Called for ACTIVE_ONLY transfers (those are the only ones we enable the 435 * IRQ for) when a transfer is done. 436 * 437 * Return: IRQ_HANDLED 438 */ 439 static irqreturn_t tcs_tx_done(int irq, void *p) 440 { 441 struct rsc_drv *drv = p; 442 int i; 443 unsigned long irq_status; 444 const struct tcs_request *req; 445 446 irq_status = readl_relaxed(drv->tcs_base + drv->regs[RSC_DRV_IRQ_STATUS]); 447 448 for_each_set_bit(i, &irq_status, BITS_PER_TYPE(u32)) { 449 req = get_req_from_tcs(drv, i); 450 if (WARN_ON(!req)) 451 goto skip; 452 453 trace_rpmh_tx_done(drv, i, req); 454 455 /* 456 * If wake tcs was re-purposed for sending active 457 * votes, clear AMC trigger & enable modes and 458 * disable interrupt for this TCS 459 */ 460 if (!drv->tcs[ACTIVE_TCS].num_tcs) 461 __tcs_set_trigger(drv, i, false); 462 skip: 463 /* Reclaim the TCS */ 464 write_tcs_reg(drv, drv->regs[RSC_DRV_CMD_ENABLE], i, 0); 465 writel_relaxed(BIT(i), drv->tcs_base + drv->regs[RSC_DRV_IRQ_CLEAR]); 466 spin_lock(&drv->lock); 467 clear_bit(i, drv->tcs_in_use); 468 /* 469 * Disable interrupt for WAKE TCS to avoid being 470 * spammed with interrupts coming when the solver 471 * sends its wake votes. 472 */ 473 if (!drv->tcs[ACTIVE_TCS].num_tcs) 474 enable_tcs_irq(drv, i, false); 475 spin_unlock(&drv->lock); 476 wake_up(&drv->tcs_wait); 477 if (req) 478 rpmh_tx_done(req); 479 } 480 481 return IRQ_HANDLED; 482 } 483 484 /** 485 * __tcs_buffer_write() - Write to TCS hardware from a request; don't trigger. 486 * @drv: The controller. 487 * @tcs_id: The global ID of this TCS. 488 * @cmd_id: The index within the TCS to start writing. 489 * @msg: The message we want to send, which will contain several addr/data 490 * pairs to program (but few enough that they all fit in one TCS). 491 * 492 * This is used for all types of transfers (active, sleep, and wake). 493 */ 494 static void __tcs_buffer_write(struct rsc_drv *drv, int tcs_id, int cmd_id, 495 const struct tcs_request *msg) 496 { 497 u32 msgid; 498 u32 cmd_msgid = CMD_MSGID_LEN | CMD_MSGID_WRITE; 499 u32 cmd_enable = 0; 500 struct tcs_cmd *cmd; 501 int i, j; 502 503 /* Convert all commands to RR when the request has wait_for_compl set */ 504 cmd_msgid |= msg->wait_for_compl ? CMD_MSGID_RESP_REQ : 0; 505 506 for (i = 0, j = cmd_id; i < msg->num_cmds; i++, j++) { 507 cmd = &msg->cmds[i]; 508 cmd_enable |= BIT(j); 509 msgid = cmd_msgid; 510 /* 511 * Additionally, if the cmd->wait is set, make the command 512 * response reqd even if the overall request was fire-n-forget. 513 */ 514 msgid |= cmd->wait ? CMD_MSGID_RESP_REQ : 0; 515 516 write_tcs_cmd(drv, drv->regs[RSC_DRV_CMD_MSGID], tcs_id, j, msgid); 517 write_tcs_cmd(drv, drv->regs[RSC_DRV_CMD_ADDR], tcs_id, j, cmd->addr); 518 write_tcs_cmd(drv, drv->regs[RSC_DRV_CMD_DATA], tcs_id, j, cmd->data); 519 trace_rpmh_send_msg(drv, tcs_id, msg->state, j, msgid, cmd); 520 } 521 522 cmd_enable |= read_tcs_reg(drv, drv->regs[RSC_DRV_CMD_ENABLE], tcs_id); 523 write_tcs_reg(drv, drv->regs[RSC_DRV_CMD_ENABLE], tcs_id, cmd_enable); 524 } 525 526 /** 527 * check_for_req_inflight() - Look to see if conflicting cmds are in flight. 528 * @drv: The controller. 529 * @tcs: A pointer to the tcs_group used for ACTIVE_ONLY transfers. 530 * @msg: The message we want to send, which will contain several addr/data 531 * pairs to program (but few enough that they all fit in one TCS). 532 * 533 * This will walk through the TCSes in the group and check if any of them 534 * appear to be sending to addresses referenced in the message. If it finds 535 * one it'll return -EBUSY. 536 * 537 * Only for use for active-only transfers. 538 * 539 * Must be called with the drv->lock held since that protects tcs_in_use. 540 * 541 * Return: 0 if nothing in flight or -EBUSY if we should try again later. 542 * The caller must re-enable interrupts between tries since that's 543 * the only way tcs_in_use will ever be updated and the only way 544 * RSC_DRV_CMD_ENABLE will ever be cleared. 545 */ 546 static int check_for_req_inflight(struct rsc_drv *drv, struct tcs_group *tcs, 547 const struct tcs_request *msg) 548 { 549 unsigned long curr_enabled; 550 u32 addr; 551 int j, k; 552 int i = tcs->offset; 553 554 for_each_set_bit_from(i, drv->tcs_in_use, tcs->offset + tcs->num_tcs) { 555 curr_enabled = read_tcs_reg(drv, drv->regs[RSC_DRV_CMD_ENABLE], i); 556 557 for_each_set_bit(j, &curr_enabled, MAX_CMDS_PER_TCS) { 558 addr = read_tcs_cmd(drv, drv->regs[RSC_DRV_CMD_ADDR], i, j); 559 for (k = 0; k < msg->num_cmds; k++) { 560 if (addr == msg->cmds[k].addr) 561 return -EBUSY; 562 } 563 } 564 } 565 566 return 0; 567 } 568 569 /** 570 * find_free_tcs() - Find free tcs in the given tcs_group; only for active. 571 * @tcs: A pointer to the active-only tcs_group (or the wake tcs_group if 572 * we borrowed it because there are zero active-only ones). 573 * 574 * Must be called with the drv->lock held since that protects tcs_in_use. 575 * 576 * Return: The first tcs that's free or -EBUSY if all in use. 577 */ 578 static int find_free_tcs(struct tcs_group *tcs) 579 { 580 const struct rsc_drv *drv = tcs->drv; 581 unsigned long i; 582 unsigned long max = tcs->offset + tcs->num_tcs; 583 584 i = find_next_zero_bit(drv->tcs_in_use, max, tcs->offset); 585 if (i >= max) 586 return -EBUSY; 587 588 return i; 589 } 590 591 /** 592 * claim_tcs_for_req() - Claim a tcs in the given tcs_group; only for active. 593 * @drv: The controller. 594 * @tcs: The tcs_group used for ACTIVE_ONLY transfers. 595 * @msg: The data to be sent. 596 * 597 * Claims a tcs in the given tcs_group while making sure that no existing cmd 598 * is in flight that would conflict with the one in @msg. 599 * 600 * Context: Must be called with the drv->lock held since that protects 601 * tcs_in_use. 602 * 603 * Return: The id of the claimed tcs or -EBUSY if a matching msg is in flight 604 * or the tcs_group is full. 605 */ 606 static int claim_tcs_for_req(struct rsc_drv *drv, struct tcs_group *tcs, 607 const struct tcs_request *msg) 608 { 609 int ret; 610 611 /* 612 * The h/w does not like if we send a request to the same address, 613 * when one is already in-flight or being processed. 614 */ 615 ret = check_for_req_inflight(drv, tcs, msg); 616 if (ret) 617 return ret; 618 619 return find_free_tcs(tcs); 620 } 621 622 /** 623 * rpmh_rsc_send_data() - Write / trigger active-only message. 624 * @drv: The controller. 625 * @msg: The data to be sent. 626 * 627 * NOTES: 628 * - This is only used for "ACTIVE_ONLY" since the limitations of this 629 * function don't make sense for sleep/wake cases. 630 * - To do the transfer, we will grab a whole TCS for ourselves--we don't 631 * try to share. If there are none available we'll wait indefinitely 632 * for a free one. 633 * - This function will not wait for the commands to be finished, only for 634 * data to be programmed into the RPMh. See rpmh_tx_done() which will 635 * be called when the transfer is fully complete. 636 * - This function must be called with interrupts enabled. If the hardware 637 * is busy doing someone else's transfer we need that transfer to fully 638 * finish so that we can have the hardware, and to fully finish it needs 639 * the interrupt handler to run. If the interrupts is set to run on the 640 * active CPU this can never happen if interrupts are disabled. 641 * 642 * Return: 0 on success, -EINVAL on error. 643 */ 644 int rpmh_rsc_send_data(struct rsc_drv *drv, const struct tcs_request *msg) 645 { 646 struct tcs_group *tcs; 647 int tcs_id; 648 unsigned long flags; 649 650 tcs = get_tcs_for_msg(drv, msg); 651 if (IS_ERR(tcs)) 652 return PTR_ERR(tcs); 653 654 spin_lock_irqsave(&drv->lock, flags); 655 656 /* Wait forever for a free tcs. It better be there eventually! */ 657 wait_event_lock_irq(drv->tcs_wait, 658 (tcs_id = claim_tcs_for_req(drv, tcs, msg)) >= 0, 659 drv->lock); 660 661 tcs->req[tcs_id - tcs->offset] = msg; 662 set_bit(tcs_id, drv->tcs_in_use); 663 if (msg->state == RPMH_ACTIVE_ONLY_STATE && tcs->type != ACTIVE_TCS) { 664 /* 665 * Clear previously programmed WAKE commands in selected 666 * repurposed TCS to avoid triggering them. tcs->slots will be 667 * cleaned from rpmh_flush() by invoking rpmh_rsc_invalidate() 668 */ 669 write_tcs_reg_sync(drv, drv->regs[RSC_DRV_CMD_ENABLE], tcs_id, 0); 670 enable_tcs_irq(drv, tcs_id, true); 671 } 672 spin_unlock_irqrestore(&drv->lock, flags); 673 674 /* 675 * These two can be done after the lock is released because: 676 * - We marked "tcs_in_use" under lock. 677 * - Once "tcs_in_use" has been marked nobody else could be writing 678 * to these registers until the interrupt goes off. 679 * - The interrupt can't go off until we trigger w/ the last line 680 * of __tcs_set_trigger() below. 681 */ 682 __tcs_buffer_write(drv, tcs_id, 0, msg); 683 __tcs_set_trigger(drv, tcs_id, true); 684 685 return 0; 686 } 687 688 /** 689 * find_slots() - Find a place to write the given message. 690 * @tcs: The tcs group to search. 691 * @msg: The message we want to find room for. 692 * @tcs_id: If we return 0 from the function, we return the global ID of the 693 * TCS to write to here. 694 * @cmd_id: If we return 0 from the function, we return the index of 695 * the command array of the returned TCS where the client should 696 * start writing the message. 697 * 698 * Only for use on sleep/wake TCSes since those are the only ones we maintain 699 * tcs->slots for. 700 * 701 * Return: -ENOMEM if there was no room, else 0. 702 */ 703 static int find_slots(struct tcs_group *tcs, const struct tcs_request *msg, 704 int *tcs_id, int *cmd_id) 705 { 706 int slot, offset; 707 int i = 0; 708 709 /* Do over, until we can fit the full payload in a single TCS */ 710 do { 711 slot = bitmap_find_next_zero_area(tcs->slots, MAX_TCS_SLOTS, 712 i, msg->num_cmds, 0); 713 if (slot >= tcs->num_tcs * tcs->ncpt) 714 return -ENOMEM; 715 i += tcs->ncpt; 716 } while (slot + msg->num_cmds - 1 >= i); 717 718 bitmap_set(tcs->slots, slot, msg->num_cmds); 719 720 offset = slot / tcs->ncpt; 721 *tcs_id = offset + tcs->offset; 722 *cmd_id = slot % tcs->ncpt; 723 724 return 0; 725 } 726 727 /** 728 * rpmh_rsc_write_ctrl_data() - Write request to controller but don't trigger. 729 * @drv: The controller. 730 * @msg: The data to be written to the controller. 731 * 732 * This should only be called for sleep/wake state, never active-only 733 * state. 734 * 735 * The caller must ensure that no other RPMH actions are happening and the 736 * controller is idle when this function is called since it runs lockless. 737 * 738 * Return: 0 if no error; else -error. 739 */ 740 int rpmh_rsc_write_ctrl_data(struct rsc_drv *drv, const struct tcs_request *msg) 741 { 742 struct tcs_group *tcs; 743 int tcs_id = 0, cmd_id = 0; 744 int ret; 745 746 tcs = get_tcs_for_msg(drv, msg); 747 if (IS_ERR(tcs)) 748 return PTR_ERR(tcs); 749 750 /* find the TCS id and the command in the TCS to write to */ 751 ret = find_slots(tcs, msg, &tcs_id, &cmd_id); 752 if (!ret) 753 __tcs_buffer_write(drv, tcs_id, cmd_id, msg); 754 755 return ret; 756 } 757 758 /** 759 * rpmh_rsc_ctrlr_is_busy() - Check if any of the AMCs are busy. 760 * @drv: The controller 761 * 762 * Checks if any of the AMCs are busy in handling ACTIVE sets. 763 * This is called from the last cpu powering down before flushing 764 * SLEEP and WAKE sets. If AMCs are busy, controller can not enter 765 * power collapse, so deny from the last cpu's pm notification. 766 * 767 * Context: Must be called with the drv->lock held. 768 * 769 * Return: 770 * * False - AMCs are idle 771 * * True - AMCs are busy 772 */ 773 static bool rpmh_rsc_ctrlr_is_busy(struct rsc_drv *drv) 774 { 775 unsigned long set; 776 const struct tcs_group *tcs = &drv->tcs[ACTIVE_TCS]; 777 unsigned long max; 778 779 /* 780 * If we made an active request on a RSC that does not have a 781 * dedicated TCS for active state use, then re-purposed wake TCSes 782 * should be checked for not busy, because we used wake TCSes for 783 * active requests in this case. 784 */ 785 if (!tcs->num_tcs) 786 tcs = &drv->tcs[WAKE_TCS]; 787 788 max = tcs->offset + tcs->num_tcs; 789 set = find_next_bit(drv->tcs_in_use, max, tcs->offset); 790 791 return set < max; 792 } 793 794 /** 795 * rpmh_rsc_write_next_wakeup() - Write next wakeup in CONTROL_TCS. 796 * @drv: The controller 797 * 798 * Writes maximum wakeup cycles when called from suspend. 799 * Writes earliest hrtimer wakeup when called from idle. 800 */ 801 void rpmh_rsc_write_next_wakeup(struct rsc_drv *drv) 802 { 803 ktime_t now, wakeup; 804 u64 wakeup_us, wakeup_cycles = ~0; 805 u32 lo, hi; 806 807 if (!drv->tcs[CONTROL_TCS].num_tcs || !drv->genpd_nb.notifier_call) 808 return; 809 810 /* Set highest time when system (timekeeping) is suspended */ 811 if (system_state == SYSTEM_SUSPEND) 812 goto exit; 813 814 /* Find the earliest hrtimer wakeup from online cpus */ 815 wakeup = dev_pm_genpd_get_next_hrtimer(drv->dev); 816 817 /* Find the relative wakeup in kernel time scale */ 818 now = ktime_get(); 819 wakeup = ktime_sub(wakeup, now); 820 wakeup_us = ktime_to_us(wakeup); 821 822 /* Convert the wakeup to arch timer scale */ 823 wakeup_cycles = USECS_TO_CYCLES(wakeup_us); 824 wakeup_cycles += arch_timer_read_counter(); 825 826 exit: 827 lo = wakeup_cycles & RSC_DRV_CTL_TCS_DATA_LO_MASK; 828 hi = wakeup_cycles >> RSC_DRV_CTL_TCS_DATA_SIZE; 829 hi &= RSC_DRV_CTL_TCS_DATA_HI_MASK; 830 hi |= RSC_DRV_CTL_TCS_DATA_HI_VALID; 831 832 writel_relaxed(lo, drv->base + RSC_DRV_CTL_TCS_DATA_LO); 833 writel_relaxed(hi, drv->base + RSC_DRV_CTL_TCS_DATA_HI); 834 } 835 836 /** 837 * rpmh_rsc_cpu_pm_callback() - Check if any of the AMCs are busy. 838 * @nfb: Pointer to the notifier block in struct rsc_drv. 839 * @action: CPU_PM_ENTER, CPU_PM_ENTER_FAILED, or CPU_PM_EXIT. 840 * @v: Unused 841 * 842 * This function is given to cpu_pm_register_notifier so we can be informed 843 * about when CPUs go down. When all CPUs go down we know no more active 844 * transfers will be started so we write sleep/wake sets. This function gets 845 * called from cpuidle code paths and also at system suspend time. 846 * 847 * If its last CPU going down and AMCs are not busy then writes cached sleep 848 * and wake messages to TCSes. The firmware then takes care of triggering 849 * them when entering deepest low power modes. 850 * 851 * Return: See cpu_pm_register_notifier() 852 */ 853 static int rpmh_rsc_cpu_pm_callback(struct notifier_block *nfb, 854 unsigned long action, void *v) 855 { 856 struct rsc_drv *drv = container_of(nfb, struct rsc_drv, rsc_pm); 857 int ret = NOTIFY_OK; 858 int cpus_in_pm; 859 860 switch (action) { 861 case CPU_PM_ENTER: 862 cpus_in_pm = atomic_inc_return(&drv->cpus_in_pm); 863 /* 864 * NOTE: comments for num_online_cpus() point out that it's 865 * only a snapshot so we need to be careful. It should be OK 866 * for us to use, though. It's important for us not to miss 867 * if we're the last CPU going down so it would only be a 868 * problem if a CPU went offline right after we did the check 869 * AND that CPU was not idle AND that CPU was the last non-idle 870 * CPU. That can't happen. CPUs would have to come out of idle 871 * before the CPU could go offline. 872 */ 873 if (cpus_in_pm < num_online_cpus()) 874 return NOTIFY_OK; 875 break; 876 case CPU_PM_ENTER_FAILED: 877 case CPU_PM_EXIT: 878 atomic_dec(&drv->cpus_in_pm); 879 return NOTIFY_OK; 880 default: 881 return NOTIFY_DONE; 882 } 883 884 /* 885 * It's likely we're on the last CPU. Grab the drv->lock and write 886 * out the sleep/wake commands to RPMH hardware. Grabbing the lock 887 * means that if we race with another CPU coming up we are still 888 * guaranteed to be safe. If another CPU came up just after we checked 889 * and has grabbed the lock or started an active transfer then we'll 890 * notice we're busy and abort. If another CPU comes up after we start 891 * flushing it will be blocked from starting an active transfer until 892 * we're done flushing. If another CPU starts an active transfer after 893 * we release the lock we're still OK because we're no longer the last 894 * CPU. 895 */ 896 if (spin_trylock(&drv->lock)) { 897 if (rpmh_rsc_ctrlr_is_busy(drv) || rpmh_flush(&drv->client)) 898 ret = NOTIFY_BAD; 899 spin_unlock(&drv->lock); 900 } else { 901 /* Another CPU must be up */ 902 return NOTIFY_OK; 903 } 904 905 if (ret == NOTIFY_BAD) { 906 /* Double-check if we're here because someone else is up */ 907 if (cpus_in_pm < num_online_cpus()) 908 ret = NOTIFY_OK; 909 else 910 /* We won't be called w/ CPU_PM_ENTER_FAILED */ 911 atomic_dec(&drv->cpus_in_pm); 912 } 913 914 return ret; 915 } 916 917 /** 918 * rpmh_rsc_pd_callback() - Check if any of the AMCs are busy. 919 * @nfb: Pointer to the genpd notifier block in struct rsc_drv. 920 * @action: GENPD_NOTIFY_PRE_OFF, GENPD_NOTIFY_OFF, GENPD_NOTIFY_PRE_ON or GENPD_NOTIFY_ON. 921 * @v: Unused 922 * 923 * This function is given to dev_pm_genpd_add_notifier() so we can be informed 924 * about when cluster-pd is going down. When cluster go down we know no more active 925 * transfers will be started so we write sleep/wake sets. This function gets 926 * called from cpuidle code paths and also at system suspend time. 927 * 928 * If AMCs are not busy then writes cached sleep and wake messages to TCSes. 929 * The firmware then takes care of triggering them when entering deepest low power modes. 930 * 931 * Return: 932 * * NOTIFY_OK - success 933 * * NOTIFY_BAD - failure 934 */ 935 static int rpmh_rsc_pd_callback(struct notifier_block *nfb, 936 unsigned long action, void *v) 937 { 938 struct rsc_drv *drv = container_of(nfb, struct rsc_drv, genpd_nb); 939 940 /* We don't need to lock as genpd on/off are serialized */ 941 if ((action == GENPD_NOTIFY_PRE_OFF) && 942 (rpmh_rsc_ctrlr_is_busy(drv) || rpmh_flush(&drv->client))) 943 return NOTIFY_BAD; 944 945 return NOTIFY_OK; 946 } 947 948 static int rpmh_rsc_pd_attach(struct rsc_drv *drv, struct device *dev) 949 { 950 int ret; 951 952 pm_runtime_enable(dev); 953 drv->genpd_nb.notifier_call = rpmh_rsc_pd_callback; 954 ret = dev_pm_genpd_add_notifier(dev, &drv->genpd_nb); 955 if (ret) 956 pm_runtime_disable(dev); 957 958 return ret; 959 } 960 961 static int rpmh_probe_tcs_config(struct platform_device *pdev, struct rsc_drv *drv) 962 { 963 struct tcs_type_config { 964 u32 type; 965 u32 n; 966 } tcs_cfg[TCS_TYPE_NR] = { { 0 } }; 967 struct device_node *dn = pdev->dev.of_node; 968 u32 config, max_tcs, ncpt, offset; 969 int i, ret, n, st = 0; 970 struct tcs_group *tcs; 971 972 ret = of_property_read_u32(dn, "qcom,tcs-offset", &offset); 973 if (ret) 974 return ret; 975 drv->tcs_base = drv->base + offset; 976 977 config = readl_relaxed(drv->base + drv->regs[DRV_PRNT_CHLD_CONFIG]); 978 979 max_tcs = config; 980 max_tcs &= DRV_NUM_TCS_MASK << (DRV_NUM_TCS_SHIFT * drv->id); 981 max_tcs = max_tcs >> (DRV_NUM_TCS_SHIFT * drv->id); 982 983 ncpt = config & (DRV_NCPT_MASK << DRV_NCPT_SHIFT); 984 ncpt = ncpt >> DRV_NCPT_SHIFT; 985 986 n = of_property_count_u32_elems(dn, "qcom,tcs-config"); 987 if (n != 2 * TCS_TYPE_NR) 988 return -EINVAL; 989 990 for (i = 0; i < TCS_TYPE_NR; i++) { 991 ret = of_property_read_u32_index(dn, "qcom,tcs-config", 992 i * 2, &tcs_cfg[i].type); 993 if (ret) 994 return ret; 995 if (tcs_cfg[i].type >= TCS_TYPE_NR) 996 return -EINVAL; 997 998 ret = of_property_read_u32_index(dn, "qcom,tcs-config", 999 i * 2 + 1, &tcs_cfg[i].n); 1000 if (ret) 1001 return ret; 1002 if (tcs_cfg[i].n > MAX_TCS_PER_TYPE) 1003 return -EINVAL; 1004 } 1005 1006 for (i = 0; i < TCS_TYPE_NR; i++) { 1007 tcs = &drv->tcs[tcs_cfg[i].type]; 1008 if (tcs->drv) 1009 return -EINVAL; 1010 tcs->drv = drv; 1011 tcs->type = tcs_cfg[i].type; 1012 tcs->num_tcs = tcs_cfg[i].n; 1013 tcs->ncpt = ncpt; 1014 1015 if (!tcs->num_tcs || tcs->type == CONTROL_TCS) 1016 continue; 1017 1018 if (st + tcs->num_tcs > max_tcs || 1019 st + tcs->num_tcs >= BITS_PER_BYTE * sizeof(tcs->mask)) 1020 return -EINVAL; 1021 1022 tcs->mask = ((1 << tcs->num_tcs) - 1) << st; 1023 tcs->offset = st; 1024 st += tcs->num_tcs; 1025 } 1026 1027 drv->num_tcs = st; 1028 1029 return 0; 1030 } 1031 1032 static int rpmh_rsc_probe(struct platform_device *pdev) 1033 { 1034 struct device_node *dn = pdev->dev.of_node; 1035 struct rsc_drv *drv; 1036 char drv_id[10] = {0}; 1037 int ret, irq; 1038 u32 solver_config; 1039 u32 rsc_id; 1040 1041 /* 1042 * Even though RPMh doesn't directly use cmd-db, all of its children 1043 * do. To avoid adding this check to our children we'll do it now. 1044 */ 1045 ret = cmd_db_ready(); 1046 if (ret) { 1047 if (ret != -EPROBE_DEFER) 1048 dev_err(&pdev->dev, "Command DB not available (%d)\n", 1049 ret); 1050 return ret; 1051 } 1052 1053 drv = devm_kzalloc(&pdev->dev, sizeof(*drv), GFP_KERNEL); 1054 if (!drv) 1055 return -ENOMEM; 1056 1057 ret = of_property_read_u32(dn, "qcom,drv-id", &drv->id); 1058 if (ret) 1059 return ret; 1060 1061 drv->name = of_get_property(dn, "label", NULL); 1062 if (!drv->name) 1063 drv->name = dev_name(&pdev->dev); 1064 1065 snprintf(drv_id, ARRAY_SIZE(drv_id), "drv-%d", drv->id); 1066 drv->base = devm_platform_ioremap_resource_byname(pdev, drv_id); 1067 if (IS_ERR(drv->base)) 1068 return PTR_ERR(drv->base); 1069 1070 rsc_id = readl_relaxed(drv->base + RSC_DRV_ID); 1071 drv->ver.major = rsc_id & (MAJOR_VER_MASK << MAJOR_VER_SHIFT); 1072 drv->ver.major >>= MAJOR_VER_SHIFT; 1073 drv->ver.minor = rsc_id & (MINOR_VER_MASK << MINOR_VER_SHIFT); 1074 drv->ver.minor >>= MINOR_VER_SHIFT; 1075 1076 if (drv->ver.major == 3) 1077 drv->regs = rpmh_rsc_reg_offset_ver_3_0; 1078 else 1079 drv->regs = rpmh_rsc_reg_offset_ver_2_7; 1080 1081 ret = rpmh_probe_tcs_config(pdev, drv); 1082 if (ret) 1083 return ret; 1084 1085 spin_lock_init(&drv->lock); 1086 init_waitqueue_head(&drv->tcs_wait); 1087 bitmap_zero(drv->tcs_in_use, MAX_TCS_NR); 1088 1089 irq = platform_get_irq(pdev, drv->id); 1090 if (irq < 0) 1091 return irq; 1092 1093 ret = devm_request_irq(&pdev->dev, irq, tcs_tx_done, 1094 IRQF_TRIGGER_HIGH | IRQF_NO_SUSPEND, 1095 drv->name, drv); 1096 if (ret) 1097 return ret; 1098 1099 /* 1100 * CPU PM/genpd notification are not required for controllers that support 1101 * 'HW solver' mode where they can be in autonomous mode executing low 1102 * power mode to power down. 1103 */ 1104 solver_config = readl_relaxed(drv->base + drv->regs[DRV_SOLVER_CONFIG]); 1105 solver_config &= DRV_HW_SOLVER_MASK << DRV_HW_SOLVER_SHIFT; 1106 solver_config = solver_config >> DRV_HW_SOLVER_SHIFT; 1107 if (!solver_config) { 1108 if (pdev->dev.pm_domain) { 1109 ret = rpmh_rsc_pd_attach(drv, &pdev->dev); 1110 if (ret) 1111 return ret; 1112 } else { 1113 drv->rsc_pm.notifier_call = rpmh_rsc_cpu_pm_callback; 1114 cpu_pm_register_notifier(&drv->rsc_pm); 1115 } 1116 } 1117 1118 /* Enable the active TCS to send requests immediately */ 1119 writel_relaxed(drv->tcs[ACTIVE_TCS].mask, 1120 drv->tcs_base + drv->regs[RSC_DRV_IRQ_ENABLE]); 1121 1122 spin_lock_init(&drv->client.cache_lock); 1123 INIT_LIST_HEAD(&drv->client.cache); 1124 INIT_LIST_HEAD(&drv->client.batch_cache); 1125 1126 dev_set_drvdata(&pdev->dev, drv); 1127 drv->dev = &pdev->dev; 1128 1129 ret = devm_of_platform_populate(&pdev->dev); 1130 if (ret && pdev->dev.pm_domain) { 1131 dev_pm_genpd_remove_notifier(&pdev->dev); 1132 pm_runtime_disable(&pdev->dev); 1133 } 1134 1135 return ret; 1136 } 1137 1138 static const struct of_device_id rpmh_drv_match[] = { 1139 { .compatible = "qcom,rpmh-rsc", }, 1140 { } 1141 }; 1142 MODULE_DEVICE_TABLE(of, rpmh_drv_match); 1143 1144 static struct platform_driver rpmh_driver = { 1145 .probe = rpmh_rsc_probe, 1146 .driver = { 1147 .name = "rpmh", 1148 .of_match_table = rpmh_drv_match, 1149 .suppress_bind_attrs = true, 1150 }, 1151 }; 1152 1153 static int __init rpmh_driver_init(void) 1154 { 1155 return platform_driver_register(&rpmh_driver); 1156 } 1157 arch_initcall(rpmh_driver_init); 1158 1159 MODULE_DESCRIPTION("Qualcomm Technologies, Inc. RPMh Driver"); 1160 MODULE_LICENSE("GPL v2"); 1161