1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (c) 2016-2018, The Linux Foundation. All rights reserved. 4 */ 5 6 #define pr_fmt(fmt) "%s " fmt, KBUILD_MODNAME 7 8 #include <linux/atomic.h> 9 #include <linux/cpu_pm.h> 10 #include <linux/delay.h> 11 #include <linux/interrupt.h> 12 #include <linux/io.h> 13 #include <linux/iopoll.h> 14 #include <linux/kernel.h> 15 #include <linux/list.h> 16 #include <linux/module.h> 17 #include <linux/of.h> 18 #include <linux/of_irq.h> 19 #include <linux/of_platform.h> 20 #include <linux/platform_device.h> 21 #include <linux/slab.h> 22 #include <linux/spinlock.h> 23 #include <linux/wait.h> 24 25 #include <soc/qcom/cmd-db.h> 26 #include <soc/qcom/tcs.h> 27 #include <dt-bindings/soc/qcom,rpmh-rsc.h> 28 29 #include "rpmh-internal.h" 30 31 #define CREATE_TRACE_POINTS 32 #include "trace-rpmh.h" 33 34 #define RSC_DRV_TCS_OFFSET 672 35 #define RSC_DRV_CMD_OFFSET 20 36 37 /* DRV HW Solver Configuration Information Register */ 38 #define DRV_SOLVER_CONFIG 0x04 39 #define DRV_HW_SOLVER_MASK 1 40 #define DRV_HW_SOLVER_SHIFT 24 41 42 /* DRV TCS Configuration Information Register */ 43 #define DRV_PRNT_CHLD_CONFIG 0x0C 44 #define DRV_NUM_TCS_MASK 0x3F 45 #define DRV_NUM_TCS_SHIFT 6 46 #define DRV_NCPT_MASK 0x1F 47 #define DRV_NCPT_SHIFT 27 48 49 /* Offsets for common TCS Registers, one bit per TCS */ 50 #define RSC_DRV_IRQ_ENABLE 0x00 51 #define RSC_DRV_IRQ_STATUS 0x04 52 #define RSC_DRV_IRQ_CLEAR 0x08 /* w/o; write 1 to clear */ 53 54 /* 55 * Offsets for per TCS Registers. 56 * 57 * TCSes start at 0x10 from tcs_base and are stored one after another. 58 * Multiply tcs_id by RSC_DRV_TCS_OFFSET to find a given TCS and add one 59 * of the below to find a register. 60 */ 61 #define RSC_DRV_CMD_WAIT_FOR_CMPL 0x10 /* 1 bit per command */ 62 #define RSC_DRV_CONTROL 0x14 63 #define RSC_DRV_STATUS 0x18 /* zero if tcs is busy */ 64 #define RSC_DRV_CMD_ENABLE 0x1C /* 1 bit per command */ 65 66 /* 67 * Offsets for per command in a TCS. 68 * 69 * Commands (up to 16) start at 0x30 in a TCS; multiply command index 70 * by RSC_DRV_CMD_OFFSET and add one of the below to find a register. 71 */ 72 #define RSC_DRV_CMD_MSGID 0x30 73 #define RSC_DRV_CMD_ADDR 0x34 74 #define RSC_DRV_CMD_DATA 0x38 75 #define RSC_DRV_CMD_STATUS 0x3C 76 #define RSC_DRV_CMD_RESP_DATA 0x40 77 78 #define TCS_AMC_MODE_ENABLE BIT(16) 79 #define TCS_AMC_MODE_TRIGGER BIT(24) 80 81 /* TCS CMD register bit mask */ 82 #define CMD_MSGID_LEN 8 83 #define CMD_MSGID_RESP_REQ BIT(8) 84 #define CMD_MSGID_WRITE BIT(16) 85 #define CMD_STATUS_ISSUED BIT(8) 86 #define CMD_STATUS_COMPL BIT(16) 87 88 /* 89 * Here's a high level overview of how all the registers in RPMH work 90 * together: 91 * 92 * - The main rpmh-rsc address is the base of a register space that can 93 * be used to find overall configuration of the hardware 94 * (DRV_PRNT_CHLD_CONFIG). Also found within the rpmh-rsc register 95 * space are all the TCS blocks. The offset of the TCS blocks is 96 * specified in the device tree by "qcom,tcs-offset" and used to 97 * compute tcs_base. 98 * - TCS blocks come one after another. Type, count, and order are 99 * specified by the device tree as "qcom,tcs-config". 100 * - Each TCS block has some registers, then space for up to 16 commands. 101 * Note that though address space is reserved for 16 commands, fewer 102 * might be present. See ncpt (num cmds per TCS). 103 * 104 * Here's a picture: 105 * 106 * +---------------------------------------------------+ 107 * |RSC | 108 * | ctrl | 109 * | | 110 * | Drvs: | 111 * | +-----------------------------------------------+ | 112 * | |DRV0 | | 113 * | | ctrl/config | | 114 * | | IRQ | | 115 * | | | | 116 * | | TCSes: | | 117 * | | +------------------------------------------+ | | 118 * | | |TCS0 | | | | | | | | | | | | | | | 119 * | | | ctrl | 0| 1| 2| 3| 4| 5| .| .| .| .|14|15| | | 120 * | | | | | | | | | | | | | | | | | | 121 * | | +------------------------------------------+ | | 122 * | | +------------------------------------------+ | | 123 * | | |TCS1 | | | | | | | | | | | | | | | 124 * | | | ctrl | 0| 1| 2| 3| 4| 5| .| .| .| .|14|15| | | 125 * | | | | | | | | | | | | | | | | | | 126 * | | +------------------------------------------+ | | 127 * | | +------------------------------------------+ | | 128 * | | |TCS2 | | | | | | | | | | | | | | | 129 * | | | ctrl | 0| 1| 2| 3| 4| 5| .| .| .| .|14|15| | | 130 * | | | | | | | | | | | | | | | | | | 131 * | | +------------------------------------------+ | | 132 * | | ...... | | 133 * | +-----------------------------------------------+ | 134 * | +-----------------------------------------------+ | 135 * | |DRV1 | | 136 * | | (same as DRV0) | | 137 * | +-----------------------------------------------+ | 138 * | ...... | 139 * +---------------------------------------------------+ 140 */ 141 142 static inline void __iomem * 143 tcs_reg_addr(const struct rsc_drv *drv, int reg, int tcs_id) 144 { 145 return drv->tcs_base + RSC_DRV_TCS_OFFSET * tcs_id + reg; 146 } 147 148 static inline void __iomem * 149 tcs_cmd_addr(const struct rsc_drv *drv, int reg, int tcs_id, int cmd_id) 150 { 151 return tcs_reg_addr(drv, reg, tcs_id) + RSC_DRV_CMD_OFFSET * cmd_id; 152 } 153 154 static u32 read_tcs_cmd(const struct rsc_drv *drv, int reg, int tcs_id, 155 int cmd_id) 156 { 157 return readl_relaxed(tcs_cmd_addr(drv, reg, tcs_id, cmd_id)); 158 } 159 160 static u32 read_tcs_reg(const struct rsc_drv *drv, int reg, int tcs_id) 161 { 162 return readl_relaxed(tcs_reg_addr(drv, reg, tcs_id)); 163 } 164 165 static void write_tcs_cmd(const struct rsc_drv *drv, int reg, int tcs_id, 166 int cmd_id, u32 data) 167 { 168 writel_relaxed(data, tcs_cmd_addr(drv, reg, tcs_id, cmd_id)); 169 } 170 171 static void write_tcs_reg(const struct rsc_drv *drv, int reg, int tcs_id, 172 u32 data) 173 { 174 writel_relaxed(data, tcs_reg_addr(drv, reg, tcs_id)); 175 } 176 177 static void write_tcs_reg_sync(const struct rsc_drv *drv, int reg, int tcs_id, 178 u32 data) 179 { 180 int i; 181 182 writel(data, tcs_reg_addr(drv, reg, tcs_id)); 183 184 /* 185 * Wait until we read back the same value. Use a counter rather than 186 * ktime for timeout since this may be called after timekeeping stops. 187 */ 188 for (i = 0; i < USEC_PER_SEC; i++) { 189 if (readl(tcs_reg_addr(drv, reg, tcs_id)) == data) 190 return; 191 udelay(1); 192 } 193 pr_err("%s: error writing %#x to %d:%#x\n", drv->name, 194 data, tcs_id, reg); 195 } 196 197 /** 198 * tcs_is_free() - Return if a TCS is totally free. 199 * @drv: The RSC controller. 200 * @tcs_id: The global ID of this TCS. 201 * 202 * Returns true if nobody has claimed this TCS (by setting tcs_in_use). 203 * 204 * Context: Must be called with the drv->lock held. 205 * 206 * Return: true if the given TCS is free. 207 */ 208 static bool tcs_is_free(struct rsc_drv *drv, int tcs_id) 209 { 210 return !test_bit(tcs_id, drv->tcs_in_use); 211 } 212 213 /** 214 * tcs_invalidate() - Invalidate all TCSes of the given type (sleep or wake). 215 * @drv: The RSC controller. 216 * @type: SLEEP_TCS or WAKE_TCS 217 * 218 * This will clear the "slots" variable of the given tcs_group and also 219 * tell the hardware to forget about all entries. 220 * 221 * The caller must ensure that no other RPMH actions are happening when this 222 * function is called, since otherwise the device may immediately become 223 * used again even before this function exits. 224 */ 225 static void tcs_invalidate(struct rsc_drv *drv, int type) 226 { 227 int m; 228 struct tcs_group *tcs = &drv->tcs[type]; 229 230 /* Caller ensures nobody else is running so no lock */ 231 if (bitmap_empty(tcs->slots, MAX_TCS_SLOTS)) 232 return; 233 234 for (m = tcs->offset; m < tcs->offset + tcs->num_tcs; m++) { 235 write_tcs_reg_sync(drv, RSC_DRV_CMD_ENABLE, m, 0); 236 write_tcs_reg_sync(drv, RSC_DRV_CMD_WAIT_FOR_CMPL, m, 0); 237 } 238 bitmap_zero(tcs->slots, MAX_TCS_SLOTS); 239 } 240 241 /** 242 * rpmh_rsc_invalidate() - Invalidate sleep and wake TCSes. 243 * @drv: The RSC controller. 244 * 245 * The caller must ensure that no other RPMH actions are happening when this 246 * function is called, since otherwise the device may immediately become 247 * used again even before this function exits. 248 */ 249 void rpmh_rsc_invalidate(struct rsc_drv *drv) 250 { 251 tcs_invalidate(drv, SLEEP_TCS); 252 tcs_invalidate(drv, WAKE_TCS); 253 } 254 255 /** 256 * get_tcs_for_msg() - Get the tcs_group used to send the given message. 257 * @drv: The RSC controller. 258 * @msg: The message we want to send. 259 * 260 * This is normally pretty straightforward except if we are trying to send 261 * an ACTIVE_ONLY message but don't have any active_only TCSes. 262 * 263 * Return: A pointer to a tcs_group or an ERR_PTR. 264 */ 265 static struct tcs_group *get_tcs_for_msg(struct rsc_drv *drv, 266 const struct tcs_request *msg) 267 { 268 int type; 269 struct tcs_group *tcs; 270 271 switch (msg->state) { 272 case RPMH_ACTIVE_ONLY_STATE: 273 type = ACTIVE_TCS; 274 break; 275 case RPMH_WAKE_ONLY_STATE: 276 type = WAKE_TCS; 277 break; 278 case RPMH_SLEEP_STATE: 279 type = SLEEP_TCS; 280 break; 281 default: 282 return ERR_PTR(-EINVAL); 283 } 284 285 /* 286 * If we are making an active request on a RSC that does not have a 287 * dedicated TCS for active state use, then re-purpose a wake TCS to 288 * send active votes. This is safe because we ensure any active-only 289 * transfers have finished before we use it (maybe by running from 290 * the last CPU in PM code). 291 */ 292 tcs = &drv->tcs[type]; 293 if (msg->state == RPMH_ACTIVE_ONLY_STATE && !tcs->num_tcs) 294 tcs = &drv->tcs[WAKE_TCS]; 295 296 return tcs; 297 } 298 299 /** 300 * get_req_from_tcs() - Get a stashed request that was xfering on the given TCS. 301 * @drv: The RSC controller. 302 * @tcs_id: The global ID of this TCS. 303 * 304 * For ACTIVE_ONLY transfers we want to call back into the client when the 305 * transfer finishes. To do this we need the "request" that the client 306 * originally provided us. This function grabs the request that we stashed 307 * when we started the transfer. 308 * 309 * This only makes sense for ACTIVE_ONLY transfers since those are the only 310 * ones we track sending (the only ones we enable interrupts for and the only 311 * ones we call back to the client for). 312 * 313 * Return: The stashed request. 314 */ 315 static const struct tcs_request *get_req_from_tcs(struct rsc_drv *drv, 316 int tcs_id) 317 { 318 struct tcs_group *tcs; 319 int i; 320 321 for (i = 0; i < TCS_TYPE_NR; i++) { 322 tcs = &drv->tcs[i]; 323 if (tcs->mask & BIT(tcs_id)) 324 return tcs->req[tcs_id - tcs->offset]; 325 } 326 327 return NULL; 328 } 329 330 /** 331 * __tcs_set_trigger() - Start xfer on a TCS or unset trigger on a borrowed TCS 332 * @drv: The controller. 333 * @tcs_id: The global ID of this TCS. 334 * @trigger: If true then untrigger/retrigger. If false then just untrigger. 335 * 336 * In the normal case we only ever call with "trigger=true" to start a 337 * transfer. That will un-trigger/disable the TCS from the last transfer 338 * then trigger/enable for this transfer. 339 * 340 * If we borrowed a wake TCS for an active-only transfer we'll also call 341 * this function with "trigger=false" to just do the un-trigger/disable 342 * before using the TCS for wake purposes again. 343 * 344 * Note that the AP is only in charge of triggering active-only transfers. 345 * The AP never triggers sleep/wake values using this function. 346 */ 347 static void __tcs_set_trigger(struct rsc_drv *drv, int tcs_id, bool trigger) 348 { 349 u32 enable; 350 351 /* 352 * HW req: Clear the DRV_CONTROL and enable TCS again 353 * While clearing ensure that the AMC mode trigger is cleared 354 * and then the mode enable is cleared. 355 */ 356 enable = read_tcs_reg(drv, RSC_DRV_CONTROL, tcs_id); 357 enable &= ~TCS_AMC_MODE_TRIGGER; 358 write_tcs_reg_sync(drv, RSC_DRV_CONTROL, tcs_id, enable); 359 enable &= ~TCS_AMC_MODE_ENABLE; 360 write_tcs_reg_sync(drv, RSC_DRV_CONTROL, tcs_id, enable); 361 362 if (trigger) { 363 /* Enable the AMC mode on the TCS and then trigger the TCS */ 364 enable = TCS_AMC_MODE_ENABLE; 365 write_tcs_reg_sync(drv, RSC_DRV_CONTROL, tcs_id, enable); 366 enable |= TCS_AMC_MODE_TRIGGER; 367 write_tcs_reg_sync(drv, RSC_DRV_CONTROL, tcs_id, enable); 368 } 369 } 370 371 /** 372 * enable_tcs_irq() - Enable or disable interrupts on the given TCS. 373 * @drv: The controller. 374 * @tcs_id: The global ID of this TCS. 375 * @enable: If true then enable; if false then disable 376 * 377 * We only ever call this when we borrow a wake TCS for an active-only 378 * transfer. For active-only TCSes interrupts are always left enabled. 379 */ 380 static void enable_tcs_irq(struct rsc_drv *drv, int tcs_id, bool enable) 381 { 382 u32 data; 383 384 data = readl_relaxed(drv->tcs_base + RSC_DRV_IRQ_ENABLE); 385 if (enable) 386 data |= BIT(tcs_id); 387 else 388 data &= ~BIT(tcs_id); 389 writel_relaxed(data, drv->tcs_base + RSC_DRV_IRQ_ENABLE); 390 } 391 392 /** 393 * tcs_tx_done() - TX Done interrupt handler. 394 * @irq: The IRQ number (ignored). 395 * @p: Pointer to "struct rsc_drv". 396 * 397 * Called for ACTIVE_ONLY transfers (those are the only ones we enable the 398 * IRQ for) when a transfer is done. 399 * 400 * Return: IRQ_HANDLED 401 */ 402 static irqreturn_t tcs_tx_done(int irq, void *p) 403 { 404 struct rsc_drv *drv = p; 405 int i, j, err = 0; 406 unsigned long irq_status; 407 const struct tcs_request *req; 408 struct tcs_cmd *cmd; 409 410 irq_status = readl_relaxed(drv->tcs_base + RSC_DRV_IRQ_STATUS); 411 412 for_each_set_bit(i, &irq_status, BITS_PER_LONG) { 413 req = get_req_from_tcs(drv, i); 414 if (!req) { 415 WARN_ON(1); 416 goto skip; 417 } 418 419 err = 0; 420 for (j = 0; j < req->num_cmds; j++) { 421 u32 sts; 422 423 cmd = &req->cmds[j]; 424 sts = read_tcs_cmd(drv, RSC_DRV_CMD_STATUS, i, j); 425 if (!(sts & CMD_STATUS_ISSUED) || 426 ((req->wait_for_compl || cmd->wait) && 427 !(sts & CMD_STATUS_COMPL))) { 428 pr_err("Incomplete request: %s: addr=%#x data=%#x", 429 drv->name, cmd->addr, cmd->data); 430 err = -EIO; 431 } 432 } 433 434 trace_rpmh_tx_done(drv, i, req, err); 435 436 /* 437 * If wake tcs was re-purposed for sending active 438 * votes, clear AMC trigger & enable modes and 439 * disable interrupt for this TCS 440 */ 441 if (!drv->tcs[ACTIVE_TCS].num_tcs) 442 __tcs_set_trigger(drv, i, false); 443 skip: 444 /* Reclaim the TCS */ 445 write_tcs_reg(drv, RSC_DRV_CMD_ENABLE, i, 0); 446 write_tcs_reg(drv, RSC_DRV_CMD_WAIT_FOR_CMPL, i, 0); 447 writel_relaxed(BIT(i), drv->tcs_base + RSC_DRV_IRQ_CLEAR); 448 spin_lock(&drv->lock); 449 clear_bit(i, drv->tcs_in_use); 450 /* 451 * Disable interrupt for WAKE TCS to avoid being 452 * spammed with interrupts coming when the solver 453 * sends its wake votes. 454 */ 455 if (!drv->tcs[ACTIVE_TCS].num_tcs) 456 enable_tcs_irq(drv, i, false); 457 spin_unlock(&drv->lock); 458 wake_up(&drv->tcs_wait); 459 if (req) 460 rpmh_tx_done(req, err); 461 } 462 463 return IRQ_HANDLED; 464 } 465 466 /** 467 * __tcs_buffer_write() - Write to TCS hardware from a request; don't trigger. 468 * @drv: The controller. 469 * @tcs_id: The global ID of this TCS. 470 * @cmd_id: The index within the TCS to start writing. 471 * @msg: The message we want to send, which will contain several addr/data 472 * pairs to program (but few enough that they all fit in one TCS). 473 * 474 * This is used for all types of transfers (active, sleep, and wake). 475 */ 476 static void __tcs_buffer_write(struct rsc_drv *drv, int tcs_id, int cmd_id, 477 const struct tcs_request *msg) 478 { 479 u32 msgid, cmd_msgid; 480 u32 cmd_enable = 0; 481 u32 cmd_complete; 482 struct tcs_cmd *cmd; 483 int i, j; 484 485 cmd_msgid = CMD_MSGID_LEN; 486 cmd_msgid |= msg->wait_for_compl ? CMD_MSGID_RESP_REQ : 0; 487 cmd_msgid |= CMD_MSGID_WRITE; 488 489 cmd_complete = read_tcs_reg(drv, RSC_DRV_CMD_WAIT_FOR_CMPL, tcs_id); 490 491 for (i = 0, j = cmd_id; i < msg->num_cmds; i++, j++) { 492 cmd = &msg->cmds[i]; 493 cmd_enable |= BIT(j); 494 cmd_complete |= cmd->wait << j; 495 msgid = cmd_msgid; 496 msgid |= cmd->wait ? CMD_MSGID_RESP_REQ : 0; 497 498 write_tcs_cmd(drv, RSC_DRV_CMD_MSGID, tcs_id, j, msgid); 499 write_tcs_cmd(drv, RSC_DRV_CMD_ADDR, tcs_id, j, cmd->addr); 500 write_tcs_cmd(drv, RSC_DRV_CMD_DATA, tcs_id, j, cmd->data); 501 trace_rpmh_send_msg(drv, tcs_id, j, msgid, cmd); 502 } 503 504 write_tcs_reg(drv, RSC_DRV_CMD_WAIT_FOR_CMPL, tcs_id, cmd_complete); 505 cmd_enable |= read_tcs_reg(drv, RSC_DRV_CMD_ENABLE, tcs_id); 506 write_tcs_reg(drv, RSC_DRV_CMD_ENABLE, tcs_id, cmd_enable); 507 } 508 509 /** 510 * check_for_req_inflight() - Look to see if conflicting cmds are in flight. 511 * @drv: The controller. 512 * @tcs: A pointer to the tcs_group used for ACTIVE_ONLY transfers. 513 * @msg: The message we want to send, which will contain several addr/data 514 * pairs to program (but few enough that they all fit in one TCS). 515 * 516 * This will walk through the TCSes in the group and check if any of them 517 * appear to be sending to addresses referenced in the message. If it finds 518 * one it'll return -EBUSY. 519 * 520 * Only for use for active-only transfers. 521 * 522 * Must be called with the drv->lock held since that protects tcs_in_use. 523 * 524 * Return: 0 if nothing in flight or -EBUSY if we should try again later. 525 * The caller must re-enable interrupts between tries since that's 526 * the only way tcs_is_free() will ever return true and the only way 527 * RSC_DRV_CMD_ENABLE will ever be cleared. 528 */ 529 static int check_for_req_inflight(struct rsc_drv *drv, struct tcs_group *tcs, 530 const struct tcs_request *msg) 531 { 532 unsigned long curr_enabled; 533 u32 addr; 534 int i, j, k; 535 int tcs_id = tcs->offset; 536 537 for (i = 0; i < tcs->num_tcs; i++, tcs_id++) { 538 if (tcs_is_free(drv, tcs_id)) 539 continue; 540 541 curr_enabled = read_tcs_reg(drv, RSC_DRV_CMD_ENABLE, tcs_id); 542 543 for_each_set_bit(j, &curr_enabled, MAX_CMDS_PER_TCS) { 544 addr = read_tcs_cmd(drv, RSC_DRV_CMD_ADDR, tcs_id, j); 545 for (k = 0; k < msg->num_cmds; k++) { 546 if (addr == msg->cmds[k].addr) 547 return -EBUSY; 548 } 549 } 550 } 551 552 return 0; 553 } 554 555 /** 556 * find_free_tcs() - Find free tcs in the given tcs_group; only for active. 557 * @tcs: A pointer to the active-only tcs_group (or the wake tcs_group if 558 * we borrowed it because there are zero active-only ones). 559 * 560 * Must be called with the drv->lock held since that protects tcs_in_use. 561 * 562 * Return: The first tcs that's free. 563 */ 564 static int find_free_tcs(struct tcs_group *tcs) 565 { 566 int i; 567 568 for (i = 0; i < tcs->num_tcs; i++) { 569 if (tcs_is_free(tcs->drv, tcs->offset + i)) 570 return tcs->offset + i; 571 } 572 573 return -EBUSY; 574 } 575 576 /** 577 * claim_tcs_for_req() - Claim a tcs in the given tcs_group; only for active. 578 * @drv: The controller. 579 * @tcs: The tcs_group used for ACTIVE_ONLY transfers. 580 * @msg: The data to be sent. 581 * 582 * Claims a tcs in the given tcs_group while making sure that no existing cmd 583 * is in flight that would conflict with the one in @msg. 584 * 585 * Context: Must be called with the drv->lock held since that protects 586 * tcs_in_use. 587 * 588 * Return: The id of the claimed tcs or -EBUSY if a matching msg is in flight 589 * or the tcs_group is full. 590 */ 591 static int claim_tcs_for_req(struct rsc_drv *drv, struct tcs_group *tcs, 592 const struct tcs_request *msg) 593 { 594 int ret; 595 596 /* 597 * The h/w does not like if we send a request to the same address, 598 * when one is already in-flight or being processed. 599 */ 600 ret = check_for_req_inflight(drv, tcs, msg); 601 if (ret) 602 return ret; 603 604 return find_free_tcs(tcs); 605 } 606 607 /** 608 * rpmh_rsc_send_data() - Write / trigger active-only message. 609 * @drv: The controller. 610 * @msg: The data to be sent. 611 * 612 * NOTES: 613 * - This is only used for "ACTIVE_ONLY" since the limitations of this 614 * function don't make sense for sleep/wake cases. 615 * - To do the transfer, we will grab a whole TCS for ourselves--we don't 616 * try to share. If there are none available we'll wait indefinitely 617 * for a free one. 618 * - This function will not wait for the commands to be finished, only for 619 * data to be programmed into the RPMh. See rpmh_tx_done() which will 620 * be called when the transfer is fully complete. 621 * - This function must be called with interrupts enabled. If the hardware 622 * is busy doing someone else's transfer we need that transfer to fully 623 * finish so that we can have the hardware, and to fully finish it needs 624 * the interrupt handler to run. If the interrupts is set to run on the 625 * active CPU this can never happen if interrupts are disabled. 626 * 627 * Return: 0 on success, -EINVAL on error. 628 */ 629 int rpmh_rsc_send_data(struct rsc_drv *drv, const struct tcs_request *msg) 630 { 631 struct tcs_group *tcs; 632 int tcs_id; 633 unsigned long flags; 634 635 tcs = get_tcs_for_msg(drv, msg); 636 if (IS_ERR(tcs)) 637 return PTR_ERR(tcs); 638 639 spin_lock_irqsave(&drv->lock, flags); 640 641 /* Wait forever for a free tcs. It better be there eventually! */ 642 wait_event_lock_irq(drv->tcs_wait, 643 (tcs_id = claim_tcs_for_req(drv, tcs, msg)) >= 0, 644 drv->lock); 645 646 tcs->req[tcs_id - tcs->offset] = msg; 647 set_bit(tcs_id, drv->tcs_in_use); 648 if (msg->state == RPMH_ACTIVE_ONLY_STATE && tcs->type != ACTIVE_TCS) { 649 /* 650 * Clear previously programmed WAKE commands in selected 651 * repurposed TCS to avoid triggering them. tcs->slots will be 652 * cleaned from rpmh_flush() by invoking rpmh_rsc_invalidate() 653 */ 654 write_tcs_reg_sync(drv, RSC_DRV_CMD_ENABLE, tcs_id, 0); 655 write_tcs_reg_sync(drv, RSC_DRV_CMD_WAIT_FOR_CMPL, tcs_id, 0); 656 enable_tcs_irq(drv, tcs_id, true); 657 } 658 spin_unlock_irqrestore(&drv->lock, flags); 659 660 /* 661 * These two can be done after the lock is released because: 662 * - We marked "tcs_in_use" under lock. 663 * - Once "tcs_in_use" has been marked nobody else could be writing 664 * to these registers until the interrupt goes off. 665 * - The interrupt can't go off until we trigger w/ the last line 666 * of __tcs_set_trigger() below. 667 */ 668 __tcs_buffer_write(drv, tcs_id, 0, msg); 669 __tcs_set_trigger(drv, tcs_id, true); 670 671 return 0; 672 } 673 674 /** 675 * find_slots() - Find a place to write the given message. 676 * @tcs: The tcs group to search. 677 * @msg: The message we want to find room for. 678 * @tcs_id: If we return 0 from the function, we return the global ID of the 679 * TCS to write to here. 680 * @cmd_id: If we return 0 from the function, we return the index of 681 * the command array of the returned TCS where the client should 682 * start writing the message. 683 * 684 * Only for use on sleep/wake TCSes since those are the only ones we maintain 685 * tcs->slots for. 686 * 687 * Return: -ENOMEM if there was no room, else 0. 688 */ 689 static int find_slots(struct tcs_group *tcs, const struct tcs_request *msg, 690 int *tcs_id, int *cmd_id) 691 { 692 int slot, offset; 693 int i = 0; 694 695 /* Do over, until we can fit the full payload in a single TCS */ 696 do { 697 slot = bitmap_find_next_zero_area(tcs->slots, MAX_TCS_SLOTS, 698 i, msg->num_cmds, 0); 699 if (slot >= tcs->num_tcs * tcs->ncpt) 700 return -ENOMEM; 701 i += tcs->ncpt; 702 } while (slot + msg->num_cmds - 1 >= i); 703 704 bitmap_set(tcs->slots, slot, msg->num_cmds); 705 706 offset = slot / tcs->ncpt; 707 *tcs_id = offset + tcs->offset; 708 *cmd_id = slot % tcs->ncpt; 709 710 return 0; 711 } 712 713 /** 714 * rpmh_rsc_write_ctrl_data() - Write request to controller but don't trigger. 715 * @drv: The controller. 716 * @msg: The data to be written to the controller. 717 * 718 * This should only be called for for sleep/wake state, never active-only 719 * state. 720 * 721 * The caller must ensure that no other RPMH actions are happening and the 722 * controller is idle when this function is called since it runs lockless. 723 * 724 * Return: 0 if no error; else -error. 725 */ 726 int rpmh_rsc_write_ctrl_data(struct rsc_drv *drv, const struct tcs_request *msg) 727 { 728 struct tcs_group *tcs; 729 int tcs_id = 0, cmd_id = 0; 730 int ret; 731 732 tcs = get_tcs_for_msg(drv, msg); 733 if (IS_ERR(tcs)) 734 return PTR_ERR(tcs); 735 736 /* find the TCS id and the command in the TCS to write to */ 737 ret = find_slots(tcs, msg, &tcs_id, &cmd_id); 738 if (!ret) 739 __tcs_buffer_write(drv, tcs_id, cmd_id, msg); 740 741 return ret; 742 } 743 744 /** 745 * rpmh_rsc_ctrlr_is_busy() - Check if any of the AMCs are busy. 746 * @drv: The controller 747 * 748 * Checks if any of the AMCs are busy in handling ACTIVE sets. 749 * This is called from the last cpu powering down before flushing 750 * SLEEP and WAKE sets. If AMCs are busy, controller can not enter 751 * power collapse, so deny from the last cpu's pm notification. 752 * 753 * Context: Must be called with the drv->lock held. 754 * 755 * Return: 756 * * False - AMCs are idle 757 * * True - AMCs are busy 758 */ 759 static bool rpmh_rsc_ctrlr_is_busy(struct rsc_drv *drv) 760 { 761 int m; 762 struct tcs_group *tcs = &drv->tcs[ACTIVE_TCS]; 763 764 /* 765 * If we made an active request on a RSC that does not have a 766 * dedicated TCS for active state use, then re-purposed wake TCSes 767 * should be checked for not busy, because we used wake TCSes for 768 * active requests in this case. 769 */ 770 if (!tcs->num_tcs) 771 tcs = &drv->tcs[WAKE_TCS]; 772 773 for (m = tcs->offset; m < tcs->offset + tcs->num_tcs; m++) { 774 if (!tcs_is_free(drv, m)) 775 return true; 776 } 777 778 return false; 779 } 780 781 /** 782 * rpmh_rsc_cpu_pm_callback() - Check if any of the AMCs are busy. 783 * @nfb: Pointer to the notifier block in struct rsc_drv. 784 * @action: CPU_PM_ENTER, CPU_PM_ENTER_FAILED, or CPU_PM_EXIT. 785 * @v: Unused 786 * 787 * This function is given to cpu_pm_register_notifier so we can be informed 788 * about when CPUs go down. When all CPUs go down we know no more active 789 * transfers will be started so we write sleep/wake sets. This function gets 790 * called from cpuidle code paths and also at system suspend time. 791 * 792 * If its last CPU going down and AMCs are not busy then writes cached sleep 793 * and wake messages to TCSes. The firmware then takes care of triggering 794 * them when entering deepest low power modes. 795 * 796 * Return: See cpu_pm_register_notifier() 797 */ 798 static int rpmh_rsc_cpu_pm_callback(struct notifier_block *nfb, 799 unsigned long action, void *v) 800 { 801 struct rsc_drv *drv = container_of(nfb, struct rsc_drv, rsc_pm); 802 int ret = NOTIFY_OK; 803 int cpus_in_pm; 804 805 switch (action) { 806 case CPU_PM_ENTER: 807 cpus_in_pm = atomic_inc_return(&drv->cpus_in_pm); 808 /* 809 * NOTE: comments for num_online_cpus() point out that it's 810 * only a snapshot so we need to be careful. It should be OK 811 * for us to use, though. It's important for us not to miss 812 * if we're the last CPU going down so it would only be a 813 * problem if a CPU went offline right after we did the check 814 * AND that CPU was not idle AND that CPU was the last non-idle 815 * CPU. That can't happen. CPUs would have to come out of idle 816 * before the CPU could go offline. 817 */ 818 if (cpus_in_pm < num_online_cpus()) 819 return NOTIFY_OK; 820 break; 821 case CPU_PM_ENTER_FAILED: 822 case CPU_PM_EXIT: 823 atomic_dec(&drv->cpus_in_pm); 824 return NOTIFY_OK; 825 default: 826 return NOTIFY_DONE; 827 } 828 829 /* 830 * It's likely we're on the last CPU. Grab the drv->lock and write 831 * out the sleep/wake commands to RPMH hardware. Grabbing the lock 832 * means that if we race with another CPU coming up we are still 833 * guaranteed to be safe. If another CPU came up just after we checked 834 * and has grabbed the lock or started an active transfer then we'll 835 * notice we're busy and abort. If another CPU comes up after we start 836 * flushing it will be blocked from starting an active transfer until 837 * we're done flushing. If another CPU starts an active transfer after 838 * we release the lock we're still OK because we're no longer the last 839 * CPU. 840 */ 841 if (spin_trylock(&drv->lock)) { 842 if (rpmh_rsc_ctrlr_is_busy(drv) || rpmh_flush(&drv->client)) 843 ret = NOTIFY_BAD; 844 spin_unlock(&drv->lock); 845 } else { 846 /* Another CPU must be up */ 847 return NOTIFY_OK; 848 } 849 850 if (ret == NOTIFY_BAD) { 851 /* Double-check if we're here because someone else is up */ 852 if (cpus_in_pm < num_online_cpus()) 853 ret = NOTIFY_OK; 854 else 855 /* We won't be called w/ CPU_PM_ENTER_FAILED */ 856 atomic_dec(&drv->cpus_in_pm); 857 } 858 859 return ret; 860 } 861 862 static int rpmh_probe_tcs_config(struct platform_device *pdev, 863 struct rsc_drv *drv, void __iomem *base) 864 { 865 struct tcs_type_config { 866 u32 type; 867 u32 n; 868 } tcs_cfg[TCS_TYPE_NR] = { { 0 } }; 869 struct device_node *dn = pdev->dev.of_node; 870 u32 config, max_tcs, ncpt, offset; 871 int i, ret, n, st = 0; 872 struct tcs_group *tcs; 873 874 ret = of_property_read_u32(dn, "qcom,tcs-offset", &offset); 875 if (ret) 876 return ret; 877 drv->tcs_base = base + offset; 878 879 config = readl_relaxed(base + DRV_PRNT_CHLD_CONFIG); 880 881 max_tcs = config; 882 max_tcs &= DRV_NUM_TCS_MASK << (DRV_NUM_TCS_SHIFT * drv->id); 883 max_tcs = max_tcs >> (DRV_NUM_TCS_SHIFT * drv->id); 884 885 ncpt = config & (DRV_NCPT_MASK << DRV_NCPT_SHIFT); 886 ncpt = ncpt >> DRV_NCPT_SHIFT; 887 888 n = of_property_count_u32_elems(dn, "qcom,tcs-config"); 889 if (n != 2 * TCS_TYPE_NR) 890 return -EINVAL; 891 892 for (i = 0; i < TCS_TYPE_NR; i++) { 893 ret = of_property_read_u32_index(dn, "qcom,tcs-config", 894 i * 2, &tcs_cfg[i].type); 895 if (ret) 896 return ret; 897 if (tcs_cfg[i].type >= TCS_TYPE_NR) 898 return -EINVAL; 899 900 ret = of_property_read_u32_index(dn, "qcom,tcs-config", 901 i * 2 + 1, &tcs_cfg[i].n); 902 if (ret) 903 return ret; 904 if (tcs_cfg[i].n > MAX_TCS_PER_TYPE) 905 return -EINVAL; 906 } 907 908 for (i = 0; i < TCS_TYPE_NR; i++) { 909 tcs = &drv->tcs[tcs_cfg[i].type]; 910 if (tcs->drv) 911 return -EINVAL; 912 tcs->drv = drv; 913 tcs->type = tcs_cfg[i].type; 914 tcs->num_tcs = tcs_cfg[i].n; 915 tcs->ncpt = ncpt; 916 917 if (!tcs->num_tcs || tcs->type == CONTROL_TCS) 918 continue; 919 920 if (st + tcs->num_tcs > max_tcs || 921 st + tcs->num_tcs >= BITS_PER_BYTE * sizeof(tcs->mask)) 922 return -EINVAL; 923 924 tcs->mask = ((1 << tcs->num_tcs) - 1) << st; 925 tcs->offset = st; 926 st += tcs->num_tcs; 927 } 928 929 drv->num_tcs = st; 930 931 return 0; 932 } 933 934 static int rpmh_rsc_probe(struct platform_device *pdev) 935 { 936 struct device_node *dn = pdev->dev.of_node; 937 struct rsc_drv *drv; 938 struct resource *res; 939 char drv_id[10] = {0}; 940 int ret, irq; 941 u32 solver_config; 942 void __iomem *base; 943 944 /* 945 * Even though RPMh doesn't directly use cmd-db, all of its children 946 * do. To avoid adding this check to our children we'll do it now. 947 */ 948 ret = cmd_db_ready(); 949 if (ret) { 950 if (ret != -EPROBE_DEFER) 951 dev_err(&pdev->dev, "Command DB not available (%d)\n", 952 ret); 953 return ret; 954 } 955 956 drv = devm_kzalloc(&pdev->dev, sizeof(*drv), GFP_KERNEL); 957 if (!drv) 958 return -ENOMEM; 959 960 ret = of_property_read_u32(dn, "qcom,drv-id", &drv->id); 961 if (ret) 962 return ret; 963 964 drv->name = of_get_property(dn, "label", NULL); 965 if (!drv->name) 966 drv->name = dev_name(&pdev->dev); 967 968 snprintf(drv_id, ARRAY_SIZE(drv_id), "drv-%d", drv->id); 969 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, drv_id); 970 base = devm_ioremap_resource(&pdev->dev, res); 971 if (IS_ERR(base)) 972 return PTR_ERR(base); 973 974 ret = rpmh_probe_tcs_config(pdev, drv, base); 975 if (ret) 976 return ret; 977 978 spin_lock_init(&drv->lock); 979 init_waitqueue_head(&drv->tcs_wait); 980 bitmap_zero(drv->tcs_in_use, MAX_TCS_NR); 981 982 irq = platform_get_irq(pdev, drv->id); 983 if (irq < 0) 984 return irq; 985 986 ret = devm_request_irq(&pdev->dev, irq, tcs_tx_done, 987 IRQF_TRIGGER_HIGH | IRQF_NO_SUSPEND, 988 drv->name, drv); 989 if (ret) 990 return ret; 991 992 /* 993 * CPU PM notification are not required for controllers that support 994 * 'HW solver' mode where they can be in autonomous mode executing low 995 * power mode to power down. 996 */ 997 solver_config = readl_relaxed(base + DRV_SOLVER_CONFIG); 998 solver_config &= DRV_HW_SOLVER_MASK << DRV_HW_SOLVER_SHIFT; 999 solver_config = solver_config >> DRV_HW_SOLVER_SHIFT; 1000 if (!solver_config) { 1001 drv->rsc_pm.notifier_call = rpmh_rsc_cpu_pm_callback; 1002 cpu_pm_register_notifier(&drv->rsc_pm); 1003 } 1004 1005 /* Enable the active TCS to send requests immediately */ 1006 writel_relaxed(drv->tcs[ACTIVE_TCS].mask, 1007 drv->tcs_base + RSC_DRV_IRQ_ENABLE); 1008 1009 spin_lock_init(&drv->client.cache_lock); 1010 INIT_LIST_HEAD(&drv->client.cache); 1011 INIT_LIST_HEAD(&drv->client.batch_cache); 1012 1013 dev_set_drvdata(&pdev->dev, drv); 1014 1015 return devm_of_platform_populate(&pdev->dev); 1016 } 1017 1018 static const struct of_device_id rpmh_drv_match[] = { 1019 { .compatible = "qcom,rpmh-rsc", }, 1020 { } 1021 }; 1022 MODULE_DEVICE_TABLE(of, rpmh_drv_match); 1023 1024 static struct platform_driver rpmh_driver = { 1025 .probe = rpmh_rsc_probe, 1026 .driver = { 1027 .name = "rpmh", 1028 .of_match_table = rpmh_drv_match, 1029 .suppress_bind_attrs = true, 1030 }, 1031 }; 1032 1033 static int __init rpmh_driver_init(void) 1034 { 1035 return platform_driver_register(&rpmh_driver); 1036 } 1037 arch_initcall(rpmh_driver_init); 1038 1039 MODULE_DESCRIPTION("Qualcomm Technologies, Inc. RPMh Driver"); 1040 MODULE_LICENSE("GPL v2"); 1041