1 /* Broadcom FlexRM Mailbox Driver 2 * 3 * Copyright (C) 2017 Broadcom 4 * 5 * This program is free software; you can redistribute it and/or modify 6 * it under the terms of the GNU General Public License version 2 as 7 * published by the Free Software Foundation. 8 * 9 * Each Broadcom FlexSparx4 offload engine is implemented as an 10 * extension to Broadcom FlexRM ring manager. The FlexRM ring 11 * manager provides a set of rings which can be used to submit 12 * work to a FlexSparx4 offload engine. 13 * 14 * This driver creates a mailbox controller using a set of FlexRM 15 * rings where each mailbox channel represents a separate FlexRM ring. 16 */ 17 18 #include <asm/barrier.h> 19 #include <asm/byteorder.h> 20 #include <linux/atomic.h> 21 #include <linux/bitmap.h> 22 #include <linux/debugfs.h> 23 #include <linux/delay.h> 24 #include <linux/device.h> 25 #include <linux/dma-mapping.h> 26 #include <linux/dmapool.h> 27 #include <linux/err.h> 28 #include <linux/interrupt.h> 29 #include <linux/kernel.h> 30 #include <linux/mailbox_controller.h> 31 #include <linux/mailbox_client.h> 32 #include <linux/mailbox/brcm-message.h> 33 #include <linux/module.h> 34 #include <linux/msi.h> 35 #include <linux/of_address.h> 36 #include <linux/of_irq.h> 37 #include <linux/platform_device.h> 38 #include <linux/spinlock.h> 39 40 /* ====== FlexRM register defines ===== */ 41 42 /* FlexRM configuration */ 43 #define RING_REGS_SIZE 0x10000 44 #define RING_DESC_SIZE 8 45 #define RING_DESC_INDEX(offset) \ 46 ((offset) / RING_DESC_SIZE) 47 #define RING_DESC_OFFSET(index) \ 48 ((index) * RING_DESC_SIZE) 49 #define RING_MAX_REQ_COUNT 1024 50 #define RING_BD_ALIGN_ORDER 12 51 #define RING_BD_ALIGN_CHECK(addr) \ 52 (!((addr) & ((0x1 << RING_BD_ALIGN_ORDER) - 1))) 53 #define RING_BD_TOGGLE_INVALID(offset) \ 54 (((offset) >> RING_BD_ALIGN_ORDER) & 0x1) 55 #define RING_BD_TOGGLE_VALID(offset) \ 56 (!RING_BD_TOGGLE_INVALID(offset)) 57 #define RING_BD_DESC_PER_REQ 32 58 #define RING_BD_DESC_COUNT \ 59 (RING_MAX_REQ_COUNT * RING_BD_DESC_PER_REQ) 60 #define RING_BD_SIZE \ 61 (RING_BD_DESC_COUNT * RING_DESC_SIZE) 62 #define RING_CMPL_ALIGN_ORDER 13 63 #define RING_CMPL_DESC_COUNT RING_MAX_REQ_COUNT 64 #define RING_CMPL_SIZE \ 65 (RING_CMPL_DESC_COUNT * RING_DESC_SIZE) 66 #define RING_VER_MAGIC 0x76303031 67 68 /* Per-Ring register offsets */ 69 #define RING_VER 0x000 70 #define RING_BD_START_ADDR 0x004 71 #define RING_BD_READ_PTR 0x008 72 #define RING_BD_WRITE_PTR 0x00c 73 #define RING_BD_READ_PTR_DDR_LS 0x010 74 #define RING_BD_READ_PTR_DDR_MS 0x014 75 #define RING_CMPL_START_ADDR 0x018 76 #define RING_CMPL_WRITE_PTR 0x01c 77 #define RING_NUM_REQ_RECV_LS 0x020 78 #define RING_NUM_REQ_RECV_MS 0x024 79 #define RING_NUM_REQ_TRANS_LS 0x028 80 #define RING_NUM_REQ_TRANS_MS 0x02c 81 #define RING_NUM_REQ_OUTSTAND 0x030 82 #define RING_CONTROL 0x034 83 #define RING_FLUSH_DONE 0x038 84 #define RING_MSI_ADDR_LS 0x03c 85 #define RING_MSI_ADDR_MS 0x040 86 #define RING_MSI_CONTROL 0x048 87 #define RING_BD_READ_PTR_DDR_CONTROL 0x04c 88 #define RING_MSI_DATA_VALUE 0x064 89 90 /* Register RING_BD_START_ADDR fields */ 91 #define BD_LAST_UPDATE_HW_SHIFT 28 92 #define BD_LAST_UPDATE_HW_MASK 0x1 93 #define BD_START_ADDR_VALUE(pa) \ 94 ((u32)((((dma_addr_t)(pa)) >> RING_BD_ALIGN_ORDER) & 0x0fffffff)) 95 #define BD_START_ADDR_DECODE(val) \ 96 ((dma_addr_t)((val) & 0x0fffffff) << RING_BD_ALIGN_ORDER) 97 98 /* Register RING_CMPL_START_ADDR fields */ 99 #define CMPL_START_ADDR_VALUE(pa) \ 100 ((u32)((((u64)(pa)) >> RING_CMPL_ALIGN_ORDER) & 0x07ffffff)) 101 102 /* Register RING_CONTROL fields */ 103 #define CONTROL_MASK_DISABLE_CONTROL 12 104 #define CONTROL_FLUSH_SHIFT 5 105 #define CONTROL_ACTIVE_SHIFT 4 106 #define CONTROL_RATE_ADAPT_MASK 0xf 107 #define CONTROL_RATE_DYNAMIC 0x0 108 #define CONTROL_RATE_FAST 0x8 109 #define CONTROL_RATE_MEDIUM 0x9 110 #define CONTROL_RATE_SLOW 0xa 111 #define CONTROL_RATE_IDLE 0xb 112 113 /* Register RING_FLUSH_DONE fields */ 114 #define FLUSH_DONE_MASK 0x1 115 116 /* Register RING_MSI_CONTROL fields */ 117 #define MSI_TIMER_VAL_SHIFT 16 118 #define MSI_TIMER_VAL_MASK 0xffff 119 #define MSI_ENABLE_SHIFT 15 120 #define MSI_ENABLE_MASK 0x1 121 #define MSI_COUNT_SHIFT 0 122 #define MSI_COUNT_MASK 0x3ff 123 124 /* Register RING_BD_READ_PTR_DDR_CONTROL fields */ 125 #define BD_READ_PTR_DDR_TIMER_VAL_SHIFT 16 126 #define BD_READ_PTR_DDR_TIMER_VAL_MASK 0xffff 127 #define BD_READ_PTR_DDR_ENABLE_SHIFT 15 128 #define BD_READ_PTR_DDR_ENABLE_MASK 0x1 129 130 /* ====== FlexRM ring descriptor defines ===== */ 131 132 /* Completion descriptor format */ 133 #define CMPL_OPAQUE_SHIFT 0 134 #define CMPL_OPAQUE_MASK 0xffff 135 #define CMPL_ENGINE_STATUS_SHIFT 16 136 #define CMPL_ENGINE_STATUS_MASK 0xffff 137 #define CMPL_DME_STATUS_SHIFT 32 138 #define CMPL_DME_STATUS_MASK 0xffff 139 #define CMPL_RM_STATUS_SHIFT 48 140 #define CMPL_RM_STATUS_MASK 0xffff 141 142 /* Completion DME status code */ 143 #define DME_STATUS_MEM_COR_ERR BIT(0) 144 #define DME_STATUS_MEM_UCOR_ERR BIT(1) 145 #define DME_STATUS_FIFO_UNDERFLOW BIT(2) 146 #define DME_STATUS_FIFO_OVERFLOW BIT(3) 147 #define DME_STATUS_RRESP_ERR BIT(4) 148 #define DME_STATUS_BRESP_ERR BIT(5) 149 #define DME_STATUS_ERROR_MASK (DME_STATUS_MEM_COR_ERR | \ 150 DME_STATUS_MEM_UCOR_ERR | \ 151 DME_STATUS_FIFO_UNDERFLOW | \ 152 DME_STATUS_FIFO_OVERFLOW | \ 153 DME_STATUS_RRESP_ERR | \ 154 DME_STATUS_BRESP_ERR) 155 156 /* Completion RM status code */ 157 #define RM_STATUS_CODE_SHIFT 0 158 #define RM_STATUS_CODE_MASK 0x3ff 159 #define RM_STATUS_CODE_GOOD 0x0 160 #define RM_STATUS_CODE_AE_TIMEOUT 0x3ff 161 162 /* General descriptor format */ 163 #define DESC_TYPE_SHIFT 60 164 #define DESC_TYPE_MASK 0xf 165 #define DESC_PAYLOAD_SHIFT 0 166 #define DESC_PAYLOAD_MASK 0x0fffffffffffffff 167 168 /* Null descriptor format */ 169 #define NULL_TYPE 0 170 #define NULL_TOGGLE_SHIFT 58 171 #define NULL_TOGGLE_MASK 0x1 172 173 /* Header descriptor format */ 174 #define HEADER_TYPE 1 175 #define HEADER_TOGGLE_SHIFT 58 176 #define HEADER_TOGGLE_MASK 0x1 177 #define HEADER_ENDPKT_SHIFT 57 178 #define HEADER_ENDPKT_MASK 0x1 179 #define HEADER_STARTPKT_SHIFT 56 180 #define HEADER_STARTPKT_MASK 0x1 181 #define HEADER_BDCOUNT_SHIFT 36 182 #define HEADER_BDCOUNT_MASK 0x1f 183 #define HEADER_BDCOUNT_MAX HEADER_BDCOUNT_MASK 184 #define HEADER_FLAGS_SHIFT 16 185 #define HEADER_FLAGS_MASK 0xffff 186 #define HEADER_OPAQUE_SHIFT 0 187 #define HEADER_OPAQUE_MASK 0xffff 188 189 /* Source (SRC) descriptor format */ 190 #define SRC_TYPE 2 191 #define SRC_LENGTH_SHIFT 44 192 #define SRC_LENGTH_MASK 0xffff 193 #define SRC_ADDR_SHIFT 0 194 #define SRC_ADDR_MASK 0x00000fffffffffff 195 196 /* Destination (DST) descriptor format */ 197 #define DST_TYPE 3 198 #define DST_LENGTH_SHIFT 44 199 #define DST_LENGTH_MASK 0xffff 200 #define DST_ADDR_SHIFT 0 201 #define DST_ADDR_MASK 0x00000fffffffffff 202 203 /* Immediate (IMM) descriptor format */ 204 #define IMM_TYPE 4 205 #define IMM_DATA_SHIFT 0 206 #define IMM_DATA_MASK 0x0fffffffffffffff 207 208 /* Next pointer (NPTR) descriptor format */ 209 #define NPTR_TYPE 5 210 #define NPTR_TOGGLE_SHIFT 58 211 #define NPTR_TOGGLE_MASK 0x1 212 #define NPTR_ADDR_SHIFT 0 213 #define NPTR_ADDR_MASK 0x00000fffffffffff 214 215 /* Mega source (MSRC) descriptor format */ 216 #define MSRC_TYPE 6 217 #define MSRC_LENGTH_SHIFT 44 218 #define MSRC_LENGTH_MASK 0xffff 219 #define MSRC_ADDR_SHIFT 0 220 #define MSRC_ADDR_MASK 0x00000fffffffffff 221 222 /* Mega destination (MDST) descriptor format */ 223 #define MDST_TYPE 7 224 #define MDST_LENGTH_SHIFT 44 225 #define MDST_LENGTH_MASK 0xffff 226 #define MDST_ADDR_SHIFT 0 227 #define MDST_ADDR_MASK 0x00000fffffffffff 228 229 /* Source with tlast (SRCT) descriptor format */ 230 #define SRCT_TYPE 8 231 #define SRCT_LENGTH_SHIFT 44 232 #define SRCT_LENGTH_MASK 0xffff 233 #define SRCT_ADDR_SHIFT 0 234 #define SRCT_ADDR_MASK 0x00000fffffffffff 235 236 /* Destination with tlast (DSTT) descriptor format */ 237 #define DSTT_TYPE 9 238 #define DSTT_LENGTH_SHIFT 44 239 #define DSTT_LENGTH_MASK 0xffff 240 #define DSTT_ADDR_SHIFT 0 241 #define DSTT_ADDR_MASK 0x00000fffffffffff 242 243 /* Immediate with tlast (IMMT) descriptor format */ 244 #define IMMT_TYPE 10 245 #define IMMT_DATA_SHIFT 0 246 #define IMMT_DATA_MASK 0x0fffffffffffffff 247 248 /* Descriptor helper macros */ 249 #define DESC_DEC(_d, _s, _m) (((_d) >> (_s)) & (_m)) 250 #define DESC_ENC(_d, _v, _s, _m) \ 251 do { \ 252 (_d) &= ~((u64)(_m) << (_s)); \ 253 (_d) |= (((u64)(_v) & (_m)) << (_s)); \ 254 } while (0) 255 256 /* ====== FlexRM data structures ===== */ 257 258 struct flexrm_ring { 259 /* Unprotected members */ 260 int num; 261 struct flexrm_mbox *mbox; 262 void __iomem *regs; 263 bool irq_requested; 264 unsigned int irq; 265 cpumask_t irq_aff_hint; 266 unsigned int msi_timer_val; 267 unsigned int msi_count_threshold; 268 struct brcm_message *requests[RING_MAX_REQ_COUNT]; 269 void *bd_base; 270 dma_addr_t bd_dma_base; 271 u32 bd_write_offset; 272 void *cmpl_base; 273 dma_addr_t cmpl_dma_base; 274 /* Atomic stats */ 275 atomic_t msg_send_count; 276 atomic_t msg_cmpl_count; 277 /* Protected members */ 278 spinlock_t lock; 279 DECLARE_BITMAP(requests_bmap, RING_MAX_REQ_COUNT); 280 u32 cmpl_read_offset; 281 }; 282 283 struct flexrm_mbox { 284 struct device *dev; 285 void __iomem *regs; 286 u32 num_rings; 287 struct flexrm_ring *rings; 288 struct dma_pool *bd_pool; 289 struct dma_pool *cmpl_pool; 290 struct dentry *root; 291 struct dentry *config; 292 struct dentry *stats; 293 struct mbox_controller controller; 294 }; 295 296 /* ====== FlexRM ring descriptor helper routines ===== */ 297 298 static u64 flexrm_read_desc(void *desc_ptr) 299 { 300 return le64_to_cpu(*((u64 *)desc_ptr)); 301 } 302 303 static void flexrm_write_desc(void *desc_ptr, u64 desc) 304 { 305 *((u64 *)desc_ptr) = cpu_to_le64(desc); 306 } 307 308 static u32 flexrm_cmpl_desc_to_reqid(u64 cmpl_desc) 309 { 310 return (u32)(cmpl_desc & CMPL_OPAQUE_MASK); 311 } 312 313 static int flexrm_cmpl_desc_to_error(u64 cmpl_desc) 314 { 315 u32 status; 316 317 status = DESC_DEC(cmpl_desc, CMPL_DME_STATUS_SHIFT, 318 CMPL_DME_STATUS_MASK); 319 if (status & DME_STATUS_ERROR_MASK) 320 return -EIO; 321 322 status = DESC_DEC(cmpl_desc, CMPL_RM_STATUS_SHIFT, 323 CMPL_RM_STATUS_MASK); 324 status &= RM_STATUS_CODE_MASK; 325 if (status == RM_STATUS_CODE_AE_TIMEOUT) 326 return -ETIMEDOUT; 327 328 return 0; 329 } 330 331 static bool flexrm_is_next_table_desc(void *desc_ptr) 332 { 333 u64 desc = flexrm_read_desc(desc_ptr); 334 u32 type = DESC_DEC(desc, DESC_TYPE_SHIFT, DESC_TYPE_MASK); 335 336 return (type == NPTR_TYPE) ? true : false; 337 } 338 339 static u64 flexrm_next_table_desc(u32 toggle, dma_addr_t next_addr) 340 { 341 u64 desc = 0; 342 343 DESC_ENC(desc, NPTR_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK); 344 DESC_ENC(desc, toggle, NPTR_TOGGLE_SHIFT, NPTR_TOGGLE_MASK); 345 DESC_ENC(desc, next_addr, NPTR_ADDR_SHIFT, NPTR_ADDR_MASK); 346 347 return desc; 348 } 349 350 static u64 flexrm_null_desc(u32 toggle) 351 { 352 u64 desc = 0; 353 354 DESC_ENC(desc, NULL_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK); 355 DESC_ENC(desc, toggle, NULL_TOGGLE_SHIFT, NULL_TOGGLE_MASK); 356 357 return desc; 358 } 359 360 static u32 flexrm_estimate_header_desc_count(u32 nhcnt) 361 { 362 u32 hcnt = nhcnt / HEADER_BDCOUNT_MAX; 363 364 if (!(nhcnt % HEADER_BDCOUNT_MAX)) 365 hcnt += 1; 366 367 return hcnt; 368 } 369 370 static void flexrm_flip_header_toogle(void *desc_ptr) 371 { 372 u64 desc = flexrm_read_desc(desc_ptr); 373 374 if (desc & ((u64)0x1 << HEADER_TOGGLE_SHIFT)) 375 desc &= ~((u64)0x1 << HEADER_TOGGLE_SHIFT); 376 else 377 desc |= ((u64)0x1 << HEADER_TOGGLE_SHIFT); 378 379 flexrm_write_desc(desc_ptr, desc); 380 } 381 382 static u64 flexrm_header_desc(u32 toggle, u32 startpkt, u32 endpkt, 383 u32 bdcount, u32 flags, u32 opaque) 384 { 385 u64 desc = 0; 386 387 DESC_ENC(desc, HEADER_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK); 388 DESC_ENC(desc, toggle, HEADER_TOGGLE_SHIFT, HEADER_TOGGLE_MASK); 389 DESC_ENC(desc, startpkt, HEADER_STARTPKT_SHIFT, HEADER_STARTPKT_MASK); 390 DESC_ENC(desc, endpkt, HEADER_ENDPKT_SHIFT, HEADER_ENDPKT_MASK); 391 DESC_ENC(desc, bdcount, HEADER_BDCOUNT_SHIFT, HEADER_BDCOUNT_MASK); 392 DESC_ENC(desc, flags, HEADER_FLAGS_SHIFT, HEADER_FLAGS_MASK); 393 DESC_ENC(desc, opaque, HEADER_OPAQUE_SHIFT, HEADER_OPAQUE_MASK); 394 395 return desc; 396 } 397 398 static void flexrm_enqueue_desc(u32 nhpos, u32 nhcnt, u32 reqid, 399 u64 desc, void **desc_ptr, u32 *toggle, 400 void *start_desc, void *end_desc) 401 { 402 u64 d; 403 u32 nhavail, _toggle, _startpkt, _endpkt, _bdcount; 404 405 /* Sanity check */ 406 if (nhcnt <= nhpos) 407 return; 408 409 /* 410 * Each request or packet start with a HEADER descriptor followed 411 * by one or more non-HEADER descriptors (SRC, SRCT, MSRC, DST, 412 * DSTT, MDST, IMM, and IMMT). The number of non-HEADER descriptors 413 * following a HEADER descriptor is represented by BDCOUNT field 414 * of HEADER descriptor. The max value of BDCOUNT field is 31 which 415 * means we can only have 31 non-HEADER descriptors following one 416 * HEADER descriptor. 417 * 418 * In general use, number of non-HEADER descriptors can easily go 419 * beyond 31. To tackle this situation, we have packet (or request) 420 * extenstion bits (STARTPKT and ENDPKT) in the HEADER descriptor. 421 * 422 * To use packet extension, the first HEADER descriptor of request 423 * (or packet) will have STARTPKT=1 and ENDPKT=0. The intermediate 424 * HEADER descriptors will have STARTPKT=0 and ENDPKT=0. The last 425 * HEADER descriptor will have STARTPKT=0 and ENDPKT=1. Also, the 426 * TOGGLE bit of the first HEADER will be set to invalid state to 427 * ensure that FlexRM does not start fetching descriptors till all 428 * descriptors are enqueued. The user of this function will flip 429 * the TOGGLE bit of first HEADER after all descriptors are 430 * enqueued. 431 */ 432 433 if ((nhpos % HEADER_BDCOUNT_MAX == 0) && (nhcnt - nhpos)) { 434 /* Prepare the header descriptor */ 435 nhavail = (nhcnt - nhpos); 436 _toggle = (nhpos == 0) ? !(*toggle) : (*toggle); 437 _startpkt = (nhpos == 0) ? 0x1 : 0x0; 438 _endpkt = (nhavail <= HEADER_BDCOUNT_MAX) ? 0x1 : 0x0; 439 _bdcount = (nhavail <= HEADER_BDCOUNT_MAX) ? 440 nhavail : HEADER_BDCOUNT_MAX; 441 if (nhavail <= HEADER_BDCOUNT_MAX) 442 _bdcount = nhavail; 443 else 444 _bdcount = HEADER_BDCOUNT_MAX; 445 d = flexrm_header_desc(_toggle, _startpkt, _endpkt, 446 _bdcount, 0x0, reqid); 447 448 /* Write header descriptor */ 449 flexrm_write_desc(*desc_ptr, d); 450 451 /* Point to next descriptor */ 452 *desc_ptr += sizeof(desc); 453 if (*desc_ptr == end_desc) 454 *desc_ptr = start_desc; 455 456 /* Skip next pointer descriptors */ 457 while (flexrm_is_next_table_desc(*desc_ptr)) { 458 *toggle = (*toggle) ? 0 : 1; 459 *desc_ptr += sizeof(desc); 460 if (*desc_ptr == end_desc) 461 *desc_ptr = start_desc; 462 } 463 } 464 465 /* Write desired descriptor */ 466 flexrm_write_desc(*desc_ptr, desc); 467 468 /* Point to next descriptor */ 469 *desc_ptr += sizeof(desc); 470 if (*desc_ptr == end_desc) 471 *desc_ptr = start_desc; 472 473 /* Skip next pointer descriptors */ 474 while (flexrm_is_next_table_desc(*desc_ptr)) { 475 *toggle = (*toggle) ? 0 : 1; 476 *desc_ptr += sizeof(desc); 477 if (*desc_ptr == end_desc) 478 *desc_ptr = start_desc; 479 } 480 } 481 482 static u64 flexrm_src_desc(dma_addr_t addr, unsigned int length) 483 { 484 u64 desc = 0; 485 486 DESC_ENC(desc, SRC_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK); 487 DESC_ENC(desc, length, SRC_LENGTH_SHIFT, SRC_LENGTH_MASK); 488 DESC_ENC(desc, addr, SRC_ADDR_SHIFT, SRC_ADDR_MASK); 489 490 return desc; 491 } 492 493 static u64 flexrm_msrc_desc(dma_addr_t addr, unsigned int length_div_16) 494 { 495 u64 desc = 0; 496 497 DESC_ENC(desc, MSRC_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK); 498 DESC_ENC(desc, length_div_16, MSRC_LENGTH_SHIFT, MSRC_LENGTH_MASK); 499 DESC_ENC(desc, addr, MSRC_ADDR_SHIFT, MSRC_ADDR_MASK); 500 501 return desc; 502 } 503 504 static u64 flexrm_dst_desc(dma_addr_t addr, unsigned int length) 505 { 506 u64 desc = 0; 507 508 DESC_ENC(desc, DST_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK); 509 DESC_ENC(desc, length, DST_LENGTH_SHIFT, DST_LENGTH_MASK); 510 DESC_ENC(desc, addr, DST_ADDR_SHIFT, DST_ADDR_MASK); 511 512 return desc; 513 } 514 515 static u64 flexrm_mdst_desc(dma_addr_t addr, unsigned int length_div_16) 516 { 517 u64 desc = 0; 518 519 DESC_ENC(desc, MDST_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK); 520 DESC_ENC(desc, length_div_16, MDST_LENGTH_SHIFT, MDST_LENGTH_MASK); 521 DESC_ENC(desc, addr, MDST_ADDR_SHIFT, MDST_ADDR_MASK); 522 523 return desc; 524 } 525 526 static u64 flexrm_imm_desc(u64 data) 527 { 528 u64 desc = 0; 529 530 DESC_ENC(desc, IMM_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK); 531 DESC_ENC(desc, data, IMM_DATA_SHIFT, IMM_DATA_MASK); 532 533 return desc; 534 } 535 536 static u64 flexrm_srct_desc(dma_addr_t addr, unsigned int length) 537 { 538 u64 desc = 0; 539 540 DESC_ENC(desc, SRCT_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK); 541 DESC_ENC(desc, length, SRCT_LENGTH_SHIFT, SRCT_LENGTH_MASK); 542 DESC_ENC(desc, addr, SRCT_ADDR_SHIFT, SRCT_ADDR_MASK); 543 544 return desc; 545 } 546 547 static u64 flexrm_dstt_desc(dma_addr_t addr, unsigned int length) 548 { 549 u64 desc = 0; 550 551 DESC_ENC(desc, DSTT_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK); 552 DESC_ENC(desc, length, DSTT_LENGTH_SHIFT, DSTT_LENGTH_MASK); 553 DESC_ENC(desc, addr, DSTT_ADDR_SHIFT, DSTT_ADDR_MASK); 554 555 return desc; 556 } 557 558 static u64 flexrm_immt_desc(u64 data) 559 { 560 u64 desc = 0; 561 562 DESC_ENC(desc, IMMT_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK); 563 DESC_ENC(desc, data, IMMT_DATA_SHIFT, IMMT_DATA_MASK); 564 565 return desc; 566 } 567 568 static bool flexrm_spu_sanity_check(struct brcm_message *msg) 569 { 570 struct scatterlist *sg; 571 572 if (!msg->spu.src || !msg->spu.dst) 573 return false; 574 for (sg = msg->spu.src; sg; sg = sg_next(sg)) { 575 if (sg->length & 0xf) { 576 if (sg->length > SRC_LENGTH_MASK) 577 return false; 578 } else { 579 if (sg->length > (MSRC_LENGTH_MASK * 16)) 580 return false; 581 } 582 } 583 for (sg = msg->spu.dst; sg; sg = sg_next(sg)) { 584 if (sg->length & 0xf) { 585 if (sg->length > DST_LENGTH_MASK) 586 return false; 587 } else { 588 if (sg->length > (MDST_LENGTH_MASK * 16)) 589 return false; 590 } 591 } 592 593 return true; 594 } 595 596 static u32 flexrm_spu_estimate_nonheader_desc_count(struct brcm_message *msg) 597 { 598 u32 cnt = 0; 599 unsigned int dst_target = 0; 600 struct scatterlist *src_sg = msg->spu.src, *dst_sg = msg->spu.dst; 601 602 while (src_sg || dst_sg) { 603 if (src_sg) { 604 cnt++; 605 dst_target = src_sg->length; 606 src_sg = sg_next(src_sg); 607 } else 608 dst_target = UINT_MAX; 609 610 while (dst_target && dst_sg) { 611 cnt++; 612 if (dst_sg->length < dst_target) 613 dst_target -= dst_sg->length; 614 else 615 dst_target = 0; 616 dst_sg = sg_next(dst_sg); 617 } 618 } 619 620 return cnt; 621 } 622 623 static int flexrm_spu_dma_map(struct device *dev, struct brcm_message *msg) 624 { 625 int rc; 626 627 rc = dma_map_sg(dev, msg->spu.src, sg_nents(msg->spu.src), 628 DMA_TO_DEVICE); 629 if (rc < 0) 630 return rc; 631 632 rc = dma_map_sg(dev, msg->spu.dst, sg_nents(msg->spu.dst), 633 DMA_FROM_DEVICE); 634 if (rc < 0) { 635 dma_unmap_sg(dev, msg->spu.src, sg_nents(msg->spu.src), 636 DMA_TO_DEVICE); 637 return rc; 638 } 639 640 return 0; 641 } 642 643 static void flexrm_spu_dma_unmap(struct device *dev, struct brcm_message *msg) 644 { 645 dma_unmap_sg(dev, msg->spu.dst, sg_nents(msg->spu.dst), 646 DMA_FROM_DEVICE); 647 dma_unmap_sg(dev, msg->spu.src, sg_nents(msg->spu.src), 648 DMA_TO_DEVICE); 649 } 650 651 static void *flexrm_spu_write_descs(struct brcm_message *msg, u32 nhcnt, 652 u32 reqid, void *desc_ptr, u32 toggle, 653 void *start_desc, void *end_desc) 654 { 655 u64 d; 656 u32 nhpos = 0; 657 void *orig_desc_ptr = desc_ptr; 658 unsigned int dst_target = 0; 659 struct scatterlist *src_sg = msg->spu.src, *dst_sg = msg->spu.dst; 660 661 while (src_sg || dst_sg) { 662 if (src_sg) { 663 if (sg_dma_len(src_sg) & 0xf) 664 d = flexrm_src_desc(sg_dma_address(src_sg), 665 sg_dma_len(src_sg)); 666 else 667 d = flexrm_msrc_desc(sg_dma_address(src_sg), 668 sg_dma_len(src_sg)/16); 669 flexrm_enqueue_desc(nhpos, nhcnt, reqid, 670 d, &desc_ptr, &toggle, 671 start_desc, end_desc); 672 nhpos++; 673 dst_target = sg_dma_len(src_sg); 674 src_sg = sg_next(src_sg); 675 } else 676 dst_target = UINT_MAX; 677 678 while (dst_target && dst_sg) { 679 if (sg_dma_len(dst_sg) & 0xf) 680 d = flexrm_dst_desc(sg_dma_address(dst_sg), 681 sg_dma_len(dst_sg)); 682 else 683 d = flexrm_mdst_desc(sg_dma_address(dst_sg), 684 sg_dma_len(dst_sg)/16); 685 flexrm_enqueue_desc(nhpos, nhcnt, reqid, 686 d, &desc_ptr, &toggle, 687 start_desc, end_desc); 688 nhpos++; 689 if (sg_dma_len(dst_sg) < dst_target) 690 dst_target -= sg_dma_len(dst_sg); 691 else 692 dst_target = 0; 693 dst_sg = sg_next(dst_sg); 694 } 695 } 696 697 /* Null descriptor with invalid toggle bit */ 698 flexrm_write_desc(desc_ptr, flexrm_null_desc(!toggle)); 699 700 /* Ensure that descriptors have been written to memory */ 701 wmb(); 702 703 /* Flip toggle bit in header */ 704 flexrm_flip_header_toogle(orig_desc_ptr); 705 706 return desc_ptr; 707 } 708 709 static bool flexrm_sba_sanity_check(struct brcm_message *msg) 710 { 711 u32 i; 712 713 if (!msg->sba.cmds || !msg->sba.cmds_count) 714 return false; 715 716 for (i = 0; i < msg->sba.cmds_count; i++) { 717 if (((msg->sba.cmds[i].flags & BRCM_SBA_CMD_TYPE_B) || 718 (msg->sba.cmds[i].flags & BRCM_SBA_CMD_TYPE_C)) && 719 (msg->sba.cmds[i].flags & BRCM_SBA_CMD_HAS_OUTPUT)) 720 return false; 721 if ((msg->sba.cmds[i].flags & BRCM_SBA_CMD_TYPE_B) && 722 (msg->sba.cmds[i].data_len > SRCT_LENGTH_MASK)) 723 return false; 724 if ((msg->sba.cmds[i].flags & BRCM_SBA_CMD_TYPE_C) && 725 (msg->sba.cmds[i].data_len > SRCT_LENGTH_MASK)) 726 return false; 727 if ((msg->sba.cmds[i].flags & BRCM_SBA_CMD_HAS_RESP) && 728 (msg->sba.cmds[i].resp_len > DSTT_LENGTH_MASK)) 729 return false; 730 if ((msg->sba.cmds[i].flags & BRCM_SBA_CMD_HAS_OUTPUT) && 731 (msg->sba.cmds[i].data_len > DSTT_LENGTH_MASK)) 732 return false; 733 } 734 735 return true; 736 } 737 738 static u32 flexrm_sba_estimate_nonheader_desc_count(struct brcm_message *msg) 739 { 740 u32 i, cnt; 741 742 cnt = 0; 743 for (i = 0; i < msg->sba.cmds_count; i++) { 744 cnt++; 745 746 if ((msg->sba.cmds[i].flags & BRCM_SBA_CMD_TYPE_B) || 747 (msg->sba.cmds[i].flags & BRCM_SBA_CMD_TYPE_C)) 748 cnt++; 749 750 if (msg->sba.cmds[i].flags & BRCM_SBA_CMD_HAS_RESP) 751 cnt++; 752 753 if (msg->sba.cmds[i].flags & BRCM_SBA_CMD_HAS_OUTPUT) 754 cnt++; 755 } 756 757 return cnt; 758 } 759 760 static void *flexrm_sba_write_descs(struct brcm_message *msg, u32 nhcnt, 761 u32 reqid, void *desc_ptr, u32 toggle, 762 void *start_desc, void *end_desc) 763 { 764 u64 d; 765 u32 i, nhpos = 0; 766 struct brcm_sba_command *c; 767 void *orig_desc_ptr = desc_ptr; 768 769 /* Convert SBA commands into descriptors */ 770 for (i = 0; i < msg->sba.cmds_count; i++) { 771 c = &msg->sba.cmds[i]; 772 773 if ((c->flags & BRCM_SBA_CMD_HAS_RESP) && 774 (c->flags & BRCM_SBA_CMD_HAS_OUTPUT)) { 775 /* Destination response descriptor */ 776 d = flexrm_dst_desc(c->resp, c->resp_len); 777 flexrm_enqueue_desc(nhpos, nhcnt, reqid, 778 d, &desc_ptr, &toggle, 779 start_desc, end_desc); 780 nhpos++; 781 } else if (c->flags & BRCM_SBA_CMD_HAS_RESP) { 782 /* Destination response with tlast descriptor */ 783 d = flexrm_dstt_desc(c->resp, c->resp_len); 784 flexrm_enqueue_desc(nhpos, nhcnt, reqid, 785 d, &desc_ptr, &toggle, 786 start_desc, end_desc); 787 nhpos++; 788 } 789 790 if (c->flags & BRCM_SBA_CMD_HAS_OUTPUT) { 791 /* Destination with tlast descriptor */ 792 d = flexrm_dstt_desc(c->data, c->data_len); 793 flexrm_enqueue_desc(nhpos, nhcnt, reqid, 794 d, &desc_ptr, &toggle, 795 start_desc, end_desc); 796 nhpos++; 797 } 798 799 if (c->flags & BRCM_SBA_CMD_TYPE_B) { 800 /* Command as immediate descriptor */ 801 d = flexrm_imm_desc(c->cmd); 802 flexrm_enqueue_desc(nhpos, nhcnt, reqid, 803 d, &desc_ptr, &toggle, 804 start_desc, end_desc); 805 nhpos++; 806 } else { 807 /* Command as immediate descriptor with tlast */ 808 d = flexrm_immt_desc(c->cmd); 809 flexrm_enqueue_desc(nhpos, nhcnt, reqid, 810 d, &desc_ptr, &toggle, 811 start_desc, end_desc); 812 nhpos++; 813 } 814 815 if ((c->flags & BRCM_SBA_CMD_TYPE_B) || 816 (c->flags & BRCM_SBA_CMD_TYPE_C)) { 817 /* Source with tlast descriptor */ 818 d = flexrm_srct_desc(c->data, c->data_len); 819 flexrm_enqueue_desc(nhpos, nhcnt, reqid, 820 d, &desc_ptr, &toggle, 821 start_desc, end_desc); 822 nhpos++; 823 } 824 } 825 826 /* Null descriptor with invalid toggle bit */ 827 flexrm_write_desc(desc_ptr, flexrm_null_desc(!toggle)); 828 829 /* Ensure that descriptors have been written to memory */ 830 wmb(); 831 832 /* Flip toggle bit in header */ 833 flexrm_flip_header_toogle(orig_desc_ptr); 834 835 return desc_ptr; 836 } 837 838 static bool flexrm_sanity_check(struct brcm_message *msg) 839 { 840 if (!msg) 841 return false; 842 843 switch (msg->type) { 844 case BRCM_MESSAGE_SPU: 845 return flexrm_spu_sanity_check(msg); 846 case BRCM_MESSAGE_SBA: 847 return flexrm_sba_sanity_check(msg); 848 default: 849 return false; 850 }; 851 } 852 853 static u32 flexrm_estimate_nonheader_desc_count(struct brcm_message *msg) 854 { 855 if (!msg) 856 return 0; 857 858 switch (msg->type) { 859 case BRCM_MESSAGE_SPU: 860 return flexrm_spu_estimate_nonheader_desc_count(msg); 861 case BRCM_MESSAGE_SBA: 862 return flexrm_sba_estimate_nonheader_desc_count(msg); 863 default: 864 return 0; 865 }; 866 } 867 868 static int flexrm_dma_map(struct device *dev, struct brcm_message *msg) 869 { 870 if (!dev || !msg) 871 return -EINVAL; 872 873 switch (msg->type) { 874 case BRCM_MESSAGE_SPU: 875 return flexrm_spu_dma_map(dev, msg); 876 default: 877 break; 878 } 879 880 return 0; 881 } 882 883 static void flexrm_dma_unmap(struct device *dev, struct brcm_message *msg) 884 { 885 if (!dev || !msg) 886 return; 887 888 switch (msg->type) { 889 case BRCM_MESSAGE_SPU: 890 flexrm_spu_dma_unmap(dev, msg); 891 break; 892 default: 893 break; 894 } 895 } 896 897 static void *flexrm_write_descs(struct brcm_message *msg, u32 nhcnt, 898 u32 reqid, void *desc_ptr, u32 toggle, 899 void *start_desc, void *end_desc) 900 { 901 if (!msg || !desc_ptr || !start_desc || !end_desc) 902 return ERR_PTR(-ENOTSUPP); 903 904 if ((desc_ptr < start_desc) || (end_desc <= desc_ptr)) 905 return ERR_PTR(-ERANGE); 906 907 switch (msg->type) { 908 case BRCM_MESSAGE_SPU: 909 return flexrm_spu_write_descs(msg, nhcnt, reqid, 910 desc_ptr, toggle, 911 start_desc, end_desc); 912 case BRCM_MESSAGE_SBA: 913 return flexrm_sba_write_descs(msg, nhcnt, reqid, 914 desc_ptr, toggle, 915 start_desc, end_desc); 916 default: 917 return ERR_PTR(-ENOTSUPP); 918 }; 919 } 920 921 /* ====== FlexRM driver helper routines ===== */ 922 923 static void flexrm_write_config_in_seqfile(struct flexrm_mbox *mbox, 924 struct seq_file *file) 925 { 926 int i; 927 const char *state; 928 struct flexrm_ring *ring; 929 930 seq_printf(file, "%-5s %-9s %-18s %-10s %-18s %-10s\n", 931 "Ring#", "State", "BD_Addr", "BD_Size", 932 "Cmpl_Addr", "Cmpl_Size"); 933 934 for (i = 0; i < mbox->num_rings; i++) { 935 ring = &mbox->rings[i]; 936 if (readl(ring->regs + RING_CONTROL) & 937 BIT(CONTROL_ACTIVE_SHIFT)) 938 state = "active"; 939 else 940 state = "inactive"; 941 seq_printf(file, 942 "%-5d %-9s 0x%016llx 0x%08x 0x%016llx 0x%08x\n", 943 ring->num, state, 944 (unsigned long long)ring->bd_dma_base, 945 (u32)RING_BD_SIZE, 946 (unsigned long long)ring->cmpl_dma_base, 947 (u32)RING_CMPL_SIZE); 948 } 949 } 950 951 static void flexrm_write_stats_in_seqfile(struct flexrm_mbox *mbox, 952 struct seq_file *file) 953 { 954 int i; 955 u32 val, bd_read_offset; 956 struct flexrm_ring *ring; 957 958 seq_printf(file, "%-5s %-10s %-10s %-10s %-11s %-11s\n", 959 "Ring#", "BD_Read", "BD_Write", 960 "Cmpl_Read", "Submitted", "Completed"); 961 962 for (i = 0; i < mbox->num_rings; i++) { 963 ring = &mbox->rings[i]; 964 bd_read_offset = readl_relaxed(ring->regs + RING_BD_READ_PTR); 965 val = readl_relaxed(ring->regs + RING_BD_START_ADDR); 966 bd_read_offset *= RING_DESC_SIZE; 967 bd_read_offset += (u32)(BD_START_ADDR_DECODE(val) - 968 ring->bd_dma_base); 969 seq_printf(file, "%-5d 0x%08x 0x%08x 0x%08x %-11d %-11d\n", 970 ring->num, 971 (u32)bd_read_offset, 972 (u32)ring->bd_write_offset, 973 (u32)ring->cmpl_read_offset, 974 (u32)atomic_read(&ring->msg_send_count), 975 (u32)atomic_read(&ring->msg_cmpl_count)); 976 } 977 } 978 979 static int flexrm_new_request(struct flexrm_ring *ring, 980 struct brcm_message *batch_msg, 981 struct brcm_message *msg) 982 { 983 void *next; 984 unsigned long flags; 985 u32 val, count, nhcnt; 986 u32 read_offset, write_offset; 987 bool exit_cleanup = false; 988 int ret = 0, reqid; 989 990 /* Do sanity check on message */ 991 if (!flexrm_sanity_check(msg)) 992 return -EIO; 993 msg->error = 0; 994 995 /* If no requests possible then save data pointer and goto done. */ 996 spin_lock_irqsave(&ring->lock, flags); 997 reqid = bitmap_find_free_region(ring->requests_bmap, 998 RING_MAX_REQ_COUNT, 0); 999 spin_unlock_irqrestore(&ring->lock, flags); 1000 if (reqid < 0) 1001 return -ENOSPC; 1002 ring->requests[reqid] = msg; 1003 1004 /* Do DMA mappings for the message */ 1005 ret = flexrm_dma_map(ring->mbox->dev, msg); 1006 if (ret < 0) { 1007 ring->requests[reqid] = NULL; 1008 spin_lock_irqsave(&ring->lock, flags); 1009 bitmap_release_region(ring->requests_bmap, reqid, 0); 1010 spin_unlock_irqrestore(&ring->lock, flags); 1011 return ret; 1012 } 1013 1014 /* Determine current HW BD read offset */ 1015 read_offset = readl_relaxed(ring->regs + RING_BD_READ_PTR); 1016 val = readl_relaxed(ring->regs + RING_BD_START_ADDR); 1017 read_offset *= RING_DESC_SIZE; 1018 read_offset += (u32)(BD_START_ADDR_DECODE(val) - ring->bd_dma_base); 1019 1020 /* 1021 * Number required descriptors = number of non-header descriptors + 1022 * number of header descriptors + 1023 * 1x null descriptor 1024 */ 1025 nhcnt = flexrm_estimate_nonheader_desc_count(msg); 1026 count = flexrm_estimate_header_desc_count(nhcnt) + nhcnt + 1; 1027 1028 /* Check for available descriptor space. */ 1029 write_offset = ring->bd_write_offset; 1030 while (count) { 1031 if (!flexrm_is_next_table_desc(ring->bd_base + write_offset)) 1032 count--; 1033 write_offset += RING_DESC_SIZE; 1034 if (write_offset == RING_BD_SIZE) 1035 write_offset = 0x0; 1036 if (write_offset == read_offset) 1037 break; 1038 } 1039 if (count) { 1040 ret = -ENOSPC; 1041 exit_cleanup = true; 1042 goto exit; 1043 } 1044 1045 /* Write descriptors to ring */ 1046 next = flexrm_write_descs(msg, nhcnt, reqid, 1047 ring->bd_base + ring->bd_write_offset, 1048 RING_BD_TOGGLE_VALID(ring->bd_write_offset), 1049 ring->bd_base, ring->bd_base + RING_BD_SIZE); 1050 if (IS_ERR(next)) { 1051 ret = PTR_ERR(next); 1052 exit_cleanup = true; 1053 goto exit; 1054 } 1055 1056 /* Save ring BD write offset */ 1057 ring->bd_write_offset = (unsigned long)(next - ring->bd_base); 1058 1059 /* Increment number of messages sent */ 1060 atomic_inc_return(&ring->msg_send_count); 1061 1062 exit: 1063 /* Update error status in message */ 1064 msg->error = ret; 1065 1066 /* Cleanup if we failed */ 1067 if (exit_cleanup) { 1068 flexrm_dma_unmap(ring->mbox->dev, msg); 1069 ring->requests[reqid] = NULL; 1070 spin_lock_irqsave(&ring->lock, flags); 1071 bitmap_release_region(ring->requests_bmap, reqid, 0); 1072 spin_unlock_irqrestore(&ring->lock, flags); 1073 } 1074 1075 return ret; 1076 } 1077 1078 static int flexrm_process_completions(struct flexrm_ring *ring) 1079 { 1080 u64 desc; 1081 int err, count = 0; 1082 unsigned long flags; 1083 struct brcm_message *msg = NULL; 1084 u32 reqid, cmpl_read_offset, cmpl_write_offset; 1085 struct mbox_chan *chan = &ring->mbox->controller.chans[ring->num]; 1086 1087 spin_lock_irqsave(&ring->lock, flags); 1088 1089 /* 1090 * Get current completion read and write offset 1091 * 1092 * Note: We should read completion write pointer atleast once 1093 * after we get a MSI interrupt because HW maintains internal 1094 * MSI status which will allow next MSI interrupt only after 1095 * completion write pointer is read. 1096 */ 1097 cmpl_write_offset = readl_relaxed(ring->regs + RING_CMPL_WRITE_PTR); 1098 cmpl_write_offset *= RING_DESC_SIZE; 1099 cmpl_read_offset = ring->cmpl_read_offset; 1100 ring->cmpl_read_offset = cmpl_write_offset; 1101 1102 spin_unlock_irqrestore(&ring->lock, flags); 1103 1104 /* For each completed request notify mailbox clients */ 1105 reqid = 0; 1106 while (cmpl_read_offset != cmpl_write_offset) { 1107 /* Dequeue next completion descriptor */ 1108 desc = *((u64 *)(ring->cmpl_base + cmpl_read_offset)); 1109 1110 /* Next read offset */ 1111 cmpl_read_offset += RING_DESC_SIZE; 1112 if (cmpl_read_offset == RING_CMPL_SIZE) 1113 cmpl_read_offset = 0; 1114 1115 /* Decode error from completion descriptor */ 1116 err = flexrm_cmpl_desc_to_error(desc); 1117 if (err < 0) { 1118 dev_warn(ring->mbox->dev, 1119 "got completion desc=0x%lx with error %d", 1120 (unsigned long)desc, err); 1121 } 1122 1123 /* Determine request id from completion descriptor */ 1124 reqid = flexrm_cmpl_desc_to_reqid(desc); 1125 1126 /* Determine message pointer based on reqid */ 1127 msg = ring->requests[reqid]; 1128 if (!msg) { 1129 dev_warn(ring->mbox->dev, 1130 "null msg pointer for completion desc=0x%lx", 1131 (unsigned long)desc); 1132 continue; 1133 } 1134 1135 /* Release reqid for recycling */ 1136 ring->requests[reqid] = NULL; 1137 spin_lock_irqsave(&ring->lock, flags); 1138 bitmap_release_region(ring->requests_bmap, reqid, 0); 1139 spin_unlock_irqrestore(&ring->lock, flags); 1140 1141 /* Unmap DMA mappings */ 1142 flexrm_dma_unmap(ring->mbox->dev, msg); 1143 1144 /* Give-back message to mailbox client */ 1145 msg->error = err; 1146 mbox_chan_received_data(chan, msg); 1147 1148 /* Increment number of completions processed */ 1149 atomic_inc_return(&ring->msg_cmpl_count); 1150 count++; 1151 } 1152 1153 return count; 1154 } 1155 1156 /* ====== FlexRM Debugfs callbacks ====== */ 1157 1158 static int flexrm_debugfs_conf_show(struct seq_file *file, void *offset) 1159 { 1160 struct platform_device *pdev = to_platform_device(file->private); 1161 struct flexrm_mbox *mbox = platform_get_drvdata(pdev); 1162 1163 /* Write config in file */ 1164 flexrm_write_config_in_seqfile(mbox, file); 1165 1166 return 0; 1167 } 1168 1169 static int flexrm_debugfs_stats_show(struct seq_file *file, void *offset) 1170 { 1171 struct platform_device *pdev = to_platform_device(file->private); 1172 struct flexrm_mbox *mbox = platform_get_drvdata(pdev); 1173 1174 /* Write stats in file */ 1175 flexrm_write_stats_in_seqfile(mbox, file); 1176 1177 return 0; 1178 } 1179 1180 /* ====== FlexRM interrupt handler ===== */ 1181 1182 static irqreturn_t flexrm_irq_event(int irq, void *dev_id) 1183 { 1184 /* We only have MSI for completions so just wakeup IRQ thread */ 1185 /* Ring related errors will be informed via completion descriptors */ 1186 1187 return IRQ_WAKE_THREAD; 1188 } 1189 1190 static irqreturn_t flexrm_irq_thread(int irq, void *dev_id) 1191 { 1192 flexrm_process_completions(dev_id); 1193 1194 return IRQ_HANDLED; 1195 } 1196 1197 /* ====== FlexRM mailbox callbacks ===== */ 1198 1199 static int flexrm_send_data(struct mbox_chan *chan, void *data) 1200 { 1201 int i, rc; 1202 struct flexrm_ring *ring = chan->con_priv; 1203 struct brcm_message *msg = data; 1204 1205 if (msg->type == BRCM_MESSAGE_BATCH) { 1206 for (i = msg->batch.msgs_queued; 1207 i < msg->batch.msgs_count; i++) { 1208 rc = flexrm_new_request(ring, msg, 1209 &msg->batch.msgs[i]); 1210 if (rc) { 1211 msg->error = rc; 1212 return rc; 1213 } 1214 msg->batch.msgs_queued++; 1215 } 1216 return 0; 1217 } 1218 1219 return flexrm_new_request(ring, NULL, data); 1220 } 1221 1222 static bool flexrm_peek_data(struct mbox_chan *chan) 1223 { 1224 int cnt = flexrm_process_completions(chan->con_priv); 1225 1226 return (cnt > 0) ? true : false; 1227 } 1228 1229 static int flexrm_startup(struct mbox_chan *chan) 1230 { 1231 u64 d; 1232 u32 val, off; 1233 int ret = 0; 1234 dma_addr_t next_addr; 1235 struct flexrm_ring *ring = chan->con_priv; 1236 1237 /* Allocate BD memory */ 1238 ring->bd_base = dma_pool_alloc(ring->mbox->bd_pool, 1239 GFP_KERNEL, &ring->bd_dma_base); 1240 if (!ring->bd_base) { 1241 dev_err(ring->mbox->dev, "can't allocate BD memory\n"); 1242 ret = -ENOMEM; 1243 goto fail; 1244 } 1245 1246 /* Configure next table pointer entries in BD memory */ 1247 for (off = 0; off < RING_BD_SIZE; off += RING_DESC_SIZE) { 1248 next_addr = off + RING_DESC_SIZE; 1249 if (next_addr == RING_BD_SIZE) 1250 next_addr = 0; 1251 next_addr += ring->bd_dma_base; 1252 if (RING_BD_ALIGN_CHECK(next_addr)) 1253 d = flexrm_next_table_desc(RING_BD_TOGGLE_VALID(off), 1254 next_addr); 1255 else 1256 d = flexrm_null_desc(RING_BD_TOGGLE_INVALID(off)); 1257 flexrm_write_desc(ring->bd_base + off, d); 1258 } 1259 1260 /* Allocate completion memory */ 1261 ring->cmpl_base = dma_pool_alloc(ring->mbox->cmpl_pool, 1262 GFP_KERNEL, &ring->cmpl_dma_base); 1263 if (!ring->cmpl_base) { 1264 dev_err(ring->mbox->dev, "can't allocate completion memory\n"); 1265 ret = -ENOMEM; 1266 goto fail_free_bd_memory; 1267 } 1268 memset(ring->cmpl_base, 0, RING_CMPL_SIZE); 1269 1270 /* Request IRQ */ 1271 if (ring->irq == UINT_MAX) { 1272 dev_err(ring->mbox->dev, "ring IRQ not available\n"); 1273 ret = -ENODEV; 1274 goto fail_free_cmpl_memory; 1275 } 1276 ret = request_threaded_irq(ring->irq, 1277 flexrm_irq_event, 1278 flexrm_irq_thread, 1279 0, dev_name(ring->mbox->dev), ring); 1280 if (ret) { 1281 dev_err(ring->mbox->dev, "failed to request ring IRQ\n"); 1282 goto fail_free_cmpl_memory; 1283 } 1284 ring->irq_requested = true; 1285 1286 /* Set IRQ affinity hint */ 1287 ring->irq_aff_hint = CPU_MASK_NONE; 1288 val = ring->mbox->num_rings; 1289 val = (num_online_cpus() < val) ? val / num_online_cpus() : 1; 1290 cpumask_set_cpu((ring->num / val) % num_online_cpus(), 1291 &ring->irq_aff_hint); 1292 ret = irq_set_affinity_hint(ring->irq, &ring->irq_aff_hint); 1293 if (ret) { 1294 dev_err(ring->mbox->dev, "failed to set IRQ affinity hint\n"); 1295 goto fail_free_irq; 1296 } 1297 1298 /* Disable/inactivate ring */ 1299 writel_relaxed(0x0, ring->regs + RING_CONTROL); 1300 1301 /* Program BD start address */ 1302 val = BD_START_ADDR_VALUE(ring->bd_dma_base); 1303 writel_relaxed(val, ring->regs + RING_BD_START_ADDR); 1304 1305 /* BD write pointer will be same as HW write pointer */ 1306 ring->bd_write_offset = 1307 readl_relaxed(ring->regs + RING_BD_WRITE_PTR); 1308 ring->bd_write_offset *= RING_DESC_SIZE; 1309 1310 /* Program completion start address */ 1311 val = CMPL_START_ADDR_VALUE(ring->cmpl_dma_base); 1312 writel_relaxed(val, ring->regs + RING_CMPL_START_ADDR); 1313 1314 /* Completion read pointer will be same as HW write pointer */ 1315 ring->cmpl_read_offset = 1316 readl_relaxed(ring->regs + RING_CMPL_WRITE_PTR); 1317 ring->cmpl_read_offset *= RING_DESC_SIZE; 1318 1319 /* Read ring Tx, Rx, and Outstanding counts to clear */ 1320 readl_relaxed(ring->regs + RING_NUM_REQ_RECV_LS); 1321 readl_relaxed(ring->regs + RING_NUM_REQ_RECV_MS); 1322 readl_relaxed(ring->regs + RING_NUM_REQ_TRANS_LS); 1323 readl_relaxed(ring->regs + RING_NUM_REQ_TRANS_MS); 1324 readl_relaxed(ring->regs + RING_NUM_REQ_OUTSTAND); 1325 1326 /* Configure RING_MSI_CONTROL */ 1327 val = 0; 1328 val |= (ring->msi_timer_val << MSI_TIMER_VAL_SHIFT); 1329 val |= BIT(MSI_ENABLE_SHIFT); 1330 val |= (ring->msi_count_threshold & MSI_COUNT_MASK) << MSI_COUNT_SHIFT; 1331 writel_relaxed(val, ring->regs + RING_MSI_CONTROL); 1332 1333 /* Enable/activate ring */ 1334 val = BIT(CONTROL_ACTIVE_SHIFT); 1335 writel_relaxed(val, ring->regs + RING_CONTROL); 1336 1337 /* Reset stats to zero */ 1338 atomic_set(&ring->msg_send_count, 0); 1339 atomic_set(&ring->msg_cmpl_count, 0); 1340 1341 return 0; 1342 1343 fail_free_irq: 1344 free_irq(ring->irq, ring); 1345 ring->irq_requested = false; 1346 fail_free_cmpl_memory: 1347 dma_pool_free(ring->mbox->cmpl_pool, 1348 ring->cmpl_base, ring->cmpl_dma_base); 1349 ring->cmpl_base = NULL; 1350 fail_free_bd_memory: 1351 dma_pool_free(ring->mbox->bd_pool, 1352 ring->bd_base, ring->bd_dma_base); 1353 ring->bd_base = NULL; 1354 fail: 1355 return ret; 1356 } 1357 1358 static void flexrm_shutdown(struct mbox_chan *chan) 1359 { 1360 u32 reqid; 1361 unsigned int timeout; 1362 struct brcm_message *msg; 1363 struct flexrm_ring *ring = chan->con_priv; 1364 1365 /* Disable/inactivate ring */ 1366 writel_relaxed(0x0, ring->regs + RING_CONTROL); 1367 1368 /* Flush ring with timeout of 1s */ 1369 timeout = 1000; 1370 writel_relaxed(BIT(CONTROL_FLUSH_SHIFT), 1371 ring->regs + RING_CONTROL); 1372 do { 1373 if (readl_relaxed(ring->regs + RING_FLUSH_DONE) & 1374 FLUSH_DONE_MASK) 1375 break; 1376 mdelay(1); 1377 } while (timeout--); 1378 1379 /* Abort all in-flight requests */ 1380 for (reqid = 0; reqid < RING_MAX_REQ_COUNT; reqid++) { 1381 msg = ring->requests[reqid]; 1382 if (!msg) 1383 continue; 1384 1385 /* Release reqid for recycling */ 1386 ring->requests[reqid] = NULL; 1387 1388 /* Unmap DMA mappings */ 1389 flexrm_dma_unmap(ring->mbox->dev, msg); 1390 1391 /* Give-back message to mailbox client */ 1392 msg->error = -EIO; 1393 mbox_chan_received_data(chan, msg); 1394 } 1395 1396 /* Clear requests bitmap */ 1397 bitmap_zero(ring->requests_bmap, RING_MAX_REQ_COUNT); 1398 1399 /* Release IRQ */ 1400 if (ring->irq_requested) { 1401 irq_set_affinity_hint(ring->irq, NULL); 1402 free_irq(ring->irq, ring); 1403 ring->irq_requested = false; 1404 } 1405 1406 /* Free-up completion descriptor ring */ 1407 if (ring->cmpl_base) { 1408 dma_pool_free(ring->mbox->cmpl_pool, 1409 ring->cmpl_base, ring->cmpl_dma_base); 1410 ring->cmpl_base = NULL; 1411 } 1412 1413 /* Free-up BD descriptor ring */ 1414 if (ring->bd_base) { 1415 dma_pool_free(ring->mbox->bd_pool, 1416 ring->bd_base, ring->bd_dma_base); 1417 ring->bd_base = NULL; 1418 } 1419 } 1420 1421 static const struct mbox_chan_ops flexrm_mbox_chan_ops = { 1422 .send_data = flexrm_send_data, 1423 .startup = flexrm_startup, 1424 .shutdown = flexrm_shutdown, 1425 .peek_data = flexrm_peek_data, 1426 }; 1427 1428 static struct mbox_chan *flexrm_mbox_of_xlate(struct mbox_controller *cntlr, 1429 const struct of_phandle_args *pa) 1430 { 1431 struct mbox_chan *chan; 1432 struct flexrm_ring *ring; 1433 1434 if (pa->args_count < 3) 1435 return ERR_PTR(-EINVAL); 1436 1437 if (pa->args[0] >= cntlr->num_chans) 1438 return ERR_PTR(-ENOENT); 1439 1440 if (pa->args[1] > MSI_COUNT_MASK) 1441 return ERR_PTR(-EINVAL); 1442 1443 if (pa->args[2] > MSI_TIMER_VAL_MASK) 1444 return ERR_PTR(-EINVAL); 1445 1446 chan = &cntlr->chans[pa->args[0]]; 1447 ring = chan->con_priv; 1448 ring->msi_count_threshold = pa->args[1]; 1449 ring->msi_timer_val = pa->args[2]; 1450 1451 return chan; 1452 } 1453 1454 /* ====== FlexRM platform driver ===== */ 1455 1456 static void flexrm_mbox_msi_write(struct msi_desc *desc, struct msi_msg *msg) 1457 { 1458 struct device *dev = msi_desc_to_dev(desc); 1459 struct flexrm_mbox *mbox = dev_get_drvdata(dev); 1460 struct flexrm_ring *ring = &mbox->rings[desc->platform.msi_index]; 1461 1462 /* Configure per-Ring MSI registers */ 1463 writel_relaxed(msg->address_lo, ring->regs + RING_MSI_ADDR_LS); 1464 writel_relaxed(msg->address_hi, ring->regs + RING_MSI_ADDR_MS); 1465 writel_relaxed(msg->data, ring->regs + RING_MSI_DATA_VALUE); 1466 } 1467 1468 static int flexrm_mbox_probe(struct platform_device *pdev) 1469 { 1470 int index, ret = 0; 1471 void __iomem *regs; 1472 void __iomem *regs_end; 1473 struct msi_desc *desc; 1474 struct resource *iomem; 1475 struct flexrm_ring *ring; 1476 struct flexrm_mbox *mbox; 1477 struct device *dev = &pdev->dev; 1478 1479 /* Allocate driver mailbox struct */ 1480 mbox = devm_kzalloc(dev, sizeof(*mbox), GFP_KERNEL); 1481 if (!mbox) { 1482 ret = -ENOMEM; 1483 goto fail; 1484 } 1485 mbox->dev = dev; 1486 platform_set_drvdata(pdev, mbox); 1487 1488 /* Get resource for registers */ 1489 iomem = platform_get_resource(pdev, IORESOURCE_MEM, 0); 1490 if (!iomem || (resource_size(iomem) < RING_REGS_SIZE)) { 1491 ret = -ENODEV; 1492 goto fail; 1493 } 1494 1495 /* Map registers of all rings */ 1496 mbox->regs = devm_ioremap_resource(&pdev->dev, iomem); 1497 if (IS_ERR(mbox->regs)) { 1498 ret = PTR_ERR(mbox->regs); 1499 dev_err(&pdev->dev, "Failed to remap mailbox regs: %d\n", ret); 1500 goto fail; 1501 } 1502 regs_end = mbox->regs + resource_size(iomem); 1503 1504 /* Scan and count available rings */ 1505 mbox->num_rings = 0; 1506 for (regs = mbox->regs; regs < regs_end; regs += RING_REGS_SIZE) { 1507 if (readl_relaxed(regs + RING_VER) == RING_VER_MAGIC) 1508 mbox->num_rings++; 1509 } 1510 if (!mbox->num_rings) { 1511 ret = -ENODEV; 1512 goto fail; 1513 } 1514 1515 /* Allocate driver ring structs */ 1516 ring = devm_kcalloc(dev, mbox->num_rings, sizeof(*ring), GFP_KERNEL); 1517 if (!ring) { 1518 ret = -ENOMEM; 1519 goto fail; 1520 } 1521 mbox->rings = ring; 1522 1523 /* Initialize members of driver ring structs */ 1524 regs = mbox->regs; 1525 for (index = 0; index < mbox->num_rings; index++) { 1526 ring = &mbox->rings[index]; 1527 ring->num = index; 1528 ring->mbox = mbox; 1529 while ((regs < regs_end) && 1530 (readl_relaxed(regs + RING_VER) != RING_VER_MAGIC)) 1531 regs += RING_REGS_SIZE; 1532 if (regs_end <= regs) { 1533 ret = -ENODEV; 1534 goto fail; 1535 } 1536 ring->regs = regs; 1537 regs += RING_REGS_SIZE; 1538 ring->irq = UINT_MAX; 1539 ring->irq_requested = false; 1540 ring->msi_timer_val = MSI_TIMER_VAL_MASK; 1541 ring->msi_count_threshold = 0x1; 1542 memset(ring->requests, 0, sizeof(ring->requests)); 1543 ring->bd_base = NULL; 1544 ring->bd_dma_base = 0; 1545 ring->cmpl_base = NULL; 1546 ring->cmpl_dma_base = 0; 1547 atomic_set(&ring->msg_send_count, 0); 1548 atomic_set(&ring->msg_cmpl_count, 0); 1549 spin_lock_init(&ring->lock); 1550 bitmap_zero(ring->requests_bmap, RING_MAX_REQ_COUNT); 1551 ring->cmpl_read_offset = 0; 1552 } 1553 1554 /* FlexRM is capable of 40-bit physical addresses only */ 1555 ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(40)); 1556 if (ret) { 1557 ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32)); 1558 if (ret) 1559 goto fail; 1560 } 1561 1562 /* Create DMA pool for ring BD memory */ 1563 mbox->bd_pool = dma_pool_create("bd", dev, RING_BD_SIZE, 1564 1 << RING_BD_ALIGN_ORDER, 0); 1565 if (!mbox->bd_pool) { 1566 ret = -ENOMEM; 1567 goto fail; 1568 } 1569 1570 /* Create DMA pool for ring completion memory */ 1571 mbox->cmpl_pool = dma_pool_create("cmpl", dev, RING_CMPL_SIZE, 1572 1 << RING_CMPL_ALIGN_ORDER, 0); 1573 if (!mbox->cmpl_pool) { 1574 ret = -ENOMEM; 1575 goto fail_destroy_bd_pool; 1576 } 1577 1578 /* Allocate platform MSIs for each ring */ 1579 ret = platform_msi_domain_alloc_irqs(dev, mbox->num_rings, 1580 flexrm_mbox_msi_write); 1581 if (ret) 1582 goto fail_destroy_cmpl_pool; 1583 1584 /* Save alloced IRQ numbers for each ring */ 1585 for_each_msi_entry(desc, dev) { 1586 ring = &mbox->rings[desc->platform.msi_index]; 1587 ring->irq = desc->irq; 1588 } 1589 1590 /* Check availability of debugfs */ 1591 if (!debugfs_initialized()) 1592 goto skip_debugfs; 1593 1594 /* Create debugfs root entry */ 1595 mbox->root = debugfs_create_dir(dev_name(mbox->dev), NULL); 1596 if (IS_ERR_OR_NULL(mbox->root)) { 1597 ret = PTR_ERR_OR_ZERO(mbox->root); 1598 goto fail_free_msis; 1599 } 1600 1601 /* Create debugfs config entry */ 1602 mbox->config = debugfs_create_devm_seqfile(mbox->dev, 1603 "config", mbox->root, 1604 flexrm_debugfs_conf_show); 1605 if (IS_ERR_OR_NULL(mbox->config)) { 1606 ret = PTR_ERR_OR_ZERO(mbox->config); 1607 goto fail_free_debugfs_root; 1608 } 1609 1610 /* Create debugfs stats entry */ 1611 mbox->stats = debugfs_create_devm_seqfile(mbox->dev, 1612 "stats", mbox->root, 1613 flexrm_debugfs_stats_show); 1614 if (IS_ERR_OR_NULL(mbox->stats)) { 1615 ret = PTR_ERR_OR_ZERO(mbox->stats); 1616 goto fail_free_debugfs_root; 1617 } 1618 skip_debugfs: 1619 1620 /* Initialize mailbox controller */ 1621 mbox->controller.txdone_irq = false; 1622 mbox->controller.txdone_poll = false; 1623 mbox->controller.ops = &flexrm_mbox_chan_ops; 1624 mbox->controller.dev = dev; 1625 mbox->controller.num_chans = mbox->num_rings; 1626 mbox->controller.of_xlate = flexrm_mbox_of_xlate; 1627 mbox->controller.chans = devm_kcalloc(dev, mbox->num_rings, 1628 sizeof(*mbox->controller.chans), GFP_KERNEL); 1629 if (!mbox->controller.chans) { 1630 ret = -ENOMEM; 1631 goto fail_free_debugfs_root; 1632 } 1633 for (index = 0; index < mbox->num_rings; index++) 1634 mbox->controller.chans[index].con_priv = &mbox->rings[index]; 1635 1636 /* Register mailbox controller */ 1637 ret = mbox_controller_register(&mbox->controller); 1638 if (ret) 1639 goto fail_free_debugfs_root; 1640 1641 dev_info(dev, "registered flexrm mailbox with %d channels\n", 1642 mbox->controller.num_chans); 1643 1644 return 0; 1645 1646 fail_free_debugfs_root: 1647 debugfs_remove_recursive(mbox->root); 1648 fail_free_msis: 1649 platform_msi_domain_free_irqs(dev); 1650 fail_destroy_cmpl_pool: 1651 dma_pool_destroy(mbox->cmpl_pool); 1652 fail_destroy_bd_pool: 1653 dma_pool_destroy(mbox->bd_pool); 1654 fail: 1655 return ret; 1656 } 1657 1658 static int flexrm_mbox_remove(struct platform_device *pdev) 1659 { 1660 struct device *dev = &pdev->dev; 1661 struct flexrm_mbox *mbox = platform_get_drvdata(pdev); 1662 1663 mbox_controller_unregister(&mbox->controller); 1664 1665 debugfs_remove_recursive(mbox->root); 1666 1667 platform_msi_domain_free_irqs(dev); 1668 1669 dma_pool_destroy(mbox->cmpl_pool); 1670 dma_pool_destroy(mbox->bd_pool); 1671 1672 return 0; 1673 } 1674 1675 static const struct of_device_id flexrm_mbox_of_match[] = { 1676 { .compatible = "brcm,iproc-flexrm-mbox", }, 1677 {}, 1678 }; 1679 MODULE_DEVICE_TABLE(of, flexrm_mbox_of_match); 1680 1681 static struct platform_driver flexrm_mbox_driver = { 1682 .driver = { 1683 .name = "brcm-flexrm-mbox", 1684 .of_match_table = flexrm_mbox_of_match, 1685 }, 1686 .probe = flexrm_mbox_probe, 1687 .remove = flexrm_mbox_remove, 1688 }; 1689 module_platform_driver(flexrm_mbox_driver); 1690 1691 MODULE_AUTHOR("Anup Patel <anup.patel@broadcom.com>"); 1692 MODULE_DESCRIPTION("Broadcom FlexRM mailbox driver"); 1693 MODULE_LICENSE("GPL v2"); 1694