1 // SPDX-License-Identifier: GPL-2.0 2 // Copyright (c) 2017-2018 MediaTek Inc. 3 4 /* 5 * Driver for MediaTek High-Speed DMA Controller 6 * 7 * Author: Sean Wang <sean.wang@mediatek.com> 8 * 9 */ 10 11 #include <linux/bitops.h> 12 #include <linux/clk.h> 13 #include <linux/dmaengine.h> 14 #include <linux/dma-mapping.h> 15 #include <linux/err.h> 16 #include <linux/iopoll.h> 17 #include <linux/list.h> 18 #include <linux/module.h> 19 #include <linux/of.h> 20 #include <linux/of_dma.h> 21 #include <linux/platform_device.h> 22 #include <linux/pm_runtime.h> 23 #include <linux/refcount.h> 24 #include <linux/slab.h> 25 26 #include "../virt-dma.h" 27 28 #define MTK_HSDMA_USEC_POLL 20 29 #define MTK_HSDMA_TIMEOUT_POLL 200000 30 #define MTK_HSDMA_DMA_BUSWIDTHS BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) 31 32 /* The default number of virtual channel */ 33 #define MTK_HSDMA_NR_VCHANS 3 34 35 /* Only one physical channel supported */ 36 #define MTK_HSDMA_NR_MAX_PCHANS 1 37 38 /* Macro for physical descriptor (PD) manipulation */ 39 /* The number of PD which must be 2 of power */ 40 #define MTK_DMA_SIZE 64 41 #define MTK_HSDMA_NEXT_DESP_IDX(x, y) (((x) + 1) & ((y) - 1)) 42 #define MTK_HSDMA_LAST_DESP_IDX(x, y) (((x) - 1) & ((y) - 1)) 43 #define MTK_HSDMA_MAX_LEN 0x3f80 44 #define MTK_HSDMA_ALIGN_SIZE 4 45 #define MTK_HSDMA_PLEN_MASK 0x3fff 46 #define MTK_HSDMA_DESC_PLEN(x) (((x) & MTK_HSDMA_PLEN_MASK) << 16) 47 #define MTK_HSDMA_DESC_PLEN_GET(x) (((x) >> 16) & MTK_HSDMA_PLEN_MASK) 48 49 /* Registers for underlying ring manipulation */ 50 #define MTK_HSDMA_TX_BASE 0x0 51 #define MTK_HSDMA_TX_CNT 0x4 52 #define MTK_HSDMA_TX_CPU 0x8 53 #define MTK_HSDMA_TX_DMA 0xc 54 #define MTK_HSDMA_RX_BASE 0x100 55 #define MTK_HSDMA_RX_CNT 0x104 56 #define MTK_HSDMA_RX_CPU 0x108 57 #define MTK_HSDMA_RX_DMA 0x10c 58 59 /* Registers for global setup */ 60 #define MTK_HSDMA_GLO 0x204 61 #define MTK_HSDMA_GLO_MULTI_DMA BIT(10) 62 #define MTK_HSDMA_TX_WB_DDONE BIT(6) 63 #define MTK_HSDMA_BURST_64BYTES (0x2 << 4) 64 #define MTK_HSDMA_GLO_RX_BUSY BIT(3) 65 #define MTK_HSDMA_GLO_RX_DMA BIT(2) 66 #define MTK_HSDMA_GLO_TX_BUSY BIT(1) 67 #define MTK_HSDMA_GLO_TX_DMA BIT(0) 68 #define MTK_HSDMA_GLO_DMA (MTK_HSDMA_GLO_TX_DMA | \ 69 MTK_HSDMA_GLO_RX_DMA) 70 #define MTK_HSDMA_GLO_BUSY (MTK_HSDMA_GLO_RX_BUSY | \ 71 MTK_HSDMA_GLO_TX_BUSY) 72 #define MTK_HSDMA_GLO_DEFAULT (MTK_HSDMA_GLO_TX_DMA | \ 73 MTK_HSDMA_GLO_RX_DMA | \ 74 MTK_HSDMA_TX_WB_DDONE | \ 75 MTK_HSDMA_BURST_64BYTES | \ 76 MTK_HSDMA_GLO_MULTI_DMA) 77 78 /* Registers for reset */ 79 #define MTK_HSDMA_RESET 0x208 80 #define MTK_HSDMA_RST_TX BIT(0) 81 #define MTK_HSDMA_RST_RX BIT(16) 82 83 /* Registers for interrupt control */ 84 #define MTK_HSDMA_DLYINT 0x20c 85 #define MTK_HSDMA_RXDLY_INT_EN BIT(15) 86 87 /* Interrupt fires when the pending number's more than the specified */ 88 #define MTK_HSDMA_RXMAX_PINT(x) (((x) & 0x7f) << 8) 89 90 /* Interrupt fires when the pending time's more than the specified in 20 us */ 91 #define MTK_HSDMA_RXMAX_PTIME(x) ((x) & 0x7f) 92 #define MTK_HSDMA_DLYINT_DEFAULT (MTK_HSDMA_RXDLY_INT_EN | \ 93 MTK_HSDMA_RXMAX_PINT(20) | \ 94 MTK_HSDMA_RXMAX_PTIME(20)) 95 #define MTK_HSDMA_INT_STATUS 0x220 96 #define MTK_HSDMA_INT_ENABLE 0x228 97 #define MTK_HSDMA_INT_RXDONE BIT(16) 98 99 enum mtk_hsdma_vdesc_flag { 100 MTK_HSDMA_VDESC_FINISHED = 0x01, 101 }; 102 103 #define IS_MTK_HSDMA_VDESC_FINISHED(x) ((x) == MTK_HSDMA_VDESC_FINISHED) 104 105 /** 106 * struct mtk_hsdma_pdesc - This is the struct holding info describing physical 107 * descriptor (PD) and its placement must be kept at 108 * 4-bytes alignment in little endian order. 109 * @desc1: | The control pad used to indicate hardware how to 110 * @desc2: | deal with the descriptor such as source and 111 * @desc3: | destination address and data length. The maximum 112 * @desc4: | data length each pdesc can handle is 0x3f80 bytes 113 */ 114 struct mtk_hsdma_pdesc { 115 __le32 desc1; 116 __le32 desc2; 117 __le32 desc3; 118 __le32 desc4; 119 } __packed __aligned(4); 120 121 /** 122 * struct mtk_hsdma_vdesc - This is the struct holding info describing virtual 123 * descriptor (VD) 124 * @vd: An instance for struct virt_dma_desc 125 * @len: The total data size device wants to move 126 * @residue: The remaining data size device will move 127 * @dest: The destination address device wants to move to 128 * @src: The source address device wants to move from 129 */ 130 struct mtk_hsdma_vdesc { 131 struct virt_dma_desc vd; 132 size_t len; 133 size_t residue; 134 dma_addr_t dest; 135 dma_addr_t src; 136 }; 137 138 /** 139 * struct mtk_hsdma_cb - This is the struct holding extra info required for RX 140 * ring to know what relevant VD the PD is being 141 * mapped to. 142 * @vd: Pointer to the relevant VD. 143 * @flag: Flag indicating what action should be taken when VD 144 * is completed. 145 */ 146 struct mtk_hsdma_cb { 147 struct virt_dma_desc *vd; 148 enum mtk_hsdma_vdesc_flag flag; 149 }; 150 151 /** 152 * struct mtk_hsdma_ring - This struct holds info describing underlying ring 153 * space 154 * @txd: The descriptor TX ring which describes DMA source 155 * information 156 * @rxd: The descriptor RX ring which describes DMA 157 * destination information 158 * @cb: The extra information pointed at by RX ring 159 * @tphys: The physical addr of TX ring 160 * @rphys: The physical addr of RX ring 161 * @cur_tptr: Pointer to the next free descriptor used by the host 162 * @cur_rptr: Pointer to the last done descriptor by the device 163 */ 164 struct mtk_hsdma_ring { 165 struct mtk_hsdma_pdesc *txd; 166 struct mtk_hsdma_pdesc *rxd; 167 struct mtk_hsdma_cb *cb; 168 dma_addr_t tphys; 169 dma_addr_t rphys; 170 u16 cur_tptr; 171 u16 cur_rptr; 172 }; 173 174 /** 175 * struct mtk_hsdma_pchan - This is the struct holding info describing physical 176 * channel (PC) 177 * @ring: An instance for the underlying ring 178 * @sz_ring: Total size allocated for the ring 179 * @nr_free: Total number of free rooms in the ring. It would 180 * be accessed and updated frequently between IRQ 181 * context and user context to reflect whether ring 182 * can accept requests from VD. 183 */ 184 struct mtk_hsdma_pchan { 185 struct mtk_hsdma_ring ring; 186 size_t sz_ring; 187 atomic_t nr_free; 188 }; 189 190 /** 191 * struct mtk_hsdma_vchan - This is the struct holding info describing virtual 192 * channel (VC) 193 * @vc: An instance for struct virt_dma_chan 194 * @issue_completion: The wait for all issued descriptors completited 195 * @issue_synchronize: Bool indicating channel synchronization starts 196 * @desc_hw_processing: List those descriptors the hardware is processing, 197 * which is protected by vc.lock 198 */ 199 struct mtk_hsdma_vchan { 200 struct virt_dma_chan vc; 201 struct completion issue_completion; 202 bool issue_synchronize; 203 struct list_head desc_hw_processing; 204 }; 205 206 /** 207 * struct mtk_hsdma_soc - This is the struct holding differences among SoCs 208 * @ddone: Bit mask for DDONE 209 * @ls0: Bit mask for LS0 210 */ 211 struct mtk_hsdma_soc { 212 __le32 ddone; 213 __le32 ls0; 214 }; 215 216 /** 217 * struct mtk_hsdma_device - This is the struct holding info describing HSDMA 218 * device 219 * @ddev: An instance for struct dma_device 220 * @base: The mapped register I/O base 221 * @clk: The clock that device internal is using 222 * @irq: The IRQ that device are using 223 * @dma_requests: The number of VCs the device supports to 224 * @vc: The pointer to all available VCs 225 * @pc: The pointer to the underlying PC 226 * @pc_refcnt: Track how many VCs are using the PC 227 * @lock: Lock protect agaisting multiple VCs access PC 228 * @soc: The pointer to area holding differences among 229 * vaious platform 230 */ 231 struct mtk_hsdma_device { 232 struct dma_device ddev; 233 void __iomem *base; 234 struct clk *clk; 235 u32 irq; 236 237 u32 dma_requests; 238 struct mtk_hsdma_vchan *vc; 239 struct mtk_hsdma_pchan *pc; 240 refcount_t pc_refcnt; 241 242 /* Lock used to protect against multiple VCs access PC */ 243 spinlock_t lock; 244 245 const struct mtk_hsdma_soc *soc; 246 }; 247 248 static struct mtk_hsdma_device *to_hsdma_dev(struct dma_chan *chan) 249 { 250 return container_of(chan->device, struct mtk_hsdma_device, ddev); 251 } 252 253 static inline struct mtk_hsdma_vchan *to_hsdma_vchan(struct dma_chan *chan) 254 { 255 return container_of(chan, struct mtk_hsdma_vchan, vc.chan); 256 } 257 258 static struct mtk_hsdma_vdesc *to_hsdma_vdesc(struct virt_dma_desc *vd) 259 { 260 return container_of(vd, struct mtk_hsdma_vdesc, vd); 261 } 262 263 static struct device *hsdma2dev(struct mtk_hsdma_device *hsdma) 264 { 265 return hsdma->ddev.dev; 266 } 267 268 static u32 mtk_dma_read(struct mtk_hsdma_device *hsdma, u32 reg) 269 { 270 return readl(hsdma->base + reg); 271 } 272 273 static void mtk_dma_write(struct mtk_hsdma_device *hsdma, u32 reg, u32 val) 274 { 275 writel(val, hsdma->base + reg); 276 } 277 278 static void mtk_dma_rmw(struct mtk_hsdma_device *hsdma, u32 reg, 279 u32 mask, u32 set) 280 { 281 u32 val; 282 283 val = mtk_dma_read(hsdma, reg); 284 val &= ~mask; 285 val |= set; 286 mtk_dma_write(hsdma, reg, val); 287 } 288 289 static void mtk_dma_set(struct mtk_hsdma_device *hsdma, u32 reg, u32 val) 290 { 291 mtk_dma_rmw(hsdma, reg, 0, val); 292 } 293 294 static void mtk_dma_clr(struct mtk_hsdma_device *hsdma, u32 reg, u32 val) 295 { 296 mtk_dma_rmw(hsdma, reg, val, 0); 297 } 298 299 static void mtk_hsdma_vdesc_free(struct virt_dma_desc *vd) 300 { 301 kfree(container_of(vd, struct mtk_hsdma_vdesc, vd)); 302 } 303 304 static int mtk_hsdma_busy_wait(struct mtk_hsdma_device *hsdma) 305 { 306 u32 status = 0; 307 308 return readl_poll_timeout(hsdma->base + MTK_HSDMA_GLO, status, 309 !(status & MTK_HSDMA_GLO_BUSY), 310 MTK_HSDMA_USEC_POLL, 311 MTK_HSDMA_TIMEOUT_POLL); 312 } 313 314 static int mtk_hsdma_alloc_pchan(struct mtk_hsdma_device *hsdma, 315 struct mtk_hsdma_pchan *pc) 316 { 317 struct mtk_hsdma_ring *ring = &pc->ring; 318 int err; 319 320 memset(pc, 0, sizeof(*pc)); 321 322 /* 323 * Allocate ring space where [0 ... MTK_DMA_SIZE - 1] is for TX ring 324 * and [MTK_DMA_SIZE ... 2 * MTK_DMA_SIZE - 1] is for RX ring. 325 */ 326 pc->sz_ring = 2 * MTK_DMA_SIZE * sizeof(*ring->txd); 327 ring->txd = dma_alloc_coherent(hsdma2dev(hsdma), pc->sz_ring, 328 &ring->tphys, GFP_NOWAIT); 329 if (!ring->txd) 330 return -ENOMEM; 331 332 ring->rxd = &ring->txd[MTK_DMA_SIZE]; 333 ring->rphys = ring->tphys + MTK_DMA_SIZE * sizeof(*ring->txd); 334 ring->cur_tptr = 0; 335 ring->cur_rptr = MTK_DMA_SIZE - 1; 336 337 ring->cb = kcalloc(MTK_DMA_SIZE, sizeof(*ring->cb), GFP_NOWAIT); 338 if (!ring->cb) { 339 err = -ENOMEM; 340 goto err_free_dma; 341 } 342 343 atomic_set(&pc->nr_free, MTK_DMA_SIZE - 1); 344 345 /* Disable HSDMA and wait for the completion */ 346 mtk_dma_clr(hsdma, MTK_HSDMA_GLO, MTK_HSDMA_GLO_DMA); 347 err = mtk_hsdma_busy_wait(hsdma); 348 if (err) 349 goto err_free_cb; 350 351 /* Reset */ 352 mtk_dma_set(hsdma, MTK_HSDMA_RESET, 353 MTK_HSDMA_RST_TX | MTK_HSDMA_RST_RX); 354 mtk_dma_clr(hsdma, MTK_HSDMA_RESET, 355 MTK_HSDMA_RST_TX | MTK_HSDMA_RST_RX); 356 357 /* Setup HSDMA initial pointer in the ring */ 358 mtk_dma_write(hsdma, MTK_HSDMA_TX_BASE, ring->tphys); 359 mtk_dma_write(hsdma, MTK_HSDMA_TX_CNT, MTK_DMA_SIZE); 360 mtk_dma_write(hsdma, MTK_HSDMA_TX_CPU, ring->cur_tptr); 361 mtk_dma_write(hsdma, MTK_HSDMA_TX_DMA, 0); 362 mtk_dma_write(hsdma, MTK_HSDMA_RX_BASE, ring->rphys); 363 mtk_dma_write(hsdma, MTK_HSDMA_RX_CNT, MTK_DMA_SIZE); 364 mtk_dma_write(hsdma, MTK_HSDMA_RX_CPU, ring->cur_rptr); 365 mtk_dma_write(hsdma, MTK_HSDMA_RX_DMA, 0); 366 367 /* Enable HSDMA */ 368 mtk_dma_set(hsdma, MTK_HSDMA_GLO, MTK_HSDMA_GLO_DMA); 369 370 /* Setup delayed interrupt */ 371 mtk_dma_write(hsdma, MTK_HSDMA_DLYINT, MTK_HSDMA_DLYINT_DEFAULT); 372 373 /* Enable interrupt */ 374 mtk_dma_set(hsdma, MTK_HSDMA_INT_ENABLE, MTK_HSDMA_INT_RXDONE); 375 376 return 0; 377 378 err_free_cb: 379 kfree(ring->cb); 380 381 err_free_dma: 382 dma_free_coherent(hsdma2dev(hsdma), 383 pc->sz_ring, ring->txd, ring->tphys); 384 return err; 385 } 386 387 static void mtk_hsdma_free_pchan(struct mtk_hsdma_device *hsdma, 388 struct mtk_hsdma_pchan *pc) 389 { 390 struct mtk_hsdma_ring *ring = &pc->ring; 391 392 /* Disable HSDMA and then wait for the completion */ 393 mtk_dma_clr(hsdma, MTK_HSDMA_GLO, MTK_HSDMA_GLO_DMA); 394 mtk_hsdma_busy_wait(hsdma); 395 396 /* Reset pointer in the ring */ 397 mtk_dma_clr(hsdma, MTK_HSDMA_INT_ENABLE, MTK_HSDMA_INT_RXDONE); 398 mtk_dma_write(hsdma, MTK_HSDMA_TX_BASE, 0); 399 mtk_dma_write(hsdma, MTK_HSDMA_TX_CNT, 0); 400 mtk_dma_write(hsdma, MTK_HSDMA_TX_CPU, 0); 401 mtk_dma_write(hsdma, MTK_HSDMA_RX_BASE, 0); 402 mtk_dma_write(hsdma, MTK_HSDMA_RX_CNT, 0); 403 mtk_dma_write(hsdma, MTK_HSDMA_RX_CPU, MTK_DMA_SIZE - 1); 404 405 kfree(ring->cb); 406 407 dma_free_coherent(hsdma2dev(hsdma), 408 pc->sz_ring, ring->txd, ring->tphys); 409 } 410 411 static int mtk_hsdma_issue_pending_vdesc(struct mtk_hsdma_device *hsdma, 412 struct mtk_hsdma_pchan *pc, 413 struct mtk_hsdma_vdesc *hvd) 414 { 415 struct mtk_hsdma_ring *ring = &pc->ring; 416 struct mtk_hsdma_pdesc *txd, *rxd; 417 u16 reserved, prev, tlen, num_sgs; 418 unsigned long flags; 419 420 /* Protect against PC is accessed by multiple VCs simultaneously */ 421 spin_lock_irqsave(&hsdma->lock, flags); 422 423 /* 424 * Reserve rooms, where pc->nr_free is used to track how many free 425 * rooms in the ring being updated in user and IRQ context. 426 */ 427 num_sgs = DIV_ROUND_UP(hvd->len, MTK_HSDMA_MAX_LEN); 428 reserved = min_t(u16, num_sgs, atomic_read(&pc->nr_free)); 429 430 if (!reserved) { 431 spin_unlock_irqrestore(&hsdma->lock, flags); 432 return -ENOSPC; 433 } 434 435 atomic_sub(reserved, &pc->nr_free); 436 437 while (reserved--) { 438 /* Limit size by PD capability for valid data moving */ 439 tlen = (hvd->len > MTK_HSDMA_MAX_LEN) ? 440 MTK_HSDMA_MAX_LEN : hvd->len; 441 442 /* 443 * Setup PDs using the remaining VD info mapped on those 444 * reserved rooms. And since RXD is shared memory between the 445 * host and the device allocated by dma_alloc_coherent call, 446 * the helper macro WRITE_ONCE can ensure the data written to 447 * RAM would really happens. 448 */ 449 txd = &ring->txd[ring->cur_tptr]; 450 WRITE_ONCE(txd->desc1, hvd->src); 451 WRITE_ONCE(txd->desc2, 452 hsdma->soc->ls0 | MTK_HSDMA_DESC_PLEN(tlen)); 453 454 rxd = &ring->rxd[ring->cur_tptr]; 455 WRITE_ONCE(rxd->desc1, hvd->dest); 456 WRITE_ONCE(rxd->desc2, MTK_HSDMA_DESC_PLEN(tlen)); 457 458 /* Associate VD, the PD belonged to */ 459 ring->cb[ring->cur_tptr].vd = &hvd->vd; 460 461 /* Move forward the pointer of TX ring */ 462 ring->cur_tptr = MTK_HSDMA_NEXT_DESP_IDX(ring->cur_tptr, 463 MTK_DMA_SIZE); 464 465 /* Update VD with remaining data */ 466 hvd->src += tlen; 467 hvd->dest += tlen; 468 hvd->len -= tlen; 469 } 470 471 /* 472 * Tagging flag for the last PD for VD will be responsible for 473 * completing VD. 474 */ 475 if (!hvd->len) { 476 prev = MTK_HSDMA_LAST_DESP_IDX(ring->cur_tptr, MTK_DMA_SIZE); 477 ring->cb[prev].flag = MTK_HSDMA_VDESC_FINISHED; 478 } 479 480 /* Ensure all changes indeed done before we're going on */ 481 wmb(); 482 483 /* 484 * Updating into hardware the pointer of TX ring lets HSDMA to take 485 * action for those pending PDs. 486 */ 487 mtk_dma_write(hsdma, MTK_HSDMA_TX_CPU, ring->cur_tptr); 488 489 spin_unlock_irqrestore(&hsdma->lock, flags); 490 491 return 0; 492 } 493 494 static void mtk_hsdma_issue_vchan_pending(struct mtk_hsdma_device *hsdma, 495 struct mtk_hsdma_vchan *hvc) 496 { 497 struct virt_dma_desc *vd, *vd2; 498 int err; 499 500 lockdep_assert_held(&hvc->vc.lock); 501 502 list_for_each_entry_safe(vd, vd2, &hvc->vc.desc_issued, node) { 503 struct mtk_hsdma_vdesc *hvd; 504 505 hvd = to_hsdma_vdesc(vd); 506 507 /* Map VD into PC and all VCs shares a single PC */ 508 err = mtk_hsdma_issue_pending_vdesc(hsdma, hsdma->pc, hvd); 509 510 /* 511 * Move VD from desc_issued to desc_hw_processing when entire 512 * VD is fit into available PDs. Otherwise, the uncompleted 513 * VDs would stay in list desc_issued and then restart the 514 * processing as soon as possible once underlying ring space 515 * got freed. 516 */ 517 if (err == -ENOSPC || hvd->len > 0) 518 break; 519 520 /* 521 * The extra list desc_hw_processing is used because 522 * hardware can't provide sufficient information allowing us 523 * to know what VDs are still working on the underlying ring. 524 * Through the additional list, it can help us to implement 525 * terminate_all, residue calculation and such thing needed 526 * to know detail descriptor status on the hardware. 527 */ 528 list_move_tail(&vd->node, &hvc->desc_hw_processing); 529 } 530 } 531 532 static void mtk_hsdma_free_rooms_in_ring(struct mtk_hsdma_device *hsdma) 533 { 534 struct mtk_hsdma_vchan *hvc; 535 struct mtk_hsdma_pdesc *rxd; 536 struct mtk_hsdma_vdesc *hvd; 537 struct mtk_hsdma_pchan *pc; 538 struct mtk_hsdma_cb *cb; 539 int i = MTK_DMA_SIZE; 540 __le32 desc2; 541 u32 status; 542 u16 next; 543 544 /* Read IRQ status */ 545 status = mtk_dma_read(hsdma, MTK_HSDMA_INT_STATUS); 546 if (unlikely(!(status & MTK_HSDMA_INT_RXDONE))) 547 goto rx_done; 548 549 pc = hsdma->pc; 550 551 /* 552 * Using a fail-safe loop with iterations of up to MTK_DMA_SIZE to 553 * reclaim these finished descriptors: The most number of PDs the ISR 554 * can handle at one time shouldn't be more than MTK_DMA_SIZE so we 555 * take it as limited count instead of just using a dangerous infinite 556 * poll. 557 */ 558 while (i--) { 559 next = MTK_HSDMA_NEXT_DESP_IDX(pc->ring.cur_rptr, 560 MTK_DMA_SIZE); 561 rxd = &pc->ring.rxd[next]; 562 563 /* 564 * If MTK_HSDMA_DESC_DDONE is no specified, that means data 565 * moving for the PD is still under going. 566 */ 567 desc2 = READ_ONCE(rxd->desc2); 568 if (!(desc2 & hsdma->soc->ddone)) 569 break; 570 571 cb = &pc->ring.cb[next]; 572 if (unlikely(!cb->vd)) { 573 dev_err(hsdma2dev(hsdma), "cb->vd cannot be null\n"); 574 break; 575 } 576 577 /* Update residue of VD the associated PD belonged to */ 578 hvd = to_hsdma_vdesc(cb->vd); 579 hvd->residue -= MTK_HSDMA_DESC_PLEN_GET(rxd->desc2); 580 581 /* Complete VD until the relevant last PD is finished */ 582 if (IS_MTK_HSDMA_VDESC_FINISHED(cb->flag)) { 583 hvc = to_hsdma_vchan(cb->vd->tx.chan); 584 585 spin_lock(&hvc->vc.lock); 586 587 /* Remove VD from list desc_hw_processing */ 588 list_del(&cb->vd->node); 589 590 /* Add VD into list desc_completed */ 591 vchan_cookie_complete(cb->vd); 592 593 if (hvc->issue_synchronize && 594 list_empty(&hvc->desc_hw_processing)) { 595 complete(&hvc->issue_completion); 596 hvc->issue_synchronize = false; 597 } 598 spin_unlock(&hvc->vc.lock); 599 600 cb->flag = 0; 601 } 602 603 cb->vd = NULL; 604 605 /* 606 * Recycle the RXD with the helper WRITE_ONCE that can ensure 607 * data written into RAM would really happens. 608 */ 609 WRITE_ONCE(rxd->desc1, 0); 610 WRITE_ONCE(rxd->desc2, 0); 611 pc->ring.cur_rptr = next; 612 613 /* Release rooms */ 614 atomic_inc(&pc->nr_free); 615 } 616 617 /* Ensure all changes indeed done before we're going on */ 618 wmb(); 619 620 /* Update CPU pointer for those completed PDs */ 621 mtk_dma_write(hsdma, MTK_HSDMA_RX_CPU, pc->ring.cur_rptr); 622 623 /* 624 * Acking the pending IRQ allows hardware no longer to keep the used 625 * IRQ line in certain trigger state when software has completed all 626 * the finished physical descriptors. 627 */ 628 if (atomic_read(&pc->nr_free) >= MTK_DMA_SIZE - 1) 629 mtk_dma_write(hsdma, MTK_HSDMA_INT_STATUS, status); 630 631 /* ASAP handles pending VDs in all VCs after freeing some rooms */ 632 for (i = 0; i < hsdma->dma_requests; i++) { 633 hvc = &hsdma->vc[i]; 634 spin_lock(&hvc->vc.lock); 635 mtk_hsdma_issue_vchan_pending(hsdma, hvc); 636 spin_unlock(&hvc->vc.lock); 637 } 638 639 rx_done: 640 /* All completed PDs are cleaned up, so enable interrupt again */ 641 mtk_dma_set(hsdma, MTK_HSDMA_INT_ENABLE, MTK_HSDMA_INT_RXDONE); 642 } 643 644 static irqreturn_t mtk_hsdma_irq(int irq, void *devid) 645 { 646 struct mtk_hsdma_device *hsdma = devid; 647 648 /* 649 * Disable interrupt until all completed PDs are cleaned up in 650 * mtk_hsdma_free_rooms call. 651 */ 652 mtk_dma_clr(hsdma, MTK_HSDMA_INT_ENABLE, MTK_HSDMA_INT_RXDONE); 653 654 mtk_hsdma_free_rooms_in_ring(hsdma); 655 656 return IRQ_HANDLED; 657 } 658 659 static struct virt_dma_desc *mtk_hsdma_find_active_desc(struct dma_chan *c, 660 dma_cookie_t cookie) 661 { 662 struct mtk_hsdma_vchan *hvc = to_hsdma_vchan(c); 663 struct virt_dma_desc *vd; 664 665 list_for_each_entry(vd, &hvc->desc_hw_processing, node) 666 if (vd->tx.cookie == cookie) 667 return vd; 668 669 list_for_each_entry(vd, &hvc->vc.desc_issued, node) 670 if (vd->tx.cookie == cookie) 671 return vd; 672 673 return NULL; 674 } 675 676 static enum dma_status mtk_hsdma_tx_status(struct dma_chan *c, 677 dma_cookie_t cookie, 678 struct dma_tx_state *txstate) 679 { 680 struct mtk_hsdma_vchan *hvc = to_hsdma_vchan(c); 681 struct mtk_hsdma_vdesc *hvd; 682 struct virt_dma_desc *vd; 683 enum dma_status ret; 684 unsigned long flags; 685 size_t bytes = 0; 686 687 ret = dma_cookie_status(c, cookie, txstate); 688 if (ret == DMA_COMPLETE || !txstate) 689 return ret; 690 691 spin_lock_irqsave(&hvc->vc.lock, flags); 692 vd = mtk_hsdma_find_active_desc(c, cookie); 693 spin_unlock_irqrestore(&hvc->vc.lock, flags); 694 695 if (vd) { 696 hvd = to_hsdma_vdesc(vd); 697 bytes = hvd->residue; 698 } 699 700 dma_set_residue(txstate, bytes); 701 702 return ret; 703 } 704 705 static void mtk_hsdma_issue_pending(struct dma_chan *c) 706 { 707 struct mtk_hsdma_device *hsdma = to_hsdma_dev(c); 708 struct mtk_hsdma_vchan *hvc = to_hsdma_vchan(c); 709 unsigned long flags; 710 711 spin_lock_irqsave(&hvc->vc.lock, flags); 712 713 if (vchan_issue_pending(&hvc->vc)) 714 mtk_hsdma_issue_vchan_pending(hsdma, hvc); 715 716 spin_unlock_irqrestore(&hvc->vc.lock, flags); 717 } 718 719 static struct dma_async_tx_descriptor * 720 mtk_hsdma_prep_dma_memcpy(struct dma_chan *c, dma_addr_t dest, 721 dma_addr_t src, size_t len, unsigned long flags) 722 { 723 struct mtk_hsdma_vdesc *hvd; 724 725 hvd = kzalloc(sizeof(*hvd), GFP_NOWAIT); 726 if (!hvd) 727 return NULL; 728 729 hvd->len = len; 730 hvd->residue = len; 731 hvd->src = src; 732 hvd->dest = dest; 733 734 return vchan_tx_prep(to_virt_chan(c), &hvd->vd, flags); 735 } 736 737 static int mtk_hsdma_free_inactive_desc(struct dma_chan *c) 738 { 739 struct virt_dma_chan *vc = to_virt_chan(c); 740 unsigned long flags; 741 LIST_HEAD(head); 742 743 spin_lock_irqsave(&vc->lock, flags); 744 list_splice_tail_init(&vc->desc_allocated, &head); 745 list_splice_tail_init(&vc->desc_submitted, &head); 746 list_splice_tail_init(&vc->desc_issued, &head); 747 spin_unlock_irqrestore(&vc->lock, flags); 748 749 /* At the point, we don't expect users put descriptor into VC again */ 750 vchan_dma_desc_free_list(vc, &head); 751 752 return 0; 753 } 754 755 static void mtk_hsdma_free_active_desc(struct dma_chan *c) 756 { 757 struct mtk_hsdma_vchan *hvc = to_hsdma_vchan(c); 758 bool sync_needed = false; 759 760 /* 761 * Once issue_synchronize is being set, which means once the hardware 762 * consumes all descriptors for the channel in the ring, the 763 * synchronization must be notified immediately it is completed. 764 */ 765 spin_lock(&hvc->vc.lock); 766 if (!list_empty(&hvc->desc_hw_processing)) { 767 hvc->issue_synchronize = true; 768 sync_needed = true; 769 } 770 spin_unlock(&hvc->vc.lock); 771 772 if (sync_needed) 773 wait_for_completion(&hvc->issue_completion); 774 /* 775 * At the point, we expect that all remaining descriptors in the ring 776 * for the channel should be all processing done. 777 */ 778 WARN_ONCE(!list_empty(&hvc->desc_hw_processing), 779 "Desc pending still in list desc_hw_processing\n"); 780 781 /* Free all descriptors in list desc_completed */ 782 vchan_synchronize(&hvc->vc); 783 784 WARN_ONCE(!list_empty(&hvc->vc.desc_completed), 785 "Desc pending still in list desc_completed\n"); 786 } 787 788 static int mtk_hsdma_terminate_all(struct dma_chan *c) 789 { 790 /* 791 * Free pending descriptors not processed yet by hardware that have 792 * previously been submitted to the channel. 793 */ 794 mtk_hsdma_free_inactive_desc(c); 795 796 /* 797 * However, the DMA engine doesn't provide any way to stop these 798 * descriptors being processed currently by hardware. The only way is 799 * to just waiting until these descriptors are all processed completely 800 * through mtk_hsdma_free_active_desc call. 801 */ 802 mtk_hsdma_free_active_desc(c); 803 804 return 0; 805 } 806 807 static int mtk_hsdma_alloc_chan_resources(struct dma_chan *c) 808 { 809 struct mtk_hsdma_device *hsdma = to_hsdma_dev(c); 810 int err; 811 812 /* 813 * Since HSDMA has only one PC, the resource for PC is being allocated 814 * when the first VC is being created and the other VCs would run on 815 * the same PC. 816 */ 817 if (!refcount_read(&hsdma->pc_refcnt)) { 818 err = mtk_hsdma_alloc_pchan(hsdma, hsdma->pc); 819 if (err) 820 return err; 821 /* 822 * refcount_inc would complain increment on 0; use-after-free. 823 * Thus, we need to explicitly set it as 1 initially. 824 */ 825 refcount_set(&hsdma->pc_refcnt, 1); 826 } else { 827 refcount_inc(&hsdma->pc_refcnt); 828 } 829 830 return 0; 831 } 832 833 static void mtk_hsdma_free_chan_resources(struct dma_chan *c) 834 { 835 struct mtk_hsdma_device *hsdma = to_hsdma_dev(c); 836 837 /* Free all descriptors in all lists on the VC */ 838 mtk_hsdma_terminate_all(c); 839 840 /* The resource for PC is not freed until all the VCs are destroyed */ 841 if (!refcount_dec_and_test(&hsdma->pc_refcnt)) 842 return; 843 844 mtk_hsdma_free_pchan(hsdma, hsdma->pc); 845 } 846 847 static int mtk_hsdma_hw_init(struct mtk_hsdma_device *hsdma) 848 { 849 int err; 850 851 pm_runtime_enable(hsdma2dev(hsdma)); 852 pm_runtime_get_sync(hsdma2dev(hsdma)); 853 854 err = clk_prepare_enable(hsdma->clk); 855 if (err) 856 return err; 857 858 mtk_dma_write(hsdma, MTK_HSDMA_INT_ENABLE, 0); 859 mtk_dma_write(hsdma, MTK_HSDMA_GLO, MTK_HSDMA_GLO_DEFAULT); 860 861 return 0; 862 } 863 864 static int mtk_hsdma_hw_deinit(struct mtk_hsdma_device *hsdma) 865 { 866 mtk_dma_write(hsdma, MTK_HSDMA_GLO, 0); 867 868 clk_disable_unprepare(hsdma->clk); 869 870 pm_runtime_put_sync(hsdma2dev(hsdma)); 871 pm_runtime_disable(hsdma2dev(hsdma)); 872 873 return 0; 874 } 875 876 static const struct mtk_hsdma_soc mt7623_soc = { 877 .ddone = BIT(31), 878 .ls0 = BIT(30), 879 }; 880 881 static const struct mtk_hsdma_soc mt7622_soc = { 882 .ddone = BIT(15), 883 .ls0 = BIT(14), 884 }; 885 886 static const struct of_device_id mtk_hsdma_match[] = { 887 { .compatible = "mediatek,mt7623-hsdma", .data = &mt7623_soc}, 888 { .compatible = "mediatek,mt7622-hsdma", .data = &mt7622_soc}, 889 { /* sentinel */ } 890 }; 891 MODULE_DEVICE_TABLE(of, mtk_hsdma_match); 892 893 static int mtk_hsdma_probe(struct platform_device *pdev) 894 { 895 struct mtk_hsdma_device *hsdma; 896 struct mtk_hsdma_vchan *vc; 897 struct dma_device *dd; 898 int i, err; 899 900 hsdma = devm_kzalloc(&pdev->dev, sizeof(*hsdma), GFP_KERNEL); 901 if (!hsdma) 902 return -ENOMEM; 903 904 dd = &hsdma->ddev; 905 906 hsdma->base = devm_platform_ioremap_resource(pdev, 0); 907 if (IS_ERR(hsdma->base)) 908 return PTR_ERR(hsdma->base); 909 910 hsdma->soc = of_device_get_match_data(&pdev->dev); 911 if (!hsdma->soc) { 912 dev_err(&pdev->dev, "No device match found\n"); 913 return -ENODEV; 914 } 915 916 hsdma->clk = devm_clk_get(&pdev->dev, "hsdma"); 917 if (IS_ERR(hsdma->clk)) { 918 dev_err(&pdev->dev, "No clock for %s\n", 919 dev_name(&pdev->dev)); 920 return PTR_ERR(hsdma->clk); 921 } 922 923 err = platform_get_irq(pdev, 0); 924 if (err < 0) 925 return err; 926 hsdma->irq = err; 927 928 refcount_set(&hsdma->pc_refcnt, 0); 929 spin_lock_init(&hsdma->lock); 930 931 dma_cap_set(DMA_MEMCPY, dd->cap_mask); 932 933 dd->copy_align = MTK_HSDMA_ALIGN_SIZE; 934 dd->device_alloc_chan_resources = mtk_hsdma_alloc_chan_resources; 935 dd->device_free_chan_resources = mtk_hsdma_free_chan_resources; 936 dd->device_tx_status = mtk_hsdma_tx_status; 937 dd->device_issue_pending = mtk_hsdma_issue_pending; 938 dd->device_prep_dma_memcpy = mtk_hsdma_prep_dma_memcpy; 939 dd->device_terminate_all = mtk_hsdma_terminate_all; 940 dd->src_addr_widths = MTK_HSDMA_DMA_BUSWIDTHS; 941 dd->dst_addr_widths = MTK_HSDMA_DMA_BUSWIDTHS; 942 dd->directions = BIT(DMA_MEM_TO_MEM); 943 dd->residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT; 944 dd->dev = &pdev->dev; 945 INIT_LIST_HEAD(&dd->channels); 946 947 hsdma->dma_requests = MTK_HSDMA_NR_VCHANS; 948 if (pdev->dev.of_node && of_property_read_u32(pdev->dev.of_node, 949 "dma-requests", 950 &hsdma->dma_requests)) { 951 dev_info(&pdev->dev, 952 "Using %u as missing dma-requests property\n", 953 MTK_HSDMA_NR_VCHANS); 954 } 955 956 hsdma->pc = devm_kcalloc(&pdev->dev, MTK_HSDMA_NR_MAX_PCHANS, 957 sizeof(*hsdma->pc), GFP_KERNEL); 958 if (!hsdma->pc) 959 return -ENOMEM; 960 961 hsdma->vc = devm_kcalloc(&pdev->dev, hsdma->dma_requests, 962 sizeof(*hsdma->vc), GFP_KERNEL); 963 if (!hsdma->vc) 964 return -ENOMEM; 965 966 for (i = 0; i < hsdma->dma_requests; i++) { 967 vc = &hsdma->vc[i]; 968 vc->vc.desc_free = mtk_hsdma_vdesc_free; 969 vchan_init(&vc->vc, dd); 970 init_completion(&vc->issue_completion); 971 INIT_LIST_HEAD(&vc->desc_hw_processing); 972 } 973 974 err = dma_async_device_register(dd); 975 if (err) 976 return err; 977 978 err = of_dma_controller_register(pdev->dev.of_node, 979 of_dma_xlate_by_chan_id, hsdma); 980 if (err) { 981 dev_err(&pdev->dev, 982 "MediaTek HSDMA OF registration failed %d\n", err); 983 goto err_unregister; 984 } 985 986 mtk_hsdma_hw_init(hsdma); 987 988 err = devm_request_irq(&pdev->dev, hsdma->irq, 989 mtk_hsdma_irq, 0, 990 dev_name(&pdev->dev), hsdma); 991 if (err) { 992 dev_err(&pdev->dev, 993 "request_irq failed with err %d\n", err); 994 goto err_free; 995 } 996 997 platform_set_drvdata(pdev, hsdma); 998 999 dev_info(&pdev->dev, "MediaTek HSDMA driver registered\n"); 1000 1001 return 0; 1002 1003 err_free: 1004 mtk_hsdma_hw_deinit(hsdma); 1005 of_dma_controller_free(pdev->dev.of_node); 1006 err_unregister: 1007 dma_async_device_unregister(dd); 1008 1009 return err; 1010 } 1011 1012 static int mtk_hsdma_remove(struct platform_device *pdev) 1013 { 1014 struct mtk_hsdma_device *hsdma = platform_get_drvdata(pdev); 1015 struct mtk_hsdma_vchan *vc; 1016 int i; 1017 1018 /* Kill VC task */ 1019 for (i = 0; i < hsdma->dma_requests; i++) { 1020 vc = &hsdma->vc[i]; 1021 1022 list_del(&vc->vc.chan.device_node); 1023 tasklet_kill(&vc->vc.task); 1024 } 1025 1026 /* Disable DMA interrupt */ 1027 mtk_dma_write(hsdma, MTK_HSDMA_INT_ENABLE, 0); 1028 1029 /* Waits for any pending IRQ handlers to complete */ 1030 synchronize_irq(hsdma->irq); 1031 1032 /* Disable hardware */ 1033 mtk_hsdma_hw_deinit(hsdma); 1034 1035 dma_async_device_unregister(&hsdma->ddev); 1036 of_dma_controller_free(pdev->dev.of_node); 1037 1038 return 0; 1039 } 1040 1041 static struct platform_driver mtk_hsdma_driver = { 1042 .probe = mtk_hsdma_probe, 1043 .remove = mtk_hsdma_remove, 1044 .driver = { 1045 .name = KBUILD_MODNAME, 1046 .of_match_table = mtk_hsdma_match, 1047 }, 1048 }; 1049 module_platform_driver(mtk_hsdma_driver); 1050 1051 MODULE_DESCRIPTION("MediaTek High-Speed DMA Controller Driver"); 1052 MODULE_AUTHOR("Sean Wang <sean.wang@mediatek.com>"); 1053 MODULE_LICENSE("GPL v2"); 1054