1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * DMA driver for Altera mSGDMA IP core 4 * 5 * Copyright (C) 2017 Stefan Roese <sr@denx.de> 6 * 7 * Based on drivers/dma/xilinx/zynqmp_dma.c, which is: 8 * Copyright (C) 2016 Xilinx, Inc. All rights reserved. 9 */ 10 11 #include <linux/bitops.h> 12 #include <linux/delay.h> 13 #include <linux/dma-mapping.h> 14 #include <linux/dmapool.h> 15 #include <linux/init.h> 16 #include <linux/interrupt.h> 17 #include <linux/io.h> 18 #include <linux/iopoll.h> 19 #include <linux/module.h> 20 #include <linux/platform_device.h> 21 #include <linux/slab.h> 22 23 #include "dmaengine.h" 24 25 #define MSGDMA_MAX_TRANS_LEN U32_MAX 26 #define MSGDMA_DESC_NUM 1024 27 28 /** 29 * struct msgdma_extended_desc - implements an extended descriptor 30 * @read_addr_lo: data buffer source address low bits 31 * @write_addr_lo: data buffer destination address low bits 32 * @len: the number of bytes to transfer per descriptor 33 * @burst_seq_num: bit 31:24 write burst 34 * bit 23:16 read burst 35 * bit 15:00 sequence number 36 * @stride: bit 31:16 write stride 37 * bit 15:00 read stride 38 * @read_addr_hi: data buffer source address high bits 39 * @write_addr_hi: data buffer destination address high bits 40 * @control: characteristics of the transfer 41 */ 42 struct msgdma_extended_desc { 43 u32 read_addr_lo; 44 u32 write_addr_lo; 45 u32 len; 46 u32 burst_seq_num; 47 u32 stride; 48 u32 read_addr_hi; 49 u32 write_addr_hi; 50 u32 control; 51 }; 52 53 /* mSGDMA descriptor control field bit definitions */ 54 #define MSGDMA_DESC_CTL_SET_CH(x) ((x) & 0xff) 55 #define MSGDMA_DESC_CTL_GEN_SOP BIT(8) 56 #define MSGDMA_DESC_CTL_GEN_EOP BIT(9) 57 #define MSGDMA_DESC_CTL_PARK_READS BIT(10) 58 #define MSGDMA_DESC_CTL_PARK_WRITES BIT(11) 59 #define MSGDMA_DESC_CTL_END_ON_EOP BIT(12) 60 #define MSGDMA_DESC_CTL_END_ON_LEN BIT(13) 61 #define MSGDMA_DESC_CTL_TR_COMP_IRQ BIT(14) 62 #define MSGDMA_DESC_CTL_EARLY_IRQ BIT(15) 63 #define MSGDMA_DESC_CTL_TR_ERR_IRQ GENMASK(23, 16) 64 #define MSGDMA_DESC_CTL_EARLY_DONE BIT(24) 65 66 /* 67 * Writing "1" the "go" bit commits the entire descriptor into the 68 * descriptor FIFO(s) 69 */ 70 #define MSGDMA_DESC_CTL_GO BIT(31) 71 72 /* Tx buffer control flags */ 73 #define MSGDMA_DESC_CTL_TX_FIRST (MSGDMA_DESC_CTL_GEN_SOP | \ 74 MSGDMA_DESC_CTL_TR_ERR_IRQ | \ 75 MSGDMA_DESC_CTL_GO) 76 77 #define MSGDMA_DESC_CTL_TX_MIDDLE (MSGDMA_DESC_CTL_TR_ERR_IRQ | \ 78 MSGDMA_DESC_CTL_GO) 79 80 #define MSGDMA_DESC_CTL_TX_LAST (MSGDMA_DESC_CTL_GEN_EOP | \ 81 MSGDMA_DESC_CTL_TR_COMP_IRQ | \ 82 MSGDMA_DESC_CTL_TR_ERR_IRQ | \ 83 MSGDMA_DESC_CTL_GO) 84 85 #define MSGDMA_DESC_CTL_TX_SINGLE (MSGDMA_DESC_CTL_GEN_SOP | \ 86 MSGDMA_DESC_CTL_GEN_EOP | \ 87 MSGDMA_DESC_CTL_TR_COMP_IRQ | \ 88 MSGDMA_DESC_CTL_TR_ERR_IRQ | \ 89 MSGDMA_DESC_CTL_GO) 90 91 #define MSGDMA_DESC_CTL_RX_SINGLE (MSGDMA_DESC_CTL_END_ON_EOP | \ 92 MSGDMA_DESC_CTL_END_ON_LEN | \ 93 MSGDMA_DESC_CTL_TR_COMP_IRQ | \ 94 MSGDMA_DESC_CTL_EARLY_IRQ | \ 95 MSGDMA_DESC_CTL_TR_ERR_IRQ | \ 96 MSGDMA_DESC_CTL_GO) 97 98 /* mSGDMA extended descriptor stride definitions */ 99 #define MSGDMA_DESC_STRIDE_RD 0x00000001 100 #define MSGDMA_DESC_STRIDE_WR 0x00010000 101 #define MSGDMA_DESC_STRIDE_RW 0x00010001 102 103 /* mSGDMA dispatcher control and status register map */ 104 #define MSGDMA_CSR_STATUS 0x00 /* Read / Clear */ 105 #define MSGDMA_CSR_CONTROL 0x04 /* Read / Write */ 106 #define MSGDMA_CSR_RW_FILL_LEVEL 0x08 /* 31:16 - write fill level */ 107 /* 15:00 - read fill level */ 108 #define MSGDMA_CSR_RESP_FILL_LEVEL 0x0c /* response FIFO fill level */ 109 #define MSGDMA_CSR_RW_SEQ_NUM 0x10 /* 31:16 - write seq number */ 110 /* 15:00 - read seq number */ 111 112 /* mSGDMA CSR status register bit definitions */ 113 #define MSGDMA_CSR_STAT_BUSY BIT(0) 114 #define MSGDMA_CSR_STAT_DESC_BUF_EMPTY BIT(1) 115 #define MSGDMA_CSR_STAT_DESC_BUF_FULL BIT(2) 116 #define MSGDMA_CSR_STAT_RESP_BUF_EMPTY BIT(3) 117 #define MSGDMA_CSR_STAT_RESP_BUF_FULL BIT(4) 118 #define MSGDMA_CSR_STAT_STOPPED BIT(5) 119 #define MSGDMA_CSR_STAT_RESETTING BIT(6) 120 #define MSGDMA_CSR_STAT_STOPPED_ON_ERR BIT(7) 121 #define MSGDMA_CSR_STAT_STOPPED_ON_EARLY BIT(8) 122 #define MSGDMA_CSR_STAT_IRQ BIT(9) 123 #define MSGDMA_CSR_STAT_MASK GENMASK(9, 0) 124 #define MSGDMA_CSR_STAT_MASK_WITHOUT_IRQ GENMASK(8, 0) 125 126 #define DESC_EMPTY (MSGDMA_CSR_STAT_DESC_BUF_EMPTY | \ 127 MSGDMA_CSR_STAT_RESP_BUF_EMPTY) 128 129 /* mSGDMA CSR control register bit definitions */ 130 #define MSGDMA_CSR_CTL_STOP BIT(0) 131 #define MSGDMA_CSR_CTL_RESET BIT(1) 132 #define MSGDMA_CSR_CTL_STOP_ON_ERR BIT(2) 133 #define MSGDMA_CSR_CTL_STOP_ON_EARLY BIT(3) 134 #define MSGDMA_CSR_CTL_GLOBAL_INTR BIT(4) 135 #define MSGDMA_CSR_CTL_STOP_DESCS BIT(5) 136 137 /* mSGDMA CSR fill level bits */ 138 #define MSGDMA_CSR_WR_FILL_LEVEL_GET(v) (((v) & 0xffff0000) >> 16) 139 #define MSGDMA_CSR_RD_FILL_LEVEL_GET(v) ((v) & 0x0000ffff) 140 #define MSGDMA_CSR_RESP_FILL_LEVEL_GET(v) ((v) & 0x0000ffff) 141 142 #define MSGDMA_CSR_SEQ_NUM_GET(v) (((v) & 0xffff0000) >> 16) 143 144 /* mSGDMA response register map */ 145 #define MSGDMA_RESP_BYTES_TRANSFERRED 0x00 146 #define MSGDMA_RESP_STATUS 0x04 147 148 /* mSGDMA response register bit definitions */ 149 #define MSGDMA_RESP_EARLY_TERM BIT(8) 150 #define MSGDMA_RESP_ERR_MASK 0xff 151 152 /** 153 * struct msgdma_sw_desc - implements a sw descriptor 154 * @async_tx: support for the async_tx api 155 * @hw_desc: assosiated HW descriptor 156 * @node: node to move from the free list to the tx list 157 * @tx_list: transmit list node 158 */ 159 struct msgdma_sw_desc { 160 struct dma_async_tx_descriptor async_tx; 161 struct msgdma_extended_desc hw_desc; 162 struct list_head node; 163 struct list_head tx_list; 164 }; 165 166 /* 167 * struct msgdma_device - DMA device structure 168 */ 169 struct msgdma_device { 170 spinlock_t lock; 171 struct device *dev; 172 struct tasklet_struct irq_tasklet; 173 struct list_head pending_list; 174 struct list_head free_list; 175 struct list_head active_list; 176 struct list_head done_list; 177 u32 desc_free_cnt; 178 bool idle; 179 180 struct dma_device dmadev; 181 struct dma_chan dmachan; 182 dma_addr_t hw_desq; 183 struct msgdma_sw_desc *sw_desq; 184 unsigned int npendings; 185 186 struct dma_slave_config slave_cfg; 187 188 int irq; 189 190 /* mSGDMA controller */ 191 void __iomem *csr; 192 193 /* mSGDMA descriptors */ 194 void __iomem *desc; 195 196 /* mSGDMA response */ 197 void __iomem *resp; 198 }; 199 200 #define to_mdev(chan) container_of(chan, struct msgdma_device, dmachan) 201 #define tx_to_desc(tx) container_of(tx, struct msgdma_sw_desc, async_tx) 202 203 /** 204 * msgdma_get_descriptor - Get the sw descriptor from the pool 205 * @mdev: Pointer to the Altera mSGDMA device structure 206 * 207 * Return: The sw descriptor 208 */ 209 static struct msgdma_sw_desc *msgdma_get_descriptor(struct msgdma_device *mdev) 210 { 211 struct msgdma_sw_desc *desc; 212 unsigned long flags; 213 214 spin_lock_irqsave(&mdev->lock, flags); 215 desc = list_first_entry(&mdev->free_list, struct msgdma_sw_desc, node); 216 list_del(&desc->node); 217 spin_unlock_irqrestore(&mdev->lock, flags); 218 219 INIT_LIST_HEAD(&desc->tx_list); 220 221 return desc; 222 } 223 224 /** 225 * msgdma_free_descriptor - Issue pending transactions 226 * @mdev: Pointer to the Altera mSGDMA device structure 227 * @desc: Transaction descriptor pointer 228 */ 229 static void msgdma_free_descriptor(struct msgdma_device *mdev, 230 struct msgdma_sw_desc *desc) 231 { 232 struct msgdma_sw_desc *child, *next; 233 234 mdev->desc_free_cnt++; 235 list_add_tail(&desc->node, &mdev->free_list); 236 list_for_each_entry_safe(child, next, &desc->tx_list, node) { 237 mdev->desc_free_cnt++; 238 list_move_tail(&child->node, &mdev->free_list); 239 } 240 } 241 242 /** 243 * msgdma_free_desc_list - Free descriptors list 244 * @mdev: Pointer to the Altera mSGDMA device structure 245 * @list: List to parse and delete the descriptor 246 */ 247 static void msgdma_free_desc_list(struct msgdma_device *mdev, 248 struct list_head *list) 249 { 250 struct msgdma_sw_desc *desc, *next; 251 252 list_for_each_entry_safe(desc, next, list, node) 253 msgdma_free_descriptor(mdev, desc); 254 } 255 256 /** 257 * msgdma_desc_config - Configure the descriptor 258 * @desc: Hw descriptor pointer 259 * @dst: Destination buffer address 260 * @src: Source buffer address 261 * @len: Transfer length 262 * @stride: Read/write stride value to set 263 */ 264 static void msgdma_desc_config(struct msgdma_extended_desc *desc, 265 dma_addr_t dst, dma_addr_t src, size_t len, 266 u32 stride) 267 { 268 /* Set lower 32bits of src & dst addresses in the descriptor */ 269 desc->read_addr_lo = lower_32_bits(src); 270 desc->write_addr_lo = lower_32_bits(dst); 271 272 /* Set upper 32bits of src & dst addresses in the descriptor */ 273 desc->read_addr_hi = upper_32_bits(src); 274 desc->write_addr_hi = upper_32_bits(dst); 275 276 desc->len = len; 277 desc->stride = stride; 278 desc->burst_seq_num = 0; /* 0 will result in max burst length */ 279 280 /* 281 * Don't set interrupt on xfer end yet, this will be done later 282 * for the "last" descriptor 283 */ 284 desc->control = MSGDMA_DESC_CTL_TR_ERR_IRQ | MSGDMA_DESC_CTL_GO | 285 MSGDMA_DESC_CTL_END_ON_LEN; 286 } 287 288 /** 289 * msgdma_desc_config_eod - Mark the descriptor as end descriptor 290 * @desc: Hw descriptor pointer 291 */ 292 static void msgdma_desc_config_eod(struct msgdma_extended_desc *desc) 293 { 294 desc->control |= MSGDMA_DESC_CTL_TR_COMP_IRQ; 295 } 296 297 /** 298 * msgdma_tx_submit - Submit DMA transaction 299 * @tx: Async transaction descriptor pointer 300 * 301 * Return: cookie value 302 */ 303 static dma_cookie_t msgdma_tx_submit(struct dma_async_tx_descriptor *tx) 304 { 305 struct msgdma_device *mdev = to_mdev(tx->chan); 306 struct msgdma_sw_desc *new; 307 dma_cookie_t cookie; 308 unsigned long flags; 309 310 new = tx_to_desc(tx); 311 spin_lock_irqsave(&mdev->lock, flags); 312 cookie = dma_cookie_assign(tx); 313 314 list_add_tail(&new->node, &mdev->pending_list); 315 spin_unlock_irqrestore(&mdev->lock, flags); 316 317 return cookie; 318 } 319 320 /** 321 * msgdma_prep_memcpy - prepare descriptors for memcpy transaction 322 * @dchan: DMA channel 323 * @dma_dst: Destination buffer address 324 * @dma_src: Source buffer address 325 * @len: Transfer length 326 * @flags: transfer ack flags 327 * 328 * Return: Async transaction descriptor on success and NULL on failure 329 */ 330 static struct dma_async_tx_descriptor * 331 msgdma_prep_memcpy(struct dma_chan *dchan, dma_addr_t dma_dst, 332 dma_addr_t dma_src, size_t len, ulong flags) 333 { 334 struct msgdma_device *mdev = to_mdev(dchan); 335 struct msgdma_sw_desc *new, *first = NULL; 336 struct msgdma_extended_desc *desc; 337 size_t copy; 338 u32 desc_cnt; 339 unsigned long irqflags; 340 341 desc_cnt = DIV_ROUND_UP(len, MSGDMA_MAX_TRANS_LEN); 342 343 spin_lock_irqsave(&mdev->lock, irqflags); 344 if (desc_cnt > mdev->desc_free_cnt) { 345 spin_unlock_irqrestore(&mdev->lock, irqflags); 346 dev_dbg(mdev->dev, "mdev %p descs are not available\n", mdev); 347 return NULL; 348 } 349 mdev->desc_free_cnt -= desc_cnt; 350 spin_unlock_irqrestore(&mdev->lock, irqflags); 351 352 do { 353 /* Allocate and populate the descriptor */ 354 new = msgdma_get_descriptor(mdev); 355 356 copy = min_t(size_t, len, MSGDMA_MAX_TRANS_LEN); 357 desc = &new->hw_desc; 358 msgdma_desc_config(desc, dma_dst, dma_src, copy, 359 MSGDMA_DESC_STRIDE_RW); 360 len -= copy; 361 dma_src += copy; 362 dma_dst += copy; 363 if (!first) 364 first = new; 365 else 366 list_add_tail(&new->node, &first->tx_list); 367 } while (len); 368 369 msgdma_desc_config_eod(desc); 370 async_tx_ack(&first->async_tx); 371 first->async_tx.flags = flags; 372 373 return &first->async_tx; 374 } 375 376 /** 377 * msgdma_prep_slave_sg - prepare descriptors for a slave sg transaction 378 * 379 * @dchan: DMA channel 380 * @sgl: Destination scatter list 381 * @sg_len: Number of entries in destination scatter list 382 * @dir: DMA transfer direction 383 * @flags: transfer ack flags 384 * @context: transfer context (unused) 385 */ 386 static struct dma_async_tx_descriptor * 387 msgdma_prep_slave_sg(struct dma_chan *dchan, struct scatterlist *sgl, 388 unsigned int sg_len, enum dma_transfer_direction dir, 389 unsigned long flags, void *context) 390 391 { 392 struct msgdma_device *mdev = to_mdev(dchan); 393 struct dma_slave_config *cfg = &mdev->slave_cfg; 394 struct msgdma_sw_desc *new, *first = NULL; 395 void *desc = NULL; 396 size_t len, avail; 397 dma_addr_t dma_dst, dma_src; 398 u32 desc_cnt = 0, i; 399 struct scatterlist *sg; 400 u32 stride; 401 unsigned long irqflags; 402 403 for_each_sg(sgl, sg, sg_len, i) 404 desc_cnt += DIV_ROUND_UP(sg_dma_len(sg), MSGDMA_MAX_TRANS_LEN); 405 406 spin_lock_irqsave(&mdev->lock, irqflags); 407 if (desc_cnt > mdev->desc_free_cnt) { 408 spin_unlock_irqrestore(&mdev->lock, irqflags); 409 dev_dbg(mdev->dev, "mdev %p descs are not available\n", mdev); 410 return NULL; 411 } 412 mdev->desc_free_cnt -= desc_cnt; 413 spin_unlock_irqrestore(&mdev->lock, irqflags); 414 415 avail = sg_dma_len(sgl); 416 417 /* Run until we are out of scatterlist entries */ 418 while (true) { 419 /* Allocate and populate the descriptor */ 420 new = msgdma_get_descriptor(mdev); 421 422 desc = &new->hw_desc; 423 len = min_t(size_t, avail, MSGDMA_MAX_TRANS_LEN); 424 425 if (dir == DMA_MEM_TO_DEV) { 426 dma_src = sg_dma_address(sgl) + sg_dma_len(sgl) - avail; 427 dma_dst = cfg->dst_addr; 428 stride = MSGDMA_DESC_STRIDE_RD; 429 } else { 430 dma_src = cfg->src_addr; 431 dma_dst = sg_dma_address(sgl) + sg_dma_len(sgl) - avail; 432 stride = MSGDMA_DESC_STRIDE_WR; 433 } 434 msgdma_desc_config(desc, dma_dst, dma_src, len, stride); 435 avail -= len; 436 437 if (!first) 438 first = new; 439 else 440 list_add_tail(&new->node, &first->tx_list); 441 442 /* Fetch the next scatterlist entry */ 443 if (avail == 0) { 444 if (sg_len == 0) 445 break; 446 sgl = sg_next(sgl); 447 if (sgl == NULL) 448 break; 449 sg_len--; 450 avail = sg_dma_len(sgl); 451 } 452 } 453 454 msgdma_desc_config_eod(desc); 455 first->async_tx.flags = flags; 456 457 return &first->async_tx; 458 } 459 460 static int msgdma_dma_config(struct dma_chan *dchan, 461 struct dma_slave_config *config) 462 { 463 struct msgdma_device *mdev = to_mdev(dchan); 464 465 memcpy(&mdev->slave_cfg, config, sizeof(*config)); 466 467 return 0; 468 } 469 470 static void msgdma_reset(struct msgdma_device *mdev) 471 { 472 u32 val; 473 int ret; 474 475 /* Reset mSGDMA */ 476 iowrite32(MSGDMA_CSR_STAT_MASK, mdev->csr + MSGDMA_CSR_STATUS); 477 iowrite32(MSGDMA_CSR_CTL_RESET, mdev->csr + MSGDMA_CSR_CONTROL); 478 479 ret = readl_poll_timeout(mdev->csr + MSGDMA_CSR_STATUS, val, 480 (val & MSGDMA_CSR_STAT_RESETTING) == 0, 481 1, 10000); 482 if (ret) 483 dev_err(mdev->dev, "DMA channel did not reset\n"); 484 485 /* Clear all status bits */ 486 iowrite32(MSGDMA_CSR_STAT_MASK, mdev->csr + MSGDMA_CSR_STATUS); 487 488 /* Enable the DMA controller including interrupts */ 489 iowrite32(MSGDMA_CSR_CTL_STOP_ON_ERR | MSGDMA_CSR_CTL_STOP_ON_EARLY | 490 MSGDMA_CSR_CTL_GLOBAL_INTR, mdev->csr + MSGDMA_CSR_CONTROL); 491 492 mdev->idle = true; 493 }; 494 495 static void msgdma_copy_one(struct msgdma_device *mdev, 496 struct msgdma_sw_desc *desc) 497 { 498 void __iomem *hw_desc = mdev->desc; 499 500 /* 501 * Check if the DESC FIFO it not full. If its full, we need to wait 502 * for at least one entry to become free again 503 */ 504 while (ioread32(mdev->csr + MSGDMA_CSR_STATUS) & 505 MSGDMA_CSR_STAT_DESC_BUF_FULL) 506 mdelay(1); 507 508 /* 509 * The descriptor needs to get copied into the descriptor FIFO 510 * of the DMA controller. The descriptor will get flushed to the 511 * FIFO, once the last word (control word) is written. Since we 512 * are not 100% sure that memcpy() writes all word in the "correct" 513 * oder (address from low to high) on all architectures, we make 514 * sure this control word is written last by single coding it and 515 * adding some write-barriers here. 516 */ 517 memcpy((void __force *)hw_desc, &desc->hw_desc, 518 sizeof(desc->hw_desc) - sizeof(u32)); 519 520 /* Write control word last to flush this descriptor into the FIFO */ 521 mdev->idle = false; 522 wmb(); 523 iowrite32(desc->hw_desc.control, hw_desc + 524 offsetof(struct msgdma_extended_desc, control)); 525 wmb(); 526 } 527 528 /** 529 * msgdma_copy_desc_to_fifo - copy descriptor(s) into controller FIFO 530 * @mdev: Pointer to the Altera mSGDMA device structure 531 * @desc: Transaction descriptor pointer 532 */ 533 static void msgdma_copy_desc_to_fifo(struct msgdma_device *mdev, 534 struct msgdma_sw_desc *desc) 535 { 536 struct msgdma_sw_desc *sdesc, *next; 537 538 msgdma_copy_one(mdev, desc); 539 540 list_for_each_entry_safe(sdesc, next, &desc->tx_list, node) 541 msgdma_copy_one(mdev, sdesc); 542 } 543 544 /** 545 * msgdma_start_transfer - Initiate the new transfer 546 * @mdev: Pointer to the Altera mSGDMA device structure 547 */ 548 static void msgdma_start_transfer(struct msgdma_device *mdev) 549 { 550 struct msgdma_sw_desc *desc; 551 552 if (!mdev->idle) 553 return; 554 555 desc = list_first_entry_or_null(&mdev->pending_list, 556 struct msgdma_sw_desc, node); 557 if (!desc) 558 return; 559 560 list_splice_tail_init(&mdev->pending_list, &mdev->active_list); 561 msgdma_copy_desc_to_fifo(mdev, desc); 562 } 563 564 /** 565 * msgdma_issue_pending - Issue pending transactions 566 * @chan: DMA channel pointer 567 */ 568 static void msgdma_issue_pending(struct dma_chan *chan) 569 { 570 struct msgdma_device *mdev = to_mdev(chan); 571 unsigned long flags; 572 573 spin_lock_irqsave(&mdev->lock, flags); 574 msgdma_start_transfer(mdev); 575 spin_unlock_irqrestore(&mdev->lock, flags); 576 } 577 578 /** 579 * msgdma_chan_desc_cleanup - Cleanup the completed descriptors 580 * @mdev: Pointer to the Altera mSGDMA device structure 581 */ 582 static void msgdma_chan_desc_cleanup(struct msgdma_device *mdev) 583 { 584 struct msgdma_sw_desc *desc, *next; 585 586 list_for_each_entry_safe(desc, next, &mdev->done_list, node) { 587 dma_async_tx_callback callback; 588 void *callback_param; 589 590 list_del(&desc->node); 591 592 callback = desc->async_tx.callback; 593 callback_param = desc->async_tx.callback_param; 594 if (callback) { 595 spin_unlock(&mdev->lock); 596 callback(callback_param); 597 spin_lock(&mdev->lock); 598 } 599 600 /* Run any dependencies, then free the descriptor */ 601 msgdma_free_descriptor(mdev, desc); 602 } 603 } 604 605 /** 606 * msgdma_complete_descriptor - Mark the active descriptor as complete 607 * @mdev: Pointer to the Altera mSGDMA device structure 608 */ 609 static void msgdma_complete_descriptor(struct msgdma_device *mdev) 610 { 611 struct msgdma_sw_desc *desc; 612 613 desc = list_first_entry_or_null(&mdev->active_list, 614 struct msgdma_sw_desc, node); 615 if (!desc) 616 return; 617 list_del(&desc->node); 618 dma_cookie_complete(&desc->async_tx); 619 list_add_tail(&desc->node, &mdev->done_list); 620 } 621 622 /** 623 * msgdma_free_descriptors - Free channel descriptors 624 * @mdev: Pointer to the Altera mSGDMA device structure 625 */ 626 static void msgdma_free_descriptors(struct msgdma_device *mdev) 627 { 628 msgdma_free_desc_list(mdev, &mdev->active_list); 629 msgdma_free_desc_list(mdev, &mdev->pending_list); 630 msgdma_free_desc_list(mdev, &mdev->done_list); 631 } 632 633 /** 634 * msgdma_free_chan_resources - Free channel resources 635 * @dchan: DMA channel pointer 636 */ 637 static void msgdma_free_chan_resources(struct dma_chan *dchan) 638 { 639 struct msgdma_device *mdev = to_mdev(dchan); 640 unsigned long flags; 641 642 spin_lock_irqsave(&mdev->lock, flags); 643 msgdma_free_descriptors(mdev); 644 spin_unlock_irqrestore(&mdev->lock, flags); 645 kfree(mdev->sw_desq); 646 } 647 648 /** 649 * msgdma_alloc_chan_resources - Allocate channel resources 650 * @dchan: DMA channel 651 * 652 * Return: Number of descriptors on success and failure value on error 653 */ 654 static int msgdma_alloc_chan_resources(struct dma_chan *dchan) 655 { 656 struct msgdma_device *mdev = to_mdev(dchan); 657 struct msgdma_sw_desc *desc; 658 int i; 659 660 mdev->sw_desq = kcalloc(MSGDMA_DESC_NUM, sizeof(*desc), GFP_NOWAIT); 661 if (!mdev->sw_desq) 662 return -ENOMEM; 663 664 mdev->idle = true; 665 mdev->desc_free_cnt = MSGDMA_DESC_NUM; 666 667 INIT_LIST_HEAD(&mdev->free_list); 668 669 for (i = 0; i < MSGDMA_DESC_NUM; i++) { 670 desc = mdev->sw_desq + i; 671 dma_async_tx_descriptor_init(&desc->async_tx, &mdev->dmachan); 672 desc->async_tx.tx_submit = msgdma_tx_submit; 673 list_add_tail(&desc->node, &mdev->free_list); 674 } 675 676 return MSGDMA_DESC_NUM; 677 } 678 679 /** 680 * msgdma_tasklet - Schedule completion tasklet 681 * @t: Pointer to the Altera sSGDMA channel structure 682 */ 683 static void msgdma_tasklet(struct tasklet_struct *t) 684 { 685 struct msgdma_device *mdev = from_tasklet(mdev, t, irq_tasklet); 686 u32 count; 687 u32 __maybe_unused size; 688 u32 __maybe_unused status; 689 unsigned long flags; 690 691 spin_lock_irqsave(&mdev->lock, flags); 692 693 /* Read number of responses that are available */ 694 count = ioread32(mdev->csr + MSGDMA_CSR_RESP_FILL_LEVEL); 695 dev_dbg(mdev->dev, "%s (%d): response count=%d\n", 696 __func__, __LINE__, count); 697 698 while (count--) { 699 /* 700 * Read both longwords to purge this response from the FIFO 701 * On Avalon-MM implementations, size and status do not 702 * have any real values, like transferred bytes or error 703 * bits. So we need to just drop these values. 704 */ 705 size = ioread32(mdev->resp + MSGDMA_RESP_BYTES_TRANSFERRED); 706 status = ioread32(mdev->resp + MSGDMA_RESP_STATUS); 707 708 msgdma_complete_descriptor(mdev); 709 msgdma_chan_desc_cleanup(mdev); 710 } 711 712 spin_unlock_irqrestore(&mdev->lock, flags); 713 } 714 715 /** 716 * msgdma_irq_handler - Altera mSGDMA Interrupt handler 717 * @irq: IRQ number 718 * @data: Pointer to the Altera mSGDMA device structure 719 * 720 * Return: IRQ_HANDLED/IRQ_NONE 721 */ 722 static irqreturn_t msgdma_irq_handler(int irq, void *data) 723 { 724 struct msgdma_device *mdev = data; 725 u32 status; 726 727 status = ioread32(mdev->csr + MSGDMA_CSR_STATUS); 728 if ((status & MSGDMA_CSR_STAT_BUSY) == 0) { 729 /* Start next transfer if the DMA controller is idle */ 730 spin_lock(&mdev->lock); 731 mdev->idle = true; 732 msgdma_start_transfer(mdev); 733 spin_unlock(&mdev->lock); 734 } 735 736 tasklet_schedule(&mdev->irq_tasklet); 737 738 /* Clear interrupt in mSGDMA controller */ 739 iowrite32(MSGDMA_CSR_STAT_IRQ, mdev->csr + MSGDMA_CSR_STATUS); 740 741 return IRQ_HANDLED; 742 } 743 744 /** 745 * msgdma_chan_remove - Channel remove function 746 * @mdev: Pointer to the Altera mSGDMA device structure 747 */ 748 static void msgdma_dev_remove(struct msgdma_device *mdev) 749 { 750 if (!mdev) 751 return; 752 753 devm_free_irq(mdev->dev, mdev->irq, mdev); 754 tasklet_kill(&mdev->irq_tasklet); 755 list_del(&mdev->dmachan.device_node); 756 } 757 758 static int request_and_map(struct platform_device *pdev, const char *name, 759 struct resource **res, void __iomem **ptr) 760 { 761 struct resource *region; 762 struct device *device = &pdev->dev; 763 764 *res = platform_get_resource_byname(pdev, IORESOURCE_MEM, name); 765 if (*res == NULL) { 766 dev_err(device, "resource %s not defined\n", name); 767 return -ENODEV; 768 } 769 770 region = devm_request_mem_region(device, (*res)->start, 771 resource_size(*res), dev_name(device)); 772 if (region == NULL) { 773 dev_err(device, "unable to request %s\n", name); 774 return -EBUSY; 775 } 776 777 *ptr = devm_ioremap(device, region->start, 778 resource_size(region)); 779 if (*ptr == NULL) { 780 dev_err(device, "ioremap of %s failed!", name); 781 return -ENOMEM; 782 } 783 784 return 0; 785 } 786 787 /** 788 * msgdma_probe - Driver probe function 789 * @pdev: Pointer to the platform_device structure 790 * 791 * Return: '0' on success and failure value on error 792 */ 793 static int msgdma_probe(struct platform_device *pdev) 794 { 795 struct msgdma_device *mdev; 796 struct dma_device *dma_dev; 797 struct resource *dma_res; 798 int ret; 799 800 mdev = devm_kzalloc(&pdev->dev, sizeof(*mdev), GFP_NOWAIT); 801 if (!mdev) 802 return -ENOMEM; 803 804 mdev->dev = &pdev->dev; 805 806 /* Map CSR space */ 807 ret = request_and_map(pdev, "csr", &dma_res, &mdev->csr); 808 if (ret) 809 return ret; 810 811 /* Map (extended) descriptor space */ 812 ret = request_and_map(pdev, "desc", &dma_res, &mdev->desc); 813 if (ret) 814 return ret; 815 816 /* Map response space */ 817 ret = request_and_map(pdev, "resp", &dma_res, &mdev->resp); 818 if (ret) 819 return ret; 820 821 platform_set_drvdata(pdev, mdev); 822 823 /* Get interrupt nr from platform data */ 824 mdev->irq = platform_get_irq(pdev, 0); 825 if (mdev->irq < 0) 826 return -ENXIO; 827 828 ret = devm_request_irq(&pdev->dev, mdev->irq, msgdma_irq_handler, 829 0, dev_name(&pdev->dev), mdev); 830 if (ret) 831 return ret; 832 833 tasklet_setup(&mdev->irq_tasklet, msgdma_tasklet); 834 835 dma_cookie_init(&mdev->dmachan); 836 837 spin_lock_init(&mdev->lock); 838 839 INIT_LIST_HEAD(&mdev->active_list); 840 INIT_LIST_HEAD(&mdev->pending_list); 841 INIT_LIST_HEAD(&mdev->done_list); 842 INIT_LIST_HEAD(&mdev->free_list); 843 844 dma_dev = &mdev->dmadev; 845 846 /* Set DMA capabilities */ 847 dma_cap_zero(dma_dev->cap_mask); 848 dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask); 849 dma_cap_set(DMA_SLAVE, dma_dev->cap_mask); 850 851 dma_dev->src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES); 852 dma_dev->dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES); 853 dma_dev->directions = BIT(DMA_MEM_TO_DEV) | BIT(DMA_DEV_TO_MEM) | 854 BIT(DMA_MEM_TO_MEM); 855 dma_dev->residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR; 856 857 /* Init DMA link list */ 858 INIT_LIST_HEAD(&dma_dev->channels); 859 860 /* Set base routines */ 861 dma_dev->device_tx_status = dma_cookie_status; 862 dma_dev->device_issue_pending = msgdma_issue_pending; 863 dma_dev->dev = &pdev->dev; 864 865 dma_dev->copy_align = DMAENGINE_ALIGN_4_BYTES; 866 dma_dev->device_prep_dma_memcpy = msgdma_prep_memcpy; 867 dma_dev->device_prep_slave_sg = msgdma_prep_slave_sg; 868 dma_dev->device_config = msgdma_dma_config; 869 870 dma_dev->device_alloc_chan_resources = msgdma_alloc_chan_resources; 871 dma_dev->device_free_chan_resources = msgdma_free_chan_resources; 872 873 mdev->dmachan.device = dma_dev; 874 list_add_tail(&mdev->dmachan.device_node, &dma_dev->channels); 875 876 /* Set DMA mask to 64 bits */ 877 ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 878 if (ret) { 879 dev_warn(&pdev->dev, "unable to set coherent mask to 64"); 880 ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); 881 if (ret) 882 goto fail; 883 } 884 885 msgdma_reset(mdev); 886 887 ret = dma_async_device_register(dma_dev); 888 if (ret) 889 goto fail; 890 891 dev_notice(&pdev->dev, "Altera mSGDMA driver probe success\n"); 892 893 return 0; 894 895 fail: 896 msgdma_dev_remove(mdev); 897 898 return ret; 899 } 900 901 /** 902 * msgdma_dma_remove - Driver remove function 903 * @pdev: Pointer to the platform_device structure 904 * 905 * Return: Always '0' 906 */ 907 static int msgdma_remove(struct platform_device *pdev) 908 { 909 struct msgdma_device *mdev = platform_get_drvdata(pdev); 910 911 dma_async_device_unregister(&mdev->dmadev); 912 msgdma_dev_remove(mdev); 913 914 dev_notice(&pdev->dev, "Altera mSGDMA driver removed\n"); 915 916 return 0; 917 } 918 919 static struct platform_driver msgdma_driver = { 920 .driver = { 921 .name = "altera-msgdma", 922 }, 923 .probe = msgdma_probe, 924 .remove = msgdma_remove, 925 }; 926 927 module_platform_driver(msgdma_driver); 928 929 MODULE_ALIAS("platform:altera-msgdma"); 930 MODULE_DESCRIPTION("Altera mSGDMA driver"); 931 MODULE_AUTHOR("Stefan Roese <sr@denx.de>"); 932 MODULE_LICENSE("GPL"); 933