1 /* 2 * DMA Engine support for Tsi721 PCIExpress-to-SRIO bridge 3 * 4 * Copyright 2011 Integrated Device Technology, Inc. 5 * Alexandre Bounine <alexandre.bounine@idt.com> 6 * 7 * This program is free software; you can redistribute it and/or modify it 8 * under the terms of the GNU General Public License as published by the Free 9 * Software Foundation; either version 2 of the License, or (at your option) 10 * any later version. 11 * 12 * This program is distributed in the hope that it will be useful, but WITHOUT 13 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 14 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 15 * more details. 16 * 17 * You should have received a copy of the GNU General Public License along with 18 * this program; if not, write to the Free Software Foundation, Inc., 59 19 * Temple Place - Suite 330, Boston, MA 02111-1307, USA. 20 */ 21 22 #include <linux/io.h> 23 #include <linux/errno.h> 24 #include <linux/init.h> 25 #include <linux/ioport.h> 26 #include <linux/kernel.h> 27 #include <linux/module.h> 28 #include <linux/pci.h> 29 #include <linux/rio.h> 30 #include <linux/rio_drv.h> 31 #include <linux/dma-mapping.h> 32 #include <linux/interrupt.h> 33 #include <linux/kfifo.h> 34 #include <linux/delay.h> 35 36 #include "tsi721.h" 37 38 static inline struct tsi721_bdma_chan *to_tsi721_chan(struct dma_chan *chan) 39 { 40 return container_of(chan, struct tsi721_bdma_chan, dchan); 41 } 42 43 static inline struct tsi721_device *to_tsi721(struct dma_device *ddev) 44 { 45 return container_of(ddev, struct rio_mport, dma)->priv; 46 } 47 48 static inline 49 struct tsi721_tx_desc *to_tsi721_desc(struct dma_async_tx_descriptor *txd) 50 { 51 return container_of(txd, struct tsi721_tx_desc, txd); 52 } 53 54 static inline 55 struct tsi721_tx_desc *tsi721_dma_first_active( 56 struct tsi721_bdma_chan *bdma_chan) 57 { 58 return list_first_entry(&bdma_chan->active_list, 59 struct tsi721_tx_desc, desc_node); 60 } 61 62 static int tsi721_bdma_ch_init(struct tsi721_bdma_chan *bdma_chan) 63 { 64 struct tsi721_dma_desc *bd_ptr; 65 struct device *dev = bdma_chan->dchan.device->dev; 66 u64 *sts_ptr; 67 dma_addr_t bd_phys; 68 dma_addr_t sts_phys; 69 int sts_size; 70 int bd_num = bdma_chan->bd_num; 71 72 dev_dbg(dev, "Init Block DMA Engine, CH%d\n", bdma_chan->id); 73 74 /* Allocate space for DMA descriptors */ 75 bd_ptr = dma_zalloc_coherent(dev, 76 bd_num * sizeof(struct tsi721_dma_desc), 77 &bd_phys, GFP_KERNEL); 78 if (!bd_ptr) 79 return -ENOMEM; 80 81 bdma_chan->bd_phys = bd_phys; 82 bdma_chan->bd_base = bd_ptr; 83 84 dev_dbg(dev, "DMA descriptors @ %p (phys = %llx)\n", 85 bd_ptr, (unsigned long long)bd_phys); 86 87 /* Allocate space for descriptor status FIFO */ 88 sts_size = (bd_num >= TSI721_DMA_MINSTSSZ) ? 89 bd_num : TSI721_DMA_MINSTSSZ; 90 sts_size = roundup_pow_of_two(sts_size); 91 sts_ptr = dma_zalloc_coherent(dev, 92 sts_size * sizeof(struct tsi721_dma_sts), 93 &sts_phys, GFP_KERNEL); 94 if (!sts_ptr) { 95 /* Free space allocated for DMA descriptors */ 96 dma_free_coherent(dev, 97 bd_num * sizeof(struct tsi721_dma_desc), 98 bd_ptr, bd_phys); 99 bdma_chan->bd_base = NULL; 100 return -ENOMEM; 101 } 102 103 bdma_chan->sts_phys = sts_phys; 104 bdma_chan->sts_base = sts_ptr; 105 bdma_chan->sts_size = sts_size; 106 107 dev_dbg(dev, 108 "desc status FIFO @ %p (phys = %llx) size=0x%x\n", 109 sts_ptr, (unsigned long long)sts_phys, sts_size); 110 111 /* Initialize DMA descriptors ring */ 112 bd_ptr[bd_num - 1].type_id = cpu_to_le32(DTYPE3 << 29); 113 bd_ptr[bd_num - 1].next_lo = cpu_to_le32((u64)bd_phys & 114 TSI721_DMAC_DPTRL_MASK); 115 bd_ptr[bd_num - 1].next_hi = cpu_to_le32((u64)bd_phys >> 32); 116 117 /* Setup DMA descriptor pointers */ 118 iowrite32(((u64)bd_phys >> 32), 119 bdma_chan->regs + TSI721_DMAC_DPTRH); 120 iowrite32(((u64)bd_phys & TSI721_DMAC_DPTRL_MASK), 121 bdma_chan->regs + TSI721_DMAC_DPTRL); 122 123 /* Setup descriptor status FIFO */ 124 iowrite32(((u64)sts_phys >> 32), 125 bdma_chan->regs + TSI721_DMAC_DSBH); 126 iowrite32(((u64)sts_phys & TSI721_DMAC_DSBL_MASK), 127 bdma_chan->regs + TSI721_DMAC_DSBL); 128 iowrite32(TSI721_DMAC_DSSZ_SIZE(sts_size), 129 bdma_chan->regs + TSI721_DMAC_DSSZ); 130 131 /* Clear interrupt bits */ 132 iowrite32(TSI721_DMAC_INT_ALL, 133 bdma_chan->regs + TSI721_DMAC_INT); 134 135 ioread32(bdma_chan->regs + TSI721_DMAC_INT); 136 137 /* Toggle DMA channel initialization */ 138 iowrite32(TSI721_DMAC_CTL_INIT, bdma_chan->regs + TSI721_DMAC_CTL); 139 ioread32(bdma_chan->regs + TSI721_DMAC_CTL); 140 bdma_chan->wr_count = bdma_chan->wr_count_next = 0; 141 bdma_chan->sts_rdptr = 0; 142 udelay(10); 143 144 return 0; 145 } 146 147 static int tsi721_bdma_ch_free(struct tsi721_bdma_chan *bdma_chan) 148 { 149 u32 ch_stat; 150 151 if (bdma_chan->bd_base == NULL) 152 return 0; 153 154 /* Check if DMA channel still running */ 155 ch_stat = ioread32(bdma_chan->regs + TSI721_DMAC_STS); 156 if (ch_stat & TSI721_DMAC_STS_RUN) 157 return -EFAULT; 158 159 /* Put DMA channel into init state */ 160 iowrite32(TSI721_DMAC_CTL_INIT, bdma_chan->regs + TSI721_DMAC_CTL); 161 162 /* Free space allocated for DMA descriptors */ 163 dma_free_coherent(bdma_chan->dchan.device->dev, 164 bdma_chan->bd_num * sizeof(struct tsi721_dma_desc), 165 bdma_chan->bd_base, bdma_chan->bd_phys); 166 bdma_chan->bd_base = NULL; 167 168 /* Free space allocated for status FIFO */ 169 dma_free_coherent(bdma_chan->dchan.device->dev, 170 bdma_chan->sts_size * sizeof(struct tsi721_dma_sts), 171 bdma_chan->sts_base, bdma_chan->sts_phys); 172 bdma_chan->sts_base = NULL; 173 return 0; 174 } 175 176 static void 177 tsi721_bdma_interrupt_enable(struct tsi721_bdma_chan *bdma_chan, int enable) 178 { 179 if (enable) { 180 /* Clear pending BDMA channel interrupts */ 181 iowrite32(TSI721_DMAC_INT_ALL, 182 bdma_chan->regs + TSI721_DMAC_INT); 183 ioread32(bdma_chan->regs + TSI721_DMAC_INT); 184 /* Enable BDMA channel interrupts */ 185 iowrite32(TSI721_DMAC_INT_ALL, 186 bdma_chan->regs + TSI721_DMAC_INTE); 187 } else { 188 /* Disable BDMA channel interrupts */ 189 iowrite32(0, bdma_chan->regs + TSI721_DMAC_INTE); 190 /* Clear pending BDMA channel interrupts */ 191 iowrite32(TSI721_DMAC_INT_ALL, 192 bdma_chan->regs + TSI721_DMAC_INT); 193 } 194 195 } 196 197 static bool tsi721_dma_is_idle(struct tsi721_bdma_chan *bdma_chan) 198 { 199 u32 sts; 200 201 sts = ioread32(bdma_chan->regs + TSI721_DMAC_STS); 202 return ((sts & TSI721_DMAC_STS_RUN) == 0); 203 } 204 205 void tsi721_bdma_handler(struct tsi721_bdma_chan *bdma_chan) 206 { 207 /* Disable BDMA channel interrupts */ 208 iowrite32(0, bdma_chan->regs + TSI721_DMAC_INTE); 209 if (bdma_chan->active) 210 tasklet_schedule(&bdma_chan->tasklet); 211 } 212 213 #ifdef CONFIG_PCI_MSI 214 /** 215 * tsi721_omsg_msix - MSI-X interrupt handler for BDMA channels 216 * @irq: Linux interrupt number 217 * @ptr: Pointer to interrupt-specific data (BDMA channel structure) 218 * 219 * Handles BDMA channel interrupts signaled using MSI-X. 220 */ 221 static irqreturn_t tsi721_bdma_msix(int irq, void *ptr) 222 { 223 struct tsi721_bdma_chan *bdma_chan = ptr; 224 225 tsi721_bdma_handler(bdma_chan); 226 return IRQ_HANDLED; 227 } 228 #endif /* CONFIG_PCI_MSI */ 229 230 /* Must be called with the spinlock held */ 231 static void tsi721_start_dma(struct tsi721_bdma_chan *bdma_chan) 232 { 233 if (!tsi721_dma_is_idle(bdma_chan)) { 234 dev_err(bdma_chan->dchan.device->dev, 235 "BUG: Attempt to start non-idle channel\n"); 236 return; 237 } 238 239 if (bdma_chan->wr_count == bdma_chan->wr_count_next) { 240 dev_err(bdma_chan->dchan.device->dev, 241 "BUG: Attempt to start DMA with no BDs ready\n"); 242 return; 243 } 244 245 dev_dbg(bdma_chan->dchan.device->dev, 246 "tx_chan: %p, chan: %d, regs: %p\n", 247 bdma_chan, bdma_chan->dchan.chan_id, bdma_chan->regs); 248 249 iowrite32(bdma_chan->wr_count_next, 250 bdma_chan->regs + TSI721_DMAC_DWRCNT); 251 ioread32(bdma_chan->regs + TSI721_DMAC_DWRCNT); 252 253 bdma_chan->wr_count = bdma_chan->wr_count_next; 254 } 255 256 static void tsi721_desc_put(struct tsi721_bdma_chan *bdma_chan, 257 struct tsi721_tx_desc *desc) 258 { 259 dev_dbg(bdma_chan->dchan.device->dev, 260 "Put desc: %p into free list\n", desc); 261 262 if (desc) { 263 spin_lock_bh(&bdma_chan->lock); 264 list_splice_init(&desc->tx_list, &bdma_chan->free_list); 265 list_add(&desc->desc_node, &bdma_chan->free_list); 266 bdma_chan->wr_count_next = bdma_chan->wr_count; 267 spin_unlock_bh(&bdma_chan->lock); 268 } 269 } 270 271 static 272 struct tsi721_tx_desc *tsi721_desc_get(struct tsi721_bdma_chan *bdma_chan) 273 { 274 struct tsi721_tx_desc *tx_desc, *_tx_desc; 275 struct tsi721_tx_desc *ret = NULL; 276 int i; 277 278 spin_lock_bh(&bdma_chan->lock); 279 list_for_each_entry_safe(tx_desc, _tx_desc, 280 &bdma_chan->free_list, desc_node) { 281 if (async_tx_test_ack(&tx_desc->txd)) { 282 list_del(&tx_desc->desc_node); 283 ret = tx_desc; 284 break; 285 } 286 dev_dbg(bdma_chan->dchan.device->dev, 287 "desc %p not ACKed\n", tx_desc); 288 } 289 290 i = bdma_chan->wr_count_next % bdma_chan->bd_num; 291 if (i == bdma_chan->bd_num - 1) { 292 i = 0; 293 bdma_chan->wr_count_next++; /* skip link descriptor */ 294 } 295 296 bdma_chan->wr_count_next++; 297 tx_desc->txd.phys = bdma_chan->bd_phys + 298 i * sizeof(struct tsi721_dma_desc); 299 tx_desc->hw_desc = &((struct tsi721_dma_desc *)bdma_chan->bd_base)[i]; 300 301 spin_unlock_bh(&bdma_chan->lock); 302 303 return ret; 304 } 305 306 static int 307 tsi721_fill_desc(struct tsi721_bdma_chan *bdma_chan, 308 struct tsi721_tx_desc *desc, struct scatterlist *sg, 309 enum dma_rtype rtype, u32 sys_size) 310 { 311 struct tsi721_dma_desc *bd_ptr = desc->hw_desc; 312 u64 rio_addr; 313 314 if (sg_dma_len(sg) > TSI721_DMAD_BCOUNT1 + 1) { 315 dev_err(bdma_chan->dchan.device->dev, 316 "SG element is too large\n"); 317 return -EINVAL; 318 } 319 320 dev_dbg(bdma_chan->dchan.device->dev, 321 "desc: 0x%llx, addr: 0x%llx len: 0x%x\n", 322 (u64)desc->txd.phys, (unsigned long long)sg_dma_address(sg), 323 sg_dma_len(sg)); 324 325 dev_dbg(bdma_chan->dchan.device->dev, 326 "bd_ptr = %p did=%d raddr=0x%llx\n", 327 bd_ptr, desc->destid, desc->rio_addr); 328 329 /* Initialize DMA descriptor */ 330 bd_ptr->type_id = cpu_to_le32((DTYPE1 << 29) | 331 (rtype << 19) | desc->destid); 332 if (desc->interrupt) 333 bd_ptr->type_id |= cpu_to_le32(TSI721_DMAD_IOF); 334 bd_ptr->bcount = cpu_to_le32(((desc->rio_addr & 0x3) << 30) | 335 (sys_size << 26) | sg_dma_len(sg)); 336 rio_addr = (desc->rio_addr >> 2) | 337 ((u64)(desc->rio_addr_u & 0x3) << 62); 338 bd_ptr->raddr_lo = cpu_to_le32(rio_addr & 0xffffffff); 339 bd_ptr->raddr_hi = cpu_to_le32(rio_addr >> 32); 340 bd_ptr->t1.bufptr_lo = cpu_to_le32( 341 (u64)sg_dma_address(sg) & 0xffffffff); 342 bd_ptr->t1.bufptr_hi = cpu_to_le32((u64)sg_dma_address(sg) >> 32); 343 bd_ptr->t1.s_dist = 0; 344 bd_ptr->t1.s_size = 0; 345 346 return 0; 347 } 348 349 static void tsi721_dma_chain_complete(struct tsi721_bdma_chan *bdma_chan, 350 struct tsi721_tx_desc *desc) 351 { 352 struct dma_async_tx_descriptor *txd = &desc->txd; 353 dma_async_tx_callback callback = txd->callback; 354 void *param = txd->callback_param; 355 356 list_splice_init(&desc->tx_list, &bdma_chan->free_list); 357 list_move(&desc->desc_node, &bdma_chan->free_list); 358 bdma_chan->completed_cookie = txd->cookie; 359 360 if (callback) 361 callback(param); 362 } 363 364 static void tsi721_dma_complete_all(struct tsi721_bdma_chan *bdma_chan) 365 { 366 struct tsi721_tx_desc *desc, *_d; 367 LIST_HEAD(list); 368 369 BUG_ON(!tsi721_dma_is_idle(bdma_chan)); 370 371 if (!list_empty(&bdma_chan->queue)) 372 tsi721_start_dma(bdma_chan); 373 374 list_splice_init(&bdma_chan->active_list, &list); 375 list_splice_init(&bdma_chan->queue, &bdma_chan->active_list); 376 377 list_for_each_entry_safe(desc, _d, &list, desc_node) 378 tsi721_dma_chain_complete(bdma_chan, desc); 379 } 380 381 static void tsi721_clr_stat(struct tsi721_bdma_chan *bdma_chan) 382 { 383 u32 srd_ptr; 384 u64 *sts_ptr; 385 int i, j; 386 387 /* Check and clear descriptor status FIFO entries */ 388 srd_ptr = bdma_chan->sts_rdptr; 389 sts_ptr = bdma_chan->sts_base; 390 j = srd_ptr * 8; 391 while (sts_ptr[j]) { 392 for (i = 0; i < 8 && sts_ptr[j]; i++, j++) 393 sts_ptr[j] = 0; 394 395 ++srd_ptr; 396 srd_ptr %= bdma_chan->sts_size; 397 j = srd_ptr * 8; 398 } 399 400 iowrite32(srd_ptr, bdma_chan->regs + TSI721_DMAC_DSRP); 401 bdma_chan->sts_rdptr = srd_ptr; 402 } 403 404 static void tsi721_advance_work(struct tsi721_bdma_chan *bdma_chan) 405 { 406 if (list_empty(&bdma_chan->active_list) || 407 list_is_singular(&bdma_chan->active_list)) { 408 dev_dbg(bdma_chan->dchan.device->dev, 409 "%s: Active_list empty\n", __func__); 410 tsi721_dma_complete_all(bdma_chan); 411 } else { 412 dev_dbg(bdma_chan->dchan.device->dev, 413 "%s: Active_list NOT empty\n", __func__); 414 tsi721_dma_chain_complete(bdma_chan, 415 tsi721_dma_first_active(bdma_chan)); 416 tsi721_start_dma(bdma_chan); 417 } 418 } 419 420 static void tsi721_dma_tasklet(unsigned long data) 421 { 422 struct tsi721_bdma_chan *bdma_chan = (struct tsi721_bdma_chan *)data; 423 u32 dmac_int, dmac_sts; 424 425 dmac_int = ioread32(bdma_chan->regs + TSI721_DMAC_INT); 426 dev_dbg(bdma_chan->dchan.device->dev, "%s: DMAC%d_INT = 0x%x\n", 427 __func__, bdma_chan->id, dmac_int); 428 /* Clear channel interrupts */ 429 iowrite32(dmac_int, bdma_chan->regs + TSI721_DMAC_INT); 430 431 if (dmac_int & TSI721_DMAC_INT_ERR) { 432 dmac_sts = ioread32(bdma_chan->regs + TSI721_DMAC_STS); 433 dev_err(bdma_chan->dchan.device->dev, 434 "%s: DMA ERROR - DMAC%d_STS = 0x%x\n", 435 __func__, bdma_chan->id, dmac_sts); 436 } 437 438 if (dmac_int & TSI721_DMAC_INT_STFULL) { 439 dev_err(bdma_chan->dchan.device->dev, 440 "%s: DMAC%d descriptor status FIFO is full\n", 441 __func__, bdma_chan->id); 442 } 443 444 if (dmac_int & (TSI721_DMAC_INT_DONE | TSI721_DMAC_INT_IOFDONE)) { 445 tsi721_clr_stat(bdma_chan); 446 spin_lock(&bdma_chan->lock); 447 tsi721_advance_work(bdma_chan); 448 spin_unlock(&bdma_chan->lock); 449 } 450 451 /* Re-Enable BDMA channel interrupts */ 452 iowrite32(TSI721_DMAC_INT_ALL, bdma_chan->regs + TSI721_DMAC_INTE); 453 } 454 455 static dma_cookie_t tsi721_tx_submit(struct dma_async_tx_descriptor *txd) 456 { 457 struct tsi721_tx_desc *desc = to_tsi721_desc(txd); 458 struct tsi721_bdma_chan *bdma_chan = to_tsi721_chan(txd->chan); 459 dma_cookie_t cookie; 460 461 spin_lock_bh(&bdma_chan->lock); 462 463 cookie = txd->chan->cookie; 464 if (++cookie < 0) 465 cookie = 1; 466 txd->chan->cookie = cookie; 467 txd->cookie = cookie; 468 469 if (list_empty(&bdma_chan->active_list)) { 470 list_add_tail(&desc->desc_node, &bdma_chan->active_list); 471 tsi721_start_dma(bdma_chan); 472 } else { 473 list_add_tail(&desc->desc_node, &bdma_chan->queue); 474 } 475 476 spin_unlock_bh(&bdma_chan->lock); 477 return cookie; 478 } 479 480 static int tsi721_alloc_chan_resources(struct dma_chan *dchan) 481 { 482 struct tsi721_bdma_chan *bdma_chan = to_tsi721_chan(dchan); 483 #ifdef CONFIG_PCI_MSI 484 struct tsi721_device *priv = to_tsi721(dchan->device); 485 #endif 486 struct tsi721_tx_desc *desc = NULL; 487 LIST_HEAD(tmp_list); 488 int i; 489 int rc; 490 491 if (bdma_chan->bd_base) 492 return bdma_chan->bd_num - 1; 493 494 /* Initialize BDMA channel */ 495 if (tsi721_bdma_ch_init(bdma_chan)) { 496 dev_err(dchan->device->dev, "Unable to initialize data DMA" 497 " channel %d, aborting\n", bdma_chan->id); 498 return -ENOMEM; 499 } 500 501 /* Alocate matching number of logical descriptors */ 502 desc = kcalloc((bdma_chan->bd_num - 1), sizeof(struct tsi721_tx_desc), 503 GFP_KERNEL); 504 if (!desc) { 505 dev_err(dchan->device->dev, 506 "Failed to allocate logical descriptors\n"); 507 rc = -ENOMEM; 508 goto err_out; 509 } 510 511 bdma_chan->tx_desc = desc; 512 513 for (i = 0; i < bdma_chan->bd_num - 1; i++) { 514 dma_async_tx_descriptor_init(&desc[i].txd, dchan); 515 desc[i].txd.tx_submit = tsi721_tx_submit; 516 desc[i].txd.flags = DMA_CTRL_ACK; 517 INIT_LIST_HEAD(&desc[i].tx_list); 518 list_add_tail(&desc[i].desc_node, &tmp_list); 519 } 520 521 spin_lock_bh(&bdma_chan->lock); 522 list_splice(&tmp_list, &bdma_chan->free_list); 523 bdma_chan->completed_cookie = dchan->cookie = 1; 524 spin_unlock_bh(&bdma_chan->lock); 525 526 #ifdef CONFIG_PCI_MSI 527 if (priv->flags & TSI721_USING_MSIX) { 528 /* Request interrupt service if we are in MSI-X mode */ 529 rc = request_irq( 530 priv->msix[TSI721_VECT_DMA0_DONE + 531 bdma_chan->id].vector, 532 tsi721_bdma_msix, 0, 533 priv->msix[TSI721_VECT_DMA0_DONE + 534 bdma_chan->id].irq_name, 535 (void *)bdma_chan); 536 537 if (rc) { 538 dev_dbg(dchan->device->dev, 539 "Unable to allocate MSI-X interrupt for " 540 "BDMA%d-DONE\n", bdma_chan->id); 541 goto err_out; 542 } 543 544 rc = request_irq(priv->msix[TSI721_VECT_DMA0_INT + 545 bdma_chan->id].vector, 546 tsi721_bdma_msix, 0, 547 priv->msix[TSI721_VECT_DMA0_INT + 548 bdma_chan->id].irq_name, 549 (void *)bdma_chan); 550 551 if (rc) { 552 dev_dbg(dchan->device->dev, 553 "Unable to allocate MSI-X interrupt for " 554 "BDMA%d-INT\n", bdma_chan->id); 555 free_irq( 556 priv->msix[TSI721_VECT_DMA0_DONE + 557 bdma_chan->id].vector, 558 (void *)bdma_chan); 559 rc = -EIO; 560 goto err_out; 561 } 562 } 563 #endif /* CONFIG_PCI_MSI */ 564 565 bdma_chan->active = true; 566 tsi721_bdma_interrupt_enable(bdma_chan, 1); 567 568 return bdma_chan->bd_num - 1; 569 570 err_out: 571 kfree(desc); 572 tsi721_bdma_ch_free(bdma_chan); 573 return rc; 574 } 575 576 static void tsi721_free_chan_resources(struct dma_chan *dchan) 577 { 578 struct tsi721_bdma_chan *bdma_chan = to_tsi721_chan(dchan); 579 struct tsi721_device *priv = to_tsi721(dchan->device); 580 LIST_HEAD(list); 581 582 dev_dbg(dchan->device->dev, "%s: Entry\n", __func__); 583 584 if (bdma_chan->bd_base == NULL) 585 return; 586 587 BUG_ON(!list_empty(&bdma_chan->active_list)); 588 BUG_ON(!list_empty(&bdma_chan->queue)); 589 590 tsi721_bdma_interrupt_enable(bdma_chan, 0); 591 bdma_chan->active = false; 592 593 #ifdef CONFIG_PCI_MSI 594 if (priv->flags & TSI721_USING_MSIX) { 595 synchronize_irq(priv->msix[TSI721_VECT_DMA0_DONE + 596 bdma_chan->id].vector); 597 synchronize_irq(priv->msix[TSI721_VECT_DMA0_INT + 598 bdma_chan->id].vector); 599 } else 600 #endif 601 synchronize_irq(priv->pdev->irq); 602 603 tasklet_kill(&bdma_chan->tasklet); 604 605 spin_lock_bh(&bdma_chan->lock); 606 list_splice_init(&bdma_chan->free_list, &list); 607 spin_unlock_bh(&bdma_chan->lock); 608 609 #ifdef CONFIG_PCI_MSI 610 if (priv->flags & TSI721_USING_MSIX) { 611 free_irq(priv->msix[TSI721_VECT_DMA0_DONE + 612 bdma_chan->id].vector, (void *)bdma_chan); 613 free_irq(priv->msix[TSI721_VECT_DMA0_INT + 614 bdma_chan->id].vector, (void *)bdma_chan); 615 } 616 #endif /* CONFIG_PCI_MSI */ 617 618 tsi721_bdma_ch_free(bdma_chan); 619 kfree(bdma_chan->tx_desc); 620 } 621 622 static 623 enum dma_status tsi721_tx_status(struct dma_chan *dchan, dma_cookie_t cookie, 624 struct dma_tx_state *txstate) 625 { 626 struct tsi721_bdma_chan *bdma_chan = to_tsi721_chan(dchan); 627 dma_cookie_t last_used; 628 dma_cookie_t last_completed; 629 int ret; 630 631 spin_lock_bh(&bdma_chan->lock); 632 last_completed = bdma_chan->completed_cookie; 633 last_used = dchan->cookie; 634 spin_unlock_bh(&bdma_chan->lock); 635 636 ret = dma_async_is_complete(cookie, last_completed, last_used); 637 638 dma_set_tx_state(txstate, last_completed, last_used, 0); 639 640 dev_dbg(dchan->device->dev, 641 "%s: exit, ret: %d, last_completed: %d, last_used: %d\n", 642 __func__, ret, last_completed, last_used); 643 644 return ret; 645 } 646 647 static void tsi721_issue_pending(struct dma_chan *dchan) 648 { 649 struct tsi721_bdma_chan *bdma_chan = to_tsi721_chan(dchan); 650 651 dev_dbg(dchan->device->dev, "%s: Entry\n", __func__); 652 653 if (tsi721_dma_is_idle(bdma_chan)) { 654 spin_lock_bh(&bdma_chan->lock); 655 tsi721_advance_work(bdma_chan); 656 spin_unlock_bh(&bdma_chan->lock); 657 } else 658 dev_dbg(dchan->device->dev, 659 "%s: DMA channel still busy\n", __func__); 660 } 661 662 static 663 struct dma_async_tx_descriptor *tsi721_prep_rio_sg(struct dma_chan *dchan, 664 struct scatterlist *sgl, unsigned int sg_len, 665 enum dma_transfer_direction dir, unsigned long flags, 666 void *tinfo) 667 { 668 struct tsi721_bdma_chan *bdma_chan = to_tsi721_chan(dchan); 669 struct tsi721_tx_desc *desc = NULL; 670 struct tsi721_tx_desc *first = NULL; 671 struct scatterlist *sg; 672 struct rio_dma_ext *rext = tinfo; 673 u64 rio_addr = rext->rio_addr; /* limited to 64-bit rio_addr for now */ 674 unsigned int i; 675 u32 sys_size = dma_to_mport(dchan->device)->sys_size; 676 enum dma_rtype rtype; 677 678 if (!sgl || !sg_len) { 679 dev_err(dchan->device->dev, "%s: No SG list\n", __func__); 680 return NULL; 681 } 682 683 if (dir == DMA_DEV_TO_MEM) 684 rtype = NREAD; 685 else if (dir == DMA_MEM_TO_DEV) { 686 switch (rext->wr_type) { 687 case RDW_ALL_NWRITE: 688 rtype = ALL_NWRITE; 689 break; 690 case RDW_ALL_NWRITE_R: 691 rtype = ALL_NWRITE_R; 692 break; 693 case RDW_LAST_NWRITE_R: 694 default: 695 rtype = LAST_NWRITE_R; 696 break; 697 } 698 } else { 699 dev_err(dchan->device->dev, 700 "%s: Unsupported DMA direction option\n", __func__); 701 return NULL; 702 } 703 704 for_each_sg(sgl, sg, sg_len, i) { 705 int err; 706 707 dev_dbg(dchan->device->dev, "%s: sg #%d\n", __func__, i); 708 desc = tsi721_desc_get(bdma_chan); 709 if (!desc) { 710 dev_err(dchan->device->dev, 711 "Not enough descriptors available\n"); 712 goto err_desc_get; 713 } 714 715 if (sg_is_last(sg)) 716 desc->interrupt = (flags & DMA_PREP_INTERRUPT) != 0; 717 else 718 desc->interrupt = false; 719 720 desc->destid = rext->destid; 721 desc->rio_addr = rio_addr; 722 desc->rio_addr_u = 0; 723 724 err = tsi721_fill_desc(bdma_chan, desc, sg, rtype, sys_size); 725 if (err) { 726 dev_err(dchan->device->dev, 727 "Failed to build desc: %d\n", err); 728 goto err_desc_get; 729 } 730 731 rio_addr += sg_dma_len(sg); 732 733 if (!first) 734 first = desc; 735 else 736 list_add_tail(&desc->desc_node, &first->tx_list); 737 } 738 739 first->txd.cookie = -EBUSY; 740 desc->txd.flags = flags; 741 742 return &first->txd; 743 744 err_desc_get: 745 tsi721_desc_put(bdma_chan, first); 746 return NULL; 747 } 748 749 static int tsi721_device_control(struct dma_chan *dchan, enum dma_ctrl_cmd cmd, 750 unsigned long arg) 751 { 752 struct tsi721_bdma_chan *bdma_chan = to_tsi721_chan(dchan); 753 struct tsi721_tx_desc *desc, *_d; 754 LIST_HEAD(list); 755 756 dev_dbg(dchan->device->dev, "%s: Entry\n", __func__); 757 758 if (cmd != DMA_TERMINATE_ALL) 759 return -ENXIO; 760 761 spin_lock_bh(&bdma_chan->lock); 762 763 /* make sure to stop the transfer */ 764 iowrite32(TSI721_DMAC_CTL_SUSP, bdma_chan->regs + TSI721_DMAC_CTL); 765 766 list_splice_init(&bdma_chan->active_list, &list); 767 list_splice_init(&bdma_chan->queue, &list); 768 769 list_for_each_entry_safe(desc, _d, &list, desc_node) 770 tsi721_dma_chain_complete(bdma_chan, desc); 771 772 spin_unlock_bh(&bdma_chan->lock); 773 774 return 0; 775 } 776 777 int tsi721_register_dma(struct tsi721_device *priv) 778 { 779 int i; 780 int nr_channels = TSI721_DMA_MAXCH; 781 int err; 782 struct rio_mport *mport = priv->mport; 783 784 mport->dma.dev = &priv->pdev->dev; 785 mport->dma.chancnt = nr_channels; 786 787 INIT_LIST_HEAD(&mport->dma.channels); 788 789 for (i = 0; i < nr_channels; i++) { 790 struct tsi721_bdma_chan *bdma_chan = &priv->bdma[i]; 791 792 if (i == TSI721_DMACH_MAINT) 793 continue; 794 795 bdma_chan->bd_num = 64; 796 bdma_chan->regs = priv->regs + TSI721_DMAC_BASE(i); 797 798 bdma_chan->dchan.device = &mport->dma; 799 bdma_chan->dchan.cookie = 1; 800 bdma_chan->dchan.chan_id = i; 801 bdma_chan->id = i; 802 bdma_chan->active = false; 803 804 spin_lock_init(&bdma_chan->lock); 805 806 INIT_LIST_HEAD(&bdma_chan->active_list); 807 INIT_LIST_HEAD(&bdma_chan->queue); 808 INIT_LIST_HEAD(&bdma_chan->free_list); 809 810 tasklet_init(&bdma_chan->tasklet, tsi721_dma_tasklet, 811 (unsigned long)bdma_chan); 812 list_add_tail(&bdma_chan->dchan.device_node, 813 &mport->dma.channels); 814 } 815 816 dma_cap_zero(mport->dma.cap_mask); 817 dma_cap_set(DMA_PRIVATE, mport->dma.cap_mask); 818 dma_cap_set(DMA_SLAVE, mport->dma.cap_mask); 819 820 mport->dma.device_alloc_chan_resources = tsi721_alloc_chan_resources; 821 mport->dma.device_free_chan_resources = tsi721_free_chan_resources; 822 mport->dma.device_tx_status = tsi721_tx_status; 823 mport->dma.device_issue_pending = tsi721_issue_pending; 824 mport->dma.device_prep_slave_sg = tsi721_prep_rio_sg; 825 mport->dma.device_control = tsi721_device_control; 826 827 err = dma_async_device_register(&mport->dma); 828 if (err) 829 dev_err(&priv->pdev->dev, "Failed to register DMA device\n"); 830 831 return err; 832 } 833