1 /* 2 * DMA driver for Nvidia's Tegra20 APB DMA controller. 3 * 4 * Copyright (c) 2012-2013, NVIDIA CORPORATION. All rights reserved. 5 * 6 * This program is free software; you can redistribute it and/or modify it 7 * under the terms and conditions of the GNU General Public License, 8 * version 2, as published by the Free Software Foundation. 9 * 10 * This program is distributed in the hope it will be useful, but WITHOUT 11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 13 * more details. 14 * 15 * You should have received a copy of the GNU General Public License 16 * along with this program. If not, see <http://www.gnu.org/licenses/>. 17 */ 18 19 #include <linux/bitops.h> 20 #include <linux/clk.h> 21 #include <linux/delay.h> 22 #include <linux/dmaengine.h> 23 #include <linux/dma-mapping.h> 24 #include <linux/err.h> 25 #include <linux/init.h> 26 #include <linux/interrupt.h> 27 #include <linux/io.h> 28 #include <linux/mm.h> 29 #include <linux/module.h> 30 #include <linux/of.h> 31 #include <linux/of_device.h> 32 #include <linux/of_dma.h> 33 #include <linux/platform_device.h> 34 #include <linux/pm.h> 35 #include <linux/pm_runtime.h> 36 #include <linux/reset.h> 37 #include <linux/slab.h> 38 39 #include "dmaengine.h" 40 41 #define TEGRA_APBDMA_GENERAL 0x0 42 #define TEGRA_APBDMA_GENERAL_ENABLE BIT(31) 43 44 #define TEGRA_APBDMA_CONTROL 0x010 45 #define TEGRA_APBDMA_IRQ_MASK 0x01c 46 #define TEGRA_APBDMA_IRQ_MASK_SET 0x020 47 48 /* CSR register */ 49 #define TEGRA_APBDMA_CHAN_CSR 0x00 50 #define TEGRA_APBDMA_CSR_ENB BIT(31) 51 #define TEGRA_APBDMA_CSR_IE_EOC BIT(30) 52 #define TEGRA_APBDMA_CSR_HOLD BIT(29) 53 #define TEGRA_APBDMA_CSR_DIR BIT(28) 54 #define TEGRA_APBDMA_CSR_ONCE BIT(27) 55 #define TEGRA_APBDMA_CSR_FLOW BIT(21) 56 #define TEGRA_APBDMA_CSR_REQ_SEL_SHIFT 16 57 #define TEGRA_APBDMA_CSR_WCOUNT_MASK 0xFFFC 58 59 /* STATUS register */ 60 #define TEGRA_APBDMA_CHAN_STATUS 0x004 61 #define TEGRA_APBDMA_STATUS_BUSY BIT(31) 62 #define TEGRA_APBDMA_STATUS_ISE_EOC BIT(30) 63 #define TEGRA_APBDMA_STATUS_HALT BIT(29) 64 #define TEGRA_APBDMA_STATUS_PING_PONG BIT(28) 65 #define TEGRA_APBDMA_STATUS_COUNT_SHIFT 2 66 #define TEGRA_APBDMA_STATUS_COUNT_MASK 0xFFFC 67 68 #define TEGRA_APBDMA_CHAN_CSRE 0x00C 69 #define TEGRA_APBDMA_CHAN_CSRE_PAUSE (1 << 31) 70 71 /* AHB memory address */ 72 #define TEGRA_APBDMA_CHAN_AHBPTR 0x010 73 74 /* AHB sequence register */ 75 #define TEGRA_APBDMA_CHAN_AHBSEQ 0x14 76 #define TEGRA_APBDMA_AHBSEQ_INTR_ENB BIT(31) 77 #define TEGRA_APBDMA_AHBSEQ_BUS_WIDTH_8 (0 << 28) 78 #define TEGRA_APBDMA_AHBSEQ_BUS_WIDTH_16 (1 << 28) 79 #define TEGRA_APBDMA_AHBSEQ_BUS_WIDTH_32 (2 << 28) 80 #define TEGRA_APBDMA_AHBSEQ_BUS_WIDTH_64 (3 << 28) 81 #define TEGRA_APBDMA_AHBSEQ_BUS_WIDTH_128 (4 << 28) 82 #define TEGRA_APBDMA_AHBSEQ_DATA_SWAP BIT(27) 83 #define TEGRA_APBDMA_AHBSEQ_BURST_1 (4 << 24) 84 #define TEGRA_APBDMA_AHBSEQ_BURST_4 (5 << 24) 85 #define TEGRA_APBDMA_AHBSEQ_BURST_8 (6 << 24) 86 #define TEGRA_APBDMA_AHBSEQ_DBL_BUF BIT(19) 87 #define TEGRA_APBDMA_AHBSEQ_WRAP_SHIFT 16 88 #define TEGRA_APBDMA_AHBSEQ_WRAP_NONE 0 89 90 /* APB address */ 91 #define TEGRA_APBDMA_CHAN_APBPTR 0x018 92 93 /* APB sequence register */ 94 #define TEGRA_APBDMA_CHAN_APBSEQ 0x01c 95 #define TEGRA_APBDMA_APBSEQ_BUS_WIDTH_8 (0 << 28) 96 #define TEGRA_APBDMA_APBSEQ_BUS_WIDTH_16 (1 << 28) 97 #define TEGRA_APBDMA_APBSEQ_BUS_WIDTH_32 (2 << 28) 98 #define TEGRA_APBDMA_APBSEQ_BUS_WIDTH_64 (3 << 28) 99 #define TEGRA_APBDMA_APBSEQ_BUS_WIDTH_128 (4 << 28) 100 #define TEGRA_APBDMA_APBSEQ_DATA_SWAP BIT(27) 101 #define TEGRA_APBDMA_APBSEQ_WRAP_WORD_1 (1 << 16) 102 103 /* Tegra148 specific registers */ 104 #define TEGRA_APBDMA_CHAN_WCOUNT 0x20 105 106 #define TEGRA_APBDMA_CHAN_WORD_TRANSFER 0x24 107 108 /* 109 * If any burst is in flight and DMA paused then this is the time to complete 110 * on-flight burst and update DMA status register. 111 */ 112 #define TEGRA_APBDMA_BURST_COMPLETE_TIME 20 113 114 /* Channel base address offset from APBDMA base address */ 115 #define TEGRA_APBDMA_CHANNEL_BASE_ADD_OFFSET 0x1000 116 117 struct tegra_dma; 118 119 /* 120 * tegra_dma_chip_data Tegra chip specific DMA data 121 * @nr_channels: Number of channels available in the controller. 122 * @channel_reg_size: Channel register size/stride. 123 * @max_dma_count: Maximum DMA transfer count supported by DMA controller. 124 * @support_channel_pause: Support channel wise pause of dma. 125 * @support_separate_wcount_reg: Support separate word count register. 126 */ 127 struct tegra_dma_chip_data { 128 int nr_channels; 129 int channel_reg_size; 130 int max_dma_count; 131 bool support_channel_pause; 132 bool support_separate_wcount_reg; 133 }; 134 135 /* DMA channel registers */ 136 struct tegra_dma_channel_regs { 137 unsigned long csr; 138 unsigned long ahb_ptr; 139 unsigned long apb_ptr; 140 unsigned long ahb_seq; 141 unsigned long apb_seq; 142 unsigned long wcount; 143 }; 144 145 /* 146 * tegra_dma_sg_req: Dma request details to configure hardware. This 147 * contains the details for one transfer to configure DMA hw. 148 * The client's request for data transfer can be broken into multiple 149 * sub-transfer as per requester details and hw support. 150 * This sub transfer get added in the list of transfer and point to Tegra 151 * DMA descriptor which manages the transfer details. 152 */ 153 struct tegra_dma_sg_req { 154 struct tegra_dma_channel_regs ch_regs; 155 int req_len; 156 bool configured; 157 bool last_sg; 158 bool half_done; 159 struct list_head node; 160 struct tegra_dma_desc *dma_desc; 161 }; 162 163 /* 164 * tegra_dma_desc: Tegra DMA descriptors which manages the client requests. 165 * This descriptor keep track of transfer status, callbacks and request 166 * counts etc. 167 */ 168 struct tegra_dma_desc { 169 struct dma_async_tx_descriptor txd; 170 int bytes_requested; 171 int bytes_transferred; 172 enum dma_status dma_status; 173 struct list_head node; 174 struct list_head tx_list; 175 struct list_head cb_node; 176 int cb_count; 177 }; 178 179 struct tegra_dma_channel; 180 181 typedef void (*dma_isr_handler)(struct tegra_dma_channel *tdc, 182 bool to_terminate); 183 184 /* tegra_dma_channel: Channel specific information */ 185 struct tegra_dma_channel { 186 struct dma_chan dma_chan; 187 char name[30]; 188 bool config_init; 189 int id; 190 int irq; 191 unsigned long chan_base_offset; 192 spinlock_t lock; 193 bool busy; 194 struct tegra_dma *tdma; 195 bool cyclic; 196 197 /* Different lists for managing the requests */ 198 struct list_head free_sg_req; 199 struct list_head pending_sg_req; 200 struct list_head free_dma_desc; 201 struct list_head cb_desc; 202 203 /* ISR handler and tasklet for bottom half of isr handling */ 204 dma_isr_handler isr_handler; 205 struct tasklet_struct tasklet; 206 dma_async_tx_callback callback; 207 void *callback_param; 208 209 /* Channel-slave specific configuration */ 210 unsigned int slave_id; 211 struct dma_slave_config dma_sconfig; 212 struct tegra_dma_channel_regs channel_reg; 213 }; 214 215 /* tegra_dma: Tegra DMA specific information */ 216 struct tegra_dma { 217 struct dma_device dma_dev; 218 struct device *dev; 219 struct clk *dma_clk; 220 struct reset_control *rst; 221 spinlock_t global_lock; 222 void __iomem *base_addr; 223 const struct tegra_dma_chip_data *chip_data; 224 225 /* Some register need to be cache before suspend */ 226 u32 reg_gen; 227 228 /* Last member of the structure */ 229 struct tegra_dma_channel channels[0]; 230 }; 231 232 static inline void tdma_write(struct tegra_dma *tdma, u32 reg, u32 val) 233 { 234 writel(val, tdma->base_addr + reg); 235 } 236 237 static inline u32 tdma_read(struct tegra_dma *tdma, u32 reg) 238 { 239 return readl(tdma->base_addr + reg); 240 } 241 242 static inline void tdc_write(struct tegra_dma_channel *tdc, 243 u32 reg, u32 val) 244 { 245 writel(val, tdc->tdma->base_addr + tdc->chan_base_offset + reg); 246 } 247 248 static inline u32 tdc_read(struct tegra_dma_channel *tdc, u32 reg) 249 { 250 return readl(tdc->tdma->base_addr + tdc->chan_base_offset + reg); 251 } 252 253 static inline struct tegra_dma_channel *to_tegra_dma_chan(struct dma_chan *dc) 254 { 255 return container_of(dc, struct tegra_dma_channel, dma_chan); 256 } 257 258 static inline struct tegra_dma_desc *txd_to_tegra_dma_desc( 259 struct dma_async_tx_descriptor *td) 260 { 261 return container_of(td, struct tegra_dma_desc, txd); 262 } 263 264 static inline struct device *tdc2dev(struct tegra_dma_channel *tdc) 265 { 266 return &tdc->dma_chan.dev->device; 267 } 268 269 static dma_cookie_t tegra_dma_tx_submit(struct dma_async_tx_descriptor *tx); 270 static int tegra_dma_runtime_suspend(struct device *dev); 271 static int tegra_dma_runtime_resume(struct device *dev); 272 273 /* Get DMA desc from free list, if not there then allocate it. */ 274 static struct tegra_dma_desc *tegra_dma_desc_get( 275 struct tegra_dma_channel *tdc) 276 { 277 struct tegra_dma_desc *dma_desc; 278 unsigned long flags; 279 280 spin_lock_irqsave(&tdc->lock, flags); 281 282 /* Do not allocate if desc are waiting for ack */ 283 list_for_each_entry(dma_desc, &tdc->free_dma_desc, node) { 284 if (async_tx_test_ack(&dma_desc->txd)) { 285 list_del(&dma_desc->node); 286 spin_unlock_irqrestore(&tdc->lock, flags); 287 dma_desc->txd.flags = 0; 288 return dma_desc; 289 } 290 } 291 292 spin_unlock_irqrestore(&tdc->lock, flags); 293 294 /* Allocate DMA desc */ 295 dma_desc = kzalloc(sizeof(*dma_desc), GFP_ATOMIC); 296 if (!dma_desc) { 297 dev_err(tdc2dev(tdc), "dma_desc alloc failed\n"); 298 return NULL; 299 } 300 301 dma_async_tx_descriptor_init(&dma_desc->txd, &tdc->dma_chan); 302 dma_desc->txd.tx_submit = tegra_dma_tx_submit; 303 dma_desc->txd.flags = 0; 304 return dma_desc; 305 } 306 307 static void tegra_dma_desc_put(struct tegra_dma_channel *tdc, 308 struct tegra_dma_desc *dma_desc) 309 { 310 unsigned long flags; 311 312 spin_lock_irqsave(&tdc->lock, flags); 313 if (!list_empty(&dma_desc->tx_list)) 314 list_splice_init(&dma_desc->tx_list, &tdc->free_sg_req); 315 list_add_tail(&dma_desc->node, &tdc->free_dma_desc); 316 spin_unlock_irqrestore(&tdc->lock, flags); 317 } 318 319 static struct tegra_dma_sg_req *tegra_dma_sg_req_get( 320 struct tegra_dma_channel *tdc) 321 { 322 struct tegra_dma_sg_req *sg_req = NULL; 323 unsigned long flags; 324 325 spin_lock_irqsave(&tdc->lock, flags); 326 if (!list_empty(&tdc->free_sg_req)) { 327 sg_req = list_first_entry(&tdc->free_sg_req, 328 typeof(*sg_req), node); 329 list_del(&sg_req->node); 330 spin_unlock_irqrestore(&tdc->lock, flags); 331 return sg_req; 332 } 333 spin_unlock_irqrestore(&tdc->lock, flags); 334 335 sg_req = kzalloc(sizeof(struct tegra_dma_sg_req), GFP_ATOMIC); 336 if (!sg_req) 337 dev_err(tdc2dev(tdc), "sg_req alloc failed\n"); 338 return sg_req; 339 } 340 341 static int tegra_dma_slave_config(struct dma_chan *dc, 342 struct dma_slave_config *sconfig) 343 { 344 struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc); 345 346 if (!list_empty(&tdc->pending_sg_req)) { 347 dev_err(tdc2dev(tdc), "Configuration not allowed\n"); 348 return -EBUSY; 349 } 350 351 memcpy(&tdc->dma_sconfig, sconfig, sizeof(*sconfig)); 352 if (!tdc->slave_id) 353 tdc->slave_id = sconfig->slave_id; 354 tdc->config_init = true; 355 return 0; 356 } 357 358 static void tegra_dma_global_pause(struct tegra_dma_channel *tdc, 359 bool wait_for_burst_complete) 360 { 361 struct tegra_dma *tdma = tdc->tdma; 362 363 spin_lock(&tdma->global_lock); 364 tdma_write(tdma, TEGRA_APBDMA_GENERAL, 0); 365 if (wait_for_burst_complete) 366 udelay(TEGRA_APBDMA_BURST_COMPLETE_TIME); 367 } 368 369 static void tegra_dma_global_resume(struct tegra_dma_channel *tdc) 370 { 371 struct tegra_dma *tdma = tdc->tdma; 372 373 tdma_write(tdma, TEGRA_APBDMA_GENERAL, TEGRA_APBDMA_GENERAL_ENABLE); 374 spin_unlock(&tdma->global_lock); 375 } 376 377 static void tegra_dma_pause(struct tegra_dma_channel *tdc, 378 bool wait_for_burst_complete) 379 { 380 struct tegra_dma *tdma = tdc->tdma; 381 382 if (tdma->chip_data->support_channel_pause) { 383 tdc_write(tdc, TEGRA_APBDMA_CHAN_CSRE, 384 TEGRA_APBDMA_CHAN_CSRE_PAUSE); 385 if (wait_for_burst_complete) 386 udelay(TEGRA_APBDMA_BURST_COMPLETE_TIME); 387 } else { 388 tegra_dma_global_pause(tdc, wait_for_burst_complete); 389 } 390 } 391 392 static void tegra_dma_resume(struct tegra_dma_channel *tdc) 393 { 394 struct tegra_dma *tdma = tdc->tdma; 395 396 if (tdma->chip_data->support_channel_pause) { 397 tdc_write(tdc, TEGRA_APBDMA_CHAN_CSRE, 0); 398 } else { 399 tegra_dma_global_resume(tdc); 400 } 401 } 402 403 static void tegra_dma_stop(struct tegra_dma_channel *tdc) 404 { 405 u32 csr; 406 u32 status; 407 408 /* Disable interrupts */ 409 csr = tdc_read(tdc, TEGRA_APBDMA_CHAN_CSR); 410 csr &= ~TEGRA_APBDMA_CSR_IE_EOC; 411 tdc_write(tdc, TEGRA_APBDMA_CHAN_CSR, csr); 412 413 /* Disable DMA */ 414 csr &= ~TEGRA_APBDMA_CSR_ENB; 415 tdc_write(tdc, TEGRA_APBDMA_CHAN_CSR, csr); 416 417 /* Clear interrupt status if it is there */ 418 status = tdc_read(tdc, TEGRA_APBDMA_CHAN_STATUS); 419 if (status & TEGRA_APBDMA_STATUS_ISE_EOC) { 420 dev_dbg(tdc2dev(tdc), "%s():clearing interrupt\n", __func__); 421 tdc_write(tdc, TEGRA_APBDMA_CHAN_STATUS, status); 422 } 423 tdc->busy = false; 424 } 425 426 static void tegra_dma_start(struct tegra_dma_channel *tdc, 427 struct tegra_dma_sg_req *sg_req) 428 { 429 struct tegra_dma_channel_regs *ch_regs = &sg_req->ch_regs; 430 431 tdc_write(tdc, TEGRA_APBDMA_CHAN_CSR, ch_regs->csr); 432 tdc_write(tdc, TEGRA_APBDMA_CHAN_APBSEQ, ch_regs->apb_seq); 433 tdc_write(tdc, TEGRA_APBDMA_CHAN_APBPTR, ch_regs->apb_ptr); 434 tdc_write(tdc, TEGRA_APBDMA_CHAN_AHBSEQ, ch_regs->ahb_seq); 435 tdc_write(tdc, TEGRA_APBDMA_CHAN_AHBPTR, ch_regs->ahb_ptr); 436 if (tdc->tdma->chip_data->support_separate_wcount_reg) 437 tdc_write(tdc, TEGRA_APBDMA_CHAN_WCOUNT, ch_regs->wcount); 438 439 /* Start DMA */ 440 tdc_write(tdc, TEGRA_APBDMA_CHAN_CSR, 441 ch_regs->csr | TEGRA_APBDMA_CSR_ENB); 442 } 443 444 static void tegra_dma_configure_for_next(struct tegra_dma_channel *tdc, 445 struct tegra_dma_sg_req *nsg_req) 446 { 447 unsigned long status; 448 449 /* 450 * The DMA controller reloads the new configuration for next transfer 451 * after last burst of current transfer completes. 452 * If there is no IEC status then this makes sure that last burst 453 * has not be completed. There may be case that last burst is on 454 * flight and so it can complete but because DMA is paused, it 455 * will not generates interrupt as well as not reload the new 456 * configuration. 457 * If there is already IEC status then interrupt handler need to 458 * load new configuration. 459 */ 460 tegra_dma_pause(tdc, false); 461 status = tdc_read(tdc, TEGRA_APBDMA_CHAN_STATUS); 462 463 /* 464 * If interrupt is pending then do nothing as the ISR will handle 465 * the programing for new request. 466 */ 467 if (status & TEGRA_APBDMA_STATUS_ISE_EOC) { 468 dev_err(tdc2dev(tdc), 469 "Skipping new configuration as interrupt is pending\n"); 470 tegra_dma_resume(tdc); 471 return; 472 } 473 474 /* Safe to program new configuration */ 475 tdc_write(tdc, TEGRA_APBDMA_CHAN_APBPTR, nsg_req->ch_regs.apb_ptr); 476 tdc_write(tdc, TEGRA_APBDMA_CHAN_AHBPTR, nsg_req->ch_regs.ahb_ptr); 477 if (tdc->tdma->chip_data->support_separate_wcount_reg) 478 tdc_write(tdc, TEGRA_APBDMA_CHAN_WCOUNT, 479 nsg_req->ch_regs.wcount); 480 tdc_write(tdc, TEGRA_APBDMA_CHAN_CSR, 481 nsg_req->ch_regs.csr | TEGRA_APBDMA_CSR_ENB); 482 nsg_req->configured = true; 483 484 tegra_dma_resume(tdc); 485 } 486 487 static void tdc_start_head_req(struct tegra_dma_channel *tdc) 488 { 489 struct tegra_dma_sg_req *sg_req; 490 491 if (list_empty(&tdc->pending_sg_req)) 492 return; 493 494 sg_req = list_first_entry(&tdc->pending_sg_req, 495 typeof(*sg_req), node); 496 tegra_dma_start(tdc, sg_req); 497 sg_req->configured = true; 498 tdc->busy = true; 499 } 500 501 static void tdc_configure_next_head_desc(struct tegra_dma_channel *tdc) 502 { 503 struct tegra_dma_sg_req *hsgreq; 504 struct tegra_dma_sg_req *hnsgreq; 505 506 if (list_empty(&tdc->pending_sg_req)) 507 return; 508 509 hsgreq = list_first_entry(&tdc->pending_sg_req, typeof(*hsgreq), node); 510 if (!list_is_last(&hsgreq->node, &tdc->pending_sg_req)) { 511 hnsgreq = list_first_entry(&hsgreq->node, 512 typeof(*hnsgreq), node); 513 tegra_dma_configure_for_next(tdc, hnsgreq); 514 } 515 } 516 517 static inline int get_current_xferred_count(struct tegra_dma_channel *tdc, 518 struct tegra_dma_sg_req *sg_req, unsigned long status) 519 { 520 return sg_req->req_len - (status & TEGRA_APBDMA_STATUS_COUNT_MASK) - 4; 521 } 522 523 static void tegra_dma_abort_all(struct tegra_dma_channel *tdc) 524 { 525 struct tegra_dma_sg_req *sgreq; 526 struct tegra_dma_desc *dma_desc; 527 528 while (!list_empty(&tdc->pending_sg_req)) { 529 sgreq = list_first_entry(&tdc->pending_sg_req, 530 typeof(*sgreq), node); 531 list_move_tail(&sgreq->node, &tdc->free_sg_req); 532 if (sgreq->last_sg) { 533 dma_desc = sgreq->dma_desc; 534 dma_desc->dma_status = DMA_ERROR; 535 list_add_tail(&dma_desc->node, &tdc->free_dma_desc); 536 537 /* Add in cb list if it is not there. */ 538 if (!dma_desc->cb_count) 539 list_add_tail(&dma_desc->cb_node, 540 &tdc->cb_desc); 541 dma_desc->cb_count++; 542 } 543 } 544 tdc->isr_handler = NULL; 545 } 546 547 static bool handle_continuous_head_request(struct tegra_dma_channel *tdc, 548 struct tegra_dma_sg_req *last_sg_req, bool to_terminate) 549 { 550 struct tegra_dma_sg_req *hsgreq = NULL; 551 552 if (list_empty(&tdc->pending_sg_req)) { 553 dev_err(tdc2dev(tdc), "Dma is running without req\n"); 554 tegra_dma_stop(tdc); 555 return false; 556 } 557 558 /* 559 * Check that head req on list should be in flight. 560 * If it is not in flight then abort transfer as 561 * looping of transfer can not continue. 562 */ 563 hsgreq = list_first_entry(&tdc->pending_sg_req, typeof(*hsgreq), node); 564 if (!hsgreq->configured) { 565 tegra_dma_stop(tdc); 566 dev_err(tdc2dev(tdc), "Error in dma transfer, aborting dma\n"); 567 tegra_dma_abort_all(tdc); 568 return false; 569 } 570 571 /* Configure next request */ 572 if (!to_terminate) 573 tdc_configure_next_head_desc(tdc); 574 return true; 575 } 576 577 static void handle_once_dma_done(struct tegra_dma_channel *tdc, 578 bool to_terminate) 579 { 580 struct tegra_dma_sg_req *sgreq; 581 struct tegra_dma_desc *dma_desc; 582 583 tdc->busy = false; 584 sgreq = list_first_entry(&tdc->pending_sg_req, typeof(*sgreq), node); 585 dma_desc = sgreq->dma_desc; 586 dma_desc->bytes_transferred += sgreq->req_len; 587 588 list_del(&sgreq->node); 589 if (sgreq->last_sg) { 590 dma_desc->dma_status = DMA_COMPLETE; 591 dma_cookie_complete(&dma_desc->txd); 592 if (!dma_desc->cb_count) 593 list_add_tail(&dma_desc->cb_node, &tdc->cb_desc); 594 dma_desc->cb_count++; 595 list_add_tail(&dma_desc->node, &tdc->free_dma_desc); 596 } 597 list_add_tail(&sgreq->node, &tdc->free_sg_req); 598 599 /* Do not start DMA if it is going to be terminate */ 600 if (to_terminate || list_empty(&tdc->pending_sg_req)) 601 return; 602 603 tdc_start_head_req(tdc); 604 return; 605 } 606 607 static void handle_cont_sngl_cycle_dma_done(struct tegra_dma_channel *tdc, 608 bool to_terminate) 609 { 610 struct tegra_dma_sg_req *sgreq; 611 struct tegra_dma_desc *dma_desc; 612 bool st; 613 614 sgreq = list_first_entry(&tdc->pending_sg_req, typeof(*sgreq), node); 615 dma_desc = sgreq->dma_desc; 616 dma_desc->bytes_transferred += sgreq->req_len; 617 618 /* Callback need to be call */ 619 if (!dma_desc->cb_count) 620 list_add_tail(&dma_desc->cb_node, &tdc->cb_desc); 621 dma_desc->cb_count++; 622 623 /* If not last req then put at end of pending list */ 624 if (!list_is_last(&sgreq->node, &tdc->pending_sg_req)) { 625 list_move_tail(&sgreq->node, &tdc->pending_sg_req); 626 sgreq->configured = false; 627 st = handle_continuous_head_request(tdc, sgreq, to_terminate); 628 if (!st) 629 dma_desc->dma_status = DMA_ERROR; 630 } 631 return; 632 } 633 634 static void tegra_dma_tasklet(unsigned long data) 635 { 636 struct tegra_dma_channel *tdc = (struct tegra_dma_channel *)data; 637 dma_async_tx_callback callback = NULL; 638 void *callback_param = NULL; 639 struct tegra_dma_desc *dma_desc; 640 unsigned long flags; 641 int cb_count; 642 643 spin_lock_irqsave(&tdc->lock, flags); 644 while (!list_empty(&tdc->cb_desc)) { 645 dma_desc = list_first_entry(&tdc->cb_desc, 646 typeof(*dma_desc), cb_node); 647 list_del(&dma_desc->cb_node); 648 callback = dma_desc->txd.callback; 649 callback_param = dma_desc->txd.callback_param; 650 cb_count = dma_desc->cb_count; 651 dma_desc->cb_count = 0; 652 spin_unlock_irqrestore(&tdc->lock, flags); 653 while (cb_count-- && callback) 654 callback(callback_param); 655 spin_lock_irqsave(&tdc->lock, flags); 656 } 657 spin_unlock_irqrestore(&tdc->lock, flags); 658 } 659 660 static irqreturn_t tegra_dma_isr(int irq, void *dev_id) 661 { 662 struct tegra_dma_channel *tdc = dev_id; 663 unsigned long status; 664 unsigned long flags; 665 666 spin_lock_irqsave(&tdc->lock, flags); 667 668 status = tdc_read(tdc, TEGRA_APBDMA_CHAN_STATUS); 669 if (status & TEGRA_APBDMA_STATUS_ISE_EOC) { 670 tdc_write(tdc, TEGRA_APBDMA_CHAN_STATUS, status); 671 tdc->isr_handler(tdc, false); 672 tasklet_schedule(&tdc->tasklet); 673 spin_unlock_irqrestore(&tdc->lock, flags); 674 return IRQ_HANDLED; 675 } 676 677 spin_unlock_irqrestore(&tdc->lock, flags); 678 dev_info(tdc2dev(tdc), 679 "Interrupt already served status 0x%08lx\n", status); 680 return IRQ_NONE; 681 } 682 683 static dma_cookie_t tegra_dma_tx_submit(struct dma_async_tx_descriptor *txd) 684 { 685 struct tegra_dma_desc *dma_desc = txd_to_tegra_dma_desc(txd); 686 struct tegra_dma_channel *tdc = to_tegra_dma_chan(txd->chan); 687 unsigned long flags; 688 dma_cookie_t cookie; 689 690 spin_lock_irqsave(&tdc->lock, flags); 691 dma_desc->dma_status = DMA_IN_PROGRESS; 692 cookie = dma_cookie_assign(&dma_desc->txd); 693 list_splice_tail_init(&dma_desc->tx_list, &tdc->pending_sg_req); 694 spin_unlock_irqrestore(&tdc->lock, flags); 695 return cookie; 696 } 697 698 static void tegra_dma_issue_pending(struct dma_chan *dc) 699 { 700 struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc); 701 unsigned long flags; 702 703 spin_lock_irqsave(&tdc->lock, flags); 704 if (list_empty(&tdc->pending_sg_req)) { 705 dev_err(tdc2dev(tdc), "No DMA request\n"); 706 goto end; 707 } 708 if (!tdc->busy) { 709 tdc_start_head_req(tdc); 710 711 /* Continuous single mode: Configure next req */ 712 if (tdc->cyclic) { 713 /* 714 * Wait for 1 burst time for configure DMA for 715 * next transfer. 716 */ 717 udelay(TEGRA_APBDMA_BURST_COMPLETE_TIME); 718 tdc_configure_next_head_desc(tdc); 719 } 720 } 721 end: 722 spin_unlock_irqrestore(&tdc->lock, flags); 723 return; 724 } 725 726 static int tegra_dma_terminate_all(struct dma_chan *dc) 727 { 728 struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc); 729 struct tegra_dma_sg_req *sgreq; 730 struct tegra_dma_desc *dma_desc; 731 unsigned long flags; 732 unsigned long status; 733 unsigned long wcount; 734 bool was_busy; 735 736 spin_lock_irqsave(&tdc->lock, flags); 737 if (list_empty(&tdc->pending_sg_req)) { 738 spin_unlock_irqrestore(&tdc->lock, flags); 739 return 0; 740 } 741 742 if (!tdc->busy) 743 goto skip_dma_stop; 744 745 /* Pause DMA before checking the queue status */ 746 tegra_dma_pause(tdc, true); 747 748 status = tdc_read(tdc, TEGRA_APBDMA_CHAN_STATUS); 749 if (status & TEGRA_APBDMA_STATUS_ISE_EOC) { 750 dev_dbg(tdc2dev(tdc), "%s():handling isr\n", __func__); 751 tdc->isr_handler(tdc, true); 752 status = tdc_read(tdc, TEGRA_APBDMA_CHAN_STATUS); 753 } 754 if (tdc->tdma->chip_data->support_separate_wcount_reg) 755 wcount = tdc_read(tdc, TEGRA_APBDMA_CHAN_WORD_TRANSFER); 756 else 757 wcount = status; 758 759 was_busy = tdc->busy; 760 tegra_dma_stop(tdc); 761 762 if (!list_empty(&tdc->pending_sg_req) && was_busy) { 763 sgreq = list_first_entry(&tdc->pending_sg_req, 764 typeof(*sgreq), node); 765 sgreq->dma_desc->bytes_transferred += 766 get_current_xferred_count(tdc, sgreq, wcount); 767 } 768 tegra_dma_resume(tdc); 769 770 skip_dma_stop: 771 tegra_dma_abort_all(tdc); 772 773 while (!list_empty(&tdc->cb_desc)) { 774 dma_desc = list_first_entry(&tdc->cb_desc, 775 typeof(*dma_desc), cb_node); 776 list_del(&dma_desc->cb_node); 777 dma_desc->cb_count = 0; 778 } 779 spin_unlock_irqrestore(&tdc->lock, flags); 780 return 0; 781 } 782 783 static enum dma_status tegra_dma_tx_status(struct dma_chan *dc, 784 dma_cookie_t cookie, struct dma_tx_state *txstate) 785 { 786 struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc); 787 struct tegra_dma_desc *dma_desc; 788 struct tegra_dma_sg_req *sg_req; 789 enum dma_status ret; 790 unsigned long flags; 791 unsigned int residual; 792 793 ret = dma_cookie_status(dc, cookie, txstate); 794 if (ret == DMA_COMPLETE) 795 return ret; 796 797 spin_lock_irqsave(&tdc->lock, flags); 798 799 /* Check on wait_ack desc status */ 800 list_for_each_entry(dma_desc, &tdc->free_dma_desc, node) { 801 if (dma_desc->txd.cookie == cookie) { 802 residual = dma_desc->bytes_requested - 803 (dma_desc->bytes_transferred % 804 dma_desc->bytes_requested); 805 dma_set_residue(txstate, residual); 806 ret = dma_desc->dma_status; 807 spin_unlock_irqrestore(&tdc->lock, flags); 808 return ret; 809 } 810 } 811 812 /* Check in pending list */ 813 list_for_each_entry(sg_req, &tdc->pending_sg_req, node) { 814 dma_desc = sg_req->dma_desc; 815 if (dma_desc->txd.cookie == cookie) { 816 residual = dma_desc->bytes_requested - 817 (dma_desc->bytes_transferred % 818 dma_desc->bytes_requested); 819 dma_set_residue(txstate, residual); 820 ret = dma_desc->dma_status; 821 spin_unlock_irqrestore(&tdc->lock, flags); 822 return ret; 823 } 824 } 825 826 dev_dbg(tdc2dev(tdc), "cookie %d does not found\n", cookie); 827 spin_unlock_irqrestore(&tdc->lock, flags); 828 return ret; 829 } 830 831 static inline int get_bus_width(struct tegra_dma_channel *tdc, 832 enum dma_slave_buswidth slave_bw) 833 { 834 switch (slave_bw) { 835 case DMA_SLAVE_BUSWIDTH_1_BYTE: 836 return TEGRA_APBDMA_APBSEQ_BUS_WIDTH_8; 837 case DMA_SLAVE_BUSWIDTH_2_BYTES: 838 return TEGRA_APBDMA_APBSEQ_BUS_WIDTH_16; 839 case DMA_SLAVE_BUSWIDTH_4_BYTES: 840 return TEGRA_APBDMA_APBSEQ_BUS_WIDTH_32; 841 case DMA_SLAVE_BUSWIDTH_8_BYTES: 842 return TEGRA_APBDMA_APBSEQ_BUS_WIDTH_64; 843 default: 844 dev_warn(tdc2dev(tdc), 845 "slave bw is not supported, using 32bits\n"); 846 return TEGRA_APBDMA_APBSEQ_BUS_WIDTH_32; 847 } 848 } 849 850 static inline int get_burst_size(struct tegra_dma_channel *tdc, 851 u32 burst_size, enum dma_slave_buswidth slave_bw, int len) 852 { 853 int burst_byte; 854 int burst_ahb_width; 855 856 /* 857 * burst_size from client is in terms of the bus_width. 858 * convert them into AHB memory width which is 4 byte. 859 */ 860 burst_byte = burst_size * slave_bw; 861 burst_ahb_width = burst_byte / 4; 862 863 /* If burst size is 0 then calculate the burst size based on length */ 864 if (!burst_ahb_width) { 865 if (len & 0xF) 866 return TEGRA_APBDMA_AHBSEQ_BURST_1; 867 else if ((len >> 4) & 0x1) 868 return TEGRA_APBDMA_AHBSEQ_BURST_4; 869 else 870 return TEGRA_APBDMA_AHBSEQ_BURST_8; 871 } 872 if (burst_ahb_width < 4) 873 return TEGRA_APBDMA_AHBSEQ_BURST_1; 874 else if (burst_ahb_width < 8) 875 return TEGRA_APBDMA_AHBSEQ_BURST_4; 876 else 877 return TEGRA_APBDMA_AHBSEQ_BURST_8; 878 } 879 880 static int get_transfer_param(struct tegra_dma_channel *tdc, 881 enum dma_transfer_direction direction, unsigned long *apb_addr, 882 unsigned long *apb_seq, unsigned long *csr, unsigned int *burst_size, 883 enum dma_slave_buswidth *slave_bw) 884 { 885 886 switch (direction) { 887 case DMA_MEM_TO_DEV: 888 *apb_addr = tdc->dma_sconfig.dst_addr; 889 *apb_seq = get_bus_width(tdc, tdc->dma_sconfig.dst_addr_width); 890 *burst_size = tdc->dma_sconfig.dst_maxburst; 891 *slave_bw = tdc->dma_sconfig.dst_addr_width; 892 *csr = TEGRA_APBDMA_CSR_DIR; 893 return 0; 894 895 case DMA_DEV_TO_MEM: 896 *apb_addr = tdc->dma_sconfig.src_addr; 897 *apb_seq = get_bus_width(tdc, tdc->dma_sconfig.src_addr_width); 898 *burst_size = tdc->dma_sconfig.src_maxburst; 899 *slave_bw = tdc->dma_sconfig.src_addr_width; 900 *csr = 0; 901 return 0; 902 903 default: 904 dev_err(tdc2dev(tdc), "Dma direction is not supported\n"); 905 return -EINVAL; 906 } 907 return -EINVAL; 908 } 909 910 static void tegra_dma_prep_wcount(struct tegra_dma_channel *tdc, 911 struct tegra_dma_channel_regs *ch_regs, u32 len) 912 { 913 u32 len_field = (len - 4) & 0xFFFC; 914 915 if (tdc->tdma->chip_data->support_separate_wcount_reg) 916 ch_regs->wcount = len_field; 917 else 918 ch_regs->csr |= len_field; 919 } 920 921 static struct dma_async_tx_descriptor *tegra_dma_prep_slave_sg( 922 struct dma_chan *dc, struct scatterlist *sgl, unsigned int sg_len, 923 enum dma_transfer_direction direction, unsigned long flags, 924 void *context) 925 { 926 struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc); 927 struct tegra_dma_desc *dma_desc; 928 unsigned int i; 929 struct scatterlist *sg; 930 unsigned long csr, ahb_seq, apb_ptr, apb_seq; 931 struct list_head req_list; 932 struct tegra_dma_sg_req *sg_req = NULL; 933 u32 burst_size; 934 enum dma_slave_buswidth slave_bw; 935 int ret; 936 937 if (!tdc->config_init) { 938 dev_err(tdc2dev(tdc), "dma channel is not configured\n"); 939 return NULL; 940 } 941 if (sg_len < 1) { 942 dev_err(tdc2dev(tdc), "Invalid segment length %d\n", sg_len); 943 return NULL; 944 } 945 946 ret = get_transfer_param(tdc, direction, &apb_ptr, &apb_seq, &csr, 947 &burst_size, &slave_bw); 948 if (ret < 0) 949 return NULL; 950 951 INIT_LIST_HEAD(&req_list); 952 953 ahb_seq = TEGRA_APBDMA_AHBSEQ_INTR_ENB; 954 ahb_seq |= TEGRA_APBDMA_AHBSEQ_WRAP_NONE << 955 TEGRA_APBDMA_AHBSEQ_WRAP_SHIFT; 956 ahb_seq |= TEGRA_APBDMA_AHBSEQ_BUS_WIDTH_32; 957 958 csr |= TEGRA_APBDMA_CSR_ONCE | TEGRA_APBDMA_CSR_FLOW; 959 csr |= tdc->slave_id << TEGRA_APBDMA_CSR_REQ_SEL_SHIFT; 960 if (flags & DMA_PREP_INTERRUPT) 961 csr |= TEGRA_APBDMA_CSR_IE_EOC; 962 963 apb_seq |= TEGRA_APBDMA_APBSEQ_WRAP_WORD_1; 964 965 dma_desc = tegra_dma_desc_get(tdc); 966 if (!dma_desc) { 967 dev_err(tdc2dev(tdc), "Dma descriptors not available\n"); 968 return NULL; 969 } 970 INIT_LIST_HEAD(&dma_desc->tx_list); 971 INIT_LIST_HEAD(&dma_desc->cb_node); 972 dma_desc->cb_count = 0; 973 dma_desc->bytes_requested = 0; 974 dma_desc->bytes_transferred = 0; 975 dma_desc->dma_status = DMA_IN_PROGRESS; 976 977 /* Make transfer requests */ 978 for_each_sg(sgl, sg, sg_len, i) { 979 u32 len, mem; 980 981 mem = sg_dma_address(sg); 982 len = sg_dma_len(sg); 983 984 if ((len & 3) || (mem & 3) || 985 (len > tdc->tdma->chip_data->max_dma_count)) { 986 dev_err(tdc2dev(tdc), 987 "Dma length/memory address is not supported\n"); 988 tegra_dma_desc_put(tdc, dma_desc); 989 return NULL; 990 } 991 992 sg_req = tegra_dma_sg_req_get(tdc); 993 if (!sg_req) { 994 dev_err(tdc2dev(tdc), "Dma sg-req not available\n"); 995 tegra_dma_desc_put(tdc, dma_desc); 996 return NULL; 997 } 998 999 ahb_seq |= get_burst_size(tdc, burst_size, slave_bw, len); 1000 dma_desc->bytes_requested += len; 1001 1002 sg_req->ch_regs.apb_ptr = apb_ptr; 1003 sg_req->ch_regs.ahb_ptr = mem; 1004 sg_req->ch_regs.csr = csr; 1005 tegra_dma_prep_wcount(tdc, &sg_req->ch_regs, len); 1006 sg_req->ch_regs.apb_seq = apb_seq; 1007 sg_req->ch_regs.ahb_seq = ahb_seq; 1008 sg_req->configured = false; 1009 sg_req->last_sg = false; 1010 sg_req->dma_desc = dma_desc; 1011 sg_req->req_len = len; 1012 1013 list_add_tail(&sg_req->node, &dma_desc->tx_list); 1014 } 1015 sg_req->last_sg = true; 1016 if (flags & DMA_CTRL_ACK) 1017 dma_desc->txd.flags = DMA_CTRL_ACK; 1018 1019 /* 1020 * Make sure that mode should not be conflicting with currently 1021 * configured mode. 1022 */ 1023 if (!tdc->isr_handler) { 1024 tdc->isr_handler = handle_once_dma_done; 1025 tdc->cyclic = false; 1026 } else { 1027 if (tdc->cyclic) { 1028 dev_err(tdc2dev(tdc), "DMA configured in cyclic mode\n"); 1029 tegra_dma_desc_put(tdc, dma_desc); 1030 return NULL; 1031 } 1032 } 1033 1034 return &dma_desc->txd; 1035 } 1036 1037 static struct dma_async_tx_descriptor *tegra_dma_prep_dma_cyclic( 1038 struct dma_chan *dc, dma_addr_t buf_addr, size_t buf_len, 1039 size_t period_len, enum dma_transfer_direction direction, 1040 unsigned long flags) 1041 { 1042 struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc); 1043 struct tegra_dma_desc *dma_desc = NULL; 1044 struct tegra_dma_sg_req *sg_req = NULL; 1045 unsigned long csr, ahb_seq, apb_ptr, apb_seq; 1046 int len; 1047 size_t remain_len; 1048 dma_addr_t mem = buf_addr; 1049 u32 burst_size; 1050 enum dma_slave_buswidth slave_bw; 1051 int ret; 1052 1053 if (!buf_len || !period_len) { 1054 dev_err(tdc2dev(tdc), "Invalid buffer/period len\n"); 1055 return NULL; 1056 } 1057 1058 if (!tdc->config_init) { 1059 dev_err(tdc2dev(tdc), "DMA slave is not configured\n"); 1060 return NULL; 1061 } 1062 1063 /* 1064 * We allow to take more number of requests till DMA is 1065 * not started. The driver will loop over all requests. 1066 * Once DMA is started then new requests can be queued only after 1067 * terminating the DMA. 1068 */ 1069 if (tdc->busy) { 1070 dev_err(tdc2dev(tdc), "Request not allowed when dma running\n"); 1071 return NULL; 1072 } 1073 1074 /* 1075 * We only support cycle transfer when buf_len is multiple of 1076 * period_len. 1077 */ 1078 if (buf_len % period_len) { 1079 dev_err(tdc2dev(tdc), "buf_len is not multiple of period_len\n"); 1080 return NULL; 1081 } 1082 1083 len = period_len; 1084 if ((len & 3) || (buf_addr & 3) || 1085 (len > tdc->tdma->chip_data->max_dma_count)) { 1086 dev_err(tdc2dev(tdc), "Req len/mem address is not correct\n"); 1087 return NULL; 1088 } 1089 1090 ret = get_transfer_param(tdc, direction, &apb_ptr, &apb_seq, &csr, 1091 &burst_size, &slave_bw); 1092 if (ret < 0) 1093 return NULL; 1094 1095 1096 ahb_seq = TEGRA_APBDMA_AHBSEQ_INTR_ENB; 1097 ahb_seq |= TEGRA_APBDMA_AHBSEQ_WRAP_NONE << 1098 TEGRA_APBDMA_AHBSEQ_WRAP_SHIFT; 1099 ahb_seq |= TEGRA_APBDMA_AHBSEQ_BUS_WIDTH_32; 1100 1101 csr |= TEGRA_APBDMA_CSR_FLOW; 1102 if (flags & DMA_PREP_INTERRUPT) 1103 csr |= TEGRA_APBDMA_CSR_IE_EOC; 1104 csr |= tdc->slave_id << TEGRA_APBDMA_CSR_REQ_SEL_SHIFT; 1105 1106 apb_seq |= TEGRA_APBDMA_APBSEQ_WRAP_WORD_1; 1107 1108 dma_desc = tegra_dma_desc_get(tdc); 1109 if (!dma_desc) { 1110 dev_err(tdc2dev(tdc), "not enough descriptors available\n"); 1111 return NULL; 1112 } 1113 1114 INIT_LIST_HEAD(&dma_desc->tx_list); 1115 INIT_LIST_HEAD(&dma_desc->cb_node); 1116 dma_desc->cb_count = 0; 1117 1118 dma_desc->bytes_transferred = 0; 1119 dma_desc->bytes_requested = buf_len; 1120 remain_len = buf_len; 1121 1122 /* Split transfer equal to period size */ 1123 while (remain_len) { 1124 sg_req = tegra_dma_sg_req_get(tdc); 1125 if (!sg_req) { 1126 dev_err(tdc2dev(tdc), "Dma sg-req not available\n"); 1127 tegra_dma_desc_put(tdc, dma_desc); 1128 return NULL; 1129 } 1130 1131 ahb_seq |= get_burst_size(tdc, burst_size, slave_bw, len); 1132 sg_req->ch_regs.apb_ptr = apb_ptr; 1133 sg_req->ch_regs.ahb_ptr = mem; 1134 sg_req->ch_regs.csr = csr; 1135 tegra_dma_prep_wcount(tdc, &sg_req->ch_regs, len); 1136 sg_req->ch_regs.apb_seq = apb_seq; 1137 sg_req->ch_regs.ahb_seq = ahb_seq; 1138 sg_req->configured = false; 1139 sg_req->half_done = false; 1140 sg_req->last_sg = false; 1141 sg_req->dma_desc = dma_desc; 1142 sg_req->req_len = len; 1143 1144 list_add_tail(&sg_req->node, &dma_desc->tx_list); 1145 remain_len -= len; 1146 mem += len; 1147 } 1148 sg_req->last_sg = true; 1149 if (flags & DMA_CTRL_ACK) 1150 dma_desc->txd.flags = DMA_CTRL_ACK; 1151 1152 /* 1153 * Make sure that mode should not be conflicting with currently 1154 * configured mode. 1155 */ 1156 if (!tdc->isr_handler) { 1157 tdc->isr_handler = handle_cont_sngl_cycle_dma_done; 1158 tdc->cyclic = true; 1159 } else { 1160 if (!tdc->cyclic) { 1161 dev_err(tdc2dev(tdc), "DMA configuration conflict\n"); 1162 tegra_dma_desc_put(tdc, dma_desc); 1163 return NULL; 1164 } 1165 } 1166 1167 return &dma_desc->txd; 1168 } 1169 1170 static int tegra_dma_alloc_chan_resources(struct dma_chan *dc) 1171 { 1172 struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc); 1173 struct tegra_dma *tdma = tdc->tdma; 1174 int ret; 1175 1176 dma_cookie_init(&tdc->dma_chan); 1177 tdc->config_init = false; 1178 ret = clk_prepare_enable(tdma->dma_clk); 1179 if (ret < 0) 1180 dev_err(tdc2dev(tdc), "clk_prepare_enable failed: %d\n", ret); 1181 return ret; 1182 } 1183 1184 static void tegra_dma_free_chan_resources(struct dma_chan *dc) 1185 { 1186 struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc); 1187 struct tegra_dma *tdma = tdc->tdma; 1188 1189 struct tegra_dma_desc *dma_desc; 1190 struct tegra_dma_sg_req *sg_req; 1191 struct list_head dma_desc_list; 1192 struct list_head sg_req_list; 1193 unsigned long flags; 1194 1195 INIT_LIST_HEAD(&dma_desc_list); 1196 INIT_LIST_HEAD(&sg_req_list); 1197 1198 dev_dbg(tdc2dev(tdc), "Freeing channel %d\n", tdc->id); 1199 1200 if (tdc->busy) 1201 tegra_dma_terminate_all(dc); 1202 1203 spin_lock_irqsave(&tdc->lock, flags); 1204 list_splice_init(&tdc->pending_sg_req, &sg_req_list); 1205 list_splice_init(&tdc->free_sg_req, &sg_req_list); 1206 list_splice_init(&tdc->free_dma_desc, &dma_desc_list); 1207 INIT_LIST_HEAD(&tdc->cb_desc); 1208 tdc->config_init = false; 1209 tdc->isr_handler = NULL; 1210 spin_unlock_irqrestore(&tdc->lock, flags); 1211 1212 while (!list_empty(&dma_desc_list)) { 1213 dma_desc = list_first_entry(&dma_desc_list, 1214 typeof(*dma_desc), node); 1215 list_del(&dma_desc->node); 1216 kfree(dma_desc); 1217 } 1218 1219 while (!list_empty(&sg_req_list)) { 1220 sg_req = list_first_entry(&sg_req_list, typeof(*sg_req), node); 1221 list_del(&sg_req->node); 1222 kfree(sg_req); 1223 } 1224 clk_disable_unprepare(tdma->dma_clk); 1225 1226 tdc->slave_id = 0; 1227 } 1228 1229 static struct dma_chan *tegra_dma_of_xlate(struct of_phandle_args *dma_spec, 1230 struct of_dma *ofdma) 1231 { 1232 struct tegra_dma *tdma = ofdma->of_dma_data; 1233 struct dma_chan *chan; 1234 struct tegra_dma_channel *tdc; 1235 1236 chan = dma_get_any_slave_channel(&tdma->dma_dev); 1237 if (!chan) 1238 return NULL; 1239 1240 tdc = to_tegra_dma_chan(chan); 1241 tdc->slave_id = dma_spec->args[0]; 1242 1243 return chan; 1244 } 1245 1246 /* Tegra20 specific DMA controller information */ 1247 static const struct tegra_dma_chip_data tegra20_dma_chip_data = { 1248 .nr_channels = 16, 1249 .channel_reg_size = 0x20, 1250 .max_dma_count = 1024UL * 64, 1251 .support_channel_pause = false, 1252 .support_separate_wcount_reg = false, 1253 }; 1254 1255 /* Tegra30 specific DMA controller information */ 1256 static const struct tegra_dma_chip_data tegra30_dma_chip_data = { 1257 .nr_channels = 32, 1258 .channel_reg_size = 0x20, 1259 .max_dma_count = 1024UL * 64, 1260 .support_channel_pause = false, 1261 .support_separate_wcount_reg = false, 1262 }; 1263 1264 /* Tegra114 specific DMA controller information */ 1265 static const struct tegra_dma_chip_data tegra114_dma_chip_data = { 1266 .nr_channels = 32, 1267 .channel_reg_size = 0x20, 1268 .max_dma_count = 1024UL * 64, 1269 .support_channel_pause = true, 1270 .support_separate_wcount_reg = false, 1271 }; 1272 1273 /* Tegra148 specific DMA controller information */ 1274 static const struct tegra_dma_chip_data tegra148_dma_chip_data = { 1275 .nr_channels = 32, 1276 .channel_reg_size = 0x40, 1277 .max_dma_count = 1024UL * 64, 1278 .support_channel_pause = true, 1279 .support_separate_wcount_reg = true, 1280 }; 1281 1282 1283 static const struct of_device_id tegra_dma_of_match[] = { 1284 { 1285 .compatible = "nvidia,tegra148-apbdma", 1286 .data = &tegra148_dma_chip_data, 1287 }, { 1288 .compatible = "nvidia,tegra114-apbdma", 1289 .data = &tegra114_dma_chip_data, 1290 }, { 1291 .compatible = "nvidia,tegra30-apbdma", 1292 .data = &tegra30_dma_chip_data, 1293 }, { 1294 .compatible = "nvidia,tegra20-apbdma", 1295 .data = &tegra20_dma_chip_data, 1296 }, { 1297 }, 1298 }; 1299 MODULE_DEVICE_TABLE(of, tegra_dma_of_match); 1300 1301 static int tegra_dma_probe(struct platform_device *pdev) 1302 { 1303 struct resource *res; 1304 struct tegra_dma *tdma; 1305 int ret; 1306 int i; 1307 const struct tegra_dma_chip_data *cdata = NULL; 1308 const struct of_device_id *match; 1309 1310 match = of_match_device(tegra_dma_of_match, &pdev->dev); 1311 if (!match) { 1312 dev_err(&pdev->dev, "Error: No device match found\n"); 1313 return -ENODEV; 1314 } 1315 cdata = match->data; 1316 1317 tdma = devm_kzalloc(&pdev->dev, sizeof(*tdma) + cdata->nr_channels * 1318 sizeof(struct tegra_dma_channel), GFP_KERNEL); 1319 if (!tdma) { 1320 dev_err(&pdev->dev, "Error: memory allocation failed\n"); 1321 return -ENOMEM; 1322 } 1323 1324 tdma->dev = &pdev->dev; 1325 tdma->chip_data = cdata; 1326 platform_set_drvdata(pdev, tdma); 1327 1328 res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 1329 tdma->base_addr = devm_ioremap_resource(&pdev->dev, res); 1330 if (IS_ERR(tdma->base_addr)) 1331 return PTR_ERR(tdma->base_addr); 1332 1333 tdma->dma_clk = devm_clk_get(&pdev->dev, NULL); 1334 if (IS_ERR(tdma->dma_clk)) { 1335 dev_err(&pdev->dev, "Error: Missing controller clock\n"); 1336 return PTR_ERR(tdma->dma_clk); 1337 } 1338 1339 tdma->rst = devm_reset_control_get(&pdev->dev, "dma"); 1340 if (IS_ERR(tdma->rst)) { 1341 dev_err(&pdev->dev, "Error: Missing reset\n"); 1342 return PTR_ERR(tdma->rst); 1343 } 1344 1345 spin_lock_init(&tdma->global_lock); 1346 1347 pm_runtime_enable(&pdev->dev); 1348 if (!pm_runtime_enabled(&pdev->dev)) { 1349 ret = tegra_dma_runtime_resume(&pdev->dev); 1350 if (ret) { 1351 dev_err(&pdev->dev, "dma_runtime_resume failed %d\n", 1352 ret); 1353 goto err_pm_disable; 1354 } 1355 } 1356 1357 /* Enable clock before accessing registers */ 1358 ret = clk_prepare_enable(tdma->dma_clk); 1359 if (ret < 0) { 1360 dev_err(&pdev->dev, "clk_prepare_enable failed: %d\n", ret); 1361 goto err_pm_disable; 1362 } 1363 1364 /* Reset DMA controller */ 1365 reset_control_assert(tdma->rst); 1366 udelay(2); 1367 reset_control_deassert(tdma->rst); 1368 1369 /* Enable global DMA registers */ 1370 tdma_write(tdma, TEGRA_APBDMA_GENERAL, TEGRA_APBDMA_GENERAL_ENABLE); 1371 tdma_write(tdma, TEGRA_APBDMA_CONTROL, 0); 1372 tdma_write(tdma, TEGRA_APBDMA_IRQ_MASK_SET, 0xFFFFFFFFul); 1373 1374 clk_disable_unprepare(tdma->dma_clk); 1375 1376 INIT_LIST_HEAD(&tdma->dma_dev.channels); 1377 for (i = 0; i < cdata->nr_channels; i++) { 1378 struct tegra_dma_channel *tdc = &tdma->channels[i]; 1379 1380 tdc->chan_base_offset = TEGRA_APBDMA_CHANNEL_BASE_ADD_OFFSET + 1381 i * cdata->channel_reg_size; 1382 1383 res = platform_get_resource(pdev, IORESOURCE_IRQ, i); 1384 if (!res) { 1385 ret = -EINVAL; 1386 dev_err(&pdev->dev, "No irq resource for chan %d\n", i); 1387 goto err_irq; 1388 } 1389 tdc->irq = res->start; 1390 snprintf(tdc->name, sizeof(tdc->name), "apbdma.%d", i); 1391 ret = devm_request_irq(&pdev->dev, tdc->irq, 1392 tegra_dma_isr, 0, tdc->name, tdc); 1393 if (ret) { 1394 dev_err(&pdev->dev, 1395 "request_irq failed with err %d channel %d\n", 1396 ret, i); 1397 goto err_irq; 1398 } 1399 1400 tdc->dma_chan.device = &tdma->dma_dev; 1401 dma_cookie_init(&tdc->dma_chan); 1402 list_add_tail(&tdc->dma_chan.device_node, 1403 &tdma->dma_dev.channels); 1404 tdc->tdma = tdma; 1405 tdc->id = i; 1406 1407 tasklet_init(&tdc->tasklet, tegra_dma_tasklet, 1408 (unsigned long)tdc); 1409 spin_lock_init(&tdc->lock); 1410 1411 INIT_LIST_HEAD(&tdc->pending_sg_req); 1412 INIT_LIST_HEAD(&tdc->free_sg_req); 1413 INIT_LIST_HEAD(&tdc->free_dma_desc); 1414 INIT_LIST_HEAD(&tdc->cb_desc); 1415 } 1416 1417 dma_cap_set(DMA_SLAVE, tdma->dma_dev.cap_mask); 1418 dma_cap_set(DMA_PRIVATE, tdma->dma_dev.cap_mask); 1419 dma_cap_set(DMA_CYCLIC, tdma->dma_dev.cap_mask); 1420 1421 tdma->dma_dev.dev = &pdev->dev; 1422 tdma->dma_dev.device_alloc_chan_resources = 1423 tegra_dma_alloc_chan_resources; 1424 tdma->dma_dev.device_free_chan_resources = 1425 tegra_dma_free_chan_resources; 1426 tdma->dma_dev.device_prep_slave_sg = tegra_dma_prep_slave_sg; 1427 tdma->dma_dev.device_prep_dma_cyclic = tegra_dma_prep_dma_cyclic; 1428 tdma->dma_dev.src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | 1429 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | 1430 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | 1431 BIT(DMA_SLAVE_BUSWIDTH_8_BYTES); 1432 tdma->dma_dev.dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | 1433 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | 1434 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | 1435 BIT(DMA_SLAVE_BUSWIDTH_8_BYTES); 1436 tdma->dma_dev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); 1437 /* 1438 * XXX The hardware appears to support 1439 * DMA_RESIDUE_GRANULARITY_BURST-level reporting, but it's 1440 * only used by this driver during tegra_dma_terminate_all() 1441 */ 1442 tdma->dma_dev.residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT; 1443 tdma->dma_dev.device_config = tegra_dma_slave_config; 1444 tdma->dma_dev.device_terminate_all = tegra_dma_terminate_all; 1445 tdma->dma_dev.device_tx_status = tegra_dma_tx_status; 1446 tdma->dma_dev.device_issue_pending = tegra_dma_issue_pending; 1447 1448 ret = dma_async_device_register(&tdma->dma_dev); 1449 if (ret < 0) { 1450 dev_err(&pdev->dev, 1451 "Tegra20 APB DMA driver registration failed %d\n", ret); 1452 goto err_irq; 1453 } 1454 1455 ret = of_dma_controller_register(pdev->dev.of_node, 1456 tegra_dma_of_xlate, tdma); 1457 if (ret < 0) { 1458 dev_err(&pdev->dev, 1459 "Tegra20 APB DMA OF registration failed %d\n", ret); 1460 goto err_unregister_dma_dev; 1461 } 1462 1463 dev_info(&pdev->dev, "Tegra20 APB DMA driver register %d channels\n", 1464 cdata->nr_channels); 1465 return 0; 1466 1467 err_unregister_dma_dev: 1468 dma_async_device_unregister(&tdma->dma_dev); 1469 err_irq: 1470 while (--i >= 0) { 1471 struct tegra_dma_channel *tdc = &tdma->channels[i]; 1472 tasklet_kill(&tdc->tasklet); 1473 } 1474 1475 err_pm_disable: 1476 pm_runtime_disable(&pdev->dev); 1477 if (!pm_runtime_status_suspended(&pdev->dev)) 1478 tegra_dma_runtime_suspend(&pdev->dev); 1479 return ret; 1480 } 1481 1482 static int tegra_dma_remove(struct platform_device *pdev) 1483 { 1484 struct tegra_dma *tdma = platform_get_drvdata(pdev); 1485 int i; 1486 struct tegra_dma_channel *tdc; 1487 1488 dma_async_device_unregister(&tdma->dma_dev); 1489 1490 for (i = 0; i < tdma->chip_data->nr_channels; ++i) { 1491 tdc = &tdma->channels[i]; 1492 tasklet_kill(&tdc->tasklet); 1493 } 1494 1495 pm_runtime_disable(&pdev->dev); 1496 if (!pm_runtime_status_suspended(&pdev->dev)) 1497 tegra_dma_runtime_suspend(&pdev->dev); 1498 1499 return 0; 1500 } 1501 1502 static int tegra_dma_runtime_suspend(struct device *dev) 1503 { 1504 struct platform_device *pdev = to_platform_device(dev); 1505 struct tegra_dma *tdma = platform_get_drvdata(pdev); 1506 1507 clk_disable_unprepare(tdma->dma_clk); 1508 return 0; 1509 } 1510 1511 static int tegra_dma_runtime_resume(struct device *dev) 1512 { 1513 struct platform_device *pdev = to_platform_device(dev); 1514 struct tegra_dma *tdma = platform_get_drvdata(pdev); 1515 int ret; 1516 1517 ret = clk_prepare_enable(tdma->dma_clk); 1518 if (ret < 0) { 1519 dev_err(dev, "clk_enable failed: %d\n", ret); 1520 return ret; 1521 } 1522 return 0; 1523 } 1524 1525 #ifdef CONFIG_PM_SLEEP 1526 static int tegra_dma_pm_suspend(struct device *dev) 1527 { 1528 struct tegra_dma *tdma = dev_get_drvdata(dev); 1529 int i; 1530 int ret; 1531 1532 /* Enable clock before accessing register */ 1533 ret = tegra_dma_runtime_resume(dev); 1534 if (ret < 0) 1535 return ret; 1536 1537 tdma->reg_gen = tdma_read(tdma, TEGRA_APBDMA_GENERAL); 1538 for (i = 0; i < tdma->chip_data->nr_channels; i++) { 1539 struct tegra_dma_channel *tdc = &tdma->channels[i]; 1540 struct tegra_dma_channel_regs *ch_reg = &tdc->channel_reg; 1541 1542 ch_reg->csr = tdc_read(tdc, TEGRA_APBDMA_CHAN_CSR); 1543 ch_reg->ahb_ptr = tdc_read(tdc, TEGRA_APBDMA_CHAN_AHBPTR); 1544 ch_reg->apb_ptr = tdc_read(tdc, TEGRA_APBDMA_CHAN_APBPTR); 1545 ch_reg->ahb_seq = tdc_read(tdc, TEGRA_APBDMA_CHAN_AHBSEQ); 1546 ch_reg->apb_seq = tdc_read(tdc, TEGRA_APBDMA_CHAN_APBSEQ); 1547 } 1548 1549 /* Disable clock */ 1550 tegra_dma_runtime_suspend(dev); 1551 return 0; 1552 } 1553 1554 static int tegra_dma_pm_resume(struct device *dev) 1555 { 1556 struct tegra_dma *tdma = dev_get_drvdata(dev); 1557 int i; 1558 int ret; 1559 1560 /* Enable clock before accessing register */ 1561 ret = tegra_dma_runtime_resume(dev); 1562 if (ret < 0) 1563 return ret; 1564 1565 tdma_write(tdma, TEGRA_APBDMA_GENERAL, tdma->reg_gen); 1566 tdma_write(tdma, TEGRA_APBDMA_CONTROL, 0); 1567 tdma_write(tdma, TEGRA_APBDMA_IRQ_MASK_SET, 0xFFFFFFFFul); 1568 1569 for (i = 0; i < tdma->chip_data->nr_channels; i++) { 1570 struct tegra_dma_channel *tdc = &tdma->channels[i]; 1571 struct tegra_dma_channel_regs *ch_reg = &tdc->channel_reg; 1572 1573 tdc_write(tdc, TEGRA_APBDMA_CHAN_APBSEQ, ch_reg->apb_seq); 1574 tdc_write(tdc, TEGRA_APBDMA_CHAN_APBPTR, ch_reg->apb_ptr); 1575 tdc_write(tdc, TEGRA_APBDMA_CHAN_AHBSEQ, ch_reg->ahb_seq); 1576 tdc_write(tdc, TEGRA_APBDMA_CHAN_AHBPTR, ch_reg->ahb_ptr); 1577 tdc_write(tdc, TEGRA_APBDMA_CHAN_CSR, 1578 (ch_reg->csr & ~TEGRA_APBDMA_CSR_ENB)); 1579 } 1580 1581 /* Disable clock */ 1582 tegra_dma_runtime_suspend(dev); 1583 return 0; 1584 } 1585 #endif 1586 1587 static const struct dev_pm_ops tegra_dma_dev_pm_ops = { 1588 #ifdef CONFIG_PM 1589 .runtime_suspend = tegra_dma_runtime_suspend, 1590 .runtime_resume = tegra_dma_runtime_resume, 1591 #endif 1592 SET_SYSTEM_SLEEP_PM_OPS(tegra_dma_pm_suspend, tegra_dma_pm_resume) 1593 }; 1594 1595 static struct platform_driver tegra_dmac_driver = { 1596 .driver = { 1597 .name = "tegra-apbdma", 1598 .pm = &tegra_dma_dev_pm_ops, 1599 .of_match_table = tegra_dma_of_match, 1600 }, 1601 .probe = tegra_dma_probe, 1602 .remove = tegra_dma_remove, 1603 }; 1604 1605 module_platform_driver(tegra_dmac_driver); 1606 1607 MODULE_ALIAS("platform:tegra20-apbdma"); 1608 MODULE_DESCRIPTION("NVIDIA Tegra APB DMA Controller driver"); 1609 MODULE_AUTHOR("Laxman Dewangan <ldewangan@nvidia.com>"); 1610 MODULE_LICENSE("GPL v2"); 1611