1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * DMA driver for Nvidia's Tegra20 APB DMA controller. 4 * 5 * Copyright (c) 2012-2013, NVIDIA CORPORATION. All rights reserved. 6 */ 7 8 #include <linux/bitops.h> 9 #include <linux/clk.h> 10 #include <linux/delay.h> 11 #include <linux/dmaengine.h> 12 #include <linux/dma-mapping.h> 13 #include <linux/err.h> 14 #include <linux/init.h> 15 #include <linux/interrupt.h> 16 #include <linux/io.h> 17 #include <linux/mm.h> 18 #include <linux/module.h> 19 #include <linux/of.h> 20 #include <linux/of_device.h> 21 #include <linux/of_dma.h> 22 #include <linux/platform_device.h> 23 #include <linux/pm.h> 24 #include <linux/pm_runtime.h> 25 #include <linux/reset.h> 26 #include <linux/slab.h> 27 #include <linux/wait.h> 28 29 #include "dmaengine.h" 30 31 #define CREATE_TRACE_POINTS 32 #include <trace/events/tegra_apb_dma.h> 33 34 #define TEGRA_APBDMA_GENERAL 0x0 35 #define TEGRA_APBDMA_GENERAL_ENABLE BIT(31) 36 37 #define TEGRA_APBDMA_CONTROL 0x010 38 #define TEGRA_APBDMA_IRQ_MASK 0x01c 39 #define TEGRA_APBDMA_IRQ_MASK_SET 0x020 40 41 /* CSR register */ 42 #define TEGRA_APBDMA_CHAN_CSR 0x00 43 #define TEGRA_APBDMA_CSR_ENB BIT(31) 44 #define TEGRA_APBDMA_CSR_IE_EOC BIT(30) 45 #define TEGRA_APBDMA_CSR_HOLD BIT(29) 46 #define TEGRA_APBDMA_CSR_DIR BIT(28) 47 #define TEGRA_APBDMA_CSR_ONCE BIT(27) 48 #define TEGRA_APBDMA_CSR_FLOW BIT(21) 49 #define TEGRA_APBDMA_CSR_REQ_SEL_SHIFT 16 50 #define TEGRA_APBDMA_CSR_REQ_SEL_MASK 0x1F 51 #define TEGRA_APBDMA_CSR_WCOUNT_MASK 0xFFFC 52 53 /* STATUS register */ 54 #define TEGRA_APBDMA_CHAN_STATUS 0x004 55 #define TEGRA_APBDMA_STATUS_BUSY BIT(31) 56 #define TEGRA_APBDMA_STATUS_ISE_EOC BIT(30) 57 #define TEGRA_APBDMA_STATUS_HALT BIT(29) 58 #define TEGRA_APBDMA_STATUS_PING_PONG BIT(28) 59 #define TEGRA_APBDMA_STATUS_COUNT_SHIFT 2 60 #define TEGRA_APBDMA_STATUS_COUNT_MASK 0xFFFC 61 62 #define TEGRA_APBDMA_CHAN_CSRE 0x00C 63 #define TEGRA_APBDMA_CHAN_CSRE_PAUSE BIT(31) 64 65 /* AHB memory address */ 66 #define TEGRA_APBDMA_CHAN_AHBPTR 0x010 67 68 /* AHB sequence register */ 69 #define TEGRA_APBDMA_CHAN_AHBSEQ 0x14 70 #define TEGRA_APBDMA_AHBSEQ_INTR_ENB BIT(31) 71 #define TEGRA_APBDMA_AHBSEQ_BUS_WIDTH_8 (0 << 28) 72 #define TEGRA_APBDMA_AHBSEQ_BUS_WIDTH_16 (1 << 28) 73 #define TEGRA_APBDMA_AHBSEQ_BUS_WIDTH_32 (2 << 28) 74 #define TEGRA_APBDMA_AHBSEQ_BUS_WIDTH_64 (3 << 28) 75 #define TEGRA_APBDMA_AHBSEQ_BUS_WIDTH_128 (4 << 28) 76 #define TEGRA_APBDMA_AHBSEQ_DATA_SWAP BIT(27) 77 #define TEGRA_APBDMA_AHBSEQ_BURST_1 (4 << 24) 78 #define TEGRA_APBDMA_AHBSEQ_BURST_4 (5 << 24) 79 #define TEGRA_APBDMA_AHBSEQ_BURST_8 (6 << 24) 80 #define TEGRA_APBDMA_AHBSEQ_DBL_BUF BIT(19) 81 #define TEGRA_APBDMA_AHBSEQ_WRAP_SHIFT 16 82 #define TEGRA_APBDMA_AHBSEQ_WRAP_NONE 0 83 84 /* APB address */ 85 #define TEGRA_APBDMA_CHAN_APBPTR 0x018 86 87 /* APB sequence register */ 88 #define TEGRA_APBDMA_CHAN_APBSEQ 0x01c 89 #define TEGRA_APBDMA_APBSEQ_BUS_WIDTH_8 (0 << 28) 90 #define TEGRA_APBDMA_APBSEQ_BUS_WIDTH_16 (1 << 28) 91 #define TEGRA_APBDMA_APBSEQ_BUS_WIDTH_32 (2 << 28) 92 #define TEGRA_APBDMA_APBSEQ_BUS_WIDTH_64 (3 << 28) 93 #define TEGRA_APBDMA_APBSEQ_BUS_WIDTH_128 (4 << 28) 94 #define TEGRA_APBDMA_APBSEQ_DATA_SWAP BIT(27) 95 #define TEGRA_APBDMA_APBSEQ_WRAP_WORD_1 (1 << 16) 96 97 /* Tegra148 specific registers */ 98 #define TEGRA_APBDMA_CHAN_WCOUNT 0x20 99 100 #define TEGRA_APBDMA_CHAN_WORD_TRANSFER 0x24 101 102 /* 103 * If any burst is in flight and DMA paused then this is the time to complete 104 * on-flight burst and update DMA status register. 105 */ 106 #define TEGRA_APBDMA_BURST_COMPLETE_TIME 20 107 108 /* Channel base address offset from APBDMA base address */ 109 #define TEGRA_APBDMA_CHANNEL_BASE_ADD_OFFSET 0x1000 110 111 #define TEGRA_APBDMA_SLAVE_ID_INVALID (TEGRA_APBDMA_CSR_REQ_SEL_MASK + 1) 112 113 struct tegra_dma; 114 115 /* 116 * tegra_dma_chip_data Tegra chip specific DMA data 117 * @nr_channels: Number of channels available in the controller. 118 * @channel_reg_size: Channel register size/stride. 119 * @max_dma_count: Maximum DMA transfer count supported by DMA controller. 120 * @support_channel_pause: Support channel wise pause of dma. 121 * @support_separate_wcount_reg: Support separate word count register. 122 */ 123 struct tegra_dma_chip_data { 124 unsigned int nr_channels; 125 unsigned int channel_reg_size; 126 unsigned int max_dma_count; 127 bool support_channel_pause; 128 bool support_separate_wcount_reg; 129 }; 130 131 /* DMA channel registers */ 132 struct tegra_dma_channel_regs { 133 u32 csr; 134 u32 ahb_ptr; 135 u32 apb_ptr; 136 u32 ahb_seq; 137 u32 apb_seq; 138 u32 wcount; 139 }; 140 141 /* 142 * tegra_dma_sg_req: DMA request details to configure hardware. This 143 * contains the details for one transfer to configure DMA hw. 144 * The client's request for data transfer can be broken into multiple 145 * sub-transfer as per requester details and hw support. 146 * This sub transfer get added in the list of transfer and point to Tegra 147 * DMA descriptor which manages the transfer details. 148 */ 149 struct tegra_dma_sg_req { 150 struct tegra_dma_channel_regs ch_regs; 151 unsigned int req_len; 152 bool configured; 153 bool last_sg; 154 struct list_head node; 155 struct tegra_dma_desc *dma_desc; 156 unsigned int words_xferred; 157 }; 158 159 /* 160 * tegra_dma_desc: Tegra DMA descriptors which manages the client requests. 161 * This descriptor keep track of transfer status, callbacks and request 162 * counts etc. 163 */ 164 struct tegra_dma_desc { 165 struct dma_async_tx_descriptor txd; 166 unsigned int bytes_requested; 167 unsigned int bytes_transferred; 168 enum dma_status dma_status; 169 struct list_head node; 170 struct list_head tx_list; 171 struct list_head cb_node; 172 unsigned int cb_count; 173 }; 174 175 struct tegra_dma_channel; 176 177 typedef void (*dma_isr_handler)(struct tegra_dma_channel *tdc, 178 bool to_terminate); 179 180 /* tegra_dma_channel: Channel specific information */ 181 struct tegra_dma_channel { 182 struct dma_chan dma_chan; 183 char name[12]; 184 bool config_init; 185 unsigned int id; 186 void __iomem *chan_addr; 187 spinlock_t lock; 188 bool busy; 189 struct tegra_dma *tdma; 190 bool cyclic; 191 192 /* Different lists for managing the requests */ 193 struct list_head free_sg_req; 194 struct list_head pending_sg_req; 195 struct list_head free_dma_desc; 196 struct list_head cb_desc; 197 198 /* ISR handler and tasklet for bottom half of isr handling */ 199 dma_isr_handler isr_handler; 200 struct tasklet_struct tasklet; 201 202 /* Channel-slave specific configuration */ 203 unsigned int slave_id; 204 struct dma_slave_config dma_sconfig; 205 struct tegra_dma_channel_regs channel_reg; 206 207 struct wait_queue_head wq; 208 }; 209 210 /* tegra_dma: Tegra DMA specific information */ 211 struct tegra_dma { 212 struct dma_device dma_dev; 213 struct device *dev; 214 struct clk *dma_clk; 215 struct reset_control *rst; 216 spinlock_t global_lock; 217 void __iomem *base_addr; 218 const struct tegra_dma_chip_data *chip_data; 219 220 /* 221 * Counter for managing global pausing of the DMA controller. 222 * Only applicable for devices that don't support individual 223 * channel pausing. 224 */ 225 u32 global_pause_count; 226 227 /* Last member of the structure */ 228 struct tegra_dma_channel channels[]; 229 }; 230 231 static inline void tdma_write(struct tegra_dma *tdma, u32 reg, u32 val) 232 { 233 writel(val, tdma->base_addr + reg); 234 } 235 236 static inline void tdc_write(struct tegra_dma_channel *tdc, 237 u32 reg, u32 val) 238 { 239 writel(val, tdc->chan_addr + reg); 240 } 241 242 static inline u32 tdc_read(struct tegra_dma_channel *tdc, u32 reg) 243 { 244 return readl(tdc->chan_addr + reg); 245 } 246 247 static inline struct tegra_dma_channel *to_tegra_dma_chan(struct dma_chan *dc) 248 { 249 return container_of(dc, struct tegra_dma_channel, dma_chan); 250 } 251 252 static inline struct tegra_dma_desc * 253 txd_to_tegra_dma_desc(struct dma_async_tx_descriptor *td) 254 { 255 return container_of(td, struct tegra_dma_desc, txd); 256 } 257 258 static inline struct device *tdc2dev(struct tegra_dma_channel *tdc) 259 { 260 return &tdc->dma_chan.dev->device; 261 } 262 263 static dma_cookie_t tegra_dma_tx_submit(struct dma_async_tx_descriptor *tx); 264 265 /* Get DMA desc from free list, if not there then allocate it. */ 266 static struct tegra_dma_desc *tegra_dma_desc_get(struct tegra_dma_channel *tdc) 267 { 268 struct tegra_dma_desc *dma_desc; 269 unsigned long flags; 270 271 spin_lock_irqsave(&tdc->lock, flags); 272 273 /* Do not allocate if desc are waiting for ack */ 274 list_for_each_entry(dma_desc, &tdc->free_dma_desc, node) { 275 if (async_tx_test_ack(&dma_desc->txd) && !dma_desc->cb_count) { 276 list_del(&dma_desc->node); 277 spin_unlock_irqrestore(&tdc->lock, flags); 278 dma_desc->txd.flags = 0; 279 return dma_desc; 280 } 281 } 282 283 spin_unlock_irqrestore(&tdc->lock, flags); 284 285 /* Allocate DMA desc */ 286 dma_desc = kzalloc(sizeof(*dma_desc), GFP_NOWAIT); 287 if (!dma_desc) 288 return NULL; 289 290 dma_async_tx_descriptor_init(&dma_desc->txd, &tdc->dma_chan); 291 dma_desc->txd.tx_submit = tegra_dma_tx_submit; 292 dma_desc->txd.flags = 0; 293 294 return dma_desc; 295 } 296 297 static void tegra_dma_desc_put(struct tegra_dma_channel *tdc, 298 struct tegra_dma_desc *dma_desc) 299 { 300 unsigned long flags; 301 302 spin_lock_irqsave(&tdc->lock, flags); 303 if (!list_empty(&dma_desc->tx_list)) 304 list_splice_init(&dma_desc->tx_list, &tdc->free_sg_req); 305 list_add_tail(&dma_desc->node, &tdc->free_dma_desc); 306 spin_unlock_irqrestore(&tdc->lock, flags); 307 } 308 309 static struct tegra_dma_sg_req * 310 tegra_dma_sg_req_get(struct tegra_dma_channel *tdc) 311 { 312 struct tegra_dma_sg_req *sg_req; 313 unsigned long flags; 314 315 spin_lock_irqsave(&tdc->lock, flags); 316 if (!list_empty(&tdc->free_sg_req)) { 317 sg_req = list_first_entry(&tdc->free_sg_req, typeof(*sg_req), 318 node); 319 list_del(&sg_req->node); 320 spin_unlock_irqrestore(&tdc->lock, flags); 321 return sg_req; 322 } 323 spin_unlock_irqrestore(&tdc->lock, flags); 324 325 sg_req = kzalloc(sizeof(*sg_req), GFP_NOWAIT); 326 327 return sg_req; 328 } 329 330 static int tegra_dma_slave_config(struct dma_chan *dc, 331 struct dma_slave_config *sconfig) 332 { 333 struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc); 334 335 if (!list_empty(&tdc->pending_sg_req)) { 336 dev_err(tdc2dev(tdc), "Configuration not allowed\n"); 337 return -EBUSY; 338 } 339 340 memcpy(&tdc->dma_sconfig, sconfig, sizeof(*sconfig)); 341 tdc->config_init = true; 342 343 return 0; 344 } 345 346 static void tegra_dma_global_pause(struct tegra_dma_channel *tdc, 347 bool wait_for_burst_complete) 348 { 349 struct tegra_dma *tdma = tdc->tdma; 350 351 spin_lock(&tdma->global_lock); 352 353 if (tdc->tdma->global_pause_count == 0) { 354 tdma_write(tdma, TEGRA_APBDMA_GENERAL, 0); 355 if (wait_for_burst_complete) 356 udelay(TEGRA_APBDMA_BURST_COMPLETE_TIME); 357 } 358 359 tdc->tdma->global_pause_count++; 360 361 spin_unlock(&tdma->global_lock); 362 } 363 364 static void tegra_dma_global_resume(struct tegra_dma_channel *tdc) 365 { 366 struct tegra_dma *tdma = tdc->tdma; 367 368 spin_lock(&tdma->global_lock); 369 370 if (WARN_ON(tdc->tdma->global_pause_count == 0)) 371 goto out; 372 373 if (--tdc->tdma->global_pause_count == 0) 374 tdma_write(tdma, TEGRA_APBDMA_GENERAL, 375 TEGRA_APBDMA_GENERAL_ENABLE); 376 377 out: 378 spin_unlock(&tdma->global_lock); 379 } 380 381 static void tegra_dma_pause(struct tegra_dma_channel *tdc, 382 bool wait_for_burst_complete) 383 { 384 struct tegra_dma *tdma = tdc->tdma; 385 386 if (tdma->chip_data->support_channel_pause) { 387 tdc_write(tdc, TEGRA_APBDMA_CHAN_CSRE, 388 TEGRA_APBDMA_CHAN_CSRE_PAUSE); 389 if (wait_for_burst_complete) 390 udelay(TEGRA_APBDMA_BURST_COMPLETE_TIME); 391 } else { 392 tegra_dma_global_pause(tdc, wait_for_burst_complete); 393 } 394 } 395 396 static void tegra_dma_resume(struct tegra_dma_channel *tdc) 397 { 398 struct tegra_dma *tdma = tdc->tdma; 399 400 if (tdma->chip_data->support_channel_pause) 401 tdc_write(tdc, TEGRA_APBDMA_CHAN_CSRE, 0); 402 else 403 tegra_dma_global_resume(tdc); 404 } 405 406 static void tegra_dma_stop(struct tegra_dma_channel *tdc) 407 { 408 u32 csr, status; 409 410 /* Disable interrupts */ 411 csr = tdc_read(tdc, TEGRA_APBDMA_CHAN_CSR); 412 csr &= ~TEGRA_APBDMA_CSR_IE_EOC; 413 tdc_write(tdc, TEGRA_APBDMA_CHAN_CSR, csr); 414 415 /* Disable DMA */ 416 csr &= ~TEGRA_APBDMA_CSR_ENB; 417 tdc_write(tdc, TEGRA_APBDMA_CHAN_CSR, csr); 418 419 /* Clear interrupt status if it is there */ 420 status = tdc_read(tdc, TEGRA_APBDMA_CHAN_STATUS); 421 if (status & TEGRA_APBDMA_STATUS_ISE_EOC) { 422 dev_dbg(tdc2dev(tdc), "%s():clearing interrupt\n", __func__); 423 tdc_write(tdc, TEGRA_APBDMA_CHAN_STATUS, status); 424 } 425 tdc->busy = false; 426 } 427 428 static void tegra_dma_start(struct tegra_dma_channel *tdc, 429 struct tegra_dma_sg_req *sg_req) 430 { 431 struct tegra_dma_channel_regs *ch_regs = &sg_req->ch_regs; 432 433 tdc_write(tdc, TEGRA_APBDMA_CHAN_CSR, ch_regs->csr); 434 tdc_write(tdc, TEGRA_APBDMA_CHAN_APBSEQ, ch_regs->apb_seq); 435 tdc_write(tdc, TEGRA_APBDMA_CHAN_APBPTR, ch_regs->apb_ptr); 436 tdc_write(tdc, TEGRA_APBDMA_CHAN_AHBSEQ, ch_regs->ahb_seq); 437 tdc_write(tdc, TEGRA_APBDMA_CHAN_AHBPTR, ch_regs->ahb_ptr); 438 if (tdc->tdma->chip_data->support_separate_wcount_reg) 439 tdc_write(tdc, TEGRA_APBDMA_CHAN_WCOUNT, ch_regs->wcount); 440 441 /* Start DMA */ 442 tdc_write(tdc, TEGRA_APBDMA_CHAN_CSR, 443 ch_regs->csr | TEGRA_APBDMA_CSR_ENB); 444 } 445 446 static void tegra_dma_configure_for_next(struct tegra_dma_channel *tdc, 447 struct tegra_dma_sg_req *nsg_req) 448 { 449 unsigned long status; 450 451 /* 452 * The DMA controller reloads the new configuration for next transfer 453 * after last burst of current transfer completes. 454 * If there is no IEC status then this makes sure that last burst 455 * has not be completed. There may be case that last burst is on 456 * flight and so it can complete but because DMA is paused, it 457 * will not generates interrupt as well as not reload the new 458 * configuration. 459 * If there is already IEC status then interrupt handler need to 460 * load new configuration. 461 */ 462 tegra_dma_pause(tdc, false); 463 status = tdc_read(tdc, TEGRA_APBDMA_CHAN_STATUS); 464 465 /* 466 * If interrupt is pending then do nothing as the ISR will handle 467 * the programing for new request. 468 */ 469 if (status & TEGRA_APBDMA_STATUS_ISE_EOC) { 470 dev_err(tdc2dev(tdc), 471 "Skipping new configuration as interrupt is pending\n"); 472 tegra_dma_resume(tdc); 473 return; 474 } 475 476 /* Safe to program new configuration */ 477 tdc_write(tdc, TEGRA_APBDMA_CHAN_APBPTR, nsg_req->ch_regs.apb_ptr); 478 tdc_write(tdc, TEGRA_APBDMA_CHAN_AHBPTR, nsg_req->ch_regs.ahb_ptr); 479 if (tdc->tdma->chip_data->support_separate_wcount_reg) 480 tdc_write(tdc, TEGRA_APBDMA_CHAN_WCOUNT, 481 nsg_req->ch_regs.wcount); 482 tdc_write(tdc, TEGRA_APBDMA_CHAN_CSR, 483 nsg_req->ch_regs.csr | TEGRA_APBDMA_CSR_ENB); 484 nsg_req->configured = true; 485 nsg_req->words_xferred = 0; 486 487 tegra_dma_resume(tdc); 488 } 489 490 static void tdc_start_head_req(struct tegra_dma_channel *tdc) 491 { 492 struct tegra_dma_sg_req *sg_req; 493 494 sg_req = list_first_entry(&tdc->pending_sg_req, typeof(*sg_req), node); 495 tegra_dma_start(tdc, sg_req); 496 sg_req->configured = true; 497 sg_req->words_xferred = 0; 498 tdc->busy = true; 499 } 500 501 static void tdc_configure_next_head_desc(struct tegra_dma_channel *tdc) 502 { 503 struct tegra_dma_sg_req *hsgreq, *hnsgreq; 504 505 hsgreq = list_first_entry(&tdc->pending_sg_req, typeof(*hsgreq), node); 506 if (!list_is_last(&hsgreq->node, &tdc->pending_sg_req)) { 507 hnsgreq = list_first_entry(&hsgreq->node, typeof(*hnsgreq), 508 node); 509 tegra_dma_configure_for_next(tdc, hnsgreq); 510 } 511 } 512 513 static inline unsigned int 514 get_current_xferred_count(struct tegra_dma_channel *tdc, 515 struct tegra_dma_sg_req *sg_req, 516 unsigned long status) 517 { 518 return sg_req->req_len - (status & TEGRA_APBDMA_STATUS_COUNT_MASK) - 4; 519 } 520 521 static void tegra_dma_abort_all(struct tegra_dma_channel *tdc) 522 { 523 struct tegra_dma_desc *dma_desc; 524 struct tegra_dma_sg_req *sgreq; 525 526 while (!list_empty(&tdc->pending_sg_req)) { 527 sgreq = list_first_entry(&tdc->pending_sg_req, typeof(*sgreq), 528 node); 529 list_move_tail(&sgreq->node, &tdc->free_sg_req); 530 if (sgreq->last_sg) { 531 dma_desc = sgreq->dma_desc; 532 dma_desc->dma_status = DMA_ERROR; 533 list_add_tail(&dma_desc->node, &tdc->free_dma_desc); 534 535 /* Add in cb list if it is not there. */ 536 if (!dma_desc->cb_count) 537 list_add_tail(&dma_desc->cb_node, 538 &tdc->cb_desc); 539 dma_desc->cb_count++; 540 } 541 } 542 tdc->isr_handler = NULL; 543 } 544 545 static bool handle_continuous_head_request(struct tegra_dma_channel *tdc, 546 bool to_terminate) 547 { 548 struct tegra_dma_sg_req *hsgreq; 549 550 /* 551 * Check that head req on list should be in flight. 552 * If it is not in flight then abort transfer as 553 * looping of transfer can not continue. 554 */ 555 hsgreq = list_first_entry(&tdc->pending_sg_req, typeof(*hsgreq), node); 556 if (!hsgreq->configured) { 557 tegra_dma_stop(tdc); 558 pm_runtime_put(tdc->tdma->dev); 559 dev_err(tdc2dev(tdc), "DMA transfer underflow, aborting DMA\n"); 560 tegra_dma_abort_all(tdc); 561 return false; 562 } 563 564 /* Configure next request */ 565 if (!to_terminate) 566 tdc_configure_next_head_desc(tdc); 567 568 return true; 569 } 570 571 static void handle_once_dma_done(struct tegra_dma_channel *tdc, 572 bool to_terminate) 573 { 574 struct tegra_dma_desc *dma_desc; 575 struct tegra_dma_sg_req *sgreq; 576 577 tdc->busy = false; 578 sgreq = list_first_entry(&tdc->pending_sg_req, typeof(*sgreq), node); 579 dma_desc = sgreq->dma_desc; 580 dma_desc->bytes_transferred += sgreq->req_len; 581 582 list_del(&sgreq->node); 583 if (sgreq->last_sg) { 584 dma_desc->dma_status = DMA_COMPLETE; 585 dma_cookie_complete(&dma_desc->txd); 586 if (!dma_desc->cb_count) 587 list_add_tail(&dma_desc->cb_node, &tdc->cb_desc); 588 dma_desc->cb_count++; 589 list_add_tail(&dma_desc->node, &tdc->free_dma_desc); 590 } 591 list_add_tail(&sgreq->node, &tdc->free_sg_req); 592 593 /* Do not start DMA if it is going to be terminate */ 594 if (to_terminate) 595 return; 596 597 if (list_empty(&tdc->pending_sg_req)) { 598 pm_runtime_put(tdc->tdma->dev); 599 return; 600 } 601 602 tdc_start_head_req(tdc); 603 } 604 605 static void handle_cont_sngl_cycle_dma_done(struct tegra_dma_channel *tdc, 606 bool to_terminate) 607 { 608 struct tegra_dma_desc *dma_desc; 609 struct tegra_dma_sg_req *sgreq; 610 bool st; 611 612 sgreq = list_first_entry(&tdc->pending_sg_req, typeof(*sgreq), node); 613 dma_desc = sgreq->dma_desc; 614 /* if we dma for long enough the transfer count will wrap */ 615 dma_desc->bytes_transferred = 616 (dma_desc->bytes_transferred + sgreq->req_len) % 617 dma_desc->bytes_requested; 618 619 /* Callback need to be call */ 620 if (!dma_desc->cb_count) 621 list_add_tail(&dma_desc->cb_node, &tdc->cb_desc); 622 dma_desc->cb_count++; 623 624 sgreq->words_xferred = 0; 625 626 /* If not last req then put at end of pending list */ 627 if (!list_is_last(&sgreq->node, &tdc->pending_sg_req)) { 628 list_move_tail(&sgreq->node, &tdc->pending_sg_req); 629 sgreq->configured = false; 630 st = handle_continuous_head_request(tdc, to_terminate); 631 if (!st) 632 dma_desc->dma_status = DMA_ERROR; 633 } 634 } 635 636 static void tegra_dma_tasklet(struct tasklet_struct *t) 637 { 638 struct tegra_dma_channel *tdc = from_tasklet(tdc, t, tasklet); 639 struct dmaengine_desc_callback cb; 640 struct tegra_dma_desc *dma_desc; 641 unsigned int cb_count; 642 unsigned long flags; 643 644 spin_lock_irqsave(&tdc->lock, flags); 645 while (!list_empty(&tdc->cb_desc)) { 646 dma_desc = list_first_entry(&tdc->cb_desc, typeof(*dma_desc), 647 cb_node); 648 list_del(&dma_desc->cb_node); 649 dmaengine_desc_get_callback(&dma_desc->txd, &cb); 650 cb_count = dma_desc->cb_count; 651 dma_desc->cb_count = 0; 652 trace_tegra_dma_complete_cb(&tdc->dma_chan, cb_count, 653 cb.callback); 654 spin_unlock_irqrestore(&tdc->lock, flags); 655 while (cb_count--) 656 dmaengine_desc_callback_invoke(&cb, NULL); 657 spin_lock_irqsave(&tdc->lock, flags); 658 } 659 spin_unlock_irqrestore(&tdc->lock, flags); 660 } 661 662 static irqreturn_t tegra_dma_isr(int irq, void *dev_id) 663 { 664 struct tegra_dma_channel *tdc = dev_id; 665 u32 status; 666 667 spin_lock(&tdc->lock); 668 669 trace_tegra_dma_isr(&tdc->dma_chan, irq); 670 status = tdc_read(tdc, TEGRA_APBDMA_CHAN_STATUS); 671 if (status & TEGRA_APBDMA_STATUS_ISE_EOC) { 672 tdc_write(tdc, TEGRA_APBDMA_CHAN_STATUS, status); 673 tdc->isr_handler(tdc, false); 674 tasklet_schedule(&tdc->tasklet); 675 wake_up_all(&tdc->wq); 676 spin_unlock(&tdc->lock); 677 return IRQ_HANDLED; 678 } 679 680 spin_unlock(&tdc->lock); 681 dev_info(tdc2dev(tdc), "Interrupt already served status 0x%08x\n", 682 status); 683 684 return IRQ_NONE; 685 } 686 687 static dma_cookie_t tegra_dma_tx_submit(struct dma_async_tx_descriptor *txd) 688 { 689 struct tegra_dma_desc *dma_desc = txd_to_tegra_dma_desc(txd); 690 struct tegra_dma_channel *tdc = to_tegra_dma_chan(txd->chan); 691 unsigned long flags; 692 dma_cookie_t cookie; 693 694 spin_lock_irqsave(&tdc->lock, flags); 695 dma_desc->dma_status = DMA_IN_PROGRESS; 696 cookie = dma_cookie_assign(&dma_desc->txd); 697 list_splice_tail_init(&dma_desc->tx_list, &tdc->pending_sg_req); 698 spin_unlock_irqrestore(&tdc->lock, flags); 699 700 return cookie; 701 } 702 703 static void tegra_dma_issue_pending(struct dma_chan *dc) 704 { 705 struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc); 706 unsigned long flags; 707 int err; 708 709 spin_lock_irqsave(&tdc->lock, flags); 710 if (list_empty(&tdc->pending_sg_req)) { 711 dev_err(tdc2dev(tdc), "No DMA request\n"); 712 goto end; 713 } 714 if (!tdc->busy) { 715 err = pm_runtime_resume_and_get(tdc->tdma->dev); 716 if (err < 0) { 717 dev_err(tdc2dev(tdc), "Failed to enable DMA\n"); 718 goto end; 719 } 720 721 tdc_start_head_req(tdc); 722 723 /* Continuous single mode: Configure next req */ 724 if (tdc->cyclic) { 725 /* 726 * Wait for 1 burst time for configure DMA for 727 * next transfer. 728 */ 729 udelay(TEGRA_APBDMA_BURST_COMPLETE_TIME); 730 tdc_configure_next_head_desc(tdc); 731 } 732 } 733 end: 734 spin_unlock_irqrestore(&tdc->lock, flags); 735 } 736 737 static int tegra_dma_terminate_all(struct dma_chan *dc) 738 { 739 struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc); 740 struct tegra_dma_desc *dma_desc; 741 struct tegra_dma_sg_req *sgreq; 742 unsigned long flags; 743 u32 status, wcount; 744 bool was_busy; 745 746 spin_lock_irqsave(&tdc->lock, flags); 747 748 if (!tdc->busy) 749 goto skip_dma_stop; 750 751 /* Pause DMA before checking the queue status */ 752 tegra_dma_pause(tdc, true); 753 754 status = tdc_read(tdc, TEGRA_APBDMA_CHAN_STATUS); 755 if (status & TEGRA_APBDMA_STATUS_ISE_EOC) { 756 dev_dbg(tdc2dev(tdc), "%s():handling isr\n", __func__); 757 tdc->isr_handler(tdc, true); 758 status = tdc_read(tdc, TEGRA_APBDMA_CHAN_STATUS); 759 } 760 if (tdc->tdma->chip_data->support_separate_wcount_reg) 761 wcount = tdc_read(tdc, TEGRA_APBDMA_CHAN_WORD_TRANSFER); 762 else 763 wcount = status; 764 765 was_busy = tdc->busy; 766 tegra_dma_stop(tdc); 767 768 if (!list_empty(&tdc->pending_sg_req) && was_busy) { 769 sgreq = list_first_entry(&tdc->pending_sg_req, typeof(*sgreq), 770 node); 771 sgreq->dma_desc->bytes_transferred += 772 get_current_xferred_count(tdc, sgreq, wcount); 773 } 774 tegra_dma_resume(tdc); 775 776 pm_runtime_put(tdc->tdma->dev); 777 wake_up_all(&tdc->wq); 778 779 skip_dma_stop: 780 tegra_dma_abort_all(tdc); 781 782 while (!list_empty(&tdc->cb_desc)) { 783 dma_desc = list_first_entry(&tdc->cb_desc, typeof(*dma_desc), 784 cb_node); 785 list_del(&dma_desc->cb_node); 786 dma_desc->cb_count = 0; 787 } 788 spin_unlock_irqrestore(&tdc->lock, flags); 789 790 return 0; 791 } 792 793 static bool tegra_dma_eoc_interrupt_deasserted(struct tegra_dma_channel *tdc) 794 { 795 unsigned long flags; 796 u32 status; 797 798 spin_lock_irqsave(&tdc->lock, flags); 799 status = tdc_read(tdc, TEGRA_APBDMA_CHAN_STATUS); 800 spin_unlock_irqrestore(&tdc->lock, flags); 801 802 return !(status & TEGRA_APBDMA_STATUS_ISE_EOC); 803 } 804 805 static void tegra_dma_synchronize(struct dma_chan *dc) 806 { 807 struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc); 808 int err; 809 810 err = pm_runtime_resume_and_get(tdc->tdma->dev); 811 if (err < 0) { 812 dev_err(tdc2dev(tdc), "Failed to synchronize DMA: %d\n", err); 813 return; 814 } 815 816 /* 817 * CPU, which handles interrupt, could be busy in 818 * uninterruptible state, in this case sibling CPU 819 * should wait until interrupt is handled. 820 */ 821 wait_event(tdc->wq, tegra_dma_eoc_interrupt_deasserted(tdc)); 822 823 tasklet_kill(&tdc->tasklet); 824 825 pm_runtime_put(tdc->tdma->dev); 826 } 827 828 static unsigned int tegra_dma_sg_bytes_xferred(struct tegra_dma_channel *tdc, 829 struct tegra_dma_sg_req *sg_req) 830 { 831 u32 status, wcount = 0; 832 833 if (!list_is_first(&sg_req->node, &tdc->pending_sg_req)) 834 return 0; 835 836 if (tdc->tdma->chip_data->support_separate_wcount_reg) 837 wcount = tdc_read(tdc, TEGRA_APBDMA_CHAN_WORD_TRANSFER); 838 839 status = tdc_read(tdc, TEGRA_APBDMA_CHAN_STATUS); 840 841 if (!tdc->tdma->chip_data->support_separate_wcount_reg) 842 wcount = status; 843 844 if (status & TEGRA_APBDMA_STATUS_ISE_EOC) 845 return sg_req->req_len; 846 847 wcount = get_current_xferred_count(tdc, sg_req, wcount); 848 849 if (!wcount) { 850 /* 851 * If wcount wasn't ever polled for this SG before, then 852 * simply assume that transfer hasn't started yet. 853 * 854 * Otherwise it's the end of the transfer. 855 * 856 * The alternative would be to poll the status register 857 * until EOC bit is set or wcount goes UP. That's so 858 * because EOC bit is getting set only after the last 859 * burst's completion and counter is less than the actual 860 * transfer size by 4 bytes. The counter value wraps around 861 * in a cyclic mode before EOC is set(!), so we can't easily 862 * distinguish start of transfer from its end. 863 */ 864 if (sg_req->words_xferred) 865 wcount = sg_req->req_len - 4; 866 867 } else if (wcount < sg_req->words_xferred) { 868 /* 869 * This case will never happen for a non-cyclic transfer. 870 * 871 * For a cyclic transfer, although it is possible for the 872 * next transfer to have already started (resetting the word 873 * count), this case should still not happen because we should 874 * have detected that the EOC bit is set and hence the transfer 875 * was completed. 876 */ 877 WARN_ON_ONCE(1); 878 879 wcount = sg_req->req_len - 4; 880 } else { 881 sg_req->words_xferred = wcount; 882 } 883 884 return wcount; 885 } 886 887 static enum dma_status tegra_dma_tx_status(struct dma_chan *dc, 888 dma_cookie_t cookie, 889 struct dma_tx_state *txstate) 890 { 891 struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc); 892 struct tegra_dma_desc *dma_desc; 893 struct tegra_dma_sg_req *sg_req; 894 enum dma_status ret; 895 unsigned long flags; 896 unsigned int residual; 897 unsigned int bytes = 0; 898 899 ret = dma_cookie_status(dc, cookie, txstate); 900 if (ret == DMA_COMPLETE) 901 return ret; 902 903 spin_lock_irqsave(&tdc->lock, flags); 904 905 /* Check on wait_ack desc status */ 906 list_for_each_entry(dma_desc, &tdc->free_dma_desc, node) { 907 if (dma_desc->txd.cookie == cookie) { 908 ret = dma_desc->dma_status; 909 goto found; 910 } 911 } 912 913 /* Check in pending list */ 914 list_for_each_entry(sg_req, &tdc->pending_sg_req, node) { 915 dma_desc = sg_req->dma_desc; 916 if (dma_desc->txd.cookie == cookie) { 917 bytes = tegra_dma_sg_bytes_xferred(tdc, sg_req); 918 ret = dma_desc->dma_status; 919 goto found; 920 } 921 } 922 923 dev_dbg(tdc2dev(tdc), "cookie %d not found\n", cookie); 924 dma_desc = NULL; 925 926 found: 927 if (dma_desc && txstate) { 928 residual = dma_desc->bytes_requested - 929 ((dma_desc->bytes_transferred + bytes) % 930 dma_desc->bytes_requested); 931 dma_set_residue(txstate, residual); 932 } 933 934 trace_tegra_dma_tx_status(&tdc->dma_chan, cookie, txstate); 935 spin_unlock_irqrestore(&tdc->lock, flags); 936 937 return ret; 938 } 939 940 static inline unsigned int get_bus_width(struct tegra_dma_channel *tdc, 941 enum dma_slave_buswidth slave_bw) 942 { 943 switch (slave_bw) { 944 case DMA_SLAVE_BUSWIDTH_1_BYTE: 945 return TEGRA_APBDMA_APBSEQ_BUS_WIDTH_8; 946 case DMA_SLAVE_BUSWIDTH_2_BYTES: 947 return TEGRA_APBDMA_APBSEQ_BUS_WIDTH_16; 948 case DMA_SLAVE_BUSWIDTH_4_BYTES: 949 return TEGRA_APBDMA_APBSEQ_BUS_WIDTH_32; 950 case DMA_SLAVE_BUSWIDTH_8_BYTES: 951 return TEGRA_APBDMA_APBSEQ_BUS_WIDTH_64; 952 default: 953 dev_warn(tdc2dev(tdc), 954 "slave bw is not supported, using 32bits\n"); 955 return TEGRA_APBDMA_APBSEQ_BUS_WIDTH_32; 956 } 957 } 958 959 static inline unsigned int get_burst_size(struct tegra_dma_channel *tdc, 960 u32 burst_size, 961 enum dma_slave_buswidth slave_bw, 962 u32 len) 963 { 964 unsigned int burst_byte, burst_ahb_width; 965 966 /* 967 * burst_size from client is in terms of the bus_width. 968 * convert them into AHB memory width which is 4 byte. 969 */ 970 burst_byte = burst_size * slave_bw; 971 burst_ahb_width = burst_byte / 4; 972 973 /* If burst size is 0 then calculate the burst size based on length */ 974 if (!burst_ahb_width) { 975 if (len & 0xF) 976 return TEGRA_APBDMA_AHBSEQ_BURST_1; 977 else if ((len >> 4) & 0x1) 978 return TEGRA_APBDMA_AHBSEQ_BURST_4; 979 else 980 return TEGRA_APBDMA_AHBSEQ_BURST_8; 981 } 982 if (burst_ahb_width < 4) 983 return TEGRA_APBDMA_AHBSEQ_BURST_1; 984 else if (burst_ahb_width < 8) 985 return TEGRA_APBDMA_AHBSEQ_BURST_4; 986 else 987 return TEGRA_APBDMA_AHBSEQ_BURST_8; 988 } 989 990 static int get_transfer_param(struct tegra_dma_channel *tdc, 991 enum dma_transfer_direction direction, 992 u32 *apb_addr, 993 u32 *apb_seq, 994 u32 *csr, 995 unsigned int *burst_size, 996 enum dma_slave_buswidth *slave_bw) 997 { 998 switch (direction) { 999 case DMA_MEM_TO_DEV: 1000 *apb_addr = tdc->dma_sconfig.dst_addr; 1001 *apb_seq = get_bus_width(tdc, tdc->dma_sconfig.dst_addr_width); 1002 *burst_size = tdc->dma_sconfig.dst_maxburst; 1003 *slave_bw = tdc->dma_sconfig.dst_addr_width; 1004 *csr = TEGRA_APBDMA_CSR_DIR; 1005 return 0; 1006 1007 case DMA_DEV_TO_MEM: 1008 *apb_addr = tdc->dma_sconfig.src_addr; 1009 *apb_seq = get_bus_width(tdc, tdc->dma_sconfig.src_addr_width); 1010 *burst_size = tdc->dma_sconfig.src_maxburst; 1011 *slave_bw = tdc->dma_sconfig.src_addr_width; 1012 *csr = 0; 1013 return 0; 1014 1015 default: 1016 dev_err(tdc2dev(tdc), "DMA direction is not supported\n"); 1017 break; 1018 } 1019 1020 return -EINVAL; 1021 } 1022 1023 static void tegra_dma_prep_wcount(struct tegra_dma_channel *tdc, 1024 struct tegra_dma_channel_regs *ch_regs, 1025 u32 len) 1026 { 1027 u32 len_field = (len - 4) & 0xFFFC; 1028 1029 if (tdc->tdma->chip_data->support_separate_wcount_reg) 1030 ch_regs->wcount = len_field; 1031 else 1032 ch_regs->csr |= len_field; 1033 } 1034 1035 static struct dma_async_tx_descriptor * 1036 tegra_dma_prep_slave_sg(struct dma_chan *dc, 1037 struct scatterlist *sgl, 1038 unsigned int sg_len, 1039 enum dma_transfer_direction direction, 1040 unsigned long flags, 1041 void *context) 1042 { 1043 struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc); 1044 struct tegra_dma_sg_req *sg_req = NULL; 1045 u32 csr, ahb_seq, apb_ptr, apb_seq; 1046 enum dma_slave_buswidth slave_bw; 1047 struct tegra_dma_desc *dma_desc; 1048 struct list_head req_list; 1049 struct scatterlist *sg; 1050 unsigned int burst_size; 1051 unsigned int i; 1052 1053 if (!tdc->config_init) { 1054 dev_err(tdc2dev(tdc), "DMA channel is not configured\n"); 1055 return NULL; 1056 } 1057 if (sg_len < 1) { 1058 dev_err(tdc2dev(tdc), "Invalid segment length %d\n", sg_len); 1059 return NULL; 1060 } 1061 1062 if (get_transfer_param(tdc, direction, &apb_ptr, &apb_seq, &csr, 1063 &burst_size, &slave_bw) < 0) 1064 return NULL; 1065 1066 INIT_LIST_HEAD(&req_list); 1067 1068 ahb_seq = TEGRA_APBDMA_AHBSEQ_INTR_ENB; 1069 ahb_seq |= TEGRA_APBDMA_AHBSEQ_WRAP_NONE << 1070 TEGRA_APBDMA_AHBSEQ_WRAP_SHIFT; 1071 ahb_seq |= TEGRA_APBDMA_AHBSEQ_BUS_WIDTH_32; 1072 1073 csr |= TEGRA_APBDMA_CSR_ONCE; 1074 1075 if (tdc->slave_id != TEGRA_APBDMA_SLAVE_ID_INVALID) { 1076 csr |= TEGRA_APBDMA_CSR_FLOW; 1077 csr |= tdc->slave_id << TEGRA_APBDMA_CSR_REQ_SEL_SHIFT; 1078 } 1079 1080 if (flags & DMA_PREP_INTERRUPT) { 1081 csr |= TEGRA_APBDMA_CSR_IE_EOC; 1082 } else { 1083 WARN_ON_ONCE(1); 1084 return NULL; 1085 } 1086 1087 apb_seq |= TEGRA_APBDMA_APBSEQ_WRAP_WORD_1; 1088 1089 dma_desc = tegra_dma_desc_get(tdc); 1090 if (!dma_desc) { 1091 dev_err(tdc2dev(tdc), "DMA descriptors not available\n"); 1092 return NULL; 1093 } 1094 INIT_LIST_HEAD(&dma_desc->tx_list); 1095 INIT_LIST_HEAD(&dma_desc->cb_node); 1096 dma_desc->cb_count = 0; 1097 dma_desc->bytes_requested = 0; 1098 dma_desc->bytes_transferred = 0; 1099 dma_desc->dma_status = DMA_IN_PROGRESS; 1100 1101 /* Make transfer requests */ 1102 for_each_sg(sgl, sg, sg_len, i) { 1103 u32 len, mem; 1104 1105 mem = sg_dma_address(sg); 1106 len = sg_dma_len(sg); 1107 1108 if ((len & 3) || (mem & 3) || 1109 len > tdc->tdma->chip_data->max_dma_count) { 1110 dev_err(tdc2dev(tdc), 1111 "DMA length/memory address is not supported\n"); 1112 tegra_dma_desc_put(tdc, dma_desc); 1113 return NULL; 1114 } 1115 1116 sg_req = tegra_dma_sg_req_get(tdc); 1117 if (!sg_req) { 1118 dev_err(tdc2dev(tdc), "DMA sg-req not available\n"); 1119 tegra_dma_desc_put(tdc, dma_desc); 1120 return NULL; 1121 } 1122 1123 ahb_seq |= get_burst_size(tdc, burst_size, slave_bw, len); 1124 dma_desc->bytes_requested += len; 1125 1126 sg_req->ch_regs.apb_ptr = apb_ptr; 1127 sg_req->ch_regs.ahb_ptr = mem; 1128 sg_req->ch_regs.csr = csr; 1129 tegra_dma_prep_wcount(tdc, &sg_req->ch_regs, len); 1130 sg_req->ch_regs.apb_seq = apb_seq; 1131 sg_req->ch_regs.ahb_seq = ahb_seq; 1132 sg_req->configured = false; 1133 sg_req->last_sg = false; 1134 sg_req->dma_desc = dma_desc; 1135 sg_req->req_len = len; 1136 1137 list_add_tail(&sg_req->node, &dma_desc->tx_list); 1138 } 1139 sg_req->last_sg = true; 1140 if (flags & DMA_CTRL_ACK) 1141 dma_desc->txd.flags = DMA_CTRL_ACK; 1142 1143 /* 1144 * Make sure that mode should not be conflicting with currently 1145 * configured mode. 1146 */ 1147 if (!tdc->isr_handler) { 1148 tdc->isr_handler = handle_once_dma_done; 1149 tdc->cyclic = false; 1150 } else { 1151 if (tdc->cyclic) { 1152 dev_err(tdc2dev(tdc), "DMA configured in cyclic mode\n"); 1153 tegra_dma_desc_put(tdc, dma_desc); 1154 return NULL; 1155 } 1156 } 1157 1158 return &dma_desc->txd; 1159 } 1160 1161 static struct dma_async_tx_descriptor * 1162 tegra_dma_prep_dma_cyclic(struct dma_chan *dc, dma_addr_t buf_addr, 1163 size_t buf_len, 1164 size_t period_len, 1165 enum dma_transfer_direction direction, 1166 unsigned long flags) 1167 { 1168 struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc); 1169 struct tegra_dma_sg_req *sg_req = NULL; 1170 u32 csr, ahb_seq, apb_ptr, apb_seq; 1171 enum dma_slave_buswidth slave_bw; 1172 struct tegra_dma_desc *dma_desc; 1173 dma_addr_t mem = buf_addr; 1174 unsigned int burst_size; 1175 size_t len, remain_len; 1176 1177 if (!buf_len || !period_len) { 1178 dev_err(tdc2dev(tdc), "Invalid buffer/period len\n"); 1179 return NULL; 1180 } 1181 1182 if (!tdc->config_init) { 1183 dev_err(tdc2dev(tdc), "DMA slave is not configured\n"); 1184 return NULL; 1185 } 1186 1187 /* 1188 * We allow to take more number of requests till DMA is 1189 * not started. The driver will loop over all requests. 1190 * Once DMA is started then new requests can be queued only after 1191 * terminating the DMA. 1192 */ 1193 if (tdc->busy) { 1194 dev_err(tdc2dev(tdc), "Request not allowed when DMA running\n"); 1195 return NULL; 1196 } 1197 1198 /* 1199 * We only support cycle transfer when buf_len is multiple of 1200 * period_len. 1201 */ 1202 if (buf_len % period_len) { 1203 dev_err(tdc2dev(tdc), "buf_len is not multiple of period_len\n"); 1204 return NULL; 1205 } 1206 1207 len = period_len; 1208 if ((len & 3) || (buf_addr & 3) || 1209 len > tdc->tdma->chip_data->max_dma_count) { 1210 dev_err(tdc2dev(tdc), "Req len/mem address is not correct\n"); 1211 return NULL; 1212 } 1213 1214 if (get_transfer_param(tdc, direction, &apb_ptr, &apb_seq, &csr, 1215 &burst_size, &slave_bw) < 0) 1216 return NULL; 1217 1218 ahb_seq = TEGRA_APBDMA_AHBSEQ_INTR_ENB; 1219 ahb_seq |= TEGRA_APBDMA_AHBSEQ_WRAP_NONE << 1220 TEGRA_APBDMA_AHBSEQ_WRAP_SHIFT; 1221 ahb_seq |= TEGRA_APBDMA_AHBSEQ_BUS_WIDTH_32; 1222 1223 if (tdc->slave_id != TEGRA_APBDMA_SLAVE_ID_INVALID) { 1224 csr |= TEGRA_APBDMA_CSR_FLOW; 1225 csr |= tdc->slave_id << TEGRA_APBDMA_CSR_REQ_SEL_SHIFT; 1226 } 1227 1228 if (flags & DMA_PREP_INTERRUPT) { 1229 csr |= TEGRA_APBDMA_CSR_IE_EOC; 1230 } else { 1231 WARN_ON_ONCE(1); 1232 return NULL; 1233 } 1234 1235 apb_seq |= TEGRA_APBDMA_APBSEQ_WRAP_WORD_1; 1236 1237 dma_desc = tegra_dma_desc_get(tdc); 1238 if (!dma_desc) { 1239 dev_err(tdc2dev(tdc), "not enough descriptors available\n"); 1240 return NULL; 1241 } 1242 1243 INIT_LIST_HEAD(&dma_desc->tx_list); 1244 INIT_LIST_HEAD(&dma_desc->cb_node); 1245 dma_desc->cb_count = 0; 1246 1247 dma_desc->bytes_transferred = 0; 1248 dma_desc->bytes_requested = buf_len; 1249 remain_len = buf_len; 1250 1251 /* Split transfer equal to period size */ 1252 while (remain_len) { 1253 sg_req = tegra_dma_sg_req_get(tdc); 1254 if (!sg_req) { 1255 dev_err(tdc2dev(tdc), "DMA sg-req not available\n"); 1256 tegra_dma_desc_put(tdc, dma_desc); 1257 return NULL; 1258 } 1259 1260 ahb_seq |= get_burst_size(tdc, burst_size, slave_bw, len); 1261 sg_req->ch_regs.apb_ptr = apb_ptr; 1262 sg_req->ch_regs.ahb_ptr = mem; 1263 sg_req->ch_regs.csr = csr; 1264 tegra_dma_prep_wcount(tdc, &sg_req->ch_regs, len); 1265 sg_req->ch_regs.apb_seq = apb_seq; 1266 sg_req->ch_regs.ahb_seq = ahb_seq; 1267 sg_req->configured = false; 1268 sg_req->last_sg = false; 1269 sg_req->dma_desc = dma_desc; 1270 sg_req->req_len = len; 1271 1272 list_add_tail(&sg_req->node, &dma_desc->tx_list); 1273 remain_len -= len; 1274 mem += len; 1275 } 1276 sg_req->last_sg = true; 1277 if (flags & DMA_CTRL_ACK) 1278 dma_desc->txd.flags = DMA_CTRL_ACK; 1279 1280 /* 1281 * Make sure that mode should not be conflicting with currently 1282 * configured mode. 1283 */ 1284 if (!tdc->isr_handler) { 1285 tdc->isr_handler = handle_cont_sngl_cycle_dma_done; 1286 tdc->cyclic = true; 1287 } else { 1288 if (!tdc->cyclic) { 1289 dev_err(tdc2dev(tdc), "DMA configuration conflict\n"); 1290 tegra_dma_desc_put(tdc, dma_desc); 1291 return NULL; 1292 } 1293 } 1294 1295 return &dma_desc->txd; 1296 } 1297 1298 static int tegra_dma_alloc_chan_resources(struct dma_chan *dc) 1299 { 1300 struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc); 1301 1302 dma_cookie_init(&tdc->dma_chan); 1303 1304 return 0; 1305 } 1306 1307 static void tegra_dma_free_chan_resources(struct dma_chan *dc) 1308 { 1309 struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc); 1310 struct tegra_dma_desc *dma_desc; 1311 struct tegra_dma_sg_req *sg_req; 1312 struct list_head dma_desc_list; 1313 struct list_head sg_req_list; 1314 1315 INIT_LIST_HEAD(&dma_desc_list); 1316 INIT_LIST_HEAD(&sg_req_list); 1317 1318 dev_dbg(tdc2dev(tdc), "Freeing channel %d\n", tdc->id); 1319 1320 tegra_dma_terminate_all(dc); 1321 tasklet_kill(&tdc->tasklet); 1322 1323 list_splice_init(&tdc->pending_sg_req, &sg_req_list); 1324 list_splice_init(&tdc->free_sg_req, &sg_req_list); 1325 list_splice_init(&tdc->free_dma_desc, &dma_desc_list); 1326 INIT_LIST_HEAD(&tdc->cb_desc); 1327 tdc->config_init = false; 1328 tdc->isr_handler = NULL; 1329 1330 while (!list_empty(&dma_desc_list)) { 1331 dma_desc = list_first_entry(&dma_desc_list, typeof(*dma_desc), 1332 node); 1333 list_del(&dma_desc->node); 1334 kfree(dma_desc); 1335 } 1336 1337 while (!list_empty(&sg_req_list)) { 1338 sg_req = list_first_entry(&sg_req_list, typeof(*sg_req), node); 1339 list_del(&sg_req->node); 1340 kfree(sg_req); 1341 } 1342 1343 tdc->slave_id = TEGRA_APBDMA_SLAVE_ID_INVALID; 1344 } 1345 1346 static struct dma_chan *tegra_dma_of_xlate(struct of_phandle_args *dma_spec, 1347 struct of_dma *ofdma) 1348 { 1349 struct tegra_dma *tdma = ofdma->of_dma_data; 1350 struct tegra_dma_channel *tdc; 1351 struct dma_chan *chan; 1352 1353 if (dma_spec->args[0] > TEGRA_APBDMA_CSR_REQ_SEL_MASK) { 1354 dev_err(tdma->dev, "Invalid slave id: %d\n", dma_spec->args[0]); 1355 return NULL; 1356 } 1357 1358 chan = dma_get_any_slave_channel(&tdma->dma_dev); 1359 if (!chan) 1360 return NULL; 1361 1362 tdc = to_tegra_dma_chan(chan); 1363 tdc->slave_id = dma_spec->args[0]; 1364 1365 return chan; 1366 } 1367 1368 /* Tegra20 specific DMA controller information */ 1369 static const struct tegra_dma_chip_data tegra20_dma_chip_data = { 1370 .nr_channels = 16, 1371 .channel_reg_size = 0x20, 1372 .max_dma_count = 1024UL * 64, 1373 .support_channel_pause = false, 1374 .support_separate_wcount_reg = false, 1375 }; 1376 1377 /* Tegra30 specific DMA controller information */ 1378 static const struct tegra_dma_chip_data tegra30_dma_chip_data = { 1379 .nr_channels = 32, 1380 .channel_reg_size = 0x20, 1381 .max_dma_count = 1024UL * 64, 1382 .support_channel_pause = false, 1383 .support_separate_wcount_reg = false, 1384 }; 1385 1386 /* Tegra114 specific DMA controller information */ 1387 static const struct tegra_dma_chip_data tegra114_dma_chip_data = { 1388 .nr_channels = 32, 1389 .channel_reg_size = 0x20, 1390 .max_dma_count = 1024UL * 64, 1391 .support_channel_pause = true, 1392 .support_separate_wcount_reg = false, 1393 }; 1394 1395 /* Tegra148 specific DMA controller information */ 1396 static const struct tegra_dma_chip_data tegra148_dma_chip_data = { 1397 .nr_channels = 32, 1398 .channel_reg_size = 0x40, 1399 .max_dma_count = 1024UL * 64, 1400 .support_channel_pause = true, 1401 .support_separate_wcount_reg = true, 1402 }; 1403 1404 static int tegra_dma_init_hw(struct tegra_dma *tdma) 1405 { 1406 int err; 1407 1408 err = reset_control_assert(tdma->rst); 1409 if (err) { 1410 dev_err(tdma->dev, "failed to assert reset: %d\n", err); 1411 return err; 1412 } 1413 1414 err = clk_enable(tdma->dma_clk); 1415 if (err) { 1416 dev_err(tdma->dev, "failed to enable clk: %d\n", err); 1417 return err; 1418 } 1419 1420 /* reset DMA controller */ 1421 udelay(2); 1422 reset_control_deassert(tdma->rst); 1423 1424 /* enable global DMA registers */ 1425 tdma_write(tdma, TEGRA_APBDMA_GENERAL, TEGRA_APBDMA_GENERAL_ENABLE); 1426 tdma_write(tdma, TEGRA_APBDMA_CONTROL, 0); 1427 tdma_write(tdma, TEGRA_APBDMA_IRQ_MASK_SET, 0xFFFFFFFF); 1428 1429 clk_disable(tdma->dma_clk); 1430 1431 return 0; 1432 } 1433 1434 static int tegra_dma_probe(struct platform_device *pdev) 1435 { 1436 const struct tegra_dma_chip_data *cdata; 1437 struct tegra_dma *tdma; 1438 unsigned int i; 1439 size_t size; 1440 int ret; 1441 1442 cdata = of_device_get_match_data(&pdev->dev); 1443 size = struct_size(tdma, channels, cdata->nr_channels); 1444 1445 tdma = devm_kzalloc(&pdev->dev, size, GFP_KERNEL); 1446 if (!tdma) 1447 return -ENOMEM; 1448 1449 tdma->dev = &pdev->dev; 1450 tdma->chip_data = cdata; 1451 platform_set_drvdata(pdev, tdma); 1452 1453 tdma->base_addr = devm_platform_ioremap_resource(pdev, 0); 1454 if (IS_ERR(tdma->base_addr)) 1455 return PTR_ERR(tdma->base_addr); 1456 1457 tdma->dma_clk = devm_clk_get(&pdev->dev, NULL); 1458 if (IS_ERR(tdma->dma_clk)) { 1459 dev_err(&pdev->dev, "Error: Missing controller clock\n"); 1460 return PTR_ERR(tdma->dma_clk); 1461 } 1462 1463 tdma->rst = devm_reset_control_get(&pdev->dev, "dma"); 1464 if (IS_ERR(tdma->rst)) { 1465 dev_err(&pdev->dev, "Error: Missing reset\n"); 1466 return PTR_ERR(tdma->rst); 1467 } 1468 1469 spin_lock_init(&tdma->global_lock); 1470 1471 ret = clk_prepare(tdma->dma_clk); 1472 if (ret) 1473 return ret; 1474 1475 ret = tegra_dma_init_hw(tdma); 1476 if (ret) 1477 goto err_clk_unprepare; 1478 1479 pm_runtime_irq_safe(&pdev->dev); 1480 pm_runtime_enable(&pdev->dev); 1481 1482 INIT_LIST_HEAD(&tdma->dma_dev.channels); 1483 for (i = 0; i < cdata->nr_channels; i++) { 1484 struct tegra_dma_channel *tdc = &tdma->channels[i]; 1485 int irq; 1486 1487 tdc->chan_addr = tdma->base_addr + 1488 TEGRA_APBDMA_CHANNEL_BASE_ADD_OFFSET + 1489 (i * cdata->channel_reg_size); 1490 1491 irq = platform_get_irq(pdev, i); 1492 if (irq < 0) { 1493 ret = irq; 1494 goto err_pm_disable; 1495 } 1496 1497 snprintf(tdc->name, sizeof(tdc->name), "apbdma.%d", i); 1498 ret = devm_request_irq(&pdev->dev, irq, tegra_dma_isr, 0, 1499 tdc->name, tdc); 1500 if (ret) { 1501 dev_err(&pdev->dev, 1502 "request_irq failed with err %d channel %d\n", 1503 ret, i); 1504 goto err_pm_disable; 1505 } 1506 1507 tdc->dma_chan.device = &tdma->dma_dev; 1508 dma_cookie_init(&tdc->dma_chan); 1509 list_add_tail(&tdc->dma_chan.device_node, 1510 &tdma->dma_dev.channels); 1511 tdc->tdma = tdma; 1512 tdc->id = i; 1513 tdc->slave_id = TEGRA_APBDMA_SLAVE_ID_INVALID; 1514 1515 tasklet_setup(&tdc->tasklet, tegra_dma_tasklet); 1516 spin_lock_init(&tdc->lock); 1517 init_waitqueue_head(&tdc->wq); 1518 1519 INIT_LIST_HEAD(&tdc->pending_sg_req); 1520 INIT_LIST_HEAD(&tdc->free_sg_req); 1521 INIT_LIST_HEAD(&tdc->free_dma_desc); 1522 INIT_LIST_HEAD(&tdc->cb_desc); 1523 } 1524 1525 dma_cap_set(DMA_SLAVE, tdma->dma_dev.cap_mask); 1526 dma_cap_set(DMA_PRIVATE, tdma->dma_dev.cap_mask); 1527 dma_cap_set(DMA_CYCLIC, tdma->dma_dev.cap_mask); 1528 1529 tdma->global_pause_count = 0; 1530 tdma->dma_dev.dev = &pdev->dev; 1531 tdma->dma_dev.device_alloc_chan_resources = 1532 tegra_dma_alloc_chan_resources; 1533 tdma->dma_dev.device_free_chan_resources = 1534 tegra_dma_free_chan_resources; 1535 tdma->dma_dev.device_prep_slave_sg = tegra_dma_prep_slave_sg; 1536 tdma->dma_dev.device_prep_dma_cyclic = tegra_dma_prep_dma_cyclic; 1537 tdma->dma_dev.src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | 1538 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | 1539 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | 1540 BIT(DMA_SLAVE_BUSWIDTH_8_BYTES); 1541 tdma->dma_dev.dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | 1542 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | 1543 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | 1544 BIT(DMA_SLAVE_BUSWIDTH_8_BYTES); 1545 tdma->dma_dev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); 1546 tdma->dma_dev.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; 1547 tdma->dma_dev.device_config = tegra_dma_slave_config; 1548 tdma->dma_dev.device_terminate_all = tegra_dma_terminate_all; 1549 tdma->dma_dev.device_synchronize = tegra_dma_synchronize; 1550 tdma->dma_dev.device_tx_status = tegra_dma_tx_status; 1551 tdma->dma_dev.device_issue_pending = tegra_dma_issue_pending; 1552 1553 ret = dma_async_device_register(&tdma->dma_dev); 1554 if (ret < 0) { 1555 dev_err(&pdev->dev, 1556 "Tegra20 APB DMA driver registration failed %d\n", ret); 1557 goto err_pm_disable; 1558 } 1559 1560 ret = of_dma_controller_register(pdev->dev.of_node, 1561 tegra_dma_of_xlate, tdma); 1562 if (ret < 0) { 1563 dev_err(&pdev->dev, 1564 "Tegra20 APB DMA OF registration failed %d\n", ret); 1565 goto err_unregister_dma_dev; 1566 } 1567 1568 dev_info(&pdev->dev, "Tegra20 APB DMA driver registered %u channels\n", 1569 cdata->nr_channels); 1570 1571 return 0; 1572 1573 err_unregister_dma_dev: 1574 dma_async_device_unregister(&tdma->dma_dev); 1575 1576 err_pm_disable: 1577 pm_runtime_disable(&pdev->dev); 1578 1579 err_clk_unprepare: 1580 clk_unprepare(tdma->dma_clk); 1581 1582 return ret; 1583 } 1584 1585 static int tegra_dma_remove(struct platform_device *pdev) 1586 { 1587 struct tegra_dma *tdma = platform_get_drvdata(pdev); 1588 1589 of_dma_controller_free(pdev->dev.of_node); 1590 dma_async_device_unregister(&tdma->dma_dev); 1591 pm_runtime_disable(&pdev->dev); 1592 clk_unprepare(tdma->dma_clk); 1593 1594 return 0; 1595 } 1596 1597 static int __maybe_unused tegra_dma_runtime_suspend(struct device *dev) 1598 { 1599 struct tegra_dma *tdma = dev_get_drvdata(dev); 1600 1601 clk_disable(tdma->dma_clk); 1602 1603 return 0; 1604 } 1605 1606 static int __maybe_unused tegra_dma_runtime_resume(struct device *dev) 1607 { 1608 struct tegra_dma *tdma = dev_get_drvdata(dev); 1609 1610 return clk_enable(tdma->dma_clk); 1611 } 1612 1613 static int __maybe_unused tegra_dma_dev_suspend(struct device *dev) 1614 { 1615 struct tegra_dma *tdma = dev_get_drvdata(dev); 1616 unsigned long flags; 1617 unsigned int i; 1618 bool busy; 1619 1620 for (i = 0; i < tdma->chip_data->nr_channels; i++) { 1621 struct tegra_dma_channel *tdc = &tdma->channels[i]; 1622 1623 tasklet_kill(&tdc->tasklet); 1624 1625 spin_lock_irqsave(&tdc->lock, flags); 1626 busy = tdc->busy; 1627 spin_unlock_irqrestore(&tdc->lock, flags); 1628 1629 if (busy) { 1630 dev_err(tdma->dev, "channel %u busy\n", i); 1631 return -EBUSY; 1632 } 1633 } 1634 1635 return pm_runtime_force_suspend(dev); 1636 } 1637 1638 static int __maybe_unused tegra_dma_dev_resume(struct device *dev) 1639 { 1640 struct tegra_dma *tdma = dev_get_drvdata(dev); 1641 int err; 1642 1643 err = tegra_dma_init_hw(tdma); 1644 if (err) 1645 return err; 1646 1647 return pm_runtime_force_resume(dev); 1648 } 1649 1650 static const struct dev_pm_ops tegra_dma_dev_pm_ops = { 1651 SET_RUNTIME_PM_OPS(tegra_dma_runtime_suspend, tegra_dma_runtime_resume, 1652 NULL) 1653 SET_SYSTEM_SLEEP_PM_OPS(tegra_dma_dev_suspend, tegra_dma_dev_resume) 1654 }; 1655 1656 static const struct of_device_id tegra_dma_of_match[] = { 1657 { 1658 .compatible = "nvidia,tegra148-apbdma", 1659 .data = &tegra148_dma_chip_data, 1660 }, { 1661 .compatible = "nvidia,tegra114-apbdma", 1662 .data = &tegra114_dma_chip_data, 1663 }, { 1664 .compatible = "nvidia,tegra30-apbdma", 1665 .data = &tegra30_dma_chip_data, 1666 }, { 1667 .compatible = "nvidia,tegra20-apbdma", 1668 .data = &tegra20_dma_chip_data, 1669 }, { 1670 }, 1671 }; 1672 MODULE_DEVICE_TABLE(of, tegra_dma_of_match); 1673 1674 static struct platform_driver tegra_dmac_driver = { 1675 .driver = { 1676 .name = "tegra-apbdma", 1677 .pm = &tegra_dma_dev_pm_ops, 1678 .of_match_table = tegra_dma_of_match, 1679 }, 1680 .probe = tegra_dma_probe, 1681 .remove = tegra_dma_remove, 1682 }; 1683 1684 module_platform_driver(tegra_dmac_driver); 1685 1686 MODULE_DESCRIPTION("NVIDIA Tegra APB DMA Controller driver"); 1687 MODULE_AUTHOR("Laxman Dewangan <ldewangan@nvidia.com>"); 1688 MODULE_LICENSE("GPL v2"); 1689