1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * DMA driver for Nvidia's Tegra20 APB DMA controller. 4 * 5 * Copyright (c) 2012-2013, NVIDIA CORPORATION. All rights reserved. 6 */ 7 8 #include <linux/bitops.h> 9 #include <linux/clk.h> 10 #include <linux/delay.h> 11 #include <linux/dmaengine.h> 12 #include <linux/dma-mapping.h> 13 #include <linux/err.h> 14 #include <linux/init.h> 15 #include <linux/interrupt.h> 16 #include <linux/io.h> 17 #include <linux/mm.h> 18 #include <linux/module.h> 19 #include <linux/of.h> 20 #include <linux/of_device.h> 21 #include <linux/of_dma.h> 22 #include <linux/platform_device.h> 23 #include <linux/pm.h> 24 #include <linux/pm_runtime.h> 25 #include <linux/reset.h> 26 #include <linux/slab.h> 27 #include <linux/wait.h> 28 29 #include "dmaengine.h" 30 31 #define CREATE_TRACE_POINTS 32 #include <trace/events/tegra_apb_dma.h> 33 34 #define TEGRA_APBDMA_GENERAL 0x0 35 #define TEGRA_APBDMA_GENERAL_ENABLE BIT(31) 36 37 #define TEGRA_APBDMA_CONTROL 0x010 38 #define TEGRA_APBDMA_IRQ_MASK 0x01c 39 #define TEGRA_APBDMA_IRQ_MASK_SET 0x020 40 41 /* CSR register */ 42 #define TEGRA_APBDMA_CHAN_CSR 0x00 43 #define TEGRA_APBDMA_CSR_ENB BIT(31) 44 #define TEGRA_APBDMA_CSR_IE_EOC BIT(30) 45 #define TEGRA_APBDMA_CSR_HOLD BIT(29) 46 #define TEGRA_APBDMA_CSR_DIR BIT(28) 47 #define TEGRA_APBDMA_CSR_ONCE BIT(27) 48 #define TEGRA_APBDMA_CSR_FLOW BIT(21) 49 #define TEGRA_APBDMA_CSR_REQ_SEL_SHIFT 16 50 #define TEGRA_APBDMA_CSR_REQ_SEL_MASK 0x1F 51 #define TEGRA_APBDMA_CSR_WCOUNT_MASK 0xFFFC 52 53 /* STATUS register */ 54 #define TEGRA_APBDMA_CHAN_STATUS 0x004 55 #define TEGRA_APBDMA_STATUS_BUSY BIT(31) 56 #define TEGRA_APBDMA_STATUS_ISE_EOC BIT(30) 57 #define TEGRA_APBDMA_STATUS_HALT BIT(29) 58 #define TEGRA_APBDMA_STATUS_PING_PONG BIT(28) 59 #define TEGRA_APBDMA_STATUS_COUNT_SHIFT 2 60 #define TEGRA_APBDMA_STATUS_COUNT_MASK 0xFFFC 61 62 #define TEGRA_APBDMA_CHAN_CSRE 0x00C 63 #define TEGRA_APBDMA_CHAN_CSRE_PAUSE BIT(31) 64 65 /* AHB memory address */ 66 #define TEGRA_APBDMA_CHAN_AHBPTR 0x010 67 68 /* AHB sequence register */ 69 #define TEGRA_APBDMA_CHAN_AHBSEQ 0x14 70 #define TEGRA_APBDMA_AHBSEQ_INTR_ENB BIT(31) 71 #define TEGRA_APBDMA_AHBSEQ_BUS_WIDTH_8 (0 << 28) 72 #define TEGRA_APBDMA_AHBSEQ_BUS_WIDTH_16 (1 << 28) 73 #define TEGRA_APBDMA_AHBSEQ_BUS_WIDTH_32 (2 << 28) 74 #define TEGRA_APBDMA_AHBSEQ_BUS_WIDTH_64 (3 << 28) 75 #define TEGRA_APBDMA_AHBSEQ_BUS_WIDTH_128 (4 << 28) 76 #define TEGRA_APBDMA_AHBSEQ_DATA_SWAP BIT(27) 77 #define TEGRA_APBDMA_AHBSEQ_BURST_1 (4 << 24) 78 #define TEGRA_APBDMA_AHBSEQ_BURST_4 (5 << 24) 79 #define TEGRA_APBDMA_AHBSEQ_BURST_8 (6 << 24) 80 #define TEGRA_APBDMA_AHBSEQ_DBL_BUF BIT(19) 81 #define TEGRA_APBDMA_AHBSEQ_WRAP_SHIFT 16 82 #define TEGRA_APBDMA_AHBSEQ_WRAP_NONE 0 83 84 /* APB address */ 85 #define TEGRA_APBDMA_CHAN_APBPTR 0x018 86 87 /* APB sequence register */ 88 #define TEGRA_APBDMA_CHAN_APBSEQ 0x01c 89 #define TEGRA_APBDMA_APBSEQ_BUS_WIDTH_8 (0 << 28) 90 #define TEGRA_APBDMA_APBSEQ_BUS_WIDTH_16 (1 << 28) 91 #define TEGRA_APBDMA_APBSEQ_BUS_WIDTH_32 (2 << 28) 92 #define TEGRA_APBDMA_APBSEQ_BUS_WIDTH_64 (3 << 28) 93 #define TEGRA_APBDMA_APBSEQ_BUS_WIDTH_128 (4 << 28) 94 #define TEGRA_APBDMA_APBSEQ_DATA_SWAP BIT(27) 95 #define TEGRA_APBDMA_APBSEQ_WRAP_WORD_1 (1 << 16) 96 97 /* Tegra148 specific registers */ 98 #define TEGRA_APBDMA_CHAN_WCOUNT 0x20 99 100 #define TEGRA_APBDMA_CHAN_WORD_TRANSFER 0x24 101 102 /* 103 * If any burst is in flight and DMA paused then this is the time to complete 104 * on-flight burst and update DMA status register. 105 */ 106 #define TEGRA_APBDMA_BURST_COMPLETE_TIME 20 107 108 /* Channel base address offset from APBDMA base address */ 109 #define TEGRA_APBDMA_CHANNEL_BASE_ADD_OFFSET 0x1000 110 111 #define TEGRA_APBDMA_SLAVE_ID_INVALID (TEGRA_APBDMA_CSR_REQ_SEL_MASK + 1) 112 113 struct tegra_dma; 114 115 /* 116 * tegra_dma_chip_data Tegra chip specific DMA data 117 * @nr_channels: Number of channels available in the controller. 118 * @channel_reg_size: Channel register size/stride. 119 * @max_dma_count: Maximum DMA transfer count supported by DMA controller. 120 * @support_channel_pause: Support channel wise pause of dma. 121 * @support_separate_wcount_reg: Support separate word count register. 122 */ 123 struct tegra_dma_chip_data { 124 unsigned int nr_channels; 125 unsigned int channel_reg_size; 126 unsigned int max_dma_count; 127 bool support_channel_pause; 128 bool support_separate_wcount_reg; 129 }; 130 131 /* DMA channel registers */ 132 struct tegra_dma_channel_regs { 133 u32 csr; 134 u32 ahb_ptr; 135 u32 apb_ptr; 136 u32 ahb_seq; 137 u32 apb_seq; 138 u32 wcount; 139 }; 140 141 /* 142 * tegra_dma_sg_req: DMA request details to configure hardware. This 143 * contains the details for one transfer to configure DMA hw. 144 * The client's request for data transfer can be broken into multiple 145 * sub-transfer as per requester details and hw support. 146 * This sub transfer get added in the list of transfer and point to Tegra 147 * DMA descriptor which manages the transfer details. 148 */ 149 struct tegra_dma_sg_req { 150 struct tegra_dma_channel_regs ch_regs; 151 unsigned int req_len; 152 bool configured; 153 bool last_sg; 154 struct list_head node; 155 struct tegra_dma_desc *dma_desc; 156 unsigned int words_xferred; 157 }; 158 159 /* 160 * tegra_dma_desc: Tegra DMA descriptors which manages the client requests. 161 * This descriptor keep track of transfer status, callbacks and request 162 * counts etc. 163 */ 164 struct tegra_dma_desc { 165 struct dma_async_tx_descriptor txd; 166 unsigned int bytes_requested; 167 unsigned int bytes_transferred; 168 enum dma_status dma_status; 169 struct list_head node; 170 struct list_head tx_list; 171 struct list_head cb_node; 172 unsigned int cb_count; 173 }; 174 175 struct tegra_dma_channel; 176 177 typedef void (*dma_isr_handler)(struct tegra_dma_channel *tdc, 178 bool to_terminate); 179 180 /* tegra_dma_channel: Channel specific information */ 181 struct tegra_dma_channel { 182 struct dma_chan dma_chan; 183 char name[12]; 184 bool config_init; 185 unsigned int id; 186 void __iomem *chan_addr; 187 spinlock_t lock; 188 bool busy; 189 struct tegra_dma *tdma; 190 bool cyclic; 191 192 /* Different lists for managing the requests */ 193 struct list_head free_sg_req; 194 struct list_head pending_sg_req; 195 struct list_head free_dma_desc; 196 struct list_head cb_desc; 197 198 /* ISR handler and tasklet for bottom half of isr handling */ 199 dma_isr_handler isr_handler; 200 struct tasklet_struct tasklet; 201 202 /* Channel-slave specific configuration */ 203 unsigned int slave_id; 204 struct dma_slave_config dma_sconfig; 205 struct tegra_dma_channel_regs channel_reg; 206 207 struct wait_queue_head wq; 208 }; 209 210 /* tegra_dma: Tegra DMA specific information */ 211 struct tegra_dma { 212 struct dma_device dma_dev; 213 struct device *dev; 214 struct clk *dma_clk; 215 struct reset_control *rst; 216 spinlock_t global_lock; 217 void __iomem *base_addr; 218 const struct tegra_dma_chip_data *chip_data; 219 220 /* 221 * Counter for managing global pausing of the DMA controller. 222 * Only applicable for devices that don't support individual 223 * channel pausing. 224 */ 225 u32 global_pause_count; 226 227 /* Last member of the structure */ 228 struct tegra_dma_channel channels[]; 229 }; 230 231 static inline void tdma_write(struct tegra_dma *tdma, u32 reg, u32 val) 232 { 233 writel(val, tdma->base_addr + reg); 234 } 235 236 static inline u32 tdma_read(struct tegra_dma *tdma, u32 reg) 237 { 238 return readl(tdma->base_addr + reg); 239 } 240 241 static inline void tdc_write(struct tegra_dma_channel *tdc, 242 u32 reg, u32 val) 243 { 244 writel(val, tdc->chan_addr + reg); 245 } 246 247 static inline u32 tdc_read(struct tegra_dma_channel *tdc, u32 reg) 248 { 249 return readl(tdc->chan_addr + reg); 250 } 251 252 static inline struct tegra_dma_channel *to_tegra_dma_chan(struct dma_chan *dc) 253 { 254 return container_of(dc, struct tegra_dma_channel, dma_chan); 255 } 256 257 static inline struct tegra_dma_desc * 258 txd_to_tegra_dma_desc(struct dma_async_tx_descriptor *td) 259 { 260 return container_of(td, struct tegra_dma_desc, txd); 261 } 262 263 static inline struct device *tdc2dev(struct tegra_dma_channel *tdc) 264 { 265 return &tdc->dma_chan.dev->device; 266 } 267 268 static dma_cookie_t tegra_dma_tx_submit(struct dma_async_tx_descriptor *tx); 269 270 /* Get DMA desc from free list, if not there then allocate it. */ 271 static struct tegra_dma_desc *tegra_dma_desc_get(struct tegra_dma_channel *tdc) 272 { 273 struct tegra_dma_desc *dma_desc; 274 unsigned long flags; 275 276 spin_lock_irqsave(&tdc->lock, flags); 277 278 /* Do not allocate if desc are waiting for ack */ 279 list_for_each_entry(dma_desc, &tdc->free_dma_desc, node) { 280 if (async_tx_test_ack(&dma_desc->txd) && !dma_desc->cb_count) { 281 list_del(&dma_desc->node); 282 spin_unlock_irqrestore(&tdc->lock, flags); 283 dma_desc->txd.flags = 0; 284 return dma_desc; 285 } 286 } 287 288 spin_unlock_irqrestore(&tdc->lock, flags); 289 290 /* Allocate DMA desc */ 291 dma_desc = kzalloc(sizeof(*dma_desc), GFP_NOWAIT); 292 if (!dma_desc) 293 return NULL; 294 295 dma_async_tx_descriptor_init(&dma_desc->txd, &tdc->dma_chan); 296 dma_desc->txd.tx_submit = tegra_dma_tx_submit; 297 dma_desc->txd.flags = 0; 298 299 return dma_desc; 300 } 301 302 static void tegra_dma_desc_put(struct tegra_dma_channel *tdc, 303 struct tegra_dma_desc *dma_desc) 304 { 305 unsigned long flags; 306 307 spin_lock_irqsave(&tdc->lock, flags); 308 if (!list_empty(&dma_desc->tx_list)) 309 list_splice_init(&dma_desc->tx_list, &tdc->free_sg_req); 310 list_add_tail(&dma_desc->node, &tdc->free_dma_desc); 311 spin_unlock_irqrestore(&tdc->lock, flags); 312 } 313 314 static struct tegra_dma_sg_req * 315 tegra_dma_sg_req_get(struct tegra_dma_channel *tdc) 316 { 317 struct tegra_dma_sg_req *sg_req; 318 unsigned long flags; 319 320 spin_lock_irqsave(&tdc->lock, flags); 321 if (!list_empty(&tdc->free_sg_req)) { 322 sg_req = list_first_entry(&tdc->free_sg_req, typeof(*sg_req), 323 node); 324 list_del(&sg_req->node); 325 spin_unlock_irqrestore(&tdc->lock, flags); 326 return sg_req; 327 } 328 spin_unlock_irqrestore(&tdc->lock, flags); 329 330 sg_req = kzalloc(sizeof(*sg_req), GFP_NOWAIT); 331 332 return sg_req; 333 } 334 335 static int tegra_dma_slave_config(struct dma_chan *dc, 336 struct dma_slave_config *sconfig) 337 { 338 struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc); 339 340 if (!list_empty(&tdc->pending_sg_req)) { 341 dev_err(tdc2dev(tdc), "Configuration not allowed\n"); 342 return -EBUSY; 343 } 344 345 memcpy(&tdc->dma_sconfig, sconfig, sizeof(*sconfig)); 346 tdc->config_init = true; 347 348 return 0; 349 } 350 351 static void tegra_dma_global_pause(struct tegra_dma_channel *tdc, 352 bool wait_for_burst_complete) 353 { 354 struct tegra_dma *tdma = tdc->tdma; 355 356 spin_lock(&tdma->global_lock); 357 358 if (tdc->tdma->global_pause_count == 0) { 359 tdma_write(tdma, TEGRA_APBDMA_GENERAL, 0); 360 if (wait_for_burst_complete) 361 udelay(TEGRA_APBDMA_BURST_COMPLETE_TIME); 362 } 363 364 tdc->tdma->global_pause_count++; 365 366 spin_unlock(&tdma->global_lock); 367 } 368 369 static void tegra_dma_global_resume(struct tegra_dma_channel *tdc) 370 { 371 struct tegra_dma *tdma = tdc->tdma; 372 373 spin_lock(&tdma->global_lock); 374 375 if (WARN_ON(tdc->tdma->global_pause_count == 0)) 376 goto out; 377 378 if (--tdc->tdma->global_pause_count == 0) 379 tdma_write(tdma, TEGRA_APBDMA_GENERAL, 380 TEGRA_APBDMA_GENERAL_ENABLE); 381 382 out: 383 spin_unlock(&tdma->global_lock); 384 } 385 386 static void tegra_dma_pause(struct tegra_dma_channel *tdc, 387 bool wait_for_burst_complete) 388 { 389 struct tegra_dma *tdma = tdc->tdma; 390 391 if (tdma->chip_data->support_channel_pause) { 392 tdc_write(tdc, TEGRA_APBDMA_CHAN_CSRE, 393 TEGRA_APBDMA_CHAN_CSRE_PAUSE); 394 if (wait_for_burst_complete) 395 udelay(TEGRA_APBDMA_BURST_COMPLETE_TIME); 396 } else { 397 tegra_dma_global_pause(tdc, wait_for_burst_complete); 398 } 399 } 400 401 static void tegra_dma_resume(struct tegra_dma_channel *tdc) 402 { 403 struct tegra_dma *tdma = tdc->tdma; 404 405 if (tdma->chip_data->support_channel_pause) 406 tdc_write(tdc, TEGRA_APBDMA_CHAN_CSRE, 0); 407 else 408 tegra_dma_global_resume(tdc); 409 } 410 411 static void tegra_dma_stop(struct tegra_dma_channel *tdc) 412 { 413 u32 csr, status; 414 415 /* Disable interrupts */ 416 csr = tdc_read(tdc, TEGRA_APBDMA_CHAN_CSR); 417 csr &= ~TEGRA_APBDMA_CSR_IE_EOC; 418 tdc_write(tdc, TEGRA_APBDMA_CHAN_CSR, csr); 419 420 /* Disable DMA */ 421 csr &= ~TEGRA_APBDMA_CSR_ENB; 422 tdc_write(tdc, TEGRA_APBDMA_CHAN_CSR, csr); 423 424 /* Clear interrupt status if it is there */ 425 status = tdc_read(tdc, TEGRA_APBDMA_CHAN_STATUS); 426 if (status & TEGRA_APBDMA_STATUS_ISE_EOC) { 427 dev_dbg(tdc2dev(tdc), "%s():clearing interrupt\n", __func__); 428 tdc_write(tdc, TEGRA_APBDMA_CHAN_STATUS, status); 429 } 430 tdc->busy = false; 431 } 432 433 static void tegra_dma_start(struct tegra_dma_channel *tdc, 434 struct tegra_dma_sg_req *sg_req) 435 { 436 struct tegra_dma_channel_regs *ch_regs = &sg_req->ch_regs; 437 438 tdc_write(tdc, TEGRA_APBDMA_CHAN_CSR, ch_regs->csr); 439 tdc_write(tdc, TEGRA_APBDMA_CHAN_APBSEQ, ch_regs->apb_seq); 440 tdc_write(tdc, TEGRA_APBDMA_CHAN_APBPTR, ch_regs->apb_ptr); 441 tdc_write(tdc, TEGRA_APBDMA_CHAN_AHBSEQ, ch_regs->ahb_seq); 442 tdc_write(tdc, TEGRA_APBDMA_CHAN_AHBPTR, ch_regs->ahb_ptr); 443 if (tdc->tdma->chip_data->support_separate_wcount_reg) 444 tdc_write(tdc, TEGRA_APBDMA_CHAN_WCOUNT, ch_regs->wcount); 445 446 /* Start DMA */ 447 tdc_write(tdc, TEGRA_APBDMA_CHAN_CSR, 448 ch_regs->csr | TEGRA_APBDMA_CSR_ENB); 449 } 450 451 static void tegra_dma_configure_for_next(struct tegra_dma_channel *tdc, 452 struct tegra_dma_sg_req *nsg_req) 453 { 454 unsigned long status; 455 456 /* 457 * The DMA controller reloads the new configuration for next transfer 458 * after last burst of current transfer completes. 459 * If there is no IEC status then this makes sure that last burst 460 * has not be completed. There may be case that last burst is on 461 * flight and so it can complete but because DMA is paused, it 462 * will not generates interrupt as well as not reload the new 463 * configuration. 464 * If there is already IEC status then interrupt handler need to 465 * load new configuration. 466 */ 467 tegra_dma_pause(tdc, false); 468 status = tdc_read(tdc, TEGRA_APBDMA_CHAN_STATUS); 469 470 /* 471 * If interrupt is pending then do nothing as the ISR will handle 472 * the programing for new request. 473 */ 474 if (status & TEGRA_APBDMA_STATUS_ISE_EOC) { 475 dev_err(tdc2dev(tdc), 476 "Skipping new configuration as interrupt is pending\n"); 477 tegra_dma_resume(tdc); 478 return; 479 } 480 481 /* Safe to program new configuration */ 482 tdc_write(tdc, TEGRA_APBDMA_CHAN_APBPTR, nsg_req->ch_regs.apb_ptr); 483 tdc_write(tdc, TEGRA_APBDMA_CHAN_AHBPTR, nsg_req->ch_regs.ahb_ptr); 484 if (tdc->tdma->chip_data->support_separate_wcount_reg) 485 tdc_write(tdc, TEGRA_APBDMA_CHAN_WCOUNT, 486 nsg_req->ch_regs.wcount); 487 tdc_write(tdc, TEGRA_APBDMA_CHAN_CSR, 488 nsg_req->ch_regs.csr | TEGRA_APBDMA_CSR_ENB); 489 nsg_req->configured = true; 490 nsg_req->words_xferred = 0; 491 492 tegra_dma_resume(tdc); 493 } 494 495 static void tdc_start_head_req(struct tegra_dma_channel *tdc) 496 { 497 struct tegra_dma_sg_req *sg_req; 498 499 sg_req = list_first_entry(&tdc->pending_sg_req, typeof(*sg_req), node); 500 tegra_dma_start(tdc, sg_req); 501 sg_req->configured = true; 502 sg_req->words_xferred = 0; 503 tdc->busy = true; 504 } 505 506 static void tdc_configure_next_head_desc(struct tegra_dma_channel *tdc) 507 { 508 struct tegra_dma_sg_req *hsgreq, *hnsgreq; 509 510 hsgreq = list_first_entry(&tdc->pending_sg_req, typeof(*hsgreq), node); 511 if (!list_is_last(&hsgreq->node, &tdc->pending_sg_req)) { 512 hnsgreq = list_first_entry(&hsgreq->node, typeof(*hnsgreq), 513 node); 514 tegra_dma_configure_for_next(tdc, hnsgreq); 515 } 516 } 517 518 static inline unsigned int 519 get_current_xferred_count(struct tegra_dma_channel *tdc, 520 struct tegra_dma_sg_req *sg_req, 521 unsigned long status) 522 { 523 return sg_req->req_len - (status & TEGRA_APBDMA_STATUS_COUNT_MASK) - 4; 524 } 525 526 static void tegra_dma_abort_all(struct tegra_dma_channel *tdc) 527 { 528 struct tegra_dma_desc *dma_desc; 529 struct tegra_dma_sg_req *sgreq; 530 531 while (!list_empty(&tdc->pending_sg_req)) { 532 sgreq = list_first_entry(&tdc->pending_sg_req, typeof(*sgreq), 533 node); 534 list_move_tail(&sgreq->node, &tdc->free_sg_req); 535 if (sgreq->last_sg) { 536 dma_desc = sgreq->dma_desc; 537 dma_desc->dma_status = DMA_ERROR; 538 list_add_tail(&dma_desc->node, &tdc->free_dma_desc); 539 540 /* Add in cb list if it is not there. */ 541 if (!dma_desc->cb_count) 542 list_add_tail(&dma_desc->cb_node, 543 &tdc->cb_desc); 544 dma_desc->cb_count++; 545 } 546 } 547 tdc->isr_handler = NULL; 548 } 549 550 static bool handle_continuous_head_request(struct tegra_dma_channel *tdc, 551 bool to_terminate) 552 { 553 struct tegra_dma_sg_req *hsgreq; 554 555 /* 556 * Check that head req on list should be in flight. 557 * If it is not in flight then abort transfer as 558 * looping of transfer can not continue. 559 */ 560 hsgreq = list_first_entry(&tdc->pending_sg_req, typeof(*hsgreq), node); 561 if (!hsgreq->configured) { 562 tegra_dma_stop(tdc); 563 pm_runtime_put(tdc->tdma->dev); 564 dev_err(tdc2dev(tdc), "DMA transfer underflow, aborting DMA\n"); 565 tegra_dma_abort_all(tdc); 566 return false; 567 } 568 569 /* Configure next request */ 570 if (!to_terminate) 571 tdc_configure_next_head_desc(tdc); 572 573 return true; 574 } 575 576 static void handle_once_dma_done(struct tegra_dma_channel *tdc, 577 bool to_terminate) 578 { 579 struct tegra_dma_desc *dma_desc; 580 struct tegra_dma_sg_req *sgreq; 581 582 tdc->busy = false; 583 sgreq = list_first_entry(&tdc->pending_sg_req, typeof(*sgreq), node); 584 dma_desc = sgreq->dma_desc; 585 dma_desc->bytes_transferred += sgreq->req_len; 586 587 list_del(&sgreq->node); 588 if (sgreq->last_sg) { 589 dma_desc->dma_status = DMA_COMPLETE; 590 dma_cookie_complete(&dma_desc->txd); 591 if (!dma_desc->cb_count) 592 list_add_tail(&dma_desc->cb_node, &tdc->cb_desc); 593 dma_desc->cb_count++; 594 list_add_tail(&dma_desc->node, &tdc->free_dma_desc); 595 } 596 list_add_tail(&sgreq->node, &tdc->free_sg_req); 597 598 /* Do not start DMA if it is going to be terminate */ 599 if (to_terminate) 600 return; 601 602 if (list_empty(&tdc->pending_sg_req)) { 603 pm_runtime_put(tdc->tdma->dev); 604 return; 605 } 606 607 tdc_start_head_req(tdc); 608 } 609 610 static void handle_cont_sngl_cycle_dma_done(struct tegra_dma_channel *tdc, 611 bool to_terminate) 612 { 613 struct tegra_dma_desc *dma_desc; 614 struct tegra_dma_sg_req *sgreq; 615 bool st; 616 617 sgreq = list_first_entry(&tdc->pending_sg_req, typeof(*sgreq), node); 618 dma_desc = sgreq->dma_desc; 619 /* if we dma for long enough the transfer count will wrap */ 620 dma_desc->bytes_transferred = 621 (dma_desc->bytes_transferred + sgreq->req_len) % 622 dma_desc->bytes_requested; 623 624 /* Callback need to be call */ 625 if (!dma_desc->cb_count) 626 list_add_tail(&dma_desc->cb_node, &tdc->cb_desc); 627 dma_desc->cb_count++; 628 629 sgreq->words_xferred = 0; 630 631 /* If not last req then put at end of pending list */ 632 if (!list_is_last(&sgreq->node, &tdc->pending_sg_req)) { 633 list_move_tail(&sgreq->node, &tdc->pending_sg_req); 634 sgreq->configured = false; 635 st = handle_continuous_head_request(tdc, to_terminate); 636 if (!st) 637 dma_desc->dma_status = DMA_ERROR; 638 } 639 } 640 641 static void tegra_dma_tasklet(struct tasklet_struct *t) 642 { 643 struct tegra_dma_channel *tdc = from_tasklet(tdc, t, tasklet); 644 struct dmaengine_desc_callback cb; 645 struct tegra_dma_desc *dma_desc; 646 unsigned int cb_count; 647 unsigned long flags; 648 649 spin_lock_irqsave(&tdc->lock, flags); 650 while (!list_empty(&tdc->cb_desc)) { 651 dma_desc = list_first_entry(&tdc->cb_desc, typeof(*dma_desc), 652 cb_node); 653 list_del(&dma_desc->cb_node); 654 dmaengine_desc_get_callback(&dma_desc->txd, &cb); 655 cb_count = dma_desc->cb_count; 656 dma_desc->cb_count = 0; 657 trace_tegra_dma_complete_cb(&tdc->dma_chan, cb_count, 658 cb.callback); 659 spin_unlock_irqrestore(&tdc->lock, flags); 660 while (cb_count--) 661 dmaengine_desc_callback_invoke(&cb, NULL); 662 spin_lock_irqsave(&tdc->lock, flags); 663 } 664 spin_unlock_irqrestore(&tdc->lock, flags); 665 } 666 667 static irqreturn_t tegra_dma_isr(int irq, void *dev_id) 668 { 669 struct tegra_dma_channel *tdc = dev_id; 670 u32 status; 671 672 spin_lock(&tdc->lock); 673 674 trace_tegra_dma_isr(&tdc->dma_chan, irq); 675 status = tdc_read(tdc, TEGRA_APBDMA_CHAN_STATUS); 676 if (status & TEGRA_APBDMA_STATUS_ISE_EOC) { 677 tdc_write(tdc, TEGRA_APBDMA_CHAN_STATUS, status); 678 tdc->isr_handler(tdc, false); 679 tasklet_schedule(&tdc->tasklet); 680 wake_up_all(&tdc->wq); 681 spin_unlock(&tdc->lock); 682 return IRQ_HANDLED; 683 } 684 685 spin_unlock(&tdc->lock); 686 dev_info(tdc2dev(tdc), "Interrupt already served status 0x%08x\n", 687 status); 688 689 return IRQ_NONE; 690 } 691 692 static dma_cookie_t tegra_dma_tx_submit(struct dma_async_tx_descriptor *txd) 693 { 694 struct tegra_dma_desc *dma_desc = txd_to_tegra_dma_desc(txd); 695 struct tegra_dma_channel *tdc = to_tegra_dma_chan(txd->chan); 696 unsigned long flags; 697 dma_cookie_t cookie; 698 699 spin_lock_irqsave(&tdc->lock, flags); 700 dma_desc->dma_status = DMA_IN_PROGRESS; 701 cookie = dma_cookie_assign(&dma_desc->txd); 702 list_splice_tail_init(&dma_desc->tx_list, &tdc->pending_sg_req); 703 spin_unlock_irqrestore(&tdc->lock, flags); 704 705 return cookie; 706 } 707 708 static void tegra_dma_issue_pending(struct dma_chan *dc) 709 { 710 struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc); 711 unsigned long flags; 712 int err; 713 714 spin_lock_irqsave(&tdc->lock, flags); 715 if (list_empty(&tdc->pending_sg_req)) { 716 dev_err(tdc2dev(tdc), "No DMA request\n"); 717 goto end; 718 } 719 if (!tdc->busy) { 720 err = pm_runtime_resume_and_get(tdc->tdma->dev); 721 if (err < 0) { 722 dev_err(tdc2dev(tdc), "Failed to enable DMA\n"); 723 goto end; 724 } 725 726 tdc_start_head_req(tdc); 727 728 /* Continuous single mode: Configure next req */ 729 if (tdc->cyclic) { 730 /* 731 * Wait for 1 burst time for configure DMA for 732 * next transfer. 733 */ 734 udelay(TEGRA_APBDMA_BURST_COMPLETE_TIME); 735 tdc_configure_next_head_desc(tdc); 736 } 737 } 738 end: 739 spin_unlock_irqrestore(&tdc->lock, flags); 740 } 741 742 static int tegra_dma_terminate_all(struct dma_chan *dc) 743 { 744 struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc); 745 struct tegra_dma_desc *dma_desc; 746 struct tegra_dma_sg_req *sgreq; 747 unsigned long flags; 748 u32 status, wcount; 749 bool was_busy; 750 751 spin_lock_irqsave(&tdc->lock, flags); 752 753 if (!tdc->busy) 754 goto skip_dma_stop; 755 756 /* Pause DMA before checking the queue status */ 757 tegra_dma_pause(tdc, true); 758 759 status = tdc_read(tdc, TEGRA_APBDMA_CHAN_STATUS); 760 if (status & TEGRA_APBDMA_STATUS_ISE_EOC) { 761 dev_dbg(tdc2dev(tdc), "%s():handling isr\n", __func__); 762 tdc->isr_handler(tdc, true); 763 status = tdc_read(tdc, TEGRA_APBDMA_CHAN_STATUS); 764 } 765 if (tdc->tdma->chip_data->support_separate_wcount_reg) 766 wcount = tdc_read(tdc, TEGRA_APBDMA_CHAN_WORD_TRANSFER); 767 else 768 wcount = status; 769 770 was_busy = tdc->busy; 771 tegra_dma_stop(tdc); 772 773 if (!list_empty(&tdc->pending_sg_req) && was_busy) { 774 sgreq = list_first_entry(&tdc->pending_sg_req, typeof(*sgreq), 775 node); 776 sgreq->dma_desc->bytes_transferred += 777 get_current_xferred_count(tdc, sgreq, wcount); 778 } 779 tegra_dma_resume(tdc); 780 781 pm_runtime_put(tdc->tdma->dev); 782 wake_up_all(&tdc->wq); 783 784 skip_dma_stop: 785 tegra_dma_abort_all(tdc); 786 787 while (!list_empty(&tdc->cb_desc)) { 788 dma_desc = list_first_entry(&tdc->cb_desc, typeof(*dma_desc), 789 cb_node); 790 list_del(&dma_desc->cb_node); 791 dma_desc->cb_count = 0; 792 } 793 spin_unlock_irqrestore(&tdc->lock, flags); 794 795 return 0; 796 } 797 798 static bool tegra_dma_eoc_interrupt_deasserted(struct tegra_dma_channel *tdc) 799 { 800 unsigned long flags; 801 u32 status; 802 803 spin_lock_irqsave(&tdc->lock, flags); 804 status = tdc_read(tdc, TEGRA_APBDMA_CHAN_STATUS); 805 spin_unlock_irqrestore(&tdc->lock, flags); 806 807 return !(status & TEGRA_APBDMA_STATUS_ISE_EOC); 808 } 809 810 static void tegra_dma_synchronize(struct dma_chan *dc) 811 { 812 struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc); 813 int err; 814 815 err = pm_runtime_resume_and_get(tdc->tdma->dev); 816 if (err < 0) { 817 dev_err(tdc2dev(tdc), "Failed to synchronize DMA: %d\n", err); 818 return; 819 } 820 821 /* 822 * CPU, which handles interrupt, could be busy in 823 * uninterruptible state, in this case sibling CPU 824 * should wait until interrupt is handled. 825 */ 826 wait_event(tdc->wq, tegra_dma_eoc_interrupt_deasserted(tdc)); 827 828 tasklet_kill(&tdc->tasklet); 829 830 pm_runtime_put(tdc->tdma->dev); 831 } 832 833 static unsigned int tegra_dma_sg_bytes_xferred(struct tegra_dma_channel *tdc, 834 struct tegra_dma_sg_req *sg_req) 835 { 836 u32 status, wcount = 0; 837 838 if (!list_is_first(&sg_req->node, &tdc->pending_sg_req)) 839 return 0; 840 841 if (tdc->tdma->chip_data->support_separate_wcount_reg) 842 wcount = tdc_read(tdc, TEGRA_APBDMA_CHAN_WORD_TRANSFER); 843 844 status = tdc_read(tdc, TEGRA_APBDMA_CHAN_STATUS); 845 846 if (!tdc->tdma->chip_data->support_separate_wcount_reg) 847 wcount = status; 848 849 if (status & TEGRA_APBDMA_STATUS_ISE_EOC) 850 return sg_req->req_len; 851 852 wcount = get_current_xferred_count(tdc, sg_req, wcount); 853 854 if (!wcount) { 855 /* 856 * If wcount wasn't ever polled for this SG before, then 857 * simply assume that transfer hasn't started yet. 858 * 859 * Otherwise it's the end of the transfer. 860 * 861 * The alternative would be to poll the status register 862 * until EOC bit is set or wcount goes UP. That's so 863 * because EOC bit is getting set only after the last 864 * burst's completion and counter is less than the actual 865 * transfer size by 4 bytes. The counter value wraps around 866 * in a cyclic mode before EOC is set(!), so we can't easily 867 * distinguish start of transfer from its end. 868 */ 869 if (sg_req->words_xferred) 870 wcount = sg_req->req_len - 4; 871 872 } else if (wcount < sg_req->words_xferred) { 873 /* 874 * This case will never happen for a non-cyclic transfer. 875 * 876 * For a cyclic transfer, although it is possible for the 877 * next transfer to have already started (resetting the word 878 * count), this case should still not happen because we should 879 * have detected that the EOC bit is set and hence the transfer 880 * was completed. 881 */ 882 WARN_ON_ONCE(1); 883 884 wcount = sg_req->req_len - 4; 885 } else { 886 sg_req->words_xferred = wcount; 887 } 888 889 return wcount; 890 } 891 892 static enum dma_status tegra_dma_tx_status(struct dma_chan *dc, 893 dma_cookie_t cookie, 894 struct dma_tx_state *txstate) 895 { 896 struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc); 897 struct tegra_dma_desc *dma_desc; 898 struct tegra_dma_sg_req *sg_req; 899 enum dma_status ret; 900 unsigned long flags; 901 unsigned int residual; 902 unsigned int bytes = 0; 903 904 ret = dma_cookie_status(dc, cookie, txstate); 905 if (ret == DMA_COMPLETE) 906 return ret; 907 908 spin_lock_irqsave(&tdc->lock, flags); 909 910 /* Check on wait_ack desc status */ 911 list_for_each_entry(dma_desc, &tdc->free_dma_desc, node) { 912 if (dma_desc->txd.cookie == cookie) { 913 ret = dma_desc->dma_status; 914 goto found; 915 } 916 } 917 918 /* Check in pending list */ 919 list_for_each_entry(sg_req, &tdc->pending_sg_req, node) { 920 dma_desc = sg_req->dma_desc; 921 if (dma_desc->txd.cookie == cookie) { 922 bytes = tegra_dma_sg_bytes_xferred(tdc, sg_req); 923 ret = dma_desc->dma_status; 924 goto found; 925 } 926 } 927 928 dev_dbg(tdc2dev(tdc), "cookie %d not found\n", cookie); 929 dma_desc = NULL; 930 931 found: 932 if (dma_desc && txstate) { 933 residual = dma_desc->bytes_requested - 934 ((dma_desc->bytes_transferred + bytes) % 935 dma_desc->bytes_requested); 936 dma_set_residue(txstate, residual); 937 } 938 939 trace_tegra_dma_tx_status(&tdc->dma_chan, cookie, txstate); 940 spin_unlock_irqrestore(&tdc->lock, flags); 941 942 return ret; 943 } 944 945 static inline unsigned int get_bus_width(struct tegra_dma_channel *tdc, 946 enum dma_slave_buswidth slave_bw) 947 { 948 switch (slave_bw) { 949 case DMA_SLAVE_BUSWIDTH_1_BYTE: 950 return TEGRA_APBDMA_APBSEQ_BUS_WIDTH_8; 951 case DMA_SLAVE_BUSWIDTH_2_BYTES: 952 return TEGRA_APBDMA_APBSEQ_BUS_WIDTH_16; 953 case DMA_SLAVE_BUSWIDTH_4_BYTES: 954 return TEGRA_APBDMA_APBSEQ_BUS_WIDTH_32; 955 case DMA_SLAVE_BUSWIDTH_8_BYTES: 956 return TEGRA_APBDMA_APBSEQ_BUS_WIDTH_64; 957 default: 958 dev_warn(tdc2dev(tdc), 959 "slave bw is not supported, using 32bits\n"); 960 return TEGRA_APBDMA_APBSEQ_BUS_WIDTH_32; 961 } 962 } 963 964 static inline unsigned int get_burst_size(struct tegra_dma_channel *tdc, 965 u32 burst_size, 966 enum dma_slave_buswidth slave_bw, 967 u32 len) 968 { 969 unsigned int burst_byte, burst_ahb_width; 970 971 /* 972 * burst_size from client is in terms of the bus_width. 973 * convert them into AHB memory width which is 4 byte. 974 */ 975 burst_byte = burst_size * slave_bw; 976 burst_ahb_width = burst_byte / 4; 977 978 /* If burst size is 0 then calculate the burst size based on length */ 979 if (!burst_ahb_width) { 980 if (len & 0xF) 981 return TEGRA_APBDMA_AHBSEQ_BURST_1; 982 else if ((len >> 4) & 0x1) 983 return TEGRA_APBDMA_AHBSEQ_BURST_4; 984 else 985 return TEGRA_APBDMA_AHBSEQ_BURST_8; 986 } 987 if (burst_ahb_width < 4) 988 return TEGRA_APBDMA_AHBSEQ_BURST_1; 989 else if (burst_ahb_width < 8) 990 return TEGRA_APBDMA_AHBSEQ_BURST_4; 991 else 992 return TEGRA_APBDMA_AHBSEQ_BURST_8; 993 } 994 995 static int get_transfer_param(struct tegra_dma_channel *tdc, 996 enum dma_transfer_direction direction, 997 u32 *apb_addr, 998 u32 *apb_seq, 999 u32 *csr, 1000 unsigned int *burst_size, 1001 enum dma_slave_buswidth *slave_bw) 1002 { 1003 switch (direction) { 1004 case DMA_MEM_TO_DEV: 1005 *apb_addr = tdc->dma_sconfig.dst_addr; 1006 *apb_seq = get_bus_width(tdc, tdc->dma_sconfig.dst_addr_width); 1007 *burst_size = tdc->dma_sconfig.dst_maxburst; 1008 *slave_bw = tdc->dma_sconfig.dst_addr_width; 1009 *csr = TEGRA_APBDMA_CSR_DIR; 1010 return 0; 1011 1012 case DMA_DEV_TO_MEM: 1013 *apb_addr = tdc->dma_sconfig.src_addr; 1014 *apb_seq = get_bus_width(tdc, tdc->dma_sconfig.src_addr_width); 1015 *burst_size = tdc->dma_sconfig.src_maxburst; 1016 *slave_bw = tdc->dma_sconfig.src_addr_width; 1017 *csr = 0; 1018 return 0; 1019 1020 default: 1021 dev_err(tdc2dev(tdc), "DMA direction is not supported\n"); 1022 break; 1023 } 1024 1025 return -EINVAL; 1026 } 1027 1028 static void tegra_dma_prep_wcount(struct tegra_dma_channel *tdc, 1029 struct tegra_dma_channel_regs *ch_regs, 1030 u32 len) 1031 { 1032 u32 len_field = (len - 4) & 0xFFFC; 1033 1034 if (tdc->tdma->chip_data->support_separate_wcount_reg) 1035 ch_regs->wcount = len_field; 1036 else 1037 ch_regs->csr |= len_field; 1038 } 1039 1040 static struct dma_async_tx_descriptor * 1041 tegra_dma_prep_slave_sg(struct dma_chan *dc, 1042 struct scatterlist *sgl, 1043 unsigned int sg_len, 1044 enum dma_transfer_direction direction, 1045 unsigned long flags, 1046 void *context) 1047 { 1048 struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc); 1049 struct tegra_dma_sg_req *sg_req = NULL; 1050 u32 csr, ahb_seq, apb_ptr, apb_seq; 1051 enum dma_slave_buswidth slave_bw; 1052 struct tegra_dma_desc *dma_desc; 1053 struct list_head req_list; 1054 struct scatterlist *sg; 1055 unsigned int burst_size; 1056 unsigned int i; 1057 1058 if (!tdc->config_init) { 1059 dev_err(tdc2dev(tdc), "DMA channel is not configured\n"); 1060 return NULL; 1061 } 1062 if (sg_len < 1) { 1063 dev_err(tdc2dev(tdc), "Invalid segment length %d\n", sg_len); 1064 return NULL; 1065 } 1066 1067 if (get_transfer_param(tdc, direction, &apb_ptr, &apb_seq, &csr, 1068 &burst_size, &slave_bw) < 0) 1069 return NULL; 1070 1071 INIT_LIST_HEAD(&req_list); 1072 1073 ahb_seq = TEGRA_APBDMA_AHBSEQ_INTR_ENB; 1074 ahb_seq |= TEGRA_APBDMA_AHBSEQ_WRAP_NONE << 1075 TEGRA_APBDMA_AHBSEQ_WRAP_SHIFT; 1076 ahb_seq |= TEGRA_APBDMA_AHBSEQ_BUS_WIDTH_32; 1077 1078 csr |= TEGRA_APBDMA_CSR_ONCE; 1079 1080 if (tdc->slave_id != TEGRA_APBDMA_SLAVE_ID_INVALID) { 1081 csr |= TEGRA_APBDMA_CSR_FLOW; 1082 csr |= tdc->slave_id << TEGRA_APBDMA_CSR_REQ_SEL_SHIFT; 1083 } 1084 1085 if (flags & DMA_PREP_INTERRUPT) { 1086 csr |= TEGRA_APBDMA_CSR_IE_EOC; 1087 } else { 1088 WARN_ON_ONCE(1); 1089 return NULL; 1090 } 1091 1092 apb_seq |= TEGRA_APBDMA_APBSEQ_WRAP_WORD_1; 1093 1094 dma_desc = tegra_dma_desc_get(tdc); 1095 if (!dma_desc) { 1096 dev_err(tdc2dev(tdc), "DMA descriptors not available\n"); 1097 return NULL; 1098 } 1099 INIT_LIST_HEAD(&dma_desc->tx_list); 1100 INIT_LIST_HEAD(&dma_desc->cb_node); 1101 dma_desc->cb_count = 0; 1102 dma_desc->bytes_requested = 0; 1103 dma_desc->bytes_transferred = 0; 1104 dma_desc->dma_status = DMA_IN_PROGRESS; 1105 1106 /* Make transfer requests */ 1107 for_each_sg(sgl, sg, sg_len, i) { 1108 u32 len, mem; 1109 1110 mem = sg_dma_address(sg); 1111 len = sg_dma_len(sg); 1112 1113 if ((len & 3) || (mem & 3) || 1114 len > tdc->tdma->chip_data->max_dma_count) { 1115 dev_err(tdc2dev(tdc), 1116 "DMA length/memory address is not supported\n"); 1117 tegra_dma_desc_put(tdc, dma_desc); 1118 return NULL; 1119 } 1120 1121 sg_req = tegra_dma_sg_req_get(tdc); 1122 if (!sg_req) { 1123 dev_err(tdc2dev(tdc), "DMA sg-req not available\n"); 1124 tegra_dma_desc_put(tdc, dma_desc); 1125 return NULL; 1126 } 1127 1128 ahb_seq |= get_burst_size(tdc, burst_size, slave_bw, len); 1129 dma_desc->bytes_requested += len; 1130 1131 sg_req->ch_regs.apb_ptr = apb_ptr; 1132 sg_req->ch_regs.ahb_ptr = mem; 1133 sg_req->ch_regs.csr = csr; 1134 tegra_dma_prep_wcount(tdc, &sg_req->ch_regs, len); 1135 sg_req->ch_regs.apb_seq = apb_seq; 1136 sg_req->ch_regs.ahb_seq = ahb_seq; 1137 sg_req->configured = false; 1138 sg_req->last_sg = false; 1139 sg_req->dma_desc = dma_desc; 1140 sg_req->req_len = len; 1141 1142 list_add_tail(&sg_req->node, &dma_desc->tx_list); 1143 } 1144 sg_req->last_sg = true; 1145 if (flags & DMA_CTRL_ACK) 1146 dma_desc->txd.flags = DMA_CTRL_ACK; 1147 1148 /* 1149 * Make sure that mode should not be conflicting with currently 1150 * configured mode. 1151 */ 1152 if (!tdc->isr_handler) { 1153 tdc->isr_handler = handle_once_dma_done; 1154 tdc->cyclic = false; 1155 } else { 1156 if (tdc->cyclic) { 1157 dev_err(tdc2dev(tdc), "DMA configured in cyclic mode\n"); 1158 tegra_dma_desc_put(tdc, dma_desc); 1159 return NULL; 1160 } 1161 } 1162 1163 return &dma_desc->txd; 1164 } 1165 1166 static struct dma_async_tx_descriptor * 1167 tegra_dma_prep_dma_cyclic(struct dma_chan *dc, dma_addr_t buf_addr, 1168 size_t buf_len, 1169 size_t period_len, 1170 enum dma_transfer_direction direction, 1171 unsigned long flags) 1172 { 1173 struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc); 1174 struct tegra_dma_sg_req *sg_req = NULL; 1175 u32 csr, ahb_seq, apb_ptr, apb_seq; 1176 enum dma_slave_buswidth slave_bw; 1177 struct tegra_dma_desc *dma_desc; 1178 dma_addr_t mem = buf_addr; 1179 unsigned int burst_size; 1180 size_t len, remain_len; 1181 1182 if (!buf_len || !period_len) { 1183 dev_err(tdc2dev(tdc), "Invalid buffer/period len\n"); 1184 return NULL; 1185 } 1186 1187 if (!tdc->config_init) { 1188 dev_err(tdc2dev(tdc), "DMA slave is not configured\n"); 1189 return NULL; 1190 } 1191 1192 /* 1193 * We allow to take more number of requests till DMA is 1194 * not started. The driver will loop over all requests. 1195 * Once DMA is started then new requests can be queued only after 1196 * terminating the DMA. 1197 */ 1198 if (tdc->busy) { 1199 dev_err(tdc2dev(tdc), "Request not allowed when DMA running\n"); 1200 return NULL; 1201 } 1202 1203 /* 1204 * We only support cycle transfer when buf_len is multiple of 1205 * period_len. 1206 */ 1207 if (buf_len % period_len) { 1208 dev_err(tdc2dev(tdc), "buf_len is not multiple of period_len\n"); 1209 return NULL; 1210 } 1211 1212 len = period_len; 1213 if ((len & 3) || (buf_addr & 3) || 1214 len > tdc->tdma->chip_data->max_dma_count) { 1215 dev_err(tdc2dev(tdc), "Req len/mem address is not correct\n"); 1216 return NULL; 1217 } 1218 1219 if (get_transfer_param(tdc, direction, &apb_ptr, &apb_seq, &csr, 1220 &burst_size, &slave_bw) < 0) 1221 return NULL; 1222 1223 ahb_seq = TEGRA_APBDMA_AHBSEQ_INTR_ENB; 1224 ahb_seq |= TEGRA_APBDMA_AHBSEQ_WRAP_NONE << 1225 TEGRA_APBDMA_AHBSEQ_WRAP_SHIFT; 1226 ahb_seq |= TEGRA_APBDMA_AHBSEQ_BUS_WIDTH_32; 1227 1228 if (tdc->slave_id != TEGRA_APBDMA_SLAVE_ID_INVALID) { 1229 csr |= TEGRA_APBDMA_CSR_FLOW; 1230 csr |= tdc->slave_id << TEGRA_APBDMA_CSR_REQ_SEL_SHIFT; 1231 } 1232 1233 if (flags & DMA_PREP_INTERRUPT) { 1234 csr |= TEGRA_APBDMA_CSR_IE_EOC; 1235 } else { 1236 WARN_ON_ONCE(1); 1237 return NULL; 1238 } 1239 1240 apb_seq |= TEGRA_APBDMA_APBSEQ_WRAP_WORD_1; 1241 1242 dma_desc = tegra_dma_desc_get(tdc); 1243 if (!dma_desc) { 1244 dev_err(tdc2dev(tdc), "not enough descriptors available\n"); 1245 return NULL; 1246 } 1247 1248 INIT_LIST_HEAD(&dma_desc->tx_list); 1249 INIT_LIST_HEAD(&dma_desc->cb_node); 1250 dma_desc->cb_count = 0; 1251 1252 dma_desc->bytes_transferred = 0; 1253 dma_desc->bytes_requested = buf_len; 1254 remain_len = buf_len; 1255 1256 /* Split transfer equal to period size */ 1257 while (remain_len) { 1258 sg_req = tegra_dma_sg_req_get(tdc); 1259 if (!sg_req) { 1260 dev_err(tdc2dev(tdc), "DMA sg-req not available\n"); 1261 tegra_dma_desc_put(tdc, dma_desc); 1262 return NULL; 1263 } 1264 1265 ahb_seq |= get_burst_size(tdc, burst_size, slave_bw, len); 1266 sg_req->ch_regs.apb_ptr = apb_ptr; 1267 sg_req->ch_regs.ahb_ptr = mem; 1268 sg_req->ch_regs.csr = csr; 1269 tegra_dma_prep_wcount(tdc, &sg_req->ch_regs, len); 1270 sg_req->ch_regs.apb_seq = apb_seq; 1271 sg_req->ch_regs.ahb_seq = ahb_seq; 1272 sg_req->configured = false; 1273 sg_req->last_sg = false; 1274 sg_req->dma_desc = dma_desc; 1275 sg_req->req_len = len; 1276 1277 list_add_tail(&sg_req->node, &dma_desc->tx_list); 1278 remain_len -= len; 1279 mem += len; 1280 } 1281 sg_req->last_sg = true; 1282 if (flags & DMA_CTRL_ACK) 1283 dma_desc->txd.flags = DMA_CTRL_ACK; 1284 1285 /* 1286 * Make sure that mode should not be conflicting with currently 1287 * configured mode. 1288 */ 1289 if (!tdc->isr_handler) { 1290 tdc->isr_handler = handle_cont_sngl_cycle_dma_done; 1291 tdc->cyclic = true; 1292 } else { 1293 if (!tdc->cyclic) { 1294 dev_err(tdc2dev(tdc), "DMA configuration conflict\n"); 1295 tegra_dma_desc_put(tdc, dma_desc); 1296 return NULL; 1297 } 1298 } 1299 1300 return &dma_desc->txd; 1301 } 1302 1303 static int tegra_dma_alloc_chan_resources(struct dma_chan *dc) 1304 { 1305 struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc); 1306 1307 dma_cookie_init(&tdc->dma_chan); 1308 1309 return 0; 1310 } 1311 1312 static void tegra_dma_free_chan_resources(struct dma_chan *dc) 1313 { 1314 struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc); 1315 struct tegra_dma_desc *dma_desc; 1316 struct tegra_dma_sg_req *sg_req; 1317 struct list_head dma_desc_list; 1318 struct list_head sg_req_list; 1319 1320 INIT_LIST_HEAD(&dma_desc_list); 1321 INIT_LIST_HEAD(&sg_req_list); 1322 1323 dev_dbg(tdc2dev(tdc), "Freeing channel %d\n", tdc->id); 1324 1325 tegra_dma_terminate_all(dc); 1326 tasklet_kill(&tdc->tasklet); 1327 1328 list_splice_init(&tdc->pending_sg_req, &sg_req_list); 1329 list_splice_init(&tdc->free_sg_req, &sg_req_list); 1330 list_splice_init(&tdc->free_dma_desc, &dma_desc_list); 1331 INIT_LIST_HEAD(&tdc->cb_desc); 1332 tdc->config_init = false; 1333 tdc->isr_handler = NULL; 1334 1335 while (!list_empty(&dma_desc_list)) { 1336 dma_desc = list_first_entry(&dma_desc_list, typeof(*dma_desc), 1337 node); 1338 list_del(&dma_desc->node); 1339 kfree(dma_desc); 1340 } 1341 1342 while (!list_empty(&sg_req_list)) { 1343 sg_req = list_first_entry(&sg_req_list, typeof(*sg_req), node); 1344 list_del(&sg_req->node); 1345 kfree(sg_req); 1346 } 1347 1348 tdc->slave_id = TEGRA_APBDMA_SLAVE_ID_INVALID; 1349 } 1350 1351 static struct dma_chan *tegra_dma_of_xlate(struct of_phandle_args *dma_spec, 1352 struct of_dma *ofdma) 1353 { 1354 struct tegra_dma *tdma = ofdma->of_dma_data; 1355 struct tegra_dma_channel *tdc; 1356 struct dma_chan *chan; 1357 1358 if (dma_spec->args[0] > TEGRA_APBDMA_CSR_REQ_SEL_MASK) { 1359 dev_err(tdma->dev, "Invalid slave id: %d\n", dma_spec->args[0]); 1360 return NULL; 1361 } 1362 1363 chan = dma_get_any_slave_channel(&tdma->dma_dev); 1364 if (!chan) 1365 return NULL; 1366 1367 tdc = to_tegra_dma_chan(chan); 1368 tdc->slave_id = dma_spec->args[0]; 1369 1370 return chan; 1371 } 1372 1373 /* Tegra20 specific DMA controller information */ 1374 static const struct tegra_dma_chip_data tegra20_dma_chip_data = { 1375 .nr_channels = 16, 1376 .channel_reg_size = 0x20, 1377 .max_dma_count = 1024UL * 64, 1378 .support_channel_pause = false, 1379 .support_separate_wcount_reg = false, 1380 }; 1381 1382 /* Tegra30 specific DMA controller information */ 1383 static const struct tegra_dma_chip_data tegra30_dma_chip_data = { 1384 .nr_channels = 32, 1385 .channel_reg_size = 0x20, 1386 .max_dma_count = 1024UL * 64, 1387 .support_channel_pause = false, 1388 .support_separate_wcount_reg = false, 1389 }; 1390 1391 /* Tegra114 specific DMA controller information */ 1392 static const struct tegra_dma_chip_data tegra114_dma_chip_data = { 1393 .nr_channels = 32, 1394 .channel_reg_size = 0x20, 1395 .max_dma_count = 1024UL * 64, 1396 .support_channel_pause = true, 1397 .support_separate_wcount_reg = false, 1398 }; 1399 1400 /* Tegra148 specific DMA controller information */ 1401 static const struct tegra_dma_chip_data tegra148_dma_chip_data = { 1402 .nr_channels = 32, 1403 .channel_reg_size = 0x40, 1404 .max_dma_count = 1024UL * 64, 1405 .support_channel_pause = true, 1406 .support_separate_wcount_reg = true, 1407 }; 1408 1409 static int tegra_dma_init_hw(struct tegra_dma *tdma) 1410 { 1411 int err; 1412 1413 err = reset_control_assert(tdma->rst); 1414 if (err) { 1415 dev_err(tdma->dev, "failed to assert reset: %d\n", err); 1416 return err; 1417 } 1418 1419 err = clk_enable(tdma->dma_clk); 1420 if (err) { 1421 dev_err(tdma->dev, "failed to enable clk: %d\n", err); 1422 return err; 1423 } 1424 1425 /* reset DMA controller */ 1426 udelay(2); 1427 reset_control_deassert(tdma->rst); 1428 1429 /* enable global DMA registers */ 1430 tdma_write(tdma, TEGRA_APBDMA_GENERAL, TEGRA_APBDMA_GENERAL_ENABLE); 1431 tdma_write(tdma, TEGRA_APBDMA_CONTROL, 0); 1432 tdma_write(tdma, TEGRA_APBDMA_IRQ_MASK_SET, 0xFFFFFFFF); 1433 1434 clk_disable(tdma->dma_clk); 1435 1436 return 0; 1437 } 1438 1439 static int tegra_dma_probe(struct platform_device *pdev) 1440 { 1441 const struct tegra_dma_chip_data *cdata; 1442 struct tegra_dma *tdma; 1443 unsigned int i; 1444 size_t size; 1445 int ret; 1446 1447 cdata = of_device_get_match_data(&pdev->dev); 1448 size = struct_size(tdma, channels, cdata->nr_channels); 1449 1450 tdma = devm_kzalloc(&pdev->dev, size, GFP_KERNEL); 1451 if (!tdma) 1452 return -ENOMEM; 1453 1454 tdma->dev = &pdev->dev; 1455 tdma->chip_data = cdata; 1456 platform_set_drvdata(pdev, tdma); 1457 1458 tdma->base_addr = devm_platform_ioremap_resource(pdev, 0); 1459 if (IS_ERR(tdma->base_addr)) 1460 return PTR_ERR(tdma->base_addr); 1461 1462 tdma->dma_clk = devm_clk_get(&pdev->dev, NULL); 1463 if (IS_ERR(tdma->dma_clk)) { 1464 dev_err(&pdev->dev, "Error: Missing controller clock\n"); 1465 return PTR_ERR(tdma->dma_clk); 1466 } 1467 1468 tdma->rst = devm_reset_control_get(&pdev->dev, "dma"); 1469 if (IS_ERR(tdma->rst)) { 1470 dev_err(&pdev->dev, "Error: Missing reset\n"); 1471 return PTR_ERR(tdma->rst); 1472 } 1473 1474 spin_lock_init(&tdma->global_lock); 1475 1476 ret = clk_prepare(tdma->dma_clk); 1477 if (ret) 1478 return ret; 1479 1480 ret = tegra_dma_init_hw(tdma); 1481 if (ret) 1482 goto err_clk_unprepare; 1483 1484 pm_runtime_irq_safe(&pdev->dev); 1485 pm_runtime_enable(&pdev->dev); 1486 1487 INIT_LIST_HEAD(&tdma->dma_dev.channels); 1488 for (i = 0; i < cdata->nr_channels; i++) { 1489 struct tegra_dma_channel *tdc = &tdma->channels[i]; 1490 int irq; 1491 1492 tdc->chan_addr = tdma->base_addr + 1493 TEGRA_APBDMA_CHANNEL_BASE_ADD_OFFSET + 1494 (i * cdata->channel_reg_size); 1495 1496 irq = platform_get_irq(pdev, i); 1497 if (irq < 0) { 1498 ret = irq; 1499 goto err_pm_disable; 1500 } 1501 1502 snprintf(tdc->name, sizeof(tdc->name), "apbdma.%d", i); 1503 ret = devm_request_irq(&pdev->dev, irq, tegra_dma_isr, 0, 1504 tdc->name, tdc); 1505 if (ret) { 1506 dev_err(&pdev->dev, 1507 "request_irq failed with err %d channel %d\n", 1508 ret, i); 1509 goto err_pm_disable; 1510 } 1511 1512 tdc->dma_chan.device = &tdma->dma_dev; 1513 dma_cookie_init(&tdc->dma_chan); 1514 list_add_tail(&tdc->dma_chan.device_node, 1515 &tdma->dma_dev.channels); 1516 tdc->tdma = tdma; 1517 tdc->id = i; 1518 tdc->slave_id = TEGRA_APBDMA_SLAVE_ID_INVALID; 1519 1520 tasklet_setup(&tdc->tasklet, tegra_dma_tasklet); 1521 spin_lock_init(&tdc->lock); 1522 init_waitqueue_head(&tdc->wq); 1523 1524 INIT_LIST_HEAD(&tdc->pending_sg_req); 1525 INIT_LIST_HEAD(&tdc->free_sg_req); 1526 INIT_LIST_HEAD(&tdc->free_dma_desc); 1527 INIT_LIST_HEAD(&tdc->cb_desc); 1528 } 1529 1530 dma_cap_set(DMA_SLAVE, tdma->dma_dev.cap_mask); 1531 dma_cap_set(DMA_PRIVATE, tdma->dma_dev.cap_mask); 1532 dma_cap_set(DMA_CYCLIC, tdma->dma_dev.cap_mask); 1533 1534 tdma->global_pause_count = 0; 1535 tdma->dma_dev.dev = &pdev->dev; 1536 tdma->dma_dev.device_alloc_chan_resources = 1537 tegra_dma_alloc_chan_resources; 1538 tdma->dma_dev.device_free_chan_resources = 1539 tegra_dma_free_chan_resources; 1540 tdma->dma_dev.device_prep_slave_sg = tegra_dma_prep_slave_sg; 1541 tdma->dma_dev.device_prep_dma_cyclic = tegra_dma_prep_dma_cyclic; 1542 tdma->dma_dev.src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | 1543 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | 1544 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | 1545 BIT(DMA_SLAVE_BUSWIDTH_8_BYTES); 1546 tdma->dma_dev.dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | 1547 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | 1548 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | 1549 BIT(DMA_SLAVE_BUSWIDTH_8_BYTES); 1550 tdma->dma_dev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); 1551 tdma->dma_dev.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; 1552 tdma->dma_dev.device_config = tegra_dma_slave_config; 1553 tdma->dma_dev.device_terminate_all = tegra_dma_terminate_all; 1554 tdma->dma_dev.device_synchronize = tegra_dma_synchronize; 1555 tdma->dma_dev.device_tx_status = tegra_dma_tx_status; 1556 tdma->dma_dev.device_issue_pending = tegra_dma_issue_pending; 1557 1558 ret = dma_async_device_register(&tdma->dma_dev); 1559 if (ret < 0) { 1560 dev_err(&pdev->dev, 1561 "Tegra20 APB DMA driver registration failed %d\n", ret); 1562 goto err_pm_disable; 1563 } 1564 1565 ret = of_dma_controller_register(pdev->dev.of_node, 1566 tegra_dma_of_xlate, tdma); 1567 if (ret < 0) { 1568 dev_err(&pdev->dev, 1569 "Tegra20 APB DMA OF registration failed %d\n", ret); 1570 goto err_unregister_dma_dev; 1571 } 1572 1573 dev_info(&pdev->dev, "Tegra20 APB DMA driver registered %u channels\n", 1574 cdata->nr_channels); 1575 1576 return 0; 1577 1578 err_unregister_dma_dev: 1579 dma_async_device_unregister(&tdma->dma_dev); 1580 1581 err_pm_disable: 1582 pm_runtime_disable(&pdev->dev); 1583 1584 err_clk_unprepare: 1585 clk_unprepare(tdma->dma_clk); 1586 1587 return ret; 1588 } 1589 1590 static int tegra_dma_remove(struct platform_device *pdev) 1591 { 1592 struct tegra_dma *tdma = platform_get_drvdata(pdev); 1593 1594 of_dma_controller_free(pdev->dev.of_node); 1595 dma_async_device_unregister(&tdma->dma_dev); 1596 pm_runtime_disable(&pdev->dev); 1597 clk_unprepare(tdma->dma_clk); 1598 1599 return 0; 1600 } 1601 1602 static int __maybe_unused tegra_dma_runtime_suspend(struct device *dev) 1603 { 1604 struct tegra_dma *tdma = dev_get_drvdata(dev); 1605 1606 clk_disable(tdma->dma_clk); 1607 1608 return 0; 1609 } 1610 1611 static int __maybe_unused tegra_dma_runtime_resume(struct device *dev) 1612 { 1613 struct tegra_dma *tdma = dev_get_drvdata(dev); 1614 1615 return clk_enable(tdma->dma_clk); 1616 } 1617 1618 static int __maybe_unused tegra_dma_dev_suspend(struct device *dev) 1619 { 1620 struct tegra_dma *tdma = dev_get_drvdata(dev); 1621 unsigned long flags; 1622 unsigned int i; 1623 bool busy; 1624 1625 for (i = 0; i < tdma->chip_data->nr_channels; i++) { 1626 struct tegra_dma_channel *tdc = &tdma->channels[i]; 1627 1628 tasklet_kill(&tdc->tasklet); 1629 1630 spin_lock_irqsave(&tdc->lock, flags); 1631 busy = tdc->busy; 1632 spin_unlock_irqrestore(&tdc->lock, flags); 1633 1634 if (busy) { 1635 dev_err(tdma->dev, "channel %u busy\n", i); 1636 return -EBUSY; 1637 } 1638 } 1639 1640 return pm_runtime_force_suspend(dev); 1641 } 1642 1643 static int __maybe_unused tegra_dma_dev_resume(struct device *dev) 1644 { 1645 struct tegra_dma *tdma = dev_get_drvdata(dev); 1646 int err; 1647 1648 err = tegra_dma_init_hw(tdma); 1649 if (err) 1650 return err; 1651 1652 return pm_runtime_force_resume(dev); 1653 } 1654 1655 static const struct dev_pm_ops tegra_dma_dev_pm_ops = { 1656 SET_RUNTIME_PM_OPS(tegra_dma_runtime_suspend, tegra_dma_runtime_resume, 1657 NULL) 1658 SET_SYSTEM_SLEEP_PM_OPS(tegra_dma_dev_suspend, tegra_dma_dev_resume) 1659 }; 1660 1661 static const struct of_device_id tegra_dma_of_match[] = { 1662 { 1663 .compatible = "nvidia,tegra148-apbdma", 1664 .data = &tegra148_dma_chip_data, 1665 }, { 1666 .compatible = "nvidia,tegra114-apbdma", 1667 .data = &tegra114_dma_chip_data, 1668 }, { 1669 .compatible = "nvidia,tegra30-apbdma", 1670 .data = &tegra30_dma_chip_data, 1671 }, { 1672 .compatible = "nvidia,tegra20-apbdma", 1673 .data = &tegra20_dma_chip_data, 1674 }, { 1675 }, 1676 }; 1677 MODULE_DEVICE_TABLE(of, tegra_dma_of_match); 1678 1679 static struct platform_driver tegra_dmac_driver = { 1680 .driver = { 1681 .name = "tegra-apbdma", 1682 .pm = &tegra_dma_dev_pm_ops, 1683 .of_match_table = tegra_dma_of_match, 1684 }, 1685 .probe = tegra_dma_probe, 1686 .remove = tegra_dma_remove, 1687 }; 1688 1689 module_platform_driver(tegra_dmac_driver); 1690 1691 MODULE_DESCRIPTION("NVIDIA Tegra APB DMA Controller driver"); 1692 MODULE_AUTHOR("Laxman Dewangan <ldewangan@nvidia.com>"); 1693 MODULE_LICENSE("GPL v2"); 1694