1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * ADMA driver for Nvidia's Tegra210 ADMA controller. 4 * 5 * Copyright (c) 2016, NVIDIA CORPORATION. All rights reserved. 6 */ 7 8 #include <linux/clk.h> 9 #include <linux/iopoll.h> 10 #include <linux/module.h> 11 #include <linux/of_device.h> 12 #include <linux/of_dma.h> 13 #include <linux/of_irq.h> 14 #include <linux/pm_runtime.h> 15 #include <linux/slab.h> 16 17 #include "virt-dma.h" 18 19 #define ADMA_CH_CMD 0x00 20 #define ADMA_CH_STATUS 0x0c 21 #define ADMA_CH_STATUS_XFER_EN BIT(0) 22 #define ADMA_CH_STATUS_XFER_PAUSED BIT(1) 23 24 #define ADMA_CH_INT_STATUS 0x10 25 #define ADMA_CH_INT_STATUS_XFER_DONE BIT(0) 26 27 #define ADMA_CH_INT_CLEAR 0x1c 28 #define ADMA_CH_CTRL 0x24 29 #define ADMA_CH_CTRL_DIR(val) (((val) & 0xf) << 12) 30 #define ADMA_CH_CTRL_DIR_AHUB2MEM 2 31 #define ADMA_CH_CTRL_DIR_MEM2AHUB 4 32 #define ADMA_CH_CTRL_MODE_CONTINUOUS (2 << 8) 33 #define ADMA_CH_CTRL_FLOWCTRL_EN BIT(1) 34 #define ADMA_CH_CTRL_XFER_PAUSE_SHIFT 0 35 36 #define ADMA_CH_CONFIG 0x28 37 #define ADMA_CH_CONFIG_SRC_BUF(val) (((val) & 0x7) << 28) 38 #define ADMA_CH_CONFIG_TRG_BUF(val) (((val) & 0x7) << 24) 39 #define ADMA_CH_CONFIG_BURST_SIZE_SHIFT 20 40 #define ADMA_CH_CONFIG_MAX_BURST_SIZE 16 41 #define ADMA_CH_CONFIG_WEIGHT_FOR_WRR(val) ((val) & 0xf) 42 #define ADMA_CH_CONFIG_MAX_BUFS 8 43 44 #define ADMA_CH_FIFO_CTRL 0x2c 45 #define TEGRA210_ADMA_CH_FIFO_CTRL_OFLWTHRES(val) (((val) & 0xf) << 24) 46 #define TEGRA210_ADMA_CH_FIFO_CTRL_STRVTHRES(val) (((val) & 0xf) << 16) 47 #define TEGRA210_ADMA_CH_FIFO_CTRL_TXSIZE(val) (((val) & 0xf) << 8) 48 #define TEGRA210_ADMA_CH_FIFO_CTRL_RXSIZE(val) ((val) & 0xf) 49 #define TEGRA186_ADMA_CH_FIFO_CTRL_OFLWTHRES(val) (((val) & 0x1f) << 24) 50 #define TEGRA186_ADMA_CH_FIFO_CTRL_STRVTHRES(val) (((val) & 0x1f) << 16) 51 #define TEGRA186_ADMA_CH_FIFO_CTRL_TXSIZE(val) (((val) & 0x1f) << 8) 52 #define TEGRA186_ADMA_CH_FIFO_CTRL_RXSIZE(val) ((val) & 0x1f) 53 54 #define ADMA_CH_LOWER_SRC_ADDR 0x34 55 #define ADMA_CH_LOWER_TRG_ADDR 0x3c 56 #define ADMA_CH_TC 0x44 57 #define ADMA_CH_TC_COUNT_MASK 0x3ffffffc 58 59 #define ADMA_CH_XFER_STATUS 0x54 60 #define ADMA_CH_XFER_STATUS_COUNT_MASK 0xffff 61 62 #define ADMA_GLOBAL_CMD 0x00 63 #define ADMA_GLOBAL_SOFT_RESET 0x04 64 65 #define TEGRA_ADMA_BURST_COMPLETE_TIME 20 66 67 #define TEGRA210_FIFO_CTRL_DEFAULT (TEGRA210_ADMA_CH_FIFO_CTRL_OFLWTHRES(1) | \ 68 TEGRA210_ADMA_CH_FIFO_CTRL_STRVTHRES(1) | \ 69 TEGRA210_ADMA_CH_FIFO_CTRL_TXSIZE(3) | \ 70 TEGRA210_ADMA_CH_FIFO_CTRL_RXSIZE(3)) 71 72 #define TEGRA186_FIFO_CTRL_DEFAULT (TEGRA186_ADMA_CH_FIFO_CTRL_OFLWTHRES(1) | \ 73 TEGRA186_ADMA_CH_FIFO_CTRL_STRVTHRES(1) | \ 74 TEGRA186_ADMA_CH_FIFO_CTRL_TXSIZE(3) | \ 75 TEGRA186_ADMA_CH_FIFO_CTRL_RXSIZE(3)) 76 77 #define ADMA_CH_REG_FIELD_VAL(val, mask, shift) (((val) & mask) << shift) 78 79 struct tegra_adma; 80 81 /* 82 * struct tegra_adma_chip_data - Tegra chip specific data 83 * @global_reg_offset: Register offset of DMA global register. 84 * @global_int_clear: Register offset of DMA global interrupt clear. 85 * @ch_req_tx_shift: Register offset for AHUB transmit channel select. 86 * @ch_req_rx_shift: Register offset for AHUB receive channel select. 87 * @ch_base_offset: Register offset of DMA channel registers. 88 * @ch_fifo_ctrl: Default value for channel FIFO CTRL register. 89 * @ch_req_mask: Mask for Tx or Rx channel select. 90 * @ch_req_max: Maximum number of Tx or Rx channels available. 91 * @ch_reg_size: Size of DMA channel register space. 92 * @nr_channels: Number of DMA channels available. 93 */ 94 struct tegra_adma_chip_data { 95 unsigned int (*adma_get_burst_config)(unsigned int burst_size); 96 unsigned int global_reg_offset; 97 unsigned int global_int_clear; 98 unsigned int ch_req_tx_shift; 99 unsigned int ch_req_rx_shift; 100 unsigned int ch_base_offset; 101 unsigned int ch_fifo_ctrl; 102 unsigned int ch_req_mask; 103 unsigned int ch_req_max; 104 unsigned int ch_reg_size; 105 unsigned int nr_channels; 106 }; 107 108 /* 109 * struct tegra_adma_chan_regs - Tegra ADMA channel registers 110 */ 111 struct tegra_adma_chan_regs { 112 unsigned int ctrl; 113 unsigned int config; 114 unsigned int src_addr; 115 unsigned int trg_addr; 116 unsigned int fifo_ctrl; 117 unsigned int cmd; 118 unsigned int tc; 119 }; 120 121 /* 122 * struct tegra_adma_desc - Tegra ADMA descriptor to manage transfer requests. 123 */ 124 struct tegra_adma_desc { 125 struct virt_dma_desc vd; 126 struct tegra_adma_chan_regs ch_regs; 127 size_t buf_len; 128 size_t period_len; 129 size_t num_periods; 130 }; 131 132 /* 133 * struct tegra_adma_chan - Tegra ADMA channel information 134 */ 135 struct tegra_adma_chan { 136 struct virt_dma_chan vc; 137 struct tegra_adma_desc *desc; 138 struct tegra_adma *tdma; 139 int irq; 140 void __iomem *chan_addr; 141 142 /* Slave channel configuration info */ 143 struct dma_slave_config sconfig; 144 enum dma_transfer_direction sreq_dir; 145 unsigned int sreq_index; 146 bool sreq_reserved; 147 struct tegra_adma_chan_regs ch_regs; 148 149 /* Transfer count and position info */ 150 unsigned int tx_buf_count; 151 unsigned int tx_buf_pos; 152 }; 153 154 /* 155 * struct tegra_adma - Tegra ADMA controller information 156 */ 157 struct tegra_adma { 158 struct dma_device dma_dev; 159 struct device *dev; 160 void __iomem *base_addr; 161 struct clk *ahub_clk; 162 unsigned int nr_channels; 163 unsigned long rx_requests_reserved; 164 unsigned long tx_requests_reserved; 165 166 /* Used to store global command register state when suspending */ 167 unsigned int global_cmd; 168 169 const struct tegra_adma_chip_data *cdata; 170 171 /* Last member of the structure */ 172 struct tegra_adma_chan channels[0]; 173 }; 174 175 static inline void tdma_write(struct tegra_adma *tdma, u32 reg, u32 val) 176 { 177 writel(val, tdma->base_addr + tdma->cdata->global_reg_offset + reg); 178 } 179 180 static inline u32 tdma_read(struct tegra_adma *tdma, u32 reg) 181 { 182 return readl(tdma->base_addr + tdma->cdata->global_reg_offset + reg); 183 } 184 185 static inline void tdma_ch_write(struct tegra_adma_chan *tdc, u32 reg, u32 val) 186 { 187 writel(val, tdc->chan_addr + reg); 188 } 189 190 static inline u32 tdma_ch_read(struct tegra_adma_chan *tdc, u32 reg) 191 { 192 return readl(tdc->chan_addr + reg); 193 } 194 195 static inline struct tegra_adma_chan *to_tegra_adma_chan(struct dma_chan *dc) 196 { 197 return container_of(dc, struct tegra_adma_chan, vc.chan); 198 } 199 200 static inline struct tegra_adma_desc *to_tegra_adma_desc( 201 struct dma_async_tx_descriptor *td) 202 { 203 return container_of(td, struct tegra_adma_desc, vd.tx); 204 } 205 206 static inline struct device *tdc2dev(struct tegra_adma_chan *tdc) 207 { 208 return tdc->tdma->dev; 209 } 210 211 static void tegra_adma_desc_free(struct virt_dma_desc *vd) 212 { 213 kfree(container_of(vd, struct tegra_adma_desc, vd)); 214 } 215 216 static int tegra_adma_slave_config(struct dma_chan *dc, 217 struct dma_slave_config *sconfig) 218 { 219 struct tegra_adma_chan *tdc = to_tegra_adma_chan(dc); 220 221 memcpy(&tdc->sconfig, sconfig, sizeof(*sconfig)); 222 223 return 0; 224 } 225 226 static int tegra_adma_init(struct tegra_adma *tdma) 227 { 228 u32 status; 229 int ret; 230 231 /* Clear any interrupts */ 232 tdma_write(tdma, tdma->cdata->global_int_clear, 0x1); 233 234 /* Assert soft reset */ 235 tdma_write(tdma, ADMA_GLOBAL_SOFT_RESET, 0x1); 236 237 /* Wait for reset to clear */ 238 ret = readx_poll_timeout(readl, 239 tdma->base_addr + 240 tdma->cdata->global_reg_offset + 241 ADMA_GLOBAL_SOFT_RESET, 242 status, status == 0, 20, 10000); 243 if (ret) 244 return ret; 245 246 /* Enable global ADMA registers */ 247 tdma_write(tdma, ADMA_GLOBAL_CMD, 1); 248 249 return 0; 250 } 251 252 static int tegra_adma_request_alloc(struct tegra_adma_chan *tdc, 253 enum dma_transfer_direction direction) 254 { 255 struct tegra_adma *tdma = tdc->tdma; 256 unsigned int sreq_index = tdc->sreq_index; 257 258 if (tdc->sreq_reserved) 259 return tdc->sreq_dir == direction ? 0 : -EINVAL; 260 261 if (sreq_index > tdma->cdata->ch_req_max) { 262 dev_err(tdma->dev, "invalid DMA request\n"); 263 return -EINVAL; 264 } 265 266 switch (direction) { 267 case DMA_MEM_TO_DEV: 268 if (test_and_set_bit(sreq_index, &tdma->tx_requests_reserved)) { 269 dev_err(tdma->dev, "DMA request reserved\n"); 270 return -EINVAL; 271 } 272 break; 273 274 case DMA_DEV_TO_MEM: 275 if (test_and_set_bit(sreq_index, &tdma->rx_requests_reserved)) { 276 dev_err(tdma->dev, "DMA request reserved\n"); 277 return -EINVAL; 278 } 279 break; 280 281 default: 282 dev_WARN(tdma->dev, "channel %s has invalid transfer type\n", 283 dma_chan_name(&tdc->vc.chan)); 284 return -EINVAL; 285 } 286 287 tdc->sreq_dir = direction; 288 tdc->sreq_reserved = true; 289 290 return 0; 291 } 292 293 static void tegra_adma_request_free(struct tegra_adma_chan *tdc) 294 { 295 struct tegra_adma *tdma = tdc->tdma; 296 297 if (!tdc->sreq_reserved) 298 return; 299 300 switch (tdc->sreq_dir) { 301 case DMA_MEM_TO_DEV: 302 clear_bit(tdc->sreq_index, &tdma->tx_requests_reserved); 303 break; 304 305 case DMA_DEV_TO_MEM: 306 clear_bit(tdc->sreq_index, &tdma->rx_requests_reserved); 307 break; 308 309 default: 310 dev_WARN(tdma->dev, "channel %s has invalid transfer type\n", 311 dma_chan_name(&tdc->vc.chan)); 312 return; 313 } 314 315 tdc->sreq_reserved = false; 316 } 317 318 static u32 tegra_adma_irq_status(struct tegra_adma_chan *tdc) 319 { 320 u32 status = tdma_ch_read(tdc, ADMA_CH_INT_STATUS); 321 322 return status & ADMA_CH_INT_STATUS_XFER_DONE; 323 } 324 325 static u32 tegra_adma_irq_clear(struct tegra_adma_chan *tdc) 326 { 327 u32 status = tegra_adma_irq_status(tdc); 328 329 if (status) 330 tdma_ch_write(tdc, ADMA_CH_INT_CLEAR, status); 331 332 return status; 333 } 334 335 static void tegra_adma_stop(struct tegra_adma_chan *tdc) 336 { 337 unsigned int status; 338 339 /* Disable ADMA */ 340 tdma_ch_write(tdc, ADMA_CH_CMD, 0); 341 342 /* Clear interrupt status */ 343 tegra_adma_irq_clear(tdc); 344 345 if (readx_poll_timeout_atomic(readl, tdc->chan_addr + ADMA_CH_STATUS, 346 status, !(status & ADMA_CH_STATUS_XFER_EN), 347 20, 10000)) { 348 dev_err(tdc2dev(tdc), "unable to stop DMA channel\n"); 349 return; 350 } 351 352 kfree(tdc->desc); 353 tdc->desc = NULL; 354 } 355 356 static void tegra_adma_start(struct tegra_adma_chan *tdc) 357 { 358 struct virt_dma_desc *vd = vchan_next_desc(&tdc->vc); 359 struct tegra_adma_chan_regs *ch_regs; 360 struct tegra_adma_desc *desc; 361 362 if (!vd) 363 return; 364 365 list_del(&vd->node); 366 367 desc = to_tegra_adma_desc(&vd->tx); 368 369 if (!desc) { 370 dev_warn(tdc2dev(tdc), "unable to start DMA, no descriptor\n"); 371 return; 372 } 373 374 ch_regs = &desc->ch_regs; 375 376 tdc->tx_buf_pos = 0; 377 tdc->tx_buf_count = 0; 378 tdma_ch_write(tdc, ADMA_CH_TC, ch_regs->tc); 379 tdma_ch_write(tdc, ADMA_CH_CTRL, ch_regs->ctrl); 380 tdma_ch_write(tdc, ADMA_CH_LOWER_SRC_ADDR, ch_regs->src_addr); 381 tdma_ch_write(tdc, ADMA_CH_LOWER_TRG_ADDR, ch_regs->trg_addr); 382 tdma_ch_write(tdc, ADMA_CH_FIFO_CTRL, ch_regs->fifo_ctrl); 383 tdma_ch_write(tdc, ADMA_CH_CONFIG, ch_regs->config); 384 385 /* Start ADMA */ 386 tdma_ch_write(tdc, ADMA_CH_CMD, 1); 387 388 tdc->desc = desc; 389 } 390 391 static unsigned int tegra_adma_get_residue(struct tegra_adma_chan *tdc) 392 { 393 struct tegra_adma_desc *desc = tdc->desc; 394 unsigned int max = ADMA_CH_XFER_STATUS_COUNT_MASK + 1; 395 unsigned int pos = tdma_ch_read(tdc, ADMA_CH_XFER_STATUS); 396 unsigned int periods_remaining; 397 398 /* 399 * Handle wrap around of buffer count register 400 */ 401 if (pos < tdc->tx_buf_pos) 402 tdc->tx_buf_count += pos + (max - tdc->tx_buf_pos); 403 else 404 tdc->tx_buf_count += pos - tdc->tx_buf_pos; 405 406 periods_remaining = tdc->tx_buf_count % desc->num_periods; 407 tdc->tx_buf_pos = pos; 408 409 return desc->buf_len - (periods_remaining * desc->period_len); 410 } 411 412 static irqreturn_t tegra_adma_isr(int irq, void *dev_id) 413 { 414 struct tegra_adma_chan *tdc = dev_id; 415 unsigned long status; 416 unsigned long flags; 417 418 spin_lock_irqsave(&tdc->vc.lock, flags); 419 420 status = tegra_adma_irq_clear(tdc); 421 if (status == 0 || !tdc->desc) { 422 spin_unlock_irqrestore(&tdc->vc.lock, flags); 423 return IRQ_NONE; 424 } 425 426 vchan_cyclic_callback(&tdc->desc->vd); 427 428 spin_unlock_irqrestore(&tdc->vc.lock, flags); 429 430 return IRQ_HANDLED; 431 } 432 433 static void tegra_adma_issue_pending(struct dma_chan *dc) 434 { 435 struct tegra_adma_chan *tdc = to_tegra_adma_chan(dc); 436 unsigned long flags; 437 438 spin_lock_irqsave(&tdc->vc.lock, flags); 439 440 if (vchan_issue_pending(&tdc->vc)) { 441 if (!tdc->desc) 442 tegra_adma_start(tdc); 443 } 444 445 spin_unlock_irqrestore(&tdc->vc.lock, flags); 446 } 447 448 static bool tegra_adma_is_paused(struct tegra_adma_chan *tdc) 449 { 450 u32 csts; 451 452 csts = tdma_ch_read(tdc, ADMA_CH_STATUS); 453 csts &= ADMA_CH_STATUS_XFER_PAUSED; 454 455 return csts ? true : false; 456 } 457 458 static int tegra_adma_pause(struct dma_chan *dc) 459 { 460 struct tegra_adma_chan *tdc = to_tegra_adma_chan(dc); 461 struct tegra_adma_desc *desc = tdc->desc; 462 struct tegra_adma_chan_regs *ch_regs = &desc->ch_regs; 463 int dcnt = 10; 464 465 ch_regs->ctrl = tdma_ch_read(tdc, ADMA_CH_CTRL); 466 ch_regs->ctrl |= (1 << ADMA_CH_CTRL_XFER_PAUSE_SHIFT); 467 tdma_ch_write(tdc, ADMA_CH_CTRL, ch_regs->ctrl); 468 469 while (dcnt-- && !tegra_adma_is_paused(tdc)) 470 udelay(TEGRA_ADMA_BURST_COMPLETE_TIME); 471 472 if (dcnt < 0) { 473 dev_err(tdc2dev(tdc), "unable to pause DMA channel\n"); 474 return -EBUSY; 475 } 476 477 return 0; 478 } 479 480 static int tegra_adma_resume(struct dma_chan *dc) 481 { 482 struct tegra_adma_chan *tdc = to_tegra_adma_chan(dc); 483 struct tegra_adma_desc *desc = tdc->desc; 484 struct tegra_adma_chan_regs *ch_regs = &desc->ch_regs; 485 486 ch_regs->ctrl = tdma_ch_read(tdc, ADMA_CH_CTRL); 487 ch_regs->ctrl &= ~(1 << ADMA_CH_CTRL_XFER_PAUSE_SHIFT); 488 tdma_ch_write(tdc, ADMA_CH_CTRL, ch_regs->ctrl); 489 490 return 0; 491 } 492 493 static int tegra_adma_terminate_all(struct dma_chan *dc) 494 { 495 struct tegra_adma_chan *tdc = to_tegra_adma_chan(dc); 496 unsigned long flags; 497 LIST_HEAD(head); 498 499 spin_lock_irqsave(&tdc->vc.lock, flags); 500 501 if (tdc->desc) 502 tegra_adma_stop(tdc); 503 504 tegra_adma_request_free(tdc); 505 vchan_get_all_descriptors(&tdc->vc, &head); 506 spin_unlock_irqrestore(&tdc->vc.lock, flags); 507 vchan_dma_desc_free_list(&tdc->vc, &head); 508 509 return 0; 510 } 511 512 static enum dma_status tegra_adma_tx_status(struct dma_chan *dc, 513 dma_cookie_t cookie, 514 struct dma_tx_state *txstate) 515 { 516 struct tegra_adma_chan *tdc = to_tegra_adma_chan(dc); 517 struct tegra_adma_desc *desc; 518 struct virt_dma_desc *vd; 519 enum dma_status ret; 520 unsigned long flags; 521 unsigned int residual; 522 523 ret = dma_cookie_status(dc, cookie, txstate); 524 if (ret == DMA_COMPLETE || !txstate) 525 return ret; 526 527 spin_lock_irqsave(&tdc->vc.lock, flags); 528 529 vd = vchan_find_desc(&tdc->vc, cookie); 530 if (vd) { 531 desc = to_tegra_adma_desc(&vd->tx); 532 residual = desc->ch_regs.tc; 533 } else if (tdc->desc && tdc->desc->vd.tx.cookie == cookie) { 534 residual = tegra_adma_get_residue(tdc); 535 } else { 536 residual = 0; 537 } 538 539 spin_unlock_irqrestore(&tdc->vc.lock, flags); 540 541 dma_set_residue(txstate, residual); 542 543 return ret; 544 } 545 546 static unsigned int tegra210_adma_get_burst_config(unsigned int burst_size) 547 { 548 if (!burst_size || burst_size > ADMA_CH_CONFIG_MAX_BURST_SIZE) 549 burst_size = ADMA_CH_CONFIG_MAX_BURST_SIZE; 550 551 return fls(burst_size) << ADMA_CH_CONFIG_BURST_SIZE_SHIFT; 552 } 553 554 static unsigned int tegra186_adma_get_burst_config(unsigned int burst_size) 555 { 556 if (!burst_size || burst_size > ADMA_CH_CONFIG_MAX_BURST_SIZE) 557 burst_size = ADMA_CH_CONFIG_MAX_BURST_SIZE; 558 559 return (burst_size - 1) << ADMA_CH_CONFIG_BURST_SIZE_SHIFT; 560 } 561 562 static int tegra_adma_set_xfer_params(struct tegra_adma_chan *tdc, 563 struct tegra_adma_desc *desc, 564 dma_addr_t buf_addr, 565 enum dma_transfer_direction direction) 566 { 567 struct tegra_adma_chan_regs *ch_regs = &desc->ch_regs; 568 const struct tegra_adma_chip_data *cdata = tdc->tdma->cdata; 569 unsigned int burst_size, adma_dir; 570 571 if (desc->num_periods > ADMA_CH_CONFIG_MAX_BUFS) 572 return -EINVAL; 573 574 switch (direction) { 575 case DMA_MEM_TO_DEV: 576 adma_dir = ADMA_CH_CTRL_DIR_MEM2AHUB; 577 burst_size = tdc->sconfig.dst_maxburst; 578 ch_regs->config = ADMA_CH_CONFIG_SRC_BUF(desc->num_periods - 1); 579 ch_regs->ctrl = ADMA_CH_REG_FIELD_VAL(tdc->sreq_index, 580 cdata->ch_req_mask, 581 cdata->ch_req_tx_shift); 582 ch_regs->src_addr = buf_addr; 583 break; 584 585 case DMA_DEV_TO_MEM: 586 adma_dir = ADMA_CH_CTRL_DIR_AHUB2MEM; 587 burst_size = tdc->sconfig.src_maxburst; 588 ch_regs->config = ADMA_CH_CONFIG_TRG_BUF(desc->num_periods - 1); 589 ch_regs->ctrl = ADMA_CH_REG_FIELD_VAL(tdc->sreq_index, 590 cdata->ch_req_mask, 591 cdata->ch_req_rx_shift); 592 ch_regs->trg_addr = buf_addr; 593 break; 594 595 default: 596 dev_err(tdc2dev(tdc), "DMA direction is not supported\n"); 597 return -EINVAL; 598 } 599 600 ch_regs->ctrl |= ADMA_CH_CTRL_DIR(adma_dir) | 601 ADMA_CH_CTRL_MODE_CONTINUOUS | 602 ADMA_CH_CTRL_FLOWCTRL_EN; 603 ch_regs->config |= cdata->adma_get_burst_config(burst_size); 604 ch_regs->config |= ADMA_CH_CONFIG_WEIGHT_FOR_WRR(1); 605 ch_regs->fifo_ctrl = cdata->ch_fifo_ctrl; 606 ch_regs->tc = desc->period_len & ADMA_CH_TC_COUNT_MASK; 607 608 return tegra_adma_request_alloc(tdc, direction); 609 } 610 611 static struct dma_async_tx_descriptor *tegra_adma_prep_dma_cyclic( 612 struct dma_chan *dc, dma_addr_t buf_addr, size_t buf_len, 613 size_t period_len, enum dma_transfer_direction direction, 614 unsigned long flags) 615 { 616 struct tegra_adma_chan *tdc = to_tegra_adma_chan(dc); 617 struct tegra_adma_desc *desc = NULL; 618 619 if (!buf_len || !period_len || period_len > ADMA_CH_TC_COUNT_MASK) { 620 dev_err(tdc2dev(tdc), "invalid buffer/period len\n"); 621 return NULL; 622 } 623 624 if (buf_len % period_len) { 625 dev_err(tdc2dev(tdc), "buf_len not a multiple of period_len\n"); 626 return NULL; 627 } 628 629 if (!IS_ALIGNED(buf_addr, 4)) { 630 dev_err(tdc2dev(tdc), "invalid buffer alignment\n"); 631 return NULL; 632 } 633 634 desc = kzalloc(sizeof(*desc), GFP_NOWAIT); 635 if (!desc) 636 return NULL; 637 638 desc->buf_len = buf_len; 639 desc->period_len = period_len; 640 desc->num_periods = buf_len / period_len; 641 642 if (tegra_adma_set_xfer_params(tdc, desc, buf_addr, direction)) { 643 kfree(desc); 644 return NULL; 645 } 646 647 return vchan_tx_prep(&tdc->vc, &desc->vd, flags); 648 } 649 650 static int tegra_adma_alloc_chan_resources(struct dma_chan *dc) 651 { 652 struct tegra_adma_chan *tdc = to_tegra_adma_chan(dc); 653 int ret; 654 655 ret = request_irq(tdc->irq, tegra_adma_isr, 0, dma_chan_name(dc), tdc); 656 if (ret) { 657 dev_err(tdc2dev(tdc), "failed to get interrupt for %s\n", 658 dma_chan_name(dc)); 659 return ret; 660 } 661 662 ret = pm_runtime_get_sync(tdc2dev(tdc)); 663 if (ret < 0) { 664 free_irq(tdc->irq, tdc); 665 return ret; 666 } 667 668 dma_cookie_init(&tdc->vc.chan); 669 670 return 0; 671 } 672 673 static void tegra_adma_free_chan_resources(struct dma_chan *dc) 674 { 675 struct tegra_adma_chan *tdc = to_tegra_adma_chan(dc); 676 677 tegra_adma_terminate_all(dc); 678 vchan_free_chan_resources(&tdc->vc); 679 tasklet_kill(&tdc->vc.task); 680 free_irq(tdc->irq, tdc); 681 pm_runtime_put(tdc2dev(tdc)); 682 683 tdc->sreq_index = 0; 684 tdc->sreq_dir = DMA_TRANS_NONE; 685 } 686 687 static struct dma_chan *tegra_dma_of_xlate(struct of_phandle_args *dma_spec, 688 struct of_dma *ofdma) 689 { 690 struct tegra_adma *tdma = ofdma->of_dma_data; 691 struct tegra_adma_chan *tdc; 692 struct dma_chan *chan; 693 unsigned int sreq_index; 694 695 if (dma_spec->args_count != 1) 696 return NULL; 697 698 sreq_index = dma_spec->args[0]; 699 700 if (sreq_index == 0) { 701 dev_err(tdma->dev, "DMA request must not be 0\n"); 702 return NULL; 703 } 704 705 chan = dma_get_any_slave_channel(&tdma->dma_dev); 706 if (!chan) 707 return NULL; 708 709 tdc = to_tegra_adma_chan(chan); 710 tdc->sreq_index = sreq_index; 711 712 return chan; 713 } 714 715 static int tegra_adma_runtime_suspend(struct device *dev) 716 { 717 struct tegra_adma *tdma = dev_get_drvdata(dev); 718 struct tegra_adma_chan_regs *ch_reg; 719 struct tegra_adma_chan *tdc; 720 int i; 721 722 tdma->global_cmd = tdma_read(tdma, ADMA_GLOBAL_CMD); 723 if (!tdma->global_cmd) 724 goto clk_disable; 725 726 for (i = 0; i < tdma->nr_channels; i++) { 727 tdc = &tdma->channels[i]; 728 ch_reg = &tdc->ch_regs; 729 ch_reg->cmd = tdma_ch_read(tdc, ADMA_CH_CMD); 730 /* skip if channel is not active */ 731 if (!ch_reg->cmd) 732 continue; 733 ch_reg->tc = tdma_ch_read(tdc, ADMA_CH_TC); 734 ch_reg->src_addr = tdma_ch_read(tdc, ADMA_CH_LOWER_SRC_ADDR); 735 ch_reg->trg_addr = tdma_ch_read(tdc, ADMA_CH_LOWER_TRG_ADDR); 736 ch_reg->ctrl = tdma_ch_read(tdc, ADMA_CH_CTRL); 737 ch_reg->fifo_ctrl = tdma_ch_read(tdc, ADMA_CH_FIFO_CTRL); 738 ch_reg->config = tdma_ch_read(tdc, ADMA_CH_CONFIG); 739 } 740 741 clk_disable: 742 clk_disable_unprepare(tdma->ahub_clk); 743 744 return 0; 745 } 746 747 static int tegra_adma_runtime_resume(struct device *dev) 748 { 749 struct tegra_adma *tdma = dev_get_drvdata(dev); 750 struct tegra_adma_chan_regs *ch_reg; 751 struct tegra_adma_chan *tdc; 752 int ret, i; 753 754 ret = clk_prepare_enable(tdma->ahub_clk); 755 if (ret) { 756 dev_err(dev, "ahub clk_enable failed: %d\n", ret); 757 return ret; 758 } 759 tdma_write(tdma, ADMA_GLOBAL_CMD, tdma->global_cmd); 760 761 if (!tdma->global_cmd) 762 return 0; 763 764 for (i = 0; i < tdma->nr_channels; i++) { 765 tdc = &tdma->channels[i]; 766 ch_reg = &tdc->ch_regs; 767 /* skip if channel was not active earlier */ 768 if (!ch_reg->cmd) 769 continue; 770 tdma_ch_write(tdc, ADMA_CH_TC, ch_reg->tc); 771 tdma_ch_write(tdc, ADMA_CH_LOWER_SRC_ADDR, ch_reg->src_addr); 772 tdma_ch_write(tdc, ADMA_CH_LOWER_TRG_ADDR, ch_reg->trg_addr); 773 tdma_ch_write(tdc, ADMA_CH_CTRL, ch_reg->ctrl); 774 tdma_ch_write(tdc, ADMA_CH_FIFO_CTRL, ch_reg->fifo_ctrl); 775 tdma_ch_write(tdc, ADMA_CH_CONFIG, ch_reg->config); 776 tdma_ch_write(tdc, ADMA_CH_CMD, ch_reg->cmd); 777 } 778 779 return 0; 780 } 781 782 static const struct tegra_adma_chip_data tegra210_chip_data = { 783 .adma_get_burst_config = tegra210_adma_get_burst_config, 784 .global_reg_offset = 0xc00, 785 .global_int_clear = 0x20, 786 .ch_req_tx_shift = 28, 787 .ch_req_rx_shift = 24, 788 .ch_base_offset = 0, 789 .ch_fifo_ctrl = TEGRA210_FIFO_CTRL_DEFAULT, 790 .ch_req_mask = 0xf, 791 .ch_req_max = 10, 792 .ch_reg_size = 0x80, 793 .nr_channels = 22, 794 }; 795 796 static const struct tegra_adma_chip_data tegra186_chip_data = { 797 .adma_get_burst_config = tegra186_adma_get_burst_config, 798 .global_reg_offset = 0, 799 .global_int_clear = 0x402c, 800 .ch_req_tx_shift = 27, 801 .ch_req_rx_shift = 22, 802 .ch_base_offset = 0x10000, 803 .ch_fifo_ctrl = TEGRA186_FIFO_CTRL_DEFAULT, 804 .ch_req_mask = 0x1f, 805 .ch_req_max = 20, 806 .ch_reg_size = 0x100, 807 .nr_channels = 32, 808 }; 809 810 static const struct of_device_id tegra_adma_of_match[] = { 811 { .compatible = "nvidia,tegra210-adma", .data = &tegra210_chip_data }, 812 { .compatible = "nvidia,tegra186-adma", .data = &tegra186_chip_data }, 813 { }, 814 }; 815 MODULE_DEVICE_TABLE(of, tegra_adma_of_match); 816 817 static int tegra_adma_probe(struct platform_device *pdev) 818 { 819 const struct tegra_adma_chip_data *cdata; 820 struct tegra_adma *tdma; 821 struct resource *res; 822 int ret, i; 823 824 cdata = of_device_get_match_data(&pdev->dev); 825 if (!cdata) { 826 dev_err(&pdev->dev, "device match data not found\n"); 827 return -ENODEV; 828 } 829 830 tdma = devm_kzalloc(&pdev->dev, 831 struct_size(tdma, channels, cdata->nr_channels), 832 GFP_KERNEL); 833 if (!tdma) 834 return -ENOMEM; 835 836 tdma->dev = &pdev->dev; 837 tdma->cdata = cdata; 838 tdma->nr_channels = cdata->nr_channels; 839 platform_set_drvdata(pdev, tdma); 840 841 res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 842 tdma->base_addr = devm_ioremap_resource(&pdev->dev, res); 843 if (IS_ERR(tdma->base_addr)) 844 return PTR_ERR(tdma->base_addr); 845 846 tdma->ahub_clk = devm_clk_get(&pdev->dev, "d_audio"); 847 if (IS_ERR(tdma->ahub_clk)) { 848 dev_err(&pdev->dev, "Error: Missing ahub controller clock\n"); 849 return PTR_ERR(tdma->ahub_clk); 850 } 851 852 INIT_LIST_HEAD(&tdma->dma_dev.channels); 853 for (i = 0; i < tdma->nr_channels; i++) { 854 struct tegra_adma_chan *tdc = &tdma->channels[i]; 855 856 tdc->chan_addr = tdma->base_addr + cdata->ch_base_offset 857 + (cdata->ch_reg_size * i); 858 859 tdc->irq = of_irq_get(pdev->dev.of_node, i); 860 if (tdc->irq <= 0) { 861 ret = tdc->irq ?: -ENXIO; 862 goto irq_dispose; 863 } 864 865 vchan_init(&tdc->vc, &tdma->dma_dev); 866 tdc->vc.desc_free = tegra_adma_desc_free; 867 tdc->tdma = tdma; 868 } 869 870 pm_runtime_enable(&pdev->dev); 871 872 ret = pm_runtime_get_sync(&pdev->dev); 873 if (ret < 0) 874 goto rpm_disable; 875 876 ret = tegra_adma_init(tdma); 877 if (ret) 878 goto rpm_put; 879 880 dma_cap_set(DMA_SLAVE, tdma->dma_dev.cap_mask); 881 dma_cap_set(DMA_PRIVATE, tdma->dma_dev.cap_mask); 882 dma_cap_set(DMA_CYCLIC, tdma->dma_dev.cap_mask); 883 884 tdma->dma_dev.dev = &pdev->dev; 885 tdma->dma_dev.device_alloc_chan_resources = 886 tegra_adma_alloc_chan_resources; 887 tdma->dma_dev.device_free_chan_resources = 888 tegra_adma_free_chan_resources; 889 tdma->dma_dev.device_issue_pending = tegra_adma_issue_pending; 890 tdma->dma_dev.device_prep_dma_cyclic = tegra_adma_prep_dma_cyclic; 891 tdma->dma_dev.device_config = tegra_adma_slave_config; 892 tdma->dma_dev.device_tx_status = tegra_adma_tx_status; 893 tdma->dma_dev.device_terminate_all = tegra_adma_terminate_all; 894 tdma->dma_dev.src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES); 895 tdma->dma_dev.dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES); 896 tdma->dma_dev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); 897 tdma->dma_dev.residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT; 898 tdma->dma_dev.device_pause = tegra_adma_pause; 899 tdma->dma_dev.device_resume = tegra_adma_resume; 900 901 ret = dma_async_device_register(&tdma->dma_dev); 902 if (ret < 0) { 903 dev_err(&pdev->dev, "ADMA registration failed: %d\n", ret); 904 goto irq_dispose; 905 } 906 907 ret = of_dma_controller_register(pdev->dev.of_node, 908 tegra_dma_of_xlate, tdma); 909 if (ret < 0) { 910 dev_err(&pdev->dev, "ADMA OF registration failed %d\n", ret); 911 goto dma_remove; 912 } 913 914 pm_runtime_put(&pdev->dev); 915 916 dev_info(&pdev->dev, "Tegra210 ADMA driver registered %d channels\n", 917 tdma->nr_channels); 918 919 return 0; 920 921 dma_remove: 922 dma_async_device_unregister(&tdma->dma_dev); 923 rpm_put: 924 pm_runtime_put_sync(&pdev->dev); 925 rpm_disable: 926 pm_runtime_disable(&pdev->dev); 927 irq_dispose: 928 while (--i >= 0) 929 irq_dispose_mapping(tdma->channels[i].irq); 930 931 return ret; 932 } 933 934 static int tegra_adma_remove(struct platform_device *pdev) 935 { 936 struct tegra_adma *tdma = platform_get_drvdata(pdev); 937 int i; 938 939 of_dma_controller_free(pdev->dev.of_node); 940 dma_async_device_unregister(&tdma->dma_dev); 941 942 for (i = 0; i < tdma->nr_channels; ++i) 943 irq_dispose_mapping(tdma->channels[i].irq); 944 945 pm_runtime_put_sync(&pdev->dev); 946 pm_runtime_disable(&pdev->dev); 947 948 return 0; 949 } 950 951 static const struct dev_pm_ops tegra_adma_dev_pm_ops = { 952 SET_RUNTIME_PM_OPS(tegra_adma_runtime_suspend, 953 tegra_adma_runtime_resume, NULL) 954 SET_LATE_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, 955 pm_runtime_force_resume) 956 }; 957 958 static struct platform_driver tegra_admac_driver = { 959 .driver = { 960 .name = "tegra-adma", 961 .pm = &tegra_adma_dev_pm_ops, 962 .of_match_table = tegra_adma_of_match, 963 }, 964 .probe = tegra_adma_probe, 965 .remove = tegra_adma_remove, 966 }; 967 968 module_platform_driver(tegra_admac_driver); 969 970 MODULE_ALIAS("platform:tegra210-adma"); 971 MODULE_DESCRIPTION("NVIDIA Tegra ADMA driver"); 972 MODULE_AUTHOR("Dara Ramesh <dramesh@nvidia.com>"); 973 MODULE_AUTHOR("Jon Hunter <jonathanh@nvidia.com>"); 974 MODULE_LICENSE("GPL v2"); 975