1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (c) 2013-2015, The Linux Foundation. All rights reserved. 4 */ 5 6 #include <linux/clk.h> 7 #include <linux/delay.h> 8 #include <linux/device.h> 9 #include <linux/dmaengine.h> 10 #include <linux/dma-mapping.h> 11 #include <linux/init.h> 12 #include <linux/interrupt.h> 13 #include <linux/io.h> 14 #include <linux/kernel.h> 15 #include <linux/module.h> 16 #include <linux/of.h> 17 #include <linux/of_address.h> 18 #include <linux/of_irq.h> 19 #include <linux/of_dma.h> 20 #include <linux/platform_device.h> 21 #include <linux/reset.h> 22 #include <linux/scatterlist.h> 23 #include <linux/slab.h> 24 25 #include "../dmaengine.h" 26 #include "../virt-dma.h" 27 28 /* ADM registers - calculated from channel number and security domain */ 29 #define ADM_CHAN_MULTI 0x4 30 #define ADM_CI_MULTI 0x4 31 #define ADM_CRCI_MULTI 0x4 32 #define ADM_EE_MULTI 0x800 33 #define ADM_CHAN_OFFS(chan) (ADM_CHAN_MULTI * (chan)) 34 #define ADM_EE_OFFS(ee) (ADM_EE_MULTI * (ee)) 35 #define ADM_CHAN_EE_OFFS(chan, ee) (ADM_CHAN_OFFS(chan) + ADM_EE_OFFS(ee)) 36 #define ADM_CHAN_OFFS(chan) (ADM_CHAN_MULTI * (chan)) 37 #define ADM_CI_OFFS(ci) (ADM_CHAN_OFF(ci)) 38 #define ADM_CH_CMD_PTR(chan, ee) (ADM_CHAN_EE_OFFS(chan, ee)) 39 #define ADM_CH_RSLT(chan, ee) (0x40 + ADM_CHAN_EE_OFFS(chan, ee)) 40 #define ADM_CH_FLUSH_STATE0(chan, ee) (0x80 + ADM_CHAN_EE_OFFS(chan, ee)) 41 #define ADM_CH_STATUS_SD(chan, ee) (0x200 + ADM_CHAN_EE_OFFS(chan, ee)) 42 #define ADM_CH_CONF(chan) (0x240 + ADM_CHAN_OFFS(chan)) 43 #define ADM_CH_RSLT_CONF(chan, ee) (0x300 + ADM_CHAN_EE_OFFS(chan, ee)) 44 #define ADM_SEC_DOMAIN_IRQ_STATUS(ee) (0x380 + ADM_EE_OFFS(ee)) 45 #define ADM_CI_CONF(ci) (0x390 + (ci) * ADM_CI_MULTI) 46 #define ADM_GP_CTL 0x3d8 47 #define ADM_CRCI_CTL(crci, ee) (0x400 + (crci) * ADM_CRCI_MULTI + \ 48 ADM_EE_OFFS(ee)) 49 50 /* channel status */ 51 #define ADM_CH_STATUS_VALID BIT(1) 52 53 /* channel result */ 54 #define ADM_CH_RSLT_VALID BIT(31) 55 #define ADM_CH_RSLT_ERR BIT(3) 56 #define ADM_CH_RSLT_FLUSH BIT(2) 57 #define ADM_CH_RSLT_TPD BIT(1) 58 59 /* channel conf */ 60 #define ADM_CH_CONF_SHADOW_EN BIT(12) 61 #define ADM_CH_CONF_MPU_DISABLE BIT(11) 62 #define ADM_CH_CONF_PERM_MPU_CONF BIT(9) 63 #define ADM_CH_CONF_FORCE_RSLT_EN BIT(7) 64 #define ADM_CH_CONF_SEC_DOMAIN(ee) ((((ee) & 0x3) << 4) | (((ee) & 0x4) << 11)) 65 66 /* channel result conf */ 67 #define ADM_CH_RSLT_CONF_FLUSH_EN BIT(1) 68 #define ADM_CH_RSLT_CONF_IRQ_EN BIT(0) 69 70 /* CRCI CTL */ 71 #define ADM_CRCI_CTL_MUX_SEL BIT(18) 72 #define ADM_CRCI_CTL_RST BIT(17) 73 74 /* CI configuration */ 75 #define ADM_CI_RANGE_END(x) ((x) << 24) 76 #define ADM_CI_RANGE_START(x) ((x) << 16) 77 #define ADM_CI_BURST_4_WORDS BIT(2) 78 #define ADM_CI_BURST_8_WORDS BIT(3) 79 80 /* GP CTL */ 81 #define ADM_GP_CTL_LP_EN BIT(12) 82 #define ADM_GP_CTL_LP_CNT(x) ((x) << 8) 83 84 /* Command pointer list entry */ 85 #define ADM_CPLE_LP BIT(31) 86 #define ADM_CPLE_CMD_PTR_LIST BIT(29) 87 88 /* Command list entry */ 89 #define ADM_CMD_LC BIT(31) 90 #define ADM_CMD_DST_CRCI(n) (((n) & 0xf) << 7) 91 #define ADM_CMD_SRC_CRCI(n) (((n) & 0xf) << 3) 92 93 #define ADM_CMD_TYPE_SINGLE 0x0 94 #define ADM_CMD_TYPE_BOX 0x3 95 96 #define ADM_CRCI_MUX_SEL BIT(4) 97 #define ADM_DESC_ALIGN 8 98 #define ADM_MAX_XFER (SZ_64K - 1) 99 #define ADM_MAX_ROWS (SZ_64K - 1) 100 #define ADM_MAX_CHANNELS 16 101 102 struct adm_desc_hw_box { 103 u32 cmd; 104 u32 src_addr; 105 u32 dst_addr; 106 u32 row_len; 107 u32 num_rows; 108 u32 row_offset; 109 }; 110 111 struct adm_desc_hw_single { 112 u32 cmd; 113 u32 src_addr; 114 u32 dst_addr; 115 u32 len; 116 }; 117 118 struct adm_async_desc { 119 struct virt_dma_desc vd; 120 struct adm_device *adev; 121 122 size_t length; 123 enum dma_transfer_direction dir; 124 dma_addr_t dma_addr; 125 size_t dma_len; 126 127 void *cpl; 128 dma_addr_t cp_addr; 129 u32 crci; 130 u32 mux; 131 u32 blk_size; 132 }; 133 134 struct adm_chan { 135 struct virt_dma_chan vc; 136 struct adm_device *adev; 137 138 /* parsed from DT */ 139 u32 id; /* channel id */ 140 141 struct adm_async_desc *curr_txd; 142 struct dma_slave_config slave; 143 struct list_head node; 144 145 int error; 146 int initialized; 147 }; 148 149 static inline struct adm_chan *to_adm_chan(struct dma_chan *common) 150 { 151 return container_of(common, struct adm_chan, vc.chan); 152 } 153 154 struct adm_device { 155 void __iomem *regs; 156 struct device *dev; 157 struct dma_device common; 158 struct device_dma_parameters dma_parms; 159 struct adm_chan *channels; 160 161 u32 ee; 162 163 struct clk *core_clk; 164 struct clk *iface_clk; 165 166 struct reset_control *clk_reset; 167 struct reset_control *c0_reset; 168 struct reset_control *c1_reset; 169 struct reset_control *c2_reset; 170 int irq; 171 }; 172 173 /** 174 * adm_free_chan - Frees dma resources associated with the specific channel 175 * 176 * @chan: dma channel 177 * 178 * Free all allocated descriptors associated with this channel 179 */ 180 static void adm_free_chan(struct dma_chan *chan) 181 { 182 /* free all queued descriptors */ 183 vchan_free_chan_resources(to_virt_chan(chan)); 184 } 185 186 /** 187 * adm_get_blksize - Get block size from burst value 188 * 189 * @burst: Burst size of transaction 190 */ 191 static int adm_get_blksize(unsigned int burst) 192 { 193 int ret; 194 195 switch (burst) { 196 case 16: 197 case 32: 198 case 64: 199 case 128: 200 ret = ffs(burst >> 4) - 1; 201 break; 202 case 192: 203 ret = 4; 204 break; 205 case 256: 206 ret = 5; 207 break; 208 default: 209 ret = -EINVAL; 210 break; 211 } 212 213 return ret; 214 } 215 216 /** 217 * adm_process_fc_descriptors - Process descriptors for flow controlled xfers 218 * 219 * @achan: ADM channel 220 * @desc: Descriptor memory pointer 221 * @sg: Scatterlist entry 222 * @crci: CRCI value 223 * @burst: Burst size of transaction 224 * @direction: DMA transfer direction 225 */ 226 static void *adm_process_fc_descriptors(struct adm_chan *achan, void *desc, 227 struct scatterlist *sg, u32 crci, 228 u32 burst, 229 enum dma_transfer_direction direction) 230 { 231 struct adm_desc_hw_box *box_desc = NULL; 232 struct adm_desc_hw_single *single_desc; 233 u32 remainder = sg_dma_len(sg); 234 u32 rows, row_offset, crci_cmd; 235 u32 mem_addr = sg_dma_address(sg); 236 u32 *incr_addr = &mem_addr; 237 u32 *src, *dst; 238 239 if (direction == DMA_DEV_TO_MEM) { 240 crci_cmd = ADM_CMD_SRC_CRCI(crci); 241 row_offset = burst; 242 src = &achan->slave.src_addr; 243 dst = &mem_addr; 244 } else { 245 crci_cmd = ADM_CMD_DST_CRCI(crci); 246 row_offset = burst << 16; 247 src = &mem_addr; 248 dst = &achan->slave.dst_addr; 249 } 250 251 while (remainder >= burst) { 252 box_desc = desc; 253 box_desc->cmd = ADM_CMD_TYPE_BOX | crci_cmd; 254 box_desc->row_offset = row_offset; 255 box_desc->src_addr = *src; 256 box_desc->dst_addr = *dst; 257 258 rows = remainder / burst; 259 rows = min_t(u32, rows, ADM_MAX_ROWS); 260 box_desc->num_rows = rows << 16 | rows; 261 box_desc->row_len = burst << 16 | burst; 262 263 *incr_addr += burst * rows; 264 remainder -= burst * rows; 265 desc += sizeof(*box_desc); 266 } 267 268 /* if leftover bytes, do one single descriptor */ 269 if (remainder) { 270 single_desc = desc; 271 single_desc->cmd = ADM_CMD_TYPE_SINGLE | crci_cmd; 272 single_desc->len = remainder; 273 single_desc->src_addr = *src; 274 single_desc->dst_addr = *dst; 275 desc += sizeof(*single_desc); 276 277 if (sg_is_last(sg)) 278 single_desc->cmd |= ADM_CMD_LC; 279 } else { 280 if (box_desc && sg_is_last(sg)) 281 box_desc->cmd |= ADM_CMD_LC; 282 } 283 284 return desc; 285 } 286 287 /** 288 * adm_process_non_fc_descriptors - Process descriptors for non-fc xfers 289 * 290 * @achan: ADM channel 291 * @desc: Descriptor memory pointer 292 * @sg: Scatterlist entry 293 * @direction: DMA transfer direction 294 */ 295 static void *adm_process_non_fc_descriptors(struct adm_chan *achan, void *desc, 296 struct scatterlist *sg, 297 enum dma_transfer_direction direction) 298 { 299 struct adm_desc_hw_single *single_desc; 300 u32 remainder = sg_dma_len(sg); 301 u32 mem_addr = sg_dma_address(sg); 302 u32 *incr_addr = &mem_addr; 303 u32 *src, *dst; 304 305 if (direction == DMA_DEV_TO_MEM) { 306 src = &achan->slave.src_addr; 307 dst = &mem_addr; 308 } else { 309 src = &mem_addr; 310 dst = &achan->slave.dst_addr; 311 } 312 313 do { 314 single_desc = desc; 315 single_desc->cmd = ADM_CMD_TYPE_SINGLE; 316 single_desc->src_addr = *src; 317 single_desc->dst_addr = *dst; 318 single_desc->len = (remainder > ADM_MAX_XFER) ? 319 ADM_MAX_XFER : remainder; 320 321 remainder -= single_desc->len; 322 *incr_addr += single_desc->len; 323 desc += sizeof(*single_desc); 324 } while (remainder); 325 326 /* set last command if this is the end of the whole transaction */ 327 if (sg_is_last(sg)) 328 single_desc->cmd |= ADM_CMD_LC; 329 330 return desc; 331 } 332 333 /** 334 * adm_prep_slave_sg - Prep slave sg transaction 335 * 336 * @chan: dma channel 337 * @sgl: scatter gather list 338 * @sg_len: length of sg 339 * @direction: DMA transfer direction 340 * @flags: DMA flags 341 * @context: transfer context (unused) 342 */ 343 static struct dma_async_tx_descriptor *adm_prep_slave_sg(struct dma_chan *chan, 344 struct scatterlist *sgl, 345 unsigned int sg_len, 346 enum dma_transfer_direction direction, 347 unsigned long flags, 348 void *context) 349 { 350 struct adm_chan *achan = to_adm_chan(chan); 351 struct adm_device *adev = achan->adev; 352 struct adm_async_desc *async_desc; 353 struct scatterlist *sg; 354 dma_addr_t cple_addr; 355 u32 i, burst; 356 u32 single_count = 0, box_count = 0, crci = 0; 357 void *desc; 358 u32 *cple; 359 int blk_size = 0; 360 361 if (!is_slave_direction(direction)) { 362 dev_err(adev->dev, "invalid dma direction\n"); 363 return NULL; 364 } 365 366 /* 367 * get burst value from slave configuration 368 */ 369 burst = (direction == DMA_MEM_TO_DEV) ? 370 achan->slave.dst_maxburst : 371 achan->slave.src_maxburst; 372 373 /* if using flow control, validate burst and crci values */ 374 if (achan->slave.device_fc) { 375 blk_size = adm_get_blksize(burst); 376 if (blk_size < 0) { 377 dev_err(adev->dev, "invalid burst value: %d\n", 378 burst); 379 return ERR_PTR(-EINVAL); 380 } 381 382 crci = achan->slave.slave_id & 0xf; 383 if (!crci || achan->slave.slave_id > 0x1f) { 384 dev_err(adev->dev, "invalid crci value\n"); 385 return ERR_PTR(-EINVAL); 386 } 387 } 388 389 /* iterate through sgs and compute allocation size of structures */ 390 for_each_sg(sgl, sg, sg_len, i) { 391 if (achan->slave.device_fc) { 392 box_count += DIV_ROUND_UP(sg_dma_len(sg) / burst, 393 ADM_MAX_ROWS); 394 if (sg_dma_len(sg) % burst) 395 single_count++; 396 } else { 397 single_count += DIV_ROUND_UP(sg_dma_len(sg), 398 ADM_MAX_XFER); 399 } 400 } 401 402 async_desc = kzalloc(sizeof(*async_desc), GFP_NOWAIT); 403 if (!async_desc) 404 return ERR_PTR(-ENOMEM); 405 406 if (crci) 407 async_desc->mux = achan->slave.slave_id & ADM_CRCI_MUX_SEL ? 408 ADM_CRCI_CTL_MUX_SEL : 0; 409 async_desc->crci = crci; 410 async_desc->blk_size = blk_size; 411 async_desc->dma_len = single_count * sizeof(struct adm_desc_hw_single) + 412 box_count * sizeof(struct adm_desc_hw_box) + 413 sizeof(*cple) + 2 * ADM_DESC_ALIGN; 414 415 async_desc->cpl = kzalloc(async_desc->dma_len, GFP_NOWAIT); 416 if (!async_desc->cpl) 417 goto free; 418 419 async_desc->adev = adev; 420 421 /* both command list entry and descriptors must be 8 byte aligned */ 422 cple = PTR_ALIGN(async_desc->cpl, ADM_DESC_ALIGN); 423 desc = PTR_ALIGN(cple + 1, ADM_DESC_ALIGN); 424 425 for_each_sg(sgl, sg, sg_len, i) { 426 async_desc->length += sg_dma_len(sg); 427 428 if (achan->slave.device_fc) 429 desc = adm_process_fc_descriptors(achan, desc, sg, crci, 430 burst, direction); 431 else 432 desc = adm_process_non_fc_descriptors(achan, desc, sg, 433 direction); 434 } 435 436 async_desc->dma_addr = dma_map_single(adev->dev, async_desc->cpl, 437 async_desc->dma_len, 438 DMA_TO_DEVICE); 439 if (dma_mapping_error(adev->dev, async_desc->dma_addr)) 440 goto free; 441 442 cple_addr = async_desc->dma_addr + ((void *)cple - async_desc->cpl); 443 444 /* init cmd list */ 445 dma_sync_single_for_cpu(adev->dev, cple_addr, sizeof(*cple), 446 DMA_TO_DEVICE); 447 *cple = ADM_CPLE_LP; 448 *cple |= (async_desc->dma_addr + ADM_DESC_ALIGN) >> 3; 449 dma_sync_single_for_device(adev->dev, cple_addr, sizeof(*cple), 450 DMA_TO_DEVICE); 451 452 return vchan_tx_prep(&achan->vc, &async_desc->vd, flags); 453 454 free: 455 kfree(async_desc); 456 return ERR_PTR(-ENOMEM); 457 } 458 459 /** 460 * adm_terminate_all - terminate all transactions on a channel 461 * @chan: dma channel 462 * 463 * Dequeues and frees all transactions, aborts current transaction 464 * No callbacks are done 465 * 466 */ 467 static int adm_terminate_all(struct dma_chan *chan) 468 { 469 struct adm_chan *achan = to_adm_chan(chan); 470 struct adm_device *adev = achan->adev; 471 unsigned long flags; 472 LIST_HEAD(head); 473 474 spin_lock_irqsave(&achan->vc.lock, flags); 475 vchan_get_all_descriptors(&achan->vc, &head); 476 477 /* send flush command to terminate current transaction */ 478 writel_relaxed(0x0, 479 adev->regs + ADM_CH_FLUSH_STATE0(achan->id, adev->ee)); 480 481 spin_unlock_irqrestore(&achan->vc.lock, flags); 482 483 vchan_dma_desc_free_list(&achan->vc, &head); 484 485 return 0; 486 } 487 488 static int adm_slave_config(struct dma_chan *chan, struct dma_slave_config *cfg) 489 { 490 struct adm_chan *achan = to_adm_chan(chan); 491 unsigned long flag; 492 493 spin_lock_irqsave(&achan->vc.lock, flag); 494 memcpy(&achan->slave, cfg, sizeof(struct dma_slave_config)); 495 spin_unlock_irqrestore(&achan->vc.lock, flag); 496 497 return 0; 498 } 499 500 /** 501 * adm_start_dma - start next transaction 502 * @achan: ADM dma channel 503 */ 504 static void adm_start_dma(struct adm_chan *achan) 505 { 506 struct virt_dma_desc *vd = vchan_next_desc(&achan->vc); 507 struct adm_device *adev = achan->adev; 508 struct adm_async_desc *async_desc; 509 510 lockdep_assert_held(&achan->vc.lock); 511 512 if (!vd) 513 return; 514 515 list_del(&vd->node); 516 517 /* write next command list out to the CMD FIFO */ 518 async_desc = container_of(vd, struct adm_async_desc, vd); 519 achan->curr_txd = async_desc; 520 521 /* reset channel error */ 522 achan->error = 0; 523 524 if (!achan->initialized) { 525 /* enable interrupts */ 526 writel(ADM_CH_CONF_SHADOW_EN | 527 ADM_CH_CONF_PERM_MPU_CONF | 528 ADM_CH_CONF_MPU_DISABLE | 529 ADM_CH_CONF_SEC_DOMAIN(adev->ee), 530 adev->regs + ADM_CH_CONF(achan->id)); 531 532 writel(ADM_CH_RSLT_CONF_IRQ_EN | ADM_CH_RSLT_CONF_FLUSH_EN, 533 adev->regs + ADM_CH_RSLT_CONF(achan->id, adev->ee)); 534 535 achan->initialized = 1; 536 } 537 538 /* set the crci block size if this transaction requires CRCI */ 539 if (async_desc->crci) { 540 writel(async_desc->mux | async_desc->blk_size, 541 adev->regs + ADM_CRCI_CTL(async_desc->crci, adev->ee)); 542 } 543 544 /* make sure IRQ enable doesn't get reordered */ 545 wmb(); 546 547 /* write next command list out to the CMD FIFO */ 548 writel(ALIGN(async_desc->dma_addr, ADM_DESC_ALIGN) >> 3, 549 adev->regs + ADM_CH_CMD_PTR(achan->id, adev->ee)); 550 } 551 552 /** 553 * adm_dma_irq - irq handler for ADM controller 554 * @irq: IRQ of interrupt 555 * @data: callback data 556 * 557 * IRQ handler for the bam controller 558 */ 559 static irqreturn_t adm_dma_irq(int irq, void *data) 560 { 561 struct adm_device *adev = data; 562 u32 srcs, i; 563 struct adm_async_desc *async_desc; 564 unsigned long flags; 565 566 srcs = readl_relaxed(adev->regs + 567 ADM_SEC_DOMAIN_IRQ_STATUS(adev->ee)); 568 569 for (i = 0; i < ADM_MAX_CHANNELS; i++) { 570 struct adm_chan *achan = &adev->channels[i]; 571 u32 status, result; 572 573 if (srcs & BIT(i)) { 574 status = readl_relaxed(adev->regs + 575 ADM_CH_STATUS_SD(i, adev->ee)); 576 577 /* if no result present, skip */ 578 if (!(status & ADM_CH_STATUS_VALID)) 579 continue; 580 581 result = readl_relaxed(adev->regs + 582 ADM_CH_RSLT(i, adev->ee)); 583 584 /* no valid results, skip */ 585 if (!(result & ADM_CH_RSLT_VALID)) 586 continue; 587 588 /* flag error if transaction was flushed or failed */ 589 if (result & (ADM_CH_RSLT_ERR | ADM_CH_RSLT_FLUSH)) 590 achan->error = 1; 591 592 spin_lock_irqsave(&achan->vc.lock, flags); 593 async_desc = achan->curr_txd; 594 595 achan->curr_txd = NULL; 596 597 if (async_desc) { 598 vchan_cookie_complete(&async_desc->vd); 599 600 /* kick off next DMA */ 601 adm_start_dma(achan); 602 } 603 604 spin_unlock_irqrestore(&achan->vc.lock, flags); 605 } 606 } 607 608 return IRQ_HANDLED; 609 } 610 611 /** 612 * adm_tx_status - returns status of transaction 613 * @chan: dma channel 614 * @cookie: transaction cookie 615 * @txstate: DMA transaction state 616 * 617 * Return status of dma transaction 618 */ 619 static enum dma_status adm_tx_status(struct dma_chan *chan, dma_cookie_t cookie, 620 struct dma_tx_state *txstate) 621 { 622 struct adm_chan *achan = to_adm_chan(chan); 623 struct virt_dma_desc *vd; 624 enum dma_status ret; 625 unsigned long flags; 626 size_t residue = 0; 627 628 ret = dma_cookie_status(chan, cookie, txstate); 629 if (ret == DMA_COMPLETE || !txstate) 630 return ret; 631 632 spin_lock_irqsave(&achan->vc.lock, flags); 633 634 vd = vchan_find_desc(&achan->vc, cookie); 635 if (vd) 636 residue = container_of(vd, struct adm_async_desc, vd)->length; 637 638 spin_unlock_irqrestore(&achan->vc.lock, flags); 639 640 /* 641 * residue is either the full length if it is in the issued list, or 0 642 * if it is in progress. We have no reliable way of determining 643 * anything inbetween 644 */ 645 dma_set_residue(txstate, residue); 646 647 if (achan->error) 648 return DMA_ERROR; 649 650 return ret; 651 } 652 653 /** 654 * adm_issue_pending - starts pending transactions 655 * @chan: dma channel 656 * 657 * Issues all pending transactions and starts DMA 658 */ 659 static void adm_issue_pending(struct dma_chan *chan) 660 { 661 struct adm_chan *achan = to_adm_chan(chan); 662 unsigned long flags; 663 664 spin_lock_irqsave(&achan->vc.lock, flags); 665 666 if (vchan_issue_pending(&achan->vc) && !achan->curr_txd) 667 adm_start_dma(achan); 668 spin_unlock_irqrestore(&achan->vc.lock, flags); 669 } 670 671 /** 672 * adm_dma_free_desc - free descriptor memory 673 * @vd: virtual descriptor 674 * 675 */ 676 static void adm_dma_free_desc(struct virt_dma_desc *vd) 677 { 678 struct adm_async_desc *async_desc = container_of(vd, 679 struct adm_async_desc, vd); 680 681 dma_unmap_single(async_desc->adev->dev, async_desc->dma_addr, 682 async_desc->dma_len, DMA_TO_DEVICE); 683 kfree(async_desc->cpl); 684 kfree(async_desc); 685 } 686 687 static void adm_channel_init(struct adm_device *adev, struct adm_chan *achan, 688 u32 index) 689 { 690 achan->id = index; 691 achan->adev = adev; 692 693 vchan_init(&achan->vc, &adev->common); 694 achan->vc.desc_free = adm_dma_free_desc; 695 } 696 697 static int adm_dma_probe(struct platform_device *pdev) 698 { 699 struct adm_device *adev; 700 int ret; 701 u32 i; 702 703 adev = devm_kzalloc(&pdev->dev, sizeof(*adev), GFP_KERNEL); 704 if (!adev) 705 return -ENOMEM; 706 707 adev->dev = &pdev->dev; 708 709 adev->regs = devm_platform_ioremap_resource(pdev, 0); 710 if (IS_ERR(adev->regs)) 711 return PTR_ERR(adev->regs); 712 713 adev->irq = platform_get_irq(pdev, 0); 714 if (adev->irq < 0) 715 return adev->irq; 716 717 ret = of_property_read_u32(pdev->dev.of_node, "qcom,ee", &adev->ee); 718 if (ret) { 719 dev_err(adev->dev, "Execution environment unspecified\n"); 720 return ret; 721 } 722 723 adev->core_clk = devm_clk_get(adev->dev, "core"); 724 if (IS_ERR(adev->core_clk)) 725 return PTR_ERR(adev->core_clk); 726 727 adev->iface_clk = devm_clk_get(adev->dev, "iface"); 728 if (IS_ERR(adev->iface_clk)) 729 return PTR_ERR(adev->iface_clk); 730 731 adev->clk_reset = devm_reset_control_get_exclusive(&pdev->dev, "clk"); 732 if (IS_ERR(adev->clk_reset)) { 733 dev_err(adev->dev, "failed to get ADM0 reset\n"); 734 return PTR_ERR(adev->clk_reset); 735 } 736 737 adev->c0_reset = devm_reset_control_get_exclusive(&pdev->dev, "c0"); 738 if (IS_ERR(adev->c0_reset)) { 739 dev_err(adev->dev, "failed to get ADM0 C0 reset\n"); 740 return PTR_ERR(adev->c0_reset); 741 } 742 743 adev->c1_reset = devm_reset_control_get_exclusive(&pdev->dev, "c1"); 744 if (IS_ERR(adev->c1_reset)) { 745 dev_err(adev->dev, "failed to get ADM0 C1 reset\n"); 746 return PTR_ERR(adev->c1_reset); 747 } 748 749 adev->c2_reset = devm_reset_control_get_exclusive(&pdev->dev, "c2"); 750 if (IS_ERR(adev->c2_reset)) { 751 dev_err(adev->dev, "failed to get ADM0 C2 reset\n"); 752 return PTR_ERR(adev->c2_reset); 753 } 754 755 ret = clk_prepare_enable(adev->core_clk); 756 if (ret) { 757 dev_err(adev->dev, "failed to prepare/enable core clock\n"); 758 return ret; 759 } 760 761 ret = clk_prepare_enable(adev->iface_clk); 762 if (ret) { 763 dev_err(adev->dev, "failed to prepare/enable iface clock\n"); 764 goto err_disable_core_clk; 765 } 766 767 reset_control_assert(adev->clk_reset); 768 reset_control_assert(adev->c0_reset); 769 reset_control_assert(adev->c1_reset); 770 reset_control_assert(adev->c2_reset); 771 772 udelay(2); 773 774 reset_control_deassert(adev->clk_reset); 775 reset_control_deassert(adev->c0_reset); 776 reset_control_deassert(adev->c1_reset); 777 reset_control_deassert(adev->c2_reset); 778 779 adev->channels = devm_kcalloc(adev->dev, ADM_MAX_CHANNELS, 780 sizeof(*adev->channels), GFP_KERNEL); 781 782 if (!adev->channels) { 783 ret = -ENOMEM; 784 goto err_disable_clks; 785 } 786 787 /* allocate and initialize channels */ 788 INIT_LIST_HEAD(&adev->common.channels); 789 790 for (i = 0; i < ADM_MAX_CHANNELS; i++) 791 adm_channel_init(adev, &adev->channels[i], i); 792 793 /* reset CRCIs */ 794 for (i = 0; i < 16; i++) 795 writel(ADM_CRCI_CTL_RST, adev->regs + 796 ADM_CRCI_CTL(i, adev->ee)); 797 798 /* configure client interfaces */ 799 writel(ADM_CI_RANGE_START(0x40) | ADM_CI_RANGE_END(0xb0) | 800 ADM_CI_BURST_8_WORDS, adev->regs + ADM_CI_CONF(0)); 801 writel(ADM_CI_RANGE_START(0x2a) | ADM_CI_RANGE_END(0x2c) | 802 ADM_CI_BURST_8_WORDS, adev->regs + ADM_CI_CONF(1)); 803 writel(ADM_CI_RANGE_START(0x12) | ADM_CI_RANGE_END(0x28) | 804 ADM_CI_BURST_8_WORDS, adev->regs + ADM_CI_CONF(2)); 805 writel(ADM_GP_CTL_LP_EN | ADM_GP_CTL_LP_CNT(0xf), 806 adev->regs + ADM_GP_CTL); 807 808 ret = devm_request_irq(adev->dev, adev->irq, adm_dma_irq, 809 0, "adm_dma", adev); 810 if (ret) 811 goto err_disable_clks; 812 813 platform_set_drvdata(pdev, adev); 814 815 adev->common.dev = adev->dev; 816 adev->common.dev->dma_parms = &adev->dma_parms; 817 818 /* set capabilities */ 819 dma_cap_zero(adev->common.cap_mask); 820 dma_cap_set(DMA_SLAVE, adev->common.cap_mask); 821 dma_cap_set(DMA_PRIVATE, adev->common.cap_mask); 822 823 /* initialize dmaengine apis */ 824 adev->common.directions = BIT(DMA_DEV_TO_MEM | DMA_MEM_TO_DEV); 825 adev->common.residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR; 826 adev->common.src_addr_widths = DMA_SLAVE_BUSWIDTH_4_BYTES; 827 adev->common.dst_addr_widths = DMA_SLAVE_BUSWIDTH_4_BYTES; 828 adev->common.device_free_chan_resources = adm_free_chan; 829 adev->common.device_prep_slave_sg = adm_prep_slave_sg; 830 adev->common.device_issue_pending = adm_issue_pending; 831 adev->common.device_tx_status = adm_tx_status; 832 adev->common.device_terminate_all = adm_terminate_all; 833 adev->common.device_config = adm_slave_config; 834 835 ret = dma_async_device_register(&adev->common); 836 if (ret) { 837 dev_err(adev->dev, "failed to register dma async device\n"); 838 goto err_disable_clks; 839 } 840 841 ret = of_dma_controller_register(pdev->dev.of_node, 842 of_dma_xlate_by_chan_id, 843 &adev->common); 844 if (ret) 845 goto err_unregister_dma; 846 847 return 0; 848 849 err_unregister_dma: 850 dma_async_device_unregister(&adev->common); 851 err_disable_clks: 852 clk_disable_unprepare(adev->iface_clk); 853 err_disable_core_clk: 854 clk_disable_unprepare(adev->core_clk); 855 856 return ret; 857 } 858 859 static int adm_dma_remove(struct platform_device *pdev) 860 { 861 struct adm_device *adev = platform_get_drvdata(pdev); 862 struct adm_chan *achan; 863 u32 i; 864 865 of_dma_controller_free(pdev->dev.of_node); 866 dma_async_device_unregister(&adev->common); 867 868 for (i = 0; i < ADM_MAX_CHANNELS; i++) { 869 achan = &adev->channels[i]; 870 871 /* mask IRQs for this channel/EE pair */ 872 writel(0, adev->regs + ADM_CH_RSLT_CONF(achan->id, adev->ee)); 873 874 tasklet_kill(&adev->channels[i].vc.task); 875 adm_terminate_all(&adev->channels[i].vc.chan); 876 } 877 878 devm_free_irq(adev->dev, adev->irq, adev); 879 880 clk_disable_unprepare(adev->core_clk); 881 clk_disable_unprepare(adev->iface_clk); 882 883 return 0; 884 } 885 886 static const struct of_device_id adm_of_match[] = { 887 { .compatible = "qcom,adm", }, 888 {} 889 }; 890 MODULE_DEVICE_TABLE(of, adm_of_match); 891 892 static struct platform_driver adm_dma_driver = { 893 .probe = adm_dma_probe, 894 .remove = adm_dma_remove, 895 .driver = { 896 .name = "adm-dma-engine", 897 .of_match_table = adm_of_match, 898 }, 899 }; 900 901 module_platform_driver(adm_dma_driver); 902 903 MODULE_AUTHOR("Andy Gross <agross@codeaurora.org>"); 904 MODULE_DESCRIPTION("QCOM ADM DMA engine driver"); 905 MODULE_LICENSE("GPL v2"); 906