1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Driver for the Analog Devices AXI-DMAC core 4 * 5 * Copyright 2013-2019 Analog Devices Inc. 6 * Author: Lars-Peter Clausen <lars@metafoo.de> 7 */ 8 9 #include <linux/bitfield.h> 10 #include <linux/clk.h> 11 #include <linux/device.h> 12 #include <linux/dma-mapping.h> 13 #include <linux/dmaengine.h> 14 #include <linux/err.h> 15 #include <linux/interrupt.h> 16 #include <linux/io.h> 17 #include <linux/kernel.h> 18 #include <linux/module.h> 19 #include <linux/of.h> 20 #include <linux/of_dma.h> 21 #include <linux/of_address.h> 22 #include <linux/platform_device.h> 23 #include <linux/regmap.h> 24 #include <linux/slab.h> 25 #include <linux/fpga/adi-axi-common.h> 26 27 #include <dt-bindings/dma/axi-dmac.h> 28 29 #include "dmaengine.h" 30 #include "virt-dma.h" 31 32 /* 33 * The AXI-DMAC is a soft IP core that is used in FPGA designs. The core has 34 * various instantiation parameters which decided the exact feature set support 35 * by the core. 36 * 37 * Each channel of the core has a source interface and a destination interface. 38 * The number of channels and the type of the channel interfaces is selected at 39 * configuration time. A interface can either be a connected to a central memory 40 * interconnect, which allows access to system memory, or it can be connected to 41 * a dedicated bus which is directly connected to a data port on a peripheral. 42 * Given that those are configuration options of the core that are selected when 43 * it is instantiated this means that they can not be changed by software at 44 * runtime. By extension this means that each channel is uni-directional. It can 45 * either be device to memory or memory to device, but not both. Also since the 46 * device side is a dedicated data bus only connected to a single peripheral 47 * there is no address than can or needs to be configured for the device side. 48 */ 49 50 #define AXI_DMAC_REG_INTERFACE_DESC 0x10 51 #define AXI_DMAC_DMA_SRC_TYPE_MSK GENMASK(13, 12) 52 #define AXI_DMAC_DMA_SRC_TYPE_GET(x) FIELD_GET(AXI_DMAC_DMA_SRC_TYPE_MSK, x) 53 #define AXI_DMAC_DMA_SRC_WIDTH_MSK GENMASK(11, 8) 54 #define AXI_DMAC_DMA_SRC_WIDTH_GET(x) FIELD_GET(AXI_DMAC_DMA_SRC_WIDTH_MSK, x) 55 #define AXI_DMAC_DMA_DST_TYPE_MSK GENMASK(5, 4) 56 #define AXI_DMAC_DMA_DST_TYPE_GET(x) FIELD_GET(AXI_DMAC_DMA_DST_TYPE_MSK, x) 57 #define AXI_DMAC_DMA_DST_WIDTH_MSK GENMASK(3, 0) 58 #define AXI_DMAC_DMA_DST_WIDTH_GET(x) FIELD_GET(AXI_DMAC_DMA_DST_WIDTH_MSK, x) 59 #define AXI_DMAC_REG_COHERENCY_DESC 0x14 60 #define AXI_DMAC_DST_COHERENT_MSK BIT(0) 61 #define AXI_DMAC_DST_COHERENT_GET(x) FIELD_GET(AXI_DMAC_DST_COHERENT_MSK, x) 62 63 #define AXI_DMAC_REG_IRQ_MASK 0x80 64 #define AXI_DMAC_REG_IRQ_PENDING 0x84 65 #define AXI_DMAC_REG_IRQ_SOURCE 0x88 66 67 #define AXI_DMAC_REG_CTRL 0x400 68 #define AXI_DMAC_REG_TRANSFER_ID 0x404 69 #define AXI_DMAC_REG_START_TRANSFER 0x408 70 #define AXI_DMAC_REG_FLAGS 0x40c 71 #define AXI_DMAC_REG_DEST_ADDRESS 0x410 72 #define AXI_DMAC_REG_SRC_ADDRESS 0x414 73 #define AXI_DMAC_REG_X_LENGTH 0x418 74 #define AXI_DMAC_REG_Y_LENGTH 0x41c 75 #define AXI_DMAC_REG_DEST_STRIDE 0x420 76 #define AXI_DMAC_REG_SRC_STRIDE 0x424 77 #define AXI_DMAC_REG_TRANSFER_DONE 0x428 78 #define AXI_DMAC_REG_ACTIVE_TRANSFER_ID 0x42c 79 #define AXI_DMAC_REG_STATUS 0x430 80 #define AXI_DMAC_REG_CURRENT_SRC_ADDR 0x434 81 #define AXI_DMAC_REG_CURRENT_DEST_ADDR 0x438 82 #define AXI_DMAC_REG_PARTIAL_XFER_LEN 0x44c 83 #define AXI_DMAC_REG_PARTIAL_XFER_ID 0x450 84 85 #define AXI_DMAC_CTRL_ENABLE BIT(0) 86 #define AXI_DMAC_CTRL_PAUSE BIT(1) 87 88 #define AXI_DMAC_IRQ_SOT BIT(0) 89 #define AXI_DMAC_IRQ_EOT BIT(1) 90 91 #define AXI_DMAC_FLAG_CYCLIC BIT(0) 92 #define AXI_DMAC_FLAG_LAST BIT(1) 93 #define AXI_DMAC_FLAG_PARTIAL_REPORT BIT(2) 94 95 #define AXI_DMAC_FLAG_PARTIAL_XFER_DONE BIT(31) 96 97 /* The maximum ID allocated by the hardware is 31 */ 98 #define AXI_DMAC_SG_UNUSED 32U 99 100 struct axi_dmac_sg { 101 dma_addr_t src_addr; 102 dma_addr_t dest_addr; 103 unsigned int x_len; 104 unsigned int y_len; 105 unsigned int dest_stride; 106 unsigned int src_stride; 107 unsigned int id; 108 unsigned int partial_len; 109 bool schedule_when_free; 110 }; 111 112 struct axi_dmac_desc { 113 struct virt_dma_desc vdesc; 114 bool cyclic; 115 bool have_partial_xfer; 116 117 unsigned int num_submitted; 118 unsigned int num_completed; 119 unsigned int num_sgs; 120 struct axi_dmac_sg sg[]; 121 }; 122 123 struct axi_dmac_chan { 124 struct virt_dma_chan vchan; 125 126 struct axi_dmac_desc *next_desc; 127 struct list_head active_descs; 128 enum dma_transfer_direction direction; 129 130 unsigned int src_width; 131 unsigned int dest_width; 132 unsigned int src_type; 133 unsigned int dest_type; 134 135 unsigned int max_length; 136 unsigned int address_align_mask; 137 unsigned int length_align_mask; 138 139 bool hw_partial_xfer; 140 bool hw_cyclic; 141 bool hw_2d; 142 }; 143 144 struct axi_dmac { 145 void __iomem *base; 146 int irq; 147 148 struct clk *clk; 149 150 struct dma_device dma_dev; 151 struct axi_dmac_chan chan; 152 }; 153 154 static struct axi_dmac *chan_to_axi_dmac(struct axi_dmac_chan *chan) 155 { 156 return container_of(chan->vchan.chan.device, struct axi_dmac, 157 dma_dev); 158 } 159 160 static struct axi_dmac_chan *to_axi_dmac_chan(struct dma_chan *c) 161 { 162 return container_of(c, struct axi_dmac_chan, vchan.chan); 163 } 164 165 static struct axi_dmac_desc *to_axi_dmac_desc(struct virt_dma_desc *vdesc) 166 { 167 return container_of(vdesc, struct axi_dmac_desc, vdesc); 168 } 169 170 static void axi_dmac_write(struct axi_dmac *axi_dmac, unsigned int reg, 171 unsigned int val) 172 { 173 writel(val, axi_dmac->base + reg); 174 } 175 176 static int axi_dmac_read(struct axi_dmac *axi_dmac, unsigned int reg) 177 { 178 return readl(axi_dmac->base + reg); 179 } 180 181 static int axi_dmac_src_is_mem(struct axi_dmac_chan *chan) 182 { 183 return chan->src_type == AXI_DMAC_BUS_TYPE_AXI_MM; 184 } 185 186 static int axi_dmac_dest_is_mem(struct axi_dmac_chan *chan) 187 { 188 return chan->dest_type == AXI_DMAC_BUS_TYPE_AXI_MM; 189 } 190 191 static bool axi_dmac_check_len(struct axi_dmac_chan *chan, unsigned int len) 192 { 193 if (len == 0) 194 return false; 195 if ((len & chan->length_align_mask) != 0) /* Not aligned */ 196 return false; 197 return true; 198 } 199 200 static bool axi_dmac_check_addr(struct axi_dmac_chan *chan, dma_addr_t addr) 201 { 202 if ((addr & chan->address_align_mask) != 0) /* Not aligned */ 203 return false; 204 return true; 205 } 206 207 static void axi_dmac_start_transfer(struct axi_dmac_chan *chan) 208 { 209 struct axi_dmac *dmac = chan_to_axi_dmac(chan); 210 struct virt_dma_desc *vdesc; 211 struct axi_dmac_desc *desc; 212 struct axi_dmac_sg *sg; 213 unsigned int flags = 0; 214 unsigned int val; 215 216 val = axi_dmac_read(dmac, AXI_DMAC_REG_START_TRANSFER); 217 if (val) /* Queue is full, wait for the next SOT IRQ */ 218 return; 219 220 desc = chan->next_desc; 221 222 if (!desc) { 223 vdesc = vchan_next_desc(&chan->vchan); 224 if (!vdesc) 225 return; 226 list_move_tail(&vdesc->node, &chan->active_descs); 227 desc = to_axi_dmac_desc(vdesc); 228 } 229 sg = &desc->sg[desc->num_submitted]; 230 231 /* Already queued in cyclic mode. Wait for it to finish */ 232 if (sg->id != AXI_DMAC_SG_UNUSED) { 233 sg->schedule_when_free = true; 234 return; 235 } 236 237 desc->num_submitted++; 238 if (desc->num_submitted == desc->num_sgs || 239 desc->have_partial_xfer) { 240 if (desc->cyclic) 241 desc->num_submitted = 0; /* Start again */ 242 else 243 chan->next_desc = NULL; 244 flags |= AXI_DMAC_FLAG_LAST; 245 } else { 246 chan->next_desc = desc; 247 } 248 249 sg->id = axi_dmac_read(dmac, AXI_DMAC_REG_TRANSFER_ID); 250 251 if (axi_dmac_dest_is_mem(chan)) { 252 axi_dmac_write(dmac, AXI_DMAC_REG_DEST_ADDRESS, sg->dest_addr); 253 axi_dmac_write(dmac, AXI_DMAC_REG_DEST_STRIDE, sg->dest_stride); 254 } 255 256 if (axi_dmac_src_is_mem(chan)) { 257 axi_dmac_write(dmac, AXI_DMAC_REG_SRC_ADDRESS, sg->src_addr); 258 axi_dmac_write(dmac, AXI_DMAC_REG_SRC_STRIDE, sg->src_stride); 259 } 260 261 /* 262 * If the hardware supports cyclic transfers and there is no callback to 263 * call and only a single segment, enable hw cyclic mode to avoid 264 * unnecessary interrupts. 265 */ 266 if (chan->hw_cyclic && desc->cyclic && !desc->vdesc.tx.callback && 267 desc->num_sgs == 1) 268 flags |= AXI_DMAC_FLAG_CYCLIC; 269 270 if (chan->hw_partial_xfer) 271 flags |= AXI_DMAC_FLAG_PARTIAL_REPORT; 272 273 axi_dmac_write(dmac, AXI_DMAC_REG_X_LENGTH, sg->x_len - 1); 274 axi_dmac_write(dmac, AXI_DMAC_REG_Y_LENGTH, sg->y_len - 1); 275 axi_dmac_write(dmac, AXI_DMAC_REG_FLAGS, flags); 276 axi_dmac_write(dmac, AXI_DMAC_REG_START_TRANSFER, 1); 277 } 278 279 static struct axi_dmac_desc *axi_dmac_active_desc(struct axi_dmac_chan *chan) 280 { 281 return list_first_entry_or_null(&chan->active_descs, 282 struct axi_dmac_desc, vdesc.node); 283 } 284 285 static inline unsigned int axi_dmac_total_sg_bytes(struct axi_dmac_chan *chan, 286 struct axi_dmac_sg *sg) 287 { 288 if (chan->hw_2d) 289 return sg->x_len * sg->y_len; 290 else 291 return sg->x_len; 292 } 293 294 static void axi_dmac_dequeue_partial_xfers(struct axi_dmac_chan *chan) 295 { 296 struct axi_dmac *dmac = chan_to_axi_dmac(chan); 297 struct axi_dmac_desc *desc; 298 struct axi_dmac_sg *sg; 299 u32 xfer_done, len, id, i; 300 bool found_sg; 301 302 do { 303 len = axi_dmac_read(dmac, AXI_DMAC_REG_PARTIAL_XFER_LEN); 304 id = axi_dmac_read(dmac, AXI_DMAC_REG_PARTIAL_XFER_ID); 305 306 found_sg = false; 307 list_for_each_entry(desc, &chan->active_descs, vdesc.node) { 308 for (i = 0; i < desc->num_sgs; i++) { 309 sg = &desc->sg[i]; 310 if (sg->id == AXI_DMAC_SG_UNUSED) 311 continue; 312 if (sg->id == id) { 313 desc->have_partial_xfer = true; 314 sg->partial_len = len; 315 found_sg = true; 316 break; 317 } 318 } 319 if (found_sg) 320 break; 321 } 322 323 if (found_sg) { 324 dev_dbg(dmac->dma_dev.dev, 325 "Found partial segment id=%u, len=%u\n", 326 id, len); 327 } else { 328 dev_warn(dmac->dma_dev.dev, 329 "Not found partial segment id=%u, len=%u\n", 330 id, len); 331 } 332 333 /* Check if we have any more partial transfers */ 334 xfer_done = axi_dmac_read(dmac, AXI_DMAC_REG_TRANSFER_DONE); 335 xfer_done = !(xfer_done & AXI_DMAC_FLAG_PARTIAL_XFER_DONE); 336 337 } while (!xfer_done); 338 } 339 340 static void axi_dmac_compute_residue(struct axi_dmac_chan *chan, 341 struct axi_dmac_desc *active) 342 { 343 struct dmaengine_result *rslt = &active->vdesc.tx_result; 344 unsigned int start = active->num_completed - 1; 345 struct axi_dmac_sg *sg; 346 unsigned int i, total; 347 348 rslt->result = DMA_TRANS_NOERROR; 349 rslt->residue = 0; 350 351 /* 352 * We get here if the last completed segment is partial, which 353 * means we can compute the residue from that segment onwards 354 */ 355 for (i = start; i < active->num_sgs; i++) { 356 sg = &active->sg[i]; 357 total = axi_dmac_total_sg_bytes(chan, sg); 358 rslt->residue += (total - sg->partial_len); 359 } 360 } 361 362 static bool axi_dmac_transfer_done(struct axi_dmac_chan *chan, 363 unsigned int completed_transfers) 364 { 365 struct axi_dmac_desc *active; 366 struct axi_dmac_sg *sg; 367 bool start_next = false; 368 369 active = axi_dmac_active_desc(chan); 370 if (!active) 371 return false; 372 373 if (chan->hw_partial_xfer && 374 (completed_transfers & AXI_DMAC_FLAG_PARTIAL_XFER_DONE)) 375 axi_dmac_dequeue_partial_xfers(chan); 376 377 do { 378 sg = &active->sg[active->num_completed]; 379 if (sg->id == AXI_DMAC_SG_UNUSED) /* Not yet submitted */ 380 break; 381 if (!(BIT(sg->id) & completed_transfers)) 382 break; 383 active->num_completed++; 384 sg->id = AXI_DMAC_SG_UNUSED; 385 if (sg->schedule_when_free) { 386 sg->schedule_when_free = false; 387 start_next = true; 388 } 389 390 if (sg->partial_len) 391 axi_dmac_compute_residue(chan, active); 392 393 if (active->cyclic) 394 vchan_cyclic_callback(&active->vdesc); 395 396 if (active->num_completed == active->num_sgs || 397 sg->partial_len) { 398 if (active->cyclic) { 399 active->num_completed = 0; /* wrap around */ 400 } else { 401 list_del(&active->vdesc.node); 402 vchan_cookie_complete(&active->vdesc); 403 active = axi_dmac_active_desc(chan); 404 } 405 } 406 } while (active); 407 408 return start_next; 409 } 410 411 static irqreturn_t axi_dmac_interrupt_handler(int irq, void *devid) 412 { 413 struct axi_dmac *dmac = devid; 414 unsigned int pending; 415 bool start_next = false; 416 417 pending = axi_dmac_read(dmac, AXI_DMAC_REG_IRQ_PENDING); 418 if (!pending) 419 return IRQ_NONE; 420 421 axi_dmac_write(dmac, AXI_DMAC_REG_IRQ_PENDING, pending); 422 423 spin_lock(&dmac->chan.vchan.lock); 424 /* One or more transfers have finished */ 425 if (pending & AXI_DMAC_IRQ_EOT) { 426 unsigned int completed; 427 428 completed = axi_dmac_read(dmac, AXI_DMAC_REG_TRANSFER_DONE); 429 start_next = axi_dmac_transfer_done(&dmac->chan, completed); 430 } 431 /* Space has become available in the descriptor queue */ 432 if ((pending & AXI_DMAC_IRQ_SOT) || start_next) 433 axi_dmac_start_transfer(&dmac->chan); 434 spin_unlock(&dmac->chan.vchan.lock); 435 436 return IRQ_HANDLED; 437 } 438 439 static int axi_dmac_terminate_all(struct dma_chan *c) 440 { 441 struct axi_dmac_chan *chan = to_axi_dmac_chan(c); 442 struct axi_dmac *dmac = chan_to_axi_dmac(chan); 443 unsigned long flags; 444 LIST_HEAD(head); 445 446 spin_lock_irqsave(&chan->vchan.lock, flags); 447 axi_dmac_write(dmac, AXI_DMAC_REG_CTRL, 0); 448 chan->next_desc = NULL; 449 vchan_get_all_descriptors(&chan->vchan, &head); 450 list_splice_tail_init(&chan->active_descs, &head); 451 spin_unlock_irqrestore(&chan->vchan.lock, flags); 452 453 vchan_dma_desc_free_list(&chan->vchan, &head); 454 455 return 0; 456 } 457 458 static void axi_dmac_synchronize(struct dma_chan *c) 459 { 460 struct axi_dmac_chan *chan = to_axi_dmac_chan(c); 461 462 vchan_synchronize(&chan->vchan); 463 } 464 465 static void axi_dmac_issue_pending(struct dma_chan *c) 466 { 467 struct axi_dmac_chan *chan = to_axi_dmac_chan(c); 468 struct axi_dmac *dmac = chan_to_axi_dmac(chan); 469 unsigned long flags; 470 471 axi_dmac_write(dmac, AXI_DMAC_REG_CTRL, AXI_DMAC_CTRL_ENABLE); 472 473 spin_lock_irqsave(&chan->vchan.lock, flags); 474 if (vchan_issue_pending(&chan->vchan)) 475 axi_dmac_start_transfer(chan); 476 spin_unlock_irqrestore(&chan->vchan.lock, flags); 477 } 478 479 static struct axi_dmac_desc *axi_dmac_alloc_desc(unsigned int num_sgs) 480 { 481 struct axi_dmac_desc *desc; 482 unsigned int i; 483 484 desc = kzalloc(struct_size(desc, sg, num_sgs), GFP_NOWAIT); 485 if (!desc) 486 return NULL; 487 488 for (i = 0; i < num_sgs; i++) 489 desc->sg[i].id = AXI_DMAC_SG_UNUSED; 490 491 desc->num_sgs = num_sgs; 492 493 return desc; 494 } 495 496 static struct axi_dmac_sg *axi_dmac_fill_linear_sg(struct axi_dmac_chan *chan, 497 enum dma_transfer_direction direction, dma_addr_t addr, 498 unsigned int num_periods, unsigned int period_len, 499 struct axi_dmac_sg *sg) 500 { 501 unsigned int num_segments, i; 502 unsigned int segment_size; 503 unsigned int len; 504 505 /* Split into multiple equally sized segments if necessary */ 506 num_segments = DIV_ROUND_UP(period_len, chan->max_length); 507 segment_size = DIV_ROUND_UP(period_len, num_segments); 508 /* Take care of alignment */ 509 segment_size = ((segment_size - 1) | chan->length_align_mask) + 1; 510 511 for (i = 0; i < num_periods; i++) { 512 len = period_len; 513 514 while (len > segment_size) { 515 if (direction == DMA_DEV_TO_MEM) 516 sg->dest_addr = addr; 517 else 518 sg->src_addr = addr; 519 sg->x_len = segment_size; 520 sg->y_len = 1; 521 sg++; 522 addr += segment_size; 523 len -= segment_size; 524 } 525 526 if (direction == DMA_DEV_TO_MEM) 527 sg->dest_addr = addr; 528 else 529 sg->src_addr = addr; 530 sg->x_len = len; 531 sg->y_len = 1; 532 sg++; 533 addr += len; 534 } 535 536 return sg; 537 } 538 539 static struct dma_async_tx_descriptor *axi_dmac_prep_slave_sg( 540 struct dma_chan *c, struct scatterlist *sgl, 541 unsigned int sg_len, enum dma_transfer_direction direction, 542 unsigned long flags, void *context) 543 { 544 struct axi_dmac_chan *chan = to_axi_dmac_chan(c); 545 struct axi_dmac_desc *desc; 546 struct axi_dmac_sg *dsg; 547 struct scatterlist *sg; 548 unsigned int num_sgs; 549 unsigned int i; 550 551 if (direction != chan->direction) 552 return NULL; 553 554 num_sgs = 0; 555 for_each_sg(sgl, sg, sg_len, i) 556 num_sgs += DIV_ROUND_UP(sg_dma_len(sg), chan->max_length); 557 558 desc = axi_dmac_alloc_desc(num_sgs); 559 if (!desc) 560 return NULL; 561 562 dsg = desc->sg; 563 564 for_each_sg(sgl, sg, sg_len, i) { 565 if (!axi_dmac_check_addr(chan, sg_dma_address(sg)) || 566 !axi_dmac_check_len(chan, sg_dma_len(sg))) { 567 kfree(desc); 568 return NULL; 569 } 570 571 dsg = axi_dmac_fill_linear_sg(chan, direction, sg_dma_address(sg), 1, 572 sg_dma_len(sg), dsg); 573 } 574 575 desc->cyclic = false; 576 577 return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags); 578 } 579 580 static struct dma_async_tx_descriptor *axi_dmac_prep_dma_cyclic( 581 struct dma_chan *c, dma_addr_t buf_addr, size_t buf_len, 582 size_t period_len, enum dma_transfer_direction direction, 583 unsigned long flags) 584 { 585 struct axi_dmac_chan *chan = to_axi_dmac_chan(c); 586 struct axi_dmac_desc *desc; 587 unsigned int num_periods, num_segments; 588 589 if (direction != chan->direction) 590 return NULL; 591 592 if (!axi_dmac_check_len(chan, buf_len) || 593 !axi_dmac_check_addr(chan, buf_addr)) 594 return NULL; 595 596 if (period_len == 0 || buf_len % period_len) 597 return NULL; 598 599 num_periods = buf_len / period_len; 600 num_segments = DIV_ROUND_UP(period_len, chan->max_length); 601 602 desc = axi_dmac_alloc_desc(num_periods * num_segments); 603 if (!desc) 604 return NULL; 605 606 axi_dmac_fill_linear_sg(chan, direction, buf_addr, num_periods, 607 period_len, desc->sg); 608 609 desc->cyclic = true; 610 611 return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags); 612 } 613 614 static struct dma_async_tx_descriptor *axi_dmac_prep_interleaved( 615 struct dma_chan *c, struct dma_interleaved_template *xt, 616 unsigned long flags) 617 { 618 struct axi_dmac_chan *chan = to_axi_dmac_chan(c); 619 struct axi_dmac_desc *desc; 620 size_t dst_icg, src_icg; 621 622 if (xt->frame_size != 1) 623 return NULL; 624 625 if (xt->dir != chan->direction) 626 return NULL; 627 628 if (axi_dmac_src_is_mem(chan)) { 629 if (!xt->src_inc || !axi_dmac_check_addr(chan, xt->src_start)) 630 return NULL; 631 } 632 633 if (axi_dmac_dest_is_mem(chan)) { 634 if (!xt->dst_inc || !axi_dmac_check_addr(chan, xt->dst_start)) 635 return NULL; 636 } 637 638 dst_icg = dmaengine_get_dst_icg(xt, &xt->sgl[0]); 639 src_icg = dmaengine_get_src_icg(xt, &xt->sgl[0]); 640 641 if (chan->hw_2d) { 642 if (!axi_dmac_check_len(chan, xt->sgl[0].size) || 643 xt->numf == 0) 644 return NULL; 645 if (xt->sgl[0].size + dst_icg > chan->max_length || 646 xt->sgl[0].size + src_icg > chan->max_length) 647 return NULL; 648 } else { 649 if (dst_icg != 0 || src_icg != 0) 650 return NULL; 651 if (chan->max_length / xt->sgl[0].size < xt->numf) 652 return NULL; 653 if (!axi_dmac_check_len(chan, xt->sgl[0].size * xt->numf)) 654 return NULL; 655 } 656 657 desc = axi_dmac_alloc_desc(1); 658 if (!desc) 659 return NULL; 660 661 if (axi_dmac_src_is_mem(chan)) { 662 desc->sg[0].src_addr = xt->src_start; 663 desc->sg[0].src_stride = xt->sgl[0].size + src_icg; 664 } 665 666 if (axi_dmac_dest_is_mem(chan)) { 667 desc->sg[0].dest_addr = xt->dst_start; 668 desc->sg[0].dest_stride = xt->sgl[0].size + dst_icg; 669 } 670 671 if (chan->hw_2d) { 672 desc->sg[0].x_len = xt->sgl[0].size; 673 desc->sg[0].y_len = xt->numf; 674 } else { 675 desc->sg[0].x_len = xt->sgl[0].size * xt->numf; 676 desc->sg[0].y_len = 1; 677 } 678 679 if (flags & DMA_CYCLIC) 680 desc->cyclic = true; 681 682 return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags); 683 } 684 685 static void axi_dmac_free_chan_resources(struct dma_chan *c) 686 { 687 vchan_free_chan_resources(to_virt_chan(c)); 688 } 689 690 static void axi_dmac_desc_free(struct virt_dma_desc *vdesc) 691 { 692 kfree(container_of(vdesc, struct axi_dmac_desc, vdesc)); 693 } 694 695 static bool axi_dmac_regmap_rdwr(struct device *dev, unsigned int reg) 696 { 697 switch (reg) { 698 case AXI_DMAC_REG_IRQ_MASK: 699 case AXI_DMAC_REG_IRQ_SOURCE: 700 case AXI_DMAC_REG_IRQ_PENDING: 701 case AXI_DMAC_REG_CTRL: 702 case AXI_DMAC_REG_TRANSFER_ID: 703 case AXI_DMAC_REG_START_TRANSFER: 704 case AXI_DMAC_REG_FLAGS: 705 case AXI_DMAC_REG_DEST_ADDRESS: 706 case AXI_DMAC_REG_SRC_ADDRESS: 707 case AXI_DMAC_REG_X_LENGTH: 708 case AXI_DMAC_REG_Y_LENGTH: 709 case AXI_DMAC_REG_DEST_STRIDE: 710 case AXI_DMAC_REG_SRC_STRIDE: 711 case AXI_DMAC_REG_TRANSFER_DONE: 712 case AXI_DMAC_REG_ACTIVE_TRANSFER_ID: 713 case AXI_DMAC_REG_STATUS: 714 case AXI_DMAC_REG_CURRENT_SRC_ADDR: 715 case AXI_DMAC_REG_CURRENT_DEST_ADDR: 716 case AXI_DMAC_REG_PARTIAL_XFER_LEN: 717 case AXI_DMAC_REG_PARTIAL_XFER_ID: 718 return true; 719 default: 720 return false; 721 } 722 } 723 724 static const struct regmap_config axi_dmac_regmap_config = { 725 .reg_bits = 32, 726 .val_bits = 32, 727 .reg_stride = 4, 728 .max_register = AXI_DMAC_REG_PARTIAL_XFER_ID, 729 .readable_reg = axi_dmac_regmap_rdwr, 730 .writeable_reg = axi_dmac_regmap_rdwr, 731 }; 732 733 static void axi_dmac_adjust_chan_params(struct axi_dmac_chan *chan) 734 { 735 chan->address_align_mask = max(chan->dest_width, chan->src_width) - 1; 736 737 if (axi_dmac_dest_is_mem(chan) && axi_dmac_src_is_mem(chan)) 738 chan->direction = DMA_MEM_TO_MEM; 739 else if (!axi_dmac_dest_is_mem(chan) && axi_dmac_src_is_mem(chan)) 740 chan->direction = DMA_MEM_TO_DEV; 741 else if (axi_dmac_dest_is_mem(chan) && !axi_dmac_src_is_mem(chan)) 742 chan->direction = DMA_DEV_TO_MEM; 743 else 744 chan->direction = DMA_DEV_TO_DEV; 745 } 746 747 /* 748 * The configuration stored in the devicetree matches the configuration 749 * parameters of the peripheral instance and allows the driver to know which 750 * features are implemented and how it should behave. 751 */ 752 static int axi_dmac_parse_chan_dt(struct device_node *of_chan, 753 struct axi_dmac_chan *chan) 754 { 755 u32 val; 756 int ret; 757 758 ret = of_property_read_u32(of_chan, "reg", &val); 759 if (ret) 760 return ret; 761 762 /* We only support 1 channel for now */ 763 if (val != 0) 764 return -EINVAL; 765 766 ret = of_property_read_u32(of_chan, "adi,source-bus-type", &val); 767 if (ret) 768 return ret; 769 if (val > AXI_DMAC_BUS_TYPE_FIFO) 770 return -EINVAL; 771 chan->src_type = val; 772 773 ret = of_property_read_u32(of_chan, "adi,destination-bus-type", &val); 774 if (ret) 775 return ret; 776 if (val > AXI_DMAC_BUS_TYPE_FIFO) 777 return -EINVAL; 778 chan->dest_type = val; 779 780 ret = of_property_read_u32(of_chan, "adi,source-bus-width", &val); 781 if (ret) 782 return ret; 783 chan->src_width = val / 8; 784 785 ret = of_property_read_u32(of_chan, "adi,destination-bus-width", &val); 786 if (ret) 787 return ret; 788 chan->dest_width = val / 8; 789 790 axi_dmac_adjust_chan_params(chan); 791 792 return 0; 793 } 794 795 static int axi_dmac_parse_dt(struct device *dev, struct axi_dmac *dmac) 796 { 797 struct device_node *of_channels, *of_chan; 798 int ret; 799 800 of_channels = of_get_child_by_name(dev->of_node, "adi,channels"); 801 if (of_channels == NULL) 802 return -ENODEV; 803 804 for_each_child_of_node(of_channels, of_chan) { 805 ret = axi_dmac_parse_chan_dt(of_chan, &dmac->chan); 806 if (ret) { 807 of_node_put(of_chan); 808 of_node_put(of_channels); 809 return -EINVAL; 810 } 811 } 812 of_node_put(of_channels); 813 814 return 0; 815 } 816 817 static int axi_dmac_read_chan_config(struct device *dev, struct axi_dmac *dmac) 818 { 819 struct axi_dmac_chan *chan = &dmac->chan; 820 unsigned int val, desc; 821 822 desc = axi_dmac_read(dmac, AXI_DMAC_REG_INTERFACE_DESC); 823 if (desc == 0) { 824 dev_err(dev, "DMA interface register reads zero\n"); 825 return -EFAULT; 826 } 827 828 val = AXI_DMAC_DMA_SRC_TYPE_GET(desc); 829 if (val > AXI_DMAC_BUS_TYPE_FIFO) { 830 dev_err(dev, "Invalid source bus type read: %d\n", val); 831 return -EINVAL; 832 } 833 chan->src_type = val; 834 835 val = AXI_DMAC_DMA_DST_TYPE_GET(desc); 836 if (val > AXI_DMAC_BUS_TYPE_FIFO) { 837 dev_err(dev, "Invalid destination bus type read: %d\n", val); 838 return -EINVAL; 839 } 840 chan->dest_type = val; 841 842 val = AXI_DMAC_DMA_SRC_WIDTH_GET(desc); 843 if (val == 0) { 844 dev_err(dev, "Source bus width is zero\n"); 845 return -EINVAL; 846 } 847 /* widths are stored in log2 */ 848 chan->src_width = 1 << val; 849 850 val = AXI_DMAC_DMA_DST_WIDTH_GET(desc); 851 if (val == 0) { 852 dev_err(dev, "Destination bus width is zero\n"); 853 return -EINVAL; 854 } 855 chan->dest_width = 1 << val; 856 857 axi_dmac_adjust_chan_params(chan); 858 859 return 0; 860 } 861 862 static int axi_dmac_detect_caps(struct axi_dmac *dmac, unsigned int version) 863 { 864 struct axi_dmac_chan *chan = &dmac->chan; 865 866 axi_dmac_write(dmac, AXI_DMAC_REG_FLAGS, AXI_DMAC_FLAG_CYCLIC); 867 if (axi_dmac_read(dmac, AXI_DMAC_REG_FLAGS) == AXI_DMAC_FLAG_CYCLIC) 868 chan->hw_cyclic = true; 869 870 axi_dmac_write(dmac, AXI_DMAC_REG_Y_LENGTH, 1); 871 if (axi_dmac_read(dmac, AXI_DMAC_REG_Y_LENGTH) == 1) 872 chan->hw_2d = true; 873 874 axi_dmac_write(dmac, AXI_DMAC_REG_X_LENGTH, 0xffffffff); 875 chan->max_length = axi_dmac_read(dmac, AXI_DMAC_REG_X_LENGTH); 876 if (chan->max_length != UINT_MAX) 877 chan->max_length++; 878 879 axi_dmac_write(dmac, AXI_DMAC_REG_DEST_ADDRESS, 0xffffffff); 880 if (axi_dmac_read(dmac, AXI_DMAC_REG_DEST_ADDRESS) == 0 && 881 chan->dest_type == AXI_DMAC_BUS_TYPE_AXI_MM) { 882 dev_err(dmac->dma_dev.dev, 883 "Destination memory-mapped interface not supported."); 884 return -ENODEV; 885 } 886 887 axi_dmac_write(dmac, AXI_DMAC_REG_SRC_ADDRESS, 0xffffffff); 888 if (axi_dmac_read(dmac, AXI_DMAC_REG_SRC_ADDRESS) == 0 && 889 chan->src_type == AXI_DMAC_BUS_TYPE_AXI_MM) { 890 dev_err(dmac->dma_dev.dev, 891 "Source memory-mapped interface not supported."); 892 return -ENODEV; 893 } 894 895 if (version >= ADI_AXI_PCORE_VER(4, 2, 'a')) 896 chan->hw_partial_xfer = true; 897 898 if (version >= ADI_AXI_PCORE_VER(4, 1, 'a')) { 899 axi_dmac_write(dmac, AXI_DMAC_REG_X_LENGTH, 0x00); 900 chan->length_align_mask = 901 axi_dmac_read(dmac, AXI_DMAC_REG_X_LENGTH); 902 } else { 903 chan->length_align_mask = chan->address_align_mask; 904 } 905 906 return 0; 907 } 908 909 static int axi_dmac_probe(struct platform_device *pdev) 910 { 911 struct dma_device *dma_dev; 912 struct axi_dmac *dmac; 913 struct regmap *regmap; 914 unsigned int version; 915 int ret; 916 917 dmac = devm_kzalloc(&pdev->dev, sizeof(*dmac), GFP_KERNEL); 918 if (!dmac) 919 return -ENOMEM; 920 921 dmac->irq = platform_get_irq(pdev, 0); 922 if (dmac->irq < 0) 923 return dmac->irq; 924 if (dmac->irq == 0) 925 return -EINVAL; 926 927 dmac->base = devm_platform_ioremap_resource(pdev, 0); 928 if (IS_ERR(dmac->base)) 929 return PTR_ERR(dmac->base); 930 931 dmac->clk = devm_clk_get(&pdev->dev, NULL); 932 if (IS_ERR(dmac->clk)) 933 return PTR_ERR(dmac->clk); 934 935 ret = clk_prepare_enable(dmac->clk); 936 if (ret < 0) 937 return ret; 938 939 version = axi_dmac_read(dmac, ADI_AXI_REG_VERSION); 940 941 if (version >= ADI_AXI_PCORE_VER(4, 3, 'a')) 942 ret = axi_dmac_read_chan_config(&pdev->dev, dmac); 943 else 944 ret = axi_dmac_parse_dt(&pdev->dev, dmac); 945 946 if (ret < 0) 947 goto err_clk_disable; 948 949 INIT_LIST_HEAD(&dmac->chan.active_descs); 950 951 dma_set_max_seg_size(&pdev->dev, UINT_MAX); 952 953 dma_dev = &dmac->dma_dev; 954 dma_cap_set(DMA_SLAVE, dma_dev->cap_mask); 955 dma_cap_set(DMA_CYCLIC, dma_dev->cap_mask); 956 dma_cap_set(DMA_INTERLEAVE, dma_dev->cap_mask); 957 dma_dev->device_free_chan_resources = axi_dmac_free_chan_resources; 958 dma_dev->device_tx_status = dma_cookie_status; 959 dma_dev->device_issue_pending = axi_dmac_issue_pending; 960 dma_dev->device_prep_slave_sg = axi_dmac_prep_slave_sg; 961 dma_dev->device_prep_dma_cyclic = axi_dmac_prep_dma_cyclic; 962 dma_dev->device_prep_interleaved_dma = axi_dmac_prep_interleaved; 963 dma_dev->device_terminate_all = axi_dmac_terminate_all; 964 dma_dev->device_synchronize = axi_dmac_synchronize; 965 dma_dev->dev = &pdev->dev; 966 dma_dev->src_addr_widths = BIT(dmac->chan.src_width); 967 dma_dev->dst_addr_widths = BIT(dmac->chan.dest_width); 968 dma_dev->directions = BIT(dmac->chan.direction); 969 dma_dev->residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR; 970 INIT_LIST_HEAD(&dma_dev->channels); 971 972 dmac->chan.vchan.desc_free = axi_dmac_desc_free; 973 vchan_init(&dmac->chan.vchan, dma_dev); 974 975 ret = axi_dmac_detect_caps(dmac, version); 976 if (ret) 977 goto err_clk_disable; 978 979 dma_dev->copy_align = (dmac->chan.address_align_mask + 1); 980 981 axi_dmac_write(dmac, AXI_DMAC_REG_IRQ_MASK, 0x00); 982 983 if (of_dma_is_coherent(pdev->dev.of_node)) { 984 ret = axi_dmac_read(dmac, AXI_DMAC_REG_COHERENCY_DESC); 985 986 if (version < ADI_AXI_PCORE_VER(4, 4, 'a') || 987 !AXI_DMAC_DST_COHERENT_GET(ret)) { 988 dev_err(dmac->dma_dev.dev, 989 "Coherent DMA not supported in hardware"); 990 ret = -EINVAL; 991 goto err_clk_disable; 992 } 993 } 994 995 ret = dma_async_device_register(dma_dev); 996 if (ret) 997 goto err_clk_disable; 998 999 ret = of_dma_controller_register(pdev->dev.of_node, 1000 of_dma_xlate_by_chan_id, dma_dev); 1001 if (ret) 1002 goto err_unregister_device; 1003 1004 ret = request_irq(dmac->irq, axi_dmac_interrupt_handler, IRQF_SHARED, 1005 dev_name(&pdev->dev), dmac); 1006 if (ret) 1007 goto err_unregister_of; 1008 1009 platform_set_drvdata(pdev, dmac); 1010 1011 regmap = devm_regmap_init_mmio(&pdev->dev, dmac->base, 1012 &axi_dmac_regmap_config); 1013 if (IS_ERR(regmap)) { 1014 ret = PTR_ERR(regmap); 1015 goto err_free_irq; 1016 } 1017 1018 return 0; 1019 1020 err_free_irq: 1021 free_irq(dmac->irq, dmac); 1022 err_unregister_of: 1023 of_dma_controller_free(pdev->dev.of_node); 1024 err_unregister_device: 1025 dma_async_device_unregister(&dmac->dma_dev); 1026 err_clk_disable: 1027 clk_disable_unprepare(dmac->clk); 1028 1029 return ret; 1030 } 1031 1032 static int axi_dmac_remove(struct platform_device *pdev) 1033 { 1034 struct axi_dmac *dmac = platform_get_drvdata(pdev); 1035 1036 of_dma_controller_free(pdev->dev.of_node); 1037 free_irq(dmac->irq, dmac); 1038 tasklet_kill(&dmac->chan.vchan.task); 1039 dma_async_device_unregister(&dmac->dma_dev); 1040 clk_disable_unprepare(dmac->clk); 1041 1042 return 0; 1043 } 1044 1045 static const struct of_device_id axi_dmac_of_match_table[] = { 1046 { .compatible = "adi,axi-dmac-1.00.a" }, 1047 { }, 1048 }; 1049 MODULE_DEVICE_TABLE(of, axi_dmac_of_match_table); 1050 1051 static struct platform_driver axi_dmac_driver = { 1052 .driver = { 1053 .name = "dma-axi-dmac", 1054 .of_match_table = axi_dmac_of_match_table, 1055 }, 1056 .probe = axi_dmac_probe, 1057 .remove = axi_dmac_remove, 1058 }; 1059 module_platform_driver(axi_dmac_driver); 1060 1061 MODULE_AUTHOR("Lars-Peter Clausen <lars@metafoo.de>"); 1062 MODULE_DESCRIPTION("DMA controller driver for the AXI-DMAC controller"); 1063 MODULE_LICENSE("GPL v2"); 1064