1 /* 2 * Ingenic JZ4780 DMA controller 3 * 4 * Copyright (c) 2015 Imagination Technologies 5 * Author: Alex Smith <alex@alex-smith.me.uk> 6 * 7 * This program is free software; you can redistribute it and/or modify it 8 * under the terms of the GNU General Public License as published by the 9 * Free Software Foundation; either version 2 of the License, or (at your 10 * option) any later version. 11 */ 12 13 #include <linux/clk.h> 14 #include <linux/dmapool.h> 15 #include <linux/init.h> 16 #include <linux/interrupt.h> 17 #include <linux/module.h> 18 #include <linux/of.h> 19 #include <linux/of_dma.h> 20 #include <linux/platform_device.h> 21 #include <linux/slab.h> 22 23 #include "dmaengine.h" 24 #include "virt-dma.h" 25 26 #define JZ_DMA_NR_CHANNELS 32 27 28 /* Global registers. */ 29 #define JZ_DMA_REG_DMAC 0x1000 30 #define JZ_DMA_REG_DIRQP 0x1004 31 #define JZ_DMA_REG_DDR 0x1008 32 #define JZ_DMA_REG_DDRS 0x100c 33 #define JZ_DMA_REG_DMACP 0x101c 34 #define JZ_DMA_REG_DSIRQP 0x1020 35 #define JZ_DMA_REG_DSIRQM 0x1024 36 #define JZ_DMA_REG_DCIRQP 0x1028 37 #define JZ_DMA_REG_DCIRQM 0x102c 38 39 /* Per-channel registers. */ 40 #define JZ_DMA_REG_CHAN(n) (n * 0x20) 41 #define JZ_DMA_REG_DSA(n) (0x00 + JZ_DMA_REG_CHAN(n)) 42 #define JZ_DMA_REG_DTA(n) (0x04 + JZ_DMA_REG_CHAN(n)) 43 #define JZ_DMA_REG_DTC(n) (0x08 + JZ_DMA_REG_CHAN(n)) 44 #define JZ_DMA_REG_DRT(n) (0x0c + JZ_DMA_REG_CHAN(n)) 45 #define JZ_DMA_REG_DCS(n) (0x10 + JZ_DMA_REG_CHAN(n)) 46 #define JZ_DMA_REG_DCM(n) (0x14 + JZ_DMA_REG_CHAN(n)) 47 #define JZ_DMA_REG_DDA(n) (0x18 + JZ_DMA_REG_CHAN(n)) 48 #define JZ_DMA_REG_DSD(n) (0x1c + JZ_DMA_REG_CHAN(n)) 49 50 #define JZ_DMA_DMAC_DMAE BIT(0) 51 #define JZ_DMA_DMAC_AR BIT(2) 52 #define JZ_DMA_DMAC_HLT BIT(3) 53 #define JZ_DMA_DMAC_FMSC BIT(31) 54 55 #define JZ_DMA_DRT_AUTO 0x8 56 57 #define JZ_DMA_DCS_CTE BIT(0) 58 #define JZ_DMA_DCS_HLT BIT(2) 59 #define JZ_DMA_DCS_TT BIT(3) 60 #define JZ_DMA_DCS_AR BIT(4) 61 #define JZ_DMA_DCS_DES8 BIT(30) 62 63 #define JZ_DMA_DCM_LINK BIT(0) 64 #define JZ_DMA_DCM_TIE BIT(1) 65 #define JZ_DMA_DCM_STDE BIT(2) 66 #define JZ_DMA_DCM_TSZ_SHIFT 8 67 #define JZ_DMA_DCM_TSZ_MASK (0x7 << JZ_DMA_DCM_TSZ_SHIFT) 68 #define JZ_DMA_DCM_DP_SHIFT 12 69 #define JZ_DMA_DCM_SP_SHIFT 14 70 #define JZ_DMA_DCM_DAI BIT(22) 71 #define JZ_DMA_DCM_SAI BIT(23) 72 73 #define JZ_DMA_SIZE_4_BYTE 0x0 74 #define JZ_DMA_SIZE_1_BYTE 0x1 75 #define JZ_DMA_SIZE_2_BYTE 0x2 76 #define JZ_DMA_SIZE_16_BYTE 0x3 77 #define JZ_DMA_SIZE_32_BYTE 0x4 78 #define JZ_DMA_SIZE_64_BYTE 0x5 79 #define JZ_DMA_SIZE_128_BYTE 0x6 80 81 #define JZ_DMA_WIDTH_32_BIT 0x0 82 #define JZ_DMA_WIDTH_8_BIT 0x1 83 #define JZ_DMA_WIDTH_16_BIT 0x2 84 85 #define JZ_DMA_BUSWIDTHS (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \ 86 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \ 87 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES)) 88 89 /** 90 * struct jz4780_dma_hwdesc - descriptor structure read by the DMA controller. 91 * @dcm: value for the DCM (channel command) register 92 * @dsa: source address 93 * @dta: target address 94 * @dtc: transfer count (number of blocks of the transfer size specified in DCM 95 * to transfer) in the low 24 bits, offset of the next descriptor from the 96 * descriptor base address in the upper 8 bits. 97 * @sd: target/source stride difference (in stride transfer mode). 98 * @drt: request type 99 */ 100 struct jz4780_dma_hwdesc { 101 uint32_t dcm; 102 uint32_t dsa; 103 uint32_t dta; 104 uint32_t dtc; 105 uint32_t sd; 106 uint32_t drt; 107 uint32_t reserved[2]; 108 }; 109 110 /* Size of allocations for hardware descriptor blocks. */ 111 #define JZ_DMA_DESC_BLOCK_SIZE PAGE_SIZE 112 #define JZ_DMA_MAX_DESC \ 113 (JZ_DMA_DESC_BLOCK_SIZE / sizeof(struct jz4780_dma_hwdesc)) 114 115 struct jz4780_dma_desc { 116 struct virt_dma_desc vdesc; 117 118 struct jz4780_dma_hwdesc *desc; 119 dma_addr_t desc_phys; 120 unsigned int count; 121 enum dma_transaction_type type; 122 uint32_t status; 123 }; 124 125 struct jz4780_dma_chan { 126 struct virt_dma_chan vchan; 127 unsigned int id; 128 struct dma_pool *desc_pool; 129 130 uint32_t transfer_type; 131 uint32_t transfer_shift; 132 struct dma_slave_config config; 133 134 struct jz4780_dma_desc *desc; 135 unsigned int curr_hwdesc; 136 }; 137 138 struct jz4780_dma_dev { 139 struct dma_device dma_device; 140 void __iomem *base; 141 struct clk *clk; 142 unsigned int irq; 143 144 uint32_t chan_reserved; 145 struct jz4780_dma_chan chan[JZ_DMA_NR_CHANNELS]; 146 }; 147 148 struct jz4780_dma_filter_data { 149 struct device_node *of_node; 150 uint32_t transfer_type; 151 int channel; 152 }; 153 154 static inline struct jz4780_dma_chan *to_jz4780_dma_chan(struct dma_chan *chan) 155 { 156 return container_of(chan, struct jz4780_dma_chan, vchan.chan); 157 } 158 159 static inline struct jz4780_dma_desc *to_jz4780_dma_desc( 160 struct virt_dma_desc *vdesc) 161 { 162 return container_of(vdesc, struct jz4780_dma_desc, vdesc); 163 } 164 165 static inline struct jz4780_dma_dev *jz4780_dma_chan_parent( 166 struct jz4780_dma_chan *jzchan) 167 { 168 return container_of(jzchan->vchan.chan.device, struct jz4780_dma_dev, 169 dma_device); 170 } 171 172 static inline uint32_t jz4780_dma_readl(struct jz4780_dma_dev *jzdma, 173 unsigned int reg) 174 { 175 return readl(jzdma->base + reg); 176 } 177 178 static inline void jz4780_dma_writel(struct jz4780_dma_dev *jzdma, 179 unsigned int reg, uint32_t val) 180 { 181 writel(val, jzdma->base + reg); 182 } 183 184 static struct jz4780_dma_desc *jz4780_dma_desc_alloc( 185 struct jz4780_dma_chan *jzchan, unsigned int count, 186 enum dma_transaction_type type) 187 { 188 struct jz4780_dma_desc *desc; 189 190 if (count > JZ_DMA_MAX_DESC) 191 return NULL; 192 193 desc = kzalloc(sizeof(*desc), GFP_NOWAIT); 194 if (!desc) 195 return NULL; 196 197 desc->desc = dma_pool_alloc(jzchan->desc_pool, GFP_NOWAIT, 198 &desc->desc_phys); 199 if (!desc->desc) { 200 kfree(desc); 201 return NULL; 202 } 203 204 desc->count = count; 205 desc->type = type; 206 return desc; 207 } 208 209 static void jz4780_dma_desc_free(struct virt_dma_desc *vdesc) 210 { 211 struct jz4780_dma_desc *desc = to_jz4780_dma_desc(vdesc); 212 struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(vdesc->tx.chan); 213 214 dma_pool_free(jzchan->desc_pool, desc->desc, desc->desc_phys); 215 kfree(desc); 216 } 217 218 static uint32_t jz4780_dma_transfer_size(unsigned long val, uint32_t *shift) 219 { 220 int ord = ffs(val) - 1; 221 222 /* 223 * 8 byte transfer sizes unsupported so fall back on 4. If it's larger 224 * than the maximum, just limit it. It is perfectly safe to fall back 225 * in this way since we won't exceed the maximum burst size supported 226 * by the device, the only effect is reduced efficiency. This is better 227 * than refusing to perform the request at all. 228 */ 229 if (ord == 3) 230 ord = 2; 231 else if (ord > 7) 232 ord = 7; 233 234 *shift = ord; 235 236 switch (ord) { 237 case 0: 238 return JZ_DMA_SIZE_1_BYTE; 239 case 1: 240 return JZ_DMA_SIZE_2_BYTE; 241 case 2: 242 return JZ_DMA_SIZE_4_BYTE; 243 case 4: 244 return JZ_DMA_SIZE_16_BYTE; 245 case 5: 246 return JZ_DMA_SIZE_32_BYTE; 247 case 6: 248 return JZ_DMA_SIZE_64_BYTE; 249 default: 250 return JZ_DMA_SIZE_128_BYTE; 251 } 252 } 253 254 static int jz4780_dma_setup_hwdesc(struct jz4780_dma_chan *jzchan, 255 struct jz4780_dma_hwdesc *desc, dma_addr_t addr, size_t len, 256 enum dma_transfer_direction direction) 257 { 258 struct dma_slave_config *config = &jzchan->config; 259 uint32_t width, maxburst, tsz; 260 261 if (direction == DMA_MEM_TO_DEV) { 262 desc->dcm = JZ_DMA_DCM_SAI; 263 desc->dsa = addr; 264 desc->dta = config->dst_addr; 265 desc->drt = jzchan->transfer_type; 266 267 width = config->dst_addr_width; 268 maxburst = config->dst_maxburst; 269 } else { 270 desc->dcm = JZ_DMA_DCM_DAI; 271 desc->dsa = config->src_addr; 272 desc->dta = addr; 273 desc->drt = jzchan->transfer_type; 274 275 width = config->src_addr_width; 276 maxburst = config->src_maxburst; 277 } 278 279 /* 280 * This calculates the maximum transfer size that can be used with the 281 * given address, length, width and maximum burst size. The address 282 * must be aligned to the transfer size, the total length must be 283 * divisible by the transfer size, and we must not use more than the 284 * maximum burst specified by the user. 285 */ 286 tsz = jz4780_dma_transfer_size(addr | len | (width * maxburst), 287 &jzchan->transfer_shift); 288 289 switch (width) { 290 case DMA_SLAVE_BUSWIDTH_1_BYTE: 291 case DMA_SLAVE_BUSWIDTH_2_BYTES: 292 break; 293 case DMA_SLAVE_BUSWIDTH_4_BYTES: 294 width = JZ_DMA_WIDTH_32_BIT; 295 break; 296 default: 297 return -EINVAL; 298 } 299 300 desc->dcm |= tsz << JZ_DMA_DCM_TSZ_SHIFT; 301 desc->dcm |= width << JZ_DMA_DCM_SP_SHIFT; 302 desc->dcm |= width << JZ_DMA_DCM_DP_SHIFT; 303 304 desc->dtc = len >> jzchan->transfer_shift; 305 return 0; 306 } 307 308 static struct dma_async_tx_descriptor *jz4780_dma_prep_slave_sg( 309 struct dma_chan *chan, struct scatterlist *sgl, unsigned int sg_len, 310 enum dma_transfer_direction direction, unsigned long flags, 311 void *context) 312 { 313 struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan); 314 struct jz4780_dma_desc *desc; 315 unsigned int i; 316 int err; 317 318 desc = jz4780_dma_desc_alloc(jzchan, sg_len, DMA_SLAVE); 319 if (!desc) 320 return NULL; 321 322 for (i = 0; i < sg_len; i++) { 323 err = jz4780_dma_setup_hwdesc(jzchan, &desc->desc[i], 324 sg_dma_address(&sgl[i]), 325 sg_dma_len(&sgl[i]), 326 direction); 327 if (err < 0) { 328 jz4780_dma_desc_free(&jzchan->desc->vdesc); 329 return NULL; 330 } 331 332 desc->desc[i].dcm |= JZ_DMA_DCM_TIE; 333 334 if (i != (sg_len - 1)) { 335 /* Automatically proceeed to the next descriptor. */ 336 desc->desc[i].dcm |= JZ_DMA_DCM_LINK; 337 338 /* 339 * The upper 8 bits of the DTC field in the descriptor 340 * must be set to (offset from descriptor base of next 341 * descriptor >> 4). 342 */ 343 desc->desc[i].dtc |= 344 (((i + 1) * sizeof(*desc->desc)) >> 4) << 24; 345 } 346 } 347 348 return vchan_tx_prep(&jzchan->vchan, &desc->vdesc, flags); 349 } 350 351 static struct dma_async_tx_descriptor *jz4780_dma_prep_dma_cyclic( 352 struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len, 353 size_t period_len, enum dma_transfer_direction direction, 354 unsigned long flags) 355 { 356 struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan); 357 struct jz4780_dma_desc *desc; 358 unsigned int periods, i; 359 int err; 360 361 if (buf_len % period_len) 362 return NULL; 363 364 periods = buf_len / period_len; 365 366 desc = jz4780_dma_desc_alloc(jzchan, periods, DMA_CYCLIC); 367 if (!desc) 368 return NULL; 369 370 for (i = 0; i < periods; i++) { 371 err = jz4780_dma_setup_hwdesc(jzchan, &desc->desc[i], buf_addr, 372 period_len, direction); 373 if (err < 0) { 374 jz4780_dma_desc_free(&jzchan->desc->vdesc); 375 return NULL; 376 } 377 378 buf_addr += period_len; 379 380 /* 381 * Set the link bit to indicate that the controller should 382 * automatically proceed to the next descriptor. In 383 * jz4780_dma_begin(), this will be cleared if we need to issue 384 * an interrupt after each period. 385 */ 386 desc->desc[i].dcm |= JZ_DMA_DCM_TIE | JZ_DMA_DCM_LINK; 387 388 /* 389 * The upper 8 bits of the DTC field in the descriptor must be 390 * set to (offset from descriptor base of next descriptor >> 4). 391 * If this is the last descriptor, link it back to the first, 392 * i.e. leave offset set to 0, otherwise point to the next one. 393 */ 394 if (i != (periods - 1)) { 395 desc->desc[i].dtc |= 396 (((i + 1) * sizeof(*desc->desc)) >> 4) << 24; 397 } 398 } 399 400 return vchan_tx_prep(&jzchan->vchan, &desc->vdesc, flags); 401 } 402 403 static struct dma_async_tx_descriptor *jz4780_dma_prep_dma_memcpy( 404 struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, 405 size_t len, unsigned long flags) 406 { 407 struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan); 408 struct jz4780_dma_desc *desc; 409 uint32_t tsz; 410 411 desc = jz4780_dma_desc_alloc(jzchan, 1, DMA_MEMCPY); 412 if (!desc) 413 return NULL; 414 415 tsz = jz4780_dma_transfer_size(dest | src | len, 416 &jzchan->transfer_shift); 417 418 desc->desc[0].dsa = src; 419 desc->desc[0].dta = dest; 420 desc->desc[0].drt = JZ_DMA_DRT_AUTO; 421 desc->desc[0].dcm = JZ_DMA_DCM_TIE | JZ_DMA_DCM_SAI | JZ_DMA_DCM_DAI | 422 tsz << JZ_DMA_DCM_TSZ_SHIFT | 423 JZ_DMA_WIDTH_32_BIT << JZ_DMA_DCM_SP_SHIFT | 424 JZ_DMA_WIDTH_32_BIT << JZ_DMA_DCM_DP_SHIFT; 425 desc->desc[0].dtc = len >> jzchan->transfer_shift; 426 427 return vchan_tx_prep(&jzchan->vchan, &desc->vdesc, flags); 428 } 429 430 static void jz4780_dma_begin(struct jz4780_dma_chan *jzchan) 431 { 432 struct jz4780_dma_dev *jzdma = jz4780_dma_chan_parent(jzchan); 433 struct virt_dma_desc *vdesc; 434 unsigned int i; 435 dma_addr_t desc_phys; 436 437 if (!jzchan->desc) { 438 vdesc = vchan_next_desc(&jzchan->vchan); 439 if (!vdesc) 440 return; 441 442 list_del(&vdesc->node); 443 444 jzchan->desc = to_jz4780_dma_desc(vdesc); 445 jzchan->curr_hwdesc = 0; 446 447 if (jzchan->desc->type == DMA_CYCLIC && vdesc->tx.callback) { 448 /* 449 * The DMA controller doesn't support triggering an 450 * interrupt after processing each descriptor, only 451 * after processing an entire terminated list of 452 * descriptors. For a cyclic DMA setup the list of 453 * descriptors is not terminated so we can never get an 454 * interrupt. 455 * 456 * If the user requested a callback for a cyclic DMA 457 * setup then we workaround this hardware limitation 458 * here by degrading to a set of unlinked descriptors 459 * which we will submit in sequence in response to the 460 * completion of processing the previous descriptor. 461 */ 462 for (i = 0; i < jzchan->desc->count; i++) 463 jzchan->desc->desc[i].dcm &= ~JZ_DMA_DCM_LINK; 464 } 465 } else { 466 /* 467 * There is an existing transfer, therefore this must be one 468 * for which we unlinked the descriptors above. Advance to the 469 * next one in the list. 470 */ 471 jzchan->curr_hwdesc = 472 (jzchan->curr_hwdesc + 1) % jzchan->desc->count; 473 } 474 475 /* Use 8-word descriptors. */ 476 jz4780_dma_writel(jzdma, JZ_DMA_REG_DCS(jzchan->id), JZ_DMA_DCS_DES8); 477 478 /* Write descriptor address and initiate descriptor fetch. */ 479 desc_phys = jzchan->desc->desc_phys + 480 (jzchan->curr_hwdesc * sizeof(*jzchan->desc->desc)); 481 jz4780_dma_writel(jzdma, JZ_DMA_REG_DDA(jzchan->id), desc_phys); 482 jz4780_dma_writel(jzdma, JZ_DMA_REG_DDRS, BIT(jzchan->id)); 483 484 /* Enable the channel. */ 485 jz4780_dma_writel(jzdma, JZ_DMA_REG_DCS(jzchan->id), 486 JZ_DMA_DCS_DES8 | JZ_DMA_DCS_CTE); 487 } 488 489 static void jz4780_dma_issue_pending(struct dma_chan *chan) 490 { 491 struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan); 492 unsigned long flags; 493 494 spin_lock_irqsave(&jzchan->vchan.lock, flags); 495 496 if (vchan_issue_pending(&jzchan->vchan) && !jzchan->desc) 497 jz4780_dma_begin(jzchan); 498 499 spin_unlock_irqrestore(&jzchan->vchan.lock, flags); 500 } 501 502 static int jz4780_dma_terminate_all(struct dma_chan *chan) 503 { 504 struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan); 505 struct jz4780_dma_dev *jzdma = jz4780_dma_chan_parent(jzchan); 506 unsigned long flags; 507 LIST_HEAD(head); 508 509 spin_lock_irqsave(&jzchan->vchan.lock, flags); 510 511 /* Clear the DMA status and stop the transfer. */ 512 jz4780_dma_writel(jzdma, JZ_DMA_REG_DCS(jzchan->id), 0); 513 if (jzchan->desc) { 514 vchan_terminate_vdesc(&jzchan->desc->vdesc); 515 jzchan->desc = NULL; 516 } 517 518 vchan_get_all_descriptors(&jzchan->vchan, &head); 519 520 spin_unlock_irqrestore(&jzchan->vchan.lock, flags); 521 522 vchan_dma_desc_free_list(&jzchan->vchan, &head); 523 return 0; 524 } 525 526 static void jz4780_dma_synchronize(struct dma_chan *chan) 527 { 528 struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan); 529 530 vchan_synchronize(&jzchan->vchan); 531 } 532 533 static int jz4780_dma_config(struct dma_chan *chan, 534 struct dma_slave_config *config) 535 { 536 struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan); 537 538 if ((config->src_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES) 539 || (config->dst_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES)) 540 return -EINVAL; 541 542 /* Copy the reset of the slave configuration, it is used later. */ 543 memcpy(&jzchan->config, config, sizeof(jzchan->config)); 544 545 return 0; 546 } 547 548 static size_t jz4780_dma_desc_residue(struct jz4780_dma_chan *jzchan, 549 struct jz4780_dma_desc *desc, unsigned int next_sg) 550 { 551 struct jz4780_dma_dev *jzdma = jz4780_dma_chan_parent(jzchan); 552 unsigned int residue, count; 553 unsigned int i; 554 555 residue = 0; 556 557 for (i = next_sg; i < desc->count; i++) 558 residue += desc->desc[i].dtc << jzchan->transfer_shift; 559 560 if (next_sg != 0) { 561 count = jz4780_dma_readl(jzdma, 562 JZ_DMA_REG_DTC(jzchan->id)); 563 residue += count << jzchan->transfer_shift; 564 } 565 566 return residue; 567 } 568 569 static enum dma_status jz4780_dma_tx_status(struct dma_chan *chan, 570 dma_cookie_t cookie, struct dma_tx_state *txstate) 571 { 572 struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan); 573 struct virt_dma_desc *vdesc; 574 enum dma_status status; 575 unsigned long flags; 576 577 status = dma_cookie_status(chan, cookie, txstate); 578 if ((status == DMA_COMPLETE) || (txstate == NULL)) 579 return status; 580 581 spin_lock_irqsave(&jzchan->vchan.lock, flags); 582 583 vdesc = vchan_find_desc(&jzchan->vchan, cookie); 584 if (vdesc) { 585 /* On the issued list, so hasn't been processed yet */ 586 txstate->residue = jz4780_dma_desc_residue(jzchan, 587 to_jz4780_dma_desc(vdesc), 0); 588 } else if (cookie == jzchan->desc->vdesc.tx.cookie) { 589 txstate->residue = jz4780_dma_desc_residue(jzchan, jzchan->desc, 590 (jzchan->curr_hwdesc + 1) % jzchan->desc->count); 591 } else 592 txstate->residue = 0; 593 594 if (vdesc && jzchan->desc && vdesc == &jzchan->desc->vdesc 595 && jzchan->desc->status & (JZ_DMA_DCS_AR | JZ_DMA_DCS_HLT)) 596 status = DMA_ERROR; 597 598 spin_unlock_irqrestore(&jzchan->vchan.lock, flags); 599 return status; 600 } 601 602 static void jz4780_dma_chan_irq(struct jz4780_dma_dev *jzdma, 603 struct jz4780_dma_chan *jzchan) 604 { 605 uint32_t dcs; 606 607 spin_lock(&jzchan->vchan.lock); 608 609 dcs = jz4780_dma_readl(jzdma, JZ_DMA_REG_DCS(jzchan->id)); 610 jz4780_dma_writel(jzdma, JZ_DMA_REG_DCS(jzchan->id), 0); 611 612 if (dcs & JZ_DMA_DCS_AR) { 613 dev_warn(&jzchan->vchan.chan.dev->device, 614 "address error (DCS=0x%x)\n", dcs); 615 } 616 617 if (dcs & JZ_DMA_DCS_HLT) { 618 dev_warn(&jzchan->vchan.chan.dev->device, 619 "channel halt (DCS=0x%x)\n", dcs); 620 } 621 622 if (jzchan->desc) { 623 jzchan->desc->status = dcs; 624 625 if ((dcs & (JZ_DMA_DCS_AR | JZ_DMA_DCS_HLT)) == 0) { 626 if (jzchan->desc->type == DMA_CYCLIC) { 627 vchan_cyclic_callback(&jzchan->desc->vdesc); 628 } else { 629 vchan_cookie_complete(&jzchan->desc->vdesc); 630 jzchan->desc = NULL; 631 } 632 633 jz4780_dma_begin(jzchan); 634 } 635 } else { 636 dev_err(&jzchan->vchan.chan.dev->device, 637 "channel IRQ with no active transfer\n"); 638 } 639 640 spin_unlock(&jzchan->vchan.lock); 641 } 642 643 static irqreturn_t jz4780_dma_irq_handler(int irq, void *data) 644 { 645 struct jz4780_dma_dev *jzdma = data; 646 uint32_t pending, dmac; 647 int i; 648 649 pending = jz4780_dma_readl(jzdma, JZ_DMA_REG_DIRQP); 650 651 for (i = 0; i < JZ_DMA_NR_CHANNELS; i++) { 652 if (!(pending & (1<<i))) 653 continue; 654 655 jz4780_dma_chan_irq(jzdma, &jzdma->chan[i]); 656 } 657 658 /* Clear halt and address error status of all channels. */ 659 dmac = jz4780_dma_readl(jzdma, JZ_DMA_REG_DMAC); 660 dmac &= ~(JZ_DMA_DMAC_HLT | JZ_DMA_DMAC_AR); 661 jz4780_dma_writel(jzdma, JZ_DMA_REG_DMAC, dmac); 662 663 /* Clear interrupt pending status. */ 664 jz4780_dma_writel(jzdma, JZ_DMA_REG_DIRQP, 0); 665 666 return IRQ_HANDLED; 667 } 668 669 static int jz4780_dma_alloc_chan_resources(struct dma_chan *chan) 670 { 671 struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan); 672 673 jzchan->desc_pool = dma_pool_create(dev_name(&chan->dev->device), 674 chan->device->dev, 675 JZ_DMA_DESC_BLOCK_SIZE, 676 PAGE_SIZE, 0); 677 if (!jzchan->desc_pool) { 678 dev_err(&chan->dev->device, 679 "failed to allocate descriptor pool\n"); 680 return -ENOMEM; 681 } 682 683 return 0; 684 } 685 686 static void jz4780_dma_free_chan_resources(struct dma_chan *chan) 687 { 688 struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan); 689 690 vchan_free_chan_resources(&jzchan->vchan); 691 dma_pool_destroy(jzchan->desc_pool); 692 jzchan->desc_pool = NULL; 693 } 694 695 static bool jz4780_dma_filter_fn(struct dma_chan *chan, void *param) 696 { 697 struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan); 698 struct jz4780_dma_dev *jzdma = jz4780_dma_chan_parent(jzchan); 699 struct jz4780_dma_filter_data *data = param; 700 701 if (jzdma->dma_device.dev->of_node != data->of_node) 702 return false; 703 704 if (data->channel > -1) { 705 if (data->channel != jzchan->id) 706 return false; 707 } else if (jzdma->chan_reserved & BIT(jzchan->id)) { 708 return false; 709 } 710 711 jzchan->transfer_type = data->transfer_type; 712 713 return true; 714 } 715 716 static struct dma_chan *jz4780_of_dma_xlate(struct of_phandle_args *dma_spec, 717 struct of_dma *ofdma) 718 { 719 struct jz4780_dma_dev *jzdma = ofdma->of_dma_data; 720 dma_cap_mask_t mask = jzdma->dma_device.cap_mask; 721 struct jz4780_dma_filter_data data; 722 723 if (dma_spec->args_count != 2) 724 return NULL; 725 726 data.of_node = ofdma->of_node; 727 data.transfer_type = dma_spec->args[0]; 728 data.channel = dma_spec->args[1]; 729 730 if (data.channel > -1) { 731 if (data.channel >= JZ_DMA_NR_CHANNELS) { 732 dev_err(jzdma->dma_device.dev, 733 "device requested non-existent channel %u\n", 734 data.channel); 735 return NULL; 736 } 737 738 /* Can only select a channel marked as reserved. */ 739 if (!(jzdma->chan_reserved & BIT(data.channel))) { 740 dev_err(jzdma->dma_device.dev, 741 "device requested unreserved channel %u\n", 742 data.channel); 743 return NULL; 744 } 745 746 jzdma->chan[data.channel].transfer_type = data.transfer_type; 747 748 return dma_get_slave_channel( 749 &jzdma->chan[data.channel].vchan.chan); 750 } else { 751 return dma_request_channel(mask, jz4780_dma_filter_fn, &data); 752 } 753 } 754 755 static int jz4780_dma_probe(struct platform_device *pdev) 756 { 757 struct device *dev = &pdev->dev; 758 struct jz4780_dma_dev *jzdma; 759 struct jz4780_dma_chan *jzchan; 760 struct dma_device *dd; 761 struct resource *res; 762 int i, ret; 763 764 jzdma = devm_kzalloc(dev, sizeof(*jzdma), GFP_KERNEL); 765 if (!jzdma) 766 return -ENOMEM; 767 768 platform_set_drvdata(pdev, jzdma); 769 770 res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 771 if (!res) { 772 dev_err(dev, "failed to get I/O memory\n"); 773 return -EINVAL; 774 } 775 776 jzdma->base = devm_ioremap_resource(dev, res); 777 if (IS_ERR(jzdma->base)) 778 return PTR_ERR(jzdma->base); 779 780 ret = platform_get_irq(pdev, 0); 781 if (ret < 0) { 782 dev_err(dev, "failed to get IRQ: %d\n", ret); 783 return ret; 784 } 785 786 jzdma->irq = ret; 787 788 ret = request_irq(jzdma->irq, jz4780_dma_irq_handler, 0, dev_name(dev), 789 jzdma); 790 if (ret) { 791 dev_err(dev, "failed to request IRQ %u!\n", jzdma->irq); 792 return ret; 793 } 794 795 jzdma->clk = devm_clk_get(dev, NULL); 796 if (IS_ERR(jzdma->clk)) { 797 dev_err(dev, "failed to get clock\n"); 798 ret = PTR_ERR(jzdma->clk); 799 goto err_free_irq; 800 } 801 802 clk_prepare_enable(jzdma->clk); 803 804 /* Property is optional, if it doesn't exist the value will remain 0. */ 805 of_property_read_u32_index(dev->of_node, "ingenic,reserved-channels", 806 0, &jzdma->chan_reserved); 807 808 dd = &jzdma->dma_device; 809 810 dma_cap_set(DMA_MEMCPY, dd->cap_mask); 811 dma_cap_set(DMA_SLAVE, dd->cap_mask); 812 dma_cap_set(DMA_CYCLIC, dd->cap_mask); 813 814 dd->dev = dev; 815 dd->copy_align = DMAENGINE_ALIGN_4_BYTES; 816 dd->device_alloc_chan_resources = jz4780_dma_alloc_chan_resources; 817 dd->device_free_chan_resources = jz4780_dma_free_chan_resources; 818 dd->device_prep_slave_sg = jz4780_dma_prep_slave_sg; 819 dd->device_prep_dma_cyclic = jz4780_dma_prep_dma_cyclic; 820 dd->device_prep_dma_memcpy = jz4780_dma_prep_dma_memcpy; 821 dd->device_config = jz4780_dma_config; 822 dd->device_terminate_all = jz4780_dma_terminate_all; 823 dd->device_synchronize = jz4780_dma_synchronize; 824 dd->device_tx_status = jz4780_dma_tx_status; 825 dd->device_issue_pending = jz4780_dma_issue_pending; 826 dd->src_addr_widths = JZ_DMA_BUSWIDTHS; 827 dd->dst_addr_widths = JZ_DMA_BUSWIDTHS; 828 dd->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); 829 dd->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; 830 831 /* 832 * Enable DMA controller, mark all channels as not programmable. 833 * Also set the FMSC bit - it increases MSC performance, so it makes 834 * little sense not to enable it. 835 */ 836 jz4780_dma_writel(jzdma, JZ_DMA_REG_DMAC, 837 JZ_DMA_DMAC_DMAE | JZ_DMA_DMAC_FMSC); 838 jz4780_dma_writel(jzdma, JZ_DMA_REG_DMACP, 0); 839 840 INIT_LIST_HEAD(&dd->channels); 841 842 for (i = 0; i < JZ_DMA_NR_CHANNELS; i++) { 843 jzchan = &jzdma->chan[i]; 844 jzchan->id = i; 845 846 vchan_init(&jzchan->vchan, dd); 847 jzchan->vchan.desc_free = jz4780_dma_desc_free; 848 } 849 850 ret = dma_async_device_register(dd); 851 if (ret) { 852 dev_err(dev, "failed to register device\n"); 853 goto err_disable_clk; 854 } 855 856 /* Register with OF DMA helpers. */ 857 ret = of_dma_controller_register(dev->of_node, jz4780_of_dma_xlate, 858 jzdma); 859 if (ret) { 860 dev_err(dev, "failed to register OF DMA controller\n"); 861 goto err_unregister_dev; 862 } 863 864 dev_info(dev, "JZ4780 DMA controller initialised\n"); 865 return 0; 866 867 err_unregister_dev: 868 dma_async_device_unregister(dd); 869 870 err_disable_clk: 871 clk_disable_unprepare(jzdma->clk); 872 873 err_free_irq: 874 free_irq(jzdma->irq, jzdma); 875 return ret; 876 } 877 878 static int jz4780_dma_remove(struct platform_device *pdev) 879 { 880 struct jz4780_dma_dev *jzdma = platform_get_drvdata(pdev); 881 int i; 882 883 of_dma_controller_free(pdev->dev.of_node); 884 885 free_irq(jzdma->irq, jzdma); 886 887 for (i = 0; i < JZ_DMA_NR_CHANNELS; i++) 888 tasklet_kill(&jzdma->chan[i].vchan.task); 889 890 dma_async_device_unregister(&jzdma->dma_device); 891 return 0; 892 } 893 894 static const struct of_device_id jz4780_dma_dt_match[] = { 895 { .compatible = "ingenic,jz4780-dma", .data = NULL }, 896 {}, 897 }; 898 MODULE_DEVICE_TABLE(of, jz4780_dma_dt_match); 899 900 static struct platform_driver jz4780_dma_driver = { 901 .probe = jz4780_dma_probe, 902 .remove = jz4780_dma_remove, 903 .driver = { 904 .name = "jz4780-dma", 905 .of_match_table = of_match_ptr(jz4780_dma_dt_match), 906 }, 907 }; 908 909 static int __init jz4780_dma_init(void) 910 { 911 return platform_driver_register(&jz4780_dma_driver); 912 } 913 subsys_initcall(jz4780_dma_init); 914 915 static void __exit jz4780_dma_exit(void) 916 { 917 platform_driver_unregister(&jz4780_dma_driver); 918 } 919 module_exit(jz4780_dma_exit); 920 921 MODULE_AUTHOR("Alex Smith <alex@alex-smith.me.uk>"); 922 MODULE_DESCRIPTION("Ingenic JZ4780 DMA controller driver"); 923 MODULE_LICENSE("GPL"); 924