1 /* 2 * Ingenic JZ4780 DMA controller 3 * 4 * Copyright (c) 2015 Imagination Technologies 5 * Author: Alex Smith <alex@alex-smith.me.uk> 6 * 7 * This program is free software; you can redistribute it and/or modify it 8 * under the terms of the GNU General Public License as published by the 9 * Free Software Foundation; either version 2 of the License, or (at your 10 * option) any later version. 11 */ 12 13 #include <linux/clk.h> 14 #include <linux/dmapool.h> 15 #include <linux/init.h> 16 #include <linux/interrupt.h> 17 #include <linux/module.h> 18 #include <linux/of.h> 19 #include <linux/of_dma.h> 20 #include <linux/platform_device.h> 21 #include <linux/slab.h> 22 23 #include "dmaengine.h" 24 #include "virt-dma.h" 25 26 #define JZ_DMA_NR_CHANNELS 32 27 28 /* Global registers. */ 29 #define JZ_DMA_REG_DMAC 0x1000 30 #define JZ_DMA_REG_DIRQP 0x1004 31 #define JZ_DMA_REG_DDR 0x1008 32 #define JZ_DMA_REG_DDRS 0x100c 33 #define JZ_DMA_REG_DMACP 0x101c 34 #define JZ_DMA_REG_DSIRQP 0x1020 35 #define JZ_DMA_REG_DSIRQM 0x1024 36 #define JZ_DMA_REG_DCIRQP 0x1028 37 #define JZ_DMA_REG_DCIRQM 0x102c 38 39 /* Per-channel registers. */ 40 #define JZ_DMA_REG_CHAN(n) (n * 0x20) 41 #define JZ_DMA_REG_DSA(n) (0x00 + JZ_DMA_REG_CHAN(n)) 42 #define JZ_DMA_REG_DTA(n) (0x04 + JZ_DMA_REG_CHAN(n)) 43 #define JZ_DMA_REG_DTC(n) (0x08 + JZ_DMA_REG_CHAN(n)) 44 #define JZ_DMA_REG_DRT(n) (0x0c + JZ_DMA_REG_CHAN(n)) 45 #define JZ_DMA_REG_DCS(n) (0x10 + JZ_DMA_REG_CHAN(n)) 46 #define JZ_DMA_REG_DCM(n) (0x14 + JZ_DMA_REG_CHAN(n)) 47 #define JZ_DMA_REG_DDA(n) (0x18 + JZ_DMA_REG_CHAN(n)) 48 #define JZ_DMA_REG_DSD(n) (0x1c + JZ_DMA_REG_CHAN(n)) 49 50 #define JZ_DMA_DMAC_DMAE BIT(0) 51 #define JZ_DMA_DMAC_AR BIT(2) 52 #define JZ_DMA_DMAC_HLT BIT(3) 53 #define JZ_DMA_DMAC_FMSC BIT(31) 54 55 #define JZ_DMA_DRT_AUTO 0x8 56 57 #define JZ_DMA_DCS_CTE BIT(0) 58 #define JZ_DMA_DCS_HLT BIT(2) 59 #define JZ_DMA_DCS_TT BIT(3) 60 #define JZ_DMA_DCS_AR BIT(4) 61 #define JZ_DMA_DCS_DES8 BIT(30) 62 63 #define JZ_DMA_DCM_LINK BIT(0) 64 #define JZ_DMA_DCM_TIE BIT(1) 65 #define JZ_DMA_DCM_STDE BIT(2) 66 #define JZ_DMA_DCM_TSZ_SHIFT 8 67 #define JZ_DMA_DCM_TSZ_MASK (0x7 << JZ_DMA_DCM_TSZ_SHIFT) 68 #define JZ_DMA_DCM_DP_SHIFT 12 69 #define JZ_DMA_DCM_SP_SHIFT 14 70 #define JZ_DMA_DCM_DAI BIT(22) 71 #define JZ_DMA_DCM_SAI BIT(23) 72 73 #define JZ_DMA_SIZE_4_BYTE 0x0 74 #define JZ_DMA_SIZE_1_BYTE 0x1 75 #define JZ_DMA_SIZE_2_BYTE 0x2 76 #define JZ_DMA_SIZE_16_BYTE 0x3 77 #define JZ_DMA_SIZE_32_BYTE 0x4 78 #define JZ_DMA_SIZE_64_BYTE 0x5 79 #define JZ_DMA_SIZE_128_BYTE 0x6 80 81 #define JZ_DMA_WIDTH_32_BIT 0x0 82 #define JZ_DMA_WIDTH_8_BIT 0x1 83 #define JZ_DMA_WIDTH_16_BIT 0x2 84 85 #define JZ_DMA_BUSWIDTHS (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \ 86 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \ 87 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES)) 88 89 /** 90 * struct jz4780_dma_hwdesc - descriptor structure read by the DMA controller. 91 * @dcm: value for the DCM (channel command) register 92 * @dsa: source address 93 * @dta: target address 94 * @dtc: transfer count (number of blocks of the transfer size specified in DCM 95 * to transfer) in the low 24 bits, offset of the next descriptor from the 96 * descriptor base address in the upper 8 bits. 97 * @sd: target/source stride difference (in stride transfer mode). 98 * @drt: request type 99 */ 100 struct jz4780_dma_hwdesc { 101 uint32_t dcm; 102 uint32_t dsa; 103 uint32_t dta; 104 uint32_t dtc; 105 uint32_t sd; 106 uint32_t drt; 107 uint32_t reserved[2]; 108 }; 109 110 /* Size of allocations for hardware descriptor blocks. */ 111 #define JZ_DMA_DESC_BLOCK_SIZE PAGE_SIZE 112 #define JZ_DMA_MAX_DESC \ 113 (JZ_DMA_DESC_BLOCK_SIZE / sizeof(struct jz4780_dma_hwdesc)) 114 115 struct jz4780_dma_desc { 116 struct virt_dma_desc vdesc; 117 118 struct jz4780_dma_hwdesc *desc; 119 dma_addr_t desc_phys; 120 unsigned int count; 121 enum dma_transaction_type type; 122 uint32_t status; 123 }; 124 125 struct jz4780_dma_chan { 126 struct virt_dma_chan vchan; 127 unsigned int id; 128 struct dma_pool *desc_pool; 129 130 uint32_t transfer_type; 131 uint32_t transfer_shift; 132 struct dma_slave_config config; 133 134 struct jz4780_dma_desc *desc; 135 unsigned int curr_hwdesc; 136 }; 137 138 struct jz4780_dma_dev { 139 struct dma_device dma_device; 140 void __iomem *base; 141 struct clk *clk; 142 unsigned int irq; 143 144 uint32_t chan_reserved; 145 struct jz4780_dma_chan chan[JZ_DMA_NR_CHANNELS]; 146 }; 147 148 struct jz4780_dma_filter_data { 149 struct device_node *of_node; 150 uint32_t transfer_type; 151 int channel; 152 }; 153 154 static inline struct jz4780_dma_chan *to_jz4780_dma_chan(struct dma_chan *chan) 155 { 156 return container_of(chan, struct jz4780_dma_chan, vchan.chan); 157 } 158 159 static inline struct jz4780_dma_desc *to_jz4780_dma_desc( 160 struct virt_dma_desc *vdesc) 161 { 162 return container_of(vdesc, struct jz4780_dma_desc, vdesc); 163 } 164 165 static inline struct jz4780_dma_dev *jz4780_dma_chan_parent( 166 struct jz4780_dma_chan *jzchan) 167 { 168 return container_of(jzchan->vchan.chan.device, struct jz4780_dma_dev, 169 dma_device); 170 } 171 172 static inline uint32_t jz4780_dma_readl(struct jz4780_dma_dev *jzdma, 173 unsigned int reg) 174 { 175 return readl(jzdma->base + reg); 176 } 177 178 static inline void jz4780_dma_writel(struct jz4780_dma_dev *jzdma, 179 unsigned int reg, uint32_t val) 180 { 181 writel(val, jzdma->base + reg); 182 } 183 184 static struct jz4780_dma_desc *jz4780_dma_desc_alloc( 185 struct jz4780_dma_chan *jzchan, unsigned int count, 186 enum dma_transaction_type type) 187 { 188 struct jz4780_dma_desc *desc; 189 190 if (count > JZ_DMA_MAX_DESC) 191 return NULL; 192 193 desc = kzalloc(sizeof(*desc), GFP_NOWAIT); 194 if (!desc) 195 return NULL; 196 197 desc->desc = dma_pool_alloc(jzchan->desc_pool, GFP_NOWAIT, 198 &desc->desc_phys); 199 if (!desc->desc) { 200 kfree(desc); 201 return NULL; 202 } 203 204 desc->count = count; 205 desc->type = type; 206 return desc; 207 } 208 209 static void jz4780_dma_desc_free(struct virt_dma_desc *vdesc) 210 { 211 struct jz4780_dma_desc *desc = to_jz4780_dma_desc(vdesc); 212 struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(vdesc->tx.chan); 213 214 dma_pool_free(jzchan->desc_pool, desc->desc, desc->desc_phys); 215 kfree(desc); 216 } 217 218 static uint32_t jz4780_dma_transfer_size(unsigned long val, uint32_t *shift) 219 { 220 int ord = ffs(val) - 1; 221 222 /* 223 * 8 byte transfer sizes unsupported so fall back on 4. If it's larger 224 * than the maximum, just limit it. It is perfectly safe to fall back 225 * in this way since we won't exceed the maximum burst size supported 226 * by the device, the only effect is reduced efficiency. This is better 227 * than refusing to perform the request at all. 228 */ 229 if (ord == 3) 230 ord = 2; 231 else if (ord > 7) 232 ord = 7; 233 234 *shift = ord; 235 236 switch (ord) { 237 case 0: 238 return JZ_DMA_SIZE_1_BYTE; 239 case 1: 240 return JZ_DMA_SIZE_2_BYTE; 241 case 2: 242 return JZ_DMA_SIZE_4_BYTE; 243 case 4: 244 return JZ_DMA_SIZE_16_BYTE; 245 case 5: 246 return JZ_DMA_SIZE_32_BYTE; 247 case 6: 248 return JZ_DMA_SIZE_64_BYTE; 249 default: 250 return JZ_DMA_SIZE_128_BYTE; 251 } 252 } 253 254 static int jz4780_dma_setup_hwdesc(struct jz4780_dma_chan *jzchan, 255 struct jz4780_dma_hwdesc *desc, dma_addr_t addr, size_t len, 256 enum dma_transfer_direction direction) 257 { 258 struct dma_slave_config *config = &jzchan->config; 259 uint32_t width, maxburst, tsz; 260 261 if (direction == DMA_MEM_TO_DEV) { 262 desc->dcm = JZ_DMA_DCM_SAI; 263 desc->dsa = addr; 264 desc->dta = config->dst_addr; 265 desc->drt = jzchan->transfer_type; 266 267 width = config->dst_addr_width; 268 maxburst = config->dst_maxburst; 269 } else { 270 desc->dcm = JZ_DMA_DCM_DAI; 271 desc->dsa = config->src_addr; 272 desc->dta = addr; 273 desc->drt = jzchan->transfer_type; 274 275 width = config->src_addr_width; 276 maxburst = config->src_maxburst; 277 } 278 279 /* 280 * This calculates the maximum transfer size that can be used with the 281 * given address, length, width and maximum burst size. The address 282 * must be aligned to the transfer size, the total length must be 283 * divisible by the transfer size, and we must not use more than the 284 * maximum burst specified by the user. 285 */ 286 tsz = jz4780_dma_transfer_size(addr | len | (width * maxburst), 287 &jzchan->transfer_shift); 288 289 switch (width) { 290 case DMA_SLAVE_BUSWIDTH_1_BYTE: 291 case DMA_SLAVE_BUSWIDTH_2_BYTES: 292 break; 293 case DMA_SLAVE_BUSWIDTH_4_BYTES: 294 width = JZ_DMA_WIDTH_32_BIT; 295 break; 296 default: 297 return -EINVAL; 298 } 299 300 desc->dcm |= tsz << JZ_DMA_DCM_TSZ_SHIFT; 301 desc->dcm |= width << JZ_DMA_DCM_SP_SHIFT; 302 desc->dcm |= width << JZ_DMA_DCM_DP_SHIFT; 303 304 desc->dtc = len >> jzchan->transfer_shift; 305 return 0; 306 } 307 308 static struct dma_async_tx_descriptor *jz4780_dma_prep_slave_sg( 309 struct dma_chan *chan, struct scatterlist *sgl, unsigned int sg_len, 310 enum dma_transfer_direction direction, unsigned long flags, 311 void *context) 312 { 313 struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan); 314 struct jz4780_dma_desc *desc; 315 unsigned int i; 316 int err; 317 318 desc = jz4780_dma_desc_alloc(jzchan, sg_len, DMA_SLAVE); 319 if (!desc) 320 return NULL; 321 322 for (i = 0; i < sg_len; i++) { 323 err = jz4780_dma_setup_hwdesc(jzchan, &desc->desc[i], 324 sg_dma_address(&sgl[i]), 325 sg_dma_len(&sgl[i]), 326 direction); 327 if (err < 0) 328 return NULL; 329 330 desc->desc[i].dcm |= JZ_DMA_DCM_TIE; 331 332 if (i != (sg_len - 1)) { 333 /* Automatically proceeed to the next descriptor. */ 334 desc->desc[i].dcm |= JZ_DMA_DCM_LINK; 335 336 /* 337 * The upper 8 bits of the DTC field in the descriptor 338 * must be set to (offset from descriptor base of next 339 * descriptor >> 4). 340 */ 341 desc->desc[i].dtc |= 342 (((i + 1) * sizeof(*desc->desc)) >> 4) << 24; 343 } 344 } 345 346 return vchan_tx_prep(&jzchan->vchan, &desc->vdesc, flags); 347 } 348 349 static struct dma_async_tx_descriptor *jz4780_dma_prep_dma_cyclic( 350 struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len, 351 size_t period_len, enum dma_transfer_direction direction, 352 unsigned long flags) 353 { 354 struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan); 355 struct jz4780_dma_desc *desc; 356 unsigned int periods, i; 357 int err; 358 359 if (buf_len % period_len) 360 return NULL; 361 362 periods = buf_len / period_len; 363 364 desc = jz4780_dma_desc_alloc(jzchan, periods, DMA_CYCLIC); 365 if (!desc) 366 return NULL; 367 368 for (i = 0; i < periods; i++) { 369 err = jz4780_dma_setup_hwdesc(jzchan, &desc->desc[i], buf_addr, 370 period_len, direction); 371 if (err < 0) 372 return NULL; 373 374 buf_addr += period_len; 375 376 /* 377 * Set the link bit to indicate that the controller should 378 * automatically proceed to the next descriptor. In 379 * jz4780_dma_begin(), this will be cleared if we need to issue 380 * an interrupt after each period. 381 */ 382 desc->desc[i].dcm |= JZ_DMA_DCM_TIE | JZ_DMA_DCM_LINK; 383 384 /* 385 * The upper 8 bits of the DTC field in the descriptor must be 386 * set to (offset from descriptor base of next descriptor >> 4). 387 * If this is the last descriptor, link it back to the first, 388 * i.e. leave offset set to 0, otherwise point to the next one. 389 */ 390 if (i != (periods - 1)) { 391 desc->desc[i].dtc |= 392 (((i + 1) * sizeof(*desc->desc)) >> 4) << 24; 393 } 394 } 395 396 return vchan_tx_prep(&jzchan->vchan, &desc->vdesc, flags); 397 } 398 399 struct dma_async_tx_descriptor *jz4780_dma_prep_dma_memcpy( 400 struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, 401 size_t len, unsigned long flags) 402 { 403 struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan); 404 struct jz4780_dma_desc *desc; 405 uint32_t tsz; 406 407 desc = jz4780_dma_desc_alloc(jzchan, 1, DMA_MEMCPY); 408 if (!desc) 409 return NULL; 410 411 tsz = jz4780_dma_transfer_size(dest | src | len, 412 &jzchan->transfer_shift); 413 414 desc->desc[0].dsa = src; 415 desc->desc[0].dta = dest; 416 desc->desc[0].drt = JZ_DMA_DRT_AUTO; 417 desc->desc[0].dcm = JZ_DMA_DCM_TIE | JZ_DMA_DCM_SAI | JZ_DMA_DCM_DAI | 418 tsz << JZ_DMA_DCM_TSZ_SHIFT | 419 JZ_DMA_WIDTH_32_BIT << JZ_DMA_DCM_SP_SHIFT | 420 JZ_DMA_WIDTH_32_BIT << JZ_DMA_DCM_DP_SHIFT; 421 desc->desc[0].dtc = len >> jzchan->transfer_shift; 422 423 return vchan_tx_prep(&jzchan->vchan, &desc->vdesc, flags); 424 } 425 426 static void jz4780_dma_begin(struct jz4780_dma_chan *jzchan) 427 { 428 struct jz4780_dma_dev *jzdma = jz4780_dma_chan_parent(jzchan); 429 struct virt_dma_desc *vdesc; 430 unsigned int i; 431 dma_addr_t desc_phys; 432 433 if (!jzchan->desc) { 434 vdesc = vchan_next_desc(&jzchan->vchan); 435 if (!vdesc) 436 return; 437 438 list_del(&vdesc->node); 439 440 jzchan->desc = to_jz4780_dma_desc(vdesc); 441 jzchan->curr_hwdesc = 0; 442 443 if (jzchan->desc->type == DMA_CYCLIC && vdesc->tx.callback) { 444 /* 445 * The DMA controller doesn't support triggering an 446 * interrupt after processing each descriptor, only 447 * after processing an entire terminated list of 448 * descriptors. For a cyclic DMA setup the list of 449 * descriptors is not terminated so we can never get an 450 * interrupt. 451 * 452 * If the user requested a callback for a cyclic DMA 453 * setup then we workaround this hardware limitation 454 * here by degrading to a set of unlinked descriptors 455 * which we will submit in sequence in response to the 456 * completion of processing the previous descriptor. 457 */ 458 for (i = 0; i < jzchan->desc->count; i++) 459 jzchan->desc->desc[i].dcm &= ~JZ_DMA_DCM_LINK; 460 } 461 } else { 462 /* 463 * There is an existing transfer, therefore this must be one 464 * for which we unlinked the descriptors above. Advance to the 465 * next one in the list. 466 */ 467 jzchan->curr_hwdesc = 468 (jzchan->curr_hwdesc + 1) % jzchan->desc->count; 469 } 470 471 /* Use 8-word descriptors. */ 472 jz4780_dma_writel(jzdma, JZ_DMA_REG_DCS(jzchan->id), JZ_DMA_DCS_DES8); 473 474 /* Write descriptor address and initiate descriptor fetch. */ 475 desc_phys = jzchan->desc->desc_phys + 476 (jzchan->curr_hwdesc * sizeof(*jzchan->desc->desc)); 477 jz4780_dma_writel(jzdma, JZ_DMA_REG_DDA(jzchan->id), desc_phys); 478 jz4780_dma_writel(jzdma, JZ_DMA_REG_DDRS, BIT(jzchan->id)); 479 480 /* Enable the channel. */ 481 jz4780_dma_writel(jzdma, JZ_DMA_REG_DCS(jzchan->id), 482 JZ_DMA_DCS_DES8 | JZ_DMA_DCS_CTE); 483 } 484 485 static void jz4780_dma_issue_pending(struct dma_chan *chan) 486 { 487 struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan); 488 unsigned long flags; 489 490 spin_lock_irqsave(&jzchan->vchan.lock, flags); 491 492 if (vchan_issue_pending(&jzchan->vchan) && !jzchan->desc) 493 jz4780_dma_begin(jzchan); 494 495 spin_unlock_irqrestore(&jzchan->vchan.lock, flags); 496 } 497 498 static int jz4780_dma_terminate_all(struct dma_chan *chan) 499 { 500 struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan); 501 struct jz4780_dma_dev *jzdma = jz4780_dma_chan_parent(jzchan); 502 unsigned long flags; 503 LIST_HEAD(head); 504 505 spin_lock_irqsave(&jzchan->vchan.lock, flags); 506 507 /* Clear the DMA status and stop the transfer. */ 508 jz4780_dma_writel(jzdma, JZ_DMA_REG_DCS(jzchan->id), 0); 509 if (jzchan->desc) { 510 jz4780_dma_desc_free(&jzchan->desc->vdesc); 511 jzchan->desc = NULL; 512 } 513 514 vchan_get_all_descriptors(&jzchan->vchan, &head); 515 516 spin_unlock_irqrestore(&jzchan->vchan.lock, flags); 517 518 vchan_dma_desc_free_list(&jzchan->vchan, &head); 519 return 0; 520 } 521 522 static int jz4780_dma_config(struct dma_chan *chan, 523 struct dma_slave_config *config) 524 { 525 struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan); 526 527 if ((config->src_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES) 528 || (config->dst_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES)) 529 return -EINVAL; 530 531 /* Copy the reset of the slave configuration, it is used later. */ 532 memcpy(&jzchan->config, config, sizeof(jzchan->config)); 533 534 return 0; 535 } 536 537 static size_t jz4780_dma_desc_residue(struct jz4780_dma_chan *jzchan, 538 struct jz4780_dma_desc *desc, unsigned int next_sg) 539 { 540 struct jz4780_dma_dev *jzdma = jz4780_dma_chan_parent(jzchan); 541 unsigned int residue, count; 542 unsigned int i; 543 544 residue = 0; 545 546 for (i = next_sg; i < desc->count; i++) 547 residue += desc->desc[i].dtc << jzchan->transfer_shift; 548 549 if (next_sg != 0) { 550 count = jz4780_dma_readl(jzdma, 551 JZ_DMA_REG_DTC(jzchan->id)); 552 residue += count << jzchan->transfer_shift; 553 } 554 555 return residue; 556 } 557 558 static enum dma_status jz4780_dma_tx_status(struct dma_chan *chan, 559 dma_cookie_t cookie, struct dma_tx_state *txstate) 560 { 561 struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan); 562 struct virt_dma_desc *vdesc; 563 enum dma_status status; 564 unsigned long flags; 565 566 status = dma_cookie_status(chan, cookie, txstate); 567 if ((status == DMA_COMPLETE) || (txstate == NULL)) 568 return status; 569 570 spin_lock_irqsave(&jzchan->vchan.lock, flags); 571 572 vdesc = vchan_find_desc(&jzchan->vchan, cookie); 573 if (vdesc) { 574 /* On the issued list, so hasn't been processed yet */ 575 txstate->residue = jz4780_dma_desc_residue(jzchan, 576 to_jz4780_dma_desc(vdesc), 0); 577 } else if (cookie == jzchan->desc->vdesc.tx.cookie) { 578 txstate->residue = jz4780_dma_desc_residue(jzchan, jzchan->desc, 579 (jzchan->curr_hwdesc + 1) % jzchan->desc->count); 580 } else 581 txstate->residue = 0; 582 583 if (vdesc && jzchan->desc && vdesc == &jzchan->desc->vdesc 584 && jzchan->desc->status & (JZ_DMA_DCS_AR | JZ_DMA_DCS_HLT)) 585 status = DMA_ERROR; 586 587 spin_unlock_irqrestore(&jzchan->vchan.lock, flags); 588 return status; 589 } 590 591 static void jz4780_dma_chan_irq(struct jz4780_dma_dev *jzdma, 592 struct jz4780_dma_chan *jzchan) 593 { 594 uint32_t dcs; 595 596 spin_lock(&jzchan->vchan.lock); 597 598 dcs = jz4780_dma_readl(jzdma, JZ_DMA_REG_DCS(jzchan->id)); 599 jz4780_dma_writel(jzdma, JZ_DMA_REG_DCS(jzchan->id), 0); 600 601 if (dcs & JZ_DMA_DCS_AR) { 602 dev_warn(&jzchan->vchan.chan.dev->device, 603 "address error (DCS=0x%x)\n", dcs); 604 } 605 606 if (dcs & JZ_DMA_DCS_HLT) { 607 dev_warn(&jzchan->vchan.chan.dev->device, 608 "channel halt (DCS=0x%x)\n", dcs); 609 } 610 611 if (jzchan->desc) { 612 jzchan->desc->status = dcs; 613 614 if ((dcs & (JZ_DMA_DCS_AR | JZ_DMA_DCS_HLT)) == 0) { 615 if (jzchan->desc->type == DMA_CYCLIC) { 616 vchan_cyclic_callback(&jzchan->desc->vdesc); 617 } else { 618 vchan_cookie_complete(&jzchan->desc->vdesc); 619 jzchan->desc = NULL; 620 } 621 622 jz4780_dma_begin(jzchan); 623 } 624 } else { 625 dev_err(&jzchan->vchan.chan.dev->device, 626 "channel IRQ with no active transfer\n"); 627 } 628 629 spin_unlock(&jzchan->vchan.lock); 630 } 631 632 static irqreturn_t jz4780_dma_irq_handler(int irq, void *data) 633 { 634 struct jz4780_dma_dev *jzdma = data; 635 uint32_t pending, dmac; 636 int i; 637 638 pending = jz4780_dma_readl(jzdma, JZ_DMA_REG_DIRQP); 639 640 for (i = 0; i < JZ_DMA_NR_CHANNELS; i++) { 641 if (!(pending & (1<<i))) 642 continue; 643 644 jz4780_dma_chan_irq(jzdma, &jzdma->chan[i]); 645 } 646 647 /* Clear halt and address error status of all channels. */ 648 dmac = jz4780_dma_readl(jzdma, JZ_DMA_REG_DMAC); 649 dmac &= ~(JZ_DMA_DMAC_HLT | JZ_DMA_DMAC_AR); 650 jz4780_dma_writel(jzdma, JZ_DMA_REG_DMAC, dmac); 651 652 /* Clear interrupt pending status. */ 653 jz4780_dma_writel(jzdma, JZ_DMA_REG_DIRQP, 0); 654 655 return IRQ_HANDLED; 656 } 657 658 static int jz4780_dma_alloc_chan_resources(struct dma_chan *chan) 659 { 660 struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan); 661 662 jzchan->desc_pool = dma_pool_create(dev_name(&chan->dev->device), 663 chan->device->dev, 664 JZ_DMA_DESC_BLOCK_SIZE, 665 PAGE_SIZE, 0); 666 if (!jzchan->desc_pool) { 667 dev_err(&chan->dev->device, 668 "failed to allocate descriptor pool\n"); 669 return -ENOMEM; 670 } 671 672 return 0; 673 } 674 675 static void jz4780_dma_free_chan_resources(struct dma_chan *chan) 676 { 677 struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan); 678 679 vchan_free_chan_resources(&jzchan->vchan); 680 dma_pool_destroy(jzchan->desc_pool); 681 jzchan->desc_pool = NULL; 682 } 683 684 static bool jz4780_dma_filter_fn(struct dma_chan *chan, void *param) 685 { 686 struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan); 687 struct jz4780_dma_dev *jzdma = jz4780_dma_chan_parent(jzchan); 688 struct jz4780_dma_filter_data *data = param; 689 690 if (jzdma->dma_device.dev->of_node != data->of_node) 691 return false; 692 693 if (data->channel > -1) { 694 if (data->channel != jzchan->id) 695 return false; 696 } else if (jzdma->chan_reserved & BIT(jzchan->id)) { 697 return false; 698 } 699 700 jzchan->transfer_type = data->transfer_type; 701 702 return true; 703 } 704 705 static struct dma_chan *jz4780_of_dma_xlate(struct of_phandle_args *dma_spec, 706 struct of_dma *ofdma) 707 { 708 struct jz4780_dma_dev *jzdma = ofdma->of_dma_data; 709 dma_cap_mask_t mask = jzdma->dma_device.cap_mask; 710 struct jz4780_dma_filter_data data; 711 712 if (dma_spec->args_count != 2) 713 return NULL; 714 715 data.of_node = ofdma->of_node; 716 data.transfer_type = dma_spec->args[0]; 717 data.channel = dma_spec->args[1]; 718 719 if (data.channel > -1) { 720 if (data.channel >= JZ_DMA_NR_CHANNELS) { 721 dev_err(jzdma->dma_device.dev, 722 "device requested non-existent channel %u\n", 723 data.channel); 724 return NULL; 725 } 726 727 /* Can only select a channel marked as reserved. */ 728 if (!(jzdma->chan_reserved & BIT(data.channel))) { 729 dev_err(jzdma->dma_device.dev, 730 "device requested unreserved channel %u\n", 731 data.channel); 732 return NULL; 733 } 734 735 jzdma->chan[data.channel].transfer_type = data.transfer_type; 736 737 return dma_get_slave_channel( 738 &jzdma->chan[data.channel].vchan.chan); 739 } else { 740 return dma_request_channel(mask, jz4780_dma_filter_fn, &data); 741 } 742 } 743 744 static int jz4780_dma_probe(struct platform_device *pdev) 745 { 746 struct device *dev = &pdev->dev; 747 struct jz4780_dma_dev *jzdma; 748 struct jz4780_dma_chan *jzchan; 749 struct dma_device *dd; 750 struct resource *res; 751 int i, ret; 752 753 jzdma = devm_kzalloc(dev, sizeof(*jzdma), GFP_KERNEL); 754 if (!jzdma) 755 return -ENOMEM; 756 757 platform_set_drvdata(pdev, jzdma); 758 759 res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 760 if (!res) { 761 dev_err(dev, "failed to get I/O memory\n"); 762 return -EINVAL; 763 } 764 765 jzdma->base = devm_ioremap_resource(dev, res); 766 if (IS_ERR(jzdma->base)) 767 return PTR_ERR(jzdma->base); 768 769 ret = platform_get_irq(pdev, 0); 770 if (ret < 0) { 771 dev_err(dev, "failed to get IRQ: %d\n", ret); 772 return ret; 773 } 774 775 jzdma->irq = ret; 776 777 ret = devm_request_irq(dev, jzdma->irq, jz4780_dma_irq_handler, 0, 778 dev_name(dev), jzdma); 779 if (ret) { 780 dev_err(dev, "failed to request IRQ %u!\n", jzdma->irq); 781 return ret; 782 } 783 784 jzdma->clk = devm_clk_get(dev, NULL); 785 if (IS_ERR(jzdma->clk)) { 786 dev_err(dev, "failed to get clock\n"); 787 return PTR_ERR(jzdma->clk); 788 } 789 790 clk_prepare_enable(jzdma->clk); 791 792 /* Property is optional, if it doesn't exist the value will remain 0. */ 793 of_property_read_u32_index(dev->of_node, "ingenic,reserved-channels", 794 0, &jzdma->chan_reserved); 795 796 dd = &jzdma->dma_device; 797 798 dma_cap_set(DMA_MEMCPY, dd->cap_mask); 799 dma_cap_set(DMA_SLAVE, dd->cap_mask); 800 dma_cap_set(DMA_CYCLIC, dd->cap_mask); 801 802 dd->dev = dev; 803 dd->copy_align = DMAENGINE_ALIGN_4_BYTES; 804 dd->device_alloc_chan_resources = jz4780_dma_alloc_chan_resources; 805 dd->device_free_chan_resources = jz4780_dma_free_chan_resources; 806 dd->device_prep_slave_sg = jz4780_dma_prep_slave_sg; 807 dd->device_prep_dma_cyclic = jz4780_dma_prep_dma_cyclic; 808 dd->device_prep_dma_memcpy = jz4780_dma_prep_dma_memcpy; 809 dd->device_config = jz4780_dma_config; 810 dd->device_terminate_all = jz4780_dma_terminate_all; 811 dd->device_tx_status = jz4780_dma_tx_status; 812 dd->device_issue_pending = jz4780_dma_issue_pending; 813 dd->src_addr_widths = JZ_DMA_BUSWIDTHS; 814 dd->dst_addr_widths = JZ_DMA_BUSWIDTHS; 815 dd->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); 816 dd->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; 817 818 /* 819 * Enable DMA controller, mark all channels as not programmable. 820 * Also set the FMSC bit - it increases MSC performance, so it makes 821 * little sense not to enable it. 822 */ 823 jz4780_dma_writel(jzdma, JZ_DMA_REG_DMAC, 824 JZ_DMA_DMAC_DMAE | JZ_DMA_DMAC_FMSC); 825 jz4780_dma_writel(jzdma, JZ_DMA_REG_DMACP, 0); 826 827 INIT_LIST_HEAD(&dd->channels); 828 829 for (i = 0; i < JZ_DMA_NR_CHANNELS; i++) { 830 jzchan = &jzdma->chan[i]; 831 jzchan->id = i; 832 833 vchan_init(&jzchan->vchan, dd); 834 jzchan->vchan.desc_free = jz4780_dma_desc_free; 835 } 836 837 ret = dma_async_device_register(dd); 838 if (ret) { 839 dev_err(dev, "failed to register device\n"); 840 goto err_disable_clk; 841 } 842 843 /* Register with OF DMA helpers. */ 844 ret = of_dma_controller_register(dev->of_node, jz4780_of_dma_xlate, 845 jzdma); 846 if (ret) { 847 dev_err(dev, "failed to register OF DMA controller\n"); 848 goto err_unregister_dev; 849 } 850 851 dev_info(dev, "JZ4780 DMA controller initialised\n"); 852 return 0; 853 854 err_unregister_dev: 855 dma_async_device_unregister(dd); 856 857 err_disable_clk: 858 clk_disable_unprepare(jzdma->clk); 859 return ret; 860 } 861 862 static int jz4780_dma_remove(struct platform_device *pdev) 863 { 864 struct jz4780_dma_dev *jzdma = platform_get_drvdata(pdev); 865 866 of_dma_controller_free(pdev->dev.of_node); 867 devm_free_irq(&pdev->dev, jzdma->irq, jzdma); 868 dma_async_device_unregister(&jzdma->dma_device); 869 return 0; 870 } 871 872 static const struct of_device_id jz4780_dma_dt_match[] = { 873 { .compatible = "ingenic,jz4780-dma", .data = NULL }, 874 {}, 875 }; 876 MODULE_DEVICE_TABLE(of, jz4780_dma_dt_match); 877 878 static struct platform_driver jz4780_dma_driver = { 879 .probe = jz4780_dma_probe, 880 .remove = jz4780_dma_remove, 881 .driver = { 882 .name = "jz4780-dma", 883 .of_match_table = of_match_ptr(jz4780_dma_dt_match), 884 }, 885 }; 886 887 static int __init jz4780_dma_init(void) 888 { 889 return platform_driver_register(&jz4780_dma_driver); 890 } 891 subsys_initcall(jz4780_dma_init); 892 893 static void __exit jz4780_dma_exit(void) 894 { 895 platform_driver_unregister(&jz4780_dma_driver); 896 } 897 module_exit(jz4780_dma_exit); 898 899 MODULE_AUTHOR("Alex Smith <alex@alex-smith.me.uk>"); 900 MODULE_DESCRIPTION("Ingenic JZ4780 DMA controller driver"); 901 MODULE_LICENSE("GPL"); 902