1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Ingenic JZ4780 DMA controller 4 * 5 * Copyright (c) 2015 Imagination Technologies 6 * Author: Alex Smith <alex@alex-smith.me.uk> 7 */ 8 9 #include <linux/clk.h> 10 #include <linux/dmapool.h> 11 #include <linux/init.h> 12 #include <linux/interrupt.h> 13 #include <linux/module.h> 14 #include <linux/of.h> 15 #include <linux/of_device.h> 16 #include <linux/of_dma.h> 17 #include <linux/platform_device.h> 18 #include <linux/slab.h> 19 20 #include "dmaengine.h" 21 #include "virt-dma.h" 22 23 /* Global registers. */ 24 #define JZ_DMA_REG_DMAC 0x00 25 #define JZ_DMA_REG_DIRQP 0x04 26 #define JZ_DMA_REG_DDR 0x08 27 #define JZ_DMA_REG_DDRS 0x0c 28 #define JZ_DMA_REG_DCKE 0x10 29 #define JZ_DMA_REG_DCKES 0x14 30 #define JZ_DMA_REG_DCKEC 0x18 31 #define JZ_DMA_REG_DMACP 0x1c 32 #define JZ_DMA_REG_DSIRQP 0x20 33 #define JZ_DMA_REG_DSIRQM 0x24 34 #define JZ_DMA_REG_DCIRQP 0x28 35 #define JZ_DMA_REG_DCIRQM 0x2c 36 37 /* Per-channel registers. */ 38 #define JZ_DMA_REG_CHAN(n) (n * 0x20) 39 #define JZ_DMA_REG_DSA 0x00 40 #define JZ_DMA_REG_DTA 0x04 41 #define JZ_DMA_REG_DTC 0x08 42 #define JZ_DMA_REG_DRT 0x0c 43 #define JZ_DMA_REG_DCS 0x10 44 #define JZ_DMA_REG_DCM 0x14 45 #define JZ_DMA_REG_DDA 0x18 46 #define JZ_DMA_REG_DSD 0x1c 47 48 #define JZ_DMA_DMAC_DMAE BIT(0) 49 #define JZ_DMA_DMAC_AR BIT(2) 50 #define JZ_DMA_DMAC_HLT BIT(3) 51 #define JZ_DMA_DMAC_FAIC BIT(27) 52 #define JZ_DMA_DMAC_FMSC BIT(31) 53 54 #define JZ_DMA_DRT_AUTO 0x8 55 56 #define JZ_DMA_DCS_CTE BIT(0) 57 #define JZ_DMA_DCS_HLT BIT(2) 58 #define JZ_DMA_DCS_TT BIT(3) 59 #define JZ_DMA_DCS_AR BIT(4) 60 #define JZ_DMA_DCS_DES8 BIT(30) 61 62 #define JZ_DMA_DCM_LINK BIT(0) 63 #define JZ_DMA_DCM_TIE BIT(1) 64 #define JZ_DMA_DCM_STDE BIT(2) 65 #define JZ_DMA_DCM_TSZ_SHIFT 8 66 #define JZ_DMA_DCM_TSZ_MASK (0x7 << JZ_DMA_DCM_TSZ_SHIFT) 67 #define JZ_DMA_DCM_DP_SHIFT 12 68 #define JZ_DMA_DCM_SP_SHIFT 14 69 #define JZ_DMA_DCM_DAI BIT(22) 70 #define JZ_DMA_DCM_SAI BIT(23) 71 72 #define JZ_DMA_SIZE_4_BYTE 0x0 73 #define JZ_DMA_SIZE_1_BYTE 0x1 74 #define JZ_DMA_SIZE_2_BYTE 0x2 75 #define JZ_DMA_SIZE_16_BYTE 0x3 76 #define JZ_DMA_SIZE_32_BYTE 0x4 77 #define JZ_DMA_SIZE_64_BYTE 0x5 78 #define JZ_DMA_SIZE_128_BYTE 0x6 79 80 #define JZ_DMA_WIDTH_32_BIT 0x0 81 #define JZ_DMA_WIDTH_8_BIT 0x1 82 #define JZ_DMA_WIDTH_16_BIT 0x2 83 84 #define JZ_DMA_BUSWIDTHS (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \ 85 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \ 86 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES)) 87 88 #define JZ4780_DMA_CTRL_OFFSET 0x1000 89 90 /* macros for use with jz4780_dma_soc_data.flags */ 91 #define JZ_SOC_DATA_ALLOW_LEGACY_DT BIT(0) 92 #define JZ_SOC_DATA_PROGRAMMABLE_DMA BIT(1) 93 #define JZ_SOC_DATA_PER_CHAN_PM BIT(2) 94 #define JZ_SOC_DATA_NO_DCKES_DCKEC BIT(3) 95 #define JZ_SOC_DATA_BREAK_LINKS BIT(4) 96 97 /** 98 * struct jz4780_dma_hwdesc - descriptor structure read by the DMA controller. 99 * @dcm: value for the DCM (channel command) register 100 * @dsa: source address 101 * @dta: target address 102 * @dtc: transfer count (number of blocks of the transfer size specified in DCM 103 * to transfer) in the low 24 bits, offset of the next descriptor from the 104 * descriptor base address in the upper 8 bits. 105 */ 106 struct jz4780_dma_hwdesc { 107 uint32_t dcm; 108 uint32_t dsa; 109 uint32_t dta; 110 uint32_t dtc; 111 }; 112 113 /* Size of allocations for hardware descriptor blocks. */ 114 #define JZ_DMA_DESC_BLOCK_SIZE PAGE_SIZE 115 #define JZ_DMA_MAX_DESC \ 116 (JZ_DMA_DESC_BLOCK_SIZE / sizeof(struct jz4780_dma_hwdesc)) 117 118 struct jz4780_dma_desc { 119 struct virt_dma_desc vdesc; 120 121 struct jz4780_dma_hwdesc *desc; 122 dma_addr_t desc_phys; 123 unsigned int count; 124 enum dma_transaction_type type; 125 uint32_t status; 126 }; 127 128 struct jz4780_dma_chan { 129 struct virt_dma_chan vchan; 130 unsigned int id; 131 struct dma_pool *desc_pool; 132 133 uint32_t transfer_type; 134 uint32_t transfer_shift; 135 struct dma_slave_config config; 136 137 struct jz4780_dma_desc *desc; 138 unsigned int curr_hwdesc; 139 }; 140 141 struct jz4780_dma_soc_data { 142 unsigned int nb_channels; 143 unsigned int transfer_ord_max; 144 unsigned long flags; 145 }; 146 147 struct jz4780_dma_dev { 148 struct dma_device dma_device; 149 void __iomem *chn_base; 150 void __iomem *ctrl_base; 151 struct clk *clk; 152 unsigned int irq; 153 const struct jz4780_dma_soc_data *soc_data; 154 155 uint32_t chan_reserved; 156 struct jz4780_dma_chan chan[]; 157 }; 158 159 struct jz4780_dma_filter_data { 160 uint32_t transfer_type; 161 int channel; 162 }; 163 164 static inline struct jz4780_dma_chan *to_jz4780_dma_chan(struct dma_chan *chan) 165 { 166 return container_of(chan, struct jz4780_dma_chan, vchan.chan); 167 } 168 169 static inline struct jz4780_dma_desc *to_jz4780_dma_desc( 170 struct virt_dma_desc *vdesc) 171 { 172 return container_of(vdesc, struct jz4780_dma_desc, vdesc); 173 } 174 175 static inline struct jz4780_dma_dev *jz4780_dma_chan_parent( 176 struct jz4780_dma_chan *jzchan) 177 { 178 return container_of(jzchan->vchan.chan.device, struct jz4780_dma_dev, 179 dma_device); 180 } 181 182 static inline uint32_t jz4780_dma_chn_readl(struct jz4780_dma_dev *jzdma, 183 unsigned int chn, unsigned int reg) 184 { 185 return readl(jzdma->chn_base + reg + JZ_DMA_REG_CHAN(chn)); 186 } 187 188 static inline void jz4780_dma_chn_writel(struct jz4780_dma_dev *jzdma, 189 unsigned int chn, unsigned int reg, uint32_t val) 190 { 191 writel(val, jzdma->chn_base + reg + JZ_DMA_REG_CHAN(chn)); 192 } 193 194 static inline uint32_t jz4780_dma_ctrl_readl(struct jz4780_dma_dev *jzdma, 195 unsigned int reg) 196 { 197 return readl(jzdma->ctrl_base + reg); 198 } 199 200 static inline void jz4780_dma_ctrl_writel(struct jz4780_dma_dev *jzdma, 201 unsigned int reg, uint32_t val) 202 { 203 writel(val, jzdma->ctrl_base + reg); 204 } 205 206 static inline void jz4780_dma_chan_enable(struct jz4780_dma_dev *jzdma, 207 unsigned int chn) 208 { 209 if (jzdma->soc_data->flags & JZ_SOC_DATA_PER_CHAN_PM) { 210 unsigned int reg; 211 212 if (jzdma->soc_data->flags & JZ_SOC_DATA_NO_DCKES_DCKEC) 213 reg = JZ_DMA_REG_DCKE; 214 else 215 reg = JZ_DMA_REG_DCKES; 216 217 jz4780_dma_ctrl_writel(jzdma, reg, BIT(chn)); 218 } 219 } 220 221 static inline void jz4780_dma_chan_disable(struct jz4780_dma_dev *jzdma, 222 unsigned int chn) 223 { 224 if ((jzdma->soc_data->flags & JZ_SOC_DATA_PER_CHAN_PM) && 225 !(jzdma->soc_data->flags & JZ_SOC_DATA_NO_DCKES_DCKEC)) 226 jz4780_dma_ctrl_writel(jzdma, JZ_DMA_REG_DCKEC, BIT(chn)); 227 } 228 229 static struct jz4780_dma_desc *jz4780_dma_desc_alloc( 230 struct jz4780_dma_chan *jzchan, unsigned int count, 231 enum dma_transaction_type type) 232 { 233 struct jz4780_dma_desc *desc; 234 235 if (count > JZ_DMA_MAX_DESC) 236 return NULL; 237 238 desc = kzalloc(sizeof(*desc), GFP_NOWAIT); 239 if (!desc) 240 return NULL; 241 242 desc->desc = dma_pool_alloc(jzchan->desc_pool, GFP_NOWAIT, 243 &desc->desc_phys); 244 if (!desc->desc) { 245 kfree(desc); 246 return NULL; 247 } 248 249 desc->count = count; 250 desc->type = type; 251 return desc; 252 } 253 254 static void jz4780_dma_desc_free(struct virt_dma_desc *vdesc) 255 { 256 struct jz4780_dma_desc *desc = to_jz4780_dma_desc(vdesc); 257 struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(vdesc->tx.chan); 258 259 dma_pool_free(jzchan->desc_pool, desc->desc, desc->desc_phys); 260 kfree(desc); 261 } 262 263 static uint32_t jz4780_dma_transfer_size(struct jz4780_dma_chan *jzchan, 264 unsigned long val, uint32_t *shift) 265 { 266 struct jz4780_dma_dev *jzdma = jz4780_dma_chan_parent(jzchan); 267 int ord = ffs(val) - 1; 268 269 /* 270 * 8 byte transfer sizes unsupported so fall back on 4. If it's larger 271 * than the maximum, just limit it. It is perfectly safe to fall back 272 * in this way since we won't exceed the maximum burst size supported 273 * by the device, the only effect is reduced efficiency. This is better 274 * than refusing to perform the request at all. 275 */ 276 if (ord == 3) 277 ord = 2; 278 else if (ord > jzdma->soc_data->transfer_ord_max) 279 ord = jzdma->soc_data->transfer_ord_max; 280 281 *shift = ord; 282 283 switch (ord) { 284 case 0: 285 return JZ_DMA_SIZE_1_BYTE; 286 case 1: 287 return JZ_DMA_SIZE_2_BYTE; 288 case 2: 289 return JZ_DMA_SIZE_4_BYTE; 290 case 4: 291 return JZ_DMA_SIZE_16_BYTE; 292 case 5: 293 return JZ_DMA_SIZE_32_BYTE; 294 case 6: 295 return JZ_DMA_SIZE_64_BYTE; 296 default: 297 return JZ_DMA_SIZE_128_BYTE; 298 } 299 } 300 301 static int jz4780_dma_setup_hwdesc(struct jz4780_dma_chan *jzchan, 302 struct jz4780_dma_hwdesc *desc, dma_addr_t addr, size_t len, 303 enum dma_transfer_direction direction) 304 { 305 struct dma_slave_config *config = &jzchan->config; 306 uint32_t width, maxburst, tsz; 307 308 if (direction == DMA_MEM_TO_DEV) { 309 desc->dcm = JZ_DMA_DCM_SAI; 310 desc->dsa = addr; 311 desc->dta = config->dst_addr; 312 313 width = config->dst_addr_width; 314 maxburst = config->dst_maxburst; 315 } else { 316 desc->dcm = JZ_DMA_DCM_DAI; 317 desc->dsa = config->src_addr; 318 desc->dta = addr; 319 320 width = config->src_addr_width; 321 maxburst = config->src_maxburst; 322 } 323 324 /* 325 * This calculates the maximum transfer size that can be used with the 326 * given address, length, width and maximum burst size. The address 327 * must be aligned to the transfer size, the total length must be 328 * divisible by the transfer size, and we must not use more than the 329 * maximum burst specified by the user. 330 */ 331 tsz = jz4780_dma_transfer_size(jzchan, addr | len | (width * maxburst), 332 &jzchan->transfer_shift); 333 334 switch (width) { 335 case DMA_SLAVE_BUSWIDTH_1_BYTE: 336 case DMA_SLAVE_BUSWIDTH_2_BYTES: 337 break; 338 case DMA_SLAVE_BUSWIDTH_4_BYTES: 339 width = JZ_DMA_WIDTH_32_BIT; 340 break; 341 default: 342 return -EINVAL; 343 } 344 345 desc->dcm |= tsz << JZ_DMA_DCM_TSZ_SHIFT; 346 desc->dcm |= width << JZ_DMA_DCM_SP_SHIFT; 347 desc->dcm |= width << JZ_DMA_DCM_DP_SHIFT; 348 349 desc->dtc = len >> jzchan->transfer_shift; 350 return 0; 351 } 352 353 static struct dma_async_tx_descriptor *jz4780_dma_prep_slave_sg( 354 struct dma_chan *chan, struct scatterlist *sgl, unsigned int sg_len, 355 enum dma_transfer_direction direction, unsigned long flags, 356 void *context) 357 { 358 struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan); 359 struct jz4780_dma_dev *jzdma = jz4780_dma_chan_parent(jzchan); 360 struct jz4780_dma_desc *desc; 361 unsigned int i; 362 int err; 363 364 desc = jz4780_dma_desc_alloc(jzchan, sg_len, DMA_SLAVE); 365 if (!desc) 366 return NULL; 367 368 for (i = 0; i < sg_len; i++) { 369 err = jz4780_dma_setup_hwdesc(jzchan, &desc->desc[i], 370 sg_dma_address(&sgl[i]), 371 sg_dma_len(&sgl[i]), 372 direction); 373 if (err < 0) { 374 jz4780_dma_desc_free(&jzchan->desc->vdesc); 375 return NULL; 376 } 377 378 desc->desc[i].dcm |= JZ_DMA_DCM_TIE; 379 380 if (i != (sg_len - 1) && 381 !(jzdma->soc_data->flags & JZ_SOC_DATA_BREAK_LINKS)) { 382 /* Automatically proceeed to the next descriptor. */ 383 desc->desc[i].dcm |= JZ_DMA_DCM_LINK; 384 385 /* 386 * The upper 8 bits of the DTC field in the descriptor 387 * must be set to (offset from descriptor base of next 388 * descriptor >> 4). 389 */ 390 desc->desc[i].dtc |= 391 (((i + 1) * sizeof(*desc->desc)) >> 4) << 24; 392 } 393 } 394 395 return vchan_tx_prep(&jzchan->vchan, &desc->vdesc, flags); 396 } 397 398 static struct dma_async_tx_descriptor *jz4780_dma_prep_dma_cyclic( 399 struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len, 400 size_t period_len, enum dma_transfer_direction direction, 401 unsigned long flags) 402 { 403 struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan); 404 struct jz4780_dma_desc *desc; 405 unsigned int periods, i; 406 int err; 407 408 if (buf_len % period_len) 409 return NULL; 410 411 periods = buf_len / period_len; 412 413 desc = jz4780_dma_desc_alloc(jzchan, periods, DMA_CYCLIC); 414 if (!desc) 415 return NULL; 416 417 for (i = 0; i < periods; i++) { 418 err = jz4780_dma_setup_hwdesc(jzchan, &desc->desc[i], buf_addr, 419 period_len, direction); 420 if (err < 0) { 421 jz4780_dma_desc_free(&jzchan->desc->vdesc); 422 return NULL; 423 } 424 425 buf_addr += period_len; 426 427 /* 428 * Set the link bit to indicate that the controller should 429 * automatically proceed to the next descriptor. In 430 * jz4780_dma_begin(), this will be cleared if we need to issue 431 * an interrupt after each period. 432 */ 433 desc->desc[i].dcm |= JZ_DMA_DCM_TIE | JZ_DMA_DCM_LINK; 434 435 /* 436 * The upper 8 bits of the DTC field in the descriptor must be 437 * set to (offset from descriptor base of next descriptor >> 4). 438 * If this is the last descriptor, link it back to the first, 439 * i.e. leave offset set to 0, otherwise point to the next one. 440 */ 441 if (i != (periods - 1)) { 442 desc->desc[i].dtc |= 443 (((i + 1) * sizeof(*desc->desc)) >> 4) << 24; 444 } 445 } 446 447 return vchan_tx_prep(&jzchan->vchan, &desc->vdesc, flags); 448 } 449 450 static struct dma_async_tx_descriptor *jz4780_dma_prep_dma_memcpy( 451 struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, 452 size_t len, unsigned long flags) 453 { 454 struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan); 455 struct jz4780_dma_desc *desc; 456 uint32_t tsz; 457 458 desc = jz4780_dma_desc_alloc(jzchan, 1, DMA_MEMCPY); 459 if (!desc) 460 return NULL; 461 462 tsz = jz4780_dma_transfer_size(jzchan, dest | src | len, 463 &jzchan->transfer_shift); 464 465 jzchan->transfer_type = JZ_DMA_DRT_AUTO; 466 467 desc->desc[0].dsa = src; 468 desc->desc[0].dta = dest; 469 desc->desc[0].dcm = JZ_DMA_DCM_TIE | JZ_DMA_DCM_SAI | JZ_DMA_DCM_DAI | 470 tsz << JZ_DMA_DCM_TSZ_SHIFT | 471 JZ_DMA_WIDTH_32_BIT << JZ_DMA_DCM_SP_SHIFT | 472 JZ_DMA_WIDTH_32_BIT << JZ_DMA_DCM_DP_SHIFT; 473 desc->desc[0].dtc = len >> jzchan->transfer_shift; 474 475 return vchan_tx_prep(&jzchan->vchan, &desc->vdesc, flags); 476 } 477 478 static void jz4780_dma_begin(struct jz4780_dma_chan *jzchan) 479 { 480 struct jz4780_dma_dev *jzdma = jz4780_dma_chan_parent(jzchan); 481 struct virt_dma_desc *vdesc; 482 unsigned int i; 483 dma_addr_t desc_phys; 484 485 if (!jzchan->desc) { 486 vdesc = vchan_next_desc(&jzchan->vchan); 487 if (!vdesc) 488 return; 489 490 list_del(&vdesc->node); 491 492 jzchan->desc = to_jz4780_dma_desc(vdesc); 493 jzchan->curr_hwdesc = 0; 494 495 if (jzchan->desc->type == DMA_CYCLIC && vdesc->tx.callback) { 496 /* 497 * The DMA controller doesn't support triggering an 498 * interrupt after processing each descriptor, only 499 * after processing an entire terminated list of 500 * descriptors. For a cyclic DMA setup the list of 501 * descriptors is not terminated so we can never get an 502 * interrupt. 503 * 504 * If the user requested a callback for a cyclic DMA 505 * setup then we workaround this hardware limitation 506 * here by degrading to a set of unlinked descriptors 507 * which we will submit in sequence in response to the 508 * completion of processing the previous descriptor. 509 */ 510 for (i = 0; i < jzchan->desc->count; i++) 511 jzchan->desc->desc[i].dcm &= ~JZ_DMA_DCM_LINK; 512 } 513 } else { 514 /* 515 * There is an existing transfer, therefore this must be one 516 * for which we unlinked the descriptors above. Advance to the 517 * next one in the list. 518 */ 519 jzchan->curr_hwdesc = 520 (jzchan->curr_hwdesc + 1) % jzchan->desc->count; 521 } 522 523 /* Enable the channel's clock. */ 524 jz4780_dma_chan_enable(jzdma, jzchan->id); 525 526 /* Use 4-word descriptors. */ 527 jz4780_dma_chn_writel(jzdma, jzchan->id, JZ_DMA_REG_DCS, 0); 528 529 /* Set transfer type. */ 530 jz4780_dma_chn_writel(jzdma, jzchan->id, JZ_DMA_REG_DRT, 531 jzchan->transfer_type); 532 533 /* 534 * Set the transfer count. This is redundant for a descriptor-driven 535 * transfer. However, there can be a delay between the transfer start 536 * time and when DTCn reg contains the new transfer count. Setting 537 * it explicitly ensures residue is computed correctly at all times. 538 */ 539 jz4780_dma_chn_writel(jzdma, jzchan->id, JZ_DMA_REG_DTC, 540 jzchan->desc->desc[jzchan->curr_hwdesc].dtc); 541 542 /* Write descriptor address and initiate descriptor fetch. */ 543 desc_phys = jzchan->desc->desc_phys + 544 (jzchan->curr_hwdesc * sizeof(*jzchan->desc->desc)); 545 jz4780_dma_chn_writel(jzdma, jzchan->id, JZ_DMA_REG_DDA, desc_phys); 546 jz4780_dma_ctrl_writel(jzdma, JZ_DMA_REG_DDRS, BIT(jzchan->id)); 547 548 /* Enable the channel. */ 549 jz4780_dma_chn_writel(jzdma, jzchan->id, JZ_DMA_REG_DCS, 550 JZ_DMA_DCS_CTE); 551 } 552 553 static void jz4780_dma_issue_pending(struct dma_chan *chan) 554 { 555 struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan); 556 unsigned long flags; 557 558 spin_lock_irqsave(&jzchan->vchan.lock, flags); 559 560 if (vchan_issue_pending(&jzchan->vchan) && !jzchan->desc) 561 jz4780_dma_begin(jzchan); 562 563 spin_unlock_irqrestore(&jzchan->vchan.lock, flags); 564 } 565 566 static int jz4780_dma_terminate_all(struct dma_chan *chan) 567 { 568 struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan); 569 struct jz4780_dma_dev *jzdma = jz4780_dma_chan_parent(jzchan); 570 unsigned long flags; 571 LIST_HEAD(head); 572 573 spin_lock_irqsave(&jzchan->vchan.lock, flags); 574 575 /* Clear the DMA status and stop the transfer. */ 576 jz4780_dma_chn_writel(jzdma, jzchan->id, JZ_DMA_REG_DCS, 0); 577 if (jzchan->desc) { 578 vchan_terminate_vdesc(&jzchan->desc->vdesc); 579 jzchan->desc = NULL; 580 } 581 582 jz4780_dma_chan_disable(jzdma, jzchan->id); 583 584 vchan_get_all_descriptors(&jzchan->vchan, &head); 585 586 spin_unlock_irqrestore(&jzchan->vchan.lock, flags); 587 588 vchan_dma_desc_free_list(&jzchan->vchan, &head); 589 return 0; 590 } 591 592 static void jz4780_dma_synchronize(struct dma_chan *chan) 593 { 594 struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan); 595 struct jz4780_dma_dev *jzdma = jz4780_dma_chan_parent(jzchan); 596 597 vchan_synchronize(&jzchan->vchan); 598 jz4780_dma_chan_disable(jzdma, jzchan->id); 599 } 600 601 static int jz4780_dma_config(struct dma_chan *chan, 602 struct dma_slave_config *config) 603 { 604 struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan); 605 606 if ((config->src_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES) 607 || (config->dst_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES)) 608 return -EINVAL; 609 610 /* Copy the reset of the slave configuration, it is used later. */ 611 memcpy(&jzchan->config, config, sizeof(jzchan->config)); 612 613 return 0; 614 } 615 616 static size_t jz4780_dma_desc_residue(struct jz4780_dma_chan *jzchan, 617 struct jz4780_dma_desc *desc, unsigned int next_sg) 618 { 619 struct jz4780_dma_dev *jzdma = jz4780_dma_chan_parent(jzchan); 620 unsigned int count = 0; 621 unsigned int i; 622 623 for (i = next_sg; i < desc->count; i++) 624 count += desc->desc[i].dtc & GENMASK(23, 0); 625 626 if (next_sg != 0) 627 count += jz4780_dma_chn_readl(jzdma, jzchan->id, 628 JZ_DMA_REG_DTC); 629 630 return count << jzchan->transfer_shift; 631 } 632 633 static enum dma_status jz4780_dma_tx_status(struct dma_chan *chan, 634 dma_cookie_t cookie, struct dma_tx_state *txstate) 635 { 636 struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan); 637 struct virt_dma_desc *vdesc; 638 enum dma_status status; 639 unsigned long flags; 640 unsigned long residue = 0; 641 642 spin_lock_irqsave(&jzchan->vchan.lock, flags); 643 644 status = dma_cookie_status(chan, cookie, txstate); 645 if ((status == DMA_COMPLETE) || (txstate == NULL)) 646 goto out_unlock_irqrestore; 647 648 vdesc = vchan_find_desc(&jzchan->vchan, cookie); 649 if (vdesc) { 650 /* On the issued list, so hasn't been processed yet */ 651 residue = jz4780_dma_desc_residue(jzchan, 652 to_jz4780_dma_desc(vdesc), 0); 653 } else if (cookie == jzchan->desc->vdesc.tx.cookie) { 654 residue = jz4780_dma_desc_residue(jzchan, jzchan->desc, 655 jzchan->curr_hwdesc + 1); 656 } 657 dma_set_residue(txstate, residue); 658 659 if (vdesc && jzchan->desc && vdesc == &jzchan->desc->vdesc 660 && jzchan->desc->status & (JZ_DMA_DCS_AR | JZ_DMA_DCS_HLT)) 661 status = DMA_ERROR; 662 663 out_unlock_irqrestore: 664 spin_unlock_irqrestore(&jzchan->vchan.lock, flags); 665 return status; 666 } 667 668 static bool jz4780_dma_chan_irq(struct jz4780_dma_dev *jzdma, 669 struct jz4780_dma_chan *jzchan) 670 { 671 const unsigned int soc_flags = jzdma->soc_data->flags; 672 struct jz4780_dma_desc *desc = jzchan->desc; 673 uint32_t dcs; 674 bool ack = true; 675 676 spin_lock(&jzchan->vchan.lock); 677 678 dcs = jz4780_dma_chn_readl(jzdma, jzchan->id, JZ_DMA_REG_DCS); 679 jz4780_dma_chn_writel(jzdma, jzchan->id, JZ_DMA_REG_DCS, 0); 680 681 if (dcs & JZ_DMA_DCS_AR) { 682 dev_warn(&jzchan->vchan.chan.dev->device, 683 "address error (DCS=0x%x)\n", dcs); 684 } 685 686 if (dcs & JZ_DMA_DCS_HLT) { 687 dev_warn(&jzchan->vchan.chan.dev->device, 688 "channel halt (DCS=0x%x)\n", dcs); 689 } 690 691 if (jzchan->desc) { 692 jzchan->desc->status = dcs; 693 694 if ((dcs & (JZ_DMA_DCS_AR | JZ_DMA_DCS_HLT)) == 0) { 695 if (jzchan->desc->type == DMA_CYCLIC) { 696 vchan_cyclic_callback(&jzchan->desc->vdesc); 697 698 jz4780_dma_begin(jzchan); 699 } else if (dcs & JZ_DMA_DCS_TT) { 700 if (!(soc_flags & JZ_SOC_DATA_BREAK_LINKS) || 701 (jzchan->curr_hwdesc + 1 == desc->count)) { 702 vchan_cookie_complete(&desc->vdesc); 703 jzchan->desc = NULL; 704 } 705 706 jz4780_dma_begin(jzchan); 707 } else { 708 /* False positive - continue the transfer */ 709 ack = false; 710 jz4780_dma_chn_writel(jzdma, jzchan->id, 711 JZ_DMA_REG_DCS, 712 JZ_DMA_DCS_CTE); 713 } 714 } 715 } else { 716 dev_err(&jzchan->vchan.chan.dev->device, 717 "channel IRQ with no active transfer\n"); 718 } 719 720 spin_unlock(&jzchan->vchan.lock); 721 722 return ack; 723 } 724 725 static irqreturn_t jz4780_dma_irq_handler(int irq, void *data) 726 { 727 struct jz4780_dma_dev *jzdma = data; 728 unsigned int nb_channels = jzdma->soc_data->nb_channels; 729 unsigned long pending; 730 uint32_t dmac; 731 int i; 732 733 pending = jz4780_dma_ctrl_readl(jzdma, JZ_DMA_REG_DIRQP); 734 735 for_each_set_bit(i, &pending, nb_channels) { 736 if (jz4780_dma_chan_irq(jzdma, &jzdma->chan[i])) 737 pending &= ~BIT(i); 738 } 739 740 /* Clear halt and address error status of all channels. */ 741 dmac = jz4780_dma_ctrl_readl(jzdma, JZ_DMA_REG_DMAC); 742 dmac &= ~(JZ_DMA_DMAC_HLT | JZ_DMA_DMAC_AR); 743 jz4780_dma_ctrl_writel(jzdma, JZ_DMA_REG_DMAC, dmac); 744 745 /* Clear interrupt pending status. */ 746 jz4780_dma_ctrl_writel(jzdma, JZ_DMA_REG_DIRQP, pending); 747 748 return IRQ_HANDLED; 749 } 750 751 static int jz4780_dma_alloc_chan_resources(struct dma_chan *chan) 752 { 753 struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan); 754 755 jzchan->desc_pool = dma_pool_create(dev_name(&chan->dev->device), 756 chan->device->dev, 757 JZ_DMA_DESC_BLOCK_SIZE, 758 PAGE_SIZE, 0); 759 if (!jzchan->desc_pool) { 760 dev_err(&chan->dev->device, 761 "failed to allocate descriptor pool\n"); 762 return -ENOMEM; 763 } 764 765 return 0; 766 } 767 768 static void jz4780_dma_free_chan_resources(struct dma_chan *chan) 769 { 770 struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan); 771 772 vchan_free_chan_resources(&jzchan->vchan); 773 dma_pool_destroy(jzchan->desc_pool); 774 jzchan->desc_pool = NULL; 775 } 776 777 static bool jz4780_dma_filter_fn(struct dma_chan *chan, void *param) 778 { 779 struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan); 780 struct jz4780_dma_dev *jzdma = jz4780_dma_chan_parent(jzchan); 781 struct jz4780_dma_filter_data *data = param; 782 783 784 if (data->channel > -1) { 785 if (data->channel != jzchan->id) 786 return false; 787 } else if (jzdma->chan_reserved & BIT(jzchan->id)) { 788 return false; 789 } 790 791 jzchan->transfer_type = data->transfer_type; 792 793 return true; 794 } 795 796 static struct dma_chan *jz4780_of_dma_xlate(struct of_phandle_args *dma_spec, 797 struct of_dma *ofdma) 798 { 799 struct jz4780_dma_dev *jzdma = ofdma->of_dma_data; 800 dma_cap_mask_t mask = jzdma->dma_device.cap_mask; 801 struct jz4780_dma_filter_data data; 802 803 if (dma_spec->args_count != 2) 804 return NULL; 805 806 data.transfer_type = dma_spec->args[0]; 807 data.channel = dma_spec->args[1]; 808 809 if (data.channel > -1) { 810 if (data.channel >= jzdma->soc_data->nb_channels) { 811 dev_err(jzdma->dma_device.dev, 812 "device requested non-existent channel %u\n", 813 data.channel); 814 return NULL; 815 } 816 817 /* Can only select a channel marked as reserved. */ 818 if (!(jzdma->chan_reserved & BIT(data.channel))) { 819 dev_err(jzdma->dma_device.dev, 820 "device requested unreserved channel %u\n", 821 data.channel); 822 return NULL; 823 } 824 825 jzdma->chan[data.channel].transfer_type = data.transfer_type; 826 827 return dma_get_slave_channel( 828 &jzdma->chan[data.channel].vchan.chan); 829 } else { 830 return __dma_request_channel(&mask, jz4780_dma_filter_fn, &data, 831 ofdma->of_node); 832 } 833 } 834 835 static int jz4780_dma_probe(struct platform_device *pdev) 836 { 837 struct device *dev = &pdev->dev; 838 const struct jz4780_dma_soc_data *soc_data; 839 struct jz4780_dma_dev *jzdma; 840 struct jz4780_dma_chan *jzchan; 841 struct dma_device *dd; 842 struct resource *res; 843 int i, ret; 844 845 if (!dev->of_node) { 846 dev_err(dev, "This driver must be probed from devicetree\n"); 847 return -EINVAL; 848 } 849 850 soc_data = device_get_match_data(dev); 851 if (!soc_data) 852 return -EINVAL; 853 854 jzdma = devm_kzalloc(dev, struct_size(jzdma, chan, 855 soc_data->nb_channels), GFP_KERNEL); 856 if (!jzdma) 857 return -ENOMEM; 858 859 jzdma->soc_data = soc_data; 860 platform_set_drvdata(pdev, jzdma); 861 862 jzdma->chn_base = devm_platform_ioremap_resource(pdev, 0); 863 if (IS_ERR(jzdma->chn_base)) 864 return PTR_ERR(jzdma->chn_base); 865 866 res = platform_get_resource(pdev, IORESOURCE_MEM, 1); 867 if (res) { 868 jzdma->ctrl_base = devm_ioremap_resource(dev, res); 869 if (IS_ERR(jzdma->ctrl_base)) 870 return PTR_ERR(jzdma->ctrl_base); 871 } else if (soc_data->flags & JZ_SOC_DATA_ALLOW_LEGACY_DT) { 872 /* 873 * On JZ4780, if the second memory resource was not supplied, 874 * assume we're using an old devicetree, and calculate the 875 * offset to the control registers. 876 */ 877 jzdma->ctrl_base = jzdma->chn_base + JZ4780_DMA_CTRL_OFFSET; 878 } else { 879 dev_err(dev, "failed to get I/O memory\n"); 880 return -EINVAL; 881 } 882 883 jzdma->clk = devm_clk_get(dev, NULL); 884 if (IS_ERR(jzdma->clk)) { 885 dev_err(dev, "failed to get clock\n"); 886 ret = PTR_ERR(jzdma->clk); 887 return ret; 888 } 889 890 clk_prepare_enable(jzdma->clk); 891 892 /* Property is optional, if it doesn't exist the value will remain 0. */ 893 of_property_read_u32_index(dev->of_node, "ingenic,reserved-channels", 894 0, &jzdma->chan_reserved); 895 896 dd = &jzdma->dma_device; 897 898 dma_cap_set(DMA_MEMCPY, dd->cap_mask); 899 dma_cap_set(DMA_SLAVE, dd->cap_mask); 900 dma_cap_set(DMA_CYCLIC, dd->cap_mask); 901 902 dd->dev = dev; 903 dd->copy_align = DMAENGINE_ALIGN_4_BYTES; 904 dd->device_alloc_chan_resources = jz4780_dma_alloc_chan_resources; 905 dd->device_free_chan_resources = jz4780_dma_free_chan_resources; 906 dd->device_prep_slave_sg = jz4780_dma_prep_slave_sg; 907 dd->device_prep_dma_cyclic = jz4780_dma_prep_dma_cyclic; 908 dd->device_prep_dma_memcpy = jz4780_dma_prep_dma_memcpy; 909 dd->device_config = jz4780_dma_config; 910 dd->device_terminate_all = jz4780_dma_terminate_all; 911 dd->device_synchronize = jz4780_dma_synchronize; 912 dd->device_tx_status = jz4780_dma_tx_status; 913 dd->device_issue_pending = jz4780_dma_issue_pending; 914 dd->src_addr_widths = JZ_DMA_BUSWIDTHS; 915 dd->dst_addr_widths = JZ_DMA_BUSWIDTHS; 916 dd->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); 917 dd->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; 918 919 /* 920 * Enable DMA controller, mark all channels as not programmable. 921 * Also set the FMSC bit - it increases MSC performance, so it makes 922 * little sense not to enable it. 923 */ 924 jz4780_dma_ctrl_writel(jzdma, JZ_DMA_REG_DMAC, JZ_DMA_DMAC_DMAE | 925 JZ_DMA_DMAC_FAIC | JZ_DMA_DMAC_FMSC); 926 927 if (soc_data->flags & JZ_SOC_DATA_PROGRAMMABLE_DMA) 928 jz4780_dma_ctrl_writel(jzdma, JZ_DMA_REG_DMACP, 0); 929 930 INIT_LIST_HEAD(&dd->channels); 931 932 for (i = 0; i < soc_data->nb_channels; i++) { 933 jzchan = &jzdma->chan[i]; 934 jzchan->id = i; 935 936 vchan_init(&jzchan->vchan, dd); 937 jzchan->vchan.desc_free = jz4780_dma_desc_free; 938 } 939 940 ret = platform_get_irq(pdev, 0); 941 if (ret < 0) 942 goto err_disable_clk; 943 944 jzdma->irq = ret; 945 946 ret = request_irq(jzdma->irq, jz4780_dma_irq_handler, 0, dev_name(dev), 947 jzdma); 948 if (ret) { 949 dev_err(dev, "failed to request IRQ %u!\n", jzdma->irq); 950 goto err_disable_clk; 951 } 952 953 ret = dmaenginem_async_device_register(dd); 954 if (ret) { 955 dev_err(dev, "failed to register device\n"); 956 goto err_free_irq; 957 } 958 959 /* Register with OF DMA helpers. */ 960 ret = of_dma_controller_register(dev->of_node, jz4780_of_dma_xlate, 961 jzdma); 962 if (ret) { 963 dev_err(dev, "failed to register OF DMA controller\n"); 964 goto err_free_irq; 965 } 966 967 dev_info(dev, "JZ4780 DMA controller initialised\n"); 968 return 0; 969 970 err_free_irq: 971 free_irq(jzdma->irq, jzdma); 972 973 err_disable_clk: 974 clk_disable_unprepare(jzdma->clk); 975 return ret; 976 } 977 978 static int jz4780_dma_remove(struct platform_device *pdev) 979 { 980 struct jz4780_dma_dev *jzdma = platform_get_drvdata(pdev); 981 int i; 982 983 of_dma_controller_free(pdev->dev.of_node); 984 985 clk_disable_unprepare(jzdma->clk); 986 free_irq(jzdma->irq, jzdma); 987 988 for (i = 0; i < jzdma->soc_data->nb_channels; i++) 989 tasklet_kill(&jzdma->chan[i].vchan.task); 990 991 return 0; 992 } 993 994 static const struct jz4780_dma_soc_data jz4740_dma_soc_data = { 995 .nb_channels = 6, 996 .transfer_ord_max = 5, 997 .flags = JZ_SOC_DATA_BREAK_LINKS, 998 }; 999 1000 static const struct jz4780_dma_soc_data jz4725b_dma_soc_data = { 1001 .nb_channels = 6, 1002 .transfer_ord_max = 5, 1003 .flags = JZ_SOC_DATA_PER_CHAN_PM | JZ_SOC_DATA_NO_DCKES_DCKEC | 1004 JZ_SOC_DATA_BREAK_LINKS, 1005 }; 1006 1007 static const struct jz4780_dma_soc_data jz4760_dma_soc_data = { 1008 .nb_channels = 5, 1009 .transfer_ord_max = 6, 1010 .flags = JZ_SOC_DATA_PER_CHAN_PM | JZ_SOC_DATA_NO_DCKES_DCKEC, 1011 }; 1012 1013 static const struct jz4780_dma_soc_data jz4760b_dma_soc_data = { 1014 .nb_channels = 5, 1015 .transfer_ord_max = 6, 1016 .flags = JZ_SOC_DATA_PER_CHAN_PM, 1017 }; 1018 1019 static const struct jz4780_dma_soc_data jz4770_dma_soc_data = { 1020 .nb_channels = 6, 1021 .transfer_ord_max = 6, 1022 .flags = JZ_SOC_DATA_PER_CHAN_PM, 1023 }; 1024 1025 static const struct jz4780_dma_soc_data jz4780_dma_soc_data = { 1026 .nb_channels = 32, 1027 .transfer_ord_max = 7, 1028 .flags = JZ_SOC_DATA_ALLOW_LEGACY_DT | JZ_SOC_DATA_PROGRAMMABLE_DMA, 1029 }; 1030 1031 static const struct jz4780_dma_soc_data x1000_dma_soc_data = { 1032 .nb_channels = 8, 1033 .transfer_ord_max = 7, 1034 .flags = JZ_SOC_DATA_PROGRAMMABLE_DMA, 1035 }; 1036 1037 static const struct jz4780_dma_soc_data x1830_dma_soc_data = { 1038 .nb_channels = 32, 1039 .transfer_ord_max = 7, 1040 .flags = JZ_SOC_DATA_PROGRAMMABLE_DMA, 1041 }; 1042 1043 static const struct of_device_id jz4780_dma_dt_match[] = { 1044 { .compatible = "ingenic,jz4740-dma", .data = &jz4740_dma_soc_data }, 1045 { .compatible = "ingenic,jz4725b-dma", .data = &jz4725b_dma_soc_data }, 1046 { .compatible = "ingenic,jz4760-dma", .data = &jz4760_dma_soc_data }, 1047 { .compatible = "ingenic,jz4760b-dma", .data = &jz4760b_dma_soc_data }, 1048 { .compatible = "ingenic,jz4770-dma", .data = &jz4770_dma_soc_data }, 1049 { .compatible = "ingenic,jz4780-dma", .data = &jz4780_dma_soc_data }, 1050 { .compatible = "ingenic,x1000-dma", .data = &x1000_dma_soc_data }, 1051 { .compatible = "ingenic,x1830-dma", .data = &x1830_dma_soc_data }, 1052 {}, 1053 }; 1054 MODULE_DEVICE_TABLE(of, jz4780_dma_dt_match); 1055 1056 static struct platform_driver jz4780_dma_driver = { 1057 .probe = jz4780_dma_probe, 1058 .remove = jz4780_dma_remove, 1059 .driver = { 1060 .name = "jz4780-dma", 1061 .of_match_table = jz4780_dma_dt_match, 1062 }, 1063 }; 1064 1065 static int __init jz4780_dma_init(void) 1066 { 1067 return platform_driver_register(&jz4780_dma_driver); 1068 } 1069 subsys_initcall(jz4780_dma_init); 1070 1071 static void __exit jz4780_dma_exit(void) 1072 { 1073 platform_driver_unregister(&jz4780_dma_driver); 1074 } 1075 module_exit(jz4780_dma_exit); 1076 1077 MODULE_AUTHOR("Alex Smith <alex@alex-smith.me.uk>"); 1078 MODULE_DESCRIPTION("Ingenic JZ4780 DMA controller driver"); 1079 MODULE_LICENSE("GPL"); 1080