1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Ingenic JZ4780 DMA controller 4 * 5 * Copyright (c) 2015 Imagination Technologies 6 * Author: Alex Smith <alex@alex-smith.me.uk> 7 */ 8 9 #include <linux/clk.h> 10 #include <linux/dmapool.h> 11 #include <linux/dma-mapping.h> 12 #include <linux/init.h> 13 #include <linux/interrupt.h> 14 #include <linux/module.h> 15 #include <linux/of.h> 16 #include <linux/of_device.h> 17 #include <linux/of_dma.h> 18 #include <linux/platform_device.h> 19 #include <linux/slab.h> 20 21 #include "dmaengine.h" 22 #include "virt-dma.h" 23 24 /* Global registers. */ 25 #define JZ_DMA_REG_DMAC 0x00 26 #define JZ_DMA_REG_DIRQP 0x04 27 #define JZ_DMA_REG_DDR 0x08 28 #define JZ_DMA_REG_DDRS 0x0c 29 #define JZ_DMA_REG_DCKE 0x10 30 #define JZ_DMA_REG_DCKES 0x14 31 #define JZ_DMA_REG_DCKEC 0x18 32 #define JZ_DMA_REG_DMACP 0x1c 33 #define JZ_DMA_REG_DSIRQP 0x20 34 #define JZ_DMA_REG_DSIRQM 0x24 35 #define JZ_DMA_REG_DCIRQP 0x28 36 #define JZ_DMA_REG_DCIRQM 0x2c 37 38 /* Per-channel registers. */ 39 #define JZ_DMA_REG_CHAN(n) (n * 0x20) 40 #define JZ_DMA_REG_DSA 0x00 41 #define JZ_DMA_REG_DTA 0x04 42 #define JZ_DMA_REG_DTC 0x08 43 #define JZ_DMA_REG_DRT 0x0c 44 #define JZ_DMA_REG_DCS 0x10 45 #define JZ_DMA_REG_DCM 0x14 46 #define JZ_DMA_REG_DDA 0x18 47 #define JZ_DMA_REG_DSD 0x1c 48 49 #define JZ_DMA_DMAC_DMAE BIT(0) 50 #define JZ_DMA_DMAC_AR BIT(2) 51 #define JZ_DMA_DMAC_HLT BIT(3) 52 #define JZ_DMA_DMAC_FAIC BIT(27) 53 #define JZ_DMA_DMAC_FMSC BIT(31) 54 55 #define JZ_DMA_DRT_AUTO 0x8 56 57 #define JZ_DMA_DCS_CTE BIT(0) 58 #define JZ_DMA_DCS_HLT BIT(2) 59 #define JZ_DMA_DCS_TT BIT(3) 60 #define JZ_DMA_DCS_AR BIT(4) 61 #define JZ_DMA_DCS_DES8 BIT(30) 62 63 #define JZ_DMA_DCM_LINK BIT(0) 64 #define JZ_DMA_DCM_TIE BIT(1) 65 #define JZ_DMA_DCM_STDE BIT(2) 66 #define JZ_DMA_DCM_TSZ_SHIFT 8 67 #define JZ_DMA_DCM_TSZ_MASK (0x7 << JZ_DMA_DCM_TSZ_SHIFT) 68 #define JZ_DMA_DCM_DP_SHIFT 12 69 #define JZ_DMA_DCM_SP_SHIFT 14 70 #define JZ_DMA_DCM_DAI BIT(22) 71 #define JZ_DMA_DCM_SAI BIT(23) 72 73 #define JZ_DMA_SIZE_4_BYTE 0x0 74 #define JZ_DMA_SIZE_1_BYTE 0x1 75 #define JZ_DMA_SIZE_2_BYTE 0x2 76 #define JZ_DMA_SIZE_16_BYTE 0x3 77 #define JZ_DMA_SIZE_32_BYTE 0x4 78 #define JZ_DMA_SIZE_64_BYTE 0x5 79 #define JZ_DMA_SIZE_128_BYTE 0x6 80 81 #define JZ_DMA_WIDTH_32_BIT 0x0 82 #define JZ_DMA_WIDTH_8_BIT 0x1 83 #define JZ_DMA_WIDTH_16_BIT 0x2 84 85 #define JZ_DMA_BUSWIDTHS (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \ 86 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \ 87 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES)) 88 89 #define JZ4780_DMA_CTRL_OFFSET 0x1000 90 91 /* macros for use with jz4780_dma_soc_data.flags */ 92 #define JZ_SOC_DATA_ALLOW_LEGACY_DT BIT(0) 93 #define JZ_SOC_DATA_PROGRAMMABLE_DMA BIT(1) 94 #define JZ_SOC_DATA_PER_CHAN_PM BIT(2) 95 #define JZ_SOC_DATA_NO_DCKES_DCKEC BIT(3) 96 #define JZ_SOC_DATA_BREAK_LINKS BIT(4) 97 98 /** 99 * struct jz4780_dma_hwdesc - descriptor structure read by the DMA controller. 100 * @dcm: value for the DCM (channel command) register 101 * @dsa: source address 102 * @dta: target address 103 * @dtc: transfer count (number of blocks of the transfer size specified in DCM 104 * to transfer) in the low 24 bits, offset of the next descriptor from the 105 * descriptor base address in the upper 8 bits. 106 */ 107 struct jz4780_dma_hwdesc { 108 u32 dcm; 109 u32 dsa; 110 u32 dta; 111 u32 dtc; 112 }; 113 114 /* Size of allocations for hardware descriptor blocks. */ 115 #define JZ_DMA_DESC_BLOCK_SIZE PAGE_SIZE 116 #define JZ_DMA_MAX_DESC \ 117 (JZ_DMA_DESC_BLOCK_SIZE / sizeof(struct jz4780_dma_hwdesc)) 118 119 struct jz4780_dma_desc { 120 struct virt_dma_desc vdesc; 121 122 struct jz4780_dma_hwdesc *desc; 123 dma_addr_t desc_phys; 124 unsigned int count; 125 enum dma_transaction_type type; 126 u32 transfer_type; 127 u32 status; 128 }; 129 130 struct jz4780_dma_chan { 131 struct virt_dma_chan vchan; 132 unsigned int id; 133 struct dma_pool *desc_pool; 134 135 u32 transfer_type_tx, transfer_type_rx; 136 u32 transfer_shift; 137 struct dma_slave_config config; 138 139 struct jz4780_dma_desc *desc; 140 unsigned int curr_hwdesc; 141 }; 142 143 struct jz4780_dma_soc_data { 144 unsigned int nb_channels; 145 unsigned int transfer_ord_max; 146 unsigned long flags; 147 }; 148 149 struct jz4780_dma_dev { 150 struct dma_device dma_device; 151 void __iomem *chn_base; 152 void __iomem *ctrl_base; 153 struct clk *clk; 154 unsigned int irq; 155 const struct jz4780_dma_soc_data *soc_data; 156 157 u32 chan_reserved; 158 struct jz4780_dma_chan chan[]; 159 }; 160 161 struct jz4780_dma_filter_data { 162 u32 transfer_type_tx, transfer_type_rx; 163 int channel; 164 }; 165 166 static inline struct jz4780_dma_chan *to_jz4780_dma_chan(struct dma_chan *chan) 167 { 168 return container_of(chan, struct jz4780_dma_chan, vchan.chan); 169 } 170 171 static inline struct jz4780_dma_desc *to_jz4780_dma_desc( 172 struct virt_dma_desc *vdesc) 173 { 174 return container_of(vdesc, struct jz4780_dma_desc, vdesc); 175 } 176 177 static inline struct jz4780_dma_dev *jz4780_dma_chan_parent( 178 struct jz4780_dma_chan *jzchan) 179 { 180 return container_of(jzchan->vchan.chan.device, struct jz4780_dma_dev, 181 dma_device); 182 } 183 184 static inline u32 jz4780_dma_chn_readl(struct jz4780_dma_dev *jzdma, 185 unsigned int chn, unsigned int reg) 186 { 187 return readl(jzdma->chn_base + reg + JZ_DMA_REG_CHAN(chn)); 188 } 189 190 static inline void jz4780_dma_chn_writel(struct jz4780_dma_dev *jzdma, 191 unsigned int chn, unsigned int reg, u32 val) 192 { 193 writel(val, jzdma->chn_base + reg + JZ_DMA_REG_CHAN(chn)); 194 } 195 196 static inline u32 jz4780_dma_ctrl_readl(struct jz4780_dma_dev *jzdma, 197 unsigned int reg) 198 { 199 return readl(jzdma->ctrl_base + reg); 200 } 201 202 static inline void jz4780_dma_ctrl_writel(struct jz4780_dma_dev *jzdma, 203 unsigned int reg, u32 val) 204 { 205 writel(val, jzdma->ctrl_base + reg); 206 } 207 208 static inline void jz4780_dma_chan_enable(struct jz4780_dma_dev *jzdma, 209 unsigned int chn) 210 { 211 if (jzdma->soc_data->flags & JZ_SOC_DATA_PER_CHAN_PM) { 212 unsigned int reg; 213 214 if (jzdma->soc_data->flags & JZ_SOC_DATA_NO_DCKES_DCKEC) 215 reg = JZ_DMA_REG_DCKE; 216 else 217 reg = JZ_DMA_REG_DCKES; 218 219 jz4780_dma_ctrl_writel(jzdma, reg, BIT(chn)); 220 } 221 } 222 223 static inline void jz4780_dma_chan_disable(struct jz4780_dma_dev *jzdma, 224 unsigned int chn) 225 { 226 if ((jzdma->soc_data->flags & JZ_SOC_DATA_PER_CHAN_PM) && 227 !(jzdma->soc_data->flags & JZ_SOC_DATA_NO_DCKES_DCKEC)) 228 jz4780_dma_ctrl_writel(jzdma, JZ_DMA_REG_DCKEC, BIT(chn)); 229 } 230 231 static struct jz4780_dma_desc * 232 jz4780_dma_desc_alloc(struct jz4780_dma_chan *jzchan, unsigned int count, 233 enum dma_transaction_type type, 234 enum dma_transfer_direction direction) 235 { 236 struct jz4780_dma_desc *desc; 237 238 if (count > JZ_DMA_MAX_DESC) 239 return NULL; 240 241 desc = kzalloc(sizeof(*desc), GFP_NOWAIT); 242 if (!desc) 243 return NULL; 244 245 desc->desc = dma_pool_alloc(jzchan->desc_pool, GFP_NOWAIT, 246 &desc->desc_phys); 247 if (!desc->desc) { 248 kfree(desc); 249 return NULL; 250 } 251 252 desc->count = count; 253 desc->type = type; 254 255 if (direction == DMA_DEV_TO_MEM) 256 desc->transfer_type = jzchan->transfer_type_rx; 257 else 258 desc->transfer_type = jzchan->transfer_type_tx; 259 260 return desc; 261 } 262 263 static void jz4780_dma_desc_free(struct virt_dma_desc *vdesc) 264 { 265 struct jz4780_dma_desc *desc = to_jz4780_dma_desc(vdesc); 266 struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(vdesc->tx.chan); 267 268 dma_pool_free(jzchan->desc_pool, desc->desc, desc->desc_phys); 269 kfree(desc); 270 } 271 272 static u32 jz4780_dma_transfer_size(struct jz4780_dma_chan *jzchan, 273 unsigned long val, u32 *shift) 274 { 275 struct jz4780_dma_dev *jzdma = jz4780_dma_chan_parent(jzchan); 276 int ord = ffs(val) - 1; 277 278 /* 279 * 8 byte transfer sizes unsupported so fall back on 4. If it's larger 280 * than the maximum, just limit it. It is perfectly safe to fall back 281 * in this way since we won't exceed the maximum burst size supported 282 * by the device, the only effect is reduced efficiency. This is better 283 * than refusing to perform the request at all. 284 */ 285 if (ord == 3) 286 ord = 2; 287 else if (ord > jzdma->soc_data->transfer_ord_max) 288 ord = jzdma->soc_data->transfer_ord_max; 289 290 *shift = ord; 291 292 switch (ord) { 293 case 0: 294 return JZ_DMA_SIZE_1_BYTE; 295 case 1: 296 return JZ_DMA_SIZE_2_BYTE; 297 case 2: 298 return JZ_DMA_SIZE_4_BYTE; 299 case 4: 300 return JZ_DMA_SIZE_16_BYTE; 301 case 5: 302 return JZ_DMA_SIZE_32_BYTE; 303 case 6: 304 return JZ_DMA_SIZE_64_BYTE; 305 default: 306 return JZ_DMA_SIZE_128_BYTE; 307 } 308 } 309 310 static int jz4780_dma_setup_hwdesc(struct jz4780_dma_chan *jzchan, 311 struct jz4780_dma_hwdesc *desc, dma_addr_t addr, size_t len, 312 enum dma_transfer_direction direction) 313 { 314 struct dma_slave_config *config = &jzchan->config; 315 u32 width, maxburst, tsz; 316 317 if (direction == DMA_MEM_TO_DEV) { 318 desc->dcm = JZ_DMA_DCM_SAI; 319 desc->dsa = addr; 320 desc->dta = config->dst_addr; 321 322 width = config->dst_addr_width; 323 maxburst = config->dst_maxburst; 324 } else { 325 desc->dcm = JZ_DMA_DCM_DAI; 326 desc->dsa = config->src_addr; 327 desc->dta = addr; 328 329 width = config->src_addr_width; 330 maxburst = config->src_maxburst; 331 } 332 333 /* 334 * This calculates the maximum transfer size that can be used with the 335 * given address, length, width and maximum burst size. The address 336 * must be aligned to the transfer size, the total length must be 337 * divisible by the transfer size, and we must not use more than the 338 * maximum burst specified by the user. 339 */ 340 tsz = jz4780_dma_transfer_size(jzchan, addr | len | (width * maxburst), 341 &jzchan->transfer_shift); 342 343 switch (width) { 344 case DMA_SLAVE_BUSWIDTH_1_BYTE: 345 case DMA_SLAVE_BUSWIDTH_2_BYTES: 346 break; 347 case DMA_SLAVE_BUSWIDTH_4_BYTES: 348 width = JZ_DMA_WIDTH_32_BIT; 349 break; 350 default: 351 return -EINVAL; 352 } 353 354 desc->dcm |= tsz << JZ_DMA_DCM_TSZ_SHIFT; 355 desc->dcm |= width << JZ_DMA_DCM_SP_SHIFT; 356 desc->dcm |= width << JZ_DMA_DCM_DP_SHIFT; 357 358 desc->dtc = len >> jzchan->transfer_shift; 359 return 0; 360 } 361 362 static struct dma_async_tx_descriptor *jz4780_dma_prep_slave_sg( 363 struct dma_chan *chan, struct scatterlist *sgl, unsigned int sg_len, 364 enum dma_transfer_direction direction, unsigned long flags, 365 void *context) 366 { 367 struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan); 368 struct jz4780_dma_dev *jzdma = jz4780_dma_chan_parent(jzchan); 369 struct jz4780_dma_desc *desc; 370 unsigned int i; 371 int err; 372 373 desc = jz4780_dma_desc_alloc(jzchan, sg_len, DMA_SLAVE, direction); 374 if (!desc) 375 return NULL; 376 377 for (i = 0; i < sg_len; i++) { 378 err = jz4780_dma_setup_hwdesc(jzchan, &desc->desc[i], 379 sg_dma_address(&sgl[i]), 380 sg_dma_len(&sgl[i]), 381 direction); 382 if (err < 0) { 383 jz4780_dma_desc_free(&jzchan->desc->vdesc); 384 return NULL; 385 } 386 387 desc->desc[i].dcm |= JZ_DMA_DCM_TIE; 388 389 if (i != (sg_len - 1) && 390 !(jzdma->soc_data->flags & JZ_SOC_DATA_BREAK_LINKS)) { 391 /* Automatically proceed to the next descriptor. */ 392 desc->desc[i].dcm |= JZ_DMA_DCM_LINK; 393 394 /* 395 * The upper 8 bits of the DTC field in the descriptor 396 * must be set to (offset from descriptor base of next 397 * descriptor >> 4). 398 */ 399 desc->desc[i].dtc |= 400 (((i + 1) * sizeof(*desc->desc)) >> 4) << 24; 401 } 402 } 403 404 return vchan_tx_prep(&jzchan->vchan, &desc->vdesc, flags); 405 } 406 407 static struct dma_async_tx_descriptor *jz4780_dma_prep_dma_cyclic( 408 struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len, 409 size_t period_len, enum dma_transfer_direction direction, 410 unsigned long flags) 411 { 412 struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan); 413 struct jz4780_dma_desc *desc; 414 unsigned int periods, i; 415 int err; 416 417 if (buf_len % period_len) 418 return NULL; 419 420 periods = buf_len / period_len; 421 422 desc = jz4780_dma_desc_alloc(jzchan, periods, DMA_CYCLIC, direction); 423 if (!desc) 424 return NULL; 425 426 for (i = 0; i < periods; i++) { 427 err = jz4780_dma_setup_hwdesc(jzchan, &desc->desc[i], buf_addr, 428 period_len, direction); 429 if (err < 0) { 430 jz4780_dma_desc_free(&jzchan->desc->vdesc); 431 return NULL; 432 } 433 434 buf_addr += period_len; 435 436 /* 437 * Set the link bit to indicate that the controller should 438 * automatically proceed to the next descriptor. In 439 * jz4780_dma_begin(), this will be cleared if we need to issue 440 * an interrupt after each period. 441 */ 442 desc->desc[i].dcm |= JZ_DMA_DCM_TIE | JZ_DMA_DCM_LINK; 443 444 /* 445 * The upper 8 bits of the DTC field in the descriptor must be 446 * set to (offset from descriptor base of next descriptor >> 4). 447 * If this is the last descriptor, link it back to the first, 448 * i.e. leave offset set to 0, otherwise point to the next one. 449 */ 450 if (i != (periods - 1)) { 451 desc->desc[i].dtc |= 452 (((i + 1) * sizeof(*desc->desc)) >> 4) << 24; 453 } 454 } 455 456 return vchan_tx_prep(&jzchan->vchan, &desc->vdesc, flags); 457 } 458 459 static struct dma_async_tx_descriptor *jz4780_dma_prep_dma_memcpy( 460 struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, 461 size_t len, unsigned long flags) 462 { 463 struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan); 464 struct jz4780_dma_desc *desc; 465 u32 tsz; 466 467 desc = jz4780_dma_desc_alloc(jzchan, 1, DMA_MEMCPY, 0); 468 if (!desc) 469 return NULL; 470 471 tsz = jz4780_dma_transfer_size(jzchan, dest | src | len, 472 &jzchan->transfer_shift); 473 474 desc->transfer_type = JZ_DMA_DRT_AUTO; 475 476 desc->desc[0].dsa = src; 477 desc->desc[0].dta = dest; 478 desc->desc[0].dcm = JZ_DMA_DCM_TIE | JZ_DMA_DCM_SAI | JZ_DMA_DCM_DAI | 479 tsz << JZ_DMA_DCM_TSZ_SHIFT | 480 JZ_DMA_WIDTH_32_BIT << JZ_DMA_DCM_SP_SHIFT | 481 JZ_DMA_WIDTH_32_BIT << JZ_DMA_DCM_DP_SHIFT; 482 desc->desc[0].dtc = len >> jzchan->transfer_shift; 483 484 return vchan_tx_prep(&jzchan->vchan, &desc->vdesc, flags); 485 } 486 487 static void jz4780_dma_begin(struct jz4780_dma_chan *jzchan) 488 { 489 struct jz4780_dma_dev *jzdma = jz4780_dma_chan_parent(jzchan); 490 struct virt_dma_desc *vdesc; 491 unsigned int i; 492 dma_addr_t desc_phys; 493 494 if (!jzchan->desc) { 495 vdesc = vchan_next_desc(&jzchan->vchan); 496 if (!vdesc) 497 return; 498 499 list_del(&vdesc->node); 500 501 jzchan->desc = to_jz4780_dma_desc(vdesc); 502 jzchan->curr_hwdesc = 0; 503 504 if (jzchan->desc->type == DMA_CYCLIC && vdesc->tx.callback) { 505 /* 506 * The DMA controller doesn't support triggering an 507 * interrupt after processing each descriptor, only 508 * after processing an entire terminated list of 509 * descriptors. For a cyclic DMA setup the list of 510 * descriptors is not terminated so we can never get an 511 * interrupt. 512 * 513 * If the user requested a callback for a cyclic DMA 514 * setup then we workaround this hardware limitation 515 * here by degrading to a set of unlinked descriptors 516 * which we will submit in sequence in response to the 517 * completion of processing the previous descriptor. 518 */ 519 for (i = 0; i < jzchan->desc->count; i++) 520 jzchan->desc->desc[i].dcm &= ~JZ_DMA_DCM_LINK; 521 } 522 } else { 523 /* 524 * There is an existing transfer, therefore this must be one 525 * for which we unlinked the descriptors above. Advance to the 526 * next one in the list. 527 */ 528 jzchan->curr_hwdesc = 529 (jzchan->curr_hwdesc + 1) % jzchan->desc->count; 530 } 531 532 /* Enable the channel's clock. */ 533 jz4780_dma_chan_enable(jzdma, jzchan->id); 534 535 /* Use 4-word descriptors. */ 536 jz4780_dma_chn_writel(jzdma, jzchan->id, JZ_DMA_REG_DCS, 0); 537 538 /* Set transfer type. */ 539 jz4780_dma_chn_writel(jzdma, jzchan->id, JZ_DMA_REG_DRT, 540 jzchan->desc->transfer_type); 541 542 /* 543 * Set the transfer count. This is redundant for a descriptor-driven 544 * transfer. However, there can be a delay between the transfer start 545 * time and when DTCn reg contains the new transfer count. Setting 546 * it explicitly ensures residue is computed correctly at all times. 547 */ 548 jz4780_dma_chn_writel(jzdma, jzchan->id, JZ_DMA_REG_DTC, 549 jzchan->desc->desc[jzchan->curr_hwdesc].dtc); 550 551 /* Write descriptor address and initiate descriptor fetch. */ 552 desc_phys = jzchan->desc->desc_phys + 553 (jzchan->curr_hwdesc * sizeof(*jzchan->desc->desc)); 554 jz4780_dma_chn_writel(jzdma, jzchan->id, JZ_DMA_REG_DDA, desc_phys); 555 jz4780_dma_ctrl_writel(jzdma, JZ_DMA_REG_DDRS, BIT(jzchan->id)); 556 557 /* Enable the channel. */ 558 jz4780_dma_chn_writel(jzdma, jzchan->id, JZ_DMA_REG_DCS, 559 JZ_DMA_DCS_CTE); 560 } 561 562 static void jz4780_dma_issue_pending(struct dma_chan *chan) 563 { 564 struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan); 565 unsigned long flags; 566 567 spin_lock_irqsave(&jzchan->vchan.lock, flags); 568 569 if (vchan_issue_pending(&jzchan->vchan) && !jzchan->desc) 570 jz4780_dma_begin(jzchan); 571 572 spin_unlock_irqrestore(&jzchan->vchan.lock, flags); 573 } 574 575 static int jz4780_dma_terminate_all(struct dma_chan *chan) 576 { 577 struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan); 578 struct jz4780_dma_dev *jzdma = jz4780_dma_chan_parent(jzchan); 579 unsigned long flags; 580 LIST_HEAD(head); 581 582 spin_lock_irqsave(&jzchan->vchan.lock, flags); 583 584 /* Clear the DMA status and stop the transfer. */ 585 jz4780_dma_chn_writel(jzdma, jzchan->id, JZ_DMA_REG_DCS, 0); 586 if (jzchan->desc) { 587 vchan_terminate_vdesc(&jzchan->desc->vdesc); 588 jzchan->desc = NULL; 589 } 590 591 jz4780_dma_chan_disable(jzdma, jzchan->id); 592 593 vchan_get_all_descriptors(&jzchan->vchan, &head); 594 595 spin_unlock_irqrestore(&jzchan->vchan.lock, flags); 596 597 vchan_dma_desc_free_list(&jzchan->vchan, &head); 598 return 0; 599 } 600 601 static void jz4780_dma_synchronize(struct dma_chan *chan) 602 { 603 struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan); 604 struct jz4780_dma_dev *jzdma = jz4780_dma_chan_parent(jzchan); 605 606 vchan_synchronize(&jzchan->vchan); 607 jz4780_dma_chan_disable(jzdma, jzchan->id); 608 } 609 610 static int jz4780_dma_config(struct dma_chan *chan, 611 struct dma_slave_config *config) 612 { 613 struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan); 614 615 if ((config->src_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES) 616 || (config->dst_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES)) 617 return -EINVAL; 618 619 /* Copy the reset of the slave configuration, it is used later. */ 620 memcpy(&jzchan->config, config, sizeof(jzchan->config)); 621 622 return 0; 623 } 624 625 static size_t jz4780_dma_desc_residue(struct jz4780_dma_chan *jzchan, 626 struct jz4780_dma_desc *desc, unsigned int next_sg) 627 { 628 struct jz4780_dma_dev *jzdma = jz4780_dma_chan_parent(jzchan); 629 unsigned int count = 0; 630 unsigned int i; 631 632 for (i = next_sg; i < desc->count; i++) 633 count += desc->desc[i].dtc & GENMASK(23, 0); 634 635 if (next_sg != 0) 636 count += jz4780_dma_chn_readl(jzdma, jzchan->id, 637 JZ_DMA_REG_DTC); 638 639 return count << jzchan->transfer_shift; 640 } 641 642 static enum dma_status jz4780_dma_tx_status(struct dma_chan *chan, 643 dma_cookie_t cookie, struct dma_tx_state *txstate) 644 { 645 struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan); 646 struct virt_dma_desc *vdesc; 647 enum dma_status status; 648 unsigned long flags; 649 unsigned long residue = 0; 650 651 spin_lock_irqsave(&jzchan->vchan.lock, flags); 652 653 status = dma_cookie_status(chan, cookie, txstate); 654 if ((status == DMA_COMPLETE) || (txstate == NULL)) 655 goto out_unlock_irqrestore; 656 657 vdesc = vchan_find_desc(&jzchan->vchan, cookie); 658 if (vdesc) { 659 /* On the issued list, so hasn't been processed yet */ 660 residue = jz4780_dma_desc_residue(jzchan, 661 to_jz4780_dma_desc(vdesc), 0); 662 } else if (cookie == jzchan->desc->vdesc.tx.cookie) { 663 residue = jz4780_dma_desc_residue(jzchan, jzchan->desc, 664 jzchan->curr_hwdesc + 1); 665 } 666 dma_set_residue(txstate, residue); 667 668 if (vdesc && jzchan->desc && vdesc == &jzchan->desc->vdesc 669 && jzchan->desc->status & (JZ_DMA_DCS_AR | JZ_DMA_DCS_HLT)) 670 status = DMA_ERROR; 671 672 out_unlock_irqrestore: 673 spin_unlock_irqrestore(&jzchan->vchan.lock, flags); 674 return status; 675 } 676 677 static bool jz4780_dma_chan_irq(struct jz4780_dma_dev *jzdma, 678 struct jz4780_dma_chan *jzchan) 679 { 680 const unsigned int soc_flags = jzdma->soc_data->flags; 681 struct jz4780_dma_desc *desc = jzchan->desc; 682 u32 dcs; 683 bool ack = true; 684 685 spin_lock(&jzchan->vchan.lock); 686 687 dcs = jz4780_dma_chn_readl(jzdma, jzchan->id, JZ_DMA_REG_DCS); 688 jz4780_dma_chn_writel(jzdma, jzchan->id, JZ_DMA_REG_DCS, 0); 689 690 if (dcs & JZ_DMA_DCS_AR) { 691 dev_warn(&jzchan->vchan.chan.dev->device, 692 "address error (DCS=0x%x)\n", dcs); 693 } 694 695 if (dcs & JZ_DMA_DCS_HLT) { 696 dev_warn(&jzchan->vchan.chan.dev->device, 697 "channel halt (DCS=0x%x)\n", dcs); 698 } 699 700 if (jzchan->desc) { 701 jzchan->desc->status = dcs; 702 703 if ((dcs & (JZ_DMA_DCS_AR | JZ_DMA_DCS_HLT)) == 0) { 704 if (jzchan->desc->type == DMA_CYCLIC) { 705 vchan_cyclic_callback(&jzchan->desc->vdesc); 706 707 jz4780_dma_begin(jzchan); 708 } else if (dcs & JZ_DMA_DCS_TT) { 709 if (!(soc_flags & JZ_SOC_DATA_BREAK_LINKS) || 710 (jzchan->curr_hwdesc + 1 == desc->count)) { 711 vchan_cookie_complete(&desc->vdesc); 712 jzchan->desc = NULL; 713 } 714 715 jz4780_dma_begin(jzchan); 716 } else { 717 /* False positive - continue the transfer */ 718 ack = false; 719 jz4780_dma_chn_writel(jzdma, jzchan->id, 720 JZ_DMA_REG_DCS, 721 JZ_DMA_DCS_CTE); 722 } 723 } 724 } else { 725 dev_err(&jzchan->vchan.chan.dev->device, 726 "channel IRQ with no active transfer\n"); 727 } 728 729 spin_unlock(&jzchan->vchan.lock); 730 731 return ack; 732 } 733 734 static irqreturn_t jz4780_dma_irq_handler(int irq, void *data) 735 { 736 struct jz4780_dma_dev *jzdma = data; 737 unsigned int nb_channels = jzdma->soc_data->nb_channels; 738 unsigned long pending; 739 u32 dmac; 740 int i; 741 742 pending = jz4780_dma_ctrl_readl(jzdma, JZ_DMA_REG_DIRQP); 743 744 for_each_set_bit(i, &pending, nb_channels) { 745 if (jz4780_dma_chan_irq(jzdma, &jzdma->chan[i])) 746 pending &= ~BIT(i); 747 } 748 749 /* Clear halt and address error status of all channels. */ 750 dmac = jz4780_dma_ctrl_readl(jzdma, JZ_DMA_REG_DMAC); 751 dmac &= ~(JZ_DMA_DMAC_HLT | JZ_DMA_DMAC_AR); 752 jz4780_dma_ctrl_writel(jzdma, JZ_DMA_REG_DMAC, dmac); 753 754 /* Clear interrupt pending status. */ 755 jz4780_dma_ctrl_writel(jzdma, JZ_DMA_REG_DIRQP, pending); 756 757 return IRQ_HANDLED; 758 } 759 760 static int jz4780_dma_alloc_chan_resources(struct dma_chan *chan) 761 { 762 struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan); 763 764 jzchan->desc_pool = dma_pool_create(dev_name(&chan->dev->device), 765 chan->device->dev, 766 JZ_DMA_DESC_BLOCK_SIZE, 767 PAGE_SIZE, 0); 768 if (!jzchan->desc_pool) { 769 dev_err(&chan->dev->device, 770 "failed to allocate descriptor pool\n"); 771 return -ENOMEM; 772 } 773 774 return 0; 775 } 776 777 static void jz4780_dma_free_chan_resources(struct dma_chan *chan) 778 { 779 struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan); 780 781 vchan_free_chan_resources(&jzchan->vchan); 782 dma_pool_destroy(jzchan->desc_pool); 783 jzchan->desc_pool = NULL; 784 } 785 786 static bool jz4780_dma_filter_fn(struct dma_chan *chan, void *param) 787 { 788 struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan); 789 struct jz4780_dma_dev *jzdma = jz4780_dma_chan_parent(jzchan); 790 struct jz4780_dma_filter_data *data = param; 791 792 793 if (data->channel > -1) { 794 if (data->channel != jzchan->id) 795 return false; 796 } else if (jzdma->chan_reserved & BIT(jzchan->id)) { 797 return false; 798 } 799 800 jzchan->transfer_type_tx = data->transfer_type_tx; 801 jzchan->transfer_type_rx = data->transfer_type_rx; 802 803 return true; 804 } 805 806 static struct dma_chan *jz4780_of_dma_xlate(struct of_phandle_args *dma_spec, 807 struct of_dma *ofdma) 808 { 809 struct jz4780_dma_dev *jzdma = ofdma->of_dma_data; 810 dma_cap_mask_t mask = jzdma->dma_device.cap_mask; 811 struct jz4780_dma_filter_data data; 812 813 if (dma_spec->args_count == 2) { 814 data.transfer_type_tx = dma_spec->args[0]; 815 data.transfer_type_rx = dma_spec->args[0]; 816 data.channel = dma_spec->args[1]; 817 } else if (dma_spec->args_count == 3) { 818 data.transfer_type_tx = dma_spec->args[0]; 819 data.transfer_type_rx = dma_spec->args[1]; 820 data.channel = dma_spec->args[2]; 821 } else { 822 return NULL; 823 } 824 825 if (data.channel > -1) { 826 if (data.channel >= jzdma->soc_data->nb_channels) { 827 dev_err(jzdma->dma_device.dev, 828 "device requested non-existent channel %u\n", 829 data.channel); 830 return NULL; 831 } 832 833 /* Can only select a channel marked as reserved. */ 834 if (!(jzdma->chan_reserved & BIT(data.channel))) { 835 dev_err(jzdma->dma_device.dev, 836 "device requested unreserved channel %u\n", 837 data.channel); 838 return NULL; 839 } 840 841 jzdma->chan[data.channel].transfer_type_tx = data.transfer_type_tx; 842 jzdma->chan[data.channel].transfer_type_rx = data.transfer_type_rx; 843 844 return dma_get_slave_channel( 845 &jzdma->chan[data.channel].vchan.chan); 846 } else { 847 return __dma_request_channel(&mask, jz4780_dma_filter_fn, &data, 848 ofdma->of_node); 849 } 850 } 851 852 static int jz4780_dma_probe(struct platform_device *pdev) 853 { 854 struct device *dev = &pdev->dev; 855 const struct jz4780_dma_soc_data *soc_data; 856 struct jz4780_dma_dev *jzdma; 857 struct jz4780_dma_chan *jzchan; 858 struct dma_device *dd; 859 struct resource *res; 860 int i, ret; 861 862 if (!dev->of_node) { 863 dev_err(dev, "This driver must be probed from devicetree\n"); 864 return -EINVAL; 865 } 866 867 soc_data = device_get_match_data(dev); 868 if (!soc_data) 869 return -EINVAL; 870 871 jzdma = devm_kzalloc(dev, struct_size(jzdma, chan, 872 soc_data->nb_channels), GFP_KERNEL); 873 if (!jzdma) 874 return -ENOMEM; 875 876 jzdma->soc_data = soc_data; 877 platform_set_drvdata(pdev, jzdma); 878 879 jzdma->chn_base = devm_platform_ioremap_resource(pdev, 0); 880 if (IS_ERR(jzdma->chn_base)) 881 return PTR_ERR(jzdma->chn_base); 882 883 res = platform_get_resource(pdev, IORESOURCE_MEM, 1); 884 if (res) { 885 jzdma->ctrl_base = devm_ioremap_resource(dev, res); 886 if (IS_ERR(jzdma->ctrl_base)) 887 return PTR_ERR(jzdma->ctrl_base); 888 } else if (soc_data->flags & JZ_SOC_DATA_ALLOW_LEGACY_DT) { 889 /* 890 * On JZ4780, if the second memory resource was not supplied, 891 * assume we're using an old devicetree, and calculate the 892 * offset to the control registers. 893 */ 894 jzdma->ctrl_base = jzdma->chn_base + JZ4780_DMA_CTRL_OFFSET; 895 } else { 896 dev_err(dev, "failed to get I/O memory\n"); 897 return -EINVAL; 898 } 899 900 jzdma->clk = devm_clk_get(dev, NULL); 901 if (IS_ERR(jzdma->clk)) { 902 dev_err(dev, "failed to get clock\n"); 903 ret = PTR_ERR(jzdma->clk); 904 return ret; 905 } 906 907 clk_prepare_enable(jzdma->clk); 908 909 /* Property is optional, if it doesn't exist the value will remain 0. */ 910 of_property_read_u32_index(dev->of_node, "ingenic,reserved-channels", 911 0, &jzdma->chan_reserved); 912 913 dd = &jzdma->dma_device; 914 915 /* 916 * The real segment size limit is dependent on the size unit selected 917 * for the transfer. Because the size unit is selected automatically 918 * and may be as small as 1 byte, use a safe limit of 2^24-1 bytes to 919 * ensure the 24-bit transfer count in the descriptor cannot overflow. 920 */ 921 dma_set_max_seg_size(dev, 0xffffff); 922 923 dma_cap_set(DMA_MEMCPY, dd->cap_mask); 924 dma_cap_set(DMA_SLAVE, dd->cap_mask); 925 dma_cap_set(DMA_CYCLIC, dd->cap_mask); 926 927 dd->dev = dev; 928 dd->copy_align = DMAENGINE_ALIGN_4_BYTES; 929 dd->device_alloc_chan_resources = jz4780_dma_alloc_chan_resources; 930 dd->device_free_chan_resources = jz4780_dma_free_chan_resources; 931 dd->device_prep_slave_sg = jz4780_dma_prep_slave_sg; 932 dd->device_prep_dma_cyclic = jz4780_dma_prep_dma_cyclic; 933 dd->device_prep_dma_memcpy = jz4780_dma_prep_dma_memcpy; 934 dd->device_config = jz4780_dma_config; 935 dd->device_terminate_all = jz4780_dma_terminate_all; 936 dd->device_synchronize = jz4780_dma_synchronize; 937 dd->device_tx_status = jz4780_dma_tx_status; 938 dd->device_issue_pending = jz4780_dma_issue_pending; 939 dd->src_addr_widths = JZ_DMA_BUSWIDTHS; 940 dd->dst_addr_widths = JZ_DMA_BUSWIDTHS; 941 dd->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); 942 dd->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; 943 dd->max_sg_burst = JZ_DMA_MAX_DESC; 944 945 /* 946 * Enable DMA controller, mark all channels as not programmable. 947 * Also set the FMSC bit - it increases MSC performance, so it makes 948 * little sense not to enable it. 949 */ 950 jz4780_dma_ctrl_writel(jzdma, JZ_DMA_REG_DMAC, JZ_DMA_DMAC_DMAE | 951 JZ_DMA_DMAC_FAIC | JZ_DMA_DMAC_FMSC); 952 953 if (soc_data->flags & JZ_SOC_DATA_PROGRAMMABLE_DMA) 954 jz4780_dma_ctrl_writel(jzdma, JZ_DMA_REG_DMACP, 0); 955 956 INIT_LIST_HEAD(&dd->channels); 957 958 for (i = 0; i < soc_data->nb_channels; i++) { 959 jzchan = &jzdma->chan[i]; 960 jzchan->id = i; 961 962 vchan_init(&jzchan->vchan, dd); 963 jzchan->vchan.desc_free = jz4780_dma_desc_free; 964 } 965 966 /* 967 * On JZ4760, chan0 won't enable properly the first time. 968 * Enabling then disabling chan1 will magically make chan0 work 969 * correctly. 970 */ 971 jz4780_dma_chan_enable(jzdma, 1); 972 jz4780_dma_chan_disable(jzdma, 1); 973 974 ret = platform_get_irq(pdev, 0); 975 if (ret < 0) 976 goto err_disable_clk; 977 978 jzdma->irq = ret; 979 980 ret = request_irq(jzdma->irq, jz4780_dma_irq_handler, 0, dev_name(dev), 981 jzdma); 982 if (ret) { 983 dev_err(dev, "failed to request IRQ %u!\n", jzdma->irq); 984 goto err_disable_clk; 985 } 986 987 ret = dmaenginem_async_device_register(dd); 988 if (ret) { 989 dev_err(dev, "failed to register device\n"); 990 goto err_free_irq; 991 } 992 993 /* Register with OF DMA helpers. */ 994 ret = of_dma_controller_register(dev->of_node, jz4780_of_dma_xlate, 995 jzdma); 996 if (ret) { 997 dev_err(dev, "failed to register OF DMA controller\n"); 998 goto err_free_irq; 999 } 1000 1001 dev_info(dev, "JZ4780 DMA controller initialised\n"); 1002 return 0; 1003 1004 err_free_irq: 1005 free_irq(jzdma->irq, jzdma); 1006 1007 err_disable_clk: 1008 clk_disable_unprepare(jzdma->clk); 1009 return ret; 1010 } 1011 1012 static int jz4780_dma_remove(struct platform_device *pdev) 1013 { 1014 struct jz4780_dma_dev *jzdma = platform_get_drvdata(pdev); 1015 int i; 1016 1017 of_dma_controller_free(pdev->dev.of_node); 1018 1019 clk_disable_unprepare(jzdma->clk); 1020 free_irq(jzdma->irq, jzdma); 1021 1022 for (i = 0; i < jzdma->soc_data->nb_channels; i++) 1023 tasklet_kill(&jzdma->chan[i].vchan.task); 1024 1025 return 0; 1026 } 1027 1028 static const struct jz4780_dma_soc_data jz4740_dma_soc_data = { 1029 .nb_channels = 6, 1030 .transfer_ord_max = 5, 1031 .flags = JZ_SOC_DATA_BREAK_LINKS, 1032 }; 1033 1034 static const struct jz4780_dma_soc_data jz4725b_dma_soc_data = { 1035 .nb_channels = 6, 1036 .transfer_ord_max = 5, 1037 .flags = JZ_SOC_DATA_PER_CHAN_PM | JZ_SOC_DATA_NO_DCKES_DCKEC | 1038 JZ_SOC_DATA_BREAK_LINKS, 1039 }; 1040 1041 static const struct jz4780_dma_soc_data jz4760_dma_soc_data = { 1042 .nb_channels = 5, 1043 .transfer_ord_max = 6, 1044 .flags = JZ_SOC_DATA_PER_CHAN_PM | JZ_SOC_DATA_NO_DCKES_DCKEC, 1045 }; 1046 1047 static const struct jz4780_dma_soc_data jz4760_mdma_soc_data = { 1048 .nb_channels = 2, 1049 .transfer_ord_max = 6, 1050 .flags = JZ_SOC_DATA_PER_CHAN_PM | JZ_SOC_DATA_NO_DCKES_DCKEC, 1051 }; 1052 1053 static const struct jz4780_dma_soc_data jz4760_bdma_soc_data = { 1054 .nb_channels = 3, 1055 .transfer_ord_max = 6, 1056 .flags = JZ_SOC_DATA_PER_CHAN_PM | JZ_SOC_DATA_NO_DCKES_DCKEC, 1057 }; 1058 1059 static const struct jz4780_dma_soc_data jz4760b_dma_soc_data = { 1060 .nb_channels = 5, 1061 .transfer_ord_max = 6, 1062 .flags = JZ_SOC_DATA_PER_CHAN_PM, 1063 }; 1064 1065 static const struct jz4780_dma_soc_data jz4760b_mdma_soc_data = { 1066 .nb_channels = 2, 1067 .transfer_ord_max = 6, 1068 .flags = JZ_SOC_DATA_PER_CHAN_PM, 1069 }; 1070 1071 static const struct jz4780_dma_soc_data jz4760b_bdma_soc_data = { 1072 .nb_channels = 3, 1073 .transfer_ord_max = 6, 1074 .flags = JZ_SOC_DATA_PER_CHAN_PM, 1075 }; 1076 1077 static const struct jz4780_dma_soc_data jz4770_dma_soc_data = { 1078 .nb_channels = 6, 1079 .transfer_ord_max = 6, 1080 .flags = JZ_SOC_DATA_PER_CHAN_PM, 1081 }; 1082 1083 static const struct jz4780_dma_soc_data jz4780_dma_soc_data = { 1084 .nb_channels = 32, 1085 .transfer_ord_max = 7, 1086 .flags = JZ_SOC_DATA_ALLOW_LEGACY_DT | JZ_SOC_DATA_PROGRAMMABLE_DMA, 1087 }; 1088 1089 static const struct jz4780_dma_soc_data x1000_dma_soc_data = { 1090 .nb_channels = 8, 1091 .transfer_ord_max = 7, 1092 .flags = JZ_SOC_DATA_PROGRAMMABLE_DMA, 1093 }; 1094 1095 static const struct jz4780_dma_soc_data x1830_dma_soc_data = { 1096 .nb_channels = 32, 1097 .transfer_ord_max = 7, 1098 .flags = JZ_SOC_DATA_PROGRAMMABLE_DMA, 1099 }; 1100 1101 static const struct of_device_id jz4780_dma_dt_match[] = { 1102 { .compatible = "ingenic,jz4740-dma", .data = &jz4740_dma_soc_data }, 1103 { .compatible = "ingenic,jz4725b-dma", .data = &jz4725b_dma_soc_data }, 1104 { .compatible = "ingenic,jz4760-dma", .data = &jz4760_dma_soc_data }, 1105 { .compatible = "ingenic,jz4760-mdma", .data = &jz4760_mdma_soc_data }, 1106 { .compatible = "ingenic,jz4760-bdma", .data = &jz4760_bdma_soc_data }, 1107 { .compatible = "ingenic,jz4760b-dma", .data = &jz4760b_dma_soc_data }, 1108 { .compatible = "ingenic,jz4760b-mdma", .data = &jz4760b_mdma_soc_data }, 1109 { .compatible = "ingenic,jz4760b-bdma", .data = &jz4760b_bdma_soc_data }, 1110 { .compatible = "ingenic,jz4770-dma", .data = &jz4770_dma_soc_data }, 1111 { .compatible = "ingenic,jz4780-dma", .data = &jz4780_dma_soc_data }, 1112 { .compatible = "ingenic,x1000-dma", .data = &x1000_dma_soc_data }, 1113 { .compatible = "ingenic,x1830-dma", .data = &x1830_dma_soc_data }, 1114 {}, 1115 }; 1116 MODULE_DEVICE_TABLE(of, jz4780_dma_dt_match); 1117 1118 static struct platform_driver jz4780_dma_driver = { 1119 .probe = jz4780_dma_probe, 1120 .remove = jz4780_dma_remove, 1121 .driver = { 1122 .name = "jz4780-dma", 1123 .of_match_table = jz4780_dma_dt_match, 1124 }, 1125 }; 1126 1127 static int __init jz4780_dma_init(void) 1128 { 1129 return platform_driver_register(&jz4780_dma_driver); 1130 } 1131 subsys_initcall(jz4780_dma_init); 1132 1133 static void __exit jz4780_dma_exit(void) 1134 { 1135 platform_driver_unregister(&jz4780_dma_driver); 1136 } 1137 module_exit(jz4780_dma_exit); 1138 1139 MODULE_AUTHOR("Alex Smith <alex@alex-smith.me.uk>"); 1140 MODULE_DESCRIPTION("Ingenic JZ4780 DMA controller driver"); 1141 MODULE_LICENSE("GPL"); 1142