1 /* 2 * Copyright (C) 2017 Spreadtrum Communications Inc. 3 * 4 * SPDX-License-Identifier: GPL-2.0 5 */ 6 7 #include <linux/clk.h> 8 #include <linux/dma-mapping.h> 9 #include <linux/dma/sprd-dma.h> 10 #include <linux/errno.h> 11 #include <linux/init.h> 12 #include <linux/interrupt.h> 13 #include <linux/io.h> 14 #include <linux/kernel.h> 15 #include <linux/module.h> 16 #include <linux/of.h> 17 #include <linux/of_dma.h> 18 #include <linux/of_device.h> 19 #include <linux/pm_runtime.h> 20 #include <linux/slab.h> 21 22 #include "virt-dma.h" 23 24 #define SPRD_DMA_CHN_REG_OFFSET 0x1000 25 #define SPRD_DMA_CHN_REG_LENGTH 0x40 26 #define SPRD_DMA_MEMCPY_MIN_SIZE 64 27 28 /* DMA global registers definition */ 29 #define SPRD_DMA_GLB_PAUSE 0x0 30 #define SPRD_DMA_GLB_FRAG_WAIT 0x4 31 #define SPRD_DMA_GLB_REQ_PEND0_EN 0x8 32 #define SPRD_DMA_GLB_REQ_PEND1_EN 0xc 33 #define SPRD_DMA_GLB_INT_RAW_STS 0x10 34 #define SPRD_DMA_GLB_INT_MSK_STS 0x14 35 #define SPRD_DMA_GLB_REQ_STS 0x18 36 #define SPRD_DMA_GLB_CHN_EN_STS 0x1c 37 #define SPRD_DMA_GLB_DEBUG_STS 0x20 38 #define SPRD_DMA_GLB_ARB_SEL_STS 0x24 39 #define SPRD_DMA_GLB_REQ_UID(uid) (0x4 * ((uid) - 1)) 40 #define SPRD_DMA_GLB_REQ_UID_OFFSET 0x2000 41 42 /* DMA channel registers definition */ 43 #define SPRD_DMA_CHN_PAUSE 0x0 44 #define SPRD_DMA_CHN_REQ 0x4 45 #define SPRD_DMA_CHN_CFG 0x8 46 #define SPRD_DMA_CHN_INTC 0xc 47 #define SPRD_DMA_CHN_SRC_ADDR 0x10 48 #define SPRD_DMA_CHN_DES_ADDR 0x14 49 #define SPRD_DMA_CHN_FRG_LEN 0x18 50 #define SPRD_DMA_CHN_BLK_LEN 0x1c 51 #define SPRD_DMA_CHN_TRSC_LEN 0x20 52 #define SPRD_DMA_CHN_TRSF_STEP 0x24 53 #define SPRD_DMA_CHN_WARP_PTR 0x28 54 #define SPRD_DMA_CHN_WARP_TO 0x2c 55 #define SPRD_DMA_CHN_LLIST_PTR 0x30 56 #define SPRD_DMA_CHN_FRAG_STEP 0x34 57 #define SPRD_DMA_CHN_SRC_BLK_STEP 0x38 58 #define SPRD_DMA_CHN_DES_BLK_STEP 0x3c 59 60 /* SPRD_DMA_CHN_INTC register definition */ 61 #define SPRD_DMA_INT_MASK GENMASK(4, 0) 62 #define SPRD_DMA_INT_CLR_OFFSET 24 63 #define SPRD_DMA_FRAG_INT_EN BIT(0) 64 #define SPRD_DMA_BLK_INT_EN BIT(1) 65 #define SPRD_DMA_TRANS_INT_EN BIT(2) 66 #define SPRD_DMA_LIST_INT_EN BIT(3) 67 #define SPRD_DMA_CFG_ERR_INT_EN BIT(4) 68 69 /* SPRD_DMA_CHN_CFG register definition */ 70 #define SPRD_DMA_CHN_EN BIT(0) 71 #define SPRD_DMA_LINKLIST_EN BIT(4) 72 #define SPRD_DMA_WAIT_BDONE_OFFSET 24 73 #define SPRD_DMA_DONOT_WAIT_BDONE 1 74 75 /* SPRD_DMA_CHN_REQ register definition */ 76 #define SPRD_DMA_REQ_EN BIT(0) 77 78 /* SPRD_DMA_CHN_PAUSE register definition */ 79 #define SPRD_DMA_PAUSE_EN BIT(0) 80 #define SPRD_DMA_PAUSE_STS BIT(2) 81 #define SPRD_DMA_PAUSE_CNT 0x2000 82 83 /* DMA_CHN_WARP_* register definition */ 84 #define SPRD_DMA_HIGH_ADDR_MASK GENMASK(31, 28) 85 #define SPRD_DMA_LOW_ADDR_MASK GENMASK(31, 0) 86 #define SPRD_DMA_HIGH_ADDR_OFFSET 4 87 88 /* SPRD_DMA_CHN_INTC register definition */ 89 #define SPRD_DMA_FRAG_INT_STS BIT(16) 90 #define SPRD_DMA_BLK_INT_STS BIT(17) 91 #define SPRD_DMA_TRSC_INT_STS BIT(18) 92 #define SPRD_DMA_LIST_INT_STS BIT(19) 93 #define SPRD_DMA_CFGERR_INT_STS BIT(20) 94 #define SPRD_DMA_CHN_INT_STS \ 95 (SPRD_DMA_FRAG_INT_STS | SPRD_DMA_BLK_INT_STS | \ 96 SPRD_DMA_TRSC_INT_STS | SPRD_DMA_LIST_INT_STS | \ 97 SPRD_DMA_CFGERR_INT_STS) 98 99 /* SPRD_DMA_CHN_FRG_LEN register definition */ 100 #define SPRD_DMA_SRC_DATAWIDTH_OFFSET 30 101 #define SPRD_DMA_DES_DATAWIDTH_OFFSET 28 102 #define SPRD_DMA_SWT_MODE_OFFSET 26 103 #define SPRD_DMA_REQ_MODE_OFFSET 24 104 #define SPRD_DMA_REQ_MODE_MASK GENMASK(1, 0) 105 #define SPRD_DMA_FIX_SEL_OFFSET 21 106 #define SPRD_DMA_FIX_EN_OFFSET 20 107 #define SPRD_DMA_LLIST_END BIT(19) 108 #define SPRD_DMA_FRG_LEN_MASK GENMASK(16, 0) 109 110 /* SPRD_DMA_CHN_BLK_LEN register definition */ 111 #define SPRD_DMA_BLK_LEN_MASK GENMASK(16, 0) 112 113 /* SPRD_DMA_CHN_TRSC_LEN register definition */ 114 #define SPRD_DMA_TRSC_LEN_MASK GENMASK(27, 0) 115 116 /* SPRD_DMA_CHN_TRSF_STEP register definition */ 117 #define SPRD_DMA_DEST_TRSF_STEP_OFFSET 16 118 #define SPRD_DMA_SRC_TRSF_STEP_OFFSET 0 119 #define SPRD_DMA_TRSF_STEP_MASK GENMASK(15, 0) 120 121 /* define the DMA transfer step type */ 122 #define SPRD_DMA_NONE_STEP 0 123 #define SPRD_DMA_BYTE_STEP 1 124 #define SPRD_DMA_SHORT_STEP 2 125 #define SPRD_DMA_WORD_STEP 4 126 #define SPRD_DMA_DWORD_STEP 8 127 128 #define SPRD_DMA_SOFTWARE_UID 0 129 130 /* dma data width values */ 131 enum sprd_dma_datawidth { 132 SPRD_DMA_DATAWIDTH_1_BYTE, 133 SPRD_DMA_DATAWIDTH_2_BYTES, 134 SPRD_DMA_DATAWIDTH_4_BYTES, 135 SPRD_DMA_DATAWIDTH_8_BYTES, 136 }; 137 138 /* dma channel hardware configuration */ 139 struct sprd_dma_chn_hw { 140 u32 pause; 141 u32 req; 142 u32 cfg; 143 u32 intc; 144 u32 src_addr; 145 u32 des_addr; 146 u32 frg_len; 147 u32 blk_len; 148 u32 trsc_len; 149 u32 trsf_step; 150 u32 wrap_ptr; 151 u32 wrap_to; 152 u32 llist_ptr; 153 u32 frg_step; 154 u32 src_blk_step; 155 u32 des_blk_step; 156 }; 157 158 /* dma request description */ 159 struct sprd_dma_desc { 160 struct virt_dma_desc vd; 161 struct sprd_dma_chn_hw chn_hw; 162 }; 163 164 /* dma channel description */ 165 struct sprd_dma_chn { 166 struct virt_dma_chan vc; 167 void __iomem *chn_base; 168 struct sprd_dma_linklist linklist; 169 struct dma_slave_config slave_cfg; 170 u32 chn_num; 171 u32 dev_id; 172 struct sprd_dma_desc *cur_desc; 173 }; 174 175 /* SPRD dma device */ 176 struct sprd_dma_dev { 177 struct dma_device dma_dev; 178 void __iomem *glb_base; 179 struct clk *clk; 180 struct clk *ashb_clk; 181 int irq; 182 u32 total_chns; 183 struct sprd_dma_chn channels[0]; 184 }; 185 186 static bool sprd_dma_filter_fn(struct dma_chan *chan, void *param); 187 static struct of_dma_filter_info sprd_dma_info = { 188 .filter_fn = sprd_dma_filter_fn, 189 }; 190 191 static inline struct sprd_dma_chn *to_sprd_dma_chan(struct dma_chan *c) 192 { 193 return container_of(c, struct sprd_dma_chn, vc.chan); 194 } 195 196 static inline struct sprd_dma_dev *to_sprd_dma_dev(struct dma_chan *c) 197 { 198 struct sprd_dma_chn *schan = to_sprd_dma_chan(c); 199 200 return container_of(schan, struct sprd_dma_dev, channels[c->chan_id]); 201 } 202 203 static inline struct sprd_dma_desc *to_sprd_dma_desc(struct virt_dma_desc *vd) 204 { 205 return container_of(vd, struct sprd_dma_desc, vd); 206 } 207 208 static void sprd_dma_chn_update(struct sprd_dma_chn *schan, u32 reg, 209 u32 mask, u32 val) 210 { 211 u32 orig = readl(schan->chn_base + reg); 212 u32 tmp; 213 214 tmp = (orig & ~mask) | val; 215 writel(tmp, schan->chn_base + reg); 216 } 217 218 static int sprd_dma_enable(struct sprd_dma_dev *sdev) 219 { 220 int ret; 221 222 ret = clk_prepare_enable(sdev->clk); 223 if (ret) 224 return ret; 225 226 /* 227 * The ashb_clk is optional and only for AGCP DMA controller, so we 228 * need add one condition to check if the ashb_clk need enable. 229 */ 230 if (!IS_ERR(sdev->ashb_clk)) 231 ret = clk_prepare_enable(sdev->ashb_clk); 232 233 return ret; 234 } 235 236 static void sprd_dma_disable(struct sprd_dma_dev *sdev) 237 { 238 clk_disable_unprepare(sdev->clk); 239 240 /* 241 * Need to check if we need disable the optional ashb_clk for AGCP DMA. 242 */ 243 if (!IS_ERR(sdev->ashb_clk)) 244 clk_disable_unprepare(sdev->ashb_clk); 245 } 246 247 static void sprd_dma_set_uid(struct sprd_dma_chn *schan) 248 { 249 struct sprd_dma_dev *sdev = to_sprd_dma_dev(&schan->vc.chan); 250 u32 dev_id = schan->dev_id; 251 252 if (dev_id != SPRD_DMA_SOFTWARE_UID) { 253 u32 uid_offset = SPRD_DMA_GLB_REQ_UID_OFFSET + 254 SPRD_DMA_GLB_REQ_UID(dev_id); 255 256 writel(schan->chn_num + 1, sdev->glb_base + uid_offset); 257 } 258 } 259 260 static void sprd_dma_unset_uid(struct sprd_dma_chn *schan) 261 { 262 struct sprd_dma_dev *sdev = to_sprd_dma_dev(&schan->vc.chan); 263 u32 dev_id = schan->dev_id; 264 265 if (dev_id != SPRD_DMA_SOFTWARE_UID) { 266 u32 uid_offset = SPRD_DMA_GLB_REQ_UID_OFFSET + 267 SPRD_DMA_GLB_REQ_UID(dev_id); 268 269 writel(0, sdev->glb_base + uid_offset); 270 } 271 } 272 273 static void sprd_dma_clear_int(struct sprd_dma_chn *schan) 274 { 275 sprd_dma_chn_update(schan, SPRD_DMA_CHN_INTC, 276 SPRD_DMA_INT_MASK << SPRD_DMA_INT_CLR_OFFSET, 277 SPRD_DMA_INT_MASK << SPRD_DMA_INT_CLR_OFFSET); 278 } 279 280 static void sprd_dma_enable_chn(struct sprd_dma_chn *schan) 281 { 282 sprd_dma_chn_update(schan, SPRD_DMA_CHN_CFG, SPRD_DMA_CHN_EN, 283 SPRD_DMA_CHN_EN); 284 } 285 286 static void sprd_dma_disable_chn(struct sprd_dma_chn *schan) 287 { 288 sprd_dma_chn_update(schan, SPRD_DMA_CHN_CFG, SPRD_DMA_CHN_EN, 0); 289 } 290 291 static void sprd_dma_soft_request(struct sprd_dma_chn *schan) 292 { 293 sprd_dma_chn_update(schan, SPRD_DMA_CHN_REQ, SPRD_DMA_REQ_EN, 294 SPRD_DMA_REQ_EN); 295 } 296 297 static void sprd_dma_pause_resume(struct sprd_dma_chn *schan, bool enable) 298 { 299 struct sprd_dma_dev *sdev = to_sprd_dma_dev(&schan->vc.chan); 300 u32 pause, timeout = SPRD_DMA_PAUSE_CNT; 301 302 if (enable) { 303 sprd_dma_chn_update(schan, SPRD_DMA_CHN_PAUSE, 304 SPRD_DMA_PAUSE_EN, SPRD_DMA_PAUSE_EN); 305 306 do { 307 pause = readl(schan->chn_base + SPRD_DMA_CHN_PAUSE); 308 if (pause & SPRD_DMA_PAUSE_STS) 309 break; 310 311 cpu_relax(); 312 } while (--timeout > 0); 313 314 if (!timeout) 315 dev_warn(sdev->dma_dev.dev, 316 "pause dma controller timeout\n"); 317 } else { 318 sprd_dma_chn_update(schan, SPRD_DMA_CHN_PAUSE, 319 SPRD_DMA_PAUSE_EN, 0); 320 } 321 } 322 323 static void sprd_dma_stop_and_disable(struct sprd_dma_chn *schan) 324 { 325 u32 cfg = readl(schan->chn_base + SPRD_DMA_CHN_CFG); 326 327 if (!(cfg & SPRD_DMA_CHN_EN)) 328 return; 329 330 sprd_dma_pause_resume(schan, true); 331 sprd_dma_disable_chn(schan); 332 } 333 334 static unsigned long sprd_dma_get_dst_addr(struct sprd_dma_chn *schan) 335 { 336 unsigned long addr, addr_high; 337 338 addr = readl(schan->chn_base + SPRD_DMA_CHN_DES_ADDR); 339 addr_high = readl(schan->chn_base + SPRD_DMA_CHN_WARP_TO) & 340 SPRD_DMA_HIGH_ADDR_MASK; 341 342 return addr | (addr_high << SPRD_DMA_HIGH_ADDR_OFFSET); 343 } 344 345 static enum sprd_dma_int_type sprd_dma_get_int_type(struct sprd_dma_chn *schan) 346 { 347 struct sprd_dma_dev *sdev = to_sprd_dma_dev(&schan->vc.chan); 348 u32 intc_sts = readl(schan->chn_base + SPRD_DMA_CHN_INTC) & 349 SPRD_DMA_CHN_INT_STS; 350 351 switch (intc_sts) { 352 case SPRD_DMA_CFGERR_INT_STS: 353 return SPRD_DMA_CFGERR_INT; 354 355 case SPRD_DMA_LIST_INT_STS: 356 return SPRD_DMA_LIST_INT; 357 358 case SPRD_DMA_TRSC_INT_STS: 359 return SPRD_DMA_TRANS_INT; 360 361 case SPRD_DMA_BLK_INT_STS: 362 return SPRD_DMA_BLK_INT; 363 364 case SPRD_DMA_FRAG_INT_STS: 365 return SPRD_DMA_FRAG_INT; 366 367 default: 368 dev_warn(sdev->dma_dev.dev, "incorrect dma interrupt type\n"); 369 return SPRD_DMA_NO_INT; 370 } 371 } 372 373 static enum sprd_dma_req_mode sprd_dma_get_req_type(struct sprd_dma_chn *schan) 374 { 375 u32 frag_reg = readl(schan->chn_base + SPRD_DMA_CHN_FRG_LEN); 376 377 return (frag_reg >> SPRD_DMA_REQ_MODE_OFFSET) & SPRD_DMA_REQ_MODE_MASK; 378 } 379 380 static void sprd_dma_set_chn_config(struct sprd_dma_chn *schan, 381 struct sprd_dma_desc *sdesc) 382 { 383 struct sprd_dma_chn_hw *cfg = &sdesc->chn_hw; 384 385 writel(cfg->pause, schan->chn_base + SPRD_DMA_CHN_PAUSE); 386 writel(cfg->cfg, schan->chn_base + SPRD_DMA_CHN_CFG); 387 writel(cfg->intc, schan->chn_base + SPRD_DMA_CHN_INTC); 388 writel(cfg->src_addr, schan->chn_base + SPRD_DMA_CHN_SRC_ADDR); 389 writel(cfg->des_addr, schan->chn_base + SPRD_DMA_CHN_DES_ADDR); 390 writel(cfg->frg_len, schan->chn_base + SPRD_DMA_CHN_FRG_LEN); 391 writel(cfg->blk_len, schan->chn_base + SPRD_DMA_CHN_BLK_LEN); 392 writel(cfg->trsc_len, schan->chn_base + SPRD_DMA_CHN_TRSC_LEN); 393 writel(cfg->trsf_step, schan->chn_base + SPRD_DMA_CHN_TRSF_STEP); 394 writel(cfg->wrap_ptr, schan->chn_base + SPRD_DMA_CHN_WARP_PTR); 395 writel(cfg->wrap_to, schan->chn_base + SPRD_DMA_CHN_WARP_TO); 396 writel(cfg->llist_ptr, schan->chn_base + SPRD_DMA_CHN_LLIST_PTR); 397 writel(cfg->frg_step, schan->chn_base + SPRD_DMA_CHN_FRAG_STEP); 398 writel(cfg->src_blk_step, schan->chn_base + SPRD_DMA_CHN_SRC_BLK_STEP); 399 writel(cfg->des_blk_step, schan->chn_base + SPRD_DMA_CHN_DES_BLK_STEP); 400 writel(cfg->req, schan->chn_base + SPRD_DMA_CHN_REQ); 401 } 402 403 static void sprd_dma_start(struct sprd_dma_chn *schan) 404 { 405 struct virt_dma_desc *vd = vchan_next_desc(&schan->vc); 406 407 if (!vd) 408 return; 409 410 list_del(&vd->node); 411 schan->cur_desc = to_sprd_dma_desc(vd); 412 413 /* 414 * Copy the DMA configuration from DMA descriptor to this hardware 415 * channel. 416 */ 417 sprd_dma_set_chn_config(schan, schan->cur_desc); 418 sprd_dma_set_uid(schan); 419 sprd_dma_enable_chn(schan); 420 421 if (schan->dev_id == SPRD_DMA_SOFTWARE_UID) 422 sprd_dma_soft_request(schan); 423 } 424 425 static void sprd_dma_stop(struct sprd_dma_chn *schan) 426 { 427 sprd_dma_stop_and_disable(schan); 428 sprd_dma_unset_uid(schan); 429 sprd_dma_clear_int(schan); 430 } 431 432 static bool sprd_dma_check_trans_done(struct sprd_dma_desc *sdesc, 433 enum sprd_dma_int_type int_type, 434 enum sprd_dma_req_mode req_mode) 435 { 436 if (int_type == SPRD_DMA_NO_INT) 437 return false; 438 439 if (int_type >= req_mode + 1) 440 return true; 441 else 442 return false; 443 } 444 445 static irqreturn_t dma_irq_handle(int irq, void *dev_id) 446 { 447 struct sprd_dma_dev *sdev = (struct sprd_dma_dev *)dev_id; 448 u32 irq_status = readl(sdev->glb_base + SPRD_DMA_GLB_INT_MSK_STS); 449 struct sprd_dma_chn *schan; 450 struct sprd_dma_desc *sdesc; 451 enum sprd_dma_req_mode req_type; 452 enum sprd_dma_int_type int_type; 453 bool trans_done = false; 454 u32 i; 455 456 while (irq_status) { 457 i = __ffs(irq_status); 458 irq_status &= (irq_status - 1); 459 schan = &sdev->channels[i]; 460 461 spin_lock(&schan->vc.lock); 462 int_type = sprd_dma_get_int_type(schan); 463 req_type = sprd_dma_get_req_type(schan); 464 sprd_dma_clear_int(schan); 465 466 sdesc = schan->cur_desc; 467 468 /* Check if the dma request descriptor is done. */ 469 trans_done = sprd_dma_check_trans_done(sdesc, int_type, 470 req_type); 471 if (trans_done == true) { 472 vchan_cookie_complete(&sdesc->vd); 473 schan->cur_desc = NULL; 474 sprd_dma_start(schan); 475 } 476 spin_unlock(&schan->vc.lock); 477 } 478 479 return IRQ_HANDLED; 480 } 481 482 static int sprd_dma_alloc_chan_resources(struct dma_chan *chan) 483 { 484 struct sprd_dma_chn *schan = to_sprd_dma_chan(chan); 485 int ret; 486 487 ret = pm_runtime_get_sync(chan->device->dev); 488 if (ret < 0) 489 return ret; 490 491 schan->dev_id = SPRD_DMA_SOFTWARE_UID; 492 return 0; 493 } 494 495 static void sprd_dma_free_chan_resources(struct dma_chan *chan) 496 { 497 struct sprd_dma_chn *schan = to_sprd_dma_chan(chan); 498 unsigned long flags; 499 500 spin_lock_irqsave(&schan->vc.lock, flags); 501 sprd_dma_stop(schan); 502 spin_unlock_irqrestore(&schan->vc.lock, flags); 503 504 vchan_free_chan_resources(&schan->vc); 505 pm_runtime_put(chan->device->dev); 506 } 507 508 static enum dma_status sprd_dma_tx_status(struct dma_chan *chan, 509 dma_cookie_t cookie, 510 struct dma_tx_state *txstate) 511 { 512 struct sprd_dma_chn *schan = to_sprd_dma_chan(chan); 513 struct virt_dma_desc *vd; 514 unsigned long flags; 515 enum dma_status ret; 516 u32 pos; 517 518 ret = dma_cookie_status(chan, cookie, txstate); 519 if (ret == DMA_COMPLETE || !txstate) 520 return ret; 521 522 spin_lock_irqsave(&schan->vc.lock, flags); 523 vd = vchan_find_desc(&schan->vc, cookie); 524 if (vd) { 525 struct sprd_dma_desc *sdesc = to_sprd_dma_desc(vd); 526 struct sprd_dma_chn_hw *hw = &sdesc->chn_hw; 527 528 if (hw->trsc_len > 0) 529 pos = hw->trsc_len; 530 else if (hw->blk_len > 0) 531 pos = hw->blk_len; 532 else if (hw->frg_len > 0) 533 pos = hw->frg_len; 534 else 535 pos = 0; 536 } else if (schan->cur_desc && schan->cur_desc->vd.tx.cookie == cookie) { 537 pos = sprd_dma_get_dst_addr(schan); 538 } else { 539 pos = 0; 540 } 541 spin_unlock_irqrestore(&schan->vc.lock, flags); 542 543 dma_set_residue(txstate, pos); 544 return ret; 545 } 546 547 static void sprd_dma_issue_pending(struct dma_chan *chan) 548 { 549 struct sprd_dma_chn *schan = to_sprd_dma_chan(chan); 550 unsigned long flags; 551 552 spin_lock_irqsave(&schan->vc.lock, flags); 553 if (vchan_issue_pending(&schan->vc) && !schan->cur_desc) 554 sprd_dma_start(schan); 555 spin_unlock_irqrestore(&schan->vc.lock, flags); 556 } 557 558 static int sprd_dma_get_datawidth(enum dma_slave_buswidth buswidth) 559 { 560 switch (buswidth) { 561 case DMA_SLAVE_BUSWIDTH_1_BYTE: 562 case DMA_SLAVE_BUSWIDTH_2_BYTES: 563 case DMA_SLAVE_BUSWIDTH_4_BYTES: 564 case DMA_SLAVE_BUSWIDTH_8_BYTES: 565 return ffs(buswidth) - 1; 566 567 default: 568 return -EINVAL; 569 } 570 } 571 572 static int sprd_dma_get_step(enum dma_slave_buswidth buswidth) 573 { 574 switch (buswidth) { 575 case DMA_SLAVE_BUSWIDTH_1_BYTE: 576 case DMA_SLAVE_BUSWIDTH_2_BYTES: 577 case DMA_SLAVE_BUSWIDTH_4_BYTES: 578 case DMA_SLAVE_BUSWIDTH_8_BYTES: 579 return buswidth; 580 581 default: 582 return -EINVAL; 583 } 584 } 585 586 static int sprd_dma_fill_desc(struct dma_chan *chan, 587 struct sprd_dma_chn_hw *hw, 588 unsigned int sglen, int sg_index, 589 dma_addr_t src, dma_addr_t dst, u32 len, 590 enum dma_transfer_direction dir, 591 unsigned long flags, 592 struct dma_slave_config *slave_cfg) 593 { 594 struct sprd_dma_dev *sdev = to_sprd_dma_dev(chan); 595 struct sprd_dma_chn *schan = to_sprd_dma_chan(chan); 596 u32 req_mode = (flags >> SPRD_DMA_REQ_SHIFT) & SPRD_DMA_REQ_MODE_MASK; 597 u32 int_mode = flags & SPRD_DMA_INT_MASK; 598 int src_datawidth, dst_datawidth, src_step, dst_step; 599 u32 temp, fix_mode = 0, fix_en = 0; 600 601 if (dir == DMA_MEM_TO_DEV) { 602 src_step = sprd_dma_get_step(slave_cfg->src_addr_width); 603 if (src_step < 0) { 604 dev_err(sdev->dma_dev.dev, "invalid source step\n"); 605 return src_step; 606 } 607 dst_step = SPRD_DMA_NONE_STEP; 608 } else { 609 dst_step = sprd_dma_get_step(slave_cfg->dst_addr_width); 610 if (dst_step < 0) { 611 dev_err(sdev->dma_dev.dev, "invalid destination step\n"); 612 return dst_step; 613 } 614 src_step = SPRD_DMA_NONE_STEP; 615 } 616 617 src_datawidth = sprd_dma_get_datawidth(slave_cfg->src_addr_width); 618 if (src_datawidth < 0) { 619 dev_err(sdev->dma_dev.dev, "invalid source datawidth\n"); 620 return src_datawidth; 621 } 622 623 dst_datawidth = sprd_dma_get_datawidth(slave_cfg->dst_addr_width); 624 if (dst_datawidth < 0) { 625 dev_err(sdev->dma_dev.dev, "invalid destination datawidth\n"); 626 return dst_datawidth; 627 } 628 629 if (slave_cfg->slave_id) 630 schan->dev_id = slave_cfg->slave_id; 631 632 hw->cfg = SPRD_DMA_DONOT_WAIT_BDONE << SPRD_DMA_WAIT_BDONE_OFFSET; 633 634 /* 635 * wrap_ptr and wrap_to will save the high 4 bits source address and 636 * destination address. 637 */ 638 hw->wrap_ptr = (src >> SPRD_DMA_HIGH_ADDR_OFFSET) & SPRD_DMA_HIGH_ADDR_MASK; 639 hw->wrap_to = (dst >> SPRD_DMA_HIGH_ADDR_OFFSET) & SPRD_DMA_HIGH_ADDR_MASK; 640 hw->src_addr = src & SPRD_DMA_LOW_ADDR_MASK; 641 hw->des_addr = dst & SPRD_DMA_LOW_ADDR_MASK; 642 643 /* 644 * If the src step and dst step both are 0 or both are not 0, that means 645 * we can not enable the fix mode. If one is 0 and another one is not, 646 * we can enable the fix mode. 647 */ 648 if ((src_step != 0 && dst_step != 0) || (src_step | dst_step) == 0) { 649 fix_en = 0; 650 } else { 651 fix_en = 1; 652 if (src_step) 653 fix_mode = 1; 654 else 655 fix_mode = 0; 656 } 657 658 hw->intc = int_mode | SPRD_DMA_CFG_ERR_INT_EN; 659 660 temp = src_datawidth << SPRD_DMA_SRC_DATAWIDTH_OFFSET; 661 temp |= dst_datawidth << SPRD_DMA_DES_DATAWIDTH_OFFSET; 662 temp |= req_mode << SPRD_DMA_REQ_MODE_OFFSET; 663 temp |= fix_mode << SPRD_DMA_FIX_SEL_OFFSET; 664 temp |= fix_en << SPRD_DMA_FIX_EN_OFFSET; 665 temp |= slave_cfg->src_maxburst & SPRD_DMA_FRG_LEN_MASK; 666 hw->frg_len = temp; 667 668 hw->blk_len = len & SPRD_DMA_BLK_LEN_MASK; 669 hw->trsc_len = len & SPRD_DMA_TRSC_LEN_MASK; 670 671 temp = (dst_step & SPRD_DMA_TRSF_STEP_MASK) << SPRD_DMA_DEST_TRSF_STEP_OFFSET; 672 temp |= (src_step & SPRD_DMA_TRSF_STEP_MASK) << SPRD_DMA_SRC_TRSF_STEP_OFFSET; 673 hw->trsf_step = temp; 674 675 /* link-list configuration */ 676 if (schan->linklist.phy_addr) { 677 if (sg_index == sglen - 1) 678 hw->frg_len |= SPRD_DMA_LLIST_END; 679 680 hw->cfg |= SPRD_DMA_LINKLIST_EN; 681 682 /* link-list index */ 683 temp = (sg_index + 1) % sglen; 684 /* Next link-list configuration's physical address offset */ 685 temp = temp * sizeof(*hw) + SPRD_DMA_CHN_SRC_ADDR; 686 /* 687 * Set the link-list pointer point to next link-list 688 * configuration's physical address. 689 */ 690 hw->llist_ptr = schan->linklist.phy_addr + temp; 691 } else { 692 hw->llist_ptr = 0; 693 } 694 695 hw->frg_step = 0; 696 hw->src_blk_step = 0; 697 hw->des_blk_step = 0; 698 return 0; 699 } 700 701 static int sprd_dma_fill_linklist_desc(struct dma_chan *chan, 702 unsigned int sglen, int sg_index, 703 dma_addr_t src, dma_addr_t dst, u32 len, 704 enum dma_transfer_direction dir, 705 unsigned long flags, 706 struct dma_slave_config *slave_cfg) 707 { 708 struct sprd_dma_chn *schan = to_sprd_dma_chan(chan); 709 struct sprd_dma_chn_hw *hw; 710 711 if (!schan->linklist.virt_addr) 712 return -EINVAL; 713 714 hw = (struct sprd_dma_chn_hw *)(schan->linklist.virt_addr + 715 sg_index * sizeof(*hw)); 716 717 return sprd_dma_fill_desc(chan, hw, sglen, sg_index, src, dst, len, 718 dir, flags, slave_cfg); 719 } 720 721 static struct dma_async_tx_descriptor * 722 sprd_dma_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, 723 size_t len, unsigned long flags) 724 { 725 struct sprd_dma_chn *schan = to_sprd_dma_chan(chan); 726 struct sprd_dma_desc *sdesc; 727 struct sprd_dma_chn_hw *hw; 728 enum sprd_dma_datawidth datawidth; 729 u32 step, temp; 730 731 sdesc = kzalloc(sizeof(*sdesc), GFP_NOWAIT); 732 if (!sdesc) 733 return NULL; 734 735 hw = &sdesc->chn_hw; 736 737 hw->cfg = SPRD_DMA_DONOT_WAIT_BDONE << SPRD_DMA_WAIT_BDONE_OFFSET; 738 hw->intc = SPRD_DMA_TRANS_INT | SPRD_DMA_CFG_ERR_INT_EN; 739 hw->src_addr = src & SPRD_DMA_LOW_ADDR_MASK; 740 hw->des_addr = dest & SPRD_DMA_LOW_ADDR_MASK; 741 hw->wrap_ptr = (src >> SPRD_DMA_HIGH_ADDR_OFFSET) & 742 SPRD_DMA_HIGH_ADDR_MASK; 743 hw->wrap_to = (dest >> SPRD_DMA_HIGH_ADDR_OFFSET) & 744 SPRD_DMA_HIGH_ADDR_MASK; 745 746 if (IS_ALIGNED(len, 8)) { 747 datawidth = SPRD_DMA_DATAWIDTH_8_BYTES; 748 step = SPRD_DMA_DWORD_STEP; 749 } else if (IS_ALIGNED(len, 4)) { 750 datawidth = SPRD_DMA_DATAWIDTH_4_BYTES; 751 step = SPRD_DMA_WORD_STEP; 752 } else if (IS_ALIGNED(len, 2)) { 753 datawidth = SPRD_DMA_DATAWIDTH_2_BYTES; 754 step = SPRD_DMA_SHORT_STEP; 755 } else { 756 datawidth = SPRD_DMA_DATAWIDTH_1_BYTE; 757 step = SPRD_DMA_BYTE_STEP; 758 } 759 760 temp = datawidth << SPRD_DMA_SRC_DATAWIDTH_OFFSET; 761 temp |= datawidth << SPRD_DMA_DES_DATAWIDTH_OFFSET; 762 temp |= SPRD_DMA_TRANS_REQ << SPRD_DMA_REQ_MODE_OFFSET; 763 temp |= len & SPRD_DMA_FRG_LEN_MASK; 764 hw->frg_len = temp; 765 766 hw->blk_len = len & SPRD_DMA_BLK_LEN_MASK; 767 hw->trsc_len = len & SPRD_DMA_TRSC_LEN_MASK; 768 769 temp = (step & SPRD_DMA_TRSF_STEP_MASK) << SPRD_DMA_DEST_TRSF_STEP_OFFSET; 770 temp |= (step & SPRD_DMA_TRSF_STEP_MASK) << SPRD_DMA_SRC_TRSF_STEP_OFFSET; 771 hw->trsf_step = temp; 772 773 return vchan_tx_prep(&schan->vc, &sdesc->vd, flags); 774 } 775 776 static struct dma_async_tx_descriptor * 777 sprd_dma_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, 778 unsigned int sglen, enum dma_transfer_direction dir, 779 unsigned long flags, void *context) 780 { 781 struct sprd_dma_chn *schan = to_sprd_dma_chan(chan); 782 struct dma_slave_config *slave_cfg = &schan->slave_cfg; 783 dma_addr_t src = 0, dst = 0; 784 struct sprd_dma_desc *sdesc; 785 struct scatterlist *sg; 786 u32 len = 0; 787 int ret, i; 788 789 if (!is_slave_direction(dir)) 790 return NULL; 791 792 if (context) { 793 struct sprd_dma_linklist *ll_cfg = 794 (struct sprd_dma_linklist *)context; 795 796 schan->linklist.phy_addr = ll_cfg->phy_addr; 797 schan->linklist.virt_addr = ll_cfg->virt_addr; 798 } else { 799 schan->linklist.phy_addr = 0; 800 schan->linklist.virt_addr = 0; 801 } 802 803 sdesc = kzalloc(sizeof(*sdesc), GFP_NOWAIT); 804 if (!sdesc) 805 return NULL; 806 807 for_each_sg(sgl, sg, sglen, i) { 808 len = sg_dma_len(sg); 809 810 if (dir == DMA_MEM_TO_DEV) { 811 src = sg_dma_address(sg); 812 dst = slave_cfg->dst_addr; 813 } else { 814 src = slave_cfg->src_addr; 815 dst = sg_dma_address(sg); 816 } 817 818 /* 819 * The link-list mode needs at least 2 link-list 820 * configurations. If there is only one sg, it doesn't 821 * need to fill the link-list configuration. 822 */ 823 if (sglen < 2) 824 break; 825 826 ret = sprd_dma_fill_linklist_desc(chan, sglen, i, src, dst, len, 827 dir, flags, slave_cfg); 828 if (ret) { 829 kfree(sdesc); 830 return NULL; 831 } 832 } 833 834 ret = sprd_dma_fill_desc(chan, &sdesc->chn_hw, 0, 0, src, dst, len, 835 dir, flags, slave_cfg); 836 if (ret) { 837 kfree(sdesc); 838 return NULL; 839 } 840 841 return vchan_tx_prep(&schan->vc, &sdesc->vd, flags); 842 } 843 844 static int sprd_dma_slave_config(struct dma_chan *chan, 845 struct dma_slave_config *config) 846 { 847 struct sprd_dma_chn *schan = to_sprd_dma_chan(chan); 848 struct dma_slave_config *slave_cfg = &schan->slave_cfg; 849 850 if (!is_slave_direction(config->direction)) 851 return -EINVAL; 852 853 memcpy(slave_cfg, config, sizeof(*config)); 854 return 0; 855 } 856 857 static int sprd_dma_pause(struct dma_chan *chan) 858 { 859 struct sprd_dma_chn *schan = to_sprd_dma_chan(chan); 860 unsigned long flags; 861 862 spin_lock_irqsave(&schan->vc.lock, flags); 863 sprd_dma_pause_resume(schan, true); 864 spin_unlock_irqrestore(&schan->vc.lock, flags); 865 866 return 0; 867 } 868 869 static int sprd_dma_resume(struct dma_chan *chan) 870 { 871 struct sprd_dma_chn *schan = to_sprd_dma_chan(chan); 872 unsigned long flags; 873 874 spin_lock_irqsave(&schan->vc.lock, flags); 875 sprd_dma_pause_resume(schan, false); 876 spin_unlock_irqrestore(&schan->vc.lock, flags); 877 878 return 0; 879 } 880 881 static int sprd_dma_terminate_all(struct dma_chan *chan) 882 { 883 struct sprd_dma_chn *schan = to_sprd_dma_chan(chan); 884 unsigned long flags; 885 LIST_HEAD(head); 886 887 spin_lock_irqsave(&schan->vc.lock, flags); 888 sprd_dma_stop(schan); 889 890 vchan_get_all_descriptors(&schan->vc, &head); 891 spin_unlock_irqrestore(&schan->vc.lock, flags); 892 893 vchan_dma_desc_free_list(&schan->vc, &head); 894 return 0; 895 } 896 897 static void sprd_dma_free_desc(struct virt_dma_desc *vd) 898 { 899 struct sprd_dma_desc *sdesc = to_sprd_dma_desc(vd); 900 901 kfree(sdesc); 902 } 903 904 static bool sprd_dma_filter_fn(struct dma_chan *chan, void *param) 905 { 906 struct sprd_dma_chn *schan = to_sprd_dma_chan(chan); 907 struct sprd_dma_dev *sdev = to_sprd_dma_dev(&schan->vc.chan); 908 u32 req = *(u32 *)param; 909 910 if (req < sdev->total_chns) 911 return req == schan->chn_num + 1; 912 else 913 return false; 914 } 915 916 static int sprd_dma_probe(struct platform_device *pdev) 917 { 918 struct device_node *np = pdev->dev.of_node; 919 struct sprd_dma_dev *sdev; 920 struct sprd_dma_chn *dma_chn; 921 struct resource *res; 922 u32 chn_count; 923 int ret, i; 924 925 ret = device_property_read_u32(&pdev->dev, "#dma-channels", &chn_count); 926 if (ret) { 927 dev_err(&pdev->dev, "get dma channels count failed\n"); 928 return ret; 929 } 930 931 sdev = devm_kzalloc(&pdev->dev, 932 struct_size(sdev, channels, chn_count), 933 GFP_KERNEL); 934 if (!sdev) 935 return -ENOMEM; 936 937 sdev->clk = devm_clk_get(&pdev->dev, "enable"); 938 if (IS_ERR(sdev->clk)) { 939 dev_err(&pdev->dev, "get enable clock failed\n"); 940 return PTR_ERR(sdev->clk); 941 } 942 943 /* ashb clock is optional for AGCP DMA */ 944 sdev->ashb_clk = devm_clk_get(&pdev->dev, "ashb_eb"); 945 if (IS_ERR(sdev->ashb_clk)) 946 dev_warn(&pdev->dev, "no optional ashb eb clock\n"); 947 948 /* 949 * We have three DMA controllers: AP DMA, AON DMA and AGCP DMA. For AGCP 950 * DMA controller, it can or do not request the irq, which will save 951 * system power without resuming system by DMA interrupts if AGCP DMA 952 * does not request the irq. Thus the DMA interrupts property should 953 * be optional. 954 */ 955 sdev->irq = platform_get_irq(pdev, 0); 956 if (sdev->irq > 0) { 957 ret = devm_request_irq(&pdev->dev, sdev->irq, dma_irq_handle, 958 0, "sprd_dma", (void *)sdev); 959 if (ret < 0) { 960 dev_err(&pdev->dev, "request dma irq failed\n"); 961 return ret; 962 } 963 } else { 964 dev_warn(&pdev->dev, "no interrupts for the dma controller\n"); 965 } 966 967 res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 968 sdev->glb_base = devm_ioremap_resource(&pdev->dev, res); 969 if (IS_ERR(sdev->glb_base)) 970 return PTR_ERR(sdev->glb_base); 971 972 dma_cap_set(DMA_MEMCPY, sdev->dma_dev.cap_mask); 973 sdev->total_chns = chn_count; 974 sdev->dma_dev.chancnt = chn_count; 975 INIT_LIST_HEAD(&sdev->dma_dev.channels); 976 INIT_LIST_HEAD(&sdev->dma_dev.global_node); 977 sdev->dma_dev.dev = &pdev->dev; 978 sdev->dma_dev.device_alloc_chan_resources = sprd_dma_alloc_chan_resources; 979 sdev->dma_dev.device_free_chan_resources = sprd_dma_free_chan_resources; 980 sdev->dma_dev.device_tx_status = sprd_dma_tx_status; 981 sdev->dma_dev.device_issue_pending = sprd_dma_issue_pending; 982 sdev->dma_dev.device_prep_dma_memcpy = sprd_dma_prep_dma_memcpy; 983 sdev->dma_dev.device_prep_slave_sg = sprd_dma_prep_slave_sg; 984 sdev->dma_dev.device_config = sprd_dma_slave_config; 985 sdev->dma_dev.device_pause = sprd_dma_pause; 986 sdev->dma_dev.device_resume = sprd_dma_resume; 987 sdev->dma_dev.device_terminate_all = sprd_dma_terminate_all; 988 989 for (i = 0; i < chn_count; i++) { 990 dma_chn = &sdev->channels[i]; 991 dma_chn->chn_num = i; 992 dma_chn->cur_desc = NULL; 993 /* get each channel's registers base address. */ 994 dma_chn->chn_base = sdev->glb_base + SPRD_DMA_CHN_REG_OFFSET + 995 SPRD_DMA_CHN_REG_LENGTH * i; 996 997 dma_chn->vc.desc_free = sprd_dma_free_desc; 998 vchan_init(&dma_chn->vc, &sdev->dma_dev); 999 } 1000 1001 platform_set_drvdata(pdev, sdev); 1002 ret = sprd_dma_enable(sdev); 1003 if (ret) 1004 return ret; 1005 1006 pm_runtime_set_active(&pdev->dev); 1007 pm_runtime_enable(&pdev->dev); 1008 1009 ret = pm_runtime_get_sync(&pdev->dev); 1010 if (ret < 0) 1011 goto err_rpm; 1012 1013 ret = dma_async_device_register(&sdev->dma_dev); 1014 if (ret < 0) { 1015 dev_err(&pdev->dev, "register dma device failed:%d\n", ret); 1016 goto err_register; 1017 } 1018 1019 sprd_dma_info.dma_cap = sdev->dma_dev.cap_mask; 1020 ret = of_dma_controller_register(np, of_dma_simple_xlate, 1021 &sprd_dma_info); 1022 if (ret) 1023 goto err_of_register; 1024 1025 pm_runtime_put(&pdev->dev); 1026 return 0; 1027 1028 err_of_register: 1029 dma_async_device_unregister(&sdev->dma_dev); 1030 err_register: 1031 pm_runtime_put_noidle(&pdev->dev); 1032 pm_runtime_disable(&pdev->dev); 1033 err_rpm: 1034 sprd_dma_disable(sdev); 1035 return ret; 1036 } 1037 1038 static int sprd_dma_remove(struct platform_device *pdev) 1039 { 1040 struct sprd_dma_dev *sdev = platform_get_drvdata(pdev); 1041 struct sprd_dma_chn *c, *cn; 1042 int ret; 1043 1044 ret = pm_runtime_get_sync(&pdev->dev); 1045 if (ret < 0) 1046 return ret; 1047 1048 /* explicitly free the irq */ 1049 if (sdev->irq > 0) 1050 devm_free_irq(&pdev->dev, sdev->irq, sdev); 1051 1052 list_for_each_entry_safe(c, cn, &sdev->dma_dev.channels, 1053 vc.chan.device_node) { 1054 list_del(&c->vc.chan.device_node); 1055 tasklet_kill(&c->vc.task); 1056 } 1057 1058 of_dma_controller_free(pdev->dev.of_node); 1059 dma_async_device_unregister(&sdev->dma_dev); 1060 sprd_dma_disable(sdev); 1061 1062 pm_runtime_put_noidle(&pdev->dev); 1063 pm_runtime_disable(&pdev->dev); 1064 return 0; 1065 } 1066 1067 static const struct of_device_id sprd_dma_match[] = { 1068 { .compatible = "sprd,sc9860-dma", }, 1069 {}, 1070 }; 1071 1072 static int __maybe_unused sprd_dma_runtime_suspend(struct device *dev) 1073 { 1074 struct sprd_dma_dev *sdev = dev_get_drvdata(dev); 1075 1076 sprd_dma_disable(sdev); 1077 return 0; 1078 } 1079 1080 static int __maybe_unused sprd_dma_runtime_resume(struct device *dev) 1081 { 1082 struct sprd_dma_dev *sdev = dev_get_drvdata(dev); 1083 int ret; 1084 1085 ret = sprd_dma_enable(sdev); 1086 if (ret) 1087 dev_err(sdev->dma_dev.dev, "enable dma failed\n"); 1088 1089 return ret; 1090 } 1091 1092 static const struct dev_pm_ops sprd_dma_pm_ops = { 1093 SET_RUNTIME_PM_OPS(sprd_dma_runtime_suspend, 1094 sprd_dma_runtime_resume, 1095 NULL) 1096 }; 1097 1098 static struct platform_driver sprd_dma_driver = { 1099 .probe = sprd_dma_probe, 1100 .remove = sprd_dma_remove, 1101 .driver = { 1102 .name = "sprd-dma", 1103 .of_match_table = sprd_dma_match, 1104 .pm = &sprd_dma_pm_ops, 1105 }, 1106 }; 1107 module_platform_driver(sprd_dma_driver); 1108 1109 MODULE_LICENSE("GPL v2"); 1110 MODULE_DESCRIPTION("DMA driver for Spreadtrum"); 1111 MODULE_AUTHOR("Baolin Wang <baolin.wang@spreadtrum.com>"); 1112 MODULE_ALIAS("platform:sprd-dma"); 1113