1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Lightning Mountain centralized DMA controller driver 4 * 5 * Copyright (c) 2016 - 2020 Intel Corporation. 6 */ 7 8 #include <linux/bitfield.h> 9 #include <linux/clk.h> 10 #include <linux/dma-mapping.h> 11 #include <linux/dmapool.h> 12 #include <linux/err.h> 13 #include <linux/export.h> 14 #include <linux/init.h> 15 #include <linux/interrupt.h> 16 #include <linux/iopoll.h> 17 #include <linux/of_dma.h> 18 #include <linux/of_irq.h> 19 #include <linux/platform_device.h> 20 #include <linux/reset.h> 21 22 #include "../dmaengine.h" 23 #include "../virt-dma.h" 24 25 #define DRIVER_NAME "lgm-dma" 26 27 #define DMA_ID 0x0008 28 #define DMA_ID_REV GENMASK(7, 0) 29 #define DMA_ID_PNR GENMASK(19, 16) 30 #define DMA_ID_CHNR GENMASK(26, 20) 31 #define DMA_ID_DW_128B BIT(27) 32 #define DMA_ID_AW_36B BIT(28) 33 #define DMA_VER32 0x32 34 #define DMA_VER31 0x31 35 #define DMA_VER22 0x0A 36 37 #define DMA_CTRL 0x0010 38 #define DMA_CTRL_RST BIT(0) 39 #define DMA_CTRL_DSRAM_PATH BIT(1) 40 #define DMA_CTRL_DBURST_WR BIT(3) 41 #define DMA_CTRL_VLD_DF_ACK BIT(4) 42 #define DMA_CTRL_CH_FL BIT(6) 43 #define DMA_CTRL_DS_FOD BIT(7) 44 #define DMA_CTRL_DRB BIT(8) 45 #define DMA_CTRL_ENBE BIT(9) 46 #define DMA_CTRL_DESC_TMOUT_CNT_V31 GENMASK(27, 16) 47 #define DMA_CTRL_DESC_TMOUT_EN_V31 BIT(30) 48 #define DMA_CTRL_PKTARB BIT(31) 49 50 #define DMA_CPOLL 0x0014 51 #define DMA_CPOLL_CNT GENMASK(15, 4) 52 #define DMA_CPOLL_EN BIT(31) 53 54 #define DMA_CS 0x0018 55 #define DMA_CS_MASK GENMASK(5, 0) 56 57 #define DMA_CCTRL 0x001C 58 #define DMA_CCTRL_ON BIT(0) 59 #define DMA_CCTRL_RST BIT(1) 60 #define DMA_CCTRL_CH_POLL_EN BIT(2) 61 #define DMA_CCTRL_CH_ABC BIT(3) /* Adaptive Burst Chop */ 62 #define DMA_CDBA_MSB GENMASK(7, 4) 63 #define DMA_CCTRL_DIR_TX BIT(8) 64 #define DMA_CCTRL_CLASS GENMASK(11, 9) 65 #define DMA_CCTRL_CLASSH GENMASK(19, 18) 66 #define DMA_CCTRL_WR_NP_EN BIT(21) 67 #define DMA_CCTRL_PDEN BIT(23) 68 #define DMA_MAX_CLASS (SZ_32 - 1) 69 70 #define DMA_CDBA 0x0020 71 #define DMA_CDLEN 0x0024 72 #define DMA_CIS 0x0028 73 #define DMA_CIE 0x002C 74 #define DMA_CI_EOP BIT(1) 75 #define DMA_CI_DUR BIT(2) 76 #define DMA_CI_DESCPT BIT(3) 77 #define DMA_CI_CHOFF BIT(4) 78 #define DMA_CI_RDERR BIT(5) 79 #define DMA_CI_ALL \ 80 (DMA_CI_EOP | DMA_CI_DUR | DMA_CI_DESCPT | DMA_CI_CHOFF | DMA_CI_RDERR) 81 82 #define DMA_PS 0x0040 83 #define DMA_PCTRL 0x0044 84 #define DMA_PCTRL_RXBL16 BIT(0) 85 #define DMA_PCTRL_TXBL16 BIT(1) 86 #define DMA_PCTRL_RXBL GENMASK(3, 2) 87 #define DMA_PCTRL_RXBL_8 3 88 #define DMA_PCTRL_TXBL GENMASK(5, 4) 89 #define DMA_PCTRL_TXBL_8 3 90 #define DMA_PCTRL_PDEN BIT(6) 91 #define DMA_PCTRL_RXBL32 BIT(7) 92 #define DMA_PCTRL_RXENDI GENMASK(9, 8) 93 #define DMA_PCTRL_TXENDI GENMASK(11, 10) 94 #define DMA_PCTRL_TXBL32 BIT(15) 95 #define DMA_PCTRL_MEM_FLUSH BIT(16) 96 97 #define DMA_IRNEN1 0x00E8 98 #define DMA_IRNCR1 0x00EC 99 #define DMA_IRNEN 0x00F4 100 #define DMA_IRNCR 0x00F8 101 #define DMA_C_DP_TICK 0x100 102 #define DMA_C_DP_TICK_TIKNARB GENMASK(15, 0) 103 #define DMA_C_DP_TICK_TIKARB GENMASK(31, 16) 104 105 #define DMA_C_HDRM 0x110 106 /* 107 * If header mode is set in DMA descriptor, 108 * If bit 30 is disabled, HDR_LEN must be configured according to channel 109 * requirement. 110 * If bit 30 is enabled(checksum with heade mode), HDR_LEN has no need to 111 * be configured. It will enable check sum for switch 112 * If header mode is not set in DMA descriptor, 113 * This register setting doesn't matter 114 */ 115 #define DMA_C_HDRM_HDR_SUM BIT(30) 116 117 #define DMA_C_BOFF 0x120 118 #define DMA_C_BOFF_BOF_LEN GENMASK(7, 0) 119 #define DMA_C_BOFF_EN BIT(31) 120 121 #define DMA_ORRC 0x190 122 #define DMA_ORRC_ORRCNT GENMASK(8, 4) 123 #define DMA_ORRC_EN BIT(31) 124 125 #define DMA_C_ENDIAN 0x200 126 #define DMA_C_END_DATAENDI GENMASK(1, 0) 127 #define DMA_C_END_DE_EN BIT(7) 128 #define DMA_C_END_DESENDI GENMASK(9, 8) 129 #define DMA_C_END_DES_EN BIT(16) 130 131 /* DMA controller capability */ 132 #define DMA_ADDR_36BIT BIT(0) 133 #define DMA_DATA_128BIT BIT(1) 134 #define DMA_CHAN_FLOW_CTL BIT(2) 135 #define DMA_DESC_FOD BIT(3) 136 #define DMA_DESC_IN_SRAM BIT(4) 137 #define DMA_EN_BYTE_EN BIT(5) 138 #define DMA_DBURST_WR BIT(6) 139 #define DMA_VALID_DESC_FETCH_ACK BIT(7) 140 #define DMA_DFT_DRB BIT(8) 141 142 #define DMA_ORRC_MAX_CNT (SZ_32 - 1) 143 #define DMA_DFT_POLL_CNT SZ_4 144 #define DMA_DFT_BURST_V22 SZ_2 145 #define DMA_BURSTL_8DW SZ_8 146 #define DMA_BURSTL_16DW SZ_16 147 #define DMA_BURSTL_32DW SZ_32 148 #define DMA_DFT_BURST DMA_BURSTL_16DW 149 #define DMA_MAX_DESC_NUM (SZ_8K - 1) 150 #define DMA_CHAN_BOFF_MAX (SZ_256 - 1) 151 #define DMA_DFT_ENDIAN 0 152 153 #define DMA_DFT_DESC_TCNT 50 154 #define DMA_HDR_LEN_MAX (SZ_16K - 1) 155 156 /* DMA flags */ 157 #define DMA_TX_CH BIT(0) 158 #define DMA_RX_CH BIT(1) 159 #define DEVICE_ALLOC_DESC BIT(2) 160 #define CHAN_IN_USE BIT(3) 161 #define DMA_HW_DESC BIT(4) 162 163 /* Descriptor fields */ 164 #define DESC_DATA_LEN GENMASK(15, 0) 165 #define DESC_BYTE_OFF GENMASK(25, 23) 166 #define DESC_EOP BIT(28) 167 #define DESC_SOP BIT(29) 168 #define DESC_C BIT(30) 169 #define DESC_OWN BIT(31) 170 171 #define DMA_CHAN_RST 1 172 #define DMA_MAX_SIZE (BIT(16) - 1) 173 #define MAX_LOWER_CHANS 32 174 #define MASK_LOWER_CHANS GENMASK(4, 0) 175 #define DMA_OWN 1 176 #define HIGH_4_BITS GENMASK(3, 0) 177 #define DMA_DFT_DESC_NUM 1 178 #define DMA_PKT_DROP_DIS 0 179 180 enum ldma_chan_on_off { 181 DMA_CH_OFF = 0, 182 DMA_CH_ON = 1, 183 }; 184 185 enum { 186 DMA_TYPE_TX = 0, 187 DMA_TYPE_RX, 188 DMA_TYPE_MCPY, 189 }; 190 191 struct ldma_dev; 192 struct ldma_port; 193 194 struct ldma_chan { 195 struct virt_dma_chan vchan; 196 struct ldma_port *port; /* back pointer */ 197 char name[8]; /* Channel name */ 198 int nr; /* Channel id in hardware */ 199 u32 flags; /* central way or channel based way */ 200 enum ldma_chan_on_off onoff; 201 dma_addr_t desc_phys; 202 void *desc_base; /* Virtual address */ 203 u32 desc_cnt; /* Number of descriptors */ 204 int rst; 205 u32 hdrm_len; 206 bool hdrm_csum; 207 u32 boff_len; 208 u32 data_endian; 209 u32 desc_endian; 210 bool pden; 211 bool desc_rx_np; 212 bool data_endian_en; 213 bool desc_endian_en; 214 bool abc_en; 215 bool desc_init; 216 struct dma_pool *desc_pool; /* Descriptors pool */ 217 u32 desc_num; 218 struct dw2_desc_sw *ds; 219 struct work_struct work; 220 struct dma_slave_config config; 221 }; 222 223 struct ldma_port { 224 struct ldma_dev *ldev; /* back pointer */ 225 u32 portid; 226 u32 rxbl; 227 u32 txbl; 228 u32 rxendi; 229 u32 txendi; 230 u32 pkt_drop; 231 }; 232 233 /* Instance specific data */ 234 struct ldma_inst_data { 235 bool desc_in_sram; 236 bool chan_fc; 237 bool desc_fod; /* Fetch On Demand */ 238 bool valid_desc_fetch_ack; 239 u32 orrc; /* Outstanding read count */ 240 const char *name; 241 u32 type; 242 }; 243 244 struct ldma_dev { 245 struct device *dev; 246 void __iomem *base; 247 struct reset_control *rst; 248 struct clk *core_clk; 249 struct dma_device dma_dev; 250 u32 ver; 251 int irq; 252 struct ldma_port *ports; 253 struct ldma_chan *chans; /* channel list on this DMA or port */ 254 spinlock_t dev_lock; /* Controller register exclusive */ 255 u32 chan_nrs; 256 u32 port_nrs; 257 u32 channels_mask; 258 u32 flags; 259 u32 pollcnt; 260 const struct ldma_inst_data *inst; 261 struct workqueue_struct *wq; 262 }; 263 264 struct dw2_desc { 265 u32 field; 266 u32 addr; 267 } __packed __aligned(8); 268 269 struct dw2_desc_sw { 270 struct virt_dma_desc vdesc; 271 struct ldma_chan *chan; 272 dma_addr_t desc_phys; 273 size_t desc_cnt; 274 size_t size; 275 struct dw2_desc *desc_hw; 276 }; 277 278 static inline void 279 ldma_update_bits(struct ldma_dev *d, u32 mask, u32 val, u32 ofs) 280 { 281 u32 old_val, new_val; 282 283 old_val = readl(d->base + ofs); 284 new_val = (old_val & ~mask) | (val & mask); 285 286 if (new_val != old_val) 287 writel(new_val, d->base + ofs); 288 } 289 290 static inline struct ldma_chan *to_ldma_chan(struct dma_chan *chan) 291 { 292 return container_of(chan, struct ldma_chan, vchan.chan); 293 } 294 295 static inline struct ldma_dev *to_ldma_dev(struct dma_device *dma_dev) 296 { 297 return container_of(dma_dev, struct ldma_dev, dma_dev); 298 } 299 300 static inline struct dw2_desc_sw *to_lgm_dma_desc(struct virt_dma_desc *vdesc) 301 { 302 return container_of(vdesc, struct dw2_desc_sw, vdesc); 303 } 304 305 static inline bool ldma_chan_tx(struct ldma_chan *c) 306 { 307 return !!(c->flags & DMA_TX_CH); 308 } 309 310 static inline bool ldma_chan_is_hw_desc(struct ldma_chan *c) 311 { 312 return !!(c->flags & DMA_HW_DESC); 313 } 314 315 static void ldma_dev_reset(struct ldma_dev *d) 316 317 { 318 unsigned long flags; 319 320 spin_lock_irqsave(&d->dev_lock, flags); 321 ldma_update_bits(d, DMA_CTRL_RST, DMA_CTRL_RST, DMA_CTRL); 322 spin_unlock_irqrestore(&d->dev_lock, flags); 323 } 324 325 static void ldma_dev_pkt_arb_cfg(struct ldma_dev *d, bool enable) 326 { 327 unsigned long flags; 328 u32 mask = DMA_CTRL_PKTARB; 329 u32 val = enable ? DMA_CTRL_PKTARB : 0; 330 331 spin_lock_irqsave(&d->dev_lock, flags); 332 ldma_update_bits(d, mask, val, DMA_CTRL); 333 spin_unlock_irqrestore(&d->dev_lock, flags); 334 } 335 336 static void ldma_dev_sram_desc_cfg(struct ldma_dev *d, bool enable) 337 { 338 unsigned long flags; 339 u32 mask = DMA_CTRL_DSRAM_PATH; 340 u32 val = enable ? DMA_CTRL_DSRAM_PATH : 0; 341 342 spin_lock_irqsave(&d->dev_lock, flags); 343 ldma_update_bits(d, mask, val, DMA_CTRL); 344 spin_unlock_irqrestore(&d->dev_lock, flags); 345 } 346 347 static void ldma_dev_chan_flow_ctl_cfg(struct ldma_dev *d, bool enable) 348 { 349 unsigned long flags; 350 u32 mask, val; 351 352 if (d->inst->type != DMA_TYPE_TX) 353 return; 354 355 mask = DMA_CTRL_CH_FL; 356 val = enable ? DMA_CTRL_CH_FL : 0; 357 358 spin_lock_irqsave(&d->dev_lock, flags); 359 ldma_update_bits(d, mask, val, DMA_CTRL); 360 spin_unlock_irqrestore(&d->dev_lock, flags); 361 } 362 363 static void ldma_dev_global_polling_enable(struct ldma_dev *d) 364 { 365 unsigned long flags; 366 u32 mask = DMA_CPOLL_EN | DMA_CPOLL_CNT; 367 u32 val = DMA_CPOLL_EN; 368 369 val |= FIELD_PREP(DMA_CPOLL_CNT, d->pollcnt); 370 371 spin_lock_irqsave(&d->dev_lock, flags); 372 ldma_update_bits(d, mask, val, DMA_CPOLL); 373 spin_unlock_irqrestore(&d->dev_lock, flags); 374 } 375 376 static void ldma_dev_desc_fetch_on_demand_cfg(struct ldma_dev *d, bool enable) 377 { 378 unsigned long flags; 379 u32 mask, val; 380 381 if (d->inst->type == DMA_TYPE_MCPY) 382 return; 383 384 mask = DMA_CTRL_DS_FOD; 385 val = enable ? DMA_CTRL_DS_FOD : 0; 386 387 spin_lock_irqsave(&d->dev_lock, flags); 388 ldma_update_bits(d, mask, val, DMA_CTRL); 389 spin_unlock_irqrestore(&d->dev_lock, flags); 390 } 391 392 static void ldma_dev_byte_enable_cfg(struct ldma_dev *d, bool enable) 393 { 394 unsigned long flags; 395 u32 mask = DMA_CTRL_ENBE; 396 u32 val = enable ? DMA_CTRL_ENBE : 0; 397 398 spin_lock_irqsave(&d->dev_lock, flags); 399 ldma_update_bits(d, mask, val, DMA_CTRL); 400 spin_unlock_irqrestore(&d->dev_lock, flags); 401 } 402 403 static void ldma_dev_orrc_cfg(struct ldma_dev *d) 404 { 405 unsigned long flags; 406 u32 val = 0; 407 u32 mask; 408 409 if (d->inst->type == DMA_TYPE_RX) 410 return; 411 412 mask = DMA_ORRC_EN | DMA_ORRC_ORRCNT; 413 if (d->inst->orrc > 0 && d->inst->orrc <= DMA_ORRC_MAX_CNT) 414 val = DMA_ORRC_EN | FIELD_PREP(DMA_ORRC_ORRCNT, d->inst->orrc); 415 416 spin_lock_irqsave(&d->dev_lock, flags); 417 ldma_update_bits(d, mask, val, DMA_ORRC); 418 spin_unlock_irqrestore(&d->dev_lock, flags); 419 } 420 421 static void ldma_dev_df_tout_cfg(struct ldma_dev *d, bool enable, int tcnt) 422 { 423 u32 mask = DMA_CTRL_DESC_TMOUT_CNT_V31; 424 unsigned long flags; 425 u32 val; 426 427 if (enable) 428 val = DMA_CTRL_DESC_TMOUT_EN_V31 | FIELD_PREP(DMA_CTRL_DESC_TMOUT_CNT_V31, tcnt); 429 else 430 val = 0; 431 432 spin_lock_irqsave(&d->dev_lock, flags); 433 ldma_update_bits(d, mask, val, DMA_CTRL); 434 spin_unlock_irqrestore(&d->dev_lock, flags); 435 } 436 437 static void ldma_dev_dburst_wr_cfg(struct ldma_dev *d, bool enable) 438 { 439 unsigned long flags; 440 u32 mask, val; 441 442 if (d->inst->type != DMA_TYPE_RX && d->inst->type != DMA_TYPE_MCPY) 443 return; 444 445 mask = DMA_CTRL_DBURST_WR; 446 val = enable ? DMA_CTRL_DBURST_WR : 0; 447 448 spin_lock_irqsave(&d->dev_lock, flags); 449 ldma_update_bits(d, mask, val, DMA_CTRL); 450 spin_unlock_irqrestore(&d->dev_lock, flags); 451 } 452 453 static void ldma_dev_vld_fetch_ack_cfg(struct ldma_dev *d, bool enable) 454 { 455 unsigned long flags; 456 u32 mask, val; 457 458 if (d->inst->type != DMA_TYPE_TX) 459 return; 460 461 mask = DMA_CTRL_VLD_DF_ACK; 462 val = enable ? DMA_CTRL_VLD_DF_ACK : 0; 463 464 spin_lock_irqsave(&d->dev_lock, flags); 465 ldma_update_bits(d, mask, val, DMA_CTRL); 466 spin_unlock_irqrestore(&d->dev_lock, flags); 467 } 468 469 static void ldma_dev_drb_cfg(struct ldma_dev *d, int enable) 470 { 471 unsigned long flags; 472 u32 mask = DMA_CTRL_DRB; 473 u32 val = enable ? DMA_CTRL_DRB : 0; 474 475 spin_lock_irqsave(&d->dev_lock, flags); 476 ldma_update_bits(d, mask, val, DMA_CTRL); 477 spin_unlock_irqrestore(&d->dev_lock, flags); 478 } 479 480 static int ldma_dev_cfg(struct ldma_dev *d) 481 { 482 bool enable; 483 484 ldma_dev_pkt_arb_cfg(d, true); 485 ldma_dev_global_polling_enable(d); 486 487 enable = !!(d->flags & DMA_DFT_DRB); 488 ldma_dev_drb_cfg(d, enable); 489 490 enable = !!(d->flags & DMA_EN_BYTE_EN); 491 ldma_dev_byte_enable_cfg(d, enable); 492 493 enable = !!(d->flags & DMA_CHAN_FLOW_CTL); 494 ldma_dev_chan_flow_ctl_cfg(d, enable); 495 496 enable = !!(d->flags & DMA_DESC_FOD); 497 ldma_dev_desc_fetch_on_demand_cfg(d, enable); 498 499 enable = !!(d->flags & DMA_DESC_IN_SRAM); 500 ldma_dev_sram_desc_cfg(d, enable); 501 502 enable = !!(d->flags & DMA_DBURST_WR); 503 ldma_dev_dburst_wr_cfg(d, enable); 504 505 enable = !!(d->flags & DMA_VALID_DESC_FETCH_ACK); 506 ldma_dev_vld_fetch_ack_cfg(d, enable); 507 508 if (d->ver > DMA_VER22) { 509 ldma_dev_orrc_cfg(d); 510 ldma_dev_df_tout_cfg(d, true, DMA_DFT_DESC_TCNT); 511 } 512 513 dev_dbg(d->dev, "%s Controller 0x%08x configuration done\n", 514 d->inst->name, readl(d->base + DMA_CTRL)); 515 516 return 0; 517 } 518 519 static int ldma_chan_cctrl_cfg(struct ldma_chan *c, u32 val) 520 { 521 struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device); 522 u32 class_low, class_high; 523 unsigned long flags; 524 u32 reg; 525 526 spin_lock_irqsave(&d->dev_lock, flags); 527 ldma_update_bits(d, DMA_CS_MASK, c->nr, DMA_CS); 528 reg = readl(d->base + DMA_CCTRL); 529 /* Read from hardware */ 530 if (reg & DMA_CCTRL_DIR_TX) 531 c->flags |= DMA_TX_CH; 532 else 533 c->flags |= DMA_RX_CH; 534 535 /* Keep the class value unchanged */ 536 class_low = FIELD_GET(DMA_CCTRL_CLASS, reg); 537 class_high = FIELD_GET(DMA_CCTRL_CLASSH, reg); 538 val &= ~DMA_CCTRL_CLASS; 539 val |= FIELD_PREP(DMA_CCTRL_CLASS, class_low); 540 val &= ~DMA_CCTRL_CLASSH; 541 val |= FIELD_PREP(DMA_CCTRL_CLASSH, class_high); 542 writel(val, d->base + DMA_CCTRL); 543 spin_unlock_irqrestore(&d->dev_lock, flags); 544 545 return 0; 546 } 547 548 static void ldma_chan_irq_init(struct ldma_chan *c) 549 { 550 struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device); 551 unsigned long flags; 552 u32 enofs, crofs; 553 u32 cn_bit; 554 555 if (c->nr < MAX_LOWER_CHANS) { 556 enofs = DMA_IRNEN; 557 crofs = DMA_IRNCR; 558 } else { 559 enofs = DMA_IRNEN1; 560 crofs = DMA_IRNCR1; 561 } 562 563 cn_bit = BIT(c->nr & MASK_LOWER_CHANS); 564 spin_lock_irqsave(&d->dev_lock, flags); 565 ldma_update_bits(d, DMA_CS_MASK, c->nr, DMA_CS); 566 567 /* Clear all interrupts and disabled it */ 568 writel(0, d->base + DMA_CIE); 569 writel(DMA_CI_ALL, d->base + DMA_CIS); 570 571 ldma_update_bits(d, cn_bit, 0, enofs); 572 writel(cn_bit, d->base + crofs); 573 spin_unlock_irqrestore(&d->dev_lock, flags); 574 } 575 576 static void ldma_chan_set_class(struct ldma_chan *c, u32 val) 577 { 578 struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device); 579 u32 class_val; 580 581 if (d->inst->type == DMA_TYPE_MCPY || val > DMA_MAX_CLASS) 582 return; 583 584 /* 3 bits low */ 585 class_val = FIELD_PREP(DMA_CCTRL_CLASS, val & 0x7); 586 /* 2 bits high */ 587 class_val |= FIELD_PREP(DMA_CCTRL_CLASSH, (val >> 3) & 0x3); 588 589 ldma_update_bits(d, DMA_CS_MASK, c->nr, DMA_CS); 590 ldma_update_bits(d, DMA_CCTRL_CLASS | DMA_CCTRL_CLASSH, class_val, 591 DMA_CCTRL); 592 } 593 594 static int ldma_chan_on(struct ldma_chan *c) 595 { 596 struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device); 597 unsigned long flags; 598 599 /* If descriptors not configured, not allow to turn on channel */ 600 if (WARN_ON(!c->desc_init)) 601 return -EINVAL; 602 603 spin_lock_irqsave(&d->dev_lock, flags); 604 ldma_update_bits(d, DMA_CS_MASK, c->nr, DMA_CS); 605 ldma_update_bits(d, DMA_CCTRL_ON, DMA_CCTRL_ON, DMA_CCTRL); 606 spin_unlock_irqrestore(&d->dev_lock, flags); 607 608 c->onoff = DMA_CH_ON; 609 610 return 0; 611 } 612 613 static int ldma_chan_off(struct ldma_chan *c) 614 { 615 struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device); 616 unsigned long flags; 617 u32 val; 618 int ret; 619 620 spin_lock_irqsave(&d->dev_lock, flags); 621 ldma_update_bits(d, DMA_CS_MASK, c->nr, DMA_CS); 622 ldma_update_bits(d, DMA_CCTRL_ON, 0, DMA_CCTRL); 623 spin_unlock_irqrestore(&d->dev_lock, flags); 624 625 ret = readl_poll_timeout_atomic(d->base + DMA_CCTRL, val, 626 !(val & DMA_CCTRL_ON), 0, 10000); 627 if (ret) 628 return ret; 629 630 c->onoff = DMA_CH_OFF; 631 632 return 0; 633 } 634 635 static void ldma_chan_desc_hw_cfg(struct ldma_chan *c, dma_addr_t desc_base, 636 int desc_num) 637 { 638 struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device); 639 unsigned long flags; 640 641 spin_lock_irqsave(&d->dev_lock, flags); 642 ldma_update_bits(d, DMA_CS_MASK, c->nr, DMA_CS); 643 writel(lower_32_bits(desc_base), d->base + DMA_CDBA); 644 645 /* Higher 4 bits of 36 bit addressing */ 646 if (IS_ENABLED(CONFIG_64BIT)) { 647 u32 hi = upper_32_bits(desc_base) & HIGH_4_BITS; 648 649 ldma_update_bits(d, DMA_CDBA_MSB, 650 FIELD_PREP(DMA_CDBA_MSB, hi), DMA_CCTRL); 651 } 652 writel(desc_num, d->base + DMA_CDLEN); 653 spin_unlock_irqrestore(&d->dev_lock, flags); 654 655 c->desc_init = true; 656 } 657 658 static struct dma_async_tx_descriptor * 659 ldma_chan_desc_cfg(struct dma_chan *chan, dma_addr_t desc_base, int desc_num) 660 { 661 struct ldma_chan *c = to_ldma_chan(chan); 662 struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device); 663 struct dma_async_tx_descriptor *tx; 664 struct dw2_desc_sw *ds; 665 666 if (!desc_num) { 667 dev_err(d->dev, "Channel %d must allocate descriptor first\n", 668 c->nr); 669 return NULL; 670 } 671 672 if (desc_num > DMA_MAX_DESC_NUM) { 673 dev_err(d->dev, "Channel %d descriptor number out of range %d\n", 674 c->nr, desc_num); 675 return NULL; 676 } 677 678 ldma_chan_desc_hw_cfg(c, desc_base, desc_num); 679 680 c->flags |= DMA_HW_DESC; 681 c->desc_cnt = desc_num; 682 c->desc_phys = desc_base; 683 684 ds = kzalloc(sizeof(*ds), GFP_NOWAIT); 685 if (!ds) 686 return NULL; 687 688 tx = &ds->vdesc.tx; 689 dma_async_tx_descriptor_init(tx, chan); 690 691 return tx; 692 } 693 694 static int ldma_chan_reset(struct ldma_chan *c) 695 { 696 struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device); 697 unsigned long flags; 698 u32 val; 699 int ret; 700 701 ret = ldma_chan_off(c); 702 if (ret) 703 return ret; 704 705 spin_lock_irqsave(&d->dev_lock, flags); 706 ldma_update_bits(d, DMA_CS_MASK, c->nr, DMA_CS); 707 ldma_update_bits(d, DMA_CCTRL_RST, DMA_CCTRL_RST, DMA_CCTRL); 708 spin_unlock_irqrestore(&d->dev_lock, flags); 709 710 ret = readl_poll_timeout_atomic(d->base + DMA_CCTRL, val, 711 !(val & DMA_CCTRL_RST), 0, 10000); 712 if (ret) 713 return ret; 714 715 c->rst = 1; 716 c->desc_init = false; 717 718 return 0; 719 } 720 721 static void ldma_chan_byte_offset_cfg(struct ldma_chan *c, u32 boff_len) 722 { 723 struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device); 724 u32 mask = DMA_C_BOFF_EN | DMA_C_BOFF_BOF_LEN; 725 u32 val; 726 727 if (boff_len > 0 && boff_len <= DMA_CHAN_BOFF_MAX) 728 val = FIELD_PREP(DMA_C_BOFF_BOF_LEN, boff_len) | DMA_C_BOFF_EN; 729 else 730 val = 0; 731 732 ldma_update_bits(d, DMA_CS_MASK, c->nr, DMA_CS); 733 ldma_update_bits(d, mask, val, DMA_C_BOFF); 734 } 735 736 static void ldma_chan_data_endian_cfg(struct ldma_chan *c, bool enable, 737 u32 endian_type) 738 { 739 struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device); 740 u32 mask = DMA_C_END_DE_EN | DMA_C_END_DATAENDI; 741 u32 val; 742 743 if (enable) 744 val = DMA_C_END_DE_EN | FIELD_PREP(DMA_C_END_DATAENDI, endian_type); 745 else 746 val = 0; 747 748 ldma_update_bits(d, DMA_CS_MASK, c->nr, DMA_CS); 749 ldma_update_bits(d, mask, val, DMA_C_ENDIAN); 750 } 751 752 static void ldma_chan_desc_endian_cfg(struct ldma_chan *c, bool enable, 753 u32 endian_type) 754 { 755 struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device); 756 u32 mask = DMA_C_END_DES_EN | DMA_C_END_DESENDI; 757 u32 val; 758 759 if (enable) 760 val = DMA_C_END_DES_EN | FIELD_PREP(DMA_C_END_DESENDI, endian_type); 761 else 762 val = 0; 763 764 ldma_update_bits(d, DMA_CS_MASK, c->nr, DMA_CS); 765 ldma_update_bits(d, mask, val, DMA_C_ENDIAN); 766 } 767 768 static void ldma_chan_hdr_mode_cfg(struct ldma_chan *c, u32 hdr_len, bool csum) 769 { 770 struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device); 771 u32 mask, val; 772 773 /* NB, csum disabled, hdr length must be provided */ 774 if (!csum && (!hdr_len || hdr_len > DMA_HDR_LEN_MAX)) 775 return; 776 777 mask = DMA_C_HDRM_HDR_SUM; 778 val = DMA_C_HDRM_HDR_SUM; 779 780 if (!csum && hdr_len) 781 val = hdr_len; 782 783 ldma_update_bits(d, DMA_CS_MASK, c->nr, DMA_CS); 784 ldma_update_bits(d, mask, val, DMA_C_HDRM); 785 } 786 787 static void ldma_chan_rxwr_np_cfg(struct ldma_chan *c, bool enable) 788 { 789 struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device); 790 u32 mask, val; 791 792 /* Only valid for RX channel */ 793 if (ldma_chan_tx(c)) 794 return; 795 796 mask = DMA_CCTRL_WR_NP_EN; 797 val = enable ? DMA_CCTRL_WR_NP_EN : 0; 798 799 ldma_update_bits(d, DMA_CS_MASK, c->nr, DMA_CS); 800 ldma_update_bits(d, mask, val, DMA_CCTRL); 801 } 802 803 static void ldma_chan_abc_cfg(struct ldma_chan *c, bool enable) 804 { 805 struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device); 806 u32 mask, val; 807 808 if (d->ver < DMA_VER32 || ldma_chan_tx(c)) 809 return; 810 811 mask = DMA_CCTRL_CH_ABC; 812 val = enable ? DMA_CCTRL_CH_ABC : 0; 813 814 ldma_update_bits(d, DMA_CS_MASK, c->nr, DMA_CS); 815 ldma_update_bits(d, mask, val, DMA_CCTRL); 816 } 817 818 static int ldma_port_cfg(struct ldma_port *p) 819 { 820 unsigned long flags; 821 struct ldma_dev *d; 822 u32 reg; 823 824 d = p->ldev; 825 reg = FIELD_PREP(DMA_PCTRL_TXENDI, p->txendi); 826 reg |= FIELD_PREP(DMA_PCTRL_RXENDI, p->rxendi); 827 828 if (d->ver == DMA_VER22) { 829 reg |= FIELD_PREP(DMA_PCTRL_TXBL, p->txbl); 830 reg |= FIELD_PREP(DMA_PCTRL_RXBL, p->rxbl); 831 } else { 832 reg |= FIELD_PREP(DMA_PCTRL_PDEN, p->pkt_drop); 833 834 if (p->txbl == DMA_BURSTL_32DW) 835 reg |= DMA_PCTRL_TXBL32; 836 else if (p->txbl == DMA_BURSTL_16DW) 837 reg |= DMA_PCTRL_TXBL16; 838 else 839 reg |= FIELD_PREP(DMA_PCTRL_TXBL, DMA_PCTRL_TXBL_8); 840 841 if (p->rxbl == DMA_BURSTL_32DW) 842 reg |= DMA_PCTRL_RXBL32; 843 else if (p->rxbl == DMA_BURSTL_16DW) 844 reg |= DMA_PCTRL_RXBL16; 845 else 846 reg |= FIELD_PREP(DMA_PCTRL_RXBL, DMA_PCTRL_RXBL_8); 847 } 848 849 spin_lock_irqsave(&d->dev_lock, flags); 850 writel(p->portid, d->base + DMA_PS); 851 writel(reg, d->base + DMA_PCTRL); 852 spin_unlock_irqrestore(&d->dev_lock, flags); 853 854 reg = readl(d->base + DMA_PCTRL); /* read back */ 855 dev_dbg(d->dev, "Port Control 0x%08x configuration done\n", reg); 856 857 return 0; 858 } 859 860 static int ldma_chan_cfg(struct ldma_chan *c) 861 { 862 struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device); 863 unsigned long flags; 864 u32 reg; 865 866 reg = c->pden ? DMA_CCTRL_PDEN : 0; 867 reg |= c->onoff ? DMA_CCTRL_ON : 0; 868 reg |= c->rst ? DMA_CCTRL_RST : 0; 869 870 ldma_chan_cctrl_cfg(c, reg); 871 ldma_chan_irq_init(c); 872 873 if (d->ver <= DMA_VER22) 874 return 0; 875 876 spin_lock_irqsave(&d->dev_lock, flags); 877 ldma_chan_set_class(c, c->nr); 878 ldma_chan_byte_offset_cfg(c, c->boff_len); 879 ldma_chan_data_endian_cfg(c, c->data_endian_en, c->data_endian); 880 ldma_chan_desc_endian_cfg(c, c->desc_endian_en, c->desc_endian); 881 ldma_chan_hdr_mode_cfg(c, c->hdrm_len, c->hdrm_csum); 882 ldma_chan_rxwr_np_cfg(c, c->desc_rx_np); 883 ldma_chan_abc_cfg(c, c->abc_en); 884 spin_unlock_irqrestore(&d->dev_lock, flags); 885 886 if (ldma_chan_is_hw_desc(c)) 887 ldma_chan_desc_hw_cfg(c, c->desc_phys, c->desc_cnt); 888 889 return 0; 890 } 891 892 static void ldma_dev_init(struct ldma_dev *d) 893 { 894 unsigned long ch_mask = (unsigned long)d->channels_mask; 895 struct ldma_port *p; 896 struct ldma_chan *c; 897 int i; 898 u32 j; 899 900 spin_lock_init(&d->dev_lock); 901 ldma_dev_reset(d); 902 ldma_dev_cfg(d); 903 904 /* DMA port initialization */ 905 for (i = 0; i < d->port_nrs; i++) { 906 p = &d->ports[i]; 907 ldma_port_cfg(p); 908 } 909 910 /* DMA channel initialization */ 911 for_each_set_bit(j, &ch_mask, d->chan_nrs) { 912 c = &d->chans[j]; 913 ldma_chan_cfg(c); 914 } 915 } 916 917 static int ldma_parse_dt(struct ldma_dev *d) 918 { 919 struct fwnode_handle *fwnode = dev_fwnode(d->dev); 920 struct ldma_port *p; 921 int i; 922 923 if (fwnode_property_read_bool(fwnode, "intel,dma-byte-en")) 924 d->flags |= DMA_EN_BYTE_EN; 925 926 if (fwnode_property_read_bool(fwnode, "intel,dma-dburst-wr")) 927 d->flags |= DMA_DBURST_WR; 928 929 if (fwnode_property_read_bool(fwnode, "intel,dma-drb")) 930 d->flags |= DMA_DFT_DRB; 931 932 if (fwnode_property_read_u32(fwnode, "intel,dma-poll-cnt", 933 &d->pollcnt)) 934 d->pollcnt = DMA_DFT_POLL_CNT; 935 936 if (d->inst->chan_fc) 937 d->flags |= DMA_CHAN_FLOW_CTL; 938 939 if (d->inst->desc_fod) 940 d->flags |= DMA_DESC_FOD; 941 942 if (d->inst->desc_in_sram) 943 d->flags |= DMA_DESC_IN_SRAM; 944 945 if (d->inst->valid_desc_fetch_ack) 946 d->flags |= DMA_VALID_DESC_FETCH_ACK; 947 948 if (d->ver > DMA_VER22) { 949 if (!d->port_nrs) 950 return -EINVAL; 951 952 for (i = 0; i < d->port_nrs; i++) { 953 p = &d->ports[i]; 954 p->rxendi = DMA_DFT_ENDIAN; 955 p->txendi = DMA_DFT_ENDIAN; 956 p->rxbl = DMA_DFT_BURST; 957 p->txbl = DMA_DFT_BURST; 958 p->pkt_drop = DMA_PKT_DROP_DIS; 959 } 960 } 961 962 return 0; 963 } 964 965 static void dma_free_desc_resource(struct virt_dma_desc *vdesc) 966 { 967 struct dw2_desc_sw *ds = to_lgm_dma_desc(vdesc); 968 struct ldma_chan *c = ds->chan; 969 970 dma_pool_free(c->desc_pool, ds->desc_hw, ds->desc_phys); 971 kfree(ds); 972 } 973 974 static struct dw2_desc_sw * 975 dma_alloc_desc_resource(int num, struct ldma_chan *c) 976 { 977 struct device *dev = c->vchan.chan.device->dev; 978 struct dw2_desc_sw *ds; 979 980 if (num > c->desc_num) { 981 dev_err(dev, "sg num %d exceed max %d\n", num, c->desc_num); 982 return NULL; 983 } 984 985 ds = kzalloc(sizeof(*ds), GFP_NOWAIT); 986 if (!ds) 987 return NULL; 988 989 ds->chan = c; 990 ds->desc_hw = dma_pool_zalloc(c->desc_pool, GFP_ATOMIC, 991 &ds->desc_phys); 992 if (!ds->desc_hw) { 993 dev_dbg(dev, "out of memory for link descriptor\n"); 994 kfree(ds); 995 return NULL; 996 } 997 ds->desc_cnt = num; 998 999 return ds; 1000 } 1001 1002 static void ldma_chan_irq_en(struct ldma_chan *c) 1003 { 1004 struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device); 1005 unsigned long flags; 1006 1007 spin_lock_irqsave(&d->dev_lock, flags); 1008 writel(c->nr, d->base + DMA_CS); 1009 writel(DMA_CI_EOP, d->base + DMA_CIE); 1010 writel(BIT(c->nr), d->base + DMA_IRNEN); 1011 spin_unlock_irqrestore(&d->dev_lock, flags); 1012 } 1013 1014 static void ldma_issue_pending(struct dma_chan *chan) 1015 { 1016 struct ldma_chan *c = to_ldma_chan(chan); 1017 struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device); 1018 unsigned long flags; 1019 1020 if (d->ver == DMA_VER22) { 1021 spin_lock_irqsave(&c->vchan.lock, flags); 1022 if (vchan_issue_pending(&c->vchan)) { 1023 struct virt_dma_desc *vdesc; 1024 1025 /* Get the next descriptor */ 1026 vdesc = vchan_next_desc(&c->vchan); 1027 if (!vdesc) { 1028 c->ds = NULL; 1029 spin_unlock_irqrestore(&c->vchan.lock, flags); 1030 return; 1031 } 1032 list_del(&vdesc->node); 1033 c->ds = to_lgm_dma_desc(vdesc); 1034 ldma_chan_desc_hw_cfg(c, c->ds->desc_phys, c->ds->desc_cnt); 1035 ldma_chan_irq_en(c); 1036 } 1037 spin_unlock_irqrestore(&c->vchan.lock, flags); 1038 } 1039 ldma_chan_on(c); 1040 } 1041 1042 static void ldma_synchronize(struct dma_chan *chan) 1043 { 1044 struct ldma_chan *c = to_ldma_chan(chan); 1045 1046 /* 1047 * clear any pending work if any. In that 1048 * case the resource needs to be free here. 1049 */ 1050 cancel_work_sync(&c->work); 1051 vchan_synchronize(&c->vchan); 1052 if (c->ds) 1053 dma_free_desc_resource(&c->ds->vdesc); 1054 } 1055 1056 static int ldma_terminate_all(struct dma_chan *chan) 1057 { 1058 struct ldma_chan *c = to_ldma_chan(chan); 1059 unsigned long flags; 1060 LIST_HEAD(head); 1061 1062 spin_lock_irqsave(&c->vchan.lock, flags); 1063 vchan_get_all_descriptors(&c->vchan, &head); 1064 spin_unlock_irqrestore(&c->vchan.lock, flags); 1065 vchan_dma_desc_free_list(&c->vchan, &head); 1066 1067 return ldma_chan_reset(c); 1068 } 1069 1070 static int ldma_resume_chan(struct dma_chan *chan) 1071 { 1072 struct ldma_chan *c = to_ldma_chan(chan); 1073 1074 ldma_chan_on(c); 1075 1076 return 0; 1077 } 1078 1079 static int ldma_pause_chan(struct dma_chan *chan) 1080 { 1081 struct ldma_chan *c = to_ldma_chan(chan); 1082 1083 return ldma_chan_off(c); 1084 } 1085 1086 static enum dma_status 1087 ldma_tx_status(struct dma_chan *chan, dma_cookie_t cookie, 1088 struct dma_tx_state *txstate) 1089 { 1090 struct ldma_chan *c = to_ldma_chan(chan); 1091 struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device); 1092 enum dma_status status = DMA_COMPLETE; 1093 1094 if (d->ver == DMA_VER22) 1095 status = dma_cookie_status(chan, cookie, txstate); 1096 1097 return status; 1098 } 1099 1100 static void dma_chan_irq(int irq, void *data) 1101 { 1102 struct ldma_chan *c = data; 1103 struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device); 1104 u32 stat; 1105 1106 /* Disable channel interrupts */ 1107 writel(c->nr, d->base + DMA_CS); 1108 stat = readl(d->base + DMA_CIS); 1109 if (!stat) 1110 return; 1111 1112 writel(readl(d->base + DMA_CIE) & ~DMA_CI_ALL, d->base + DMA_CIE); 1113 writel(stat, d->base + DMA_CIS); 1114 queue_work(d->wq, &c->work); 1115 } 1116 1117 static irqreturn_t dma_interrupt(int irq, void *dev_id) 1118 { 1119 struct ldma_dev *d = dev_id; 1120 struct ldma_chan *c; 1121 unsigned long irncr; 1122 u32 cid; 1123 1124 irncr = readl(d->base + DMA_IRNCR); 1125 if (!irncr) { 1126 dev_err(d->dev, "dummy interrupt\n"); 1127 return IRQ_NONE; 1128 } 1129 1130 for_each_set_bit(cid, &irncr, d->chan_nrs) { 1131 /* Mask */ 1132 writel(readl(d->base + DMA_IRNEN) & ~BIT(cid), d->base + DMA_IRNEN); 1133 /* Ack */ 1134 writel(readl(d->base + DMA_IRNCR) | BIT(cid), d->base + DMA_IRNCR); 1135 1136 c = &d->chans[cid]; 1137 dma_chan_irq(irq, c); 1138 } 1139 1140 return IRQ_HANDLED; 1141 } 1142 1143 static void prep_slave_burst_len(struct ldma_chan *c) 1144 { 1145 struct ldma_port *p = c->port; 1146 struct dma_slave_config *cfg = &c->config; 1147 1148 if (cfg->dst_maxburst) 1149 cfg->src_maxburst = cfg->dst_maxburst; 1150 1151 /* TX and RX has the same burst length */ 1152 p->txbl = ilog2(cfg->src_maxburst); 1153 p->rxbl = p->txbl; 1154 } 1155 1156 static struct dma_async_tx_descriptor * 1157 ldma_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, 1158 unsigned int sglen, enum dma_transfer_direction dir, 1159 unsigned long flags, void *context) 1160 { 1161 struct ldma_chan *c = to_ldma_chan(chan); 1162 struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device); 1163 size_t len, avail, total = 0; 1164 struct dw2_desc *hw_ds; 1165 struct dw2_desc_sw *ds; 1166 struct scatterlist *sg; 1167 int num = sglen, i; 1168 dma_addr_t addr; 1169 1170 if (!sgl) 1171 return NULL; 1172 1173 if (d->ver > DMA_VER22) 1174 return ldma_chan_desc_cfg(chan, sgl->dma_address, sglen); 1175 1176 for_each_sg(sgl, sg, sglen, i) { 1177 avail = sg_dma_len(sg); 1178 if (avail > DMA_MAX_SIZE) 1179 num += DIV_ROUND_UP(avail, DMA_MAX_SIZE) - 1; 1180 } 1181 1182 ds = dma_alloc_desc_resource(num, c); 1183 if (!ds) 1184 return NULL; 1185 1186 c->ds = ds; 1187 1188 num = 0; 1189 /* sop and eop has to be handled nicely */ 1190 for_each_sg(sgl, sg, sglen, i) { 1191 addr = sg_dma_address(sg); 1192 avail = sg_dma_len(sg); 1193 total += avail; 1194 1195 do { 1196 len = min_t(size_t, avail, DMA_MAX_SIZE); 1197 1198 hw_ds = &ds->desc_hw[num]; 1199 switch (sglen) { 1200 case 1: 1201 hw_ds->field &= ~DESC_SOP; 1202 hw_ds->field |= FIELD_PREP(DESC_SOP, 1); 1203 1204 hw_ds->field &= ~DESC_EOP; 1205 hw_ds->field |= FIELD_PREP(DESC_EOP, 1); 1206 break; 1207 default: 1208 if (num == 0) { 1209 hw_ds->field &= ~DESC_SOP; 1210 hw_ds->field |= FIELD_PREP(DESC_SOP, 1); 1211 1212 hw_ds->field &= ~DESC_EOP; 1213 hw_ds->field |= FIELD_PREP(DESC_EOP, 0); 1214 } else if (num == (sglen - 1)) { 1215 hw_ds->field &= ~DESC_SOP; 1216 hw_ds->field |= FIELD_PREP(DESC_SOP, 0); 1217 hw_ds->field &= ~DESC_EOP; 1218 hw_ds->field |= FIELD_PREP(DESC_EOP, 1); 1219 } else { 1220 hw_ds->field &= ~DESC_SOP; 1221 hw_ds->field |= FIELD_PREP(DESC_SOP, 0); 1222 1223 hw_ds->field &= ~DESC_EOP; 1224 hw_ds->field |= FIELD_PREP(DESC_EOP, 0); 1225 } 1226 break; 1227 } 1228 /* Only 32 bit address supported */ 1229 hw_ds->addr = (u32)addr; 1230 1231 hw_ds->field &= ~DESC_DATA_LEN; 1232 hw_ds->field |= FIELD_PREP(DESC_DATA_LEN, len); 1233 1234 hw_ds->field &= ~DESC_C; 1235 hw_ds->field |= FIELD_PREP(DESC_C, 0); 1236 1237 hw_ds->field &= ~DESC_BYTE_OFF; 1238 hw_ds->field |= FIELD_PREP(DESC_BYTE_OFF, addr & 0x3); 1239 1240 /* Ensure data ready before ownership change */ 1241 wmb(); 1242 hw_ds->field &= ~DESC_OWN; 1243 hw_ds->field |= FIELD_PREP(DESC_OWN, DMA_OWN); 1244 1245 /* Ensure ownership changed before moving forward */ 1246 wmb(); 1247 num++; 1248 addr += len; 1249 avail -= len; 1250 } while (avail); 1251 } 1252 1253 ds->size = total; 1254 prep_slave_burst_len(c); 1255 1256 return vchan_tx_prep(&c->vchan, &ds->vdesc, DMA_CTRL_ACK); 1257 } 1258 1259 static int 1260 ldma_slave_config(struct dma_chan *chan, struct dma_slave_config *cfg) 1261 { 1262 struct ldma_chan *c = to_ldma_chan(chan); 1263 1264 memcpy(&c->config, cfg, sizeof(c->config)); 1265 1266 return 0; 1267 } 1268 1269 static int ldma_alloc_chan_resources(struct dma_chan *chan) 1270 { 1271 struct ldma_chan *c = to_ldma_chan(chan); 1272 struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device); 1273 struct device *dev = c->vchan.chan.device->dev; 1274 size_t desc_sz; 1275 1276 if (d->ver > DMA_VER22) { 1277 c->flags |= CHAN_IN_USE; 1278 return 0; 1279 } 1280 1281 if (c->desc_pool) 1282 return c->desc_num; 1283 1284 desc_sz = c->desc_num * sizeof(struct dw2_desc); 1285 c->desc_pool = dma_pool_create(c->name, dev, desc_sz, 1286 __alignof__(struct dw2_desc), 0); 1287 1288 if (!c->desc_pool) { 1289 dev_err(dev, "unable to allocate descriptor pool\n"); 1290 return -ENOMEM; 1291 } 1292 1293 return c->desc_num; 1294 } 1295 1296 static void ldma_free_chan_resources(struct dma_chan *chan) 1297 { 1298 struct ldma_chan *c = to_ldma_chan(chan); 1299 struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device); 1300 1301 if (d->ver == DMA_VER22) { 1302 dma_pool_destroy(c->desc_pool); 1303 c->desc_pool = NULL; 1304 vchan_free_chan_resources(to_virt_chan(chan)); 1305 ldma_chan_reset(c); 1306 } else { 1307 c->flags &= ~CHAN_IN_USE; 1308 } 1309 } 1310 1311 static void dma_work(struct work_struct *work) 1312 { 1313 struct ldma_chan *c = container_of(work, struct ldma_chan, work); 1314 struct dma_async_tx_descriptor *tx = &c->ds->vdesc.tx; 1315 struct virt_dma_chan *vc = &c->vchan; 1316 struct dmaengine_desc_callback cb; 1317 struct virt_dma_desc *vd, *_vd; 1318 unsigned long flags; 1319 LIST_HEAD(head); 1320 1321 spin_lock_irqsave(&c->vchan.lock, flags); 1322 list_splice_tail_init(&vc->desc_completed, &head); 1323 spin_unlock_irqrestore(&c->vchan.lock, flags); 1324 dmaengine_desc_get_callback(tx, &cb); 1325 dma_cookie_complete(tx); 1326 dmaengine_desc_callback_invoke(&cb, NULL); 1327 1328 list_for_each_entry_safe(vd, _vd, &head, node) { 1329 dmaengine_desc_get_callback(tx, &cb); 1330 dma_cookie_complete(tx); 1331 list_del(&vd->node); 1332 dmaengine_desc_callback_invoke(&cb, NULL); 1333 1334 vchan_vdesc_fini(vd); 1335 } 1336 c->ds = NULL; 1337 } 1338 1339 static void 1340 update_burst_len_v22(struct ldma_chan *c, struct ldma_port *p, u32 burst) 1341 { 1342 if (ldma_chan_tx(c)) 1343 p->txbl = ilog2(burst); 1344 else 1345 p->rxbl = ilog2(burst); 1346 } 1347 1348 static void 1349 update_burst_len_v3X(struct ldma_chan *c, struct ldma_port *p, u32 burst) 1350 { 1351 if (ldma_chan_tx(c)) 1352 p->txbl = burst; 1353 else 1354 p->rxbl = burst; 1355 } 1356 1357 static int 1358 update_client_configs(struct of_dma *ofdma, struct of_phandle_args *spec) 1359 { 1360 struct ldma_dev *d = ofdma->of_dma_data; 1361 u32 chan_id = spec->args[0]; 1362 u32 port_id = spec->args[1]; 1363 u32 burst = spec->args[2]; 1364 struct ldma_port *p; 1365 struct ldma_chan *c; 1366 1367 if (chan_id >= d->chan_nrs || port_id >= d->port_nrs) 1368 return 0; 1369 1370 p = &d->ports[port_id]; 1371 c = &d->chans[chan_id]; 1372 c->port = p; 1373 1374 if (d->ver == DMA_VER22) 1375 update_burst_len_v22(c, p, burst); 1376 else 1377 update_burst_len_v3X(c, p, burst); 1378 1379 ldma_port_cfg(p); 1380 1381 return 1; 1382 } 1383 1384 static struct dma_chan *ldma_xlate(struct of_phandle_args *spec, 1385 struct of_dma *ofdma) 1386 { 1387 struct ldma_dev *d = ofdma->of_dma_data; 1388 u32 chan_id = spec->args[0]; 1389 int ret; 1390 1391 if (!spec->args_count) 1392 return NULL; 1393 1394 /* if args_count is 1 driver use default settings */ 1395 if (spec->args_count > 1) { 1396 ret = update_client_configs(ofdma, spec); 1397 if (!ret) 1398 return NULL; 1399 } 1400 1401 return dma_get_slave_channel(&d->chans[chan_id].vchan.chan); 1402 } 1403 1404 static void ldma_dma_init_v22(int i, struct ldma_dev *d) 1405 { 1406 struct ldma_chan *c; 1407 1408 c = &d->chans[i]; 1409 c->nr = i; /* Real channel number */ 1410 c->rst = DMA_CHAN_RST; 1411 c->desc_num = DMA_DFT_DESC_NUM; 1412 snprintf(c->name, sizeof(c->name), "chan%d", c->nr); 1413 INIT_WORK(&c->work, dma_work); 1414 c->vchan.desc_free = dma_free_desc_resource; 1415 vchan_init(&c->vchan, &d->dma_dev); 1416 } 1417 1418 static void ldma_dma_init_v3X(int i, struct ldma_dev *d) 1419 { 1420 struct ldma_chan *c; 1421 1422 c = &d->chans[i]; 1423 c->data_endian = DMA_DFT_ENDIAN; 1424 c->desc_endian = DMA_DFT_ENDIAN; 1425 c->data_endian_en = false; 1426 c->desc_endian_en = false; 1427 c->desc_rx_np = false; 1428 c->flags |= DEVICE_ALLOC_DESC; 1429 c->onoff = DMA_CH_OFF; 1430 c->rst = DMA_CHAN_RST; 1431 c->abc_en = true; 1432 c->hdrm_csum = false; 1433 c->boff_len = 0; 1434 c->nr = i; 1435 c->vchan.desc_free = dma_free_desc_resource; 1436 vchan_init(&c->vchan, &d->dma_dev); 1437 } 1438 1439 static int ldma_init_v22(struct ldma_dev *d, struct platform_device *pdev) 1440 { 1441 int ret; 1442 1443 ret = device_property_read_u32(d->dev, "dma-channels", &d->chan_nrs); 1444 if (ret < 0) { 1445 dev_err(d->dev, "unable to read dma-channels property\n"); 1446 return ret; 1447 } 1448 1449 d->irq = platform_get_irq(pdev, 0); 1450 if (d->irq < 0) 1451 return d->irq; 1452 1453 ret = devm_request_irq(&pdev->dev, d->irq, dma_interrupt, 0, 1454 DRIVER_NAME, d); 1455 if (ret) 1456 return ret; 1457 1458 d->wq = alloc_ordered_workqueue("dma_wq", WQ_MEM_RECLAIM | 1459 WQ_HIGHPRI); 1460 if (!d->wq) 1461 return -ENOMEM; 1462 1463 return 0; 1464 } 1465 1466 static void ldma_clk_disable(void *data) 1467 { 1468 struct ldma_dev *d = data; 1469 1470 clk_disable_unprepare(d->core_clk); 1471 reset_control_assert(d->rst); 1472 } 1473 1474 static const struct ldma_inst_data dma0 = { 1475 .name = "dma0", 1476 .chan_fc = false, 1477 .desc_fod = false, 1478 .desc_in_sram = false, 1479 .valid_desc_fetch_ack = false, 1480 }; 1481 1482 static const struct ldma_inst_data dma2tx = { 1483 .name = "dma2tx", 1484 .type = DMA_TYPE_TX, 1485 .orrc = 16, 1486 .chan_fc = true, 1487 .desc_fod = true, 1488 .desc_in_sram = true, 1489 .valid_desc_fetch_ack = true, 1490 }; 1491 1492 static const struct ldma_inst_data dma1rx = { 1493 .name = "dma1rx", 1494 .type = DMA_TYPE_RX, 1495 .orrc = 16, 1496 .chan_fc = false, 1497 .desc_fod = true, 1498 .desc_in_sram = true, 1499 .valid_desc_fetch_ack = false, 1500 }; 1501 1502 static const struct ldma_inst_data dma1tx = { 1503 .name = "dma1tx", 1504 .type = DMA_TYPE_TX, 1505 .orrc = 16, 1506 .chan_fc = true, 1507 .desc_fod = true, 1508 .desc_in_sram = true, 1509 .valid_desc_fetch_ack = true, 1510 }; 1511 1512 static const struct ldma_inst_data dma0tx = { 1513 .name = "dma0tx", 1514 .type = DMA_TYPE_TX, 1515 .orrc = 16, 1516 .chan_fc = true, 1517 .desc_fod = true, 1518 .desc_in_sram = true, 1519 .valid_desc_fetch_ack = true, 1520 }; 1521 1522 static const struct ldma_inst_data dma3 = { 1523 .name = "dma3", 1524 .type = DMA_TYPE_MCPY, 1525 .orrc = 16, 1526 .chan_fc = false, 1527 .desc_fod = false, 1528 .desc_in_sram = true, 1529 .valid_desc_fetch_ack = false, 1530 }; 1531 1532 static const struct ldma_inst_data toe_dma30 = { 1533 .name = "toe_dma30", 1534 .type = DMA_TYPE_MCPY, 1535 .orrc = 16, 1536 .chan_fc = false, 1537 .desc_fod = false, 1538 .desc_in_sram = true, 1539 .valid_desc_fetch_ack = true, 1540 }; 1541 1542 static const struct ldma_inst_data toe_dma31 = { 1543 .name = "toe_dma31", 1544 .type = DMA_TYPE_MCPY, 1545 .orrc = 16, 1546 .chan_fc = false, 1547 .desc_fod = false, 1548 .desc_in_sram = true, 1549 .valid_desc_fetch_ack = true, 1550 }; 1551 1552 static const struct of_device_id intel_ldma_match[] = { 1553 { .compatible = "intel,lgm-cdma", .data = &dma0}, 1554 { .compatible = "intel,lgm-dma2tx", .data = &dma2tx}, 1555 { .compatible = "intel,lgm-dma1rx", .data = &dma1rx}, 1556 { .compatible = "intel,lgm-dma1tx", .data = &dma1tx}, 1557 { .compatible = "intel,lgm-dma0tx", .data = &dma0tx}, 1558 { .compatible = "intel,lgm-dma3", .data = &dma3}, 1559 { .compatible = "intel,lgm-toe-dma30", .data = &toe_dma30}, 1560 { .compatible = "intel,lgm-toe-dma31", .data = &toe_dma31}, 1561 {} 1562 }; 1563 1564 static int intel_ldma_probe(struct platform_device *pdev) 1565 { 1566 struct device *dev = &pdev->dev; 1567 struct dma_device *dma_dev; 1568 unsigned long ch_mask; 1569 struct ldma_chan *c; 1570 struct ldma_port *p; 1571 struct ldma_dev *d; 1572 u32 id, bitn = 32, j; 1573 int i, ret; 1574 1575 d = devm_kzalloc(dev, sizeof(*d), GFP_KERNEL); 1576 if (!d) 1577 return -ENOMEM; 1578 1579 /* Link controller to platform device */ 1580 d->dev = &pdev->dev; 1581 1582 d->inst = device_get_match_data(dev); 1583 if (!d->inst) { 1584 dev_err(dev, "No device match found\n"); 1585 return -ENODEV; 1586 } 1587 1588 d->base = devm_platform_ioremap_resource(pdev, 0); 1589 if (IS_ERR(d->base)) 1590 return PTR_ERR(d->base); 1591 1592 /* Power up and reset the dma engine, some DMAs always on?? */ 1593 d->core_clk = devm_clk_get_optional(dev, NULL); 1594 if (IS_ERR(d->core_clk)) 1595 return PTR_ERR(d->core_clk); 1596 1597 d->rst = devm_reset_control_get_optional(dev, NULL); 1598 if (IS_ERR(d->rst)) 1599 return PTR_ERR(d->rst); 1600 1601 clk_prepare_enable(d->core_clk); 1602 reset_control_deassert(d->rst); 1603 1604 ret = devm_add_action_or_reset(dev, ldma_clk_disable, d); 1605 if (ret) { 1606 dev_err(dev, "Failed to devm_add_action_or_reset, %d\n", ret); 1607 return ret; 1608 } 1609 1610 id = readl(d->base + DMA_ID); 1611 d->chan_nrs = FIELD_GET(DMA_ID_CHNR, id); 1612 d->port_nrs = FIELD_GET(DMA_ID_PNR, id); 1613 d->ver = FIELD_GET(DMA_ID_REV, id); 1614 1615 if (id & DMA_ID_AW_36B) 1616 d->flags |= DMA_ADDR_36BIT; 1617 1618 if (IS_ENABLED(CONFIG_64BIT) && (id & DMA_ID_AW_36B)) 1619 bitn = 36; 1620 1621 if (id & DMA_ID_DW_128B) 1622 d->flags |= DMA_DATA_128BIT; 1623 1624 ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(bitn)); 1625 if (ret) { 1626 dev_err(dev, "No usable DMA configuration\n"); 1627 return ret; 1628 } 1629 1630 if (d->ver == DMA_VER22) { 1631 ret = ldma_init_v22(d, pdev); 1632 if (ret) 1633 return ret; 1634 } 1635 1636 ret = device_property_read_u32(dev, "dma-channel-mask", &d->channels_mask); 1637 if (ret < 0) 1638 d->channels_mask = GENMASK(d->chan_nrs - 1, 0); 1639 1640 dma_dev = &d->dma_dev; 1641 1642 dma_cap_zero(dma_dev->cap_mask); 1643 dma_cap_set(DMA_SLAVE, dma_dev->cap_mask); 1644 1645 /* Channel initializations */ 1646 INIT_LIST_HEAD(&dma_dev->channels); 1647 1648 /* Port Initializations */ 1649 d->ports = devm_kcalloc(dev, d->port_nrs, sizeof(*p), GFP_KERNEL); 1650 if (!d->ports) 1651 return -ENOMEM; 1652 1653 /* Channels Initializations */ 1654 d->chans = devm_kcalloc(d->dev, d->chan_nrs, sizeof(*c), GFP_KERNEL); 1655 if (!d->chans) 1656 return -ENOMEM; 1657 1658 for (i = 0; i < d->port_nrs; i++) { 1659 p = &d->ports[i]; 1660 p->portid = i; 1661 p->ldev = d; 1662 } 1663 1664 dma_dev->dev = &pdev->dev; 1665 1666 ch_mask = (unsigned long)d->channels_mask; 1667 for_each_set_bit(j, &ch_mask, d->chan_nrs) { 1668 if (d->ver == DMA_VER22) 1669 ldma_dma_init_v22(j, d); 1670 else 1671 ldma_dma_init_v3X(j, d); 1672 } 1673 1674 ret = ldma_parse_dt(d); 1675 if (ret) 1676 return ret; 1677 1678 dma_dev->device_alloc_chan_resources = ldma_alloc_chan_resources; 1679 dma_dev->device_free_chan_resources = ldma_free_chan_resources; 1680 dma_dev->device_terminate_all = ldma_terminate_all; 1681 dma_dev->device_issue_pending = ldma_issue_pending; 1682 dma_dev->device_tx_status = ldma_tx_status; 1683 dma_dev->device_resume = ldma_resume_chan; 1684 dma_dev->device_pause = ldma_pause_chan; 1685 dma_dev->device_prep_slave_sg = ldma_prep_slave_sg; 1686 1687 if (d->ver == DMA_VER22) { 1688 dma_dev->device_config = ldma_slave_config; 1689 dma_dev->device_synchronize = ldma_synchronize; 1690 dma_dev->src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES); 1691 dma_dev->dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES); 1692 dma_dev->directions = BIT(DMA_MEM_TO_DEV) | 1693 BIT(DMA_DEV_TO_MEM); 1694 dma_dev->residue_granularity = 1695 DMA_RESIDUE_GRANULARITY_DESCRIPTOR; 1696 } 1697 1698 platform_set_drvdata(pdev, d); 1699 1700 ldma_dev_init(d); 1701 1702 ret = dma_async_device_register(dma_dev); 1703 if (ret) { 1704 dev_err(dev, "Failed to register slave DMA engine device\n"); 1705 return ret; 1706 } 1707 1708 ret = of_dma_controller_register(pdev->dev.of_node, ldma_xlate, d); 1709 if (ret) { 1710 dev_err(dev, "Failed to register of DMA controller\n"); 1711 dma_async_device_unregister(dma_dev); 1712 return ret; 1713 } 1714 1715 dev_info(dev, "Init done - rev: %x, ports: %d channels: %d\n", d->ver, 1716 d->port_nrs, d->chan_nrs); 1717 1718 return 0; 1719 } 1720 1721 static struct platform_driver intel_ldma_driver = { 1722 .probe = intel_ldma_probe, 1723 .driver = { 1724 .name = DRIVER_NAME, 1725 .of_match_table = intel_ldma_match, 1726 }, 1727 }; 1728 1729 /* 1730 * Perform this driver as device_initcall to make sure initialization happens 1731 * before its DMA clients of some are platform specific and also to provide 1732 * registered DMA channels and DMA capabilities to clients before their 1733 * initialization. 1734 */ 1735 builtin_platform_driver(intel_ldma_driver); 1736