1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (c) 2018-2019 Synopsys, Inc. and/or its affiliates. 4 * Synopsys DesignWare eDMA v0 core 5 * 6 * Author: Gustavo Pimentel <gustavo.pimentel@synopsys.com> 7 */ 8 9 #include <linux/bitfield.h> 10 11 #include <linux/io-64-nonatomic-lo-hi.h> 12 13 #include "dw-edma-core.h" 14 #include "dw-edma-v0-core.h" 15 #include "dw-edma-v0-regs.h" 16 #include "dw-edma-v0-debugfs.h" 17 18 enum dw_edma_control { 19 DW_EDMA_V0_CB = BIT(0), 20 DW_EDMA_V0_TCB = BIT(1), 21 DW_EDMA_V0_LLP = BIT(2), 22 DW_EDMA_V0_LIE = BIT(3), 23 DW_EDMA_V0_RIE = BIT(4), 24 DW_EDMA_V0_CCS = BIT(8), 25 DW_EDMA_V0_LLE = BIT(9), 26 }; 27 28 static inline struct dw_edma_v0_regs __iomem *__dw_regs(struct dw_edma *dw) 29 { 30 return dw->chip->reg_base; 31 } 32 33 #define SET_32(dw, name, value) \ 34 writel(value, &(__dw_regs(dw)->name)) 35 36 #define GET_32(dw, name) \ 37 readl(&(__dw_regs(dw)->name)) 38 39 #define SET_RW_32(dw, dir, name, value) \ 40 do { \ 41 if ((dir) == EDMA_DIR_WRITE) \ 42 SET_32(dw, wr_##name, value); \ 43 else \ 44 SET_32(dw, rd_##name, value); \ 45 } while (0) 46 47 #define GET_RW_32(dw, dir, name) \ 48 ((dir) == EDMA_DIR_WRITE \ 49 ? GET_32(dw, wr_##name) \ 50 : GET_32(dw, rd_##name)) 51 52 #define SET_BOTH_32(dw, name, value) \ 53 do { \ 54 SET_32(dw, wr_##name, value); \ 55 SET_32(dw, rd_##name, value); \ 56 } while (0) 57 58 #define SET_64(dw, name, value) \ 59 writeq(value, &(__dw_regs(dw)->name)) 60 61 #define GET_64(dw, name) \ 62 readq(&(__dw_regs(dw)->name)) 63 64 #define SET_RW_64(dw, dir, name, value) \ 65 do { \ 66 if ((dir) == EDMA_DIR_WRITE) \ 67 SET_64(dw, wr_##name, value); \ 68 else \ 69 SET_64(dw, rd_##name, value); \ 70 } while (0) 71 72 #define GET_RW_64(dw, dir, name) \ 73 ((dir) == EDMA_DIR_WRITE \ 74 ? GET_64(dw, wr_##name) \ 75 : GET_64(dw, rd_##name)) 76 77 #define SET_BOTH_64(dw, name, value) \ 78 do { \ 79 SET_64(dw, wr_##name, value); \ 80 SET_64(dw, rd_##name, value); \ 81 } while (0) 82 83 #define SET_COMPAT(dw, name, value) \ 84 writel(value, &(__dw_regs(dw)->type.unroll.name)) 85 86 #define SET_RW_COMPAT(dw, dir, name, value) \ 87 do { \ 88 if ((dir) == EDMA_DIR_WRITE) \ 89 SET_COMPAT(dw, wr_##name, value); \ 90 else \ 91 SET_COMPAT(dw, rd_##name, value); \ 92 } while (0) 93 94 static inline struct dw_edma_v0_ch_regs __iomem * 95 __dw_ch_regs(struct dw_edma *dw, enum dw_edma_dir dir, u16 ch) 96 { 97 if (dw->chip->mf == EDMA_MF_EDMA_LEGACY) 98 return &(__dw_regs(dw)->type.legacy.ch); 99 100 if (dir == EDMA_DIR_WRITE) 101 return &__dw_regs(dw)->type.unroll.ch[ch].wr; 102 103 return &__dw_regs(dw)->type.unroll.ch[ch].rd; 104 } 105 106 static inline void writel_ch(struct dw_edma *dw, enum dw_edma_dir dir, u16 ch, 107 u32 value, void __iomem *addr) 108 { 109 if (dw->chip->mf == EDMA_MF_EDMA_LEGACY) { 110 u32 viewport_sel; 111 unsigned long flags; 112 113 raw_spin_lock_irqsave(&dw->lock, flags); 114 115 viewport_sel = FIELD_PREP(EDMA_V0_VIEWPORT_MASK, ch); 116 if (dir == EDMA_DIR_READ) 117 viewport_sel |= BIT(31); 118 119 writel(viewport_sel, 120 &(__dw_regs(dw)->type.legacy.viewport_sel)); 121 writel(value, addr); 122 123 raw_spin_unlock_irqrestore(&dw->lock, flags); 124 } else { 125 writel(value, addr); 126 } 127 } 128 129 static inline u32 readl_ch(struct dw_edma *dw, enum dw_edma_dir dir, u16 ch, 130 const void __iomem *addr) 131 { 132 u32 value; 133 134 if (dw->chip->mf == EDMA_MF_EDMA_LEGACY) { 135 u32 viewport_sel; 136 unsigned long flags; 137 138 raw_spin_lock_irqsave(&dw->lock, flags); 139 140 viewport_sel = FIELD_PREP(EDMA_V0_VIEWPORT_MASK, ch); 141 if (dir == EDMA_DIR_READ) 142 viewport_sel |= BIT(31); 143 144 writel(viewport_sel, 145 &(__dw_regs(dw)->type.legacy.viewport_sel)); 146 value = readl(addr); 147 148 raw_spin_unlock_irqrestore(&dw->lock, flags); 149 } else { 150 value = readl(addr); 151 } 152 153 return value; 154 } 155 156 #define SET_CH_32(dw, dir, ch, name, value) \ 157 writel_ch(dw, dir, ch, value, &(__dw_ch_regs(dw, dir, ch)->name)) 158 159 #define GET_CH_32(dw, dir, ch, name) \ 160 readl_ch(dw, dir, ch, &(__dw_ch_regs(dw, dir, ch)->name)) 161 162 /* eDMA management callbacks */ 163 void dw_edma_v0_core_off(struct dw_edma *dw) 164 { 165 SET_BOTH_32(dw, int_mask, 166 EDMA_V0_DONE_INT_MASK | EDMA_V0_ABORT_INT_MASK); 167 SET_BOTH_32(dw, int_clear, 168 EDMA_V0_DONE_INT_MASK | EDMA_V0_ABORT_INT_MASK); 169 SET_BOTH_32(dw, engine_en, 0); 170 } 171 172 u16 dw_edma_v0_core_ch_count(struct dw_edma *dw, enum dw_edma_dir dir) 173 { 174 u32 num_ch; 175 176 if (dir == EDMA_DIR_WRITE) 177 num_ch = FIELD_GET(EDMA_V0_WRITE_CH_COUNT_MASK, 178 GET_32(dw, ctrl)); 179 else 180 num_ch = FIELD_GET(EDMA_V0_READ_CH_COUNT_MASK, 181 GET_32(dw, ctrl)); 182 183 if (num_ch > EDMA_V0_MAX_NR_CH) 184 num_ch = EDMA_V0_MAX_NR_CH; 185 186 return (u16)num_ch; 187 } 188 189 enum dma_status dw_edma_v0_core_ch_status(struct dw_edma_chan *chan) 190 { 191 struct dw_edma *dw = chan->dw; 192 u32 tmp; 193 194 tmp = FIELD_GET(EDMA_V0_CH_STATUS_MASK, 195 GET_CH_32(dw, chan->dir, chan->id, ch_control1)); 196 197 if (tmp == 1) 198 return DMA_IN_PROGRESS; 199 else if (tmp == 3) 200 return DMA_COMPLETE; 201 else 202 return DMA_ERROR; 203 } 204 205 void dw_edma_v0_core_clear_done_int(struct dw_edma_chan *chan) 206 { 207 struct dw_edma *dw = chan->dw; 208 209 SET_RW_32(dw, chan->dir, int_clear, 210 FIELD_PREP(EDMA_V0_DONE_INT_MASK, BIT(chan->id))); 211 } 212 213 void dw_edma_v0_core_clear_abort_int(struct dw_edma_chan *chan) 214 { 215 struct dw_edma *dw = chan->dw; 216 217 SET_RW_32(dw, chan->dir, int_clear, 218 FIELD_PREP(EDMA_V0_ABORT_INT_MASK, BIT(chan->id))); 219 } 220 221 u32 dw_edma_v0_core_status_done_int(struct dw_edma *dw, enum dw_edma_dir dir) 222 { 223 return FIELD_GET(EDMA_V0_DONE_INT_MASK, 224 GET_RW_32(dw, dir, int_status)); 225 } 226 227 u32 dw_edma_v0_core_status_abort_int(struct dw_edma *dw, enum dw_edma_dir dir) 228 { 229 return FIELD_GET(EDMA_V0_ABORT_INT_MASK, 230 GET_RW_32(dw, dir, int_status)); 231 } 232 233 static void dw_edma_v0_write_ll_data(struct dw_edma_chunk *chunk, int i, 234 u32 control, u32 size, u64 sar, u64 dar) 235 { 236 ptrdiff_t ofs = i * sizeof(struct dw_edma_v0_lli); 237 238 if (chunk->chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) { 239 struct dw_edma_v0_lli *lli = chunk->ll_region.vaddr.mem + ofs; 240 241 lli->control = control; 242 lli->transfer_size = size; 243 lli->sar.reg = sar; 244 lli->dar.reg = dar; 245 } else { 246 struct dw_edma_v0_lli __iomem *lli = chunk->ll_region.vaddr.io + ofs; 247 248 writel(control, &lli->control); 249 writel(size, &lli->transfer_size); 250 writeq(sar, &lli->sar.reg); 251 writeq(dar, &lli->dar.reg); 252 } 253 } 254 255 static void dw_edma_v0_write_ll_link(struct dw_edma_chunk *chunk, 256 int i, u32 control, u64 pointer) 257 { 258 ptrdiff_t ofs = i * sizeof(struct dw_edma_v0_lli); 259 260 if (chunk->chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) { 261 struct dw_edma_v0_llp *llp = chunk->ll_region.vaddr.mem + ofs; 262 263 llp->control = control; 264 llp->llp.reg = pointer; 265 } else { 266 struct dw_edma_v0_llp __iomem *llp = chunk->ll_region.vaddr.io + ofs; 267 268 writel(control, &llp->control); 269 writeq(pointer, &llp->llp.reg); 270 } 271 } 272 273 static void dw_edma_v0_core_write_chunk(struct dw_edma_chunk *chunk) 274 { 275 struct dw_edma_burst *child; 276 struct dw_edma_chan *chan = chunk->chan; 277 u32 control = 0, i = 0; 278 int j; 279 280 if (chunk->cb) 281 control = DW_EDMA_V0_CB; 282 283 j = chunk->bursts_alloc; 284 list_for_each_entry(child, &chunk->burst->list, list) { 285 j--; 286 if (!j) { 287 control |= DW_EDMA_V0_LIE; 288 if (!(chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL)) 289 control |= DW_EDMA_V0_RIE; 290 } 291 292 dw_edma_v0_write_ll_data(chunk, i++, control, child->sz, 293 child->sar, child->dar); 294 } 295 296 control = DW_EDMA_V0_LLP | DW_EDMA_V0_TCB; 297 if (!chunk->cb) 298 control |= DW_EDMA_V0_CB; 299 300 dw_edma_v0_write_ll_link(chunk, i, control, chunk->ll_region.paddr); 301 } 302 303 void dw_edma_v0_core_start(struct dw_edma_chunk *chunk, bool first) 304 { 305 struct dw_edma_chan *chan = chunk->chan; 306 struct dw_edma *dw = chan->dw; 307 u32 tmp; 308 309 dw_edma_v0_core_write_chunk(chunk); 310 311 if (first) { 312 /* Enable engine */ 313 SET_RW_32(dw, chan->dir, engine_en, BIT(0)); 314 if (dw->chip->mf == EDMA_MF_HDMA_COMPAT) { 315 switch (chan->id) { 316 case 0: 317 SET_RW_COMPAT(dw, chan->dir, ch0_pwr_en, 318 BIT(0)); 319 break; 320 case 1: 321 SET_RW_COMPAT(dw, chan->dir, ch1_pwr_en, 322 BIT(0)); 323 break; 324 case 2: 325 SET_RW_COMPAT(dw, chan->dir, ch2_pwr_en, 326 BIT(0)); 327 break; 328 case 3: 329 SET_RW_COMPAT(dw, chan->dir, ch3_pwr_en, 330 BIT(0)); 331 break; 332 case 4: 333 SET_RW_COMPAT(dw, chan->dir, ch4_pwr_en, 334 BIT(0)); 335 break; 336 case 5: 337 SET_RW_COMPAT(dw, chan->dir, ch5_pwr_en, 338 BIT(0)); 339 break; 340 case 6: 341 SET_RW_COMPAT(dw, chan->dir, ch6_pwr_en, 342 BIT(0)); 343 break; 344 case 7: 345 SET_RW_COMPAT(dw, chan->dir, ch7_pwr_en, 346 BIT(0)); 347 break; 348 } 349 } 350 /* Interrupt unmask - done, abort */ 351 tmp = GET_RW_32(dw, chan->dir, int_mask); 352 tmp &= ~FIELD_PREP(EDMA_V0_DONE_INT_MASK, BIT(chan->id)); 353 tmp &= ~FIELD_PREP(EDMA_V0_ABORT_INT_MASK, BIT(chan->id)); 354 SET_RW_32(dw, chan->dir, int_mask, tmp); 355 /* Linked list error */ 356 tmp = GET_RW_32(dw, chan->dir, linked_list_err_en); 357 tmp |= FIELD_PREP(EDMA_V0_LINKED_LIST_ERR_MASK, BIT(chan->id)); 358 SET_RW_32(dw, chan->dir, linked_list_err_en, tmp); 359 /* Channel control */ 360 SET_CH_32(dw, chan->dir, chan->id, ch_control1, 361 (DW_EDMA_V0_CCS | DW_EDMA_V0_LLE)); 362 /* Linked list */ 363 /* llp is not aligned on 64bit -> keep 32bit accesses */ 364 SET_CH_32(dw, chan->dir, chan->id, llp.lsb, 365 lower_32_bits(chunk->ll_region.paddr)); 366 SET_CH_32(dw, chan->dir, chan->id, llp.msb, 367 upper_32_bits(chunk->ll_region.paddr)); 368 } 369 /* Doorbell */ 370 SET_RW_32(dw, chan->dir, doorbell, 371 FIELD_PREP(EDMA_V0_DOORBELL_CH_MASK, chan->id)); 372 } 373 374 int dw_edma_v0_core_device_config(struct dw_edma_chan *chan) 375 { 376 struct dw_edma *dw = chan->dw; 377 u32 tmp = 0; 378 379 /* MSI done addr - low, high */ 380 SET_RW_32(dw, chan->dir, done_imwr.lsb, chan->msi.address_lo); 381 SET_RW_32(dw, chan->dir, done_imwr.msb, chan->msi.address_hi); 382 /* MSI abort addr - low, high */ 383 SET_RW_32(dw, chan->dir, abort_imwr.lsb, chan->msi.address_lo); 384 SET_RW_32(dw, chan->dir, abort_imwr.msb, chan->msi.address_hi); 385 /* MSI data - low, high */ 386 switch (chan->id) { 387 case 0: 388 case 1: 389 tmp = GET_RW_32(dw, chan->dir, ch01_imwr_data); 390 break; 391 392 case 2: 393 case 3: 394 tmp = GET_RW_32(dw, chan->dir, ch23_imwr_data); 395 break; 396 397 case 4: 398 case 5: 399 tmp = GET_RW_32(dw, chan->dir, ch45_imwr_data); 400 break; 401 402 case 6: 403 case 7: 404 tmp = GET_RW_32(dw, chan->dir, ch67_imwr_data); 405 break; 406 } 407 408 if (chan->id & BIT(0)) { 409 /* Channel odd {1, 3, 5, 7} */ 410 tmp &= EDMA_V0_CH_EVEN_MSI_DATA_MASK; 411 tmp |= FIELD_PREP(EDMA_V0_CH_ODD_MSI_DATA_MASK, 412 chan->msi.data); 413 } else { 414 /* Channel even {0, 2, 4, 6} */ 415 tmp &= EDMA_V0_CH_ODD_MSI_DATA_MASK; 416 tmp |= FIELD_PREP(EDMA_V0_CH_EVEN_MSI_DATA_MASK, 417 chan->msi.data); 418 } 419 420 switch (chan->id) { 421 case 0: 422 case 1: 423 SET_RW_32(dw, chan->dir, ch01_imwr_data, tmp); 424 break; 425 426 case 2: 427 case 3: 428 SET_RW_32(dw, chan->dir, ch23_imwr_data, tmp); 429 break; 430 431 case 4: 432 case 5: 433 SET_RW_32(dw, chan->dir, ch45_imwr_data, tmp); 434 break; 435 436 case 6: 437 case 7: 438 SET_RW_32(dw, chan->dir, ch67_imwr_data, tmp); 439 break; 440 } 441 442 return 0; 443 } 444 445 /* eDMA debugfs callbacks */ 446 void dw_edma_v0_core_debugfs_on(struct dw_edma *dw) 447 { 448 dw_edma_v0_debugfs_on(dw); 449 } 450