1 /* 2 * Copyright (C) 2013-2014 Renesas Electronics Europe Ltd. 3 * Author: Guennadi Liakhovetski <g.liakhovetski@gmx.de> 4 * 5 * This program is free software; you can redistribute it and/or modify 6 * it under the terms of version 2 of the GNU General Public License as 7 * published by the Free Software Foundation. 8 */ 9 10 #include <linux/bitmap.h> 11 #include <linux/bitops.h> 12 #include <linux/clk.h> 13 #include <linux/dma-mapping.h> 14 #include <linux/dmaengine.h> 15 #include <linux/err.h> 16 #include <linux/interrupt.h> 17 #include <linux/io.h> 18 #include <linux/log2.h> 19 #include <linux/module.h> 20 #include <linux/of.h> 21 #include <linux/of_device.h> 22 #include <linux/of_dma.h> 23 #include <linux/platform_device.h> 24 #include <linux/slab.h> 25 26 #include <dt-bindings/dma/nbpfaxi.h> 27 28 #include "dmaengine.h" 29 30 #define NBPF_REG_CHAN_OFFSET 0 31 #define NBPF_REG_CHAN_SIZE 0x40 32 33 /* Channel Current Transaction Byte register */ 34 #define NBPF_CHAN_CUR_TR_BYTE 0x20 35 36 /* Channel Status register */ 37 #define NBPF_CHAN_STAT 0x24 38 #define NBPF_CHAN_STAT_EN 1 39 #define NBPF_CHAN_STAT_TACT 4 40 #define NBPF_CHAN_STAT_ERR 0x10 41 #define NBPF_CHAN_STAT_END 0x20 42 #define NBPF_CHAN_STAT_TC 0x40 43 #define NBPF_CHAN_STAT_DER 0x400 44 45 /* Channel Control register */ 46 #define NBPF_CHAN_CTRL 0x28 47 #define NBPF_CHAN_CTRL_SETEN 1 48 #define NBPF_CHAN_CTRL_CLREN 2 49 #define NBPF_CHAN_CTRL_STG 4 50 #define NBPF_CHAN_CTRL_SWRST 8 51 #define NBPF_CHAN_CTRL_CLRRQ 0x10 52 #define NBPF_CHAN_CTRL_CLREND 0x20 53 #define NBPF_CHAN_CTRL_CLRTC 0x40 54 #define NBPF_CHAN_CTRL_SETSUS 0x100 55 #define NBPF_CHAN_CTRL_CLRSUS 0x200 56 57 /* Channel Configuration register */ 58 #define NBPF_CHAN_CFG 0x2c 59 #define NBPF_CHAN_CFG_SEL 7 /* terminal SELect: 0..7 */ 60 #define NBPF_CHAN_CFG_REQD 8 /* REQuest Direction: DMAREQ is 0: input, 1: output */ 61 #define NBPF_CHAN_CFG_LOEN 0x10 /* LOw ENable: low DMA request line is: 0: inactive, 1: active */ 62 #define NBPF_CHAN_CFG_HIEN 0x20 /* HIgh ENable: high DMA request line is: 0: inactive, 1: active */ 63 #define NBPF_CHAN_CFG_LVL 0x40 /* LeVeL: DMA request line is sensed as 0: edge, 1: level */ 64 #define NBPF_CHAN_CFG_AM 0x700 /* ACK Mode: 0: Pulse mode, 1: Level mode, b'1x: Bus Cycle */ 65 #define NBPF_CHAN_CFG_SDS 0xf000 /* Source Data Size: 0: 8 bits,... , 7: 1024 bits */ 66 #define NBPF_CHAN_CFG_DDS 0xf0000 /* Destination Data Size: as above */ 67 #define NBPF_CHAN_CFG_SAD 0x100000 /* Source ADdress counting: 0: increment, 1: fixed */ 68 #define NBPF_CHAN_CFG_DAD 0x200000 /* Destination ADdress counting: 0: increment, 1: fixed */ 69 #define NBPF_CHAN_CFG_TM 0x400000 /* Transfer Mode: 0: single, 1: block TM */ 70 #define NBPF_CHAN_CFG_DEM 0x1000000 /* DMAEND interrupt Mask */ 71 #define NBPF_CHAN_CFG_TCM 0x2000000 /* DMATCO interrupt Mask */ 72 #define NBPF_CHAN_CFG_SBE 0x8000000 /* Sweep Buffer Enable */ 73 #define NBPF_CHAN_CFG_RSEL 0x10000000 /* RM: Register Set sELect */ 74 #define NBPF_CHAN_CFG_RSW 0x20000000 /* RM: Register Select sWitch */ 75 #define NBPF_CHAN_CFG_REN 0x40000000 /* RM: Register Set Enable */ 76 #define NBPF_CHAN_CFG_DMS 0x80000000 /* 0: register mode (RM), 1: link mode (LM) */ 77 78 #define NBPF_CHAN_NXLA 0x38 79 #define NBPF_CHAN_CRLA 0x3c 80 81 /* Link Header field */ 82 #define NBPF_HEADER_LV 1 83 #define NBPF_HEADER_LE 2 84 #define NBPF_HEADER_WBD 4 85 #define NBPF_HEADER_DIM 8 86 87 #define NBPF_CTRL 0x300 88 #define NBPF_CTRL_PR 1 /* 0: fixed priority, 1: round robin */ 89 #define NBPF_CTRL_LVINT 2 /* DMAEND and DMAERR signalling: 0: pulse, 1: level */ 90 91 #define NBPF_DSTAT_ER 0x314 92 #define NBPF_DSTAT_END 0x318 93 94 #define NBPF_DMA_BUSWIDTHS \ 95 (BIT(DMA_SLAVE_BUSWIDTH_UNDEFINED) | \ 96 BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \ 97 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \ 98 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | \ 99 BIT(DMA_SLAVE_BUSWIDTH_8_BYTES)) 100 101 struct nbpf_config { 102 int num_channels; 103 int buffer_size; 104 }; 105 106 /* 107 * We've got 3 types of objects, used to describe DMA transfers: 108 * 1. high-level descriptor, containing a struct dma_async_tx_descriptor object 109 * in it, used to communicate with the user 110 * 2. hardware DMA link descriptors, that we pass to DMAC for DMA transfer 111 * queuing, these must be DMAable, using either the streaming DMA API or 112 * allocated from coherent memory - one per SG segment 113 * 3. one per SG segment descriptors, used to manage HW link descriptors from 114 * (2). They do not have to be DMAable. They can either be (a) allocated 115 * together with link descriptors as mixed (DMA / CPU) objects, or (b) 116 * separately. Even if allocated separately it would be best to link them 117 * to link descriptors once during channel resource allocation and always 118 * use them as a single object. 119 * Therefore for both cases (a) and (b) at run-time objects (2) and (3) shall be 120 * treated as a single SG segment descriptor. 121 */ 122 123 struct nbpf_link_reg { 124 u32 header; 125 u32 src_addr; 126 u32 dst_addr; 127 u32 transaction_size; 128 u32 config; 129 u32 interval; 130 u32 extension; 131 u32 next; 132 } __packed; 133 134 struct nbpf_device; 135 struct nbpf_channel; 136 struct nbpf_desc; 137 138 struct nbpf_link_desc { 139 struct nbpf_link_reg *hwdesc; 140 dma_addr_t hwdesc_dma_addr; 141 struct nbpf_desc *desc; 142 struct list_head node; 143 }; 144 145 /** 146 * struct nbpf_desc - DMA transfer descriptor 147 * @async_tx: dmaengine object 148 * @user_wait: waiting for a user ack 149 * @length: total transfer length 150 * @sg: list of hardware descriptors, represented by struct nbpf_link_desc 151 * @node: member in channel descriptor lists 152 */ 153 struct nbpf_desc { 154 struct dma_async_tx_descriptor async_tx; 155 bool user_wait; 156 size_t length; 157 struct nbpf_channel *chan; 158 struct list_head sg; 159 struct list_head node; 160 }; 161 162 /* Take a wild guess: allocate 4 segments per descriptor */ 163 #define NBPF_SEGMENTS_PER_DESC 4 164 #define NBPF_DESCS_PER_PAGE ((PAGE_SIZE - sizeof(struct list_head)) / \ 165 (sizeof(struct nbpf_desc) + \ 166 NBPF_SEGMENTS_PER_DESC * \ 167 (sizeof(struct nbpf_link_desc) + sizeof(struct nbpf_link_reg)))) 168 #define NBPF_SEGMENTS_PER_PAGE (NBPF_SEGMENTS_PER_DESC * NBPF_DESCS_PER_PAGE) 169 170 struct nbpf_desc_page { 171 struct list_head node; 172 struct nbpf_desc desc[NBPF_DESCS_PER_PAGE]; 173 struct nbpf_link_desc ldesc[NBPF_SEGMENTS_PER_PAGE]; 174 struct nbpf_link_reg hwdesc[NBPF_SEGMENTS_PER_PAGE]; 175 }; 176 177 /** 178 * struct nbpf_channel - one DMAC channel 179 * @dma_chan: standard dmaengine channel object 180 * @base: register address base 181 * @nbpf: DMAC 182 * @name: IRQ name 183 * @irq: IRQ number 184 * @slave_addr: address for slave DMA 185 * @slave_width:slave data size in bytes 186 * @slave_burst:maximum slave burst size in bytes 187 * @terminal: DMA terminal, assigned to this channel 188 * @dmarq_cfg: DMA request line configuration - high / low, edge / level for NBPF_CHAN_CFG 189 * @flags: configuration flags from DT 190 * @lock: protect descriptor lists 191 * @free_links: list of free link descriptors 192 * @free: list of free descriptors 193 * @queued: list of queued descriptors 194 * @active: list of descriptors, scheduled for processing 195 * @done: list of completed descriptors, waiting post-processing 196 * @desc_page: list of additionally allocated descriptor pages - if any 197 */ 198 struct nbpf_channel { 199 struct dma_chan dma_chan; 200 struct tasklet_struct tasklet; 201 void __iomem *base; 202 struct nbpf_device *nbpf; 203 char name[16]; 204 int irq; 205 dma_addr_t slave_src_addr; 206 size_t slave_src_width; 207 size_t slave_src_burst; 208 dma_addr_t slave_dst_addr; 209 size_t slave_dst_width; 210 size_t slave_dst_burst; 211 unsigned int terminal; 212 u32 dmarq_cfg; 213 unsigned long flags; 214 spinlock_t lock; 215 struct list_head free_links; 216 struct list_head free; 217 struct list_head queued; 218 struct list_head active; 219 struct list_head done; 220 struct list_head desc_page; 221 struct nbpf_desc *running; 222 bool paused; 223 }; 224 225 struct nbpf_device { 226 struct dma_device dma_dev; 227 void __iomem *base; 228 struct clk *clk; 229 const struct nbpf_config *config; 230 struct nbpf_channel chan[]; 231 }; 232 233 enum nbpf_model { 234 NBPF1B4, 235 NBPF1B8, 236 NBPF1B16, 237 NBPF4B4, 238 NBPF4B8, 239 NBPF4B16, 240 NBPF8B4, 241 NBPF8B8, 242 NBPF8B16, 243 }; 244 245 static struct nbpf_config nbpf_cfg[] = { 246 [NBPF1B4] = { 247 .num_channels = 1, 248 .buffer_size = 4, 249 }, 250 [NBPF1B8] = { 251 .num_channels = 1, 252 .buffer_size = 8, 253 }, 254 [NBPF1B16] = { 255 .num_channels = 1, 256 .buffer_size = 16, 257 }, 258 [NBPF4B4] = { 259 .num_channels = 4, 260 .buffer_size = 4, 261 }, 262 [NBPF4B8] = { 263 .num_channels = 4, 264 .buffer_size = 8, 265 }, 266 [NBPF4B16] = { 267 .num_channels = 4, 268 .buffer_size = 16, 269 }, 270 [NBPF8B4] = { 271 .num_channels = 8, 272 .buffer_size = 4, 273 }, 274 [NBPF8B8] = { 275 .num_channels = 8, 276 .buffer_size = 8, 277 }, 278 [NBPF8B16] = { 279 .num_channels = 8, 280 .buffer_size = 16, 281 }, 282 }; 283 284 #define nbpf_to_chan(d) container_of(d, struct nbpf_channel, dma_chan) 285 286 /* 287 * dmaengine drivers seem to have a lot in common and instead of sharing more 288 * code, they reimplement those common algorithms independently. In this driver 289 * we try to separate the hardware-specific part from the (largely) generic 290 * part. This improves code readability and makes it possible in the future to 291 * reuse the generic code in form of a helper library. That generic code should 292 * be suitable for various DMA controllers, using transfer descriptors in RAM 293 * and pushing one SG list at a time to the DMA controller. 294 */ 295 296 /* Hardware-specific part */ 297 298 static inline u32 nbpf_chan_read(struct nbpf_channel *chan, 299 unsigned int offset) 300 { 301 u32 data = ioread32(chan->base + offset); 302 dev_dbg(chan->dma_chan.device->dev, "%s(0x%p + 0x%x) = 0x%x\n", 303 __func__, chan->base, offset, data); 304 return data; 305 } 306 307 static inline void nbpf_chan_write(struct nbpf_channel *chan, 308 unsigned int offset, u32 data) 309 { 310 iowrite32(data, chan->base + offset); 311 dev_dbg(chan->dma_chan.device->dev, "%s(0x%p + 0x%x) = 0x%x\n", 312 __func__, chan->base, offset, data); 313 } 314 315 static inline u32 nbpf_read(struct nbpf_device *nbpf, 316 unsigned int offset) 317 { 318 u32 data = ioread32(nbpf->base + offset); 319 dev_dbg(nbpf->dma_dev.dev, "%s(0x%p + 0x%x) = 0x%x\n", 320 __func__, nbpf->base, offset, data); 321 return data; 322 } 323 324 static inline void nbpf_write(struct nbpf_device *nbpf, 325 unsigned int offset, u32 data) 326 { 327 iowrite32(data, nbpf->base + offset); 328 dev_dbg(nbpf->dma_dev.dev, "%s(0x%p + 0x%x) = 0x%x\n", 329 __func__, nbpf->base, offset, data); 330 } 331 332 static void nbpf_chan_halt(struct nbpf_channel *chan) 333 { 334 nbpf_chan_write(chan, NBPF_CHAN_CTRL, NBPF_CHAN_CTRL_CLREN); 335 } 336 337 static bool nbpf_status_get(struct nbpf_channel *chan) 338 { 339 u32 status = nbpf_read(chan->nbpf, NBPF_DSTAT_END); 340 341 return status & BIT(chan - chan->nbpf->chan); 342 } 343 344 static void nbpf_status_ack(struct nbpf_channel *chan) 345 { 346 nbpf_chan_write(chan, NBPF_CHAN_CTRL, NBPF_CHAN_CTRL_CLREND); 347 } 348 349 static u32 nbpf_error_get(struct nbpf_device *nbpf) 350 { 351 return nbpf_read(nbpf, NBPF_DSTAT_ER); 352 } 353 354 static struct nbpf_channel *nbpf_error_get_channel(struct nbpf_device *nbpf, u32 error) 355 { 356 return nbpf->chan + __ffs(error); 357 } 358 359 static void nbpf_error_clear(struct nbpf_channel *chan) 360 { 361 u32 status; 362 int i; 363 364 /* Stop the channel, make sure DMA has been aborted */ 365 nbpf_chan_halt(chan); 366 367 for (i = 1000; i; i--) { 368 status = nbpf_chan_read(chan, NBPF_CHAN_STAT); 369 if (!(status & NBPF_CHAN_STAT_TACT)) 370 break; 371 cpu_relax(); 372 } 373 374 if (!i) 375 dev_err(chan->dma_chan.device->dev, 376 "%s(): abort timeout, channel status 0x%x\n", __func__, status); 377 378 nbpf_chan_write(chan, NBPF_CHAN_CTRL, NBPF_CHAN_CTRL_SWRST); 379 } 380 381 static int nbpf_start(struct nbpf_desc *desc) 382 { 383 struct nbpf_channel *chan = desc->chan; 384 struct nbpf_link_desc *ldesc = list_first_entry(&desc->sg, struct nbpf_link_desc, node); 385 386 nbpf_chan_write(chan, NBPF_CHAN_NXLA, (u32)ldesc->hwdesc_dma_addr); 387 nbpf_chan_write(chan, NBPF_CHAN_CTRL, NBPF_CHAN_CTRL_SETEN | NBPF_CHAN_CTRL_CLRSUS); 388 chan->paused = false; 389 390 /* Software trigger MEMCPY - only MEMCPY uses the block mode */ 391 if (ldesc->hwdesc->config & NBPF_CHAN_CFG_TM) 392 nbpf_chan_write(chan, NBPF_CHAN_CTRL, NBPF_CHAN_CTRL_STG); 393 394 dev_dbg(chan->nbpf->dma_dev.dev, "%s(): next 0x%x, cur 0x%x\n", __func__, 395 nbpf_chan_read(chan, NBPF_CHAN_NXLA), nbpf_chan_read(chan, NBPF_CHAN_CRLA)); 396 397 return 0; 398 } 399 400 static void nbpf_chan_prepare(struct nbpf_channel *chan) 401 { 402 chan->dmarq_cfg = (chan->flags & NBPF_SLAVE_RQ_HIGH ? NBPF_CHAN_CFG_HIEN : 0) | 403 (chan->flags & NBPF_SLAVE_RQ_LOW ? NBPF_CHAN_CFG_LOEN : 0) | 404 (chan->flags & NBPF_SLAVE_RQ_LEVEL ? 405 NBPF_CHAN_CFG_LVL | (NBPF_CHAN_CFG_AM & 0x200) : 0) | 406 chan->terminal; 407 } 408 409 static void nbpf_chan_prepare_default(struct nbpf_channel *chan) 410 { 411 /* Don't output DMAACK */ 412 chan->dmarq_cfg = NBPF_CHAN_CFG_AM & 0x400; 413 chan->terminal = 0; 414 chan->flags = 0; 415 } 416 417 static void nbpf_chan_configure(struct nbpf_channel *chan) 418 { 419 /* 420 * We assume, that only the link mode and DMA request line configuration 421 * have to be set in the configuration register manually. Dynamic 422 * per-transfer configuration will be loaded from transfer descriptors. 423 */ 424 nbpf_chan_write(chan, NBPF_CHAN_CFG, NBPF_CHAN_CFG_DMS | chan->dmarq_cfg); 425 } 426 427 static u32 nbpf_xfer_ds(struct nbpf_device *nbpf, size_t size) 428 { 429 /* Maximum supported bursts depend on the buffer size */ 430 return min_t(int, __ffs(size), ilog2(nbpf->config->buffer_size * 8)); 431 } 432 433 static size_t nbpf_xfer_size(struct nbpf_device *nbpf, 434 enum dma_slave_buswidth width, u32 burst) 435 { 436 size_t size; 437 438 if (!burst) 439 burst = 1; 440 441 switch (width) { 442 case DMA_SLAVE_BUSWIDTH_8_BYTES: 443 size = 8 * burst; 444 break; 445 446 case DMA_SLAVE_BUSWIDTH_4_BYTES: 447 size = 4 * burst; 448 break; 449 450 case DMA_SLAVE_BUSWIDTH_2_BYTES: 451 size = 2 * burst; 452 break; 453 454 default: 455 pr_warn("%s(): invalid bus width %u\n", __func__, width); 456 case DMA_SLAVE_BUSWIDTH_1_BYTE: 457 size = burst; 458 } 459 460 return nbpf_xfer_ds(nbpf, size); 461 } 462 463 /* 464 * We need a way to recognise slaves, whose data is sent "raw" over the bus, 465 * i.e. it isn't known in advance how many bytes will be received. Therefore 466 * the slave driver has to provide a "large enough" buffer and either read the 467 * buffer, when it is full, or detect, that some data has arrived, then wait for 468 * a timeout, if no more data arrives - receive what's already there. We want to 469 * handle such slaves in a special way to allow an optimised mode for other 470 * users, for whom the amount of data is known in advance. So far there's no way 471 * to recognise such slaves. We use a data-width check to distinguish between 472 * the SD host and the PL011 UART. 473 */ 474 475 static int nbpf_prep_one(struct nbpf_link_desc *ldesc, 476 enum dma_transfer_direction direction, 477 dma_addr_t src, dma_addr_t dst, size_t size, bool last) 478 { 479 struct nbpf_link_reg *hwdesc = ldesc->hwdesc; 480 struct nbpf_desc *desc = ldesc->desc; 481 struct nbpf_channel *chan = desc->chan; 482 struct device *dev = chan->dma_chan.device->dev; 483 size_t mem_xfer, slave_xfer; 484 bool can_burst; 485 486 hwdesc->header = NBPF_HEADER_WBD | NBPF_HEADER_LV | 487 (last ? NBPF_HEADER_LE : 0); 488 489 hwdesc->src_addr = src; 490 hwdesc->dst_addr = dst; 491 hwdesc->transaction_size = size; 492 493 /* 494 * set config: SAD, DAD, DDS, SDS, etc. 495 * Note on transfer sizes: the DMAC can perform unaligned DMA transfers, 496 * but it is important to have transaction size a multiple of both 497 * receiver and transmitter transfer sizes. It is also possible to use 498 * different RAM and device transfer sizes, and it does work well with 499 * some devices, e.g. with V08R07S01E SD host controllers, which can use 500 * 128 byte transfers. But this doesn't work with other devices, 501 * especially when the transaction size is unknown. This is the case, 502 * e.g. with serial drivers like amba-pl011.c. For reception it sets up 503 * the transaction size of 4K and if fewer bytes are received, it 504 * pauses DMA and reads out data received via DMA as well as those left 505 * in the Rx FIFO. For this to work with the RAM side using burst 506 * transfers we enable the SBE bit and terminate the transfer in our 507 * DMA_PAUSE handler. 508 */ 509 mem_xfer = nbpf_xfer_ds(chan->nbpf, size); 510 511 switch (direction) { 512 case DMA_DEV_TO_MEM: 513 can_burst = chan->slave_src_width >= 3; 514 slave_xfer = min(mem_xfer, can_burst ? 515 chan->slave_src_burst : chan->slave_src_width); 516 /* 517 * Is the slave narrower than 64 bits, i.e. isn't using the full 518 * bus width and cannot use bursts? 519 */ 520 if (mem_xfer > chan->slave_src_burst && !can_burst) 521 mem_xfer = chan->slave_src_burst; 522 /* Device-to-RAM DMA is unreliable without REQD set */ 523 hwdesc->config = NBPF_CHAN_CFG_SAD | (NBPF_CHAN_CFG_DDS & (mem_xfer << 16)) | 524 (NBPF_CHAN_CFG_SDS & (slave_xfer << 12)) | NBPF_CHAN_CFG_REQD | 525 NBPF_CHAN_CFG_SBE; 526 break; 527 528 case DMA_MEM_TO_DEV: 529 slave_xfer = min(mem_xfer, chan->slave_dst_width >= 3 ? 530 chan->slave_dst_burst : chan->slave_dst_width); 531 hwdesc->config = NBPF_CHAN_CFG_DAD | (NBPF_CHAN_CFG_SDS & (mem_xfer << 12)) | 532 (NBPF_CHAN_CFG_DDS & (slave_xfer << 16)) | NBPF_CHAN_CFG_REQD; 533 break; 534 535 case DMA_MEM_TO_MEM: 536 hwdesc->config = NBPF_CHAN_CFG_TCM | NBPF_CHAN_CFG_TM | 537 (NBPF_CHAN_CFG_SDS & (mem_xfer << 12)) | 538 (NBPF_CHAN_CFG_DDS & (mem_xfer << 16)); 539 break; 540 541 default: 542 return -EINVAL; 543 } 544 545 hwdesc->config |= chan->dmarq_cfg | (last ? 0 : NBPF_CHAN_CFG_DEM) | 546 NBPF_CHAN_CFG_DMS; 547 548 dev_dbg(dev, "%s(): desc @ %pad: hdr 0x%x, cfg 0x%x, %zu @ %pad -> %pad\n", 549 __func__, &ldesc->hwdesc_dma_addr, hwdesc->header, 550 hwdesc->config, size, &src, &dst); 551 552 dma_sync_single_for_device(dev, ldesc->hwdesc_dma_addr, sizeof(*hwdesc), 553 DMA_TO_DEVICE); 554 555 return 0; 556 } 557 558 static size_t nbpf_bytes_left(struct nbpf_channel *chan) 559 { 560 return nbpf_chan_read(chan, NBPF_CHAN_CUR_TR_BYTE); 561 } 562 563 static void nbpf_configure(struct nbpf_device *nbpf) 564 { 565 nbpf_write(nbpf, NBPF_CTRL, NBPF_CTRL_LVINT); 566 } 567 568 static void nbpf_pause(struct nbpf_channel *chan) 569 { 570 nbpf_chan_write(chan, NBPF_CHAN_CTRL, NBPF_CHAN_CTRL_SETSUS); 571 /* See comment in nbpf_prep_one() */ 572 nbpf_chan_write(chan, NBPF_CHAN_CTRL, NBPF_CHAN_CTRL_CLREN); 573 } 574 575 /* Generic part */ 576 577 /* DMA ENGINE functions */ 578 static void nbpf_issue_pending(struct dma_chan *dchan) 579 { 580 struct nbpf_channel *chan = nbpf_to_chan(dchan); 581 unsigned long flags; 582 583 dev_dbg(dchan->device->dev, "Entry %s()\n", __func__); 584 585 spin_lock_irqsave(&chan->lock, flags); 586 if (list_empty(&chan->queued)) 587 goto unlock; 588 589 list_splice_tail_init(&chan->queued, &chan->active); 590 591 if (!chan->running) { 592 struct nbpf_desc *desc = list_first_entry(&chan->active, 593 struct nbpf_desc, node); 594 if (!nbpf_start(desc)) 595 chan->running = desc; 596 } 597 598 unlock: 599 spin_unlock_irqrestore(&chan->lock, flags); 600 } 601 602 static enum dma_status nbpf_tx_status(struct dma_chan *dchan, 603 dma_cookie_t cookie, struct dma_tx_state *state) 604 { 605 struct nbpf_channel *chan = nbpf_to_chan(dchan); 606 enum dma_status status = dma_cookie_status(dchan, cookie, state); 607 608 if (state) { 609 dma_cookie_t running; 610 unsigned long flags; 611 612 spin_lock_irqsave(&chan->lock, flags); 613 running = chan->running ? chan->running->async_tx.cookie : -EINVAL; 614 615 if (cookie == running) { 616 state->residue = nbpf_bytes_left(chan); 617 dev_dbg(dchan->device->dev, "%s(): residue %u\n", __func__, 618 state->residue); 619 } else if (status == DMA_IN_PROGRESS) { 620 struct nbpf_desc *desc; 621 bool found = false; 622 623 list_for_each_entry(desc, &chan->active, node) 624 if (desc->async_tx.cookie == cookie) { 625 found = true; 626 break; 627 } 628 629 if (!found) 630 list_for_each_entry(desc, &chan->queued, node) 631 if (desc->async_tx.cookie == cookie) { 632 found = true; 633 break; 634 635 } 636 637 state->residue = found ? desc->length : 0; 638 } 639 640 spin_unlock_irqrestore(&chan->lock, flags); 641 } 642 643 if (chan->paused) 644 status = DMA_PAUSED; 645 646 return status; 647 } 648 649 static dma_cookie_t nbpf_tx_submit(struct dma_async_tx_descriptor *tx) 650 { 651 struct nbpf_desc *desc = container_of(tx, struct nbpf_desc, async_tx); 652 struct nbpf_channel *chan = desc->chan; 653 unsigned long flags; 654 dma_cookie_t cookie; 655 656 spin_lock_irqsave(&chan->lock, flags); 657 cookie = dma_cookie_assign(tx); 658 list_add_tail(&desc->node, &chan->queued); 659 spin_unlock_irqrestore(&chan->lock, flags); 660 661 dev_dbg(chan->dma_chan.device->dev, "Entry %s(%d)\n", __func__, cookie); 662 663 return cookie; 664 } 665 666 static int nbpf_desc_page_alloc(struct nbpf_channel *chan) 667 { 668 struct dma_chan *dchan = &chan->dma_chan; 669 struct nbpf_desc_page *dpage = (void *)get_zeroed_page(GFP_KERNEL | GFP_DMA); 670 struct nbpf_link_desc *ldesc; 671 struct nbpf_link_reg *hwdesc; 672 struct nbpf_desc *desc; 673 LIST_HEAD(head); 674 LIST_HEAD(lhead); 675 int i; 676 struct device *dev = dchan->device->dev; 677 678 if (!dpage) 679 return -ENOMEM; 680 681 dev_dbg(dev, "%s(): alloc %lu descriptors, %lu segments, total alloc %zu\n", 682 __func__, NBPF_DESCS_PER_PAGE, NBPF_SEGMENTS_PER_PAGE, sizeof(*dpage)); 683 684 for (i = 0, ldesc = dpage->ldesc, hwdesc = dpage->hwdesc; 685 i < ARRAY_SIZE(dpage->ldesc); 686 i++, ldesc++, hwdesc++) { 687 ldesc->hwdesc = hwdesc; 688 list_add_tail(&ldesc->node, &lhead); 689 ldesc->hwdesc_dma_addr = dma_map_single(dchan->device->dev, 690 hwdesc, sizeof(*hwdesc), DMA_TO_DEVICE); 691 692 dev_dbg(dev, "%s(): mapped 0x%p to %pad\n", __func__, 693 hwdesc, &ldesc->hwdesc_dma_addr); 694 } 695 696 for (i = 0, desc = dpage->desc; 697 i < ARRAY_SIZE(dpage->desc); 698 i++, desc++) { 699 dma_async_tx_descriptor_init(&desc->async_tx, dchan); 700 desc->async_tx.tx_submit = nbpf_tx_submit; 701 desc->chan = chan; 702 INIT_LIST_HEAD(&desc->sg); 703 list_add_tail(&desc->node, &head); 704 } 705 706 /* 707 * This function cannot be called from interrupt context, so, no need to 708 * save flags 709 */ 710 spin_lock_irq(&chan->lock); 711 list_splice_tail(&lhead, &chan->free_links); 712 list_splice_tail(&head, &chan->free); 713 list_add(&dpage->node, &chan->desc_page); 714 spin_unlock_irq(&chan->lock); 715 716 return ARRAY_SIZE(dpage->desc); 717 } 718 719 static void nbpf_desc_put(struct nbpf_desc *desc) 720 { 721 struct nbpf_channel *chan = desc->chan; 722 struct nbpf_link_desc *ldesc, *tmp; 723 unsigned long flags; 724 725 spin_lock_irqsave(&chan->lock, flags); 726 list_for_each_entry_safe(ldesc, tmp, &desc->sg, node) 727 list_move(&ldesc->node, &chan->free_links); 728 729 list_add(&desc->node, &chan->free); 730 spin_unlock_irqrestore(&chan->lock, flags); 731 } 732 733 static void nbpf_scan_acked(struct nbpf_channel *chan) 734 { 735 struct nbpf_desc *desc, *tmp; 736 unsigned long flags; 737 LIST_HEAD(head); 738 739 spin_lock_irqsave(&chan->lock, flags); 740 list_for_each_entry_safe(desc, tmp, &chan->done, node) 741 if (async_tx_test_ack(&desc->async_tx) && desc->user_wait) { 742 list_move(&desc->node, &head); 743 desc->user_wait = false; 744 } 745 spin_unlock_irqrestore(&chan->lock, flags); 746 747 list_for_each_entry_safe(desc, tmp, &head, node) { 748 list_del(&desc->node); 749 nbpf_desc_put(desc); 750 } 751 } 752 753 /* 754 * We have to allocate descriptors with the channel lock dropped. This means, 755 * before we re-acquire the lock buffers can be taken already, so we have to 756 * re-check after re-acquiring the lock and possibly retry, if buffers are gone 757 * again. 758 */ 759 static struct nbpf_desc *nbpf_desc_get(struct nbpf_channel *chan, size_t len) 760 { 761 struct nbpf_desc *desc = NULL; 762 struct nbpf_link_desc *ldesc, *prev = NULL; 763 764 nbpf_scan_acked(chan); 765 766 spin_lock_irq(&chan->lock); 767 768 do { 769 int i = 0, ret; 770 771 if (list_empty(&chan->free)) { 772 /* No more free descriptors */ 773 spin_unlock_irq(&chan->lock); 774 ret = nbpf_desc_page_alloc(chan); 775 if (ret < 0) 776 return NULL; 777 spin_lock_irq(&chan->lock); 778 continue; 779 } 780 desc = list_first_entry(&chan->free, struct nbpf_desc, node); 781 list_del(&desc->node); 782 783 do { 784 if (list_empty(&chan->free_links)) { 785 /* No more free link descriptors */ 786 spin_unlock_irq(&chan->lock); 787 ret = nbpf_desc_page_alloc(chan); 788 if (ret < 0) { 789 nbpf_desc_put(desc); 790 return NULL; 791 } 792 spin_lock_irq(&chan->lock); 793 continue; 794 } 795 796 ldesc = list_first_entry(&chan->free_links, 797 struct nbpf_link_desc, node); 798 ldesc->desc = desc; 799 if (prev) 800 prev->hwdesc->next = (u32)ldesc->hwdesc_dma_addr; 801 802 prev = ldesc; 803 list_move_tail(&ldesc->node, &desc->sg); 804 805 i++; 806 } while (i < len); 807 } while (!desc); 808 809 prev->hwdesc->next = 0; 810 811 spin_unlock_irq(&chan->lock); 812 813 return desc; 814 } 815 816 static void nbpf_chan_idle(struct nbpf_channel *chan) 817 { 818 struct nbpf_desc *desc, *tmp; 819 unsigned long flags; 820 LIST_HEAD(head); 821 822 spin_lock_irqsave(&chan->lock, flags); 823 824 list_splice_init(&chan->done, &head); 825 list_splice_init(&chan->active, &head); 826 list_splice_init(&chan->queued, &head); 827 828 chan->running = NULL; 829 830 spin_unlock_irqrestore(&chan->lock, flags); 831 832 list_for_each_entry_safe(desc, tmp, &head, node) { 833 dev_dbg(chan->nbpf->dma_dev.dev, "%s(): force-free desc %p cookie %d\n", 834 __func__, desc, desc->async_tx.cookie); 835 list_del(&desc->node); 836 nbpf_desc_put(desc); 837 } 838 } 839 840 static int nbpf_control(struct dma_chan *dchan, enum dma_ctrl_cmd cmd, 841 unsigned long arg) 842 { 843 struct nbpf_channel *chan = nbpf_to_chan(dchan); 844 struct dma_slave_config *config; 845 846 dev_dbg(dchan->device->dev, "Entry %s(%d)\n", __func__, cmd); 847 848 switch (cmd) { 849 case DMA_TERMINATE_ALL: 850 dev_dbg(dchan->device->dev, "Terminating\n"); 851 nbpf_chan_halt(chan); 852 nbpf_chan_idle(chan); 853 break; 854 855 case DMA_SLAVE_CONFIG: 856 if (!arg) 857 return -EINVAL; 858 config = (struct dma_slave_config *)arg; 859 860 /* 861 * We could check config->slave_id to match chan->terminal here, 862 * but with DT they would be coming from the same source, so 863 * such a check would be superflous 864 */ 865 866 chan->slave_dst_addr = config->dst_addr; 867 chan->slave_dst_width = nbpf_xfer_size(chan->nbpf, 868 config->dst_addr_width, 1); 869 chan->slave_dst_burst = nbpf_xfer_size(chan->nbpf, 870 config->dst_addr_width, 871 config->dst_maxburst); 872 chan->slave_src_addr = config->src_addr; 873 chan->slave_src_width = nbpf_xfer_size(chan->nbpf, 874 config->src_addr_width, 1); 875 chan->slave_src_burst = nbpf_xfer_size(chan->nbpf, 876 config->src_addr_width, 877 config->src_maxburst); 878 break; 879 880 case DMA_PAUSE: 881 chan->paused = true; 882 nbpf_pause(chan); 883 break; 884 885 default: 886 return -ENXIO; 887 } 888 889 return 0; 890 } 891 892 static struct dma_async_tx_descriptor *nbpf_prep_sg(struct nbpf_channel *chan, 893 struct scatterlist *src_sg, struct scatterlist *dst_sg, 894 size_t len, enum dma_transfer_direction direction, 895 unsigned long flags) 896 { 897 struct nbpf_link_desc *ldesc; 898 struct scatterlist *mem_sg; 899 struct nbpf_desc *desc; 900 bool inc_src, inc_dst; 901 size_t data_len = 0; 902 int i = 0; 903 904 switch (direction) { 905 case DMA_DEV_TO_MEM: 906 mem_sg = dst_sg; 907 inc_src = false; 908 inc_dst = true; 909 break; 910 911 case DMA_MEM_TO_DEV: 912 mem_sg = src_sg; 913 inc_src = true; 914 inc_dst = false; 915 break; 916 917 default: 918 case DMA_MEM_TO_MEM: 919 mem_sg = src_sg; 920 inc_src = true; 921 inc_dst = true; 922 } 923 924 desc = nbpf_desc_get(chan, len); 925 if (!desc) 926 return NULL; 927 928 desc->async_tx.flags = flags; 929 desc->async_tx.cookie = -EBUSY; 930 desc->user_wait = false; 931 932 /* 933 * This is a private descriptor list, and we own the descriptor. No need 934 * to lock. 935 */ 936 list_for_each_entry(ldesc, &desc->sg, node) { 937 int ret = nbpf_prep_one(ldesc, direction, 938 sg_dma_address(src_sg), 939 sg_dma_address(dst_sg), 940 sg_dma_len(mem_sg), 941 i == len - 1); 942 if (ret < 0) { 943 nbpf_desc_put(desc); 944 return NULL; 945 } 946 data_len += sg_dma_len(mem_sg); 947 if (inc_src) 948 src_sg = sg_next(src_sg); 949 if (inc_dst) 950 dst_sg = sg_next(dst_sg); 951 mem_sg = direction == DMA_DEV_TO_MEM ? dst_sg : src_sg; 952 i++; 953 } 954 955 desc->length = data_len; 956 957 /* The user has to return the descriptor to us ASAP via .tx_submit() */ 958 return &desc->async_tx; 959 } 960 961 static struct dma_async_tx_descriptor *nbpf_prep_memcpy( 962 struct dma_chan *dchan, dma_addr_t dst, dma_addr_t src, 963 size_t len, unsigned long flags) 964 { 965 struct nbpf_channel *chan = nbpf_to_chan(dchan); 966 struct scatterlist dst_sg; 967 struct scatterlist src_sg; 968 969 sg_init_table(&dst_sg, 1); 970 sg_init_table(&src_sg, 1); 971 972 sg_dma_address(&dst_sg) = dst; 973 sg_dma_address(&src_sg) = src; 974 975 sg_dma_len(&dst_sg) = len; 976 sg_dma_len(&src_sg) = len; 977 978 dev_dbg(dchan->device->dev, "%s(): %zu @ %pad -> %pad\n", 979 __func__, len, &src, &dst); 980 981 return nbpf_prep_sg(chan, &src_sg, &dst_sg, 1, 982 DMA_MEM_TO_MEM, flags); 983 } 984 985 static struct dma_async_tx_descriptor *nbpf_prep_memcpy_sg( 986 struct dma_chan *dchan, 987 struct scatterlist *dst_sg, unsigned int dst_nents, 988 struct scatterlist *src_sg, unsigned int src_nents, 989 unsigned long flags) 990 { 991 struct nbpf_channel *chan = nbpf_to_chan(dchan); 992 993 if (dst_nents != src_nents) 994 return NULL; 995 996 return nbpf_prep_sg(chan, src_sg, dst_sg, src_nents, 997 DMA_MEM_TO_MEM, flags); 998 } 999 1000 static struct dma_async_tx_descriptor *nbpf_prep_slave_sg( 1001 struct dma_chan *dchan, struct scatterlist *sgl, unsigned int sg_len, 1002 enum dma_transfer_direction direction, unsigned long flags, void *context) 1003 { 1004 struct nbpf_channel *chan = nbpf_to_chan(dchan); 1005 struct scatterlist slave_sg; 1006 1007 dev_dbg(dchan->device->dev, "Entry %s()\n", __func__); 1008 1009 sg_init_table(&slave_sg, 1); 1010 1011 switch (direction) { 1012 case DMA_MEM_TO_DEV: 1013 sg_dma_address(&slave_sg) = chan->slave_dst_addr; 1014 return nbpf_prep_sg(chan, sgl, &slave_sg, sg_len, 1015 direction, flags); 1016 1017 case DMA_DEV_TO_MEM: 1018 sg_dma_address(&slave_sg) = chan->slave_src_addr; 1019 return nbpf_prep_sg(chan, &slave_sg, sgl, sg_len, 1020 direction, flags); 1021 1022 default: 1023 return NULL; 1024 } 1025 } 1026 1027 static int nbpf_alloc_chan_resources(struct dma_chan *dchan) 1028 { 1029 struct nbpf_channel *chan = nbpf_to_chan(dchan); 1030 int ret; 1031 1032 INIT_LIST_HEAD(&chan->free); 1033 INIT_LIST_HEAD(&chan->free_links); 1034 INIT_LIST_HEAD(&chan->queued); 1035 INIT_LIST_HEAD(&chan->active); 1036 INIT_LIST_HEAD(&chan->done); 1037 1038 ret = nbpf_desc_page_alloc(chan); 1039 if (ret < 0) 1040 return ret; 1041 1042 dev_dbg(dchan->device->dev, "Entry %s(): terminal %u\n", __func__, 1043 chan->terminal); 1044 1045 nbpf_chan_configure(chan); 1046 1047 return ret; 1048 } 1049 1050 static void nbpf_free_chan_resources(struct dma_chan *dchan) 1051 { 1052 struct nbpf_channel *chan = nbpf_to_chan(dchan); 1053 struct nbpf_desc_page *dpage, *tmp; 1054 1055 dev_dbg(dchan->device->dev, "Entry %s()\n", __func__); 1056 1057 nbpf_chan_halt(chan); 1058 nbpf_chan_idle(chan); 1059 /* Clean up for if a channel is re-used for MEMCPY after slave DMA */ 1060 nbpf_chan_prepare_default(chan); 1061 1062 list_for_each_entry_safe(dpage, tmp, &chan->desc_page, node) { 1063 struct nbpf_link_desc *ldesc; 1064 int i; 1065 list_del(&dpage->node); 1066 for (i = 0, ldesc = dpage->ldesc; 1067 i < ARRAY_SIZE(dpage->ldesc); 1068 i++, ldesc++) 1069 dma_unmap_single(dchan->device->dev, ldesc->hwdesc_dma_addr, 1070 sizeof(*ldesc->hwdesc), DMA_TO_DEVICE); 1071 free_page((unsigned long)dpage); 1072 } 1073 } 1074 1075 static int nbpf_slave_caps(struct dma_chan *dchan, 1076 struct dma_slave_caps *caps) 1077 { 1078 caps->src_addr_widths = NBPF_DMA_BUSWIDTHS; 1079 caps->dstn_addr_widths = NBPF_DMA_BUSWIDTHS; 1080 caps->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); 1081 caps->cmd_pause = false; 1082 caps->cmd_terminate = true; 1083 1084 return 0; 1085 } 1086 1087 static struct dma_chan *nbpf_of_xlate(struct of_phandle_args *dma_spec, 1088 struct of_dma *ofdma) 1089 { 1090 struct nbpf_device *nbpf = ofdma->of_dma_data; 1091 struct dma_chan *dchan; 1092 struct nbpf_channel *chan; 1093 1094 if (dma_spec->args_count != 2) 1095 return NULL; 1096 1097 dchan = dma_get_any_slave_channel(&nbpf->dma_dev); 1098 if (!dchan) 1099 return NULL; 1100 1101 dev_dbg(dchan->device->dev, "Entry %s(%s)\n", __func__, 1102 dma_spec->np->name); 1103 1104 chan = nbpf_to_chan(dchan); 1105 1106 chan->terminal = dma_spec->args[0]; 1107 chan->flags = dma_spec->args[1]; 1108 1109 nbpf_chan_prepare(chan); 1110 nbpf_chan_configure(chan); 1111 1112 return dchan; 1113 } 1114 1115 static void nbpf_chan_tasklet(unsigned long data) 1116 { 1117 struct nbpf_channel *chan = (struct nbpf_channel *)data; 1118 struct nbpf_desc *desc, *tmp; 1119 dma_async_tx_callback callback; 1120 void *param; 1121 1122 while (!list_empty(&chan->done)) { 1123 bool found = false, must_put, recycling = false; 1124 1125 spin_lock_irq(&chan->lock); 1126 1127 list_for_each_entry_safe(desc, tmp, &chan->done, node) { 1128 if (!desc->user_wait) { 1129 /* Newly completed descriptor, have to process */ 1130 found = true; 1131 break; 1132 } else if (async_tx_test_ack(&desc->async_tx)) { 1133 /* 1134 * This descriptor was waiting for a user ACK, 1135 * it can be recycled now. 1136 */ 1137 list_del(&desc->node); 1138 spin_unlock_irq(&chan->lock); 1139 nbpf_desc_put(desc); 1140 recycling = true; 1141 break; 1142 } 1143 } 1144 1145 if (recycling) 1146 continue; 1147 1148 if (!found) { 1149 /* This can happen if TERMINATE_ALL has been called */ 1150 spin_unlock_irq(&chan->lock); 1151 break; 1152 } 1153 1154 dma_cookie_complete(&desc->async_tx); 1155 1156 /* 1157 * With released lock we cannot dereference desc, maybe it's 1158 * still on the "done" list 1159 */ 1160 if (async_tx_test_ack(&desc->async_tx)) { 1161 list_del(&desc->node); 1162 must_put = true; 1163 } else { 1164 desc->user_wait = true; 1165 must_put = false; 1166 } 1167 1168 callback = desc->async_tx.callback; 1169 param = desc->async_tx.callback_param; 1170 1171 /* ack and callback completed descriptor */ 1172 spin_unlock_irq(&chan->lock); 1173 1174 if (callback) 1175 callback(param); 1176 1177 if (must_put) 1178 nbpf_desc_put(desc); 1179 } 1180 } 1181 1182 static irqreturn_t nbpf_chan_irq(int irq, void *dev) 1183 { 1184 struct nbpf_channel *chan = dev; 1185 bool done = nbpf_status_get(chan); 1186 struct nbpf_desc *desc; 1187 irqreturn_t ret; 1188 bool bh = false; 1189 1190 if (!done) 1191 return IRQ_NONE; 1192 1193 nbpf_status_ack(chan); 1194 1195 dev_dbg(&chan->dma_chan.dev->device, "%s()\n", __func__); 1196 1197 spin_lock(&chan->lock); 1198 desc = chan->running; 1199 if (WARN_ON(!desc)) { 1200 ret = IRQ_NONE; 1201 goto unlock; 1202 } else { 1203 ret = IRQ_HANDLED; 1204 bh = true; 1205 } 1206 1207 list_move_tail(&desc->node, &chan->done); 1208 chan->running = NULL; 1209 1210 if (!list_empty(&chan->active)) { 1211 desc = list_first_entry(&chan->active, 1212 struct nbpf_desc, node); 1213 if (!nbpf_start(desc)) 1214 chan->running = desc; 1215 } 1216 1217 unlock: 1218 spin_unlock(&chan->lock); 1219 1220 if (bh) 1221 tasklet_schedule(&chan->tasklet); 1222 1223 return ret; 1224 } 1225 1226 static irqreturn_t nbpf_err_irq(int irq, void *dev) 1227 { 1228 struct nbpf_device *nbpf = dev; 1229 u32 error = nbpf_error_get(nbpf); 1230 1231 dev_warn(nbpf->dma_dev.dev, "DMA error IRQ %u\n", irq); 1232 1233 if (!error) 1234 return IRQ_NONE; 1235 1236 do { 1237 struct nbpf_channel *chan = nbpf_error_get_channel(nbpf, error); 1238 /* On error: abort all queued transfers, no callback */ 1239 nbpf_error_clear(chan); 1240 nbpf_chan_idle(chan); 1241 error = nbpf_error_get(nbpf); 1242 } while (error); 1243 1244 return IRQ_HANDLED; 1245 } 1246 1247 static int nbpf_chan_probe(struct nbpf_device *nbpf, int n) 1248 { 1249 struct dma_device *dma_dev = &nbpf->dma_dev; 1250 struct nbpf_channel *chan = nbpf->chan + n; 1251 int ret; 1252 1253 chan->nbpf = nbpf; 1254 chan->base = nbpf->base + NBPF_REG_CHAN_OFFSET + NBPF_REG_CHAN_SIZE * n; 1255 INIT_LIST_HEAD(&chan->desc_page); 1256 spin_lock_init(&chan->lock); 1257 chan->dma_chan.device = dma_dev; 1258 dma_cookie_init(&chan->dma_chan); 1259 nbpf_chan_prepare_default(chan); 1260 1261 dev_dbg(dma_dev->dev, "%s(): channel %d: -> %p\n", __func__, n, chan->base); 1262 1263 snprintf(chan->name, sizeof(chan->name), "nbpf %d", n); 1264 1265 tasklet_init(&chan->tasklet, nbpf_chan_tasklet, (unsigned long)chan); 1266 ret = devm_request_irq(dma_dev->dev, chan->irq, 1267 nbpf_chan_irq, IRQF_SHARED, 1268 chan->name, chan); 1269 if (ret < 0) 1270 return ret; 1271 1272 /* Add the channel to DMA device channel list */ 1273 list_add_tail(&chan->dma_chan.device_node, 1274 &dma_dev->channels); 1275 1276 return 0; 1277 } 1278 1279 static const struct of_device_id nbpf_match[] = { 1280 {.compatible = "renesas,nbpfaxi64dmac1b4", .data = &nbpf_cfg[NBPF1B4]}, 1281 {.compatible = "renesas,nbpfaxi64dmac1b8", .data = &nbpf_cfg[NBPF1B8]}, 1282 {.compatible = "renesas,nbpfaxi64dmac1b16", .data = &nbpf_cfg[NBPF1B16]}, 1283 {.compatible = "renesas,nbpfaxi64dmac4b4", .data = &nbpf_cfg[NBPF4B4]}, 1284 {.compatible = "renesas,nbpfaxi64dmac4b8", .data = &nbpf_cfg[NBPF4B8]}, 1285 {.compatible = "renesas,nbpfaxi64dmac4b16", .data = &nbpf_cfg[NBPF4B16]}, 1286 {.compatible = "renesas,nbpfaxi64dmac8b4", .data = &nbpf_cfg[NBPF8B4]}, 1287 {.compatible = "renesas,nbpfaxi64dmac8b8", .data = &nbpf_cfg[NBPF8B8]}, 1288 {.compatible = "renesas,nbpfaxi64dmac8b16", .data = &nbpf_cfg[NBPF8B16]}, 1289 {} 1290 }; 1291 MODULE_DEVICE_TABLE(of, nbpf_match); 1292 1293 static int nbpf_probe(struct platform_device *pdev) 1294 { 1295 struct device *dev = &pdev->dev; 1296 const struct of_device_id *of_id = of_match_device(nbpf_match, dev); 1297 struct device_node *np = dev->of_node; 1298 struct nbpf_device *nbpf; 1299 struct dma_device *dma_dev; 1300 struct resource *iomem, *irq_res; 1301 const struct nbpf_config *cfg; 1302 int num_channels; 1303 int ret, irq, eirq, i; 1304 int irqbuf[9] /* maximum 8 channels + error IRQ */; 1305 unsigned int irqs = 0; 1306 1307 BUILD_BUG_ON(sizeof(struct nbpf_desc_page) > PAGE_SIZE); 1308 1309 /* DT only */ 1310 if (!np || !of_id || !of_id->data) 1311 return -ENODEV; 1312 1313 cfg = of_id->data; 1314 num_channels = cfg->num_channels; 1315 1316 nbpf = devm_kzalloc(dev, sizeof(*nbpf) + num_channels * 1317 sizeof(nbpf->chan[0]), GFP_KERNEL); 1318 if (!nbpf) { 1319 dev_err(dev, "Memory allocation failed\n"); 1320 return -ENOMEM; 1321 } 1322 dma_dev = &nbpf->dma_dev; 1323 dma_dev->dev = dev; 1324 1325 iomem = platform_get_resource(pdev, IORESOURCE_MEM, 0); 1326 nbpf->base = devm_ioremap_resource(dev, iomem); 1327 if (IS_ERR(nbpf->base)) 1328 return PTR_ERR(nbpf->base); 1329 1330 nbpf->clk = devm_clk_get(dev, NULL); 1331 if (IS_ERR(nbpf->clk)) 1332 return PTR_ERR(nbpf->clk); 1333 1334 nbpf->config = cfg; 1335 1336 for (i = 0; irqs < ARRAY_SIZE(irqbuf); i++) { 1337 irq_res = platform_get_resource(pdev, IORESOURCE_IRQ, i); 1338 if (!irq_res) 1339 break; 1340 1341 for (irq = irq_res->start; irq <= irq_res->end; 1342 irq++, irqs++) 1343 irqbuf[irqs] = irq; 1344 } 1345 1346 /* 1347 * 3 IRQ resource schemes are supported: 1348 * 1. 1 shared IRQ for error and all channels 1349 * 2. 2 IRQs: one for error and one shared for all channels 1350 * 3. 1 IRQ for error and an own IRQ for each channel 1351 */ 1352 if (irqs != 1 && irqs != 2 && irqs != num_channels + 1) 1353 return -ENXIO; 1354 1355 if (irqs == 1) { 1356 eirq = irqbuf[0]; 1357 1358 for (i = 0; i <= num_channels; i++) 1359 nbpf->chan[i].irq = irqbuf[0]; 1360 } else { 1361 eirq = platform_get_irq_byname(pdev, "error"); 1362 if (eirq < 0) 1363 return eirq; 1364 1365 if (irqs == num_channels + 1) { 1366 struct nbpf_channel *chan; 1367 1368 for (i = 0, chan = nbpf->chan; i <= num_channels; 1369 i++, chan++) { 1370 /* Skip the error IRQ */ 1371 if (irqbuf[i] == eirq) 1372 i++; 1373 chan->irq = irqbuf[i]; 1374 } 1375 1376 if (chan != nbpf->chan + num_channels) 1377 return -EINVAL; 1378 } else { 1379 /* 2 IRQs and more than one channel */ 1380 if (irqbuf[0] == eirq) 1381 irq = irqbuf[1]; 1382 else 1383 irq = irqbuf[0]; 1384 1385 for (i = 0; i <= num_channels; i++) 1386 nbpf->chan[i].irq = irq; 1387 } 1388 } 1389 1390 ret = devm_request_irq(dev, eirq, nbpf_err_irq, 1391 IRQF_SHARED, "dma error", nbpf); 1392 if (ret < 0) 1393 return ret; 1394 1395 INIT_LIST_HEAD(&dma_dev->channels); 1396 1397 /* Create DMA Channel */ 1398 for (i = 0; i < num_channels; i++) { 1399 ret = nbpf_chan_probe(nbpf, i); 1400 if (ret < 0) 1401 return ret; 1402 } 1403 1404 dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask); 1405 dma_cap_set(DMA_SLAVE, dma_dev->cap_mask); 1406 dma_cap_set(DMA_PRIVATE, dma_dev->cap_mask); 1407 dma_cap_set(DMA_SG, dma_dev->cap_mask); 1408 1409 /* Common and MEMCPY operations */ 1410 dma_dev->device_alloc_chan_resources 1411 = nbpf_alloc_chan_resources; 1412 dma_dev->device_free_chan_resources = nbpf_free_chan_resources; 1413 dma_dev->device_prep_dma_sg = nbpf_prep_memcpy_sg; 1414 dma_dev->device_prep_dma_memcpy = nbpf_prep_memcpy; 1415 dma_dev->device_tx_status = nbpf_tx_status; 1416 dma_dev->device_issue_pending = nbpf_issue_pending; 1417 dma_dev->device_slave_caps = nbpf_slave_caps; 1418 1419 /* 1420 * If we drop support for unaligned MEMCPY buffer addresses and / or 1421 * lengths by setting 1422 * dma_dev->copy_align = 4; 1423 * then we can set transfer length to 4 bytes in nbpf_prep_one() for 1424 * DMA_MEM_TO_MEM 1425 */ 1426 1427 /* Compulsory for DMA_SLAVE fields */ 1428 dma_dev->device_prep_slave_sg = nbpf_prep_slave_sg; 1429 dma_dev->device_control = nbpf_control; 1430 1431 platform_set_drvdata(pdev, nbpf); 1432 1433 ret = clk_prepare_enable(nbpf->clk); 1434 if (ret < 0) 1435 return ret; 1436 1437 nbpf_configure(nbpf); 1438 1439 ret = dma_async_device_register(dma_dev); 1440 if (ret < 0) 1441 goto e_clk_off; 1442 1443 ret = of_dma_controller_register(np, nbpf_of_xlate, nbpf); 1444 if (ret < 0) 1445 goto e_dma_dev_unreg; 1446 1447 return 0; 1448 1449 e_dma_dev_unreg: 1450 dma_async_device_unregister(dma_dev); 1451 e_clk_off: 1452 clk_disable_unprepare(nbpf->clk); 1453 1454 return ret; 1455 } 1456 1457 static int nbpf_remove(struct platform_device *pdev) 1458 { 1459 struct nbpf_device *nbpf = platform_get_drvdata(pdev); 1460 1461 of_dma_controller_free(pdev->dev.of_node); 1462 dma_async_device_unregister(&nbpf->dma_dev); 1463 clk_disable_unprepare(nbpf->clk); 1464 1465 return 0; 1466 } 1467 1468 static struct platform_device_id nbpf_ids[] = { 1469 {"nbpfaxi64dmac1b4", (kernel_ulong_t)&nbpf_cfg[NBPF1B4]}, 1470 {"nbpfaxi64dmac1b8", (kernel_ulong_t)&nbpf_cfg[NBPF1B8]}, 1471 {"nbpfaxi64dmac1b16", (kernel_ulong_t)&nbpf_cfg[NBPF1B16]}, 1472 {"nbpfaxi64dmac4b4", (kernel_ulong_t)&nbpf_cfg[NBPF4B4]}, 1473 {"nbpfaxi64dmac4b8", (kernel_ulong_t)&nbpf_cfg[NBPF4B8]}, 1474 {"nbpfaxi64dmac4b16", (kernel_ulong_t)&nbpf_cfg[NBPF4B16]}, 1475 {"nbpfaxi64dmac8b4", (kernel_ulong_t)&nbpf_cfg[NBPF8B4]}, 1476 {"nbpfaxi64dmac8b8", (kernel_ulong_t)&nbpf_cfg[NBPF8B8]}, 1477 {"nbpfaxi64dmac8b16", (kernel_ulong_t)&nbpf_cfg[NBPF8B16]}, 1478 {}, 1479 }; 1480 MODULE_DEVICE_TABLE(platform, nbpf_ids); 1481 1482 #ifdef CONFIG_PM 1483 static int nbpf_runtime_suspend(struct device *dev) 1484 { 1485 struct nbpf_device *nbpf = platform_get_drvdata(to_platform_device(dev)); 1486 clk_disable_unprepare(nbpf->clk); 1487 return 0; 1488 } 1489 1490 static int nbpf_runtime_resume(struct device *dev) 1491 { 1492 struct nbpf_device *nbpf = platform_get_drvdata(to_platform_device(dev)); 1493 return clk_prepare_enable(nbpf->clk); 1494 } 1495 #endif 1496 1497 static const struct dev_pm_ops nbpf_pm_ops = { 1498 SET_RUNTIME_PM_OPS(nbpf_runtime_suspend, nbpf_runtime_resume, NULL) 1499 }; 1500 1501 static struct platform_driver nbpf_driver = { 1502 .driver = { 1503 .name = "dma-nbpf", 1504 .of_match_table = nbpf_match, 1505 .pm = &nbpf_pm_ops, 1506 }, 1507 .id_table = nbpf_ids, 1508 .probe = nbpf_probe, 1509 .remove = nbpf_remove, 1510 }; 1511 1512 module_platform_driver(nbpf_driver); 1513 1514 MODULE_AUTHOR("Guennadi Liakhovetski <g.liakhovetski@gmx.de>"); 1515 MODULE_DESCRIPTION("dmaengine driver for NBPFAXI64* DMACs"); 1516 MODULE_LICENSE("GPL v2"); 1517