1 /* 2 * Copyright (C) Ericsson AB 2007-2008 3 * Copyright (C) ST-Ericsson SA 2008-2010 4 * Author: Per Forlin <per.forlin@stericsson.com> for ST-Ericsson 5 * Author: Jonas Aaberg <jonas.aberg@stericsson.com> for ST-Ericsson 6 * License terms: GNU General Public License (GPL) version 2 7 */ 8 9 #include <linux/dma-mapping.h> 10 #include <linux/kernel.h> 11 #include <linux/slab.h> 12 #include <linux/dmaengine.h> 13 #include <linux/platform_device.h> 14 #include <linux/clk.h> 15 #include <linux/delay.h> 16 #include <linux/err.h> 17 #include <linux/amba/bus.h> 18 19 #include <plat/ste_dma40.h> 20 21 #include "ste_dma40_ll.h" 22 23 #define D40_NAME "dma40" 24 25 #define D40_PHY_CHAN -1 26 27 /* For masking out/in 2 bit channel positions */ 28 #define D40_CHAN_POS(chan) (2 * (chan / 2)) 29 #define D40_CHAN_POS_MASK(chan) (0x3 << D40_CHAN_POS(chan)) 30 31 /* Maximum iterations taken before giving up suspending a channel */ 32 #define D40_SUSPEND_MAX_IT 500 33 34 /* Hardware requirement on LCLA alignment */ 35 #define LCLA_ALIGNMENT 0x40000 36 37 /* Max number of links per event group */ 38 #define D40_LCLA_LINK_PER_EVENT_GRP 128 39 #define D40_LCLA_END D40_LCLA_LINK_PER_EVENT_GRP 40 41 /* Attempts before giving up to trying to get pages that are aligned */ 42 #define MAX_LCLA_ALLOC_ATTEMPTS 256 43 44 /* Bit markings for allocation map */ 45 #define D40_ALLOC_FREE (1 << 31) 46 #define D40_ALLOC_PHY (1 << 30) 47 #define D40_ALLOC_LOG_FREE 0 48 49 /** 50 * enum 40_command - The different commands and/or statuses. 51 * 52 * @D40_DMA_STOP: DMA channel command STOP or status STOPPED, 53 * @D40_DMA_RUN: The DMA channel is RUNNING of the command RUN. 54 * @D40_DMA_SUSPEND_REQ: Request the DMA to SUSPEND as soon as possible. 55 * @D40_DMA_SUSPENDED: The DMA channel is SUSPENDED. 56 */ 57 enum d40_command { 58 D40_DMA_STOP = 0, 59 D40_DMA_RUN = 1, 60 D40_DMA_SUSPEND_REQ = 2, 61 D40_DMA_SUSPENDED = 3 62 }; 63 64 /** 65 * struct d40_lli_pool - Structure for keeping LLIs in memory 66 * 67 * @base: Pointer to memory area when the pre_alloc_lli's are not large 68 * enough, IE bigger than the most common case, 1 dst and 1 src. NULL if 69 * pre_alloc_lli is used. 70 * @dma_addr: DMA address, if mapped 71 * @size: The size in bytes of the memory at base or the size of pre_alloc_lli. 72 * @pre_alloc_lli: Pre allocated area for the most common case of transfers, 73 * one buffer to one buffer. 74 */ 75 struct d40_lli_pool { 76 void *base; 77 int size; 78 dma_addr_t dma_addr; 79 /* Space for dst and src, plus an extra for padding */ 80 u8 pre_alloc_lli[3 * sizeof(struct d40_phy_lli)]; 81 }; 82 83 /** 84 * struct d40_desc - A descriptor is one DMA job. 85 * 86 * @lli_phy: LLI settings for physical channel. Both src and dst= 87 * points into the lli_pool, to base if lli_len > 1 or to pre_alloc_lli if 88 * lli_len equals one. 89 * @lli_log: Same as above but for logical channels. 90 * @lli_pool: The pool with two entries pre-allocated. 91 * @lli_len: Number of llis of current descriptor. 92 * @lli_current: Number of transferred llis. 93 * @lcla_alloc: Number of LCLA entries allocated. 94 * @txd: DMA engine struct. Used for among other things for communication 95 * during a transfer. 96 * @node: List entry. 97 * @is_in_client_list: true if the client owns this descriptor. 98 * the previous one. 99 * 100 * This descriptor is used for both logical and physical transfers. 101 */ 102 struct d40_desc { 103 /* LLI physical */ 104 struct d40_phy_lli_bidir lli_phy; 105 /* LLI logical */ 106 struct d40_log_lli_bidir lli_log; 107 108 struct d40_lli_pool lli_pool; 109 int lli_len; 110 int lli_current; 111 int lcla_alloc; 112 113 struct dma_async_tx_descriptor txd; 114 struct list_head node; 115 116 bool is_in_client_list; 117 bool cyclic; 118 }; 119 120 /** 121 * struct d40_lcla_pool - LCLA pool settings and data. 122 * 123 * @base: The virtual address of LCLA. 18 bit aligned. 124 * @base_unaligned: The orignal kmalloc pointer, if kmalloc is used. 125 * This pointer is only there for clean-up on error. 126 * @pages: The number of pages needed for all physical channels. 127 * Only used later for clean-up on error 128 * @lock: Lock to protect the content in this struct. 129 * @alloc_map: big map over which LCLA entry is own by which job. 130 */ 131 struct d40_lcla_pool { 132 void *base; 133 dma_addr_t dma_addr; 134 void *base_unaligned; 135 int pages; 136 spinlock_t lock; 137 struct d40_desc **alloc_map; 138 }; 139 140 /** 141 * struct d40_phy_res - struct for handling eventlines mapped to physical 142 * channels. 143 * 144 * @lock: A lock protection this entity. 145 * @num: The physical channel number of this entity. 146 * @allocated_src: Bit mapped to show which src event line's are mapped to 147 * this physical channel. Can also be free or physically allocated. 148 * @allocated_dst: Same as for src but is dst. 149 * allocated_dst and allocated_src uses the D40_ALLOC* defines as well as 150 * event line number. 151 */ 152 struct d40_phy_res { 153 spinlock_t lock; 154 int num; 155 u32 allocated_src; 156 u32 allocated_dst; 157 }; 158 159 struct d40_base; 160 161 /** 162 * struct d40_chan - Struct that describes a channel. 163 * 164 * @lock: A spinlock to protect this struct. 165 * @log_num: The logical number, if any of this channel. 166 * @completed: Starts with 1, after first interrupt it is set to dma engine's 167 * current cookie. 168 * @pending_tx: The number of pending transfers. Used between interrupt handler 169 * and tasklet. 170 * @busy: Set to true when transfer is ongoing on this channel. 171 * @phy_chan: Pointer to physical channel which this instance runs on. If this 172 * point is NULL, then the channel is not allocated. 173 * @chan: DMA engine handle. 174 * @tasklet: Tasklet that gets scheduled from interrupt context to complete a 175 * transfer and call client callback. 176 * @client: Cliented owned descriptor list. 177 * @pending_queue: Submitted jobs, to be issued by issue_pending() 178 * @active: Active descriptor. 179 * @queue: Queued jobs. 180 * @prepare_queue: Prepared jobs. 181 * @dma_cfg: The client configuration of this dma channel. 182 * @configured: whether the dma_cfg configuration is valid 183 * @base: Pointer to the device instance struct. 184 * @src_def_cfg: Default cfg register setting for src. 185 * @dst_def_cfg: Default cfg register setting for dst. 186 * @log_def: Default logical channel settings. 187 * @lcla: Space for one dst src pair for logical channel transfers. 188 * @lcpa: Pointer to dst and src lcpa settings. 189 * @runtime_addr: runtime configured address. 190 * @runtime_direction: runtime configured direction. 191 * 192 * This struct can either "be" a logical or a physical channel. 193 */ 194 struct d40_chan { 195 spinlock_t lock; 196 int log_num; 197 /* ID of the most recent completed transfer */ 198 int completed; 199 int pending_tx; 200 bool busy; 201 struct d40_phy_res *phy_chan; 202 struct dma_chan chan; 203 struct tasklet_struct tasklet; 204 struct list_head client; 205 struct list_head pending_queue; 206 struct list_head active; 207 struct list_head queue; 208 struct list_head prepare_queue; 209 struct stedma40_chan_cfg dma_cfg; 210 bool configured; 211 struct d40_base *base; 212 /* Default register configurations */ 213 u32 src_def_cfg; 214 u32 dst_def_cfg; 215 struct d40_def_lcsp log_def; 216 struct d40_log_lli_full *lcpa; 217 /* Runtime reconfiguration */ 218 dma_addr_t runtime_addr; 219 enum dma_data_direction runtime_direction; 220 }; 221 222 /** 223 * struct d40_base - The big global struct, one for each probe'd instance. 224 * 225 * @interrupt_lock: Lock used to make sure one interrupt is handle a time. 226 * @execmd_lock: Lock for execute command usage since several channels share 227 * the same physical register. 228 * @dev: The device structure. 229 * @virtbase: The virtual base address of the DMA's register. 230 * @rev: silicon revision detected. 231 * @clk: Pointer to the DMA clock structure. 232 * @phy_start: Physical memory start of the DMA registers. 233 * @phy_size: Size of the DMA register map. 234 * @irq: The IRQ number. 235 * @num_phy_chans: The number of physical channels. Read from HW. This 236 * is the number of available channels for this driver, not counting "Secure 237 * mode" allocated physical channels. 238 * @num_log_chans: The number of logical channels. Calculated from 239 * num_phy_chans. 240 * @dma_both: dma_device channels that can do both memcpy and slave transfers. 241 * @dma_slave: dma_device channels that can do only do slave transfers. 242 * @dma_memcpy: dma_device channels that can do only do memcpy transfers. 243 * @log_chans: Room for all possible logical channels in system. 244 * @lookup_log_chans: Used to map interrupt number to logical channel. Points 245 * to log_chans entries. 246 * @lookup_phy_chans: Used to map interrupt number to physical channel. Points 247 * to phy_chans entries. 248 * @plat_data: Pointer to provided platform_data which is the driver 249 * configuration. 250 * @phy_res: Vector containing all physical channels. 251 * @lcla_pool: lcla pool settings and data. 252 * @lcpa_base: The virtual mapped address of LCPA. 253 * @phy_lcpa: The physical address of the LCPA. 254 * @lcpa_size: The size of the LCPA area. 255 * @desc_slab: cache for descriptors. 256 */ 257 struct d40_base { 258 spinlock_t interrupt_lock; 259 spinlock_t execmd_lock; 260 struct device *dev; 261 void __iomem *virtbase; 262 u8 rev:4; 263 struct clk *clk; 264 phys_addr_t phy_start; 265 resource_size_t phy_size; 266 int irq; 267 int num_phy_chans; 268 int num_log_chans; 269 struct dma_device dma_both; 270 struct dma_device dma_slave; 271 struct dma_device dma_memcpy; 272 struct d40_chan *phy_chans; 273 struct d40_chan *log_chans; 274 struct d40_chan **lookup_log_chans; 275 struct d40_chan **lookup_phy_chans; 276 struct stedma40_platform_data *plat_data; 277 /* Physical half channels */ 278 struct d40_phy_res *phy_res; 279 struct d40_lcla_pool lcla_pool; 280 void *lcpa_base; 281 dma_addr_t phy_lcpa; 282 resource_size_t lcpa_size; 283 struct kmem_cache *desc_slab; 284 }; 285 286 /** 287 * struct d40_interrupt_lookup - lookup table for interrupt handler 288 * 289 * @src: Interrupt mask register. 290 * @clr: Interrupt clear register. 291 * @is_error: true if this is an error interrupt. 292 * @offset: start delta in the lookup_log_chans in d40_base. If equals to 293 * D40_PHY_CHAN, the lookup_phy_chans shall be used instead. 294 */ 295 struct d40_interrupt_lookup { 296 u32 src; 297 u32 clr; 298 bool is_error; 299 int offset; 300 }; 301 302 /** 303 * struct d40_reg_val - simple lookup struct 304 * 305 * @reg: The register. 306 * @val: The value that belongs to the register in reg. 307 */ 308 struct d40_reg_val { 309 unsigned int reg; 310 unsigned int val; 311 }; 312 313 static struct device *chan2dev(struct d40_chan *d40c) 314 { 315 return &d40c->chan.dev->device; 316 } 317 318 static bool chan_is_physical(struct d40_chan *chan) 319 { 320 return chan->log_num == D40_PHY_CHAN; 321 } 322 323 static bool chan_is_logical(struct d40_chan *chan) 324 { 325 return !chan_is_physical(chan); 326 } 327 328 static void __iomem *chan_base(struct d40_chan *chan) 329 { 330 return chan->base->virtbase + D40_DREG_PCBASE + 331 chan->phy_chan->num * D40_DREG_PCDELTA; 332 } 333 334 #define d40_err(dev, format, arg...) \ 335 dev_err(dev, "[%s] " format, __func__, ## arg) 336 337 #define chan_err(d40c, format, arg...) \ 338 d40_err(chan2dev(d40c), format, ## arg) 339 340 static int d40_pool_lli_alloc(struct d40_chan *d40c, struct d40_desc *d40d, 341 int lli_len) 342 { 343 bool is_log = chan_is_logical(d40c); 344 u32 align; 345 void *base; 346 347 if (is_log) 348 align = sizeof(struct d40_log_lli); 349 else 350 align = sizeof(struct d40_phy_lli); 351 352 if (lli_len == 1) { 353 base = d40d->lli_pool.pre_alloc_lli; 354 d40d->lli_pool.size = sizeof(d40d->lli_pool.pre_alloc_lli); 355 d40d->lli_pool.base = NULL; 356 } else { 357 d40d->lli_pool.size = lli_len * 2 * align; 358 359 base = kmalloc(d40d->lli_pool.size + align, GFP_NOWAIT); 360 d40d->lli_pool.base = base; 361 362 if (d40d->lli_pool.base == NULL) 363 return -ENOMEM; 364 } 365 366 if (is_log) { 367 d40d->lli_log.src = PTR_ALIGN(base, align); 368 d40d->lli_log.dst = d40d->lli_log.src + lli_len; 369 370 d40d->lli_pool.dma_addr = 0; 371 } else { 372 d40d->lli_phy.src = PTR_ALIGN(base, align); 373 d40d->lli_phy.dst = d40d->lli_phy.src + lli_len; 374 375 d40d->lli_pool.dma_addr = dma_map_single(d40c->base->dev, 376 d40d->lli_phy.src, 377 d40d->lli_pool.size, 378 DMA_TO_DEVICE); 379 380 if (dma_mapping_error(d40c->base->dev, 381 d40d->lli_pool.dma_addr)) { 382 kfree(d40d->lli_pool.base); 383 d40d->lli_pool.base = NULL; 384 d40d->lli_pool.dma_addr = 0; 385 return -ENOMEM; 386 } 387 } 388 389 return 0; 390 } 391 392 static void d40_pool_lli_free(struct d40_chan *d40c, struct d40_desc *d40d) 393 { 394 if (d40d->lli_pool.dma_addr) 395 dma_unmap_single(d40c->base->dev, d40d->lli_pool.dma_addr, 396 d40d->lli_pool.size, DMA_TO_DEVICE); 397 398 kfree(d40d->lli_pool.base); 399 d40d->lli_pool.base = NULL; 400 d40d->lli_pool.size = 0; 401 d40d->lli_log.src = NULL; 402 d40d->lli_log.dst = NULL; 403 d40d->lli_phy.src = NULL; 404 d40d->lli_phy.dst = NULL; 405 } 406 407 static int d40_lcla_alloc_one(struct d40_chan *d40c, 408 struct d40_desc *d40d) 409 { 410 unsigned long flags; 411 int i; 412 int ret = -EINVAL; 413 int p; 414 415 spin_lock_irqsave(&d40c->base->lcla_pool.lock, flags); 416 417 p = d40c->phy_chan->num * D40_LCLA_LINK_PER_EVENT_GRP; 418 419 /* 420 * Allocate both src and dst at the same time, therefore the half 421 * start on 1 since 0 can't be used since zero is used as end marker. 422 */ 423 for (i = 1 ; i < D40_LCLA_LINK_PER_EVENT_GRP / 2; i++) { 424 if (!d40c->base->lcla_pool.alloc_map[p + i]) { 425 d40c->base->lcla_pool.alloc_map[p + i] = d40d; 426 d40d->lcla_alloc++; 427 ret = i; 428 break; 429 } 430 } 431 432 spin_unlock_irqrestore(&d40c->base->lcla_pool.lock, flags); 433 434 return ret; 435 } 436 437 static int d40_lcla_free_all(struct d40_chan *d40c, 438 struct d40_desc *d40d) 439 { 440 unsigned long flags; 441 int i; 442 int ret = -EINVAL; 443 444 if (chan_is_physical(d40c)) 445 return 0; 446 447 spin_lock_irqsave(&d40c->base->lcla_pool.lock, flags); 448 449 for (i = 1 ; i < D40_LCLA_LINK_PER_EVENT_GRP / 2; i++) { 450 if (d40c->base->lcla_pool.alloc_map[d40c->phy_chan->num * 451 D40_LCLA_LINK_PER_EVENT_GRP + i] == d40d) { 452 d40c->base->lcla_pool.alloc_map[d40c->phy_chan->num * 453 D40_LCLA_LINK_PER_EVENT_GRP + i] = NULL; 454 d40d->lcla_alloc--; 455 if (d40d->lcla_alloc == 0) { 456 ret = 0; 457 break; 458 } 459 } 460 } 461 462 spin_unlock_irqrestore(&d40c->base->lcla_pool.lock, flags); 463 464 return ret; 465 466 } 467 468 static void d40_desc_remove(struct d40_desc *d40d) 469 { 470 list_del(&d40d->node); 471 } 472 473 static struct d40_desc *d40_desc_get(struct d40_chan *d40c) 474 { 475 struct d40_desc *desc = NULL; 476 477 if (!list_empty(&d40c->client)) { 478 struct d40_desc *d; 479 struct d40_desc *_d; 480 481 list_for_each_entry_safe(d, _d, &d40c->client, node) 482 if (async_tx_test_ack(&d->txd)) { 483 d40_desc_remove(d); 484 desc = d; 485 memset(desc, 0, sizeof(*desc)); 486 break; 487 } 488 } 489 490 if (!desc) 491 desc = kmem_cache_zalloc(d40c->base->desc_slab, GFP_NOWAIT); 492 493 if (desc) 494 INIT_LIST_HEAD(&desc->node); 495 496 return desc; 497 } 498 499 static void d40_desc_free(struct d40_chan *d40c, struct d40_desc *d40d) 500 { 501 502 d40_pool_lli_free(d40c, d40d); 503 d40_lcla_free_all(d40c, d40d); 504 kmem_cache_free(d40c->base->desc_slab, d40d); 505 } 506 507 static void d40_desc_submit(struct d40_chan *d40c, struct d40_desc *desc) 508 { 509 list_add_tail(&desc->node, &d40c->active); 510 } 511 512 static void d40_phy_lli_load(struct d40_chan *chan, struct d40_desc *desc) 513 { 514 struct d40_phy_lli *lli_dst = desc->lli_phy.dst; 515 struct d40_phy_lli *lli_src = desc->lli_phy.src; 516 void __iomem *base = chan_base(chan); 517 518 writel(lli_src->reg_cfg, base + D40_CHAN_REG_SSCFG); 519 writel(lli_src->reg_elt, base + D40_CHAN_REG_SSELT); 520 writel(lli_src->reg_ptr, base + D40_CHAN_REG_SSPTR); 521 writel(lli_src->reg_lnk, base + D40_CHAN_REG_SSLNK); 522 523 writel(lli_dst->reg_cfg, base + D40_CHAN_REG_SDCFG); 524 writel(lli_dst->reg_elt, base + D40_CHAN_REG_SDELT); 525 writel(lli_dst->reg_ptr, base + D40_CHAN_REG_SDPTR); 526 writel(lli_dst->reg_lnk, base + D40_CHAN_REG_SDLNK); 527 } 528 529 static void d40_log_lli_to_lcxa(struct d40_chan *chan, struct d40_desc *desc) 530 { 531 struct d40_lcla_pool *pool = &chan->base->lcla_pool; 532 struct d40_log_lli_bidir *lli = &desc->lli_log; 533 int lli_current = desc->lli_current; 534 int lli_len = desc->lli_len; 535 bool cyclic = desc->cyclic; 536 int curr_lcla = -EINVAL; 537 int first_lcla = 0; 538 bool linkback; 539 540 /* 541 * We may have partially running cyclic transfers, in case we did't get 542 * enough LCLA entries. 543 */ 544 linkback = cyclic && lli_current == 0; 545 546 /* 547 * For linkback, we need one LCLA even with only one link, because we 548 * can't link back to the one in LCPA space 549 */ 550 if (linkback || (lli_len - lli_current > 1)) { 551 curr_lcla = d40_lcla_alloc_one(chan, desc); 552 first_lcla = curr_lcla; 553 } 554 555 /* 556 * For linkback, we normally load the LCPA in the loop since we need to 557 * link it to the second LCLA and not the first. However, if we 558 * couldn't even get a first LCLA, then we have to run in LCPA and 559 * reload manually. 560 */ 561 if (!linkback || curr_lcla == -EINVAL) { 562 unsigned int flags = 0; 563 564 if (curr_lcla == -EINVAL) 565 flags |= LLI_TERM_INT; 566 567 d40_log_lli_lcpa_write(chan->lcpa, 568 &lli->dst[lli_current], 569 &lli->src[lli_current], 570 curr_lcla, 571 flags); 572 lli_current++; 573 } 574 575 if (curr_lcla < 0) 576 goto out; 577 578 for (; lli_current < lli_len; lli_current++) { 579 unsigned int lcla_offset = chan->phy_chan->num * 1024 + 580 8 * curr_lcla * 2; 581 struct d40_log_lli *lcla = pool->base + lcla_offset; 582 unsigned int flags = 0; 583 int next_lcla; 584 585 if (lli_current + 1 < lli_len) 586 next_lcla = d40_lcla_alloc_one(chan, desc); 587 else 588 next_lcla = linkback ? first_lcla : -EINVAL; 589 590 if (cyclic || next_lcla == -EINVAL) 591 flags |= LLI_TERM_INT; 592 593 if (linkback && curr_lcla == first_lcla) { 594 /* First link goes in both LCPA and LCLA */ 595 d40_log_lli_lcpa_write(chan->lcpa, 596 &lli->dst[lli_current], 597 &lli->src[lli_current], 598 next_lcla, flags); 599 } 600 601 /* 602 * One unused LCLA in the cyclic case if the very first 603 * next_lcla fails... 604 */ 605 d40_log_lli_lcla_write(lcla, 606 &lli->dst[lli_current], 607 &lli->src[lli_current], 608 next_lcla, flags); 609 610 dma_sync_single_range_for_device(chan->base->dev, 611 pool->dma_addr, lcla_offset, 612 2 * sizeof(struct d40_log_lli), 613 DMA_TO_DEVICE); 614 615 curr_lcla = next_lcla; 616 617 if (curr_lcla == -EINVAL || curr_lcla == first_lcla) { 618 lli_current++; 619 break; 620 } 621 } 622 623 out: 624 desc->lli_current = lli_current; 625 } 626 627 static void d40_desc_load(struct d40_chan *d40c, struct d40_desc *d40d) 628 { 629 if (chan_is_physical(d40c)) { 630 d40_phy_lli_load(d40c, d40d); 631 d40d->lli_current = d40d->lli_len; 632 } else 633 d40_log_lli_to_lcxa(d40c, d40d); 634 } 635 636 static struct d40_desc *d40_first_active_get(struct d40_chan *d40c) 637 { 638 struct d40_desc *d; 639 640 if (list_empty(&d40c->active)) 641 return NULL; 642 643 d = list_first_entry(&d40c->active, 644 struct d40_desc, 645 node); 646 return d; 647 } 648 649 /* remove desc from current queue and add it to the pending_queue */ 650 static void d40_desc_queue(struct d40_chan *d40c, struct d40_desc *desc) 651 { 652 d40_desc_remove(desc); 653 desc->is_in_client_list = false; 654 list_add_tail(&desc->node, &d40c->pending_queue); 655 } 656 657 static struct d40_desc *d40_first_pending(struct d40_chan *d40c) 658 { 659 struct d40_desc *d; 660 661 if (list_empty(&d40c->pending_queue)) 662 return NULL; 663 664 d = list_first_entry(&d40c->pending_queue, 665 struct d40_desc, 666 node); 667 return d; 668 } 669 670 static struct d40_desc *d40_first_queued(struct d40_chan *d40c) 671 { 672 struct d40_desc *d; 673 674 if (list_empty(&d40c->queue)) 675 return NULL; 676 677 d = list_first_entry(&d40c->queue, 678 struct d40_desc, 679 node); 680 return d; 681 } 682 683 static int d40_psize_2_burst_size(bool is_log, int psize) 684 { 685 if (is_log) { 686 if (psize == STEDMA40_PSIZE_LOG_1) 687 return 1; 688 } else { 689 if (psize == STEDMA40_PSIZE_PHY_1) 690 return 1; 691 } 692 693 return 2 << psize; 694 } 695 696 /* 697 * The dma only supports transmitting packages up to 698 * STEDMA40_MAX_SEG_SIZE << data_width. Calculate the total number of 699 * dma elements required to send the entire sg list 700 */ 701 static int d40_size_2_dmalen(int size, u32 data_width1, u32 data_width2) 702 { 703 int dmalen; 704 u32 max_w = max(data_width1, data_width2); 705 u32 min_w = min(data_width1, data_width2); 706 u32 seg_max = ALIGN(STEDMA40_MAX_SEG_SIZE << min_w, 1 << max_w); 707 708 if (seg_max > STEDMA40_MAX_SEG_SIZE) 709 seg_max -= (1 << max_w); 710 711 if (!IS_ALIGNED(size, 1 << max_w)) 712 return -EINVAL; 713 714 if (size <= seg_max) 715 dmalen = 1; 716 else { 717 dmalen = size / seg_max; 718 if (dmalen * seg_max < size) 719 dmalen++; 720 } 721 return dmalen; 722 } 723 724 static int d40_sg_2_dmalen(struct scatterlist *sgl, int sg_len, 725 u32 data_width1, u32 data_width2) 726 { 727 struct scatterlist *sg; 728 int i; 729 int len = 0; 730 int ret; 731 732 for_each_sg(sgl, sg, sg_len, i) { 733 ret = d40_size_2_dmalen(sg_dma_len(sg), 734 data_width1, data_width2); 735 if (ret < 0) 736 return ret; 737 len += ret; 738 } 739 return len; 740 } 741 742 /* Support functions for logical channels */ 743 744 static int d40_channel_execute_command(struct d40_chan *d40c, 745 enum d40_command command) 746 { 747 u32 status; 748 int i; 749 void __iomem *active_reg; 750 int ret = 0; 751 unsigned long flags; 752 u32 wmask; 753 754 spin_lock_irqsave(&d40c->base->execmd_lock, flags); 755 756 if (d40c->phy_chan->num % 2 == 0) 757 active_reg = d40c->base->virtbase + D40_DREG_ACTIVE; 758 else 759 active_reg = d40c->base->virtbase + D40_DREG_ACTIVO; 760 761 if (command == D40_DMA_SUSPEND_REQ) { 762 status = (readl(active_reg) & 763 D40_CHAN_POS_MASK(d40c->phy_chan->num)) >> 764 D40_CHAN_POS(d40c->phy_chan->num); 765 766 if (status == D40_DMA_SUSPENDED || status == D40_DMA_STOP) 767 goto done; 768 } 769 770 wmask = 0xffffffff & ~(D40_CHAN_POS_MASK(d40c->phy_chan->num)); 771 writel(wmask | (command << D40_CHAN_POS(d40c->phy_chan->num)), 772 active_reg); 773 774 if (command == D40_DMA_SUSPEND_REQ) { 775 776 for (i = 0 ; i < D40_SUSPEND_MAX_IT; i++) { 777 status = (readl(active_reg) & 778 D40_CHAN_POS_MASK(d40c->phy_chan->num)) >> 779 D40_CHAN_POS(d40c->phy_chan->num); 780 781 cpu_relax(); 782 /* 783 * Reduce the number of bus accesses while 784 * waiting for the DMA to suspend. 785 */ 786 udelay(3); 787 788 if (status == D40_DMA_STOP || 789 status == D40_DMA_SUSPENDED) 790 break; 791 } 792 793 if (i == D40_SUSPEND_MAX_IT) { 794 chan_err(d40c, 795 "unable to suspend the chl %d (log: %d) status %x\n", 796 d40c->phy_chan->num, d40c->log_num, 797 status); 798 dump_stack(); 799 ret = -EBUSY; 800 } 801 802 } 803 done: 804 spin_unlock_irqrestore(&d40c->base->execmd_lock, flags); 805 return ret; 806 } 807 808 static void d40_term_all(struct d40_chan *d40c) 809 { 810 struct d40_desc *d40d; 811 struct d40_desc *_d; 812 813 /* Release active descriptors */ 814 while ((d40d = d40_first_active_get(d40c))) { 815 d40_desc_remove(d40d); 816 d40_desc_free(d40c, d40d); 817 } 818 819 /* Release queued descriptors waiting for transfer */ 820 while ((d40d = d40_first_queued(d40c))) { 821 d40_desc_remove(d40d); 822 d40_desc_free(d40c, d40d); 823 } 824 825 /* Release pending descriptors */ 826 while ((d40d = d40_first_pending(d40c))) { 827 d40_desc_remove(d40d); 828 d40_desc_free(d40c, d40d); 829 } 830 831 /* Release client owned descriptors */ 832 if (!list_empty(&d40c->client)) 833 list_for_each_entry_safe(d40d, _d, &d40c->client, node) { 834 d40_desc_remove(d40d); 835 d40_desc_free(d40c, d40d); 836 } 837 838 /* Release descriptors in prepare queue */ 839 if (!list_empty(&d40c->prepare_queue)) 840 list_for_each_entry_safe(d40d, _d, 841 &d40c->prepare_queue, node) { 842 d40_desc_remove(d40d); 843 d40_desc_free(d40c, d40d); 844 } 845 846 d40c->pending_tx = 0; 847 d40c->busy = false; 848 } 849 850 static void __d40_config_set_event(struct d40_chan *d40c, bool enable, 851 u32 event, int reg) 852 { 853 void __iomem *addr = chan_base(d40c) + reg; 854 int tries; 855 856 if (!enable) { 857 writel((D40_DEACTIVATE_EVENTLINE << D40_EVENTLINE_POS(event)) 858 | ~D40_EVENTLINE_MASK(event), addr); 859 return; 860 } 861 862 /* 863 * The hardware sometimes doesn't register the enable when src and dst 864 * event lines are active on the same logical channel. Retry to ensure 865 * it does. Usually only one retry is sufficient. 866 */ 867 tries = 100; 868 while (--tries) { 869 writel((D40_ACTIVATE_EVENTLINE << D40_EVENTLINE_POS(event)) 870 | ~D40_EVENTLINE_MASK(event), addr); 871 872 if (readl(addr) & D40_EVENTLINE_MASK(event)) 873 break; 874 } 875 876 if (tries != 99) 877 dev_dbg(chan2dev(d40c), 878 "[%s] workaround enable S%cLNK (%d tries)\n", 879 __func__, reg == D40_CHAN_REG_SSLNK ? 'S' : 'D', 880 100 - tries); 881 882 WARN_ON(!tries); 883 } 884 885 static void d40_config_set_event(struct d40_chan *d40c, bool do_enable) 886 { 887 unsigned long flags; 888 889 spin_lock_irqsave(&d40c->phy_chan->lock, flags); 890 891 /* Enable event line connected to device (or memcpy) */ 892 if ((d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM) || 893 (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_PERIPH)) { 894 u32 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.src_dev_type); 895 896 __d40_config_set_event(d40c, do_enable, event, 897 D40_CHAN_REG_SSLNK); 898 } 899 900 if (d40c->dma_cfg.dir != STEDMA40_PERIPH_TO_MEM) { 901 u32 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.dst_dev_type); 902 903 __d40_config_set_event(d40c, do_enable, event, 904 D40_CHAN_REG_SDLNK); 905 } 906 907 spin_unlock_irqrestore(&d40c->phy_chan->lock, flags); 908 } 909 910 static u32 d40_chan_has_events(struct d40_chan *d40c) 911 { 912 void __iomem *chanbase = chan_base(d40c); 913 u32 val; 914 915 val = readl(chanbase + D40_CHAN_REG_SSLNK); 916 val |= readl(chanbase + D40_CHAN_REG_SDLNK); 917 918 return val; 919 } 920 921 static u32 d40_get_prmo(struct d40_chan *d40c) 922 { 923 static const unsigned int phy_map[] = { 924 [STEDMA40_PCHAN_BASIC_MODE] 925 = D40_DREG_PRMO_PCHAN_BASIC, 926 [STEDMA40_PCHAN_MODULO_MODE] 927 = D40_DREG_PRMO_PCHAN_MODULO, 928 [STEDMA40_PCHAN_DOUBLE_DST_MODE] 929 = D40_DREG_PRMO_PCHAN_DOUBLE_DST, 930 }; 931 static const unsigned int log_map[] = { 932 [STEDMA40_LCHAN_SRC_PHY_DST_LOG] 933 = D40_DREG_PRMO_LCHAN_SRC_PHY_DST_LOG, 934 [STEDMA40_LCHAN_SRC_LOG_DST_PHY] 935 = D40_DREG_PRMO_LCHAN_SRC_LOG_DST_PHY, 936 [STEDMA40_LCHAN_SRC_LOG_DST_LOG] 937 = D40_DREG_PRMO_LCHAN_SRC_LOG_DST_LOG, 938 }; 939 940 if (chan_is_physical(d40c)) 941 return phy_map[d40c->dma_cfg.mode_opt]; 942 else 943 return log_map[d40c->dma_cfg.mode_opt]; 944 } 945 946 static void d40_config_write(struct d40_chan *d40c) 947 { 948 u32 addr_base; 949 u32 var; 950 951 /* Odd addresses are even addresses + 4 */ 952 addr_base = (d40c->phy_chan->num % 2) * 4; 953 /* Setup channel mode to logical or physical */ 954 var = ((u32)(chan_is_logical(d40c)) + 1) << 955 D40_CHAN_POS(d40c->phy_chan->num); 956 writel(var, d40c->base->virtbase + D40_DREG_PRMSE + addr_base); 957 958 /* Setup operational mode option register */ 959 var = d40_get_prmo(d40c) << D40_CHAN_POS(d40c->phy_chan->num); 960 961 writel(var, d40c->base->virtbase + D40_DREG_PRMOE + addr_base); 962 963 if (chan_is_logical(d40c)) { 964 int lidx = (d40c->phy_chan->num << D40_SREG_ELEM_LOG_LIDX_POS) 965 & D40_SREG_ELEM_LOG_LIDX_MASK; 966 void __iomem *chanbase = chan_base(d40c); 967 968 /* Set default config for CFG reg */ 969 writel(d40c->src_def_cfg, chanbase + D40_CHAN_REG_SSCFG); 970 writel(d40c->dst_def_cfg, chanbase + D40_CHAN_REG_SDCFG); 971 972 /* Set LIDX for lcla */ 973 writel(lidx, chanbase + D40_CHAN_REG_SSELT); 974 writel(lidx, chanbase + D40_CHAN_REG_SDELT); 975 } 976 } 977 978 static u32 d40_residue(struct d40_chan *d40c) 979 { 980 u32 num_elt; 981 982 if (chan_is_logical(d40c)) 983 num_elt = (readl(&d40c->lcpa->lcsp2) & D40_MEM_LCSP2_ECNT_MASK) 984 >> D40_MEM_LCSP2_ECNT_POS; 985 else { 986 u32 val = readl(chan_base(d40c) + D40_CHAN_REG_SDELT); 987 num_elt = (val & D40_SREG_ELEM_PHY_ECNT_MASK) 988 >> D40_SREG_ELEM_PHY_ECNT_POS; 989 } 990 991 return num_elt * (1 << d40c->dma_cfg.dst_info.data_width); 992 } 993 994 static bool d40_tx_is_linked(struct d40_chan *d40c) 995 { 996 bool is_link; 997 998 if (chan_is_logical(d40c)) 999 is_link = readl(&d40c->lcpa->lcsp3) & D40_MEM_LCSP3_DLOS_MASK; 1000 else 1001 is_link = readl(chan_base(d40c) + D40_CHAN_REG_SDLNK) 1002 & D40_SREG_LNK_PHYS_LNK_MASK; 1003 1004 return is_link; 1005 } 1006 1007 static int d40_pause(struct d40_chan *d40c) 1008 { 1009 int res = 0; 1010 unsigned long flags; 1011 1012 if (!d40c->busy) 1013 return 0; 1014 1015 spin_lock_irqsave(&d40c->lock, flags); 1016 1017 res = d40_channel_execute_command(d40c, D40_DMA_SUSPEND_REQ); 1018 if (res == 0) { 1019 if (chan_is_logical(d40c)) { 1020 d40_config_set_event(d40c, false); 1021 /* Resume the other logical channels if any */ 1022 if (d40_chan_has_events(d40c)) 1023 res = d40_channel_execute_command(d40c, 1024 D40_DMA_RUN); 1025 } 1026 } 1027 1028 spin_unlock_irqrestore(&d40c->lock, flags); 1029 return res; 1030 } 1031 1032 static int d40_resume(struct d40_chan *d40c) 1033 { 1034 int res = 0; 1035 unsigned long flags; 1036 1037 if (!d40c->busy) 1038 return 0; 1039 1040 spin_lock_irqsave(&d40c->lock, flags); 1041 1042 if (d40c->base->rev == 0) 1043 if (chan_is_logical(d40c)) { 1044 res = d40_channel_execute_command(d40c, 1045 D40_DMA_SUSPEND_REQ); 1046 goto no_suspend; 1047 } 1048 1049 /* If bytes left to transfer or linked tx resume job */ 1050 if (d40_residue(d40c) || d40_tx_is_linked(d40c)) { 1051 1052 if (chan_is_logical(d40c)) 1053 d40_config_set_event(d40c, true); 1054 1055 res = d40_channel_execute_command(d40c, D40_DMA_RUN); 1056 } 1057 1058 no_suspend: 1059 spin_unlock_irqrestore(&d40c->lock, flags); 1060 return res; 1061 } 1062 1063 static int d40_terminate_all(struct d40_chan *chan) 1064 { 1065 unsigned long flags; 1066 int ret = 0; 1067 1068 ret = d40_pause(chan); 1069 if (!ret && chan_is_physical(chan)) 1070 ret = d40_channel_execute_command(chan, D40_DMA_STOP); 1071 1072 spin_lock_irqsave(&chan->lock, flags); 1073 d40_term_all(chan); 1074 spin_unlock_irqrestore(&chan->lock, flags); 1075 1076 return ret; 1077 } 1078 1079 static dma_cookie_t d40_tx_submit(struct dma_async_tx_descriptor *tx) 1080 { 1081 struct d40_chan *d40c = container_of(tx->chan, 1082 struct d40_chan, 1083 chan); 1084 struct d40_desc *d40d = container_of(tx, struct d40_desc, txd); 1085 unsigned long flags; 1086 1087 spin_lock_irqsave(&d40c->lock, flags); 1088 1089 d40c->chan.cookie++; 1090 1091 if (d40c->chan.cookie < 0) 1092 d40c->chan.cookie = 1; 1093 1094 d40d->txd.cookie = d40c->chan.cookie; 1095 1096 d40_desc_queue(d40c, d40d); 1097 1098 spin_unlock_irqrestore(&d40c->lock, flags); 1099 1100 return tx->cookie; 1101 } 1102 1103 static int d40_start(struct d40_chan *d40c) 1104 { 1105 if (d40c->base->rev == 0) { 1106 int err; 1107 1108 if (chan_is_logical(d40c)) { 1109 err = d40_channel_execute_command(d40c, 1110 D40_DMA_SUSPEND_REQ); 1111 if (err) 1112 return err; 1113 } 1114 } 1115 1116 if (chan_is_logical(d40c)) 1117 d40_config_set_event(d40c, true); 1118 1119 return d40_channel_execute_command(d40c, D40_DMA_RUN); 1120 } 1121 1122 static struct d40_desc *d40_queue_start(struct d40_chan *d40c) 1123 { 1124 struct d40_desc *d40d; 1125 int err; 1126 1127 /* Start queued jobs, if any */ 1128 d40d = d40_first_queued(d40c); 1129 1130 if (d40d != NULL) { 1131 d40c->busy = true; 1132 1133 /* Remove from queue */ 1134 d40_desc_remove(d40d); 1135 1136 /* Add to active queue */ 1137 d40_desc_submit(d40c, d40d); 1138 1139 /* Initiate DMA job */ 1140 d40_desc_load(d40c, d40d); 1141 1142 /* Start dma job */ 1143 err = d40_start(d40c); 1144 1145 if (err) 1146 return NULL; 1147 } 1148 1149 return d40d; 1150 } 1151 1152 /* called from interrupt context */ 1153 static void dma_tc_handle(struct d40_chan *d40c) 1154 { 1155 struct d40_desc *d40d; 1156 1157 /* Get first active entry from list */ 1158 d40d = d40_first_active_get(d40c); 1159 1160 if (d40d == NULL) 1161 return; 1162 1163 if (d40d->cyclic) { 1164 /* 1165 * If this was a paritially loaded list, we need to reloaded 1166 * it, and only when the list is completed. We need to check 1167 * for done because the interrupt will hit for every link, and 1168 * not just the last one. 1169 */ 1170 if (d40d->lli_current < d40d->lli_len 1171 && !d40_tx_is_linked(d40c) 1172 && !d40_residue(d40c)) { 1173 d40_lcla_free_all(d40c, d40d); 1174 d40_desc_load(d40c, d40d); 1175 (void) d40_start(d40c); 1176 1177 if (d40d->lli_current == d40d->lli_len) 1178 d40d->lli_current = 0; 1179 } 1180 } else { 1181 d40_lcla_free_all(d40c, d40d); 1182 1183 if (d40d->lli_current < d40d->lli_len) { 1184 d40_desc_load(d40c, d40d); 1185 /* Start dma job */ 1186 (void) d40_start(d40c); 1187 return; 1188 } 1189 1190 if (d40_queue_start(d40c) == NULL) 1191 d40c->busy = false; 1192 } 1193 1194 d40c->pending_tx++; 1195 tasklet_schedule(&d40c->tasklet); 1196 1197 } 1198 1199 static void dma_tasklet(unsigned long data) 1200 { 1201 struct d40_chan *d40c = (struct d40_chan *) data; 1202 struct d40_desc *d40d; 1203 unsigned long flags; 1204 dma_async_tx_callback callback; 1205 void *callback_param; 1206 1207 spin_lock_irqsave(&d40c->lock, flags); 1208 1209 /* Get first active entry from list */ 1210 d40d = d40_first_active_get(d40c); 1211 if (d40d == NULL) 1212 goto err; 1213 1214 if (!d40d->cyclic) 1215 d40c->completed = d40d->txd.cookie; 1216 1217 /* 1218 * If terminating a channel pending_tx is set to zero. 1219 * This prevents any finished active jobs to return to the client. 1220 */ 1221 if (d40c->pending_tx == 0) { 1222 spin_unlock_irqrestore(&d40c->lock, flags); 1223 return; 1224 } 1225 1226 /* Callback to client */ 1227 callback = d40d->txd.callback; 1228 callback_param = d40d->txd.callback_param; 1229 1230 if (!d40d->cyclic) { 1231 if (async_tx_test_ack(&d40d->txd)) { 1232 d40_desc_remove(d40d); 1233 d40_desc_free(d40c, d40d); 1234 } else { 1235 if (!d40d->is_in_client_list) { 1236 d40_desc_remove(d40d); 1237 d40_lcla_free_all(d40c, d40d); 1238 list_add_tail(&d40d->node, &d40c->client); 1239 d40d->is_in_client_list = true; 1240 } 1241 } 1242 } 1243 1244 d40c->pending_tx--; 1245 1246 if (d40c->pending_tx) 1247 tasklet_schedule(&d40c->tasklet); 1248 1249 spin_unlock_irqrestore(&d40c->lock, flags); 1250 1251 if (callback && (d40d->txd.flags & DMA_PREP_INTERRUPT)) 1252 callback(callback_param); 1253 1254 return; 1255 1256 err: 1257 /* Rescue manoeuvre if receiving double interrupts */ 1258 if (d40c->pending_tx > 0) 1259 d40c->pending_tx--; 1260 spin_unlock_irqrestore(&d40c->lock, flags); 1261 } 1262 1263 static irqreturn_t d40_handle_interrupt(int irq, void *data) 1264 { 1265 static const struct d40_interrupt_lookup il[] = { 1266 {D40_DREG_LCTIS0, D40_DREG_LCICR0, false, 0}, 1267 {D40_DREG_LCTIS1, D40_DREG_LCICR1, false, 32}, 1268 {D40_DREG_LCTIS2, D40_DREG_LCICR2, false, 64}, 1269 {D40_DREG_LCTIS3, D40_DREG_LCICR3, false, 96}, 1270 {D40_DREG_LCEIS0, D40_DREG_LCICR0, true, 0}, 1271 {D40_DREG_LCEIS1, D40_DREG_LCICR1, true, 32}, 1272 {D40_DREG_LCEIS2, D40_DREG_LCICR2, true, 64}, 1273 {D40_DREG_LCEIS3, D40_DREG_LCICR3, true, 96}, 1274 {D40_DREG_PCTIS, D40_DREG_PCICR, false, D40_PHY_CHAN}, 1275 {D40_DREG_PCEIS, D40_DREG_PCICR, true, D40_PHY_CHAN}, 1276 }; 1277 1278 int i; 1279 u32 regs[ARRAY_SIZE(il)]; 1280 u32 idx; 1281 u32 row; 1282 long chan = -1; 1283 struct d40_chan *d40c; 1284 unsigned long flags; 1285 struct d40_base *base = data; 1286 1287 spin_lock_irqsave(&base->interrupt_lock, flags); 1288 1289 /* Read interrupt status of both logical and physical channels */ 1290 for (i = 0; i < ARRAY_SIZE(il); i++) 1291 regs[i] = readl(base->virtbase + il[i].src); 1292 1293 for (;;) { 1294 1295 chan = find_next_bit((unsigned long *)regs, 1296 BITS_PER_LONG * ARRAY_SIZE(il), chan + 1); 1297 1298 /* No more set bits found? */ 1299 if (chan == BITS_PER_LONG * ARRAY_SIZE(il)) 1300 break; 1301 1302 row = chan / BITS_PER_LONG; 1303 idx = chan & (BITS_PER_LONG - 1); 1304 1305 /* ACK interrupt */ 1306 writel(1 << idx, base->virtbase + il[row].clr); 1307 1308 if (il[row].offset == D40_PHY_CHAN) 1309 d40c = base->lookup_phy_chans[idx]; 1310 else 1311 d40c = base->lookup_log_chans[il[row].offset + idx]; 1312 spin_lock(&d40c->lock); 1313 1314 if (!il[row].is_error) 1315 dma_tc_handle(d40c); 1316 else 1317 d40_err(base->dev, "IRQ chan: %ld offset %d idx %d\n", 1318 chan, il[row].offset, idx); 1319 1320 spin_unlock(&d40c->lock); 1321 } 1322 1323 spin_unlock_irqrestore(&base->interrupt_lock, flags); 1324 1325 return IRQ_HANDLED; 1326 } 1327 1328 static int d40_validate_conf(struct d40_chan *d40c, 1329 struct stedma40_chan_cfg *conf) 1330 { 1331 int res = 0; 1332 u32 dst_event_group = D40_TYPE_TO_GROUP(conf->dst_dev_type); 1333 u32 src_event_group = D40_TYPE_TO_GROUP(conf->src_dev_type); 1334 bool is_log = conf->mode == STEDMA40_MODE_LOGICAL; 1335 1336 if (!conf->dir) { 1337 chan_err(d40c, "Invalid direction.\n"); 1338 res = -EINVAL; 1339 } 1340 1341 if (conf->dst_dev_type != STEDMA40_DEV_DST_MEMORY && 1342 d40c->base->plat_data->dev_tx[conf->dst_dev_type] == 0 && 1343 d40c->runtime_addr == 0) { 1344 1345 chan_err(d40c, "Invalid TX channel address (%d)\n", 1346 conf->dst_dev_type); 1347 res = -EINVAL; 1348 } 1349 1350 if (conf->src_dev_type != STEDMA40_DEV_SRC_MEMORY && 1351 d40c->base->plat_data->dev_rx[conf->src_dev_type] == 0 && 1352 d40c->runtime_addr == 0) { 1353 chan_err(d40c, "Invalid RX channel address (%d)\n", 1354 conf->src_dev_type); 1355 res = -EINVAL; 1356 } 1357 1358 if (conf->dir == STEDMA40_MEM_TO_PERIPH && 1359 dst_event_group == STEDMA40_DEV_DST_MEMORY) { 1360 chan_err(d40c, "Invalid dst\n"); 1361 res = -EINVAL; 1362 } 1363 1364 if (conf->dir == STEDMA40_PERIPH_TO_MEM && 1365 src_event_group == STEDMA40_DEV_SRC_MEMORY) { 1366 chan_err(d40c, "Invalid src\n"); 1367 res = -EINVAL; 1368 } 1369 1370 if (src_event_group == STEDMA40_DEV_SRC_MEMORY && 1371 dst_event_group == STEDMA40_DEV_DST_MEMORY && is_log) { 1372 chan_err(d40c, "No event line\n"); 1373 res = -EINVAL; 1374 } 1375 1376 if (conf->dir == STEDMA40_PERIPH_TO_PERIPH && 1377 (src_event_group != dst_event_group)) { 1378 chan_err(d40c, "Invalid event group\n"); 1379 res = -EINVAL; 1380 } 1381 1382 if (conf->dir == STEDMA40_PERIPH_TO_PERIPH) { 1383 /* 1384 * DMAC HW supports it. Will be added to this driver, 1385 * in case any dma client requires it. 1386 */ 1387 chan_err(d40c, "periph to periph not supported\n"); 1388 res = -EINVAL; 1389 } 1390 1391 if (d40_psize_2_burst_size(is_log, conf->src_info.psize) * 1392 (1 << conf->src_info.data_width) != 1393 d40_psize_2_burst_size(is_log, conf->dst_info.psize) * 1394 (1 << conf->dst_info.data_width)) { 1395 /* 1396 * The DMAC hardware only supports 1397 * src (burst x width) == dst (burst x width) 1398 */ 1399 1400 chan_err(d40c, "src (burst x width) != dst (burst x width)\n"); 1401 res = -EINVAL; 1402 } 1403 1404 return res; 1405 } 1406 1407 static bool d40_alloc_mask_set(struct d40_phy_res *phy, bool is_src, 1408 int log_event_line, bool is_log) 1409 { 1410 unsigned long flags; 1411 spin_lock_irqsave(&phy->lock, flags); 1412 if (!is_log) { 1413 /* Physical interrupts are masked per physical full channel */ 1414 if (phy->allocated_src == D40_ALLOC_FREE && 1415 phy->allocated_dst == D40_ALLOC_FREE) { 1416 phy->allocated_dst = D40_ALLOC_PHY; 1417 phy->allocated_src = D40_ALLOC_PHY; 1418 goto found; 1419 } else 1420 goto not_found; 1421 } 1422 1423 /* Logical channel */ 1424 if (is_src) { 1425 if (phy->allocated_src == D40_ALLOC_PHY) 1426 goto not_found; 1427 1428 if (phy->allocated_src == D40_ALLOC_FREE) 1429 phy->allocated_src = D40_ALLOC_LOG_FREE; 1430 1431 if (!(phy->allocated_src & (1 << log_event_line))) { 1432 phy->allocated_src |= 1 << log_event_line; 1433 goto found; 1434 } else 1435 goto not_found; 1436 } else { 1437 if (phy->allocated_dst == D40_ALLOC_PHY) 1438 goto not_found; 1439 1440 if (phy->allocated_dst == D40_ALLOC_FREE) 1441 phy->allocated_dst = D40_ALLOC_LOG_FREE; 1442 1443 if (!(phy->allocated_dst & (1 << log_event_line))) { 1444 phy->allocated_dst |= 1 << log_event_line; 1445 goto found; 1446 } else 1447 goto not_found; 1448 } 1449 1450 not_found: 1451 spin_unlock_irqrestore(&phy->lock, flags); 1452 return false; 1453 found: 1454 spin_unlock_irqrestore(&phy->lock, flags); 1455 return true; 1456 } 1457 1458 static bool d40_alloc_mask_free(struct d40_phy_res *phy, bool is_src, 1459 int log_event_line) 1460 { 1461 unsigned long flags; 1462 bool is_free = false; 1463 1464 spin_lock_irqsave(&phy->lock, flags); 1465 if (!log_event_line) { 1466 phy->allocated_dst = D40_ALLOC_FREE; 1467 phy->allocated_src = D40_ALLOC_FREE; 1468 is_free = true; 1469 goto out; 1470 } 1471 1472 /* Logical channel */ 1473 if (is_src) { 1474 phy->allocated_src &= ~(1 << log_event_line); 1475 if (phy->allocated_src == D40_ALLOC_LOG_FREE) 1476 phy->allocated_src = D40_ALLOC_FREE; 1477 } else { 1478 phy->allocated_dst &= ~(1 << log_event_line); 1479 if (phy->allocated_dst == D40_ALLOC_LOG_FREE) 1480 phy->allocated_dst = D40_ALLOC_FREE; 1481 } 1482 1483 is_free = ((phy->allocated_src | phy->allocated_dst) == 1484 D40_ALLOC_FREE); 1485 1486 out: 1487 spin_unlock_irqrestore(&phy->lock, flags); 1488 1489 return is_free; 1490 } 1491 1492 static int d40_allocate_channel(struct d40_chan *d40c) 1493 { 1494 int dev_type; 1495 int event_group; 1496 int event_line; 1497 struct d40_phy_res *phys; 1498 int i; 1499 int j; 1500 int log_num; 1501 bool is_src; 1502 bool is_log = d40c->dma_cfg.mode == STEDMA40_MODE_LOGICAL; 1503 1504 phys = d40c->base->phy_res; 1505 1506 if (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM) { 1507 dev_type = d40c->dma_cfg.src_dev_type; 1508 log_num = 2 * dev_type; 1509 is_src = true; 1510 } else if (d40c->dma_cfg.dir == STEDMA40_MEM_TO_PERIPH || 1511 d40c->dma_cfg.dir == STEDMA40_MEM_TO_MEM) { 1512 /* dst event lines are used for logical memcpy */ 1513 dev_type = d40c->dma_cfg.dst_dev_type; 1514 log_num = 2 * dev_type + 1; 1515 is_src = false; 1516 } else 1517 return -EINVAL; 1518 1519 event_group = D40_TYPE_TO_GROUP(dev_type); 1520 event_line = D40_TYPE_TO_EVENT(dev_type); 1521 1522 if (!is_log) { 1523 if (d40c->dma_cfg.dir == STEDMA40_MEM_TO_MEM) { 1524 /* Find physical half channel */ 1525 for (i = 0; i < d40c->base->num_phy_chans; i++) { 1526 1527 if (d40_alloc_mask_set(&phys[i], is_src, 1528 0, is_log)) 1529 goto found_phy; 1530 } 1531 } else 1532 for (j = 0; j < d40c->base->num_phy_chans; j += 8) { 1533 int phy_num = j + event_group * 2; 1534 for (i = phy_num; i < phy_num + 2; i++) { 1535 if (d40_alloc_mask_set(&phys[i], 1536 is_src, 1537 0, 1538 is_log)) 1539 goto found_phy; 1540 } 1541 } 1542 return -EINVAL; 1543 found_phy: 1544 d40c->phy_chan = &phys[i]; 1545 d40c->log_num = D40_PHY_CHAN; 1546 goto out; 1547 } 1548 if (dev_type == -1) 1549 return -EINVAL; 1550 1551 /* Find logical channel */ 1552 for (j = 0; j < d40c->base->num_phy_chans; j += 8) { 1553 int phy_num = j + event_group * 2; 1554 /* 1555 * Spread logical channels across all available physical rather 1556 * than pack every logical channel at the first available phy 1557 * channels. 1558 */ 1559 if (is_src) { 1560 for (i = phy_num; i < phy_num + 2; i++) { 1561 if (d40_alloc_mask_set(&phys[i], is_src, 1562 event_line, is_log)) 1563 goto found_log; 1564 } 1565 } else { 1566 for (i = phy_num + 1; i >= phy_num; i--) { 1567 if (d40_alloc_mask_set(&phys[i], is_src, 1568 event_line, is_log)) 1569 goto found_log; 1570 } 1571 } 1572 } 1573 return -EINVAL; 1574 1575 found_log: 1576 d40c->phy_chan = &phys[i]; 1577 d40c->log_num = log_num; 1578 out: 1579 1580 if (is_log) 1581 d40c->base->lookup_log_chans[d40c->log_num] = d40c; 1582 else 1583 d40c->base->lookup_phy_chans[d40c->phy_chan->num] = d40c; 1584 1585 return 0; 1586 1587 } 1588 1589 static int d40_config_memcpy(struct d40_chan *d40c) 1590 { 1591 dma_cap_mask_t cap = d40c->chan.device->cap_mask; 1592 1593 if (dma_has_cap(DMA_MEMCPY, cap) && !dma_has_cap(DMA_SLAVE, cap)) { 1594 d40c->dma_cfg = *d40c->base->plat_data->memcpy_conf_log; 1595 d40c->dma_cfg.src_dev_type = STEDMA40_DEV_SRC_MEMORY; 1596 d40c->dma_cfg.dst_dev_type = d40c->base->plat_data-> 1597 memcpy[d40c->chan.chan_id]; 1598 1599 } else if (dma_has_cap(DMA_MEMCPY, cap) && 1600 dma_has_cap(DMA_SLAVE, cap)) { 1601 d40c->dma_cfg = *d40c->base->plat_data->memcpy_conf_phy; 1602 } else { 1603 chan_err(d40c, "No memcpy\n"); 1604 return -EINVAL; 1605 } 1606 1607 return 0; 1608 } 1609 1610 1611 static int d40_free_dma(struct d40_chan *d40c) 1612 { 1613 1614 int res = 0; 1615 u32 event; 1616 struct d40_phy_res *phy = d40c->phy_chan; 1617 bool is_src; 1618 1619 /* Terminate all queued and active transfers */ 1620 d40_term_all(d40c); 1621 1622 if (phy == NULL) { 1623 chan_err(d40c, "phy == null\n"); 1624 return -EINVAL; 1625 } 1626 1627 if (phy->allocated_src == D40_ALLOC_FREE && 1628 phy->allocated_dst == D40_ALLOC_FREE) { 1629 chan_err(d40c, "channel already free\n"); 1630 return -EINVAL; 1631 } 1632 1633 if (d40c->dma_cfg.dir == STEDMA40_MEM_TO_PERIPH || 1634 d40c->dma_cfg.dir == STEDMA40_MEM_TO_MEM) { 1635 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.dst_dev_type); 1636 is_src = false; 1637 } else if (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM) { 1638 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.src_dev_type); 1639 is_src = true; 1640 } else { 1641 chan_err(d40c, "Unknown direction\n"); 1642 return -EINVAL; 1643 } 1644 1645 res = d40_channel_execute_command(d40c, D40_DMA_SUSPEND_REQ); 1646 if (res) { 1647 chan_err(d40c, "suspend failed\n"); 1648 return res; 1649 } 1650 1651 if (chan_is_logical(d40c)) { 1652 /* Release logical channel, deactivate the event line */ 1653 1654 d40_config_set_event(d40c, false); 1655 d40c->base->lookup_log_chans[d40c->log_num] = NULL; 1656 1657 /* 1658 * Check if there are more logical allocation 1659 * on this phy channel. 1660 */ 1661 if (!d40_alloc_mask_free(phy, is_src, event)) { 1662 /* Resume the other logical channels if any */ 1663 if (d40_chan_has_events(d40c)) { 1664 res = d40_channel_execute_command(d40c, 1665 D40_DMA_RUN); 1666 if (res) { 1667 chan_err(d40c, 1668 "Executing RUN command\n"); 1669 return res; 1670 } 1671 } 1672 return 0; 1673 } 1674 } else { 1675 (void) d40_alloc_mask_free(phy, is_src, 0); 1676 } 1677 1678 /* Release physical channel */ 1679 res = d40_channel_execute_command(d40c, D40_DMA_STOP); 1680 if (res) { 1681 chan_err(d40c, "Failed to stop channel\n"); 1682 return res; 1683 } 1684 d40c->phy_chan = NULL; 1685 d40c->configured = false; 1686 d40c->base->lookup_phy_chans[phy->num] = NULL; 1687 1688 return 0; 1689 } 1690 1691 static bool d40_is_paused(struct d40_chan *d40c) 1692 { 1693 void __iomem *chanbase = chan_base(d40c); 1694 bool is_paused = false; 1695 unsigned long flags; 1696 void __iomem *active_reg; 1697 u32 status; 1698 u32 event; 1699 1700 spin_lock_irqsave(&d40c->lock, flags); 1701 1702 if (chan_is_physical(d40c)) { 1703 if (d40c->phy_chan->num % 2 == 0) 1704 active_reg = d40c->base->virtbase + D40_DREG_ACTIVE; 1705 else 1706 active_reg = d40c->base->virtbase + D40_DREG_ACTIVO; 1707 1708 status = (readl(active_reg) & 1709 D40_CHAN_POS_MASK(d40c->phy_chan->num)) >> 1710 D40_CHAN_POS(d40c->phy_chan->num); 1711 if (status == D40_DMA_SUSPENDED || status == D40_DMA_STOP) 1712 is_paused = true; 1713 1714 goto _exit; 1715 } 1716 1717 if (d40c->dma_cfg.dir == STEDMA40_MEM_TO_PERIPH || 1718 d40c->dma_cfg.dir == STEDMA40_MEM_TO_MEM) { 1719 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.dst_dev_type); 1720 status = readl(chanbase + D40_CHAN_REG_SDLNK); 1721 } else if (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM) { 1722 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.src_dev_type); 1723 status = readl(chanbase + D40_CHAN_REG_SSLNK); 1724 } else { 1725 chan_err(d40c, "Unknown direction\n"); 1726 goto _exit; 1727 } 1728 1729 status = (status & D40_EVENTLINE_MASK(event)) >> 1730 D40_EVENTLINE_POS(event); 1731 1732 if (status != D40_DMA_RUN) 1733 is_paused = true; 1734 _exit: 1735 spin_unlock_irqrestore(&d40c->lock, flags); 1736 return is_paused; 1737 1738 } 1739 1740 1741 static u32 stedma40_residue(struct dma_chan *chan) 1742 { 1743 struct d40_chan *d40c = 1744 container_of(chan, struct d40_chan, chan); 1745 u32 bytes_left; 1746 unsigned long flags; 1747 1748 spin_lock_irqsave(&d40c->lock, flags); 1749 bytes_left = d40_residue(d40c); 1750 spin_unlock_irqrestore(&d40c->lock, flags); 1751 1752 return bytes_left; 1753 } 1754 1755 static int 1756 d40_prep_sg_log(struct d40_chan *chan, struct d40_desc *desc, 1757 struct scatterlist *sg_src, struct scatterlist *sg_dst, 1758 unsigned int sg_len, dma_addr_t src_dev_addr, 1759 dma_addr_t dst_dev_addr) 1760 { 1761 struct stedma40_chan_cfg *cfg = &chan->dma_cfg; 1762 struct stedma40_half_channel_info *src_info = &cfg->src_info; 1763 struct stedma40_half_channel_info *dst_info = &cfg->dst_info; 1764 int ret; 1765 1766 ret = d40_log_sg_to_lli(sg_src, sg_len, 1767 src_dev_addr, 1768 desc->lli_log.src, 1769 chan->log_def.lcsp1, 1770 src_info->data_width, 1771 dst_info->data_width); 1772 1773 ret = d40_log_sg_to_lli(sg_dst, sg_len, 1774 dst_dev_addr, 1775 desc->lli_log.dst, 1776 chan->log_def.lcsp3, 1777 dst_info->data_width, 1778 src_info->data_width); 1779 1780 return ret < 0 ? ret : 0; 1781 } 1782 1783 static int 1784 d40_prep_sg_phy(struct d40_chan *chan, struct d40_desc *desc, 1785 struct scatterlist *sg_src, struct scatterlist *sg_dst, 1786 unsigned int sg_len, dma_addr_t src_dev_addr, 1787 dma_addr_t dst_dev_addr) 1788 { 1789 struct stedma40_chan_cfg *cfg = &chan->dma_cfg; 1790 struct stedma40_half_channel_info *src_info = &cfg->src_info; 1791 struct stedma40_half_channel_info *dst_info = &cfg->dst_info; 1792 unsigned long flags = 0; 1793 int ret; 1794 1795 if (desc->cyclic) 1796 flags |= LLI_CYCLIC | LLI_TERM_INT; 1797 1798 ret = d40_phy_sg_to_lli(sg_src, sg_len, src_dev_addr, 1799 desc->lli_phy.src, 1800 virt_to_phys(desc->lli_phy.src), 1801 chan->src_def_cfg, 1802 src_info, dst_info, flags); 1803 1804 ret = d40_phy_sg_to_lli(sg_dst, sg_len, dst_dev_addr, 1805 desc->lli_phy.dst, 1806 virt_to_phys(desc->lli_phy.dst), 1807 chan->dst_def_cfg, 1808 dst_info, src_info, flags); 1809 1810 dma_sync_single_for_device(chan->base->dev, desc->lli_pool.dma_addr, 1811 desc->lli_pool.size, DMA_TO_DEVICE); 1812 1813 return ret < 0 ? ret : 0; 1814 } 1815 1816 1817 static struct d40_desc * 1818 d40_prep_desc(struct d40_chan *chan, struct scatterlist *sg, 1819 unsigned int sg_len, unsigned long dma_flags) 1820 { 1821 struct stedma40_chan_cfg *cfg = &chan->dma_cfg; 1822 struct d40_desc *desc; 1823 int ret; 1824 1825 desc = d40_desc_get(chan); 1826 if (!desc) 1827 return NULL; 1828 1829 desc->lli_len = d40_sg_2_dmalen(sg, sg_len, cfg->src_info.data_width, 1830 cfg->dst_info.data_width); 1831 if (desc->lli_len < 0) { 1832 chan_err(chan, "Unaligned size\n"); 1833 goto err; 1834 } 1835 1836 ret = d40_pool_lli_alloc(chan, desc, desc->lli_len); 1837 if (ret < 0) { 1838 chan_err(chan, "Could not allocate lli\n"); 1839 goto err; 1840 } 1841 1842 1843 desc->lli_current = 0; 1844 desc->txd.flags = dma_flags; 1845 desc->txd.tx_submit = d40_tx_submit; 1846 1847 dma_async_tx_descriptor_init(&desc->txd, &chan->chan); 1848 1849 return desc; 1850 1851 err: 1852 d40_desc_free(chan, desc); 1853 return NULL; 1854 } 1855 1856 static dma_addr_t 1857 d40_get_dev_addr(struct d40_chan *chan, enum dma_data_direction direction) 1858 { 1859 struct stedma40_platform_data *plat = chan->base->plat_data; 1860 struct stedma40_chan_cfg *cfg = &chan->dma_cfg; 1861 dma_addr_t addr = 0; 1862 1863 if (chan->runtime_addr) 1864 return chan->runtime_addr; 1865 1866 if (direction == DMA_FROM_DEVICE) 1867 addr = plat->dev_rx[cfg->src_dev_type]; 1868 else if (direction == DMA_TO_DEVICE) 1869 addr = plat->dev_tx[cfg->dst_dev_type]; 1870 1871 return addr; 1872 } 1873 1874 static struct dma_async_tx_descriptor * 1875 d40_prep_sg(struct dma_chan *dchan, struct scatterlist *sg_src, 1876 struct scatterlist *sg_dst, unsigned int sg_len, 1877 enum dma_data_direction direction, unsigned long dma_flags) 1878 { 1879 struct d40_chan *chan = container_of(dchan, struct d40_chan, chan); 1880 dma_addr_t src_dev_addr = 0; 1881 dma_addr_t dst_dev_addr = 0; 1882 struct d40_desc *desc; 1883 unsigned long flags; 1884 int ret; 1885 1886 if (!chan->phy_chan) { 1887 chan_err(chan, "Cannot prepare unallocated channel\n"); 1888 return NULL; 1889 } 1890 1891 1892 spin_lock_irqsave(&chan->lock, flags); 1893 1894 desc = d40_prep_desc(chan, sg_src, sg_len, dma_flags); 1895 if (desc == NULL) 1896 goto err; 1897 1898 if (sg_next(&sg_src[sg_len - 1]) == sg_src) 1899 desc->cyclic = true; 1900 1901 if (direction != DMA_NONE) { 1902 dma_addr_t dev_addr = d40_get_dev_addr(chan, direction); 1903 1904 if (direction == DMA_FROM_DEVICE) 1905 src_dev_addr = dev_addr; 1906 else if (direction == DMA_TO_DEVICE) 1907 dst_dev_addr = dev_addr; 1908 } 1909 1910 if (chan_is_logical(chan)) 1911 ret = d40_prep_sg_log(chan, desc, sg_src, sg_dst, 1912 sg_len, src_dev_addr, dst_dev_addr); 1913 else 1914 ret = d40_prep_sg_phy(chan, desc, sg_src, sg_dst, 1915 sg_len, src_dev_addr, dst_dev_addr); 1916 1917 if (ret) { 1918 chan_err(chan, "Failed to prepare %s sg job: %d\n", 1919 chan_is_logical(chan) ? "log" : "phy", ret); 1920 goto err; 1921 } 1922 1923 /* 1924 * add descriptor to the prepare queue in order to be able 1925 * to free them later in terminate_all 1926 */ 1927 list_add_tail(&desc->node, &chan->prepare_queue); 1928 1929 spin_unlock_irqrestore(&chan->lock, flags); 1930 1931 return &desc->txd; 1932 1933 err: 1934 if (desc) 1935 d40_desc_free(chan, desc); 1936 spin_unlock_irqrestore(&chan->lock, flags); 1937 return NULL; 1938 } 1939 1940 bool stedma40_filter(struct dma_chan *chan, void *data) 1941 { 1942 struct stedma40_chan_cfg *info = data; 1943 struct d40_chan *d40c = 1944 container_of(chan, struct d40_chan, chan); 1945 int err; 1946 1947 if (data) { 1948 err = d40_validate_conf(d40c, info); 1949 if (!err) 1950 d40c->dma_cfg = *info; 1951 } else 1952 err = d40_config_memcpy(d40c); 1953 1954 if (!err) 1955 d40c->configured = true; 1956 1957 return err == 0; 1958 } 1959 EXPORT_SYMBOL(stedma40_filter); 1960 1961 static void __d40_set_prio_rt(struct d40_chan *d40c, int dev_type, bool src) 1962 { 1963 bool realtime = d40c->dma_cfg.realtime; 1964 bool highprio = d40c->dma_cfg.high_priority; 1965 u32 prioreg = highprio ? D40_DREG_PSEG1 : D40_DREG_PCEG1; 1966 u32 rtreg = realtime ? D40_DREG_RSEG1 : D40_DREG_RCEG1; 1967 u32 event = D40_TYPE_TO_EVENT(dev_type); 1968 u32 group = D40_TYPE_TO_GROUP(dev_type); 1969 u32 bit = 1 << event; 1970 1971 /* Destination event lines are stored in the upper halfword */ 1972 if (!src) 1973 bit <<= 16; 1974 1975 writel(bit, d40c->base->virtbase + prioreg + group * 4); 1976 writel(bit, d40c->base->virtbase + rtreg + group * 4); 1977 } 1978 1979 static void d40_set_prio_realtime(struct d40_chan *d40c) 1980 { 1981 if (d40c->base->rev < 3) 1982 return; 1983 1984 if ((d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM) || 1985 (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_PERIPH)) 1986 __d40_set_prio_rt(d40c, d40c->dma_cfg.src_dev_type, true); 1987 1988 if ((d40c->dma_cfg.dir == STEDMA40_MEM_TO_PERIPH) || 1989 (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_PERIPH)) 1990 __d40_set_prio_rt(d40c, d40c->dma_cfg.dst_dev_type, false); 1991 } 1992 1993 /* DMA ENGINE functions */ 1994 static int d40_alloc_chan_resources(struct dma_chan *chan) 1995 { 1996 int err; 1997 unsigned long flags; 1998 struct d40_chan *d40c = 1999 container_of(chan, struct d40_chan, chan); 2000 bool is_free_phy; 2001 spin_lock_irqsave(&d40c->lock, flags); 2002 2003 d40c->completed = chan->cookie = 1; 2004 2005 /* If no dma configuration is set use default configuration (memcpy) */ 2006 if (!d40c->configured) { 2007 err = d40_config_memcpy(d40c); 2008 if (err) { 2009 chan_err(d40c, "Failed to configure memcpy channel\n"); 2010 goto fail; 2011 } 2012 } 2013 is_free_phy = (d40c->phy_chan == NULL); 2014 2015 err = d40_allocate_channel(d40c); 2016 if (err) { 2017 chan_err(d40c, "Failed to allocate channel\n"); 2018 goto fail; 2019 } 2020 2021 /* Fill in basic CFG register values */ 2022 d40_phy_cfg(&d40c->dma_cfg, &d40c->src_def_cfg, 2023 &d40c->dst_def_cfg, chan_is_logical(d40c)); 2024 2025 d40_set_prio_realtime(d40c); 2026 2027 if (chan_is_logical(d40c)) { 2028 d40_log_cfg(&d40c->dma_cfg, 2029 &d40c->log_def.lcsp1, &d40c->log_def.lcsp3); 2030 2031 if (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM) 2032 d40c->lcpa = d40c->base->lcpa_base + 2033 d40c->dma_cfg.src_dev_type * D40_LCPA_CHAN_SIZE; 2034 else 2035 d40c->lcpa = d40c->base->lcpa_base + 2036 d40c->dma_cfg.dst_dev_type * 2037 D40_LCPA_CHAN_SIZE + D40_LCPA_CHAN_DST_DELTA; 2038 } 2039 2040 /* 2041 * Only write channel configuration to the DMA if the physical 2042 * resource is free. In case of multiple logical channels 2043 * on the same physical resource, only the first write is necessary. 2044 */ 2045 if (is_free_phy) 2046 d40_config_write(d40c); 2047 fail: 2048 spin_unlock_irqrestore(&d40c->lock, flags); 2049 return err; 2050 } 2051 2052 static void d40_free_chan_resources(struct dma_chan *chan) 2053 { 2054 struct d40_chan *d40c = 2055 container_of(chan, struct d40_chan, chan); 2056 int err; 2057 unsigned long flags; 2058 2059 if (d40c->phy_chan == NULL) { 2060 chan_err(d40c, "Cannot free unallocated channel\n"); 2061 return; 2062 } 2063 2064 2065 spin_lock_irqsave(&d40c->lock, flags); 2066 2067 err = d40_free_dma(d40c); 2068 2069 if (err) 2070 chan_err(d40c, "Failed to free channel\n"); 2071 spin_unlock_irqrestore(&d40c->lock, flags); 2072 } 2073 2074 static struct dma_async_tx_descriptor *d40_prep_memcpy(struct dma_chan *chan, 2075 dma_addr_t dst, 2076 dma_addr_t src, 2077 size_t size, 2078 unsigned long dma_flags) 2079 { 2080 struct scatterlist dst_sg; 2081 struct scatterlist src_sg; 2082 2083 sg_init_table(&dst_sg, 1); 2084 sg_init_table(&src_sg, 1); 2085 2086 sg_dma_address(&dst_sg) = dst; 2087 sg_dma_address(&src_sg) = src; 2088 2089 sg_dma_len(&dst_sg) = size; 2090 sg_dma_len(&src_sg) = size; 2091 2092 return d40_prep_sg(chan, &src_sg, &dst_sg, 1, DMA_NONE, dma_flags); 2093 } 2094 2095 static struct dma_async_tx_descriptor * 2096 d40_prep_memcpy_sg(struct dma_chan *chan, 2097 struct scatterlist *dst_sg, unsigned int dst_nents, 2098 struct scatterlist *src_sg, unsigned int src_nents, 2099 unsigned long dma_flags) 2100 { 2101 if (dst_nents != src_nents) 2102 return NULL; 2103 2104 return d40_prep_sg(chan, src_sg, dst_sg, src_nents, DMA_NONE, dma_flags); 2105 } 2106 2107 static struct dma_async_tx_descriptor *d40_prep_slave_sg(struct dma_chan *chan, 2108 struct scatterlist *sgl, 2109 unsigned int sg_len, 2110 enum dma_data_direction direction, 2111 unsigned long dma_flags) 2112 { 2113 if (direction != DMA_FROM_DEVICE && direction != DMA_TO_DEVICE) 2114 return NULL; 2115 2116 return d40_prep_sg(chan, sgl, sgl, sg_len, direction, dma_flags); 2117 } 2118 2119 static struct dma_async_tx_descriptor * 2120 dma40_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t dma_addr, 2121 size_t buf_len, size_t period_len, 2122 enum dma_data_direction direction) 2123 { 2124 unsigned int periods = buf_len / period_len; 2125 struct dma_async_tx_descriptor *txd; 2126 struct scatterlist *sg; 2127 int i; 2128 2129 sg = kcalloc(periods + 1, sizeof(struct scatterlist), GFP_NOWAIT); 2130 for (i = 0; i < periods; i++) { 2131 sg_dma_address(&sg[i]) = dma_addr; 2132 sg_dma_len(&sg[i]) = period_len; 2133 dma_addr += period_len; 2134 } 2135 2136 sg[periods].offset = 0; 2137 sg[periods].length = 0; 2138 sg[periods].page_link = 2139 ((unsigned long)sg | 0x01) & ~0x02; 2140 2141 txd = d40_prep_sg(chan, sg, sg, periods, direction, 2142 DMA_PREP_INTERRUPT); 2143 2144 kfree(sg); 2145 2146 return txd; 2147 } 2148 2149 static enum dma_status d40_tx_status(struct dma_chan *chan, 2150 dma_cookie_t cookie, 2151 struct dma_tx_state *txstate) 2152 { 2153 struct d40_chan *d40c = container_of(chan, struct d40_chan, chan); 2154 dma_cookie_t last_used; 2155 dma_cookie_t last_complete; 2156 int ret; 2157 2158 if (d40c->phy_chan == NULL) { 2159 chan_err(d40c, "Cannot read status of unallocated channel\n"); 2160 return -EINVAL; 2161 } 2162 2163 last_complete = d40c->completed; 2164 last_used = chan->cookie; 2165 2166 if (d40_is_paused(d40c)) 2167 ret = DMA_PAUSED; 2168 else 2169 ret = dma_async_is_complete(cookie, last_complete, last_used); 2170 2171 dma_set_tx_state(txstate, last_complete, last_used, 2172 stedma40_residue(chan)); 2173 2174 return ret; 2175 } 2176 2177 static void d40_issue_pending(struct dma_chan *chan) 2178 { 2179 struct d40_chan *d40c = container_of(chan, struct d40_chan, chan); 2180 unsigned long flags; 2181 2182 if (d40c->phy_chan == NULL) { 2183 chan_err(d40c, "Channel is not allocated!\n"); 2184 return; 2185 } 2186 2187 spin_lock_irqsave(&d40c->lock, flags); 2188 2189 list_splice_tail_init(&d40c->pending_queue, &d40c->queue); 2190 2191 /* Busy means that queued jobs are already being processed */ 2192 if (!d40c->busy) 2193 (void) d40_queue_start(d40c); 2194 2195 spin_unlock_irqrestore(&d40c->lock, flags); 2196 } 2197 2198 static int 2199 dma40_config_to_halfchannel(struct d40_chan *d40c, 2200 struct stedma40_half_channel_info *info, 2201 enum dma_slave_buswidth width, 2202 u32 maxburst) 2203 { 2204 enum stedma40_periph_data_width addr_width; 2205 int psize; 2206 2207 switch (width) { 2208 case DMA_SLAVE_BUSWIDTH_1_BYTE: 2209 addr_width = STEDMA40_BYTE_WIDTH; 2210 break; 2211 case DMA_SLAVE_BUSWIDTH_2_BYTES: 2212 addr_width = STEDMA40_HALFWORD_WIDTH; 2213 break; 2214 case DMA_SLAVE_BUSWIDTH_4_BYTES: 2215 addr_width = STEDMA40_WORD_WIDTH; 2216 break; 2217 case DMA_SLAVE_BUSWIDTH_8_BYTES: 2218 addr_width = STEDMA40_DOUBLEWORD_WIDTH; 2219 break; 2220 default: 2221 dev_err(d40c->base->dev, 2222 "illegal peripheral address width " 2223 "requested (%d)\n", 2224 width); 2225 return -EINVAL; 2226 } 2227 2228 if (chan_is_logical(d40c)) { 2229 if (maxburst >= 16) 2230 psize = STEDMA40_PSIZE_LOG_16; 2231 else if (maxburst >= 8) 2232 psize = STEDMA40_PSIZE_LOG_8; 2233 else if (maxburst >= 4) 2234 psize = STEDMA40_PSIZE_LOG_4; 2235 else 2236 psize = STEDMA40_PSIZE_LOG_1; 2237 } else { 2238 if (maxburst >= 16) 2239 psize = STEDMA40_PSIZE_PHY_16; 2240 else if (maxburst >= 8) 2241 psize = STEDMA40_PSIZE_PHY_8; 2242 else if (maxburst >= 4) 2243 psize = STEDMA40_PSIZE_PHY_4; 2244 else 2245 psize = STEDMA40_PSIZE_PHY_1; 2246 } 2247 2248 info->data_width = addr_width; 2249 info->psize = psize; 2250 info->flow_ctrl = STEDMA40_NO_FLOW_CTRL; 2251 2252 return 0; 2253 } 2254 2255 /* Runtime reconfiguration extension */ 2256 static int d40_set_runtime_config(struct dma_chan *chan, 2257 struct dma_slave_config *config) 2258 { 2259 struct d40_chan *d40c = container_of(chan, struct d40_chan, chan); 2260 struct stedma40_chan_cfg *cfg = &d40c->dma_cfg; 2261 enum dma_slave_buswidth src_addr_width, dst_addr_width; 2262 dma_addr_t config_addr; 2263 u32 src_maxburst, dst_maxburst; 2264 int ret; 2265 2266 src_addr_width = config->src_addr_width; 2267 src_maxburst = config->src_maxburst; 2268 dst_addr_width = config->dst_addr_width; 2269 dst_maxburst = config->dst_maxburst; 2270 2271 if (config->direction == DMA_FROM_DEVICE) { 2272 dma_addr_t dev_addr_rx = 2273 d40c->base->plat_data->dev_rx[cfg->src_dev_type]; 2274 2275 config_addr = config->src_addr; 2276 if (dev_addr_rx) 2277 dev_dbg(d40c->base->dev, 2278 "channel has a pre-wired RX address %08x " 2279 "overriding with %08x\n", 2280 dev_addr_rx, config_addr); 2281 if (cfg->dir != STEDMA40_PERIPH_TO_MEM) 2282 dev_dbg(d40c->base->dev, 2283 "channel was not configured for peripheral " 2284 "to memory transfer (%d) overriding\n", 2285 cfg->dir); 2286 cfg->dir = STEDMA40_PERIPH_TO_MEM; 2287 2288 /* Configure the memory side */ 2289 if (dst_addr_width == DMA_SLAVE_BUSWIDTH_UNDEFINED) 2290 dst_addr_width = src_addr_width; 2291 if (dst_maxburst == 0) 2292 dst_maxburst = src_maxburst; 2293 2294 } else if (config->direction == DMA_TO_DEVICE) { 2295 dma_addr_t dev_addr_tx = 2296 d40c->base->plat_data->dev_tx[cfg->dst_dev_type]; 2297 2298 config_addr = config->dst_addr; 2299 if (dev_addr_tx) 2300 dev_dbg(d40c->base->dev, 2301 "channel has a pre-wired TX address %08x " 2302 "overriding with %08x\n", 2303 dev_addr_tx, config_addr); 2304 if (cfg->dir != STEDMA40_MEM_TO_PERIPH) 2305 dev_dbg(d40c->base->dev, 2306 "channel was not configured for memory " 2307 "to peripheral transfer (%d) overriding\n", 2308 cfg->dir); 2309 cfg->dir = STEDMA40_MEM_TO_PERIPH; 2310 2311 /* Configure the memory side */ 2312 if (src_addr_width == DMA_SLAVE_BUSWIDTH_UNDEFINED) 2313 src_addr_width = dst_addr_width; 2314 if (src_maxburst == 0) 2315 src_maxburst = dst_maxburst; 2316 } else { 2317 dev_err(d40c->base->dev, 2318 "unrecognized channel direction %d\n", 2319 config->direction); 2320 return -EINVAL; 2321 } 2322 2323 if (src_maxburst * src_addr_width != dst_maxburst * dst_addr_width) { 2324 dev_err(d40c->base->dev, 2325 "src/dst width/maxburst mismatch: %d*%d != %d*%d\n", 2326 src_maxburst, 2327 src_addr_width, 2328 dst_maxburst, 2329 dst_addr_width); 2330 return -EINVAL; 2331 } 2332 2333 ret = dma40_config_to_halfchannel(d40c, &cfg->src_info, 2334 src_addr_width, 2335 src_maxburst); 2336 if (ret) 2337 return ret; 2338 2339 ret = dma40_config_to_halfchannel(d40c, &cfg->dst_info, 2340 dst_addr_width, 2341 dst_maxburst); 2342 if (ret) 2343 return ret; 2344 2345 /* Fill in register values */ 2346 if (chan_is_logical(d40c)) 2347 d40_log_cfg(cfg, &d40c->log_def.lcsp1, &d40c->log_def.lcsp3); 2348 else 2349 d40_phy_cfg(cfg, &d40c->src_def_cfg, 2350 &d40c->dst_def_cfg, false); 2351 2352 /* These settings will take precedence later */ 2353 d40c->runtime_addr = config_addr; 2354 d40c->runtime_direction = config->direction; 2355 dev_dbg(d40c->base->dev, 2356 "configured channel %s for %s, data width %d/%d, " 2357 "maxburst %d/%d elements, LE, no flow control\n", 2358 dma_chan_name(chan), 2359 (config->direction == DMA_FROM_DEVICE) ? "RX" : "TX", 2360 src_addr_width, dst_addr_width, 2361 src_maxburst, dst_maxburst); 2362 2363 return 0; 2364 } 2365 2366 static int d40_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd, 2367 unsigned long arg) 2368 { 2369 struct d40_chan *d40c = container_of(chan, struct d40_chan, chan); 2370 2371 if (d40c->phy_chan == NULL) { 2372 chan_err(d40c, "Channel is not allocated!\n"); 2373 return -EINVAL; 2374 } 2375 2376 switch (cmd) { 2377 case DMA_TERMINATE_ALL: 2378 return d40_terminate_all(d40c); 2379 case DMA_PAUSE: 2380 return d40_pause(d40c); 2381 case DMA_RESUME: 2382 return d40_resume(d40c); 2383 case DMA_SLAVE_CONFIG: 2384 return d40_set_runtime_config(chan, 2385 (struct dma_slave_config *) arg); 2386 default: 2387 break; 2388 } 2389 2390 /* Other commands are unimplemented */ 2391 return -ENXIO; 2392 } 2393 2394 /* Initialization functions */ 2395 2396 static void __init d40_chan_init(struct d40_base *base, struct dma_device *dma, 2397 struct d40_chan *chans, int offset, 2398 int num_chans) 2399 { 2400 int i = 0; 2401 struct d40_chan *d40c; 2402 2403 INIT_LIST_HEAD(&dma->channels); 2404 2405 for (i = offset; i < offset + num_chans; i++) { 2406 d40c = &chans[i]; 2407 d40c->base = base; 2408 d40c->chan.device = dma; 2409 2410 spin_lock_init(&d40c->lock); 2411 2412 d40c->log_num = D40_PHY_CHAN; 2413 2414 INIT_LIST_HEAD(&d40c->active); 2415 INIT_LIST_HEAD(&d40c->queue); 2416 INIT_LIST_HEAD(&d40c->pending_queue); 2417 INIT_LIST_HEAD(&d40c->client); 2418 INIT_LIST_HEAD(&d40c->prepare_queue); 2419 2420 tasklet_init(&d40c->tasklet, dma_tasklet, 2421 (unsigned long) d40c); 2422 2423 list_add_tail(&d40c->chan.device_node, 2424 &dma->channels); 2425 } 2426 } 2427 2428 static void d40_ops_init(struct d40_base *base, struct dma_device *dev) 2429 { 2430 if (dma_has_cap(DMA_SLAVE, dev->cap_mask)) 2431 dev->device_prep_slave_sg = d40_prep_slave_sg; 2432 2433 if (dma_has_cap(DMA_MEMCPY, dev->cap_mask)) { 2434 dev->device_prep_dma_memcpy = d40_prep_memcpy; 2435 2436 /* 2437 * This controller can only access address at even 2438 * 32bit boundaries, i.e. 2^2 2439 */ 2440 dev->copy_align = 2; 2441 } 2442 2443 if (dma_has_cap(DMA_SG, dev->cap_mask)) 2444 dev->device_prep_dma_sg = d40_prep_memcpy_sg; 2445 2446 if (dma_has_cap(DMA_CYCLIC, dev->cap_mask)) 2447 dev->device_prep_dma_cyclic = dma40_prep_dma_cyclic; 2448 2449 dev->device_alloc_chan_resources = d40_alloc_chan_resources; 2450 dev->device_free_chan_resources = d40_free_chan_resources; 2451 dev->device_issue_pending = d40_issue_pending; 2452 dev->device_tx_status = d40_tx_status; 2453 dev->device_control = d40_control; 2454 dev->dev = base->dev; 2455 } 2456 2457 static int __init d40_dmaengine_init(struct d40_base *base, 2458 int num_reserved_chans) 2459 { 2460 int err ; 2461 2462 d40_chan_init(base, &base->dma_slave, base->log_chans, 2463 0, base->num_log_chans); 2464 2465 dma_cap_zero(base->dma_slave.cap_mask); 2466 dma_cap_set(DMA_SLAVE, base->dma_slave.cap_mask); 2467 dma_cap_set(DMA_CYCLIC, base->dma_slave.cap_mask); 2468 2469 d40_ops_init(base, &base->dma_slave); 2470 2471 err = dma_async_device_register(&base->dma_slave); 2472 2473 if (err) { 2474 d40_err(base->dev, "Failed to register slave channels\n"); 2475 goto failure1; 2476 } 2477 2478 d40_chan_init(base, &base->dma_memcpy, base->log_chans, 2479 base->num_log_chans, base->plat_data->memcpy_len); 2480 2481 dma_cap_zero(base->dma_memcpy.cap_mask); 2482 dma_cap_set(DMA_MEMCPY, base->dma_memcpy.cap_mask); 2483 dma_cap_set(DMA_SG, base->dma_memcpy.cap_mask); 2484 2485 d40_ops_init(base, &base->dma_memcpy); 2486 2487 err = dma_async_device_register(&base->dma_memcpy); 2488 2489 if (err) { 2490 d40_err(base->dev, 2491 "Failed to regsiter memcpy only channels\n"); 2492 goto failure2; 2493 } 2494 2495 d40_chan_init(base, &base->dma_both, base->phy_chans, 2496 0, num_reserved_chans); 2497 2498 dma_cap_zero(base->dma_both.cap_mask); 2499 dma_cap_set(DMA_SLAVE, base->dma_both.cap_mask); 2500 dma_cap_set(DMA_MEMCPY, base->dma_both.cap_mask); 2501 dma_cap_set(DMA_SG, base->dma_both.cap_mask); 2502 dma_cap_set(DMA_CYCLIC, base->dma_slave.cap_mask); 2503 2504 d40_ops_init(base, &base->dma_both); 2505 err = dma_async_device_register(&base->dma_both); 2506 2507 if (err) { 2508 d40_err(base->dev, 2509 "Failed to register logical and physical capable channels\n"); 2510 goto failure3; 2511 } 2512 return 0; 2513 failure3: 2514 dma_async_device_unregister(&base->dma_memcpy); 2515 failure2: 2516 dma_async_device_unregister(&base->dma_slave); 2517 failure1: 2518 return err; 2519 } 2520 2521 /* Initialization functions. */ 2522 2523 static int __init d40_phy_res_init(struct d40_base *base) 2524 { 2525 int i; 2526 int num_phy_chans_avail = 0; 2527 u32 val[2]; 2528 int odd_even_bit = -2; 2529 2530 val[0] = readl(base->virtbase + D40_DREG_PRSME); 2531 val[1] = readl(base->virtbase + D40_DREG_PRSMO); 2532 2533 for (i = 0; i < base->num_phy_chans; i++) { 2534 base->phy_res[i].num = i; 2535 odd_even_bit += 2 * ((i % 2) == 0); 2536 if (((val[i % 2] >> odd_even_bit) & 3) == 1) { 2537 /* Mark security only channels as occupied */ 2538 base->phy_res[i].allocated_src = D40_ALLOC_PHY; 2539 base->phy_res[i].allocated_dst = D40_ALLOC_PHY; 2540 } else { 2541 base->phy_res[i].allocated_src = D40_ALLOC_FREE; 2542 base->phy_res[i].allocated_dst = D40_ALLOC_FREE; 2543 num_phy_chans_avail++; 2544 } 2545 spin_lock_init(&base->phy_res[i].lock); 2546 } 2547 2548 /* Mark disabled channels as occupied */ 2549 for (i = 0; base->plat_data->disabled_channels[i] != -1; i++) { 2550 int chan = base->plat_data->disabled_channels[i]; 2551 2552 base->phy_res[chan].allocated_src = D40_ALLOC_PHY; 2553 base->phy_res[chan].allocated_dst = D40_ALLOC_PHY; 2554 num_phy_chans_avail--; 2555 } 2556 2557 dev_info(base->dev, "%d of %d physical DMA channels available\n", 2558 num_phy_chans_avail, base->num_phy_chans); 2559 2560 /* Verify settings extended vs standard */ 2561 val[0] = readl(base->virtbase + D40_DREG_PRTYP); 2562 2563 for (i = 0; i < base->num_phy_chans; i++) { 2564 2565 if (base->phy_res[i].allocated_src == D40_ALLOC_FREE && 2566 (val[0] & 0x3) != 1) 2567 dev_info(base->dev, 2568 "[%s] INFO: channel %d is misconfigured (%d)\n", 2569 __func__, i, val[0] & 0x3); 2570 2571 val[0] = val[0] >> 2; 2572 } 2573 2574 return num_phy_chans_avail; 2575 } 2576 2577 static struct d40_base * __init d40_hw_detect_init(struct platform_device *pdev) 2578 { 2579 struct stedma40_platform_data *plat_data; 2580 struct clk *clk = NULL; 2581 void __iomem *virtbase = NULL; 2582 struct resource *res = NULL; 2583 struct d40_base *base = NULL; 2584 int num_log_chans = 0; 2585 int num_phy_chans; 2586 int i; 2587 u32 pid; 2588 u32 cid; 2589 u8 rev; 2590 2591 clk = clk_get(&pdev->dev, NULL); 2592 2593 if (IS_ERR(clk)) { 2594 d40_err(&pdev->dev, "No matching clock found\n"); 2595 goto failure; 2596 } 2597 2598 clk_enable(clk); 2599 2600 /* Get IO for DMAC base address */ 2601 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "base"); 2602 if (!res) 2603 goto failure; 2604 2605 if (request_mem_region(res->start, resource_size(res), 2606 D40_NAME " I/O base") == NULL) 2607 goto failure; 2608 2609 virtbase = ioremap(res->start, resource_size(res)); 2610 if (!virtbase) 2611 goto failure; 2612 2613 /* This is just a regular AMBA PrimeCell ID actually */ 2614 for (pid = 0, i = 0; i < 4; i++) 2615 pid |= (readl(virtbase + resource_size(res) - 0x20 + 4 * i) 2616 & 255) << (i * 8); 2617 for (cid = 0, i = 0; i < 4; i++) 2618 cid |= (readl(virtbase + resource_size(res) - 0x10 + 4 * i) 2619 & 255) << (i * 8); 2620 2621 if (cid != AMBA_CID) { 2622 d40_err(&pdev->dev, "Unknown hardware! No PrimeCell ID\n"); 2623 goto failure; 2624 } 2625 if (AMBA_MANF_BITS(pid) != AMBA_VENDOR_ST) { 2626 d40_err(&pdev->dev, "Unknown designer! Got %x wanted %x\n", 2627 AMBA_MANF_BITS(pid), 2628 AMBA_VENDOR_ST); 2629 goto failure; 2630 } 2631 /* 2632 * HW revision: 2633 * DB8500ed has revision 0 2634 * ? has revision 1 2635 * DB8500v1 has revision 2 2636 * DB8500v2 has revision 3 2637 */ 2638 rev = AMBA_REV_BITS(pid); 2639 2640 /* The number of physical channels on this HW */ 2641 num_phy_chans = 4 * (readl(virtbase + D40_DREG_ICFG) & 0x7) + 4; 2642 2643 dev_info(&pdev->dev, "hardware revision: %d @ 0x%x\n", 2644 rev, res->start); 2645 2646 plat_data = pdev->dev.platform_data; 2647 2648 /* Count the number of logical channels in use */ 2649 for (i = 0; i < plat_data->dev_len; i++) 2650 if (plat_data->dev_rx[i] != 0) 2651 num_log_chans++; 2652 2653 for (i = 0; i < plat_data->dev_len; i++) 2654 if (plat_data->dev_tx[i] != 0) 2655 num_log_chans++; 2656 2657 base = kzalloc(ALIGN(sizeof(struct d40_base), 4) + 2658 (num_phy_chans + num_log_chans + plat_data->memcpy_len) * 2659 sizeof(struct d40_chan), GFP_KERNEL); 2660 2661 if (base == NULL) { 2662 d40_err(&pdev->dev, "Out of memory\n"); 2663 goto failure; 2664 } 2665 2666 base->rev = rev; 2667 base->clk = clk; 2668 base->num_phy_chans = num_phy_chans; 2669 base->num_log_chans = num_log_chans; 2670 base->phy_start = res->start; 2671 base->phy_size = resource_size(res); 2672 base->virtbase = virtbase; 2673 base->plat_data = plat_data; 2674 base->dev = &pdev->dev; 2675 base->phy_chans = ((void *)base) + ALIGN(sizeof(struct d40_base), 4); 2676 base->log_chans = &base->phy_chans[num_phy_chans]; 2677 2678 base->phy_res = kzalloc(num_phy_chans * sizeof(struct d40_phy_res), 2679 GFP_KERNEL); 2680 if (!base->phy_res) 2681 goto failure; 2682 2683 base->lookup_phy_chans = kzalloc(num_phy_chans * 2684 sizeof(struct d40_chan *), 2685 GFP_KERNEL); 2686 if (!base->lookup_phy_chans) 2687 goto failure; 2688 2689 if (num_log_chans + plat_data->memcpy_len) { 2690 /* 2691 * The max number of logical channels are event lines for all 2692 * src devices and dst devices 2693 */ 2694 base->lookup_log_chans = kzalloc(plat_data->dev_len * 2 * 2695 sizeof(struct d40_chan *), 2696 GFP_KERNEL); 2697 if (!base->lookup_log_chans) 2698 goto failure; 2699 } 2700 2701 base->lcla_pool.alloc_map = kzalloc(num_phy_chans * 2702 sizeof(struct d40_desc *) * 2703 D40_LCLA_LINK_PER_EVENT_GRP, 2704 GFP_KERNEL); 2705 if (!base->lcla_pool.alloc_map) 2706 goto failure; 2707 2708 base->desc_slab = kmem_cache_create(D40_NAME, sizeof(struct d40_desc), 2709 0, SLAB_HWCACHE_ALIGN, 2710 NULL); 2711 if (base->desc_slab == NULL) 2712 goto failure; 2713 2714 return base; 2715 2716 failure: 2717 if (!IS_ERR(clk)) { 2718 clk_disable(clk); 2719 clk_put(clk); 2720 } 2721 if (virtbase) 2722 iounmap(virtbase); 2723 if (res) 2724 release_mem_region(res->start, 2725 resource_size(res)); 2726 if (virtbase) 2727 iounmap(virtbase); 2728 2729 if (base) { 2730 kfree(base->lcla_pool.alloc_map); 2731 kfree(base->lookup_log_chans); 2732 kfree(base->lookup_phy_chans); 2733 kfree(base->phy_res); 2734 kfree(base); 2735 } 2736 2737 return NULL; 2738 } 2739 2740 static void __init d40_hw_init(struct d40_base *base) 2741 { 2742 2743 static const struct d40_reg_val dma_init_reg[] = { 2744 /* Clock every part of the DMA block from start */ 2745 { .reg = D40_DREG_GCC, .val = 0x0000ff01}, 2746 2747 /* Interrupts on all logical channels */ 2748 { .reg = D40_DREG_LCMIS0, .val = 0xFFFFFFFF}, 2749 { .reg = D40_DREG_LCMIS1, .val = 0xFFFFFFFF}, 2750 { .reg = D40_DREG_LCMIS2, .val = 0xFFFFFFFF}, 2751 { .reg = D40_DREG_LCMIS3, .val = 0xFFFFFFFF}, 2752 { .reg = D40_DREG_LCICR0, .val = 0xFFFFFFFF}, 2753 { .reg = D40_DREG_LCICR1, .val = 0xFFFFFFFF}, 2754 { .reg = D40_DREG_LCICR2, .val = 0xFFFFFFFF}, 2755 { .reg = D40_DREG_LCICR3, .val = 0xFFFFFFFF}, 2756 { .reg = D40_DREG_LCTIS0, .val = 0xFFFFFFFF}, 2757 { .reg = D40_DREG_LCTIS1, .val = 0xFFFFFFFF}, 2758 { .reg = D40_DREG_LCTIS2, .val = 0xFFFFFFFF}, 2759 { .reg = D40_DREG_LCTIS3, .val = 0xFFFFFFFF} 2760 }; 2761 int i; 2762 u32 prmseo[2] = {0, 0}; 2763 u32 activeo[2] = {0xFFFFFFFF, 0xFFFFFFFF}; 2764 u32 pcmis = 0; 2765 u32 pcicr = 0; 2766 2767 for (i = 0; i < ARRAY_SIZE(dma_init_reg); i++) 2768 writel(dma_init_reg[i].val, 2769 base->virtbase + dma_init_reg[i].reg); 2770 2771 /* Configure all our dma channels to default settings */ 2772 for (i = 0; i < base->num_phy_chans; i++) { 2773 2774 activeo[i % 2] = activeo[i % 2] << 2; 2775 2776 if (base->phy_res[base->num_phy_chans - i - 1].allocated_src 2777 == D40_ALLOC_PHY) { 2778 activeo[i % 2] |= 3; 2779 continue; 2780 } 2781 2782 /* Enable interrupt # */ 2783 pcmis = (pcmis << 1) | 1; 2784 2785 /* Clear interrupt # */ 2786 pcicr = (pcicr << 1) | 1; 2787 2788 /* Set channel to physical mode */ 2789 prmseo[i % 2] = prmseo[i % 2] << 2; 2790 prmseo[i % 2] |= 1; 2791 2792 } 2793 2794 writel(prmseo[1], base->virtbase + D40_DREG_PRMSE); 2795 writel(prmseo[0], base->virtbase + D40_DREG_PRMSO); 2796 writel(activeo[1], base->virtbase + D40_DREG_ACTIVE); 2797 writel(activeo[0], base->virtbase + D40_DREG_ACTIVO); 2798 2799 /* Write which interrupt to enable */ 2800 writel(pcmis, base->virtbase + D40_DREG_PCMIS); 2801 2802 /* Write which interrupt to clear */ 2803 writel(pcicr, base->virtbase + D40_DREG_PCICR); 2804 2805 } 2806 2807 static int __init d40_lcla_allocate(struct d40_base *base) 2808 { 2809 struct d40_lcla_pool *pool = &base->lcla_pool; 2810 unsigned long *page_list; 2811 int i, j; 2812 int ret = 0; 2813 2814 /* 2815 * This is somewhat ugly. We need 8192 bytes that are 18 bit aligned, 2816 * To full fill this hardware requirement without wasting 256 kb 2817 * we allocate pages until we get an aligned one. 2818 */ 2819 page_list = kmalloc(sizeof(unsigned long) * MAX_LCLA_ALLOC_ATTEMPTS, 2820 GFP_KERNEL); 2821 2822 if (!page_list) { 2823 ret = -ENOMEM; 2824 goto failure; 2825 } 2826 2827 /* Calculating how many pages that are required */ 2828 base->lcla_pool.pages = SZ_1K * base->num_phy_chans / PAGE_SIZE; 2829 2830 for (i = 0; i < MAX_LCLA_ALLOC_ATTEMPTS; i++) { 2831 page_list[i] = __get_free_pages(GFP_KERNEL, 2832 base->lcla_pool.pages); 2833 if (!page_list[i]) { 2834 2835 d40_err(base->dev, "Failed to allocate %d pages.\n", 2836 base->lcla_pool.pages); 2837 2838 for (j = 0; j < i; j++) 2839 free_pages(page_list[j], base->lcla_pool.pages); 2840 goto failure; 2841 } 2842 2843 if ((virt_to_phys((void *)page_list[i]) & 2844 (LCLA_ALIGNMENT - 1)) == 0) 2845 break; 2846 } 2847 2848 for (j = 0; j < i; j++) 2849 free_pages(page_list[j], base->lcla_pool.pages); 2850 2851 if (i < MAX_LCLA_ALLOC_ATTEMPTS) { 2852 base->lcla_pool.base = (void *)page_list[i]; 2853 } else { 2854 /* 2855 * After many attempts and no succees with finding the correct 2856 * alignment, try with allocating a big buffer. 2857 */ 2858 dev_warn(base->dev, 2859 "[%s] Failed to get %d pages @ 18 bit align.\n", 2860 __func__, base->lcla_pool.pages); 2861 base->lcla_pool.base_unaligned = kmalloc(SZ_1K * 2862 base->num_phy_chans + 2863 LCLA_ALIGNMENT, 2864 GFP_KERNEL); 2865 if (!base->lcla_pool.base_unaligned) { 2866 ret = -ENOMEM; 2867 goto failure; 2868 } 2869 2870 base->lcla_pool.base = PTR_ALIGN(base->lcla_pool.base_unaligned, 2871 LCLA_ALIGNMENT); 2872 } 2873 2874 pool->dma_addr = dma_map_single(base->dev, pool->base, 2875 SZ_1K * base->num_phy_chans, 2876 DMA_TO_DEVICE); 2877 if (dma_mapping_error(base->dev, pool->dma_addr)) { 2878 pool->dma_addr = 0; 2879 ret = -ENOMEM; 2880 goto failure; 2881 } 2882 2883 writel(virt_to_phys(base->lcla_pool.base), 2884 base->virtbase + D40_DREG_LCLA); 2885 failure: 2886 kfree(page_list); 2887 return ret; 2888 } 2889 2890 static int __init d40_probe(struct platform_device *pdev) 2891 { 2892 int err; 2893 int ret = -ENOENT; 2894 struct d40_base *base; 2895 struct resource *res = NULL; 2896 int num_reserved_chans; 2897 u32 val; 2898 2899 base = d40_hw_detect_init(pdev); 2900 2901 if (!base) 2902 goto failure; 2903 2904 num_reserved_chans = d40_phy_res_init(base); 2905 2906 platform_set_drvdata(pdev, base); 2907 2908 spin_lock_init(&base->interrupt_lock); 2909 spin_lock_init(&base->execmd_lock); 2910 2911 /* Get IO for logical channel parameter address */ 2912 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "lcpa"); 2913 if (!res) { 2914 ret = -ENOENT; 2915 d40_err(&pdev->dev, "No \"lcpa\" memory resource\n"); 2916 goto failure; 2917 } 2918 base->lcpa_size = resource_size(res); 2919 base->phy_lcpa = res->start; 2920 2921 if (request_mem_region(res->start, resource_size(res), 2922 D40_NAME " I/O lcpa") == NULL) { 2923 ret = -EBUSY; 2924 d40_err(&pdev->dev, 2925 "Failed to request LCPA region 0x%x-0x%x\n", 2926 res->start, res->end); 2927 goto failure; 2928 } 2929 2930 /* We make use of ESRAM memory for this. */ 2931 val = readl(base->virtbase + D40_DREG_LCPA); 2932 if (res->start != val && val != 0) { 2933 dev_warn(&pdev->dev, 2934 "[%s] Mismatch LCPA dma 0x%x, def 0x%x\n", 2935 __func__, val, res->start); 2936 } else 2937 writel(res->start, base->virtbase + D40_DREG_LCPA); 2938 2939 base->lcpa_base = ioremap(res->start, resource_size(res)); 2940 if (!base->lcpa_base) { 2941 ret = -ENOMEM; 2942 d40_err(&pdev->dev, "Failed to ioremap LCPA region\n"); 2943 goto failure; 2944 } 2945 2946 ret = d40_lcla_allocate(base); 2947 if (ret) { 2948 d40_err(&pdev->dev, "Failed to allocate LCLA area\n"); 2949 goto failure; 2950 } 2951 2952 spin_lock_init(&base->lcla_pool.lock); 2953 2954 base->irq = platform_get_irq(pdev, 0); 2955 2956 ret = request_irq(base->irq, d40_handle_interrupt, 0, D40_NAME, base); 2957 if (ret) { 2958 d40_err(&pdev->dev, "No IRQ defined\n"); 2959 goto failure; 2960 } 2961 2962 err = d40_dmaengine_init(base, num_reserved_chans); 2963 if (err) 2964 goto failure; 2965 2966 d40_hw_init(base); 2967 2968 dev_info(base->dev, "initialized\n"); 2969 return 0; 2970 2971 failure: 2972 if (base) { 2973 if (base->desc_slab) 2974 kmem_cache_destroy(base->desc_slab); 2975 if (base->virtbase) 2976 iounmap(base->virtbase); 2977 2978 if (base->lcla_pool.dma_addr) 2979 dma_unmap_single(base->dev, base->lcla_pool.dma_addr, 2980 SZ_1K * base->num_phy_chans, 2981 DMA_TO_DEVICE); 2982 2983 if (!base->lcla_pool.base_unaligned && base->lcla_pool.base) 2984 free_pages((unsigned long)base->lcla_pool.base, 2985 base->lcla_pool.pages); 2986 2987 kfree(base->lcla_pool.base_unaligned); 2988 2989 if (base->phy_lcpa) 2990 release_mem_region(base->phy_lcpa, 2991 base->lcpa_size); 2992 if (base->phy_start) 2993 release_mem_region(base->phy_start, 2994 base->phy_size); 2995 if (base->clk) { 2996 clk_disable(base->clk); 2997 clk_put(base->clk); 2998 } 2999 3000 kfree(base->lcla_pool.alloc_map); 3001 kfree(base->lookup_log_chans); 3002 kfree(base->lookup_phy_chans); 3003 kfree(base->phy_res); 3004 kfree(base); 3005 } 3006 3007 d40_err(&pdev->dev, "probe failed\n"); 3008 return ret; 3009 } 3010 3011 static struct platform_driver d40_driver = { 3012 .driver = { 3013 .owner = THIS_MODULE, 3014 .name = D40_NAME, 3015 }, 3016 }; 3017 3018 static int __init stedma40_init(void) 3019 { 3020 return platform_driver_probe(&d40_driver, d40_probe); 3021 } 3022 subsys_initcall(stedma40_init); 3023