1 /* 2 * offload engine driver for the Marvell XOR engine 3 * Copyright (C) 2007, 2008, Marvell International Ltd. 4 * 5 * This program is free software; you can redistribute it and/or modify it 6 * under the terms and conditions of the GNU General Public License, 7 * version 2, as published by the Free Software Foundation. 8 * 9 * This program is distributed in the hope it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 12 * more details. 13 */ 14 15 #include <linux/init.h> 16 #include <linux/slab.h> 17 #include <linux/delay.h> 18 #include <linux/dma-mapping.h> 19 #include <linux/spinlock.h> 20 #include <linux/interrupt.h> 21 #include <linux/of_device.h> 22 #include <linux/platform_device.h> 23 #include <linux/memory.h> 24 #include <linux/clk.h> 25 #include <linux/of.h> 26 #include <linux/of_irq.h> 27 #include <linux/irqdomain.h> 28 #include <linux/cpumask.h> 29 #include <linux/platform_data/dma-mv_xor.h> 30 31 #include "dmaengine.h" 32 #include "mv_xor.h" 33 34 enum mv_xor_type { 35 XOR_ORION, 36 XOR_ARMADA_38X, 37 XOR_ARMADA_37XX, 38 }; 39 40 enum mv_xor_mode { 41 XOR_MODE_IN_REG, 42 XOR_MODE_IN_DESC, 43 }; 44 45 static void mv_xor_issue_pending(struct dma_chan *chan); 46 47 #define to_mv_xor_chan(chan) \ 48 container_of(chan, struct mv_xor_chan, dmachan) 49 50 #define to_mv_xor_slot(tx) \ 51 container_of(tx, struct mv_xor_desc_slot, async_tx) 52 53 #define mv_chan_to_devp(chan) \ 54 ((chan)->dmadev.dev) 55 56 static void mv_desc_init(struct mv_xor_desc_slot *desc, 57 dma_addr_t addr, u32 byte_count, 58 enum dma_ctrl_flags flags) 59 { 60 struct mv_xor_desc *hw_desc = desc->hw_desc; 61 62 hw_desc->status = XOR_DESC_DMA_OWNED; 63 hw_desc->phy_next_desc = 0; 64 /* Enable end-of-descriptor interrupts only for DMA_PREP_INTERRUPT */ 65 hw_desc->desc_command = (flags & DMA_PREP_INTERRUPT) ? 66 XOR_DESC_EOD_INT_EN : 0; 67 hw_desc->phy_dest_addr = addr; 68 hw_desc->byte_count = byte_count; 69 } 70 71 static void mv_desc_set_mode(struct mv_xor_desc_slot *desc) 72 { 73 struct mv_xor_desc *hw_desc = desc->hw_desc; 74 75 switch (desc->type) { 76 case DMA_XOR: 77 case DMA_INTERRUPT: 78 hw_desc->desc_command |= XOR_DESC_OPERATION_XOR; 79 break; 80 case DMA_MEMCPY: 81 hw_desc->desc_command |= XOR_DESC_OPERATION_MEMCPY; 82 break; 83 default: 84 BUG(); 85 return; 86 } 87 } 88 89 static void mv_desc_set_next_desc(struct mv_xor_desc_slot *desc, 90 u32 next_desc_addr) 91 { 92 struct mv_xor_desc *hw_desc = desc->hw_desc; 93 BUG_ON(hw_desc->phy_next_desc); 94 hw_desc->phy_next_desc = next_desc_addr; 95 } 96 97 static void mv_desc_set_src_addr(struct mv_xor_desc_slot *desc, 98 int index, dma_addr_t addr) 99 { 100 struct mv_xor_desc *hw_desc = desc->hw_desc; 101 hw_desc->phy_src_addr[mv_phy_src_idx(index)] = addr; 102 if (desc->type == DMA_XOR) 103 hw_desc->desc_command |= (1 << index); 104 } 105 106 static u32 mv_chan_get_current_desc(struct mv_xor_chan *chan) 107 { 108 return readl_relaxed(XOR_CURR_DESC(chan)); 109 } 110 111 static void mv_chan_set_next_descriptor(struct mv_xor_chan *chan, 112 u32 next_desc_addr) 113 { 114 writel_relaxed(next_desc_addr, XOR_NEXT_DESC(chan)); 115 } 116 117 static void mv_chan_unmask_interrupts(struct mv_xor_chan *chan) 118 { 119 u32 val = readl_relaxed(XOR_INTR_MASK(chan)); 120 val |= XOR_INTR_MASK_VALUE << (chan->idx * 16); 121 writel_relaxed(val, XOR_INTR_MASK(chan)); 122 } 123 124 static u32 mv_chan_get_intr_cause(struct mv_xor_chan *chan) 125 { 126 u32 intr_cause = readl_relaxed(XOR_INTR_CAUSE(chan)); 127 intr_cause = (intr_cause >> (chan->idx * 16)) & 0xFFFF; 128 return intr_cause; 129 } 130 131 static void mv_chan_clear_eoc_cause(struct mv_xor_chan *chan) 132 { 133 u32 val; 134 135 val = XOR_INT_END_OF_DESC | XOR_INT_END_OF_CHAIN | XOR_INT_STOPPED; 136 val = ~(val << (chan->idx * 16)); 137 dev_dbg(mv_chan_to_devp(chan), "%s, val 0x%08x\n", __func__, val); 138 writel_relaxed(val, XOR_INTR_CAUSE(chan)); 139 } 140 141 static void mv_chan_clear_err_status(struct mv_xor_chan *chan) 142 { 143 u32 val = 0xFFFF0000 >> (chan->idx * 16); 144 writel_relaxed(val, XOR_INTR_CAUSE(chan)); 145 } 146 147 static void mv_chan_set_mode(struct mv_xor_chan *chan, 148 u32 op_mode) 149 { 150 u32 config = readl_relaxed(XOR_CONFIG(chan)); 151 152 config &= ~0x7; 153 config |= op_mode; 154 155 #if defined(__BIG_ENDIAN) 156 config |= XOR_DESCRIPTOR_SWAP; 157 #else 158 config &= ~XOR_DESCRIPTOR_SWAP; 159 #endif 160 161 writel_relaxed(config, XOR_CONFIG(chan)); 162 } 163 164 static void mv_chan_activate(struct mv_xor_chan *chan) 165 { 166 dev_dbg(mv_chan_to_devp(chan), " activate chan.\n"); 167 168 /* writel ensures all descriptors are flushed before activation */ 169 writel(BIT(0), XOR_ACTIVATION(chan)); 170 } 171 172 static char mv_chan_is_busy(struct mv_xor_chan *chan) 173 { 174 u32 state = readl_relaxed(XOR_ACTIVATION(chan)); 175 176 state = (state >> 4) & 0x3; 177 178 return (state == 1) ? 1 : 0; 179 } 180 181 /* 182 * mv_chan_start_new_chain - program the engine to operate on new 183 * chain headed by sw_desc 184 * Caller must hold &mv_chan->lock while calling this function 185 */ 186 static void mv_chan_start_new_chain(struct mv_xor_chan *mv_chan, 187 struct mv_xor_desc_slot *sw_desc) 188 { 189 dev_dbg(mv_chan_to_devp(mv_chan), "%s %d: sw_desc %p\n", 190 __func__, __LINE__, sw_desc); 191 192 /* set the hardware chain */ 193 mv_chan_set_next_descriptor(mv_chan, sw_desc->async_tx.phys); 194 195 mv_chan->pending++; 196 mv_xor_issue_pending(&mv_chan->dmachan); 197 } 198 199 static dma_cookie_t 200 mv_desc_run_tx_complete_actions(struct mv_xor_desc_slot *desc, 201 struct mv_xor_chan *mv_chan, 202 dma_cookie_t cookie) 203 { 204 BUG_ON(desc->async_tx.cookie < 0); 205 206 if (desc->async_tx.cookie > 0) { 207 cookie = desc->async_tx.cookie; 208 209 /* call the callback (must not sleep or submit new 210 * operations to this channel) 211 */ 212 if (desc->async_tx.callback) 213 desc->async_tx.callback( 214 desc->async_tx.callback_param); 215 216 dma_descriptor_unmap(&desc->async_tx); 217 } 218 219 /* run dependent operations */ 220 dma_run_dependencies(&desc->async_tx); 221 222 return cookie; 223 } 224 225 static int 226 mv_chan_clean_completed_slots(struct mv_xor_chan *mv_chan) 227 { 228 struct mv_xor_desc_slot *iter, *_iter; 229 230 dev_dbg(mv_chan_to_devp(mv_chan), "%s %d\n", __func__, __LINE__); 231 list_for_each_entry_safe(iter, _iter, &mv_chan->completed_slots, 232 node) { 233 234 if (async_tx_test_ack(&iter->async_tx)) 235 list_move_tail(&iter->node, &mv_chan->free_slots); 236 } 237 return 0; 238 } 239 240 static int 241 mv_desc_clean_slot(struct mv_xor_desc_slot *desc, 242 struct mv_xor_chan *mv_chan) 243 { 244 dev_dbg(mv_chan_to_devp(mv_chan), "%s %d: desc %p flags %d\n", 245 __func__, __LINE__, desc, desc->async_tx.flags); 246 247 /* the client is allowed to attach dependent operations 248 * until 'ack' is set 249 */ 250 if (!async_tx_test_ack(&desc->async_tx)) 251 /* move this slot to the completed_slots */ 252 list_move_tail(&desc->node, &mv_chan->completed_slots); 253 else 254 list_move_tail(&desc->node, &mv_chan->free_slots); 255 256 return 0; 257 } 258 259 /* This function must be called with the mv_xor_chan spinlock held */ 260 static void mv_chan_slot_cleanup(struct mv_xor_chan *mv_chan) 261 { 262 struct mv_xor_desc_slot *iter, *_iter; 263 dma_cookie_t cookie = 0; 264 int busy = mv_chan_is_busy(mv_chan); 265 u32 current_desc = mv_chan_get_current_desc(mv_chan); 266 int current_cleaned = 0; 267 struct mv_xor_desc *hw_desc; 268 269 dev_dbg(mv_chan_to_devp(mv_chan), "%s %d\n", __func__, __LINE__); 270 dev_dbg(mv_chan_to_devp(mv_chan), "current_desc %x\n", current_desc); 271 mv_chan_clean_completed_slots(mv_chan); 272 273 /* free completed slots from the chain starting with 274 * the oldest descriptor 275 */ 276 277 list_for_each_entry_safe(iter, _iter, &mv_chan->chain, 278 node) { 279 280 /* clean finished descriptors */ 281 hw_desc = iter->hw_desc; 282 if (hw_desc->status & XOR_DESC_SUCCESS) { 283 cookie = mv_desc_run_tx_complete_actions(iter, mv_chan, 284 cookie); 285 286 /* done processing desc, clean slot */ 287 mv_desc_clean_slot(iter, mv_chan); 288 289 /* break if we did cleaned the current */ 290 if (iter->async_tx.phys == current_desc) { 291 current_cleaned = 1; 292 break; 293 } 294 } else { 295 if (iter->async_tx.phys == current_desc) { 296 current_cleaned = 0; 297 break; 298 } 299 } 300 } 301 302 if ((busy == 0) && !list_empty(&mv_chan->chain)) { 303 if (current_cleaned) { 304 /* 305 * current descriptor cleaned and removed, run 306 * from list head 307 */ 308 iter = list_entry(mv_chan->chain.next, 309 struct mv_xor_desc_slot, 310 node); 311 mv_chan_start_new_chain(mv_chan, iter); 312 } else { 313 if (!list_is_last(&iter->node, &mv_chan->chain)) { 314 /* 315 * descriptors are still waiting after 316 * current, trigger them 317 */ 318 iter = list_entry(iter->node.next, 319 struct mv_xor_desc_slot, 320 node); 321 mv_chan_start_new_chain(mv_chan, iter); 322 } else { 323 /* 324 * some descriptors are still waiting 325 * to be cleaned 326 */ 327 tasklet_schedule(&mv_chan->irq_tasklet); 328 } 329 } 330 } 331 332 if (cookie > 0) 333 mv_chan->dmachan.completed_cookie = cookie; 334 } 335 336 static void mv_xor_tasklet(unsigned long data) 337 { 338 struct mv_xor_chan *chan = (struct mv_xor_chan *) data; 339 340 spin_lock_bh(&chan->lock); 341 mv_chan_slot_cleanup(chan); 342 spin_unlock_bh(&chan->lock); 343 } 344 345 static struct mv_xor_desc_slot * 346 mv_chan_alloc_slot(struct mv_xor_chan *mv_chan) 347 { 348 struct mv_xor_desc_slot *iter; 349 350 spin_lock_bh(&mv_chan->lock); 351 352 if (!list_empty(&mv_chan->free_slots)) { 353 iter = list_first_entry(&mv_chan->free_slots, 354 struct mv_xor_desc_slot, 355 node); 356 357 list_move_tail(&iter->node, &mv_chan->allocated_slots); 358 359 spin_unlock_bh(&mv_chan->lock); 360 361 /* pre-ack descriptor */ 362 async_tx_ack(&iter->async_tx); 363 iter->async_tx.cookie = -EBUSY; 364 365 return iter; 366 367 } 368 369 spin_unlock_bh(&mv_chan->lock); 370 371 /* try to free some slots if the allocation fails */ 372 tasklet_schedule(&mv_chan->irq_tasklet); 373 374 return NULL; 375 } 376 377 /************************ DMA engine API functions ****************************/ 378 static dma_cookie_t 379 mv_xor_tx_submit(struct dma_async_tx_descriptor *tx) 380 { 381 struct mv_xor_desc_slot *sw_desc = to_mv_xor_slot(tx); 382 struct mv_xor_chan *mv_chan = to_mv_xor_chan(tx->chan); 383 struct mv_xor_desc_slot *old_chain_tail; 384 dma_cookie_t cookie; 385 int new_hw_chain = 1; 386 387 dev_dbg(mv_chan_to_devp(mv_chan), 388 "%s sw_desc %p: async_tx %p\n", 389 __func__, sw_desc, &sw_desc->async_tx); 390 391 spin_lock_bh(&mv_chan->lock); 392 cookie = dma_cookie_assign(tx); 393 394 if (list_empty(&mv_chan->chain)) 395 list_move_tail(&sw_desc->node, &mv_chan->chain); 396 else { 397 new_hw_chain = 0; 398 399 old_chain_tail = list_entry(mv_chan->chain.prev, 400 struct mv_xor_desc_slot, 401 node); 402 list_move_tail(&sw_desc->node, &mv_chan->chain); 403 404 dev_dbg(mv_chan_to_devp(mv_chan), "Append to last desc %pa\n", 405 &old_chain_tail->async_tx.phys); 406 407 /* fix up the hardware chain */ 408 mv_desc_set_next_desc(old_chain_tail, sw_desc->async_tx.phys); 409 410 /* if the channel is not busy */ 411 if (!mv_chan_is_busy(mv_chan)) { 412 u32 current_desc = mv_chan_get_current_desc(mv_chan); 413 /* 414 * and the curren desc is the end of the chain before 415 * the append, then we need to start the channel 416 */ 417 if (current_desc == old_chain_tail->async_tx.phys) 418 new_hw_chain = 1; 419 } 420 } 421 422 if (new_hw_chain) 423 mv_chan_start_new_chain(mv_chan, sw_desc); 424 425 spin_unlock_bh(&mv_chan->lock); 426 427 return cookie; 428 } 429 430 /* returns the number of allocated descriptors */ 431 static int mv_xor_alloc_chan_resources(struct dma_chan *chan) 432 { 433 void *virt_desc; 434 dma_addr_t dma_desc; 435 int idx; 436 struct mv_xor_chan *mv_chan = to_mv_xor_chan(chan); 437 struct mv_xor_desc_slot *slot = NULL; 438 int num_descs_in_pool = MV_XOR_POOL_SIZE/MV_XOR_SLOT_SIZE; 439 440 /* Allocate descriptor slots */ 441 idx = mv_chan->slots_allocated; 442 while (idx < num_descs_in_pool) { 443 slot = kzalloc(sizeof(*slot), GFP_KERNEL); 444 if (!slot) { 445 dev_info(mv_chan_to_devp(mv_chan), 446 "channel only initialized %d descriptor slots", 447 idx); 448 break; 449 } 450 virt_desc = mv_chan->dma_desc_pool_virt; 451 slot->hw_desc = virt_desc + idx * MV_XOR_SLOT_SIZE; 452 453 dma_async_tx_descriptor_init(&slot->async_tx, chan); 454 slot->async_tx.tx_submit = mv_xor_tx_submit; 455 INIT_LIST_HEAD(&slot->node); 456 dma_desc = mv_chan->dma_desc_pool; 457 slot->async_tx.phys = dma_desc + idx * MV_XOR_SLOT_SIZE; 458 slot->idx = idx++; 459 460 spin_lock_bh(&mv_chan->lock); 461 mv_chan->slots_allocated = idx; 462 list_add_tail(&slot->node, &mv_chan->free_slots); 463 spin_unlock_bh(&mv_chan->lock); 464 } 465 466 dev_dbg(mv_chan_to_devp(mv_chan), 467 "allocated %d descriptor slots\n", 468 mv_chan->slots_allocated); 469 470 return mv_chan->slots_allocated ? : -ENOMEM; 471 } 472 473 /* 474 * Check if source or destination is an PCIe/IO address (non-SDRAM) and add 475 * a new MBus window if necessary. Use a cache for these check so that 476 * the MMIO mapped registers don't have to be accessed for this check 477 * to speed up this process. 478 */ 479 static int mv_xor_add_io_win(struct mv_xor_chan *mv_chan, u32 addr) 480 { 481 struct mv_xor_device *xordev = mv_chan->xordev; 482 void __iomem *base = mv_chan->mmr_high_base; 483 u32 win_enable; 484 u32 size; 485 u8 target, attr; 486 int ret; 487 int i; 488 489 /* Nothing needs to get done for the Armada 3700 */ 490 if (xordev->xor_type == XOR_ARMADA_37XX) 491 return 0; 492 493 /* 494 * Loop over the cached windows to check, if the requested area 495 * is already mapped. If this the case, nothing needs to be done 496 * and we can return. 497 */ 498 for (i = 0; i < WINDOW_COUNT; i++) { 499 if (addr >= xordev->win_start[i] && 500 addr <= xordev->win_end[i]) { 501 /* Window is already mapped */ 502 return 0; 503 } 504 } 505 506 /* 507 * The window is not mapped, so we need to create the new mapping 508 */ 509 510 /* If no IO window is found that addr has to be located in SDRAM */ 511 ret = mvebu_mbus_get_io_win_info(addr, &size, &target, &attr); 512 if (ret < 0) 513 return 0; 514 515 /* 516 * Mask the base addr 'addr' according to 'size' read back from the 517 * MBus window. Otherwise we might end up with an address located 518 * somewhere in the middle of this area here. 519 */ 520 size -= 1; 521 addr &= ~size; 522 523 /* 524 * Reading one of both enabled register is enough, as they are always 525 * programmed to the identical values 526 */ 527 win_enable = readl(base + WINDOW_BAR_ENABLE(0)); 528 529 /* Set 'i' to the first free window to write the new values to */ 530 i = ffs(~win_enable) - 1; 531 if (i >= WINDOW_COUNT) 532 return -ENOMEM; 533 534 writel((addr & 0xffff0000) | (attr << 8) | target, 535 base + WINDOW_BASE(i)); 536 writel(size & 0xffff0000, base + WINDOW_SIZE(i)); 537 538 /* Fill the caching variables for later use */ 539 xordev->win_start[i] = addr; 540 xordev->win_end[i] = addr + size; 541 542 win_enable |= (1 << i); 543 win_enable |= 3 << (16 + (2 * i)); 544 writel(win_enable, base + WINDOW_BAR_ENABLE(0)); 545 writel(win_enable, base + WINDOW_BAR_ENABLE(1)); 546 547 return 0; 548 } 549 550 static struct dma_async_tx_descriptor * 551 mv_xor_prep_dma_xor(struct dma_chan *chan, dma_addr_t dest, dma_addr_t *src, 552 unsigned int src_cnt, size_t len, unsigned long flags) 553 { 554 struct mv_xor_chan *mv_chan = to_mv_xor_chan(chan); 555 struct mv_xor_desc_slot *sw_desc; 556 int ret; 557 558 if (unlikely(len < MV_XOR_MIN_BYTE_COUNT)) 559 return NULL; 560 561 BUG_ON(len > MV_XOR_MAX_BYTE_COUNT); 562 563 dev_dbg(mv_chan_to_devp(mv_chan), 564 "%s src_cnt: %d len: %zu dest %pad flags: %ld\n", 565 __func__, src_cnt, len, &dest, flags); 566 567 /* Check if a new window needs to get added for 'dest' */ 568 ret = mv_xor_add_io_win(mv_chan, dest); 569 if (ret) 570 return NULL; 571 572 sw_desc = mv_chan_alloc_slot(mv_chan); 573 if (sw_desc) { 574 sw_desc->type = DMA_XOR; 575 sw_desc->async_tx.flags = flags; 576 mv_desc_init(sw_desc, dest, len, flags); 577 if (mv_chan->op_in_desc == XOR_MODE_IN_DESC) 578 mv_desc_set_mode(sw_desc); 579 while (src_cnt--) { 580 /* Check if a new window needs to get added for 'src' */ 581 ret = mv_xor_add_io_win(mv_chan, src[src_cnt]); 582 if (ret) 583 return NULL; 584 mv_desc_set_src_addr(sw_desc, src_cnt, src[src_cnt]); 585 } 586 } 587 588 dev_dbg(mv_chan_to_devp(mv_chan), 589 "%s sw_desc %p async_tx %p \n", 590 __func__, sw_desc, &sw_desc->async_tx); 591 return sw_desc ? &sw_desc->async_tx : NULL; 592 } 593 594 static struct dma_async_tx_descriptor * 595 mv_xor_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, 596 size_t len, unsigned long flags) 597 { 598 /* 599 * A MEMCPY operation is identical to an XOR operation with only 600 * a single source address. 601 */ 602 return mv_xor_prep_dma_xor(chan, dest, &src, 1, len, flags); 603 } 604 605 static struct dma_async_tx_descriptor * 606 mv_xor_prep_dma_interrupt(struct dma_chan *chan, unsigned long flags) 607 { 608 struct mv_xor_chan *mv_chan = to_mv_xor_chan(chan); 609 dma_addr_t src, dest; 610 size_t len; 611 612 src = mv_chan->dummy_src_addr; 613 dest = mv_chan->dummy_dst_addr; 614 len = MV_XOR_MIN_BYTE_COUNT; 615 616 /* 617 * We implement the DMA_INTERRUPT operation as a minimum sized 618 * XOR operation with a single dummy source address. 619 */ 620 return mv_xor_prep_dma_xor(chan, dest, &src, 1, len, flags); 621 } 622 623 static void mv_xor_free_chan_resources(struct dma_chan *chan) 624 { 625 struct mv_xor_chan *mv_chan = to_mv_xor_chan(chan); 626 struct mv_xor_desc_slot *iter, *_iter; 627 int in_use_descs = 0; 628 629 spin_lock_bh(&mv_chan->lock); 630 631 mv_chan_slot_cleanup(mv_chan); 632 633 list_for_each_entry_safe(iter, _iter, &mv_chan->chain, 634 node) { 635 in_use_descs++; 636 list_move_tail(&iter->node, &mv_chan->free_slots); 637 } 638 list_for_each_entry_safe(iter, _iter, &mv_chan->completed_slots, 639 node) { 640 in_use_descs++; 641 list_move_tail(&iter->node, &mv_chan->free_slots); 642 } 643 list_for_each_entry_safe(iter, _iter, &mv_chan->allocated_slots, 644 node) { 645 in_use_descs++; 646 list_move_tail(&iter->node, &mv_chan->free_slots); 647 } 648 list_for_each_entry_safe_reverse( 649 iter, _iter, &mv_chan->free_slots, node) { 650 list_del(&iter->node); 651 kfree(iter); 652 mv_chan->slots_allocated--; 653 } 654 655 dev_dbg(mv_chan_to_devp(mv_chan), "%s slots_allocated %d\n", 656 __func__, mv_chan->slots_allocated); 657 spin_unlock_bh(&mv_chan->lock); 658 659 if (in_use_descs) 660 dev_err(mv_chan_to_devp(mv_chan), 661 "freeing %d in use descriptors!\n", in_use_descs); 662 } 663 664 /** 665 * mv_xor_status - poll the status of an XOR transaction 666 * @chan: XOR channel handle 667 * @cookie: XOR transaction identifier 668 * @txstate: XOR transactions state holder (or NULL) 669 */ 670 static enum dma_status mv_xor_status(struct dma_chan *chan, 671 dma_cookie_t cookie, 672 struct dma_tx_state *txstate) 673 { 674 struct mv_xor_chan *mv_chan = to_mv_xor_chan(chan); 675 enum dma_status ret; 676 677 ret = dma_cookie_status(chan, cookie, txstate); 678 if (ret == DMA_COMPLETE) 679 return ret; 680 681 spin_lock_bh(&mv_chan->lock); 682 mv_chan_slot_cleanup(mv_chan); 683 spin_unlock_bh(&mv_chan->lock); 684 685 return dma_cookie_status(chan, cookie, txstate); 686 } 687 688 static void mv_chan_dump_regs(struct mv_xor_chan *chan) 689 { 690 u32 val; 691 692 val = readl_relaxed(XOR_CONFIG(chan)); 693 dev_err(mv_chan_to_devp(chan), "config 0x%08x\n", val); 694 695 val = readl_relaxed(XOR_ACTIVATION(chan)); 696 dev_err(mv_chan_to_devp(chan), "activation 0x%08x\n", val); 697 698 val = readl_relaxed(XOR_INTR_CAUSE(chan)); 699 dev_err(mv_chan_to_devp(chan), "intr cause 0x%08x\n", val); 700 701 val = readl_relaxed(XOR_INTR_MASK(chan)); 702 dev_err(mv_chan_to_devp(chan), "intr mask 0x%08x\n", val); 703 704 val = readl_relaxed(XOR_ERROR_CAUSE(chan)); 705 dev_err(mv_chan_to_devp(chan), "error cause 0x%08x\n", val); 706 707 val = readl_relaxed(XOR_ERROR_ADDR(chan)); 708 dev_err(mv_chan_to_devp(chan), "error addr 0x%08x\n", val); 709 } 710 711 static void mv_chan_err_interrupt_handler(struct mv_xor_chan *chan, 712 u32 intr_cause) 713 { 714 if (intr_cause & XOR_INT_ERR_DECODE) { 715 dev_dbg(mv_chan_to_devp(chan), "ignoring address decode error\n"); 716 return; 717 } 718 719 dev_err(mv_chan_to_devp(chan), "error on chan %d. intr cause 0x%08x\n", 720 chan->idx, intr_cause); 721 722 mv_chan_dump_regs(chan); 723 WARN_ON(1); 724 } 725 726 static irqreturn_t mv_xor_interrupt_handler(int irq, void *data) 727 { 728 struct mv_xor_chan *chan = data; 729 u32 intr_cause = mv_chan_get_intr_cause(chan); 730 731 dev_dbg(mv_chan_to_devp(chan), "intr cause %x\n", intr_cause); 732 733 if (intr_cause & XOR_INTR_ERRORS) 734 mv_chan_err_interrupt_handler(chan, intr_cause); 735 736 tasklet_schedule(&chan->irq_tasklet); 737 738 mv_chan_clear_eoc_cause(chan); 739 740 return IRQ_HANDLED; 741 } 742 743 static void mv_xor_issue_pending(struct dma_chan *chan) 744 { 745 struct mv_xor_chan *mv_chan = to_mv_xor_chan(chan); 746 747 if (mv_chan->pending >= MV_XOR_THRESHOLD) { 748 mv_chan->pending = 0; 749 mv_chan_activate(mv_chan); 750 } 751 } 752 753 /* 754 * Perform a transaction to verify the HW works. 755 */ 756 757 static int mv_chan_memcpy_self_test(struct mv_xor_chan *mv_chan) 758 { 759 int i, ret; 760 void *src, *dest; 761 dma_addr_t src_dma, dest_dma; 762 struct dma_chan *dma_chan; 763 dma_cookie_t cookie; 764 struct dma_async_tx_descriptor *tx; 765 struct dmaengine_unmap_data *unmap; 766 int err = 0; 767 768 src = kmalloc(sizeof(u8) * PAGE_SIZE, GFP_KERNEL); 769 if (!src) 770 return -ENOMEM; 771 772 dest = kzalloc(sizeof(u8) * PAGE_SIZE, GFP_KERNEL); 773 if (!dest) { 774 kfree(src); 775 return -ENOMEM; 776 } 777 778 /* Fill in src buffer */ 779 for (i = 0; i < PAGE_SIZE; i++) 780 ((u8 *) src)[i] = (u8)i; 781 782 dma_chan = &mv_chan->dmachan; 783 if (mv_xor_alloc_chan_resources(dma_chan) < 1) { 784 err = -ENODEV; 785 goto out; 786 } 787 788 unmap = dmaengine_get_unmap_data(dma_chan->device->dev, 2, GFP_KERNEL); 789 if (!unmap) { 790 err = -ENOMEM; 791 goto free_resources; 792 } 793 794 src_dma = dma_map_page(dma_chan->device->dev, virt_to_page(src), 795 (size_t)src & ~PAGE_MASK, PAGE_SIZE, 796 DMA_TO_DEVICE); 797 unmap->addr[0] = src_dma; 798 799 ret = dma_mapping_error(dma_chan->device->dev, src_dma); 800 if (ret) { 801 err = -ENOMEM; 802 goto free_resources; 803 } 804 unmap->to_cnt = 1; 805 806 dest_dma = dma_map_page(dma_chan->device->dev, virt_to_page(dest), 807 (size_t)dest & ~PAGE_MASK, PAGE_SIZE, 808 DMA_FROM_DEVICE); 809 unmap->addr[1] = dest_dma; 810 811 ret = dma_mapping_error(dma_chan->device->dev, dest_dma); 812 if (ret) { 813 err = -ENOMEM; 814 goto free_resources; 815 } 816 unmap->from_cnt = 1; 817 unmap->len = PAGE_SIZE; 818 819 tx = mv_xor_prep_dma_memcpy(dma_chan, dest_dma, src_dma, 820 PAGE_SIZE, 0); 821 if (!tx) { 822 dev_err(dma_chan->device->dev, 823 "Self-test cannot prepare operation, disabling\n"); 824 err = -ENODEV; 825 goto free_resources; 826 } 827 828 cookie = mv_xor_tx_submit(tx); 829 if (dma_submit_error(cookie)) { 830 dev_err(dma_chan->device->dev, 831 "Self-test submit error, disabling\n"); 832 err = -ENODEV; 833 goto free_resources; 834 } 835 836 mv_xor_issue_pending(dma_chan); 837 async_tx_ack(tx); 838 msleep(1); 839 840 if (mv_xor_status(dma_chan, cookie, NULL) != 841 DMA_COMPLETE) { 842 dev_err(dma_chan->device->dev, 843 "Self-test copy timed out, disabling\n"); 844 err = -ENODEV; 845 goto free_resources; 846 } 847 848 dma_sync_single_for_cpu(dma_chan->device->dev, dest_dma, 849 PAGE_SIZE, DMA_FROM_DEVICE); 850 if (memcmp(src, dest, PAGE_SIZE)) { 851 dev_err(dma_chan->device->dev, 852 "Self-test copy failed compare, disabling\n"); 853 err = -ENODEV; 854 goto free_resources; 855 } 856 857 free_resources: 858 dmaengine_unmap_put(unmap); 859 mv_xor_free_chan_resources(dma_chan); 860 out: 861 kfree(src); 862 kfree(dest); 863 return err; 864 } 865 866 #define MV_XOR_NUM_SRC_TEST 4 /* must be <= 15 */ 867 static int 868 mv_chan_xor_self_test(struct mv_xor_chan *mv_chan) 869 { 870 int i, src_idx, ret; 871 struct page *dest; 872 struct page *xor_srcs[MV_XOR_NUM_SRC_TEST]; 873 dma_addr_t dma_srcs[MV_XOR_NUM_SRC_TEST]; 874 dma_addr_t dest_dma; 875 struct dma_async_tx_descriptor *tx; 876 struct dmaengine_unmap_data *unmap; 877 struct dma_chan *dma_chan; 878 dma_cookie_t cookie; 879 u8 cmp_byte = 0; 880 u32 cmp_word; 881 int err = 0; 882 int src_count = MV_XOR_NUM_SRC_TEST; 883 884 for (src_idx = 0; src_idx < src_count; src_idx++) { 885 xor_srcs[src_idx] = alloc_page(GFP_KERNEL); 886 if (!xor_srcs[src_idx]) { 887 while (src_idx--) 888 __free_page(xor_srcs[src_idx]); 889 return -ENOMEM; 890 } 891 } 892 893 dest = alloc_page(GFP_KERNEL); 894 if (!dest) { 895 while (src_idx--) 896 __free_page(xor_srcs[src_idx]); 897 return -ENOMEM; 898 } 899 900 /* Fill in src buffers */ 901 for (src_idx = 0; src_idx < src_count; src_idx++) { 902 u8 *ptr = page_address(xor_srcs[src_idx]); 903 for (i = 0; i < PAGE_SIZE; i++) 904 ptr[i] = (1 << src_idx); 905 } 906 907 for (src_idx = 0; src_idx < src_count; src_idx++) 908 cmp_byte ^= (u8) (1 << src_idx); 909 910 cmp_word = (cmp_byte << 24) | (cmp_byte << 16) | 911 (cmp_byte << 8) | cmp_byte; 912 913 memset(page_address(dest), 0, PAGE_SIZE); 914 915 dma_chan = &mv_chan->dmachan; 916 if (mv_xor_alloc_chan_resources(dma_chan) < 1) { 917 err = -ENODEV; 918 goto out; 919 } 920 921 unmap = dmaengine_get_unmap_data(dma_chan->device->dev, src_count + 1, 922 GFP_KERNEL); 923 if (!unmap) { 924 err = -ENOMEM; 925 goto free_resources; 926 } 927 928 /* test xor */ 929 for (i = 0; i < src_count; i++) { 930 unmap->addr[i] = dma_map_page(dma_chan->device->dev, xor_srcs[i], 931 0, PAGE_SIZE, DMA_TO_DEVICE); 932 dma_srcs[i] = unmap->addr[i]; 933 ret = dma_mapping_error(dma_chan->device->dev, unmap->addr[i]); 934 if (ret) { 935 err = -ENOMEM; 936 goto free_resources; 937 } 938 unmap->to_cnt++; 939 } 940 941 unmap->addr[src_count] = dma_map_page(dma_chan->device->dev, dest, 0, PAGE_SIZE, 942 DMA_FROM_DEVICE); 943 dest_dma = unmap->addr[src_count]; 944 ret = dma_mapping_error(dma_chan->device->dev, unmap->addr[src_count]); 945 if (ret) { 946 err = -ENOMEM; 947 goto free_resources; 948 } 949 unmap->from_cnt = 1; 950 unmap->len = PAGE_SIZE; 951 952 tx = mv_xor_prep_dma_xor(dma_chan, dest_dma, dma_srcs, 953 src_count, PAGE_SIZE, 0); 954 if (!tx) { 955 dev_err(dma_chan->device->dev, 956 "Self-test cannot prepare operation, disabling\n"); 957 err = -ENODEV; 958 goto free_resources; 959 } 960 961 cookie = mv_xor_tx_submit(tx); 962 if (dma_submit_error(cookie)) { 963 dev_err(dma_chan->device->dev, 964 "Self-test submit error, disabling\n"); 965 err = -ENODEV; 966 goto free_resources; 967 } 968 969 mv_xor_issue_pending(dma_chan); 970 async_tx_ack(tx); 971 msleep(8); 972 973 if (mv_xor_status(dma_chan, cookie, NULL) != 974 DMA_COMPLETE) { 975 dev_err(dma_chan->device->dev, 976 "Self-test xor timed out, disabling\n"); 977 err = -ENODEV; 978 goto free_resources; 979 } 980 981 dma_sync_single_for_cpu(dma_chan->device->dev, dest_dma, 982 PAGE_SIZE, DMA_FROM_DEVICE); 983 for (i = 0; i < (PAGE_SIZE / sizeof(u32)); i++) { 984 u32 *ptr = page_address(dest); 985 if (ptr[i] != cmp_word) { 986 dev_err(dma_chan->device->dev, 987 "Self-test xor failed compare, disabling. index %d, data %x, expected %x\n", 988 i, ptr[i], cmp_word); 989 err = -ENODEV; 990 goto free_resources; 991 } 992 } 993 994 free_resources: 995 dmaengine_unmap_put(unmap); 996 mv_xor_free_chan_resources(dma_chan); 997 out: 998 src_idx = src_count; 999 while (src_idx--) 1000 __free_page(xor_srcs[src_idx]); 1001 __free_page(dest); 1002 return err; 1003 } 1004 1005 static int mv_xor_channel_remove(struct mv_xor_chan *mv_chan) 1006 { 1007 struct dma_chan *chan, *_chan; 1008 struct device *dev = mv_chan->dmadev.dev; 1009 1010 dma_async_device_unregister(&mv_chan->dmadev); 1011 1012 dma_free_coherent(dev, MV_XOR_POOL_SIZE, 1013 mv_chan->dma_desc_pool_virt, mv_chan->dma_desc_pool); 1014 dma_unmap_single(dev, mv_chan->dummy_src_addr, 1015 MV_XOR_MIN_BYTE_COUNT, DMA_FROM_DEVICE); 1016 dma_unmap_single(dev, mv_chan->dummy_dst_addr, 1017 MV_XOR_MIN_BYTE_COUNT, DMA_TO_DEVICE); 1018 1019 list_for_each_entry_safe(chan, _chan, &mv_chan->dmadev.channels, 1020 device_node) { 1021 list_del(&chan->device_node); 1022 } 1023 1024 free_irq(mv_chan->irq, mv_chan); 1025 1026 return 0; 1027 } 1028 1029 static struct mv_xor_chan * 1030 mv_xor_channel_add(struct mv_xor_device *xordev, 1031 struct platform_device *pdev, 1032 int idx, dma_cap_mask_t cap_mask, int irq) 1033 { 1034 int ret = 0; 1035 struct mv_xor_chan *mv_chan; 1036 struct dma_device *dma_dev; 1037 1038 mv_chan = devm_kzalloc(&pdev->dev, sizeof(*mv_chan), GFP_KERNEL); 1039 if (!mv_chan) 1040 return ERR_PTR(-ENOMEM); 1041 1042 mv_chan->idx = idx; 1043 mv_chan->irq = irq; 1044 if (xordev->xor_type == XOR_ORION) 1045 mv_chan->op_in_desc = XOR_MODE_IN_REG; 1046 else 1047 mv_chan->op_in_desc = XOR_MODE_IN_DESC; 1048 1049 dma_dev = &mv_chan->dmadev; 1050 mv_chan->xordev = xordev; 1051 1052 /* 1053 * These source and destination dummy buffers are used to implement 1054 * a DMA_INTERRUPT operation as a minimum-sized XOR operation. 1055 * Hence, we only need to map the buffers at initialization-time. 1056 */ 1057 mv_chan->dummy_src_addr = dma_map_single(dma_dev->dev, 1058 mv_chan->dummy_src, MV_XOR_MIN_BYTE_COUNT, DMA_FROM_DEVICE); 1059 mv_chan->dummy_dst_addr = dma_map_single(dma_dev->dev, 1060 mv_chan->dummy_dst, MV_XOR_MIN_BYTE_COUNT, DMA_TO_DEVICE); 1061 1062 /* allocate coherent memory for hardware descriptors 1063 * note: writecombine gives slightly better performance, but 1064 * requires that we explicitly flush the writes 1065 */ 1066 mv_chan->dma_desc_pool_virt = 1067 dma_alloc_wc(&pdev->dev, MV_XOR_POOL_SIZE, &mv_chan->dma_desc_pool, 1068 GFP_KERNEL); 1069 if (!mv_chan->dma_desc_pool_virt) 1070 return ERR_PTR(-ENOMEM); 1071 1072 /* discover transaction capabilites from the platform data */ 1073 dma_dev->cap_mask = cap_mask; 1074 1075 INIT_LIST_HEAD(&dma_dev->channels); 1076 1077 /* set base routines */ 1078 dma_dev->device_alloc_chan_resources = mv_xor_alloc_chan_resources; 1079 dma_dev->device_free_chan_resources = mv_xor_free_chan_resources; 1080 dma_dev->device_tx_status = mv_xor_status; 1081 dma_dev->device_issue_pending = mv_xor_issue_pending; 1082 dma_dev->dev = &pdev->dev; 1083 1084 /* set prep routines based on capability */ 1085 if (dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask)) 1086 dma_dev->device_prep_dma_interrupt = mv_xor_prep_dma_interrupt; 1087 if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask)) 1088 dma_dev->device_prep_dma_memcpy = mv_xor_prep_dma_memcpy; 1089 if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) { 1090 dma_dev->max_xor = 8; 1091 dma_dev->device_prep_dma_xor = mv_xor_prep_dma_xor; 1092 } 1093 1094 mv_chan->mmr_base = xordev->xor_base; 1095 mv_chan->mmr_high_base = xordev->xor_high_base; 1096 tasklet_init(&mv_chan->irq_tasklet, mv_xor_tasklet, (unsigned long) 1097 mv_chan); 1098 1099 /* clear errors before enabling interrupts */ 1100 mv_chan_clear_err_status(mv_chan); 1101 1102 ret = request_irq(mv_chan->irq, mv_xor_interrupt_handler, 1103 0, dev_name(&pdev->dev), mv_chan); 1104 if (ret) 1105 goto err_free_dma; 1106 1107 mv_chan_unmask_interrupts(mv_chan); 1108 1109 if (mv_chan->op_in_desc == XOR_MODE_IN_DESC) 1110 mv_chan_set_mode(mv_chan, XOR_OPERATION_MODE_IN_DESC); 1111 else 1112 mv_chan_set_mode(mv_chan, XOR_OPERATION_MODE_XOR); 1113 1114 spin_lock_init(&mv_chan->lock); 1115 INIT_LIST_HEAD(&mv_chan->chain); 1116 INIT_LIST_HEAD(&mv_chan->completed_slots); 1117 INIT_LIST_HEAD(&mv_chan->free_slots); 1118 INIT_LIST_HEAD(&mv_chan->allocated_slots); 1119 mv_chan->dmachan.device = dma_dev; 1120 dma_cookie_init(&mv_chan->dmachan); 1121 1122 list_add_tail(&mv_chan->dmachan.device_node, &dma_dev->channels); 1123 1124 if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask)) { 1125 ret = mv_chan_memcpy_self_test(mv_chan); 1126 dev_dbg(&pdev->dev, "memcpy self test returned %d\n", ret); 1127 if (ret) 1128 goto err_free_irq; 1129 } 1130 1131 if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) { 1132 ret = mv_chan_xor_self_test(mv_chan); 1133 dev_dbg(&pdev->dev, "xor self test returned %d\n", ret); 1134 if (ret) 1135 goto err_free_irq; 1136 } 1137 1138 dev_info(&pdev->dev, "Marvell XOR (%s): ( %s%s%s)\n", 1139 mv_chan->op_in_desc ? "Descriptor Mode" : "Registers Mode", 1140 dma_has_cap(DMA_XOR, dma_dev->cap_mask) ? "xor " : "", 1141 dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask) ? "cpy " : "", 1142 dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask) ? "intr " : ""); 1143 1144 dma_async_device_register(dma_dev); 1145 return mv_chan; 1146 1147 err_free_irq: 1148 free_irq(mv_chan->irq, mv_chan); 1149 err_free_dma: 1150 dma_free_coherent(&pdev->dev, MV_XOR_POOL_SIZE, 1151 mv_chan->dma_desc_pool_virt, mv_chan->dma_desc_pool); 1152 return ERR_PTR(ret); 1153 } 1154 1155 static void 1156 mv_xor_conf_mbus_windows(struct mv_xor_device *xordev, 1157 const struct mbus_dram_target_info *dram) 1158 { 1159 void __iomem *base = xordev->xor_high_base; 1160 u32 win_enable = 0; 1161 int i; 1162 1163 for (i = 0; i < 8; i++) { 1164 writel(0, base + WINDOW_BASE(i)); 1165 writel(0, base + WINDOW_SIZE(i)); 1166 if (i < 4) 1167 writel(0, base + WINDOW_REMAP_HIGH(i)); 1168 } 1169 1170 for (i = 0; i < dram->num_cs; i++) { 1171 const struct mbus_dram_window *cs = dram->cs + i; 1172 1173 writel((cs->base & 0xffff0000) | 1174 (cs->mbus_attr << 8) | 1175 dram->mbus_dram_target_id, base + WINDOW_BASE(i)); 1176 writel((cs->size - 1) & 0xffff0000, base + WINDOW_SIZE(i)); 1177 1178 /* Fill the caching variables for later use */ 1179 xordev->win_start[i] = cs->base; 1180 xordev->win_end[i] = cs->base + cs->size - 1; 1181 1182 win_enable |= (1 << i); 1183 win_enable |= 3 << (16 + (2 * i)); 1184 } 1185 1186 writel(win_enable, base + WINDOW_BAR_ENABLE(0)); 1187 writel(win_enable, base + WINDOW_BAR_ENABLE(1)); 1188 writel(0, base + WINDOW_OVERRIDE_CTRL(0)); 1189 writel(0, base + WINDOW_OVERRIDE_CTRL(1)); 1190 } 1191 1192 static void 1193 mv_xor_conf_mbus_windows_a3700(struct mv_xor_device *xordev) 1194 { 1195 void __iomem *base = xordev->xor_high_base; 1196 u32 win_enable = 0; 1197 int i; 1198 1199 for (i = 0; i < 8; i++) { 1200 writel(0, base + WINDOW_BASE(i)); 1201 writel(0, base + WINDOW_SIZE(i)); 1202 if (i < 4) 1203 writel(0, base + WINDOW_REMAP_HIGH(i)); 1204 } 1205 /* 1206 * For Armada3700 open default 4GB Mbus window. The dram 1207 * related configuration are done at AXIS level. 1208 */ 1209 writel(0xffff0000, base + WINDOW_SIZE(0)); 1210 win_enable |= 1; 1211 win_enable |= 3 << 16; 1212 1213 writel(win_enable, base + WINDOW_BAR_ENABLE(0)); 1214 writel(win_enable, base + WINDOW_BAR_ENABLE(1)); 1215 writel(0, base + WINDOW_OVERRIDE_CTRL(0)); 1216 writel(0, base + WINDOW_OVERRIDE_CTRL(1)); 1217 } 1218 1219 /* 1220 * Since this XOR driver is basically used only for RAID5, we don't 1221 * need to care about synchronizing ->suspend with DMA activity, 1222 * because the DMA engine will naturally be quiet due to the block 1223 * devices being suspended. 1224 */ 1225 static int mv_xor_suspend(struct platform_device *pdev, pm_message_t state) 1226 { 1227 struct mv_xor_device *xordev = platform_get_drvdata(pdev); 1228 int i; 1229 1230 for (i = 0; i < MV_XOR_MAX_CHANNELS; i++) { 1231 struct mv_xor_chan *mv_chan = xordev->channels[i]; 1232 1233 if (!mv_chan) 1234 continue; 1235 1236 mv_chan->saved_config_reg = 1237 readl_relaxed(XOR_CONFIG(mv_chan)); 1238 mv_chan->saved_int_mask_reg = 1239 readl_relaxed(XOR_INTR_MASK(mv_chan)); 1240 } 1241 1242 return 0; 1243 } 1244 1245 static int mv_xor_resume(struct platform_device *dev) 1246 { 1247 struct mv_xor_device *xordev = platform_get_drvdata(dev); 1248 const struct mbus_dram_target_info *dram; 1249 int i; 1250 1251 for (i = 0; i < MV_XOR_MAX_CHANNELS; i++) { 1252 struct mv_xor_chan *mv_chan = xordev->channels[i]; 1253 1254 if (!mv_chan) 1255 continue; 1256 1257 writel_relaxed(mv_chan->saved_config_reg, 1258 XOR_CONFIG(mv_chan)); 1259 writel_relaxed(mv_chan->saved_int_mask_reg, 1260 XOR_INTR_MASK(mv_chan)); 1261 } 1262 1263 if (xordev->xor_type == XOR_ARMADA_37XX) { 1264 mv_xor_conf_mbus_windows_a3700(xordev); 1265 return 0; 1266 } 1267 1268 dram = mv_mbus_dram_info(); 1269 if (dram) 1270 mv_xor_conf_mbus_windows(xordev, dram); 1271 1272 return 0; 1273 } 1274 1275 static const struct of_device_id mv_xor_dt_ids[] = { 1276 { .compatible = "marvell,orion-xor", .data = (void *)XOR_ORION }, 1277 { .compatible = "marvell,armada-380-xor", .data = (void *)XOR_ARMADA_38X }, 1278 { .compatible = "marvell,armada-3700-xor", .data = (void *)XOR_ARMADA_37XX }, 1279 {}, 1280 }; 1281 1282 static unsigned int mv_xor_engine_count; 1283 1284 static int mv_xor_probe(struct platform_device *pdev) 1285 { 1286 const struct mbus_dram_target_info *dram; 1287 struct mv_xor_device *xordev; 1288 struct mv_xor_platform_data *pdata = dev_get_platdata(&pdev->dev); 1289 struct resource *res; 1290 unsigned int max_engines, max_channels; 1291 int i, ret; 1292 1293 dev_notice(&pdev->dev, "Marvell shared XOR driver\n"); 1294 1295 xordev = devm_kzalloc(&pdev->dev, sizeof(*xordev), GFP_KERNEL); 1296 if (!xordev) 1297 return -ENOMEM; 1298 1299 res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 1300 if (!res) 1301 return -ENODEV; 1302 1303 xordev->xor_base = devm_ioremap(&pdev->dev, res->start, 1304 resource_size(res)); 1305 if (!xordev->xor_base) 1306 return -EBUSY; 1307 1308 res = platform_get_resource(pdev, IORESOURCE_MEM, 1); 1309 if (!res) 1310 return -ENODEV; 1311 1312 xordev->xor_high_base = devm_ioremap(&pdev->dev, res->start, 1313 resource_size(res)); 1314 if (!xordev->xor_high_base) 1315 return -EBUSY; 1316 1317 platform_set_drvdata(pdev, xordev); 1318 1319 1320 /* 1321 * We need to know which type of XOR device we use before 1322 * setting up. In non-dt case it can only be the legacy one. 1323 */ 1324 xordev->xor_type = XOR_ORION; 1325 if (pdev->dev.of_node) { 1326 const struct of_device_id *of_id = 1327 of_match_device(mv_xor_dt_ids, 1328 &pdev->dev); 1329 1330 xordev->xor_type = (uintptr_t)of_id->data; 1331 } 1332 1333 /* 1334 * (Re-)program MBUS remapping windows if we are asked to. 1335 */ 1336 if (xordev->xor_type == XOR_ARMADA_37XX) { 1337 mv_xor_conf_mbus_windows_a3700(xordev); 1338 } else { 1339 dram = mv_mbus_dram_info(); 1340 if (dram) 1341 mv_xor_conf_mbus_windows(xordev, dram); 1342 } 1343 1344 /* Not all platforms can gate the clock, so it is not 1345 * an error if the clock does not exists. 1346 */ 1347 xordev->clk = clk_get(&pdev->dev, NULL); 1348 if (!IS_ERR(xordev->clk)) 1349 clk_prepare_enable(xordev->clk); 1350 1351 /* 1352 * We don't want to have more than one channel per CPU in 1353 * order for async_tx to perform well. So we limit the number 1354 * of engines and channels so that we take into account this 1355 * constraint. Note that we also want to use channels from 1356 * separate engines when possible. For dual-CPU Armada 3700 1357 * SoC with single XOR engine allow using its both channels. 1358 */ 1359 max_engines = num_present_cpus(); 1360 if (xordev->xor_type == XOR_ARMADA_37XX) 1361 max_channels = num_present_cpus(); 1362 else 1363 max_channels = min_t(unsigned int, 1364 MV_XOR_MAX_CHANNELS, 1365 DIV_ROUND_UP(num_present_cpus(), 2)); 1366 1367 if (mv_xor_engine_count >= max_engines) 1368 return 0; 1369 1370 if (pdev->dev.of_node) { 1371 struct device_node *np; 1372 int i = 0; 1373 1374 for_each_child_of_node(pdev->dev.of_node, np) { 1375 struct mv_xor_chan *chan; 1376 dma_cap_mask_t cap_mask; 1377 int irq; 1378 1379 if (i >= max_channels) 1380 continue; 1381 1382 dma_cap_zero(cap_mask); 1383 dma_cap_set(DMA_MEMCPY, cap_mask); 1384 dma_cap_set(DMA_XOR, cap_mask); 1385 dma_cap_set(DMA_INTERRUPT, cap_mask); 1386 1387 irq = irq_of_parse_and_map(np, 0); 1388 if (!irq) { 1389 ret = -ENODEV; 1390 goto err_channel_add; 1391 } 1392 1393 chan = mv_xor_channel_add(xordev, pdev, i, 1394 cap_mask, irq); 1395 if (IS_ERR(chan)) { 1396 ret = PTR_ERR(chan); 1397 irq_dispose_mapping(irq); 1398 goto err_channel_add; 1399 } 1400 1401 xordev->channels[i] = chan; 1402 i++; 1403 } 1404 } else if (pdata && pdata->channels) { 1405 for (i = 0; i < max_channels; i++) { 1406 struct mv_xor_channel_data *cd; 1407 struct mv_xor_chan *chan; 1408 int irq; 1409 1410 cd = &pdata->channels[i]; 1411 if (!cd) { 1412 ret = -ENODEV; 1413 goto err_channel_add; 1414 } 1415 1416 irq = platform_get_irq(pdev, i); 1417 if (irq < 0) { 1418 ret = irq; 1419 goto err_channel_add; 1420 } 1421 1422 chan = mv_xor_channel_add(xordev, pdev, i, 1423 cd->cap_mask, irq); 1424 if (IS_ERR(chan)) { 1425 ret = PTR_ERR(chan); 1426 goto err_channel_add; 1427 } 1428 1429 xordev->channels[i] = chan; 1430 } 1431 } 1432 1433 return 0; 1434 1435 err_channel_add: 1436 for (i = 0; i < MV_XOR_MAX_CHANNELS; i++) 1437 if (xordev->channels[i]) { 1438 mv_xor_channel_remove(xordev->channels[i]); 1439 if (pdev->dev.of_node) 1440 irq_dispose_mapping(xordev->channels[i]->irq); 1441 } 1442 1443 if (!IS_ERR(xordev->clk)) { 1444 clk_disable_unprepare(xordev->clk); 1445 clk_put(xordev->clk); 1446 } 1447 1448 return ret; 1449 } 1450 1451 static struct platform_driver mv_xor_driver = { 1452 .probe = mv_xor_probe, 1453 .suspend = mv_xor_suspend, 1454 .resume = mv_xor_resume, 1455 .driver = { 1456 .name = MV_XOR_NAME, 1457 .of_match_table = of_match_ptr(mv_xor_dt_ids), 1458 }, 1459 }; 1460 1461 1462 static int __init mv_xor_init(void) 1463 { 1464 return platform_driver_register(&mv_xor_driver); 1465 } 1466 device_initcall(mv_xor_init); 1467 1468 /* 1469 MODULE_AUTHOR("Saeed Bishara <saeed@marvell.com>"); 1470 MODULE_DESCRIPTION("DMA engine driver for Marvell's XOR engine"); 1471 MODULE_LICENSE("GPL"); 1472 */ 1473