1 /* 2 * offload engine driver for the Marvell XOR engine 3 * Copyright (C) 2007, 2008, Marvell International Ltd. 4 * 5 * This program is free software; you can redistribute it and/or modify it 6 * under the terms and conditions of the GNU General Public License, 7 * version 2, as published by the Free Software Foundation. 8 * 9 * This program is distributed in the hope it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 12 * more details. 13 */ 14 15 #include <linux/init.h> 16 #include <linux/slab.h> 17 #include <linux/delay.h> 18 #include <linux/dma-mapping.h> 19 #include <linux/spinlock.h> 20 #include <linux/interrupt.h> 21 #include <linux/of_device.h> 22 #include <linux/platform_device.h> 23 #include <linux/memory.h> 24 #include <linux/clk.h> 25 #include <linux/of.h> 26 #include <linux/of_irq.h> 27 #include <linux/irqdomain.h> 28 #include <linux/cpumask.h> 29 #include <linux/platform_data/dma-mv_xor.h> 30 31 #include "dmaengine.h" 32 #include "mv_xor.h" 33 34 enum mv_xor_type { 35 XOR_ORION, 36 XOR_ARMADA_38X, 37 XOR_ARMADA_37XX, 38 }; 39 40 enum mv_xor_mode { 41 XOR_MODE_IN_REG, 42 XOR_MODE_IN_DESC, 43 }; 44 45 static void mv_xor_issue_pending(struct dma_chan *chan); 46 47 #define to_mv_xor_chan(chan) \ 48 container_of(chan, struct mv_xor_chan, dmachan) 49 50 #define to_mv_xor_slot(tx) \ 51 container_of(tx, struct mv_xor_desc_slot, async_tx) 52 53 #define mv_chan_to_devp(chan) \ 54 ((chan)->dmadev.dev) 55 56 static void mv_desc_init(struct mv_xor_desc_slot *desc, 57 dma_addr_t addr, u32 byte_count, 58 enum dma_ctrl_flags flags) 59 { 60 struct mv_xor_desc *hw_desc = desc->hw_desc; 61 62 hw_desc->status = XOR_DESC_DMA_OWNED; 63 hw_desc->phy_next_desc = 0; 64 /* Enable end-of-descriptor interrupts only for DMA_PREP_INTERRUPT */ 65 hw_desc->desc_command = (flags & DMA_PREP_INTERRUPT) ? 66 XOR_DESC_EOD_INT_EN : 0; 67 hw_desc->phy_dest_addr = addr; 68 hw_desc->byte_count = byte_count; 69 } 70 71 static void mv_desc_set_mode(struct mv_xor_desc_slot *desc) 72 { 73 struct mv_xor_desc *hw_desc = desc->hw_desc; 74 75 switch (desc->type) { 76 case DMA_XOR: 77 case DMA_INTERRUPT: 78 hw_desc->desc_command |= XOR_DESC_OPERATION_XOR; 79 break; 80 case DMA_MEMCPY: 81 hw_desc->desc_command |= XOR_DESC_OPERATION_MEMCPY; 82 break; 83 default: 84 BUG(); 85 return; 86 } 87 } 88 89 static void mv_desc_set_next_desc(struct mv_xor_desc_slot *desc, 90 u32 next_desc_addr) 91 { 92 struct mv_xor_desc *hw_desc = desc->hw_desc; 93 BUG_ON(hw_desc->phy_next_desc); 94 hw_desc->phy_next_desc = next_desc_addr; 95 } 96 97 static void mv_desc_set_src_addr(struct mv_xor_desc_slot *desc, 98 int index, dma_addr_t addr) 99 { 100 struct mv_xor_desc *hw_desc = desc->hw_desc; 101 hw_desc->phy_src_addr[mv_phy_src_idx(index)] = addr; 102 if (desc->type == DMA_XOR) 103 hw_desc->desc_command |= (1 << index); 104 } 105 106 static u32 mv_chan_get_current_desc(struct mv_xor_chan *chan) 107 { 108 return readl_relaxed(XOR_CURR_DESC(chan)); 109 } 110 111 static void mv_chan_set_next_descriptor(struct mv_xor_chan *chan, 112 u32 next_desc_addr) 113 { 114 writel_relaxed(next_desc_addr, XOR_NEXT_DESC(chan)); 115 } 116 117 static void mv_chan_unmask_interrupts(struct mv_xor_chan *chan) 118 { 119 u32 val = readl_relaxed(XOR_INTR_MASK(chan)); 120 val |= XOR_INTR_MASK_VALUE << (chan->idx * 16); 121 writel_relaxed(val, XOR_INTR_MASK(chan)); 122 } 123 124 static u32 mv_chan_get_intr_cause(struct mv_xor_chan *chan) 125 { 126 u32 intr_cause = readl_relaxed(XOR_INTR_CAUSE(chan)); 127 intr_cause = (intr_cause >> (chan->idx * 16)) & 0xFFFF; 128 return intr_cause; 129 } 130 131 static void mv_chan_clear_eoc_cause(struct mv_xor_chan *chan) 132 { 133 u32 val; 134 135 val = XOR_INT_END_OF_DESC | XOR_INT_END_OF_CHAIN | XOR_INT_STOPPED; 136 val = ~(val << (chan->idx * 16)); 137 dev_dbg(mv_chan_to_devp(chan), "%s, val 0x%08x\n", __func__, val); 138 writel_relaxed(val, XOR_INTR_CAUSE(chan)); 139 } 140 141 static void mv_chan_clear_err_status(struct mv_xor_chan *chan) 142 { 143 u32 val = 0xFFFF0000 >> (chan->idx * 16); 144 writel_relaxed(val, XOR_INTR_CAUSE(chan)); 145 } 146 147 static void mv_chan_set_mode(struct mv_xor_chan *chan, 148 u32 op_mode) 149 { 150 u32 config = readl_relaxed(XOR_CONFIG(chan)); 151 152 config &= ~0x7; 153 config |= op_mode; 154 155 #if defined(__BIG_ENDIAN) 156 config |= XOR_DESCRIPTOR_SWAP; 157 #else 158 config &= ~XOR_DESCRIPTOR_SWAP; 159 #endif 160 161 writel_relaxed(config, XOR_CONFIG(chan)); 162 } 163 164 static void mv_chan_activate(struct mv_xor_chan *chan) 165 { 166 dev_dbg(mv_chan_to_devp(chan), " activate chan.\n"); 167 168 /* writel ensures all descriptors are flushed before activation */ 169 writel(BIT(0), XOR_ACTIVATION(chan)); 170 } 171 172 static char mv_chan_is_busy(struct mv_xor_chan *chan) 173 { 174 u32 state = readl_relaxed(XOR_ACTIVATION(chan)); 175 176 state = (state >> 4) & 0x3; 177 178 return (state == 1) ? 1 : 0; 179 } 180 181 /* 182 * mv_chan_start_new_chain - program the engine to operate on new 183 * chain headed by sw_desc 184 * Caller must hold &mv_chan->lock while calling this function 185 */ 186 static void mv_chan_start_new_chain(struct mv_xor_chan *mv_chan, 187 struct mv_xor_desc_slot *sw_desc) 188 { 189 dev_dbg(mv_chan_to_devp(mv_chan), "%s %d: sw_desc %p\n", 190 __func__, __LINE__, sw_desc); 191 192 /* set the hardware chain */ 193 mv_chan_set_next_descriptor(mv_chan, sw_desc->async_tx.phys); 194 195 mv_chan->pending++; 196 mv_xor_issue_pending(&mv_chan->dmachan); 197 } 198 199 static dma_cookie_t 200 mv_desc_run_tx_complete_actions(struct mv_xor_desc_slot *desc, 201 struct mv_xor_chan *mv_chan, 202 dma_cookie_t cookie) 203 { 204 BUG_ON(desc->async_tx.cookie < 0); 205 206 if (desc->async_tx.cookie > 0) { 207 cookie = desc->async_tx.cookie; 208 209 dma_descriptor_unmap(&desc->async_tx); 210 /* call the callback (must not sleep or submit new 211 * operations to this channel) 212 */ 213 dmaengine_desc_get_callback_invoke(&desc->async_tx, NULL); 214 } 215 216 /* run dependent operations */ 217 dma_run_dependencies(&desc->async_tx); 218 219 return cookie; 220 } 221 222 static int 223 mv_chan_clean_completed_slots(struct mv_xor_chan *mv_chan) 224 { 225 struct mv_xor_desc_slot *iter, *_iter; 226 227 dev_dbg(mv_chan_to_devp(mv_chan), "%s %d\n", __func__, __LINE__); 228 list_for_each_entry_safe(iter, _iter, &mv_chan->completed_slots, 229 node) { 230 231 if (async_tx_test_ack(&iter->async_tx)) 232 list_move_tail(&iter->node, &mv_chan->free_slots); 233 } 234 return 0; 235 } 236 237 static int 238 mv_desc_clean_slot(struct mv_xor_desc_slot *desc, 239 struct mv_xor_chan *mv_chan) 240 { 241 dev_dbg(mv_chan_to_devp(mv_chan), "%s %d: desc %p flags %d\n", 242 __func__, __LINE__, desc, desc->async_tx.flags); 243 244 /* the client is allowed to attach dependent operations 245 * until 'ack' is set 246 */ 247 if (!async_tx_test_ack(&desc->async_tx)) 248 /* move this slot to the completed_slots */ 249 list_move_tail(&desc->node, &mv_chan->completed_slots); 250 else 251 list_move_tail(&desc->node, &mv_chan->free_slots); 252 253 return 0; 254 } 255 256 /* This function must be called with the mv_xor_chan spinlock held */ 257 static void mv_chan_slot_cleanup(struct mv_xor_chan *mv_chan) 258 { 259 struct mv_xor_desc_slot *iter, *_iter; 260 dma_cookie_t cookie = 0; 261 int busy = mv_chan_is_busy(mv_chan); 262 u32 current_desc = mv_chan_get_current_desc(mv_chan); 263 int current_cleaned = 0; 264 struct mv_xor_desc *hw_desc; 265 266 dev_dbg(mv_chan_to_devp(mv_chan), "%s %d\n", __func__, __LINE__); 267 dev_dbg(mv_chan_to_devp(mv_chan), "current_desc %x\n", current_desc); 268 mv_chan_clean_completed_slots(mv_chan); 269 270 /* free completed slots from the chain starting with 271 * the oldest descriptor 272 */ 273 274 list_for_each_entry_safe(iter, _iter, &mv_chan->chain, 275 node) { 276 277 /* clean finished descriptors */ 278 hw_desc = iter->hw_desc; 279 if (hw_desc->status & XOR_DESC_SUCCESS) { 280 cookie = mv_desc_run_tx_complete_actions(iter, mv_chan, 281 cookie); 282 283 /* done processing desc, clean slot */ 284 mv_desc_clean_slot(iter, mv_chan); 285 286 /* break if we did cleaned the current */ 287 if (iter->async_tx.phys == current_desc) { 288 current_cleaned = 1; 289 break; 290 } 291 } else { 292 if (iter->async_tx.phys == current_desc) { 293 current_cleaned = 0; 294 break; 295 } 296 } 297 } 298 299 if ((busy == 0) && !list_empty(&mv_chan->chain)) { 300 if (current_cleaned) { 301 /* 302 * current descriptor cleaned and removed, run 303 * from list head 304 */ 305 iter = list_entry(mv_chan->chain.next, 306 struct mv_xor_desc_slot, 307 node); 308 mv_chan_start_new_chain(mv_chan, iter); 309 } else { 310 if (!list_is_last(&iter->node, &mv_chan->chain)) { 311 /* 312 * descriptors are still waiting after 313 * current, trigger them 314 */ 315 iter = list_entry(iter->node.next, 316 struct mv_xor_desc_slot, 317 node); 318 mv_chan_start_new_chain(mv_chan, iter); 319 } else { 320 /* 321 * some descriptors are still waiting 322 * to be cleaned 323 */ 324 tasklet_schedule(&mv_chan->irq_tasklet); 325 } 326 } 327 } 328 329 if (cookie > 0) 330 mv_chan->dmachan.completed_cookie = cookie; 331 } 332 333 static void mv_xor_tasklet(unsigned long data) 334 { 335 struct mv_xor_chan *chan = (struct mv_xor_chan *) data; 336 337 spin_lock_bh(&chan->lock); 338 mv_chan_slot_cleanup(chan); 339 spin_unlock_bh(&chan->lock); 340 } 341 342 static struct mv_xor_desc_slot * 343 mv_chan_alloc_slot(struct mv_xor_chan *mv_chan) 344 { 345 struct mv_xor_desc_slot *iter; 346 347 spin_lock_bh(&mv_chan->lock); 348 349 if (!list_empty(&mv_chan->free_slots)) { 350 iter = list_first_entry(&mv_chan->free_slots, 351 struct mv_xor_desc_slot, 352 node); 353 354 list_move_tail(&iter->node, &mv_chan->allocated_slots); 355 356 spin_unlock_bh(&mv_chan->lock); 357 358 /* pre-ack descriptor */ 359 async_tx_ack(&iter->async_tx); 360 iter->async_tx.cookie = -EBUSY; 361 362 return iter; 363 364 } 365 366 spin_unlock_bh(&mv_chan->lock); 367 368 /* try to free some slots if the allocation fails */ 369 tasklet_schedule(&mv_chan->irq_tasklet); 370 371 return NULL; 372 } 373 374 /************************ DMA engine API functions ****************************/ 375 static dma_cookie_t 376 mv_xor_tx_submit(struct dma_async_tx_descriptor *tx) 377 { 378 struct mv_xor_desc_slot *sw_desc = to_mv_xor_slot(tx); 379 struct mv_xor_chan *mv_chan = to_mv_xor_chan(tx->chan); 380 struct mv_xor_desc_slot *old_chain_tail; 381 dma_cookie_t cookie; 382 int new_hw_chain = 1; 383 384 dev_dbg(mv_chan_to_devp(mv_chan), 385 "%s sw_desc %p: async_tx %p\n", 386 __func__, sw_desc, &sw_desc->async_tx); 387 388 spin_lock_bh(&mv_chan->lock); 389 cookie = dma_cookie_assign(tx); 390 391 if (list_empty(&mv_chan->chain)) 392 list_move_tail(&sw_desc->node, &mv_chan->chain); 393 else { 394 new_hw_chain = 0; 395 396 old_chain_tail = list_entry(mv_chan->chain.prev, 397 struct mv_xor_desc_slot, 398 node); 399 list_move_tail(&sw_desc->node, &mv_chan->chain); 400 401 dev_dbg(mv_chan_to_devp(mv_chan), "Append to last desc %pa\n", 402 &old_chain_tail->async_tx.phys); 403 404 /* fix up the hardware chain */ 405 mv_desc_set_next_desc(old_chain_tail, sw_desc->async_tx.phys); 406 407 /* if the channel is not busy */ 408 if (!mv_chan_is_busy(mv_chan)) { 409 u32 current_desc = mv_chan_get_current_desc(mv_chan); 410 /* 411 * and the curren desc is the end of the chain before 412 * the append, then we need to start the channel 413 */ 414 if (current_desc == old_chain_tail->async_tx.phys) 415 new_hw_chain = 1; 416 } 417 } 418 419 if (new_hw_chain) 420 mv_chan_start_new_chain(mv_chan, sw_desc); 421 422 spin_unlock_bh(&mv_chan->lock); 423 424 return cookie; 425 } 426 427 /* returns the number of allocated descriptors */ 428 static int mv_xor_alloc_chan_resources(struct dma_chan *chan) 429 { 430 void *virt_desc; 431 dma_addr_t dma_desc; 432 int idx; 433 struct mv_xor_chan *mv_chan = to_mv_xor_chan(chan); 434 struct mv_xor_desc_slot *slot = NULL; 435 int num_descs_in_pool = MV_XOR_POOL_SIZE/MV_XOR_SLOT_SIZE; 436 437 /* Allocate descriptor slots */ 438 idx = mv_chan->slots_allocated; 439 while (idx < num_descs_in_pool) { 440 slot = kzalloc(sizeof(*slot), GFP_KERNEL); 441 if (!slot) { 442 dev_info(mv_chan_to_devp(mv_chan), 443 "channel only initialized %d descriptor slots", 444 idx); 445 break; 446 } 447 virt_desc = mv_chan->dma_desc_pool_virt; 448 slot->hw_desc = virt_desc + idx * MV_XOR_SLOT_SIZE; 449 450 dma_async_tx_descriptor_init(&slot->async_tx, chan); 451 slot->async_tx.tx_submit = mv_xor_tx_submit; 452 INIT_LIST_HEAD(&slot->node); 453 dma_desc = mv_chan->dma_desc_pool; 454 slot->async_tx.phys = dma_desc + idx * MV_XOR_SLOT_SIZE; 455 slot->idx = idx++; 456 457 spin_lock_bh(&mv_chan->lock); 458 mv_chan->slots_allocated = idx; 459 list_add_tail(&slot->node, &mv_chan->free_slots); 460 spin_unlock_bh(&mv_chan->lock); 461 } 462 463 dev_dbg(mv_chan_to_devp(mv_chan), 464 "allocated %d descriptor slots\n", 465 mv_chan->slots_allocated); 466 467 return mv_chan->slots_allocated ? : -ENOMEM; 468 } 469 470 static struct dma_async_tx_descriptor * 471 mv_xor_prep_dma_xor(struct dma_chan *chan, dma_addr_t dest, dma_addr_t *src, 472 unsigned int src_cnt, size_t len, unsigned long flags) 473 { 474 struct mv_xor_chan *mv_chan = to_mv_xor_chan(chan); 475 struct mv_xor_desc_slot *sw_desc; 476 477 if (unlikely(len < MV_XOR_MIN_BYTE_COUNT)) 478 return NULL; 479 480 BUG_ON(len > MV_XOR_MAX_BYTE_COUNT); 481 482 dev_dbg(mv_chan_to_devp(mv_chan), 483 "%s src_cnt: %d len: %zu dest %pad flags: %ld\n", 484 __func__, src_cnt, len, &dest, flags); 485 486 sw_desc = mv_chan_alloc_slot(mv_chan); 487 if (sw_desc) { 488 sw_desc->type = DMA_XOR; 489 sw_desc->async_tx.flags = flags; 490 mv_desc_init(sw_desc, dest, len, flags); 491 if (mv_chan->op_in_desc == XOR_MODE_IN_DESC) 492 mv_desc_set_mode(sw_desc); 493 while (src_cnt--) 494 mv_desc_set_src_addr(sw_desc, src_cnt, src[src_cnt]); 495 } 496 497 dev_dbg(mv_chan_to_devp(mv_chan), 498 "%s sw_desc %p async_tx %p \n", 499 __func__, sw_desc, &sw_desc->async_tx); 500 return sw_desc ? &sw_desc->async_tx : NULL; 501 } 502 503 static struct dma_async_tx_descriptor * 504 mv_xor_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, 505 size_t len, unsigned long flags) 506 { 507 /* 508 * A MEMCPY operation is identical to an XOR operation with only 509 * a single source address. 510 */ 511 return mv_xor_prep_dma_xor(chan, dest, &src, 1, len, flags); 512 } 513 514 static struct dma_async_tx_descriptor * 515 mv_xor_prep_dma_interrupt(struct dma_chan *chan, unsigned long flags) 516 { 517 struct mv_xor_chan *mv_chan = to_mv_xor_chan(chan); 518 dma_addr_t src, dest; 519 size_t len; 520 521 src = mv_chan->dummy_src_addr; 522 dest = mv_chan->dummy_dst_addr; 523 len = MV_XOR_MIN_BYTE_COUNT; 524 525 /* 526 * We implement the DMA_INTERRUPT operation as a minimum sized 527 * XOR operation with a single dummy source address. 528 */ 529 return mv_xor_prep_dma_xor(chan, dest, &src, 1, len, flags); 530 } 531 532 static void mv_xor_free_chan_resources(struct dma_chan *chan) 533 { 534 struct mv_xor_chan *mv_chan = to_mv_xor_chan(chan); 535 struct mv_xor_desc_slot *iter, *_iter; 536 int in_use_descs = 0; 537 538 spin_lock_bh(&mv_chan->lock); 539 540 mv_chan_slot_cleanup(mv_chan); 541 542 list_for_each_entry_safe(iter, _iter, &mv_chan->chain, 543 node) { 544 in_use_descs++; 545 list_move_tail(&iter->node, &mv_chan->free_slots); 546 } 547 list_for_each_entry_safe(iter, _iter, &mv_chan->completed_slots, 548 node) { 549 in_use_descs++; 550 list_move_tail(&iter->node, &mv_chan->free_slots); 551 } 552 list_for_each_entry_safe(iter, _iter, &mv_chan->allocated_slots, 553 node) { 554 in_use_descs++; 555 list_move_tail(&iter->node, &mv_chan->free_slots); 556 } 557 list_for_each_entry_safe_reverse( 558 iter, _iter, &mv_chan->free_slots, node) { 559 list_del(&iter->node); 560 kfree(iter); 561 mv_chan->slots_allocated--; 562 } 563 564 dev_dbg(mv_chan_to_devp(mv_chan), "%s slots_allocated %d\n", 565 __func__, mv_chan->slots_allocated); 566 spin_unlock_bh(&mv_chan->lock); 567 568 if (in_use_descs) 569 dev_err(mv_chan_to_devp(mv_chan), 570 "freeing %d in use descriptors!\n", in_use_descs); 571 } 572 573 /** 574 * mv_xor_status - poll the status of an XOR transaction 575 * @chan: XOR channel handle 576 * @cookie: XOR transaction identifier 577 * @txstate: XOR transactions state holder (or NULL) 578 */ 579 static enum dma_status mv_xor_status(struct dma_chan *chan, 580 dma_cookie_t cookie, 581 struct dma_tx_state *txstate) 582 { 583 struct mv_xor_chan *mv_chan = to_mv_xor_chan(chan); 584 enum dma_status ret; 585 586 ret = dma_cookie_status(chan, cookie, txstate); 587 if (ret == DMA_COMPLETE) 588 return ret; 589 590 spin_lock_bh(&mv_chan->lock); 591 mv_chan_slot_cleanup(mv_chan); 592 spin_unlock_bh(&mv_chan->lock); 593 594 return dma_cookie_status(chan, cookie, txstate); 595 } 596 597 static void mv_chan_dump_regs(struct mv_xor_chan *chan) 598 { 599 u32 val; 600 601 val = readl_relaxed(XOR_CONFIG(chan)); 602 dev_err(mv_chan_to_devp(chan), "config 0x%08x\n", val); 603 604 val = readl_relaxed(XOR_ACTIVATION(chan)); 605 dev_err(mv_chan_to_devp(chan), "activation 0x%08x\n", val); 606 607 val = readl_relaxed(XOR_INTR_CAUSE(chan)); 608 dev_err(mv_chan_to_devp(chan), "intr cause 0x%08x\n", val); 609 610 val = readl_relaxed(XOR_INTR_MASK(chan)); 611 dev_err(mv_chan_to_devp(chan), "intr mask 0x%08x\n", val); 612 613 val = readl_relaxed(XOR_ERROR_CAUSE(chan)); 614 dev_err(mv_chan_to_devp(chan), "error cause 0x%08x\n", val); 615 616 val = readl_relaxed(XOR_ERROR_ADDR(chan)); 617 dev_err(mv_chan_to_devp(chan), "error addr 0x%08x\n", val); 618 } 619 620 static void mv_chan_err_interrupt_handler(struct mv_xor_chan *chan, 621 u32 intr_cause) 622 { 623 if (intr_cause & XOR_INT_ERR_DECODE) { 624 dev_dbg(mv_chan_to_devp(chan), "ignoring address decode error\n"); 625 return; 626 } 627 628 dev_err(mv_chan_to_devp(chan), "error on chan %d. intr cause 0x%08x\n", 629 chan->idx, intr_cause); 630 631 mv_chan_dump_regs(chan); 632 WARN_ON(1); 633 } 634 635 static irqreturn_t mv_xor_interrupt_handler(int irq, void *data) 636 { 637 struct mv_xor_chan *chan = data; 638 u32 intr_cause = mv_chan_get_intr_cause(chan); 639 640 dev_dbg(mv_chan_to_devp(chan), "intr cause %x\n", intr_cause); 641 642 if (intr_cause & XOR_INTR_ERRORS) 643 mv_chan_err_interrupt_handler(chan, intr_cause); 644 645 tasklet_schedule(&chan->irq_tasklet); 646 647 mv_chan_clear_eoc_cause(chan); 648 649 return IRQ_HANDLED; 650 } 651 652 static void mv_xor_issue_pending(struct dma_chan *chan) 653 { 654 struct mv_xor_chan *mv_chan = to_mv_xor_chan(chan); 655 656 if (mv_chan->pending >= MV_XOR_THRESHOLD) { 657 mv_chan->pending = 0; 658 mv_chan_activate(mv_chan); 659 } 660 } 661 662 /* 663 * Perform a transaction to verify the HW works. 664 */ 665 666 static int mv_chan_memcpy_self_test(struct mv_xor_chan *mv_chan) 667 { 668 int i, ret; 669 void *src, *dest; 670 dma_addr_t src_dma, dest_dma; 671 struct dma_chan *dma_chan; 672 dma_cookie_t cookie; 673 struct dma_async_tx_descriptor *tx; 674 struct dmaengine_unmap_data *unmap; 675 int err = 0; 676 677 src = kmalloc(sizeof(u8) * PAGE_SIZE, GFP_KERNEL); 678 if (!src) 679 return -ENOMEM; 680 681 dest = kzalloc(sizeof(u8) * PAGE_SIZE, GFP_KERNEL); 682 if (!dest) { 683 kfree(src); 684 return -ENOMEM; 685 } 686 687 /* Fill in src buffer */ 688 for (i = 0; i < PAGE_SIZE; i++) 689 ((u8 *) src)[i] = (u8)i; 690 691 dma_chan = &mv_chan->dmachan; 692 if (mv_xor_alloc_chan_resources(dma_chan) < 1) { 693 err = -ENODEV; 694 goto out; 695 } 696 697 unmap = dmaengine_get_unmap_data(dma_chan->device->dev, 2, GFP_KERNEL); 698 if (!unmap) { 699 err = -ENOMEM; 700 goto free_resources; 701 } 702 703 src_dma = dma_map_page(dma_chan->device->dev, virt_to_page(src), 704 (size_t)src & ~PAGE_MASK, PAGE_SIZE, 705 DMA_TO_DEVICE); 706 unmap->addr[0] = src_dma; 707 708 ret = dma_mapping_error(dma_chan->device->dev, src_dma); 709 if (ret) { 710 err = -ENOMEM; 711 goto free_resources; 712 } 713 unmap->to_cnt = 1; 714 715 dest_dma = dma_map_page(dma_chan->device->dev, virt_to_page(dest), 716 (size_t)dest & ~PAGE_MASK, PAGE_SIZE, 717 DMA_FROM_DEVICE); 718 unmap->addr[1] = dest_dma; 719 720 ret = dma_mapping_error(dma_chan->device->dev, dest_dma); 721 if (ret) { 722 err = -ENOMEM; 723 goto free_resources; 724 } 725 unmap->from_cnt = 1; 726 unmap->len = PAGE_SIZE; 727 728 tx = mv_xor_prep_dma_memcpy(dma_chan, dest_dma, src_dma, 729 PAGE_SIZE, 0); 730 if (!tx) { 731 dev_err(dma_chan->device->dev, 732 "Self-test cannot prepare operation, disabling\n"); 733 err = -ENODEV; 734 goto free_resources; 735 } 736 737 cookie = mv_xor_tx_submit(tx); 738 if (dma_submit_error(cookie)) { 739 dev_err(dma_chan->device->dev, 740 "Self-test submit error, disabling\n"); 741 err = -ENODEV; 742 goto free_resources; 743 } 744 745 mv_xor_issue_pending(dma_chan); 746 async_tx_ack(tx); 747 msleep(1); 748 749 if (mv_xor_status(dma_chan, cookie, NULL) != 750 DMA_COMPLETE) { 751 dev_err(dma_chan->device->dev, 752 "Self-test copy timed out, disabling\n"); 753 err = -ENODEV; 754 goto free_resources; 755 } 756 757 dma_sync_single_for_cpu(dma_chan->device->dev, dest_dma, 758 PAGE_SIZE, DMA_FROM_DEVICE); 759 if (memcmp(src, dest, PAGE_SIZE)) { 760 dev_err(dma_chan->device->dev, 761 "Self-test copy failed compare, disabling\n"); 762 err = -ENODEV; 763 goto free_resources; 764 } 765 766 free_resources: 767 dmaengine_unmap_put(unmap); 768 mv_xor_free_chan_resources(dma_chan); 769 out: 770 kfree(src); 771 kfree(dest); 772 return err; 773 } 774 775 #define MV_XOR_NUM_SRC_TEST 4 /* must be <= 15 */ 776 static int 777 mv_chan_xor_self_test(struct mv_xor_chan *mv_chan) 778 { 779 int i, src_idx, ret; 780 struct page *dest; 781 struct page *xor_srcs[MV_XOR_NUM_SRC_TEST]; 782 dma_addr_t dma_srcs[MV_XOR_NUM_SRC_TEST]; 783 dma_addr_t dest_dma; 784 struct dma_async_tx_descriptor *tx; 785 struct dmaengine_unmap_data *unmap; 786 struct dma_chan *dma_chan; 787 dma_cookie_t cookie; 788 u8 cmp_byte = 0; 789 u32 cmp_word; 790 int err = 0; 791 int src_count = MV_XOR_NUM_SRC_TEST; 792 793 for (src_idx = 0; src_idx < src_count; src_idx++) { 794 xor_srcs[src_idx] = alloc_page(GFP_KERNEL); 795 if (!xor_srcs[src_idx]) { 796 while (src_idx--) 797 __free_page(xor_srcs[src_idx]); 798 return -ENOMEM; 799 } 800 } 801 802 dest = alloc_page(GFP_KERNEL); 803 if (!dest) { 804 while (src_idx--) 805 __free_page(xor_srcs[src_idx]); 806 return -ENOMEM; 807 } 808 809 /* Fill in src buffers */ 810 for (src_idx = 0; src_idx < src_count; src_idx++) { 811 u8 *ptr = page_address(xor_srcs[src_idx]); 812 for (i = 0; i < PAGE_SIZE; i++) 813 ptr[i] = (1 << src_idx); 814 } 815 816 for (src_idx = 0; src_idx < src_count; src_idx++) 817 cmp_byte ^= (u8) (1 << src_idx); 818 819 cmp_word = (cmp_byte << 24) | (cmp_byte << 16) | 820 (cmp_byte << 8) | cmp_byte; 821 822 memset(page_address(dest), 0, PAGE_SIZE); 823 824 dma_chan = &mv_chan->dmachan; 825 if (mv_xor_alloc_chan_resources(dma_chan) < 1) { 826 err = -ENODEV; 827 goto out; 828 } 829 830 unmap = dmaengine_get_unmap_data(dma_chan->device->dev, src_count + 1, 831 GFP_KERNEL); 832 if (!unmap) { 833 err = -ENOMEM; 834 goto free_resources; 835 } 836 837 /* test xor */ 838 for (i = 0; i < src_count; i++) { 839 unmap->addr[i] = dma_map_page(dma_chan->device->dev, xor_srcs[i], 840 0, PAGE_SIZE, DMA_TO_DEVICE); 841 dma_srcs[i] = unmap->addr[i]; 842 ret = dma_mapping_error(dma_chan->device->dev, unmap->addr[i]); 843 if (ret) { 844 err = -ENOMEM; 845 goto free_resources; 846 } 847 unmap->to_cnt++; 848 } 849 850 unmap->addr[src_count] = dma_map_page(dma_chan->device->dev, dest, 0, PAGE_SIZE, 851 DMA_FROM_DEVICE); 852 dest_dma = unmap->addr[src_count]; 853 ret = dma_mapping_error(dma_chan->device->dev, unmap->addr[src_count]); 854 if (ret) { 855 err = -ENOMEM; 856 goto free_resources; 857 } 858 unmap->from_cnt = 1; 859 unmap->len = PAGE_SIZE; 860 861 tx = mv_xor_prep_dma_xor(dma_chan, dest_dma, dma_srcs, 862 src_count, PAGE_SIZE, 0); 863 if (!tx) { 864 dev_err(dma_chan->device->dev, 865 "Self-test cannot prepare operation, disabling\n"); 866 err = -ENODEV; 867 goto free_resources; 868 } 869 870 cookie = mv_xor_tx_submit(tx); 871 if (dma_submit_error(cookie)) { 872 dev_err(dma_chan->device->dev, 873 "Self-test submit error, disabling\n"); 874 err = -ENODEV; 875 goto free_resources; 876 } 877 878 mv_xor_issue_pending(dma_chan); 879 async_tx_ack(tx); 880 msleep(8); 881 882 if (mv_xor_status(dma_chan, cookie, NULL) != 883 DMA_COMPLETE) { 884 dev_err(dma_chan->device->dev, 885 "Self-test xor timed out, disabling\n"); 886 err = -ENODEV; 887 goto free_resources; 888 } 889 890 dma_sync_single_for_cpu(dma_chan->device->dev, dest_dma, 891 PAGE_SIZE, DMA_FROM_DEVICE); 892 for (i = 0; i < (PAGE_SIZE / sizeof(u32)); i++) { 893 u32 *ptr = page_address(dest); 894 if (ptr[i] != cmp_word) { 895 dev_err(dma_chan->device->dev, 896 "Self-test xor failed compare, disabling. index %d, data %x, expected %x\n", 897 i, ptr[i], cmp_word); 898 err = -ENODEV; 899 goto free_resources; 900 } 901 } 902 903 free_resources: 904 dmaengine_unmap_put(unmap); 905 mv_xor_free_chan_resources(dma_chan); 906 out: 907 src_idx = src_count; 908 while (src_idx--) 909 __free_page(xor_srcs[src_idx]); 910 __free_page(dest); 911 return err; 912 } 913 914 static int mv_xor_channel_remove(struct mv_xor_chan *mv_chan) 915 { 916 struct dma_chan *chan, *_chan; 917 struct device *dev = mv_chan->dmadev.dev; 918 919 dma_async_device_unregister(&mv_chan->dmadev); 920 921 dma_free_coherent(dev, MV_XOR_POOL_SIZE, 922 mv_chan->dma_desc_pool_virt, mv_chan->dma_desc_pool); 923 dma_unmap_single(dev, mv_chan->dummy_src_addr, 924 MV_XOR_MIN_BYTE_COUNT, DMA_FROM_DEVICE); 925 dma_unmap_single(dev, mv_chan->dummy_dst_addr, 926 MV_XOR_MIN_BYTE_COUNT, DMA_TO_DEVICE); 927 928 list_for_each_entry_safe(chan, _chan, &mv_chan->dmadev.channels, 929 device_node) { 930 list_del(&chan->device_node); 931 } 932 933 free_irq(mv_chan->irq, mv_chan); 934 935 return 0; 936 } 937 938 static struct mv_xor_chan * 939 mv_xor_channel_add(struct mv_xor_device *xordev, 940 struct platform_device *pdev, 941 int idx, dma_cap_mask_t cap_mask, int irq) 942 { 943 int ret = 0; 944 struct mv_xor_chan *mv_chan; 945 struct dma_device *dma_dev; 946 947 mv_chan = devm_kzalloc(&pdev->dev, sizeof(*mv_chan), GFP_KERNEL); 948 if (!mv_chan) 949 return ERR_PTR(-ENOMEM); 950 951 mv_chan->idx = idx; 952 mv_chan->irq = irq; 953 if (xordev->xor_type == XOR_ORION) 954 mv_chan->op_in_desc = XOR_MODE_IN_REG; 955 else 956 mv_chan->op_in_desc = XOR_MODE_IN_DESC; 957 958 dma_dev = &mv_chan->dmadev; 959 960 /* 961 * These source and destination dummy buffers are used to implement 962 * a DMA_INTERRUPT operation as a minimum-sized XOR operation. 963 * Hence, we only need to map the buffers at initialization-time. 964 */ 965 mv_chan->dummy_src_addr = dma_map_single(dma_dev->dev, 966 mv_chan->dummy_src, MV_XOR_MIN_BYTE_COUNT, DMA_FROM_DEVICE); 967 mv_chan->dummy_dst_addr = dma_map_single(dma_dev->dev, 968 mv_chan->dummy_dst, MV_XOR_MIN_BYTE_COUNT, DMA_TO_DEVICE); 969 970 /* allocate coherent memory for hardware descriptors 971 * note: writecombine gives slightly better performance, but 972 * requires that we explicitly flush the writes 973 */ 974 mv_chan->dma_desc_pool_virt = 975 dma_alloc_wc(&pdev->dev, MV_XOR_POOL_SIZE, &mv_chan->dma_desc_pool, 976 GFP_KERNEL); 977 if (!mv_chan->dma_desc_pool_virt) 978 return ERR_PTR(-ENOMEM); 979 980 /* discover transaction capabilites from the platform data */ 981 dma_dev->cap_mask = cap_mask; 982 983 INIT_LIST_HEAD(&dma_dev->channels); 984 985 /* set base routines */ 986 dma_dev->device_alloc_chan_resources = mv_xor_alloc_chan_resources; 987 dma_dev->device_free_chan_resources = mv_xor_free_chan_resources; 988 dma_dev->device_tx_status = mv_xor_status; 989 dma_dev->device_issue_pending = mv_xor_issue_pending; 990 dma_dev->dev = &pdev->dev; 991 992 /* set prep routines based on capability */ 993 if (dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask)) 994 dma_dev->device_prep_dma_interrupt = mv_xor_prep_dma_interrupt; 995 if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask)) 996 dma_dev->device_prep_dma_memcpy = mv_xor_prep_dma_memcpy; 997 if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) { 998 dma_dev->max_xor = 8; 999 dma_dev->device_prep_dma_xor = mv_xor_prep_dma_xor; 1000 } 1001 1002 mv_chan->mmr_base = xordev->xor_base; 1003 mv_chan->mmr_high_base = xordev->xor_high_base; 1004 tasklet_init(&mv_chan->irq_tasklet, mv_xor_tasklet, (unsigned long) 1005 mv_chan); 1006 1007 /* clear errors before enabling interrupts */ 1008 mv_chan_clear_err_status(mv_chan); 1009 1010 ret = request_irq(mv_chan->irq, mv_xor_interrupt_handler, 1011 0, dev_name(&pdev->dev), mv_chan); 1012 if (ret) 1013 goto err_free_dma; 1014 1015 mv_chan_unmask_interrupts(mv_chan); 1016 1017 if (mv_chan->op_in_desc == XOR_MODE_IN_DESC) 1018 mv_chan_set_mode(mv_chan, XOR_OPERATION_MODE_IN_DESC); 1019 else 1020 mv_chan_set_mode(mv_chan, XOR_OPERATION_MODE_XOR); 1021 1022 spin_lock_init(&mv_chan->lock); 1023 INIT_LIST_HEAD(&mv_chan->chain); 1024 INIT_LIST_HEAD(&mv_chan->completed_slots); 1025 INIT_LIST_HEAD(&mv_chan->free_slots); 1026 INIT_LIST_HEAD(&mv_chan->allocated_slots); 1027 mv_chan->dmachan.device = dma_dev; 1028 dma_cookie_init(&mv_chan->dmachan); 1029 1030 list_add_tail(&mv_chan->dmachan.device_node, &dma_dev->channels); 1031 1032 if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask)) { 1033 ret = mv_chan_memcpy_self_test(mv_chan); 1034 dev_dbg(&pdev->dev, "memcpy self test returned %d\n", ret); 1035 if (ret) 1036 goto err_free_irq; 1037 } 1038 1039 if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) { 1040 ret = mv_chan_xor_self_test(mv_chan); 1041 dev_dbg(&pdev->dev, "xor self test returned %d\n", ret); 1042 if (ret) 1043 goto err_free_irq; 1044 } 1045 1046 dev_info(&pdev->dev, "Marvell XOR (%s): ( %s%s%s)\n", 1047 mv_chan->op_in_desc ? "Descriptor Mode" : "Registers Mode", 1048 dma_has_cap(DMA_XOR, dma_dev->cap_mask) ? "xor " : "", 1049 dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask) ? "cpy " : "", 1050 dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask) ? "intr " : ""); 1051 1052 dma_async_device_register(dma_dev); 1053 return mv_chan; 1054 1055 err_free_irq: 1056 free_irq(mv_chan->irq, mv_chan); 1057 err_free_dma: 1058 dma_free_coherent(&pdev->dev, MV_XOR_POOL_SIZE, 1059 mv_chan->dma_desc_pool_virt, mv_chan->dma_desc_pool); 1060 return ERR_PTR(ret); 1061 } 1062 1063 static void 1064 mv_xor_conf_mbus_windows(struct mv_xor_device *xordev, 1065 const struct mbus_dram_target_info *dram) 1066 { 1067 void __iomem *base = xordev->xor_high_base; 1068 u32 win_enable = 0; 1069 int i; 1070 1071 for (i = 0; i < 8; i++) { 1072 writel(0, base + WINDOW_BASE(i)); 1073 writel(0, base + WINDOW_SIZE(i)); 1074 if (i < 4) 1075 writel(0, base + WINDOW_REMAP_HIGH(i)); 1076 } 1077 1078 for (i = 0; i < dram->num_cs; i++) { 1079 const struct mbus_dram_window *cs = dram->cs + i; 1080 1081 writel((cs->base & 0xffff0000) | 1082 (cs->mbus_attr << 8) | 1083 dram->mbus_dram_target_id, base + WINDOW_BASE(i)); 1084 writel((cs->size - 1) & 0xffff0000, base + WINDOW_SIZE(i)); 1085 1086 win_enable |= (1 << i); 1087 win_enable |= 3 << (16 + (2 * i)); 1088 } 1089 1090 writel(win_enable, base + WINDOW_BAR_ENABLE(0)); 1091 writel(win_enable, base + WINDOW_BAR_ENABLE(1)); 1092 writel(0, base + WINDOW_OVERRIDE_CTRL(0)); 1093 writel(0, base + WINDOW_OVERRIDE_CTRL(1)); 1094 } 1095 1096 static void 1097 mv_xor_conf_mbus_windows_a3700(struct mv_xor_device *xordev) 1098 { 1099 void __iomem *base = xordev->xor_high_base; 1100 u32 win_enable = 0; 1101 int i; 1102 1103 for (i = 0; i < 8; i++) { 1104 writel(0, base + WINDOW_BASE(i)); 1105 writel(0, base + WINDOW_SIZE(i)); 1106 if (i < 4) 1107 writel(0, base + WINDOW_REMAP_HIGH(i)); 1108 } 1109 /* 1110 * For Armada3700 open default 4GB Mbus window. The dram 1111 * related configuration are done at AXIS level. 1112 */ 1113 writel(0xffff0000, base + WINDOW_SIZE(0)); 1114 win_enable |= 1; 1115 win_enable |= 3 << 16; 1116 1117 writel(win_enable, base + WINDOW_BAR_ENABLE(0)); 1118 writel(win_enable, base + WINDOW_BAR_ENABLE(1)); 1119 writel(0, base + WINDOW_OVERRIDE_CTRL(0)); 1120 writel(0, base + WINDOW_OVERRIDE_CTRL(1)); 1121 } 1122 1123 /* 1124 * Since this XOR driver is basically used only for RAID5, we don't 1125 * need to care about synchronizing ->suspend with DMA activity, 1126 * because the DMA engine will naturally be quiet due to the block 1127 * devices being suspended. 1128 */ 1129 static int mv_xor_suspend(struct platform_device *pdev, pm_message_t state) 1130 { 1131 struct mv_xor_device *xordev = platform_get_drvdata(pdev); 1132 int i; 1133 1134 for (i = 0; i < MV_XOR_MAX_CHANNELS; i++) { 1135 struct mv_xor_chan *mv_chan = xordev->channels[i]; 1136 1137 if (!mv_chan) 1138 continue; 1139 1140 mv_chan->saved_config_reg = 1141 readl_relaxed(XOR_CONFIG(mv_chan)); 1142 mv_chan->saved_int_mask_reg = 1143 readl_relaxed(XOR_INTR_MASK(mv_chan)); 1144 } 1145 1146 return 0; 1147 } 1148 1149 static int mv_xor_resume(struct platform_device *dev) 1150 { 1151 struct mv_xor_device *xordev = platform_get_drvdata(dev); 1152 const struct mbus_dram_target_info *dram; 1153 int i; 1154 1155 for (i = 0; i < MV_XOR_MAX_CHANNELS; i++) { 1156 struct mv_xor_chan *mv_chan = xordev->channels[i]; 1157 1158 if (!mv_chan) 1159 continue; 1160 1161 writel_relaxed(mv_chan->saved_config_reg, 1162 XOR_CONFIG(mv_chan)); 1163 writel_relaxed(mv_chan->saved_int_mask_reg, 1164 XOR_INTR_MASK(mv_chan)); 1165 } 1166 1167 if (xordev->xor_type == XOR_ARMADA_37XX) { 1168 mv_xor_conf_mbus_windows_a3700(xordev); 1169 return 0; 1170 } 1171 1172 dram = mv_mbus_dram_info(); 1173 if (dram) 1174 mv_xor_conf_mbus_windows(xordev, dram); 1175 1176 return 0; 1177 } 1178 1179 static const struct of_device_id mv_xor_dt_ids[] = { 1180 { .compatible = "marvell,orion-xor", .data = (void *)XOR_ORION }, 1181 { .compatible = "marvell,armada-380-xor", .data = (void *)XOR_ARMADA_38X }, 1182 { .compatible = "marvell,armada-3700-xor", .data = (void *)XOR_ARMADA_37XX }, 1183 {}, 1184 }; 1185 1186 static unsigned int mv_xor_engine_count; 1187 1188 static int mv_xor_probe(struct platform_device *pdev) 1189 { 1190 const struct mbus_dram_target_info *dram; 1191 struct mv_xor_device *xordev; 1192 struct mv_xor_platform_data *pdata = dev_get_platdata(&pdev->dev); 1193 struct resource *res; 1194 unsigned int max_engines, max_channels; 1195 int i, ret; 1196 1197 dev_notice(&pdev->dev, "Marvell shared XOR driver\n"); 1198 1199 xordev = devm_kzalloc(&pdev->dev, sizeof(*xordev), GFP_KERNEL); 1200 if (!xordev) 1201 return -ENOMEM; 1202 1203 res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 1204 if (!res) 1205 return -ENODEV; 1206 1207 xordev->xor_base = devm_ioremap(&pdev->dev, res->start, 1208 resource_size(res)); 1209 if (!xordev->xor_base) 1210 return -EBUSY; 1211 1212 res = platform_get_resource(pdev, IORESOURCE_MEM, 1); 1213 if (!res) 1214 return -ENODEV; 1215 1216 xordev->xor_high_base = devm_ioremap(&pdev->dev, res->start, 1217 resource_size(res)); 1218 if (!xordev->xor_high_base) 1219 return -EBUSY; 1220 1221 platform_set_drvdata(pdev, xordev); 1222 1223 1224 /* 1225 * We need to know which type of XOR device we use before 1226 * setting up. In non-dt case it can only be the legacy one. 1227 */ 1228 xordev->xor_type = XOR_ORION; 1229 if (pdev->dev.of_node) { 1230 const struct of_device_id *of_id = 1231 of_match_device(mv_xor_dt_ids, 1232 &pdev->dev); 1233 1234 xordev->xor_type = (uintptr_t)of_id->data; 1235 } 1236 1237 /* 1238 * (Re-)program MBUS remapping windows if we are asked to. 1239 */ 1240 if (xordev->xor_type == XOR_ARMADA_37XX) { 1241 mv_xor_conf_mbus_windows_a3700(xordev); 1242 } else { 1243 dram = mv_mbus_dram_info(); 1244 if (dram) 1245 mv_xor_conf_mbus_windows(xordev, dram); 1246 } 1247 1248 /* Not all platforms can gate the clock, so it is not 1249 * an error if the clock does not exists. 1250 */ 1251 xordev->clk = clk_get(&pdev->dev, NULL); 1252 if (!IS_ERR(xordev->clk)) 1253 clk_prepare_enable(xordev->clk); 1254 1255 /* 1256 * We don't want to have more than one channel per CPU in 1257 * order for async_tx to perform well. So we limit the number 1258 * of engines and channels so that we take into account this 1259 * constraint. Note that we also want to use channels from 1260 * separate engines when possible. For dual-CPU Armada 3700 1261 * SoC with single XOR engine allow using its both channels. 1262 */ 1263 max_engines = num_present_cpus(); 1264 if (xordev->xor_type == XOR_ARMADA_37XX) 1265 max_channels = num_present_cpus(); 1266 else 1267 max_channels = min_t(unsigned int, 1268 MV_XOR_MAX_CHANNELS, 1269 DIV_ROUND_UP(num_present_cpus(), 2)); 1270 1271 if (mv_xor_engine_count >= max_engines) 1272 return 0; 1273 1274 if (pdev->dev.of_node) { 1275 struct device_node *np; 1276 int i = 0; 1277 1278 for_each_child_of_node(pdev->dev.of_node, np) { 1279 struct mv_xor_chan *chan; 1280 dma_cap_mask_t cap_mask; 1281 int irq; 1282 1283 if (i >= max_channels) 1284 continue; 1285 1286 dma_cap_zero(cap_mask); 1287 dma_cap_set(DMA_MEMCPY, cap_mask); 1288 dma_cap_set(DMA_XOR, cap_mask); 1289 dma_cap_set(DMA_INTERRUPT, cap_mask); 1290 1291 irq = irq_of_parse_and_map(np, 0); 1292 if (!irq) { 1293 ret = -ENODEV; 1294 goto err_channel_add; 1295 } 1296 1297 chan = mv_xor_channel_add(xordev, pdev, i, 1298 cap_mask, irq); 1299 if (IS_ERR(chan)) { 1300 ret = PTR_ERR(chan); 1301 irq_dispose_mapping(irq); 1302 goto err_channel_add; 1303 } 1304 1305 xordev->channels[i] = chan; 1306 i++; 1307 } 1308 } else if (pdata && pdata->channels) { 1309 for (i = 0; i < max_channels; i++) { 1310 struct mv_xor_channel_data *cd; 1311 struct mv_xor_chan *chan; 1312 int irq; 1313 1314 cd = &pdata->channels[i]; 1315 if (!cd) { 1316 ret = -ENODEV; 1317 goto err_channel_add; 1318 } 1319 1320 irq = platform_get_irq(pdev, i); 1321 if (irq < 0) { 1322 ret = irq; 1323 goto err_channel_add; 1324 } 1325 1326 chan = mv_xor_channel_add(xordev, pdev, i, 1327 cd->cap_mask, irq); 1328 if (IS_ERR(chan)) { 1329 ret = PTR_ERR(chan); 1330 goto err_channel_add; 1331 } 1332 1333 xordev->channels[i] = chan; 1334 } 1335 } 1336 1337 return 0; 1338 1339 err_channel_add: 1340 for (i = 0; i < MV_XOR_MAX_CHANNELS; i++) 1341 if (xordev->channels[i]) { 1342 mv_xor_channel_remove(xordev->channels[i]); 1343 if (pdev->dev.of_node) 1344 irq_dispose_mapping(xordev->channels[i]->irq); 1345 } 1346 1347 if (!IS_ERR(xordev->clk)) { 1348 clk_disable_unprepare(xordev->clk); 1349 clk_put(xordev->clk); 1350 } 1351 1352 return ret; 1353 } 1354 1355 static struct platform_driver mv_xor_driver = { 1356 .probe = mv_xor_probe, 1357 .suspend = mv_xor_suspend, 1358 .resume = mv_xor_resume, 1359 .driver = { 1360 .name = MV_XOR_NAME, 1361 .of_match_table = of_match_ptr(mv_xor_dt_ids), 1362 }, 1363 }; 1364 1365 1366 static int __init mv_xor_init(void) 1367 { 1368 return platform_driver_register(&mv_xor_driver); 1369 } 1370 device_initcall(mv_xor_init); 1371 1372 /* 1373 MODULE_AUTHOR("Saeed Bishara <saeed@marvell.com>"); 1374 MODULE_DESCRIPTION("DMA engine driver for Marvell's XOR engine"); 1375 MODULE_LICENSE("GPL"); 1376 */ 1377