1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Copyright (C) 2006-2009 DENX Software Engineering. 4 * 5 * Author: Yuri Tikhonov <yur@emcraft.com> 6 * 7 * Further porting to arch/powerpc by 8 * Anatolij Gustschin <agust@denx.de> 9 */ 10 11 /* 12 * This driver supports the asynchrounous DMA copy and RAID engines available 13 * on the AMCC PPC440SPe Processors. 14 * Based on the Intel Xscale(R) family of I/O Processors (IOP 32x, 33x, 134x) 15 * ADMA driver written by D.Williams. 16 */ 17 18 #include <linux/init.h> 19 #include <linux/module.h> 20 #include <linux/async_tx.h> 21 #include <linux/delay.h> 22 #include <linux/dma-mapping.h> 23 #include <linux/spinlock.h> 24 #include <linux/interrupt.h> 25 #include <linux/slab.h> 26 #include <linux/uaccess.h> 27 #include <linux/proc_fs.h> 28 #include <linux/of.h> 29 #include <linux/of_address.h> 30 #include <linux/of_irq.h> 31 #include <linux/of_platform.h> 32 #include <asm/dcr.h> 33 #include <asm/dcr-regs.h> 34 #include "adma.h" 35 #include "../dmaengine.h" 36 37 enum ppc_adma_init_code { 38 PPC_ADMA_INIT_OK = 0, 39 PPC_ADMA_INIT_MEMRES, 40 PPC_ADMA_INIT_MEMREG, 41 PPC_ADMA_INIT_ALLOC, 42 PPC_ADMA_INIT_COHERENT, 43 PPC_ADMA_INIT_CHANNEL, 44 PPC_ADMA_INIT_IRQ1, 45 PPC_ADMA_INIT_IRQ2, 46 PPC_ADMA_INIT_REGISTER 47 }; 48 49 static char *ppc_adma_errors[] = { 50 [PPC_ADMA_INIT_OK] = "ok", 51 [PPC_ADMA_INIT_MEMRES] = "failed to get memory resource", 52 [PPC_ADMA_INIT_MEMREG] = "failed to request memory region", 53 [PPC_ADMA_INIT_ALLOC] = "failed to allocate memory for adev " 54 "structure", 55 [PPC_ADMA_INIT_COHERENT] = "failed to allocate coherent memory for " 56 "hardware descriptors", 57 [PPC_ADMA_INIT_CHANNEL] = "failed to allocate memory for channel", 58 [PPC_ADMA_INIT_IRQ1] = "failed to request first irq", 59 [PPC_ADMA_INIT_IRQ2] = "failed to request second irq", 60 [PPC_ADMA_INIT_REGISTER] = "failed to register dma async device", 61 }; 62 63 static enum ppc_adma_init_code 64 ppc440spe_adma_devices[PPC440SPE_ADMA_ENGINES_NUM]; 65 66 struct ppc_dma_chan_ref { 67 struct dma_chan *chan; 68 struct list_head node; 69 }; 70 71 /* The list of channels exported by ppc440spe ADMA */ 72 static struct list_head 73 ppc440spe_adma_chan_list = LIST_HEAD_INIT(ppc440spe_adma_chan_list); 74 75 /* This flag is set when want to refetch the xor chain in the interrupt 76 * handler 77 */ 78 static u32 do_xor_refetch; 79 80 /* Pointer to DMA0, DMA1 CP/CS FIFO */ 81 static void *ppc440spe_dma_fifo_buf; 82 83 /* Pointers to last submitted to DMA0, DMA1 CDBs */ 84 static struct ppc440spe_adma_desc_slot *chan_last_sub[3]; 85 static struct ppc440spe_adma_desc_slot *chan_first_cdb[3]; 86 87 /* Pointer to last linked and submitted xor CB */ 88 static struct ppc440spe_adma_desc_slot *xor_last_linked; 89 static struct ppc440spe_adma_desc_slot *xor_last_submit; 90 91 /* This array is used in data-check operations for storing a pattern */ 92 static char ppc440spe_qword[16]; 93 94 static atomic_t ppc440spe_adma_err_irq_ref; 95 static dcr_host_t ppc440spe_mq_dcr_host; 96 static unsigned int ppc440spe_mq_dcr_len; 97 98 /* Since RXOR operations use the common register (MQ0_CF2H) for setting-up 99 * the block size in transactions, then we do not allow to activate more than 100 * only one RXOR transactions simultaneously. So use this var to store 101 * the information about is RXOR currently active (PPC440SPE_RXOR_RUN bit is 102 * set) or not (PPC440SPE_RXOR_RUN is clear). 103 */ 104 static unsigned long ppc440spe_rxor_state; 105 106 /* These are used in enable & check routines 107 */ 108 static u32 ppc440spe_r6_enabled; 109 static struct ppc440spe_adma_chan *ppc440spe_r6_tchan; 110 static struct completion ppc440spe_r6_test_comp; 111 112 static int ppc440spe_adma_dma2rxor_prep_src( 113 struct ppc440spe_adma_desc_slot *desc, 114 struct ppc440spe_rxor *cursor, int index, 115 int src_cnt, u32 addr); 116 static void ppc440spe_adma_dma2rxor_set_src( 117 struct ppc440spe_adma_desc_slot *desc, 118 int index, dma_addr_t addr); 119 static void ppc440spe_adma_dma2rxor_set_mult( 120 struct ppc440spe_adma_desc_slot *desc, 121 int index, u8 mult); 122 123 #ifdef ADMA_LL_DEBUG 124 #define ADMA_LL_DBG(x) ({ if (1) x; 0; }) 125 #else 126 #define ADMA_LL_DBG(x) ({ if (0) x; 0; }) 127 #endif 128 129 static void print_cb(struct ppc440spe_adma_chan *chan, void *block) 130 { 131 struct dma_cdb *cdb; 132 struct xor_cb *cb; 133 int i; 134 135 switch (chan->device->id) { 136 case 0: 137 case 1: 138 cdb = block; 139 140 pr_debug("CDB at %p [%d]:\n" 141 "\t attr 0x%02x opc 0x%02x cnt 0x%08x\n" 142 "\t sg1u 0x%08x sg1l 0x%08x\n" 143 "\t sg2u 0x%08x sg2l 0x%08x\n" 144 "\t sg3u 0x%08x sg3l 0x%08x\n", 145 cdb, chan->device->id, 146 cdb->attr, cdb->opc, le32_to_cpu(cdb->cnt), 147 le32_to_cpu(cdb->sg1u), le32_to_cpu(cdb->sg1l), 148 le32_to_cpu(cdb->sg2u), le32_to_cpu(cdb->sg2l), 149 le32_to_cpu(cdb->sg3u), le32_to_cpu(cdb->sg3l) 150 ); 151 break; 152 case 2: 153 cb = block; 154 155 pr_debug("CB at %p [%d]:\n" 156 "\t cbc 0x%08x cbbc 0x%08x cbs 0x%08x\n" 157 "\t cbtah 0x%08x cbtal 0x%08x\n" 158 "\t cblah 0x%08x cblal 0x%08x\n", 159 cb, chan->device->id, 160 cb->cbc, cb->cbbc, cb->cbs, 161 cb->cbtah, cb->cbtal, 162 cb->cblah, cb->cblal); 163 for (i = 0; i < 16; i++) { 164 if (i && !cb->ops[i].h && !cb->ops[i].l) 165 continue; 166 pr_debug("\t ops[%2d]: h 0x%08x l 0x%08x\n", 167 i, cb->ops[i].h, cb->ops[i].l); 168 } 169 break; 170 } 171 } 172 173 static void print_cb_list(struct ppc440spe_adma_chan *chan, 174 struct ppc440spe_adma_desc_slot *iter) 175 { 176 for (; iter; iter = iter->hw_next) 177 print_cb(chan, iter->hw_desc); 178 } 179 180 static void prep_dma_xor_dbg(int id, dma_addr_t dst, dma_addr_t *src, 181 unsigned int src_cnt) 182 { 183 int i; 184 185 pr_debug("\n%s(%d):\nsrc: ", __func__, id); 186 for (i = 0; i < src_cnt; i++) 187 pr_debug("\t0x%016llx ", src[i]); 188 pr_debug("dst:\n\t0x%016llx\n", dst); 189 } 190 191 static void prep_dma_pq_dbg(int id, dma_addr_t *dst, dma_addr_t *src, 192 unsigned int src_cnt) 193 { 194 int i; 195 196 pr_debug("\n%s(%d):\nsrc: ", __func__, id); 197 for (i = 0; i < src_cnt; i++) 198 pr_debug("\t0x%016llx ", src[i]); 199 pr_debug("dst: "); 200 for (i = 0; i < 2; i++) 201 pr_debug("\t0x%016llx ", dst[i]); 202 } 203 204 static void prep_dma_pqzero_sum_dbg(int id, dma_addr_t *src, 205 unsigned int src_cnt, 206 const unsigned char *scf) 207 { 208 int i; 209 210 pr_debug("\n%s(%d):\nsrc(coef): ", __func__, id); 211 if (scf) { 212 for (i = 0; i < src_cnt; i++) 213 pr_debug("\t0x%016llx(0x%02x) ", src[i], scf[i]); 214 } else { 215 for (i = 0; i < src_cnt; i++) 216 pr_debug("\t0x%016llx(no) ", src[i]); 217 } 218 219 pr_debug("dst: "); 220 for (i = 0; i < 2; i++) 221 pr_debug("\t0x%016llx ", src[src_cnt + i]); 222 } 223 224 /****************************************************************************** 225 * Command (Descriptor) Blocks low-level routines 226 ******************************************************************************/ 227 /** 228 * ppc440spe_desc_init_interrupt - initialize the descriptor for INTERRUPT 229 * pseudo operation 230 */ 231 static void ppc440spe_desc_init_interrupt(struct ppc440spe_adma_desc_slot *desc, 232 struct ppc440spe_adma_chan *chan) 233 { 234 struct xor_cb *p; 235 236 switch (chan->device->id) { 237 case PPC440SPE_XOR_ID: 238 p = desc->hw_desc; 239 memset(desc->hw_desc, 0, sizeof(struct xor_cb)); 240 /* NOP with Command Block Complete Enable */ 241 p->cbc = XOR_CBCR_CBCE_BIT; 242 break; 243 case PPC440SPE_DMA0_ID: 244 case PPC440SPE_DMA1_ID: 245 memset(desc->hw_desc, 0, sizeof(struct dma_cdb)); 246 /* NOP with interrupt */ 247 set_bit(PPC440SPE_DESC_INT, &desc->flags); 248 break; 249 default: 250 printk(KERN_ERR "Unsupported id %d in %s\n", chan->device->id, 251 __func__); 252 break; 253 } 254 } 255 256 /** 257 * ppc440spe_desc_init_null_xor - initialize the descriptor for NULL XOR 258 * pseudo operation 259 */ 260 static void ppc440spe_desc_init_null_xor(struct ppc440spe_adma_desc_slot *desc) 261 { 262 memset(desc->hw_desc, 0, sizeof(struct xor_cb)); 263 desc->hw_next = NULL; 264 desc->src_cnt = 0; 265 desc->dst_cnt = 1; 266 } 267 268 /** 269 * ppc440spe_desc_init_xor - initialize the descriptor for XOR operation 270 */ 271 static void ppc440spe_desc_init_xor(struct ppc440spe_adma_desc_slot *desc, 272 int src_cnt, unsigned long flags) 273 { 274 struct xor_cb *hw_desc = desc->hw_desc; 275 276 memset(desc->hw_desc, 0, sizeof(struct xor_cb)); 277 desc->hw_next = NULL; 278 desc->src_cnt = src_cnt; 279 desc->dst_cnt = 1; 280 281 hw_desc->cbc = XOR_CBCR_TGT_BIT | src_cnt; 282 if (flags & DMA_PREP_INTERRUPT) 283 /* Enable interrupt on completion */ 284 hw_desc->cbc |= XOR_CBCR_CBCE_BIT; 285 } 286 287 /** 288 * ppc440spe_desc_init_dma2pq - initialize the descriptor for PQ 289 * operation in DMA2 controller 290 */ 291 static void ppc440spe_desc_init_dma2pq(struct ppc440spe_adma_desc_slot *desc, 292 int dst_cnt, int src_cnt, unsigned long flags) 293 { 294 struct xor_cb *hw_desc = desc->hw_desc; 295 296 memset(desc->hw_desc, 0, sizeof(struct xor_cb)); 297 desc->hw_next = NULL; 298 desc->src_cnt = src_cnt; 299 desc->dst_cnt = dst_cnt; 300 memset(desc->reverse_flags, 0, sizeof(desc->reverse_flags)); 301 desc->descs_per_op = 0; 302 303 hw_desc->cbc = XOR_CBCR_TGT_BIT; 304 if (flags & DMA_PREP_INTERRUPT) 305 /* Enable interrupt on completion */ 306 hw_desc->cbc |= XOR_CBCR_CBCE_BIT; 307 } 308 309 #define DMA_CTRL_FLAGS_LAST DMA_PREP_FENCE 310 #define DMA_PREP_ZERO_P (DMA_CTRL_FLAGS_LAST << 1) 311 #define DMA_PREP_ZERO_Q (DMA_PREP_ZERO_P << 1) 312 313 /** 314 * ppc440spe_desc_init_dma01pq - initialize the descriptors for PQ operation 315 * with DMA0/1 316 */ 317 static void ppc440spe_desc_init_dma01pq(struct ppc440spe_adma_desc_slot *desc, 318 int dst_cnt, int src_cnt, unsigned long flags, 319 unsigned long op) 320 { 321 struct dma_cdb *hw_desc; 322 struct ppc440spe_adma_desc_slot *iter; 323 u8 dopc; 324 325 /* Common initialization of a PQ descriptors chain */ 326 set_bits(op, &desc->flags); 327 desc->src_cnt = src_cnt; 328 desc->dst_cnt = dst_cnt; 329 330 /* WXOR MULTICAST if both P and Q are being computed 331 * MV_SG1_SG2 if Q only 332 */ 333 dopc = (desc->dst_cnt == DMA_DEST_MAX_NUM) ? 334 DMA_CDB_OPC_MULTICAST : DMA_CDB_OPC_MV_SG1_SG2; 335 336 list_for_each_entry(iter, &desc->group_list, chain_node) { 337 hw_desc = iter->hw_desc; 338 memset(iter->hw_desc, 0, sizeof(struct dma_cdb)); 339 340 if (likely(!list_is_last(&iter->chain_node, 341 &desc->group_list))) { 342 /* set 'next' pointer */ 343 iter->hw_next = list_entry(iter->chain_node.next, 344 struct ppc440spe_adma_desc_slot, chain_node); 345 clear_bit(PPC440SPE_DESC_INT, &iter->flags); 346 } else { 347 /* this is the last descriptor. 348 * this slot will be pasted from ADMA level 349 * each time it wants to configure parameters 350 * of the transaction (src, dst, ...) 351 */ 352 iter->hw_next = NULL; 353 if (flags & DMA_PREP_INTERRUPT) 354 set_bit(PPC440SPE_DESC_INT, &iter->flags); 355 else 356 clear_bit(PPC440SPE_DESC_INT, &iter->flags); 357 } 358 } 359 360 /* Set OPS depending on WXOR/RXOR type of operation */ 361 if (!test_bit(PPC440SPE_DESC_RXOR, &desc->flags)) { 362 /* This is a WXOR only chain: 363 * - first descriptors are for zeroing destinations 364 * if PPC440SPE_ZERO_P/Q set; 365 * - descriptors remained are for GF-XOR operations. 366 */ 367 iter = list_first_entry(&desc->group_list, 368 struct ppc440spe_adma_desc_slot, 369 chain_node); 370 371 if (test_bit(PPC440SPE_ZERO_P, &desc->flags)) { 372 hw_desc = iter->hw_desc; 373 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; 374 iter = list_first_entry(&iter->chain_node, 375 struct ppc440spe_adma_desc_slot, 376 chain_node); 377 } 378 379 if (test_bit(PPC440SPE_ZERO_Q, &desc->flags)) { 380 hw_desc = iter->hw_desc; 381 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; 382 iter = list_first_entry(&iter->chain_node, 383 struct ppc440spe_adma_desc_slot, 384 chain_node); 385 } 386 387 list_for_each_entry_from(iter, &desc->group_list, chain_node) { 388 hw_desc = iter->hw_desc; 389 hw_desc->opc = dopc; 390 } 391 } else { 392 /* This is either RXOR-only or mixed RXOR/WXOR */ 393 394 /* The first 1 or 2 slots in chain are always RXOR, 395 * if need to calculate P & Q, then there are two 396 * RXOR slots; if only P or only Q, then there is one 397 */ 398 iter = list_first_entry(&desc->group_list, 399 struct ppc440spe_adma_desc_slot, 400 chain_node); 401 hw_desc = iter->hw_desc; 402 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; 403 404 if (desc->dst_cnt == DMA_DEST_MAX_NUM) { 405 iter = list_first_entry(&iter->chain_node, 406 struct ppc440spe_adma_desc_slot, 407 chain_node); 408 hw_desc = iter->hw_desc; 409 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; 410 } 411 412 /* The remaining descs (if any) are WXORs */ 413 if (test_bit(PPC440SPE_DESC_WXOR, &desc->flags)) { 414 iter = list_first_entry(&iter->chain_node, 415 struct ppc440spe_adma_desc_slot, 416 chain_node); 417 list_for_each_entry_from(iter, &desc->group_list, 418 chain_node) { 419 hw_desc = iter->hw_desc; 420 hw_desc->opc = dopc; 421 } 422 } 423 } 424 } 425 426 /** 427 * ppc440spe_desc_init_dma01pqzero_sum - initialize the descriptor 428 * for PQ_ZERO_SUM operation 429 */ 430 static void ppc440spe_desc_init_dma01pqzero_sum( 431 struct ppc440spe_adma_desc_slot *desc, 432 int dst_cnt, int src_cnt) 433 { 434 struct dma_cdb *hw_desc; 435 struct ppc440spe_adma_desc_slot *iter; 436 int i = 0; 437 u8 dopc = (dst_cnt == 2) ? DMA_CDB_OPC_MULTICAST : 438 DMA_CDB_OPC_MV_SG1_SG2; 439 /* 440 * Initialize starting from 2nd or 3rd descriptor dependent 441 * on dst_cnt. First one or two slots are for cloning P 442 * and/or Q to chan->pdest and/or chan->qdest as we have 443 * to preserve original P/Q. 444 */ 445 iter = list_first_entry(&desc->group_list, 446 struct ppc440spe_adma_desc_slot, chain_node); 447 iter = list_entry(iter->chain_node.next, 448 struct ppc440spe_adma_desc_slot, chain_node); 449 450 if (dst_cnt > 1) { 451 iter = list_entry(iter->chain_node.next, 452 struct ppc440spe_adma_desc_slot, chain_node); 453 } 454 /* initialize each source descriptor in chain */ 455 list_for_each_entry_from(iter, &desc->group_list, chain_node) { 456 hw_desc = iter->hw_desc; 457 memset(iter->hw_desc, 0, sizeof(struct dma_cdb)); 458 iter->src_cnt = 0; 459 iter->dst_cnt = 0; 460 461 /* This is a ZERO_SUM operation: 462 * - <src_cnt> descriptors starting from 2nd or 3rd 463 * descriptor are for GF-XOR operations; 464 * - remaining <dst_cnt> descriptors are for checking the result 465 */ 466 if (i++ < src_cnt) 467 /* MV_SG1_SG2 if only Q is being verified 468 * MULTICAST if both P and Q are being verified 469 */ 470 hw_desc->opc = dopc; 471 else 472 /* DMA_CDB_OPC_DCHECK128 operation */ 473 hw_desc->opc = DMA_CDB_OPC_DCHECK128; 474 475 if (likely(!list_is_last(&iter->chain_node, 476 &desc->group_list))) { 477 /* set 'next' pointer */ 478 iter->hw_next = list_entry(iter->chain_node.next, 479 struct ppc440spe_adma_desc_slot, 480 chain_node); 481 } else { 482 /* this is the last descriptor. 483 * this slot will be pasted from ADMA level 484 * each time it wants to configure parameters 485 * of the transaction (src, dst, ...) 486 */ 487 iter->hw_next = NULL; 488 /* always enable interrupt generation since we get 489 * the status of pqzero from the handler 490 */ 491 set_bit(PPC440SPE_DESC_INT, &iter->flags); 492 } 493 } 494 desc->src_cnt = src_cnt; 495 desc->dst_cnt = dst_cnt; 496 } 497 498 /** 499 * ppc440spe_desc_init_memcpy - initialize the descriptor for MEMCPY operation 500 */ 501 static void ppc440spe_desc_init_memcpy(struct ppc440spe_adma_desc_slot *desc, 502 unsigned long flags) 503 { 504 struct dma_cdb *hw_desc = desc->hw_desc; 505 506 memset(desc->hw_desc, 0, sizeof(struct dma_cdb)); 507 desc->hw_next = NULL; 508 desc->src_cnt = 1; 509 desc->dst_cnt = 1; 510 511 if (flags & DMA_PREP_INTERRUPT) 512 set_bit(PPC440SPE_DESC_INT, &desc->flags); 513 else 514 clear_bit(PPC440SPE_DESC_INT, &desc->flags); 515 516 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; 517 } 518 519 /** 520 * ppc440spe_desc_set_src_addr - set source address into the descriptor 521 */ 522 static void ppc440spe_desc_set_src_addr(struct ppc440spe_adma_desc_slot *desc, 523 struct ppc440spe_adma_chan *chan, 524 int src_idx, dma_addr_t addrh, 525 dma_addr_t addrl) 526 { 527 struct dma_cdb *dma_hw_desc; 528 struct xor_cb *xor_hw_desc; 529 phys_addr_t addr64, tmplow, tmphi; 530 531 switch (chan->device->id) { 532 case PPC440SPE_DMA0_ID: 533 case PPC440SPE_DMA1_ID: 534 if (!addrh) { 535 addr64 = addrl; 536 tmphi = (addr64 >> 32); 537 tmplow = (addr64 & 0xFFFFFFFF); 538 } else { 539 tmphi = addrh; 540 tmplow = addrl; 541 } 542 dma_hw_desc = desc->hw_desc; 543 dma_hw_desc->sg1l = cpu_to_le32((u32)tmplow); 544 dma_hw_desc->sg1u |= cpu_to_le32((u32)tmphi); 545 break; 546 case PPC440SPE_XOR_ID: 547 xor_hw_desc = desc->hw_desc; 548 xor_hw_desc->ops[src_idx].l = addrl; 549 xor_hw_desc->ops[src_idx].h |= addrh; 550 break; 551 } 552 } 553 554 /** 555 * ppc440spe_desc_set_src_mult - set source address mult into the descriptor 556 */ 557 static void ppc440spe_desc_set_src_mult(struct ppc440spe_adma_desc_slot *desc, 558 struct ppc440spe_adma_chan *chan, u32 mult_index, 559 int sg_index, unsigned char mult_value) 560 { 561 struct dma_cdb *dma_hw_desc; 562 u32 *psgu; 563 564 switch (chan->device->id) { 565 case PPC440SPE_DMA0_ID: 566 case PPC440SPE_DMA1_ID: 567 dma_hw_desc = desc->hw_desc; 568 569 switch (sg_index) { 570 /* for RXOR operations set multiplier 571 * into source cued address 572 */ 573 case DMA_CDB_SG_SRC: 574 psgu = &dma_hw_desc->sg1u; 575 break; 576 /* for WXOR operations set multiplier 577 * into destination cued address(es) 578 */ 579 case DMA_CDB_SG_DST1: 580 psgu = &dma_hw_desc->sg2u; 581 break; 582 case DMA_CDB_SG_DST2: 583 psgu = &dma_hw_desc->sg3u; 584 break; 585 default: 586 BUG(); 587 } 588 589 *psgu |= cpu_to_le32(mult_value << mult_index); 590 break; 591 case PPC440SPE_XOR_ID: 592 break; 593 default: 594 BUG(); 595 } 596 } 597 598 /** 599 * ppc440spe_desc_set_dest_addr - set destination address into the descriptor 600 */ 601 static void ppc440spe_desc_set_dest_addr(struct ppc440spe_adma_desc_slot *desc, 602 struct ppc440spe_adma_chan *chan, 603 dma_addr_t addrh, dma_addr_t addrl, 604 u32 dst_idx) 605 { 606 struct dma_cdb *dma_hw_desc; 607 struct xor_cb *xor_hw_desc; 608 phys_addr_t addr64, tmphi, tmplow; 609 u32 *psgu, *psgl; 610 611 switch (chan->device->id) { 612 case PPC440SPE_DMA0_ID: 613 case PPC440SPE_DMA1_ID: 614 if (!addrh) { 615 addr64 = addrl; 616 tmphi = (addr64 >> 32); 617 tmplow = (addr64 & 0xFFFFFFFF); 618 } else { 619 tmphi = addrh; 620 tmplow = addrl; 621 } 622 dma_hw_desc = desc->hw_desc; 623 624 psgu = dst_idx ? &dma_hw_desc->sg3u : &dma_hw_desc->sg2u; 625 psgl = dst_idx ? &dma_hw_desc->sg3l : &dma_hw_desc->sg2l; 626 627 *psgl = cpu_to_le32((u32)tmplow); 628 *psgu |= cpu_to_le32((u32)tmphi); 629 break; 630 case PPC440SPE_XOR_ID: 631 xor_hw_desc = desc->hw_desc; 632 xor_hw_desc->cbtal = addrl; 633 xor_hw_desc->cbtah |= addrh; 634 break; 635 } 636 } 637 638 /** 639 * ppc440spe_desc_set_byte_count - set number of data bytes involved 640 * into the operation 641 */ 642 static void ppc440spe_desc_set_byte_count(struct ppc440spe_adma_desc_slot *desc, 643 struct ppc440spe_adma_chan *chan, 644 u32 byte_count) 645 { 646 struct dma_cdb *dma_hw_desc; 647 struct xor_cb *xor_hw_desc; 648 649 switch (chan->device->id) { 650 case PPC440SPE_DMA0_ID: 651 case PPC440SPE_DMA1_ID: 652 dma_hw_desc = desc->hw_desc; 653 dma_hw_desc->cnt = cpu_to_le32(byte_count); 654 break; 655 case PPC440SPE_XOR_ID: 656 xor_hw_desc = desc->hw_desc; 657 xor_hw_desc->cbbc = byte_count; 658 break; 659 } 660 } 661 662 /** 663 * ppc440spe_desc_set_rxor_block_size - set RXOR block size 664 */ 665 static inline void ppc440spe_desc_set_rxor_block_size(u32 byte_count) 666 { 667 /* assume that byte_count is aligned on the 512-boundary; 668 * thus write it directly to the register (bits 23:31 are 669 * reserved there). 670 */ 671 dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_CF2H, byte_count); 672 } 673 674 /** 675 * ppc440spe_desc_set_dcheck - set CHECK pattern 676 */ 677 static void ppc440spe_desc_set_dcheck(struct ppc440spe_adma_desc_slot *desc, 678 struct ppc440spe_adma_chan *chan, u8 *qword) 679 { 680 struct dma_cdb *dma_hw_desc; 681 682 switch (chan->device->id) { 683 case PPC440SPE_DMA0_ID: 684 case PPC440SPE_DMA1_ID: 685 dma_hw_desc = desc->hw_desc; 686 iowrite32(qword[0], &dma_hw_desc->sg3l); 687 iowrite32(qword[4], &dma_hw_desc->sg3u); 688 iowrite32(qword[8], &dma_hw_desc->sg2l); 689 iowrite32(qword[12], &dma_hw_desc->sg2u); 690 break; 691 default: 692 BUG(); 693 } 694 } 695 696 /** 697 * ppc440spe_xor_set_link - set link address in xor CB 698 */ 699 static void ppc440spe_xor_set_link(struct ppc440spe_adma_desc_slot *prev_desc, 700 struct ppc440spe_adma_desc_slot *next_desc) 701 { 702 struct xor_cb *xor_hw_desc = prev_desc->hw_desc; 703 704 if (unlikely(!next_desc || !(next_desc->phys))) { 705 printk(KERN_ERR "%s: next_desc=0x%p; next_desc->phys=0x%llx\n", 706 __func__, next_desc, 707 next_desc ? next_desc->phys : 0); 708 BUG(); 709 } 710 711 xor_hw_desc->cbs = 0; 712 xor_hw_desc->cblal = next_desc->phys; 713 xor_hw_desc->cblah = 0; 714 xor_hw_desc->cbc |= XOR_CBCR_LNK_BIT; 715 } 716 717 /** 718 * ppc440spe_desc_set_link - set the address of descriptor following this 719 * descriptor in chain 720 */ 721 static void ppc440spe_desc_set_link(struct ppc440spe_adma_chan *chan, 722 struct ppc440spe_adma_desc_slot *prev_desc, 723 struct ppc440spe_adma_desc_slot *next_desc) 724 { 725 unsigned long flags; 726 struct ppc440spe_adma_desc_slot *tail = next_desc; 727 728 if (unlikely(!prev_desc || !next_desc || 729 (prev_desc->hw_next && prev_desc->hw_next != next_desc))) { 730 /* If previous next is overwritten something is wrong. 731 * though we may refetch from append to initiate list 732 * processing; in this case - it's ok. 733 */ 734 printk(KERN_ERR "%s: prev_desc=0x%p; next_desc=0x%p; " 735 "prev->hw_next=0x%p\n", __func__, prev_desc, 736 next_desc, prev_desc ? prev_desc->hw_next : 0); 737 BUG(); 738 } 739 740 local_irq_save(flags); 741 742 /* do s/w chaining both for DMA and XOR descriptors */ 743 prev_desc->hw_next = next_desc; 744 745 switch (chan->device->id) { 746 case PPC440SPE_DMA0_ID: 747 case PPC440SPE_DMA1_ID: 748 break; 749 case PPC440SPE_XOR_ID: 750 /* bind descriptor to the chain */ 751 while (tail->hw_next) 752 tail = tail->hw_next; 753 xor_last_linked = tail; 754 755 if (prev_desc == xor_last_submit) 756 /* do not link to the last submitted CB */ 757 break; 758 ppc440spe_xor_set_link(prev_desc, next_desc); 759 break; 760 } 761 762 local_irq_restore(flags); 763 } 764 765 /** 766 * ppc440spe_desc_get_link - get the address of the descriptor that 767 * follows this one 768 */ 769 static inline u32 ppc440spe_desc_get_link(struct ppc440spe_adma_desc_slot *desc, 770 struct ppc440spe_adma_chan *chan) 771 { 772 if (!desc->hw_next) 773 return 0; 774 775 return desc->hw_next->phys; 776 } 777 778 /** 779 * ppc440spe_desc_is_aligned - check alignment 780 */ 781 static inline int ppc440spe_desc_is_aligned( 782 struct ppc440spe_adma_desc_slot *desc, int num_slots) 783 { 784 return (desc->idx & (num_slots - 1)) ? 0 : 1; 785 } 786 787 /** 788 * ppc440spe_chan_xor_slot_count - get the number of slots necessary for 789 * XOR operation 790 */ 791 static int ppc440spe_chan_xor_slot_count(size_t len, int src_cnt, 792 int *slots_per_op) 793 { 794 int slot_cnt; 795 796 /* each XOR descriptor provides up to 16 source operands */ 797 slot_cnt = *slots_per_op = (src_cnt + XOR_MAX_OPS - 1)/XOR_MAX_OPS; 798 799 if (likely(len <= PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT)) 800 return slot_cnt; 801 802 printk(KERN_ERR "%s: len %d > max %d !!\n", 803 __func__, len, PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT); 804 BUG(); 805 return slot_cnt; 806 } 807 808 /** 809 * ppc440spe_dma2_pq_slot_count - get the number of slots necessary for 810 * DMA2 PQ operation 811 */ 812 static int ppc440spe_dma2_pq_slot_count(dma_addr_t *srcs, 813 int src_cnt, size_t len) 814 { 815 signed long long order = 0; 816 int state = 0; 817 int addr_count = 0; 818 int i; 819 for (i = 1; i < src_cnt; i++) { 820 dma_addr_t cur_addr = srcs[i]; 821 dma_addr_t old_addr = srcs[i-1]; 822 switch (state) { 823 case 0: 824 if (cur_addr == old_addr + len) { 825 /* direct RXOR */ 826 order = 1; 827 state = 1; 828 if (i == src_cnt-1) 829 addr_count++; 830 } else if (old_addr == cur_addr + len) { 831 /* reverse RXOR */ 832 order = -1; 833 state = 1; 834 if (i == src_cnt-1) 835 addr_count++; 836 } else { 837 state = 3; 838 } 839 break; 840 case 1: 841 if (i == src_cnt-2 || (order == -1 842 && cur_addr != old_addr - len)) { 843 order = 0; 844 state = 0; 845 addr_count++; 846 } else if (cur_addr == old_addr + len*order) { 847 state = 2; 848 if (i == src_cnt-1) 849 addr_count++; 850 } else if (cur_addr == old_addr + 2*len) { 851 state = 2; 852 if (i == src_cnt-1) 853 addr_count++; 854 } else if (cur_addr == old_addr + 3*len) { 855 state = 2; 856 if (i == src_cnt-1) 857 addr_count++; 858 } else { 859 order = 0; 860 state = 0; 861 addr_count++; 862 } 863 break; 864 case 2: 865 order = 0; 866 state = 0; 867 addr_count++; 868 break; 869 } 870 if (state == 3) 871 break; 872 } 873 if (src_cnt <= 1 || (state != 1 && state != 2)) { 874 pr_err("%s: src_cnt=%d, state=%d, addr_count=%d, order=%lld\n", 875 __func__, src_cnt, state, addr_count, order); 876 for (i = 0; i < src_cnt; i++) 877 pr_err("\t[%d] 0x%llx \n", i, srcs[i]); 878 BUG(); 879 } 880 881 return (addr_count + XOR_MAX_OPS - 1) / XOR_MAX_OPS; 882 } 883 884 885 /****************************************************************************** 886 * ADMA channel low-level routines 887 ******************************************************************************/ 888 889 static u32 890 ppc440spe_chan_get_current_descriptor(struct ppc440spe_adma_chan *chan); 891 static void ppc440spe_chan_append(struct ppc440spe_adma_chan *chan); 892 893 /** 894 * ppc440spe_adma_device_clear_eot_status - interrupt ack to XOR or DMA engine 895 */ 896 static void ppc440spe_adma_device_clear_eot_status( 897 struct ppc440spe_adma_chan *chan) 898 { 899 struct dma_regs *dma_reg; 900 struct xor_regs *xor_reg; 901 u8 *p = chan->device->dma_desc_pool_virt; 902 struct dma_cdb *cdb; 903 u32 rv, i; 904 905 switch (chan->device->id) { 906 case PPC440SPE_DMA0_ID: 907 case PPC440SPE_DMA1_ID: 908 /* read FIFO to ack */ 909 dma_reg = chan->device->dma_reg; 910 while ((rv = ioread32(&dma_reg->csfpl))) { 911 i = rv & DMA_CDB_ADDR_MSK; 912 cdb = (struct dma_cdb *)&p[i - 913 (u32)chan->device->dma_desc_pool]; 914 915 /* Clear opcode to ack. This is necessary for 916 * ZeroSum operations only 917 */ 918 cdb->opc = 0; 919 920 if (test_bit(PPC440SPE_RXOR_RUN, 921 &ppc440spe_rxor_state)) { 922 /* probably this is a completed RXOR op, 923 * get pointer to CDB using the fact that 924 * physical and virtual addresses of CDB 925 * in pools have the same offsets 926 */ 927 if (le32_to_cpu(cdb->sg1u) & 928 DMA_CUED_XOR_BASE) { 929 /* this is a RXOR */ 930 clear_bit(PPC440SPE_RXOR_RUN, 931 &ppc440spe_rxor_state); 932 } 933 } 934 935 if (rv & DMA_CDB_STATUS_MSK) { 936 /* ZeroSum check failed 937 */ 938 struct ppc440spe_adma_desc_slot *iter; 939 dma_addr_t phys = rv & ~DMA_CDB_MSK; 940 941 /* 942 * Update the status of corresponding 943 * descriptor. 944 */ 945 list_for_each_entry(iter, &chan->chain, 946 chain_node) { 947 if (iter->phys == phys) 948 break; 949 } 950 /* 951 * if cannot find the corresponding 952 * slot it's a bug 953 */ 954 BUG_ON(&iter->chain_node == &chan->chain); 955 956 if (iter->xor_check_result) { 957 if (test_bit(PPC440SPE_DESC_PCHECK, 958 &iter->flags)) { 959 *iter->xor_check_result |= 960 SUM_CHECK_P_RESULT; 961 } else 962 if (test_bit(PPC440SPE_DESC_QCHECK, 963 &iter->flags)) { 964 *iter->xor_check_result |= 965 SUM_CHECK_Q_RESULT; 966 } else 967 BUG(); 968 } 969 } 970 } 971 972 rv = ioread32(&dma_reg->dsts); 973 if (rv) { 974 pr_err("DMA%d err status: 0x%x\n", 975 chan->device->id, rv); 976 /* write back to clear */ 977 iowrite32(rv, &dma_reg->dsts); 978 } 979 break; 980 case PPC440SPE_XOR_ID: 981 /* reset status bits to ack */ 982 xor_reg = chan->device->xor_reg; 983 rv = ioread32be(&xor_reg->sr); 984 iowrite32be(rv, &xor_reg->sr); 985 986 if (rv & (XOR_IE_ICBIE_BIT|XOR_IE_ICIE_BIT|XOR_IE_RPTIE_BIT)) { 987 if (rv & XOR_IE_RPTIE_BIT) { 988 /* Read PLB Timeout Error. 989 * Try to resubmit the CB 990 */ 991 u32 val = ioread32be(&xor_reg->ccbalr); 992 993 iowrite32be(val, &xor_reg->cblalr); 994 995 val = ioread32be(&xor_reg->crsr); 996 iowrite32be(val | XOR_CRSR_XAE_BIT, 997 &xor_reg->crsr); 998 } else 999 pr_err("XOR ERR 0x%x status\n", rv); 1000 break; 1001 } 1002 1003 /* if the XORcore is idle, but there are unprocessed CBs 1004 * then refetch the s/w chain here 1005 */ 1006 if (!(ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT) && 1007 do_xor_refetch) 1008 ppc440spe_chan_append(chan); 1009 break; 1010 } 1011 } 1012 1013 /** 1014 * ppc440spe_chan_is_busy - get the channel status 1015 */ 1016 static int ppc440spe_chan_is_busy(struct ppc440spe_adma_chan *chan) 1017 { 1018 struct dma_regs *dma_reg; 1019 struct xor_regs *xor_reg; 1020 int busy = 0; 1021 1022 switch (chan->device->id) { 1023 case PPC440SPE_DMA0_ID: 1024 case PPC440SPE_DMA1_ID: 1025 dma_reg = chan->device->dma_reg; 1026 /* if command FIFO's head and tail pointers are equal and 1027 * status tail is the same as command, then channel is free 1028 */ 1029 if (ioread16(&dma_reg->cpfhp) != ioread16(&dma_reg->cpftp) || 1030 ioread16(&dma_reg->cpftp) != ioread16(&dma_reg->csftp)) 1031 busy = 1; 1032 break; 1033 case PPC440SPE_XOR_ID: 1034 /* use the special status bit for the XORcore 1035 */ 1036 xor_reg = chan->device->xor_reg; 1037 busy = (ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT) ? 1 : 0; 1038 break; 1039 } 1040 1041 return busy; 1042 } 1043 1044 /** 1045 * ppc440spe_chan_set_first_xor_descriptor - init XORcore chain 1046 */ 1047 static void ppc440spe_chan_set_first_xor_descriptor( 1048 struct ppc440spe_adma_chan *chan, 1049 struct ppc440spe_adma_desc_slot *next_desc) 1050 { 1051 struct xor_regs *xor_reg = chan->device->xor_reg; 1052 1053 if (ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT) 1054 printk(KERN_INFO "%s: Warn: XORcore is running " 1055 "when try to set the first CDB!\n", 1056 __func__); 1057 1058 xor_last_submit = xor_last_linked = next_desc; 1059 1060 iowrite32be(XOR_CRSR_64BA_BIT, &xor_reg->crsr); 1061 1062 iowrite32be(next_desc->phys, &xor_reg->cblalr); 1063 iowrite32be(0, &xor_reg->cblahr); 1064 iowrite32be(ioread32be(&xor_reg->cbcr) | XOR_CBCR_LNK_BIT, 1065 &xor_reg->cbcr); 1066 1067 chan->hw_chain_inited = 1; 1068 } 1069 1070 /** 1071 * ppc440spe_dma_put_desc - put DMA0,1 descriptor to FIFO. 1072 * called with irqs disabled 1073 */ 1074 static void ppc440spe_dma_put_desc(struct ppc440spe_adma_chan *chan, 1075 struct ppc440spe_adma_desc_slot *desc) 1076 { 1077 u32 pcdb; 1078 struct dma_regs *dma_reg = chan->device->dma_reg; 1079 1080 pcdb = desc->phys; 1081 if (!test_bit(PPC440SPE_DESC_INT, &desc->flags)) 1082 pcdb |= DMA_CDB_NO_INT; 1083 1084 chan_last_sub[chan->device->id] = desc; 1085 1086 ADMA_LL_DBG(print_cb(chan, desc->hw_desc)); 1087 1088 iowrite32(pcdb, &dma_reg->cpfpl); 1089 } 1090 1091 /** 1092 * ppc440spe_chan_append - update the h/w chain in the channel 1093 */ 1094 static void ppc440spe_chan_append(struct ppc440spe_adma_chan *chan) 1095 { 1096 struct xor_regs *xor_reg; 1097 struct ppc440spe_adma_desc_slot *iter; 1098 struct xor_cb *xcb; 1099 u32 cur_desc; 1100 unsigned long flags; 1101 1102 local_irq_save(flags); 1103 1104 switch (chan->device->id) { 1105 case PPC440SPE_DMA0_ID: 1106 case PPC440SPE_DMA1_ID: 1107 cur_desc = ppc440spe_chan_get_current_descriptor(chan); 1108 1109 if (likely(cur_desc)) { 1110 iter = chan_last_sub[chan->device->id]; 1111 BUG_ON(!iter); 1112 } else { 1113 /* first peer */ 1114 iter = chan_first_cdb[chan->device->id]; 1115 BUG_ON(!iter); 1116 ppc440spe_dma_put_desc(chan, iter); 1117 chan->hw_chain_inited = 1; 1118 } 1119 1120 /* is there something new to append */ 1121 if (!iter->hw_next) 1122 break; 1123 1124 /* flush descriptors from the s/w queue to fifo */ 1125 list_for_each_entry_continue(iter, &chan->chain, chain_node) { 1126 ppc440spe_dma_put_desc(chan, iter); 1127 if (!iter->hw_next) 1128 break; 1129 } 1130 break; 1131 case PPC440SPE_XOR_ID: 1132 /* update h/w links and refetch */ 1133 if (!xor_last_submit->hw_next) 1134 break; 1135 1136 xor_reg = chan->device->xor_reg; 1137 /* the last linked CDB has to generate an interrupt 1138 * that we'd be able to append the next lists to h/w 1139 * regardless of the XOR engine state at the moment of 1140 * appending of these next lists 1141 */ 1142 xcb = xor_last_linked->hw_desc; 1143 xcb->cbc |= XOR_CBCR_CBCE_BIT; 1144 1145 if (!(ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT)) { 1146 /* XORcore is idle. Refetch now */ 1147 do_xor_refetch = 0; 1148 ppc440spe_xor_set_link(xor_last_submit, 1149 xor_last_submit->hw_next); 1150 1151 ADMA_LL_DBG(print_cb_list(chan, 1152 xor_last_submit->hw_next)); 1153 1154 xor_last_submit = xor_last_linked; 1155 iowrite32be(ioread32be(&xor_reg->crsr) | 1156 XOR_CRSR_RCBE_BIT | XOR_CRSR_64BA_BIT, 1157 &xor_reg->crsr); 1158 } else { 1159 /* XORcore is running. Refetch later in the handler */ 1160 do_xor_refetch = 1; 1161 } 1162 1163 break; 1164 } 1165 1166 local_irq_restore(flags); 1167 } 1168 1169 /** 1170 * ppc440spe_chan_get_current_descriptor - get the currently executed descriptor 1171 */ 1172 static u32 1173 ppc440spe_chan_get_current_descriptor(struct ppc440spe_adma_chan *chan) 1174 { 1175 struct dma_regs *dma_reg; 1176 struct xor_regs *xor_reg; 1177 1178 if (unlikely(!chan->hw_chain_inited)) 1179 /* h/w descriptor chain is not initialized yet */ 1180 return 0; 1181 1182 switch (chan->device->id) { 1183 case PPC440SPE_DMA0_ID: 1184 case PPC440SPE_DMA1_ID: 1185 dma_reg = chan->device->dma_reg; 1186 return ioread32(&dma_reg->acpl) & (~DMA_CDB_MSK); 1187 case PPC440SPE_XOR_ID: 1188 xor_reg = chan->device->xor_reg; 1189 return ioread32be(&xor_reg->ccbalr); 1190 } 1191 return 0; 1192 } 1193 1194 /** 1195 * ppc440spe_chan_run - enable the channel 1196 */ 1197 static void ppc440spe_chan_run(struct ppc440spe_adma_chan *chan) 1198 { 1199 struct xor_regs *xor_reg; 1200 1201 switch (chan->device->id) { 1202 case PPC440SPE_DMA0_ID: 1203 case PPC440SPE_DMA1_ID: 1204 /* DMAs are always enabled, do nothing */ 1205 break; 1206 case PPC440SPE_XOR_ID: 1207 /* drain write buffer */ 1208 xor_reg = chan->device->xor_reg; 1209 1210 /* fetch descriptor pointed to in <link> */ 1211 iowrite32be(XOR_CRSR_64BA_BIT | XOR_CRSR_XAE_BIT, 1212 &xor_reg->crsr); 1213 break; 1214 } 1215 } 1216 1217 /****************************************************************************** 1218 * ADMA device level 1219 ******************************************************************************/ 1220 1221 static void ppc440spe_chan_start_null_xor(struct ppc440spe_adma_chan *chan); 1222 static int ppc440spe_adma_alloc_chan_resources(struct dma_chan *chan); 1223 1224 static dma_cookie_t 1225 ppc440spe_adma_tx_submit(struct dma_async_tx_descriptor *tx); 1226 1227 static void ppc440spe_adma_set_dest(struct ppc440spe_adma_desc_slot *tx, 1228 dma_addr_t addr, int index); 1229 static void 1230 ppc440spe_adma_memcpy_xor_set_src(struct ppc440spe_adma_desc_slot *tx, 1231 dma_addr_t addr, int index); 1232 1233 static void 1234 ppc440spe_adma_pq_set_dest(struct ppc440spe_adma_desc_slot *tx, 1235 dma_addr_t *paddr, unsigned long flags); 1236 static void 1237 ppc440spe_adma_pq_set_src(struct ppc440spe_adma_desc_slot *tx, 1238 dma_addr_t addr, int index); 1239 static void 1240 ppc440spe_adma_pq_set_src_mult(struct ppc440spe_adma_desc_slot *tx, 1241 unsigned char mult, int index, int dst_pos); 1242 static void 1243 ppc440spe_adma_pqzero_sum_set_dest(struct ppc440spe_adma_desc_slot *tx, 1244 dma_addr_t paddr, dma_addr_t qaddr); 1245 1246 static struct page *ppc440spe_rxor_srcs[32]; 1247 1248 /** 1249 * ppc440spe_can_rxor - check if the operands may be processed with RXOR 1250 */ 1251 static int ppc440spe_can_rxor(struct page **srcs, int src_cnt, size_t len) 1252 { 1253 int i, order = 0, state = 0; 1254 int idx = 0; 1255 1256 if (unlikely(!(src_cnt > 1))) 1257 return 0; 1258 1259 BUG_ON(src_cnt > ARRAY_SIZE(ppc440spe_rxor_srcs)); 1260 1261 /* Skip holes in the source list before checking */ 1262 for (i = 0; i < src_cnt; i++) { 1263 if (!srcs[i]) 1264 continue; 1265 ppc440spe_rxor_srcs[idx++] = srcs[i]; 1266 } 1267 src_cnt = idx; 1268 1269 for (i = 1; i < src_cnt; i++) { 1270 char *cur_addr = page_address(ppc440spe_rxor_srcs[i]); 1271 char *old_addr = page_address(ppc440spe_rxor_srcs[i - 1]); 1272 1273 switch (state) { 1274 case 0: 1275 if (cur_addr == old_addr + len) { 1276 /* direct RXOR */ 1277 order = 1; 1278 state = 1; 1279 } else if (old_addr == cur_addr + len) { 1280 /* reverse RXOR */ 1281 order = -1; 1282 state = 1; 1283 } else 1284 goto out; 1285 break; 1286 case 1: 1287 if ((i == src_cnt - 2) || 1288 (order == -1 && cur_addr != old_addr - len)) { 1289 order = 0; 1290 state = 0; 1291 } else if ((cur_addr == old_addr + len * order) || 1292 (cur_addr == old_addr + 2 * len) || 1293 (cur_addr == old_addr + 3 * len)) { 1294 state = 2; 1295 } else { 1296 order = 0; 1297 state = 0; 1298 } 1299 break; 1300 case 2: 1301 order = 0; 1302 state = 0; 1303 break; 1304 } 1305 } 1306 1307 out: 1308 if (state == 1 || state == 2) 1309 return 1; 1310 1311 return 0; 1312 } 1313 1314 /** 1315 * ppc440spe_adma_device_estimate - estimate the efficiency of processing 1316 * the operation given on this channel. It's assumed that 'chan' is 1317 * capable to process 'cap' type of operation. 1318 * @chan: channel to use 1319 * @cap: type of transaction 1320 * @dst_lst: array of destination pointers 1321 * @dst_cnt: number of destination operands 1322 * @src_lst: array of source pointers 1323 * @src_cnt: number of source operands 1324 * @src_sz: size of each source operand 1325 */ 1326 static int ppc440spe_adma_estimate(struct dma_chan *chan, 1327 enum dma_transaction_type cap, struct page **dst_lst, int dst_cnt, 1328 struct page **src_lst, int src_cnt, size_t src_sz) 1329 { 1330 int ef = 1; 1331 1332 if (cap == DMA_PQ || cap == DMA_PQ_VAL) { 1333 /* If RAID-6 capabilities were not activated don't try 1334 * to use them 1335 */ 1336 if (unlikely(!ppc440spe_r6_enabled)) 1337 return -1; 1338 } 1339 /* In the current implementation of ppc440spe ADMA driver it 1340 * makes sense to pick out only pq case, because it may be 1341 * processed: 1342 * (1) either using Biskup method on DMA2; 1343 * (2) or on DMA0/1. 1344 * Thus we give a favour to (1) if the sources are suitable; 1345 * else let it be processed on one of the DMA0/1 engines. 1346 * In the sum_product case where destination is also the 1347 * source process it on DMA0/1 only. 1348 */ 1349 if (cap == DMA_PQ && chan->chan_id == PPC440SPE_XOR_ID) { 1350 1351 if (dst_cnt == 1 && src_cnt == 2 && dst_lst[0] == src_lst[1]) 1352 ef = 0; /* sum_product case, process on DMA0/1 */ 1353 else if (ppc440spe_can_rxor(src_lst, src_cnt, src_sz)) 1354 ef = 3; /* override (DMA0/1 + idle) */ 1355 else 1356 ef = 0; /* can't process on DMA2 if !rxor */ 1357 } 1358 1359 /* channel idleness increases the priority */ 1360 if (likely(ef) && 1361 !ppc440spe_chan_is_busy(to_ppc440spe_adma_chan(chan))) 1362 ef++; 1363 1364 return ef; 1365 } 1366 1367 struct dma_chan * 1368 ppc440spe_async_tx_find_best_channel(enum dma_transaction_type cap, 1369 struct page **dst_lst, int dst_cnt, struct page **src_lst, 1370 int src_cnt, size_t src_sz) 1371 { 1372 struct dma_chan *best_chan = NULL; 1373 struct ppc_dma_chan_ref *ref; 1374 int best_rank = -1; 1375 1376 if (unlikely(!src_sz)) 1377 return NULL; 1378 if (src_sz > PAGE_SIZE) { 1379 /* 1380 * should a user of the api ever pass > PAGE_SIZE requests 1381 * we sort out cases where temporary page-sized buffers 1382 * are used. 1383 */ 1384 switch (cap) { 1385 case DMA_PQ: 1386 if (src_cnt == 1 && dst_lst[1] == src_lst[0]) 1387 return NULL; 1388 if (src_cnt == 2 && dst_lst[1] == src_lst[1]) 1389 return NULL; 1390 break; 1391 case DMA_PQ_VAL: 1392 case DMA_XOR_VAL: 1393 return NULL; 1394 default: 1395 break; 1396 } 1397 } 1398 1399 list_for_each_entry(ref, &ppc440spe_adma_chan_list, node) { 1400 if (dma_has_cap(cap, ref->chan->device->cap_mask)) { 1401 int rank; 1402 1403 rank = ppc440spe_adma_estimate(ref->chan, cap, dst_lst, 1404 dst_cnt, src_lst, src_cnt, src_sz); 1405 if (rank > best_rank) { 1406 best_rank = rank; 1407 best_chan = ref->chan; 1408 } 1409 } 1410 } 1411 1412 return best_chan; 1413 } 1414 EXPORT_SYMBOL_GPL(ppc440spe_async_tx_find_best_channel); 1415 1416 /** 1417 * ppc440spe_get_group_entry - get group entry with index idx 1418 * @tdesc: is the last allocated slot in the group. 1419 */ 1420 static struct ppc440spe_adma_desc_slot * 1421 ppc440spe_get_group_entry(struct ppc440spe_adma_desc_slot *tdesc, u32 entry_idx) 1422 { 1423 struct ppc440spe_adma_desc_slot *iter = tdesc->group_head; 1424 int i = 0; 1425 1426 if (entry_idx < 0 || entry_idx >= (tdesc->src_cnt + tdesc->dst_cnt)) { 1427 printk("%s: entry_idx %d, src_cnt %d, dst_cnt %d\n", 1428 __func__, entry_idx, tdesc->src_cnt, tdesc->dst_cnt); 1429 BUG(); 1430 } 1431 1432 list_for_each_entry(iter, &tdesc->group_list, chain_node) { 1433 if (i++ == entry_idx) 1434 break; 1435 } 1436 return iter; 1437 } 1438 1439 /** 1440 * ppc440spe_adma_free_slots - flags descriptor slots for reuse 1441 * @slot: Slot to free 1442 * Caller must hold &ppc440spe_chan->lock while calling this function 1443 */ 1444 static void ppc440spe_adma_free_slots(struct ppc440spe_adma_desc_slot *slot, 1445 struct ppc440spe_adma_chan *chan) 1446 { 1447 int stride = slot->slots_per_op; 1448 1449 while (stride--) { 1450 slot->slots_per_op = 0; 1451 slot = list_entry(slot->slot_node.next, 1452 struct ppc440spe_adma_desc_slot, 1453 slot_node); 1454 } 1455 } 1456 1457 /** 1458 * ppc440spe_adma_run_tx_complete_actions - call functions to be called 1459 * upon completion 1460 */ 1461 static dma_cookie_t ppc440spe_adma_run_tx_complete_actions( 1462 struct ppc440spe_adma_desc_slot *desc, 1463 struct ppc440spe_adma_chan *chan, 1464 dma_cookie_t cookie) 1465 { 1466 BUG_ON(desc->async_tx.cookie < 0); 1467 if (desc->async_tx.cookie > 0) { 1468 cookie = desc->async_tx.cookie; 1469 desc->async_tx.cookie = 0; 1470 1471 dma_descriptor_unmap(&desc->async_tx); 1472 /* call the callback (must not sleep or submit new 1473 * operations to this channel) 1474 */ 1475 dmaengine_desc_get_callback_invoke(&desc->async_tx, NULL); 1476 } 1477 1478 /* run dependent operations */ 1479 dma_run_dependencies(&desc->async_tx); 1480 1481 return cookie; 1482 } 1483 1484 /** 1485 * ppc440spe_adma_clean_slot - clean up CDB slot (if ack is set) 1486 */ 1487 static int ppc440spe_adma_clean_slot(struct ppc440spe_adma_desc_slot *desc, 1488 struct ppc440spe_adma_chan *chan) 1489 { 1490 /* the client is allowed to attach dependent operations 1491 * until 'ack' is set 1492 */ 1493 if (!async_tx_test_ack(&desc->async_tx)) 1494 return 0; 1495 1496 /* leave the last descriptor in the chain 1497 * so we can append to it 1498 */ 1499 if (list_is_last(&desc->chain_node, &chan->chain) || 1500 desc->phys == ppc440spe_chan_get_current_descriptor(chan)) 1501 return 1; 1502 1503 if (chan->device->id != PPC440SPE_XOR_ID) { 1504 /* our DMA interrupt handler clears opc field of 1505 * each processed descriptor. For all types of 1506 * operations except for ZeroSum we do not actually 1507 * need ack from the interrupt handler. ZeroSum is a 1508 * special case since the result of this operation 1509 * is available from the handler only, so if we see 1510 * such type of descriptor (which is unprocessed yet) 1511 * then leave it in chain. 1512 */ 1513 struct dma_cdb *cdb = desc->hw_desc; 1514 if (cdb->opc == DMA_CDB_OPC_DCHECK128) 1515 return 1; 1516 } 1517 1518 dev_dbg(chan->device->common.dev, "\tfree slot %llx: %d stride: %d\n", 1519 desc->phys, desc->idx, desc->slots_per_op); 1520 1521 list_del(&desc->chain_node); 1522 ppc440spe_adma_free_slots(desc, chan); 1523 return 0; 1524 } 1525 1526 /** 1527 * __ppc440spe_adma_slot_cleanup - this is the common clean-up routine 1528 * which runs through the channel CDBs list until reach the descriptor 1529 * currently processed. When routine determines that all CDBs of group 1530 * are completed then corresponding callbacks (if any) are called and slots 1531 * are freed. 1532 */ 1533 static void __ppc440spe_adma_slot_cleanup(struct ppc440spe_adma_chan *chan) 1534 { 1535 struct ppc440spe_adma_desc_slot *iter, *_iter, *group_start = NULL; 1536 dma_cookie_t cookie = 0; 1537 u32 current_desc = ppc440spe_chan_get_current_descriptor(chan); 1538 int busy = ppc440spe_chan_is_busy(chan); 1539 int seen_current = 0, slot_cnt = 0, slots_per_op = 0; 1540 1541 dev_dbg(chan->device->common.dev, "ppc440spe adma%d: %s\n", 1542 chan->device->id, __func__); 1543 1544 if (!current_desc) { 1545 /* There were no transactions yet, so 1546 * nothing to clean 1547 */ 1548 return; 1549 } 1550 1551 /* free completed slots from the chain starting with 1552 * the oldest descriptor 1553 */ 1554 list_for_each_entry_safe(iter, _iter, &chan->chain, 1555 chain_node) { 1556 dev_dbg(chan->device->common.dev, "\tcookie: %d slot: %d " 1557 "busy: %d this_desc: %#llx next_desc: %#x " 1558 "cur: %#x ack: %d\n", 1559 iter->async_tx.cookie, iter->idx, busy, iter->phys, 1560 ppc440spe_desc_get_link(iter, chan), current_desc, 1561 async_tx_test_ack(&iter->async_tx)); 1562 prefetch(_iter); 1563 prefetch(&_iter->async_tx); 1564 1565 /* do not advance past the current descriptor loaded into the 1566 * hardware channel,subsequent descriptors are either in process 1567 * or have not been submitted 1568 */ 1569 if (seen_current) 1570 break; 1571 1572 /* stop the search if we reach the current descriptor and the 1573 * channel is busy, or if it appears that the current descriptor 1574 * needs to be re-read (i.e. has been appended to) 1575 */ 1576 if (iter->phys == current_desc) { 1577 BUG_ON(seen_current++); 1578 if (busy || ppc440spe_desc_get_link(iter, chan)) { 1579 /* not all descriptors of the group have 1580 * been completed; exit. 1581 */ 1582 break; 1583 } 1584 } 1585 1586 /* detect the start of a group transaction */ 1587 if (!slot_cnt && !slots_per_op) { 1588 slot_cnt = iter->slot_cnt; 1589 slots_per_op = iter->slots_per_op; 1590 if (slot_cnt <= slots_per_op) { 1591 slot_cnt = 0; 1592 slots_per_op = 0; 1593 } 1594 } 1595 1596 if (slot_cnt) { 1597 if (!group_start) 1598 group_start = iter; 1599 slot_cnt -= slots_per_op; 1600 } 1601 1602 /* all the members of a group are complete */ 1603 if (slots_per_op != 0 && slot_cnt == 0) { 1604 struct ppc440spe_adma_desc_slot *grp_iter, *_grp_iter; 1605 int end_of_chain = 0; 1606 1607 /* clean up the group */ 1608 slot_cnt = group_start->slot_cnt; 1609 grp_iter = group_start; 1610 list_for_each_entry_safe_from(grp_iter, _grp_iter, 1611 &chan->chain, chain_node) { 1612 1613 cookie = ppc440spe_adma_run_tx_complete_actions( 1614 grp_iter, chan, cookie); 1615 1616 slot_cnt -= slots_per_op; 1617 end_of_chain = ppc440spe_adma_clean_slot( 1618 grp_iter, chan); 1619 if (end_of_chain && slot_cnt) { 1620 /* Should wait for ZeroSum completion */ 1621 if (cookie > 0) 1622 chan->common.completed_cookie = cookie; 1623 return; 1624 } 1625 1626 if (slot_cnt == 0 || end_of_chain) 1627 break; 1628 } 1629 1630 /* the group should be complete at this point */ 1631 BUG_ON(slot_cnt); 1632 1633 slots_per_op = 0; 1634 group_start = NULL; 1635 if (end_of_chain) 1636 break; 1637 else 1638 continue; 1639 } else if (slots_per_op) /* wait for group completion */ 1640 continue; 1641 1642 cookie = ppc440spe_adma_run_tx_complete_actions(iter, chan, 1643 cookie); 1644 1645 if (ppc440spe_adma_clean_slot(iter, chan)) 1646 break; 1647 } 1648 1649 BUG_ON(!seen_current); 1650 1651 if (cookie > 0) { 1652 chan->common.completed_cookie = cookie; 1653 pr_debug("\tcompleted cookie %d\n", cookie); 1654 } 1655 1656 } 1657 1658 /** 1659 * ppc440spe_adma_tasklet - clean up watch-dog initiator 1660 */ 1661 static void ppc440spe_adma_tasklet(struct tasklet_struct *t) 1662 { 1663 struct ppc440spe_adma_chan *chan = from_tasklet(chan, t, irq_tasklet); 1664 1665 spin_lock_nested(&chan->lock, SINGLE_DEPTH_NESTING); 1666 __ppc440spe_adma_slot_cleanup(chan); 1667 spin_unlock(&chan->lock); 1668 } 1669 1670 /** 1671 * ppc440spe_adma_slot_cleanup - clean up scheduled initiator 1672 */ 1673 static void ppc440spe_adma_slot_cleanup(struct ppc440spe_adma_chan *chan) 1674 { 1675 spin_lock_bh(&chan->lock); 1676 __ppc440spe_adma_slot_cleanup(chan); 1677 spin_unlock_bh(&chan->lock); 1678 } 1679 1680 /** 1681 * ppc440spe_adma_alloc_slots - allocate free slots (if any) 1682 */ 1683 static struct ppc440spe_adma_desc_slot *ppc440spe_adma_alloc_slots( 1684 struct ppc440spe_adma_chan *chan, int num_slots, 1685 int slots_per_op) 1686 { 1687 struct ppc440spe_adma_desc_slot *iter = NULL, *_iter; 1688 struct ppc440spe_adma_desc_slot *alloc_start = NULL; 1689 struct list_head chain = LIST_HEAD_INIT(chain); 1690 int slots_found, retry = 0; 1691 1692 1693 BUG_ON(!num_slots || !slots_per_op); 1694 /* start search from the last allocated descrtiptor 1695 * if a contiguous allocation can not be found start searching 1696 * from the beginning of the list 1697 */ 1698 retry: 1699 slots_found = 0; 1700 if (retry == 0) 1701 iter = chan->last_used; 1702 else 1703 iter = list_entry(&chan->all_slots, 1704 struct ppc440spe_adma_desc_slot, 1705 slot_node); 1706 list_for_each_entry_safe_continue(iter, _iter, &chan->all_slots, 1707 slot_node) { 1708 prefetch(_iter); 1709 prefetch(&_iter->async_tx); 1710 if (iter->slots_per_op) { 1711 slots_found = 0; 1712 continue; 1713 } 1714 1715 /* start the allocation if the slot is correctly aligned */ 1716 if (!slots_found++) 1717 alloc_start = iter; 1718 1719 if (slots_found == num_slots) { 1720 struct ppc440spe_adma_desc_slot *alloc_tail = NULL; 1721 struct ppc440spe_adma_desc_slot *last_used = NULL; 1722 1723 iter = alloc_start; 1724 while (num_slots) { 1725 int i; 1726 /* pre-ack all but the last descriptor */ 1727 if (num_slots != slots_per_op) 1728 async_tx_ack(&iter->async_tx); 1729 1730 list_add_tail(&iter->chain_node, &chain); 1731 alloc_tail = iter; 1732 iter->async_tx.cookie = 0; 1733 iter->hw_next = NULL; 1734 iter->flags = 0; 1735 iter->slot_cnt = num_slots; 1736 iter->xor_check_result = NULL; 1737 for (i = 0; i < slots_per_op; i++) { 1738 iter->slots_per_op = slots_per_op - i; 1739 last_used = iter; 1740 iter = list_entry(iter->slot_node.next, 1741 struct ppc440spe_adma_desc_slot, 1742 slot_node); 1743 } 1744 num_slots -= slots_per_op; 1745 } 1746 alloc_tail->group_head = alloc_start; 1747 alloc_tail->async_tx.cookie = -EBUSY; 1748 list_splice(&chain, &alloc_tail->group_list); 1749 chan->last_used = last_used; 1750 return alloc_tail; 1751 } 1752 } 1753 if (!retry++) 1754 goto retry; 1755 1756 /* try to free some slots if the allocation fails */ 1757 tasklet_schedule(&chan->irq_tasklet); 1758 return NULL; 1759 } 1760 1761 /** 1762 * ppc440spe_adma_alloc_chan_resources - allocate pools for CDB slots 1763 */ 1764 static int ppc440spe_adma_alloc_chan_resources(struct dma_chan *chan) 1765 { 1766 struct ppc440spe_adma_chan *ppc440spe_chan; 1767 struct ppc440spe_adma_desc_slot *slot = NULL; 1768 char *hw_desc; 1769 int i, db_sz; 1770 int init; 1771 1772 ppc440spe_chan = to_ppc440spe_adma_chan(chan); 1773 init = ppc440spe_chan->slots_allocated ? 0 : 1; 1774 chan->chan_id = ppc440spe_chan->device->id; 1775 1776 /* Allocate descriptor slots */ 1777 i = ppc440spe_chan->slots_allocated; 1778 if (ppc440spe_chan->device->id != PPC440SPE_XOR_ID) 1779 db_sz = sizeof(struct dma_cdb); 1780 else 1781 db_sz = sizeof(struct xor_cb); 1782 1783 for (; i < (ppc440spe_chan->device->pool_size / db_sz); i++) { 1784 slot = kzalloc(sizeof(struct ppc440spe_adma_desc_slot), 1785 GFP_KERNEL); 1786 if (!slot) { 1787 printk(KERN_INFO "SPE ADMA Channel only initialized" 1788 " %d descriptor slots", i--); 1789 break; 1790 } 1791 1792 hw_desc = (char *) ppc440spe_chan->device->dma_desc_pool_virt; 1793 slot->hw_desc = (void *) &hw_desc[i * db_sz]; 1794 dma_async_tx_descriptor_init(&slot->async_tx, chan); 1795 slot->async_tx.tx_submit = ppc440spe_adma_tx_submit; 1796 INIT_LIST_HEAD(&slot->chain_node); 1797 INIT_LIST_HEAD(&slot->slot_node); 1798 INIT_LIST_HEAD(&slot->group_list); 1799 slot->phys = ppc440spe_chan->device->dma_desc_pool + i * db_sz; 1800 slot->idx = i; 1801 1802 spin_lock_bh(&ppc440spe_chan->lock); 1803 ppc440spe_chan->slots_allocated++; 1804 list_add_tail(&slot->slot_node, &ppc440spe_chan->all_slots); 1805 spin_unlock_bh(&ppc440spe_chan->lock); 1806 } 1807 1808 if (i && !ppc440spe_chan->last_used) { 1809 ppc440spe_chan->last_used = 1810 list_entry(ppc440spe_chan->all_slots.next, 1811 struct ppc440spe_adma_desc_slot, 1812 slot_node); 1813 } 1814 1815 dev_dbg(ppc440spe_chan->device->common.dev, 1816 "ppc440spe adma%d: allocated %d descriptor slots\n", 1817 ppc440spe_chan->device->id, i); 1818 1819 /* initialize the channel and the chain with a null operation */ 1820 if (init) { 1821 switch (ppc440spe_chan->device->id) { 1822 case PPC440SPE_DMA0_ID: 1823 case PPC440SPE_DMA1_ID: 1824 ppc440spe_chan->hw_chain_inited = 0; 1825 /* Use WXOR for self-testing */ 1826 if (!ppc440spe_r6_tchan) 1827 ppc440spe_r6_tchan = ppc440spe_chan; 1828 break; 1829 case PPC440SPE_XOR_ID: 1830 ppc440spe_chan_start_null_xor(ppc440spe_chan); 1831 break; 1832 default: 1833 BUG(); 1834 } 1835 ppc440spe_chan->needs_unmap = 1; 1836 } 1837 1838 return (i > 0) ? i : -ENOMEM; 1839 } 1840 1841 /** 1842 * ppc440spe_rxor_set_region_data - 1843 */ 1844 static void ppc440spe_rxor_set_region(struct ppc440spe_adma_desc_slot *desc, 1845 u8 xor_arg_no, u32 mask) 1846 { 1847 struct xor_cb *xcb = desc->hw_desc; 1848 1849 xcb->ops[xor_arg_no].h |= mask; 1850 } 1851 1852 /** 1853 * ppc440spe_rxor_set_src - 1854 */ 1855 static void ppc440spe_rxor_set_src(struct ppc440spe_adma_desc_slot *desc, 1856 u8 xor_arg_no, dma_addr_t addr) 1857 { 1858 struct xor_cb *xcb = desc->hw_desc; 1859 1860 xcb->ops[xor_arg_no].h |= DMA_CUED_XOR_BASE; 1861 xcb->ops[xor_arg_no].l = addr; 1862 } 1863 1864 /** 1865 * ppc440spe_rxor_set_mult - 1866 */ 1867 static void ppc440spe_rxor_set_mult(struct ppc440spe_adma_desc_slot *desc, 1868 u8 xor_arg_no, u8 idx, u8 mult) 1869 { 1870 struct xor_cb *xcb = desc->hw_desc; 1871 1872 xcb->ops[xor_arg_no].h |= mult << (DMA_CUED_MULT1_OFF + idx * 8); 1873 } 1874 1875 /** 1876 * ppc440spe_adma_check_threshold - append CDBs to h/w chain if threshold 1877 * has been achieved 1878 */ 1879 static void ppc440spe_adma_check_threshold(struct ppc440spe_adma_chan *chan) 1880 { 1881 dev_dbg(chan->device->common.dev, "ppc440spe adma%d: pending: %d\n", 1882 chan->device->id, chan->pending); 1883 1884 if (chan->pending >= PPC440SPE_ADMA_THRESHOLD) { 1885 chan->pending = 0; 1886 ppc440spe_chan_append(chan); 1887 } 1888 } 1889 1890 /** 1891 * ppc440spe_adma_tx_submit - submit new descriptor group to the channel 1892 * (it's not necessary that descriptors will be submitted to the h/w 1893 * chains too right now) 1894 */ 1895 static dma_cookie_t ppc440spe_adma_tx_submit(struct dma_async_tx_descriptor *tx) 1896 { 1897 struct ppc440spe_adma_desc_slot *sw_desc; 1898 struct ppc440spe_adma_chan *chan = to_ppc440spe_adma_chan(tx->chan); 1899 struct ppc440spe_adma_desc_slot *group_start, *old_chain_tail; 1900 int slot_cnt; 1901 int slots_per_op; 1902 dma_cookie_t cookie; 1903 1904 sw_desc = tx_to_ppc440spe_adma_slot(tx); 1905 1906 group_start = sw_desc->group_head; 1907 slot_cnt = group_start->slot_cnt; 1908 slots_per_op = group_start->slots_per_op; 1909 1910 spin_lock_bh(&chan->lock); 1911 cookie = dma_cookie_assign(tx); 1912 1913 if (unlikely(list_empty(&chan->chain))) { 1914 /* first peer */ 1915 list_splice_init(&sw_desc->group_list, &chan->chain); 1916 chan_first_cdb[chan->device->id] = group_start; 1917 } else { 1918 /* isn't first peer, bind CDBs to chain */ 1919 old_chain_tail = list_entry(chan->chain.prev, 1920 struct ppc440spe_adma_desc_slot, 1921 chain_node); 1922 list_splice_init(&sw_desc->group_list, 1923 &old_chain_tail->chain_node); 1924 /* fix up the hardware chain */ 1925 ppc440spe_desc_set_link(chan, old_chain_tail, group_start); 1926 } 1927 1928 /* increment the pending count by the number of operations */ 1929 chan->pending += slot_cnt / slots_per_op; 1930 ppc440spe_adma_check_threshold(chan); 1931 spin_unlock_bh(&chan->lock); 1932 1933 dev_dbg(chan->device->common.dev, 1934 "ppc440spe adma%d: %s cookie: %d slot: %d tx %p\n", 1935 chan->device->id, __func__, 1936 sw_desc->async_tx.cookie, sw_desc->idx, sw_desc); 1937 1938 return cookie; 1939 } 1940 1941 /** 1942 * ppc440spe_adma_prep_dma_interrupt - prepare CDB for a pseudo DMA operation 1943 */ 1944 static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_interrupt( 1945 struct dma_chan *chan, unsigned long flags) 1946 { 1947 struct ppc440spe_adma_chan *ppc440spe_chan; 1948 struct ppc440spe_adma_desc_slot *sw_desc, *group_start; 1949 int slot_cnt, slots_per_op; 1950 1951 ppc440spe_chan = to_ppc440spe_adma_chan(chan); 1952 1953 dev_dbg(ppc440spe_chan->device->common.dev, 1954 "ppc440spe adma%d: %s\n", ppc440spe_chan->device->id, 1955 __func__); 1956 1957 spin_lock_bh(&ppc440spe_chan->lock); 1958 slot_cnt = slots_per_op = 1; 1959 sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1960 slots_per_op); 1961 if (sw_desc) { 1962 group_start = sw_desc->group_head; 1963 ppc440spe_desc_init_interrupt(group_start, ppc440spe_chan); 1964 group_start->unmap_len = 0; 1965 sw_desc->async_tx.flags = flags; 1966 } 1967 spin_unlock_bh(&ppc440spe_chan->lock); 1968 1969 return sw_desc ? &sw_desc->async_tx : NULL; 1970 } 1971 1972 /** 1973 * ppc440spe_adma_prep_dma_memcpy - prepare CDB for a MEMCPY operation 1974 */ 1975 static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_memcpy( 1976 struct dma_chan *chan, dma_addr_t dma_dest, 1977 dma_addr_t dma_src, size_t len, unsigned long flags) 1978 { 1979 struct ppc440spe_adma_chan *ppc440spe_chan; 1980 struct ppc440spe_adma_desc_slot *sw_desc, *group_start; 1981 int slot_cnt, slots_per_op; 1982 1983 ppc440spe_chan = to_ppc440spe_adma_chan(chan); 1984 1985 if (unlikely(!len)) 1986 return NULL; 1987 1988 BUG_ON(len > PPC440SPE_ADMA_DMA_MAX_BYTE_COUNT); 1989 1990 spin_lock_bh(&ppc440spe_chan->lock); 1991 1992 dev_dbg(ppc440spe_chan->device->common.dev, 1993 "ppc440spe adma%d: %s len: %u int_en %d\n", 1994 ppc440spe_chan->device->id, __func__, len, 1995 flags & DMA_PREP_INTERRUPT ? 1 : 0); 1996 slot_cnt = slots_per_op = 1; 1997 sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1998 slots_per_op); 1999 if (sw_desc) { 2000 group_start = sw_desc->group_head; 2001 ppc440spe_desc_init_memcpy(group_start, flags); 2002 ppc440spe_adma_set_dest(group_start, dma_dest, 0); 2003 ppc440spe_adma_memcpy_xor_set_src(group_start, dma_src, 0); 2004 ppc440spe_desc_set_byte_count(group_start, ppc440spe_chan, len); 2005 sw_desc->unmap_len = len; 2006 sw_desc->async_tx.flags = flags; 2007 } 2008 spin_unlock_bh(&ppc440spe_chan->lock); 2009 2010 return sw_desc ? &sw_desc->async_tx : NULL; 2011 } 2012 2013 /** 2014 * ppc440spe_adma_prep_dma_xor - prepare CDB for a XOR operation 2015 */ 2016 static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_xor( 2017 struct dma_chan *chan, dma_addr_t dma_dest, 2018 dma_addr_t *dma_src, u32 src_cnt, size_t len, 2019 unsigned long flags) 2020 { 2021 struct ppc440spe_adma_chan *ppc440spe_chan; 2022 struct ppc440spe_adma_desc_slot *sw_desc, *group_start; 2023 int slot_cnt, slots_per_op; 2024 2025 ppc440spe_chan = to_ppc440spe_adma_chan(chan); 2026 2027 ADMA_LL_DBG(prep_dma_xor_dbg(ppc440spe_chan->device->id, 2028 dma_dest, dma_src, src_cnt)); 2029 if (unlikely(!len)) 2030 return NULL; 2031 BUG_ON(len > PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT); 2032 2033 dev_dbg(ppc440spe_chan->device->common.dev, 2034 "ppc440spe adma%d: %s src_cnt: %d len: %u int_en: %d\n", 2035 ppc440spe_chan->device->id, __func__, src_cnt, len, 2036 flags & DMA_PREP_INTERRUPT ? 1 : 0); 2037 2038 spin_lock_bh(&ppc440spe_chan->lock); 2039 slot_cnt = ppc440spe_chan_xor_slot_count(len, src_cnt, &slots_per_op); 2040 sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 2041 slots_per_op); 2042 if (sw_desc) { 2043 group_start = sw_desc->group_head; 2044 ppc440spe_desc_init_xor(group_start, src_cnt, flags); 2045 ppc440spe_adma_set_dest(group_start, dma_dest, 0); 2046 while (src_cnt--) 2047 ppc440spe_adma_memcpy_xor_set_src(group_start, 2048 dma_src[src_cnt], src_cnt); 2049 ppc440spe_desc_set_byte_count(group_start, ppc440spe_chan, len); 2050 sw_desc->unmap_len = len; 2051 sw_desc->async_tx.flags = flags; 2052 } 2053 spin_unlock_bh(&ppc440spe_chan->lock); 2054 2055 return sw_desc ? &sw_desc->async_tx : NULL; 2056 } 2057 2058 static inline void 2059 ppc440spe_desc_set_xor_src_cnt(struct ppc440spe_adma_desc_slot *desc, 2060 int src_cnt); 2061 static void ppc440spe_init_rxor_cursor(struct ppc440spe_rxor *cursor); 2062 2063 /** 2064 * ppc440spe_adma_init_dma2rxor_slot - 2065 */ 2066 static void ppc440spe_adma_init_dma2rxor_slot( 2067 struct ppc440spe_adma_desc_slot *desc, 2068 dma_addr_t *src, int src_cnt) 2069 { 2070 int i; 2071 2072 /* initialize CDB */ 2073 for (i = 0; i < src_cnt; i++) { 2074 ppc440spe_adma_dma2rxor_prep_src(desc, &desc->rxor_cursor, i, 2075 desc->src_cnt, (u32)src[i]); 2076 } 2077 } 2078 2079 /** 2080 * ppc440spe_dma01_prep_mult - 2081 * for Q operation where destination is also the source 2082 */ 2083 static struct ppc440spe_adma_desc_slot *ppc440spe_dma01_prep_mult( 2084 struct ppc440spe_adma_chan *ppc440spe_chan, 2085 dma_addr_t *dst, int dst_cnt, dma_addr_t *src, int src_cnt, 2086 const unsigned char *scf, size_t len, unsigned long flags) 2087 { 2088 struct ppc440spe_adma_desc_slot *sw_desc = NULL; 2089 unsigned long op = 0; 2090 int slot_cnt; 2091 2092 set_bit(PPC440SPE_DESC_WXOR, &op); 2093 slot_cnt = 2; 2094 2095 spin_lock_bh(&ppc440spe_chan->lock); 2096 2097 /* use WXOR, each descriptor occupies one slot */ 2098 sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1); 2099 if (sw_desc) { 2100 struct ppc440spe_adma_chan *chan; 2101 struct ppc440spe_adma_desc_slot *iter; 2102 struct dma_cdb *hw_desc; 2103 2104 chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan); 2105 set_bits(op, &sw_desc->flags); 2106 sw_desc->src_cnt = src_cnt; 2107 sw_desc->dst_cnt = dst_cnt; 2108 /* First descriptor, zero data in the destination and copy it 2109 * to q page using MULTICAST transfer. 2110 */ 2111 iter = list_first_entry(&sw_desc->group_list, 2112 struct ppc440spe_adma_desc_slot, 2113 chain_node); 2114 memset(iter->hw_desc, 0, sizeof(struct dma_cdb)); 2115 /* set 'next' pointer */ 2116 iter->hw_next = list_entry(iter->chain_node.next, 2117 struct ppc440spe_adma_desc_slot, 2118 chain_node); 2119 clear_bit(PPC440SPE_DESC_INT, &iter->flags); 2120 hw_desc = iter->hw_desc; 2121 hw_desc->opc = DMA_CDB_OPC_MULTICAST; 2122 2123 ppc440spe_desc_set_dest_addr(iter, chan, 2124 DMA_CUED_XOR_BASE, dst[0], 0); 2125 ppc440spe_desc_set_dest_addr(iter, chan, 0, dst[1], 1); 2126 ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB, 2127 src[0]); 2128 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len); 2129 iter->unmap_len = len; 2130 2131 /* 2132 * Second descriptor, multiply data from the q page 2133 * and store the result in real destination. 2134 */ 2135 iter = list_first_entry(&iter->chain_node, 2136 struct ppc440spe_adma_desc_slot, 2137 chain_node); 2138 memset(iter->hw_desc, 0, sizeof(struct dma_cdb)); 2139 iter->hw_next = NULL; 2140 if (flags & DMA_PREP_INTERRUPT) 2141 set_bit(PPC440SPE_DESC_INT, &iter->flags); 2142 else 2143 clear_bit(PPC440SPE_DESC_INT, &iter->flags); 2144 2145 hw_desc = iter->hw_desc; 2146 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; 2147 ppc440spe_desc_set_src_addr(iter, chan, 0, 2148 DMA_CUED_XOR_HB, dst[1]); 2149 ppc440spe_desc_set_dest_addr(iter, chan, 2150 DMA_CUED_XOR_BASE, dst[0], 0); 2151 2152 ppc440spe_desc_set_src_mult(iter, chan, DMA_CUED_MULT1_OFF, 2153 DMA_CDB_SG_DST1, scf[0]); 2154 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len); 2155 iter->unmap_len = len; 2156 sw_desc->async_tx.flags = flags; 2157 } 2158 2159 spin_unlock_bh(&ppc440spe_chan->lock); 2160 2161 return sw_desc; 2162 } 2163 2164 /** 2165 * ppc440spe_dma01_prep_sum_product - 2166 * Dx = A*(P+Pxy) + B*(Q+Qxy) operation where destination is also 2167 * the source. 2168 */ 2169 static struct ppc440spe_adma_desc_slot *ppc440spe_dma01_prep_sum_product( 2170 struct ppc440spe_adma_chan *ppc440spe_chan, 2171 dma_addr_t *dst, dma_addr_t *src, int src_cnt, 2172 const unsigned char *scf, size_t len, unsigned long flags) 2173 { 2174 struct ppc440spe_adma_desc_slot *sw_desc = NULL; 2175 unsigned long op = 0; 2176 int slot_cnt; 2177 2178 set_bit(PPC440SPE_DESC_WXOR, &op); 2179 slot_cnt = 3; 2180 2181 spin_lock_bh(&ppc440spe_chan->lock); 2182 2183 /* WXOR, each descriptor occupies one slot */ 2184 sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1); 2185 if (sw_desc) { 2186 struct ppc440spe_adma_chan *chan; 2187 struct ppc440spe_adma_desc_slot *iter; 2188 struct dma_cdb *hw_desc; 2189 2190 chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan); 2191 set_bits(op, &sw_desc->flags); 2192 sw_desc->src_cnt = src_cnt; 2193 sw_desc->dst_cnt = 1; 2194 /* 1st descriptor, src[1] data to q page and zero destination */ 2195 iter = list_first_entry(&sw_desc->group_list, 2196 struct ppc440spe_adma_desc_slot, 2197 chain_node); 2198 memset(iter->hw_desc, 0, sizeof(struct dma_cdb)); 2199 iter->hw_next = list_entry(iter->chain_node.next, 2200 struct ppc440spe_adma_desc_slot, 2201 chain_node); 2202 clear_bit(PPC440SPE_DESC_INT, &iter->flags); 2203 hw_desc = iter->hw_desc; 2204 hw_desc->opc = DMA_CDB_OPC_MULTICAST; 2205 2206 ppc440spe_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE, 2207 *dst, 0); 2208 ppc440spe_desc_set_dest_addr(iter, chan, 0, 2209 ppc440spe_chan->qdest, 1); 2210 ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB, 2211 src[1]); 2212 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len); 2213 iter->unmap_len = len; 2214 2215 /* 2nd descriptor, multiply src[1] data and store the 2216 * result in destination */ 2217 iter = list_first_entry(&iter->chain_node, 2218 struct ppc440spe_adma_desc_slot, 2219 chain_node); 2220 memset(iter->hw_desc, 0, sizeof(struct dma_cdb)); 2221 /* set 'next' pointer */ 2222 iter->hw_next = list_entry(iter->chain_node.next, 2223 struct ppc440spe_adma_desc_slot, 2224 chain_node); 2225 if (flags & DMA_PREP_INTERRUPT) 2226 set_bit(PPC440SPE_DESC_INT, &iter->flags); 2227 else 2228 clear_bit(PPC440SPE_DESC_INT, &iter->flags); 2229 2230 hw_desc = iter->hw_desc; 2231 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; 2232 ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB, 2233 ppc440spe_chan->qdest); 2234 ppc440spe_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE, 2235 *dst, 0); 2236 ppc440spe_desc_set_src_mult(iter, chan, DMA_CUED_MULT1_OFF, 2237 DMA_CDB_SG_DST1, scf[1]); 2238 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len); 2239 iter->unmap_len = len; 2240 2241 /* 2242 * 3rd descriptor, multiply src[0] data and xor it 2243 * with destination 2244 */ 2245 iter = list_first_entry(&iter->chain_node, 2246 struct ppc440spe_adma_desc_slot, 2247 chain_node); 2248 memset(iter->hw_desc, 0, sizeof(struct dma_cdb)); 2249 iter->hw_next = NULL; 2250 if (flags & DMA_PREP_INTERRUPT) 2251 set_bit(PPC440SPE_DESC_INT, &iter->flags); 2252 else 2253 clear_bit(PPC440SPE_DESC_INT, &iter->flags); 2254 2255 hw_desc = iter->hw_desc; 2256 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; 2257 ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB, 2258 src[0]); 2259 ppc440spe_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE, 2260 *dst, 0); 2261 ppc440spe_desc_set_src_mult(iter, chan, DMA_CUED_MULT1_OFF, 2262 DMA_CDB_SG_DST1, scf[0]); 2263 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len); 2264 iter->unmap_len = len; 2265 sw_desc->async_tx.flags = flags; 2266 } 2267 2268 spin_unlock_bh(&ppc440spe_chan->lock); 2269 2270 return sw_desc; 2271 } 2272 2273 static struct ppc440spe_adma_desc_slot *ppc440spe_dma01_prep_pq( 2274 struct ppc440spe_adma_chan *ppc440spe_chan, 2275 dma_addr_t *dst, int dst_cnt, dma_addr_t *src, int src_cnt, 2276 const unsigned char *scf, size_t len, unsigned long flags) 2277 { 2278 int slot_cnt; 2279 struct ppc440spe_adma_desc_slot *sw_desc = NULL, *iter; 2280 unsigned long op = 0; 2281 unsigned char mult = 1; 2282 2283 pr_debug("%s: dst_cnt %d, src_cnt %d, len %d\n", 2284 __func__, dst_cnt, src_cnt, len); 2285 /* select operations WXOR/RXOR depending on the 2286 * source addresses of operators and the number 2287 * of destinations (RXOR support only Q-parity calculations) 2288 */ 2289 set_bit(PPC440SPE_DESC_WXOR, &op); 2290 if (!test_and_set_bit(PPC440SPE_RXOR_RUN, &ppc440spe_rxor_state)) { 2291 /* no active RXOR; 2292 * do RXOR if: 2293 * - there are more than 1 source, 2294 * - len is aligned on 512-byte boundary, 2295 * - source addresses fit to one of 4 possible regions. 2296 */ 2297 if (src_cnt > 1 && 2298 !(len & MQ0_CF2H_RXOR_BS_MASK) && 2299 (src[0] + len) == src[1]) { 2300 /* may do RXOR R1 R2 */ 2301 set_bit(PPC440SPE_DESC_RXOR, &op); 2302 if (src_cnt != 2) { 2303 /* may try to enhance region of RXOR */ 2304 if ((src[1] + len) == src[2]) { 2305 /* do RXOR R1 R2 R3 */ 2306 set_bit(PPC440SPE_DESC_RXOR123, 2307 &op); 2308 } else if ((src[1] + len * 2) == src[2]) { 2309 /* do RXOR R1 R2 R4 */ 2310 set_bit(PPC440SPE_DESC_RXOR124, &op); 2311 } else if ((src[1] + len * 3) == src[2]) { 2312 /* do RXOR R1 R2 R5 */ 2313 set_bit(PPC440SPE_DESC_RXOR125, 2314 &op); 2315 } else { 2316 /* do RXOR R1 R2 */ 2317 set_bit(PPC440SPE_DESC_RXOR12, 2318 &op); 2319 } 2320 } else { 2321 /* do RXOR R1 R2 */ 2322 set_bit(PPC440SPE_DESC_RXOR12, &op); 2323 } 2324 } 2325 2326 if (!test_bit(PPC440SPE_DESC_RXOR, &op)) { 2327 /* can not do this operation with RXOR */ 2328 clear_bit(PPC440SPE_RXOR_RUN, 2329 &ppc440spe_rxor_state); 2330 } else { 2331 /* can do; set block size right now */ 2332 ppc440spe_desc_set_rxor_block_size(len); 2333 } 2334 } 2335 2336 /* Number of necessary slots depends on operation type selected */ 2337 if (!test_bit(PPC440SPE_DESC_RXOR, &op)) { 2338 /* This is a WXOR only chain. Need descriptors for each 2339 * source to GF-XOR them with WXOR, and need descriptors 2340 * for each destination to zero them with WXOR 2341 */ 2342 slot_cnt = src_cnt; 2343 2344 if (flags & DMA_PREP_ZERO_P) { 2345 slot_cnt++; 2346 set_bit(PPC440SPE_ZERO_P, &op); 2347 } 2348 if (flags & DMA_PREP_ZERO_Q) { 2349 slot_cnt++; 2350 set_bit(PPC440SPE_ZERO_Q, &op); 2351 } 2352 } else { 2353 /* Need 1/2 descriptor for RXOR operation, and 2354 * need (src_cnt - (2 or 3)) for WXOR of sources 2355 * remained (if any) 2356 */ 2357 slot_cnt = dst_cnt; 2358 2359 if (flags & DMA_PREP_ZERO_P) 2360 set_bit(PPC440SPE_ZERO_P, &op); 2361 if (flags & DMA_PREP_ZERO_Q) 2362 set_bit(PPC440SPE_ZERO_Q, &op); 2363 2364 if (test_bit(PPC440SPE_DESC_RXOR12, &op)) 2365 slot_cnt += src_cnt - 2; 2366 else 2367 slot_cnt += src_cnt - 3; 2368 2369 /* Thus we have either RXOR only chain or 2370 * mixed RXOR/WXOR 2371 */ 2372 if (slot_cnt == dst_cnt) 2373 /* RXOR only chain */ 2374 clear_bit(PPC440SPE_DESC_WXOR, &op); 2375 } 2376 2377 spin_lock_bh(&ppc440spe_chan->lock); 2378 /* for both RXOR/WXOR each descriptor occupies one slot */ 2379 sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1); 2380 if (sw_desc) { 2381 ppc440spe_desc_init_dma01pq(sw_desc, dst_cnt, src_cnt, 2382 flags, op); 2383 2384 /* setup dst/src/mult */ 2385 pr_debug("%s: set dst descriptor 0, 1: 0x%016llx, 0x%016llx\n", 2386 __func__, dst[0], dst[1]); 2387 ppc440spe_adma_pq_set_dest(sw_desc, dst, flags); 2388 while (src_cnt--) { 2389 ppc440spe_adma_pq_set_src(sw_desc, src[src_cnt], 2390 src_cnt); 2391 2392 /* NOTE: "Multi = 0 is equivalent to = 1" as it 2393 * stated in 440SPSPe_RAID6_Addendum_UM_1_17.pdf 2394 * doesn't work for RXOR with DMA0/1! Instead, multi=0 2395 * leads to zeroing source data after RXOR. 2396 * So, for P case set-up mult=1 explicitly. 2397 */ 2398 if (!(flags & DMA_PREP_PQ_DISABLE_Q)) 2399 mult = scf[src_cnt]; 2400 ppc440spe_adma_pq_set_src_mult(sw_desc, 2401 mult, src_cnt, dst_cnt - 1); 2402 } 2403 2404 /* Setup byte count foreach slot just allocated */ 2405 sw_desc->async_tx.flags = flags; 2406 list_for_each_entry(iter, &sw_desc->group_list, 2407 chain_node) { 2408 ppc440spe_desc_set_byte_count(iter, 2409 ppc440spe_chan, len); 2410 iter->unmap_len = len; 2411 } 2412 } 2413 spin_unlock_bh(&ppc440spe_chan->lock); 2414 2415 return sw_desc; 2416 } 2417 2418 static struct ppc440spe_adma_desc_slot *ppc440spe_dma2_prep_pq( 2419 struct ppc440spe_adma_chan *ppc440spe_chan, 2420 dma_addr_t *dst, int dst_cnt, dma_addr_t *src, int src_cnt, 2421 const unsigned char *scf, size_t len, unsigned long flags) 2422 { 2423 int slot_cnt, descs_per_op; 2424 struct ppc440spe_adma_desc_slot *sw_desc = NULL, *iter; 2425 unsigned long op = 0; 2426 unsigned char mult = 1; 2427 2428 BUG_ON(!dst_cnt); 2429 /*pr_debug("%s: dst_cnt %d, src_cnt %d, len %d\n", 2430 __func__, dst_cnt, src_cnt, len);*/ 2431 2432 spin_lock_bh(&ppc440spe_chan->lock); 2433 descs_per_op = ppc440spe_dma2_pq_slot_count(src, src_cnt, len); 2434 if (descs_per_op < 0) { 2435 spin_unlock_bh(&ppc440spe_chan->lock); 2436 return NULL; 2437 } 2438 2439 /* depending on number of sources we have 1 or 2 RXOR chains */ 2440 slot_cnt = descs_per_op * dst_cnt; 2441 2442 sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1); 2443 if (sw_desc) { 2444 op = slot_cnt; 2445 sw_desc->async_tx.flags = flags; 2446 list_for_each_entry(iter, &sw_desc->group_list, chain_node) { 2447 ppc440spe_desc_init_dma2pq(iter, dst_cnt, src_cnt, 2448 --op ? 0 : flags); 2449 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, 2450 len); 2451 iter->unmap_len = len; 2452 2453 ppc440spe_init_rxor_cursor(&(iter->rxor_cursor)); 2454 iter->rxor_cursor.len = len; 2455 iter->descs_per_op = descs_per_op; 2456 } 2457 op = 0; 2458 list_for_each_entry(iter, &sw_desc->group_list, chain_node) { 2459 op++; 2460 if (op % descs_per_op == 0) 2461 ppc440spe_adma_init_dma2rxor_slot(iter, src, 2462 src_cnt); 2463 if (likely(!list_is_last(&iter->chain_node, 2464 &sw_desc->group_list))) { 2465 /* set 'next' pointer */ 2466 iter->hw_next = 2467 list_entry(iter->chain_node.next, 2468 struct ppc440spe_adma_desc_slot, 2469 chain_node); 2470 ppc440spe_xor_set_link(iter, iter->hw_next); 2471 } else { 2472 /* this is the last descriptor. */ 2473 iter->hw_next = NULL; 2474 } 2475 } 2476 2477 /* fixup head descriptor */ 2478 sw_desc->dst_cnt = dst_cnt; 2479 if (flags & DMA_PREP_ZERO_P) 2480 set_bit(PPC440SPE_ZERO_P, &sw_desc->flags); 2481 if (flags & DMA_PREP_ZERO_Q) 2482 set_bit(PPC440SPE_ZERO_Q, &sw_desc->flags); 2483 2484 /* setup dst/src/mult */ 2485 ppc440spe_adma_pq_set_dest(sw_desc, dst, flags); 2486 2487 while (src_cnt--) { 2488 /* handle descriptors (if dst_cnt == 2) inside 2489 * the ppc440spe_adma_pq_set_srcxxx() functions 2490 */ 2491 ppc440spe_adma_pq_set_src(sw_desc, src[src_cnt], 2492 src_cnt); 2493 if (!(flags & DMA_PREP_PQ_DISABLE_Q)) 2494 mult = scf[src_cnt]; 2495 ppc440spe_adma_pq_set_src_mult(sw_desc, 2496 mult, src_cnt, dst_cnt - 1); 2497 } 2498 } 2499 spin_unlock_bh(&ppc440spe_chan->lock); 2500 ppc440spe_desc_set_rxor_block_size(len); 2501 return sw_desc; 2502 } 2503 2504 /** 2505 * ppc440spe_adma_prep_dma_pq - prepare CDB (group) for a GF-XOR operation 2506 */ 2507 static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_pq( 2508 struct dma_chan *chan, dma_addr_t *dst, dma_addr_t *src, 2509 unsigned int src_cnt, const unsigned char *scf, 2510 size_t len, unsigned long flags) 2511 { 2512 struct ppc440spe_adma_chan *ppc440spe_chan; 2513 struct ppc440spe_adma_desc_slot *sw_desc = NULL; 2514 int dst_cnt = 0; 2515 2516 ppc440spe_chan = to_ppc440spe_adma_chan(chan); 2517 2518 ADMA_LL_DBG(prep_dma_pq_dbg(ppc440spe_chan->device->id, 2519 dst, src, src_cnt)); 2520 BUG_ON(!len); 2521 BUG_ON(len > PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT); 2522 BUG_ON(!src_cnt); 2523 2524 if (src_cnt == 1 && dst[1] == src[0]) { 2525 dma_addr_t dest[2]; 2526 2527 /* dst[1] is real destination (Q) */ 2528 dest[0] = dst[1]; 2529 /* this is the page to multicast source data to */ 2530 dest[1] = ppc440spe_chan->qdest; 2531 sw_desc = ppc440spe_dma01_prep_mult(ppc440spe_chan, 2532 dest, 2, src, src_cnt, scf, len, flags); 2533 return sw_desc ? &sw_desc->async_tx : NULL; 2534 } 2535 2536 if (src_cnt == 2 && dst[1] == src[1]) { 2537 sw_desc = ppc440spe_dma01_prep_sum_product(ppc440spe_chan, 2538 &dst[1], src, 2, scf, len, flags); 2539 return sw_desc ? &sw_desc->async_tx : NULL; 2540 } 2541 2542 if (!(flags & DMA_PREP_PQ_DISABLE_P)) { 2543 BUG_ON(!dst[0]); 2544 dst_cnt++; 2545 flags |= DMA_PREP_ZERO_P; 2546 } 2547 2548 if (!(flags & DMA_PREP_PQ_DISABLE_Q)) { 2549 BUG_ON(!dst[1]); 2550 dst_cnt++; 2551 flags |= DMA_PREP_ZERO_Q; 2552 } 2553 2554 BUG_ON(!dst_cnt); 2555 2556 dev_dbg(ppc440spe_chan->device->common.dev, 2557 "ppc440spe adma%d: %s src_cnt: %d len: %u int_en: %d\n", 2558 ppc440spe_chan->device->id, __func__, src_cnt, len, 2559 flags & DMA_PREP_INTERRUPT ? 1 : 0); 2560 2561 switch (ppc440spe_chan->device->id) { 2562 case PPC440SPE_DMA0_ID: 2563 case PPC440SPE_DMA1_ID: 2564 sw_desc = ppc440spe_dma01_prep_pq(ppc440spe_chan, 2565 dst, dst_cnt, src, src_cnt, scf, 2566 len, flags); 2567 break; 2568 2569 case PPC440SPE_XOR_ID: 2570 sw_desc = ppc440spe_dma2_prep_pq(ppc440spe_chan, 2571 dst, dst_cnt, src, src_cnt, scf, 2572 len, flags); 2573 break; 2574 } 2575 2576 return sw_desc ? &sw_desc->async_tx : NULL; 2577 } 2578 2579 /** 2580 * ppc440spe_adma_prep_dma_pqzero_sum - prepare CDB group for 2581 * a PQ_ZERO_SUM operation 2582 */ 2583 static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_pqzero_sum( 2584 struct dma_chan *chan, dma_addr_t *pq, dma_addr_t *src, 2585 unsigned int src_cnt, const unsigned char *scf, size_t len, 2586 enum sum_check_flags *pqres, unsigned long flags) 2587 { 2588 struct ppc440spe_adma_chan *ppc440spe_chan; 2589 struct ppc440spe_adma_desc_slot *sw_desc, *iter; 2590 dma_addr_t pdest, qdest; 2591 int slot_cnt, slots_per_op, idst, dst_cnt; 2592 2593 ppc440spe_chan = to_ppc440spe_adma_chan(chan); 2594 2595 if (flags & DMA_PREP_PQ_DISABLE_P) 2596 pdest = 0; 2597 else 2598 pdest = pq[0]; 2599 2600 if (flags & DMA_PREP_PQ_DISABLE_Q) 2601 qdest = 0; 2602 else 2603 qdest = pq[1]; 2604 2605 ADMA_LL_DBG(prep_dma_pqzero_sum_dbg(ppc440spe_chan->device->id, 2606 src, src_cnt, scf)); 2607 2608 /* Always use WXOR for P/Q calculations (two destinations). 2609 * Need 1 or 2 extra slots to verify results are zero. 2610 */ 2611 idst = dst_cnt = (pdest && qdest) ? 2 : 1; 2612 2613 /* One additional slot per destination to clone P/Q 2614 * before calculation (we have to preserve destinations). 2615 */ 2616 slot_cnt = src_cnt + dst_cnt * 2; 2617 slots_per_op = 1; 2618 2619 spin_lock_bh(&ppc440spe_chan->lock); 2620 sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 2621 slots_per_op); 2622 if (sw_desc) { 2623 ppc440spe_desc_init_dma01pqzero_sum(sw_desc, dst_cnt, src_cnt); 2624 2625 /* Setup byte count for each slot just allocated */ 2626 sw_desc->async_tx.flags = flags; 2627 list_for_each_entry(iter, &sw_desc->group_list, chain_node) { 2628 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, 2629 len); 2630 iter->unmap_len = len; 2631 } 2632 2633 if (pdest) { 2634 struct dma_cdb *hw_desc; 2635 struct ppc440spe_adma_chan *chan; 2636 2637 iter = sw_desc->group_head; 2638 chan = to_ppc440spe_adma_chan(iter->async_tx.chan); 2639 memset(iter->hw_desc, 0, sizeof(struct dma_cdb)); 2640 iter->hw_next = list_entry(iter->chain_node.next, 2641 struct ppc440spe_adma_desc_slot, 2642 chain_node); 2643 hw_desc = iter->hw_desc; 2644 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; 2645 iter->src_cnt = 0; 2646 iter->dst_cnt = 0; 2647 ppc440spe_desc_set_dest_addr(iter, chan, 0, 2648 ppc440spe_chan->pdest, 0); 2649 ppc440spe_desc_set_src_addr(iter, chan, 0, 0, pdest); 2650 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, 2651 len); 2652 iter->unmap_len = 0; 2653 /* override pdest to preserve original P */ 2654 pdest = ppc440spe_chan->pdest; 2655 } 2656 if (qdest) { 2657 struct dma_cdb *hw_desc; 2658 struct ppc440spe_adma_chan *chan; 2659 2660 iter = list_first_entry(&sw_desc->group_list, 2661 struct ppc440spe_adma_desc_slot, 2662 chain_node); 2663 chan = to_ppc440spe_adma_chan(iter->async_tx.chan); 2664 2665 if (pdest) { 2666 iter = list_entry(iter->chain_node.next, 2667 struct ppc440spe_adma_desc_slot, 2668 chain_node); 2669 } 2670 2671 memset(iter->hw_desc, 0, sizeof(struct dma_cdb)); 2672 iter->hw_next = list_entry(iter->chain_node.next, 2673 struct ppc440spe_adma_desc_slot, 2674 chain_node); 2675 hw_desc = iter->hw_desc; 2676 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; 2677 iter->src_cnt = 0; 2678 iter->dst_cnt = 0; 2679 ppc440spe_desc_set_dest_addr(iter, chan, 0, 2680 ppc440spe_chan->qdest, 0); 2681 ppc440spe_desc_set_src_addr(iter, chan, 0, 0, qdest); 2682 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, 2683 len); 2684 iter->unmap_len = 0; 2685 /* override qdest to preserve original Q */ 2686 qdest = ppc440spe_chan->qdest; 2687 } 2688 2689 /* Setup destinations for P/Q ops */ 2690 ppc440spe_adma_pqzero_sum_set_dest(sw_desc, pdest, qdest); 2691 2692 /* Setup zero QWORDs into DCHECK CDBs */ 2693 idst = dst_cnt; 2694 list_for_each_entry_reverse(iter, &sw_desc->group_list, 2695 chain_node) { 2696 /* 2697 * The last CDB corresponds to Q-parity check, 2698 * the one before last CDB corresponds 2699 * P-parity check 2700 */ 2701 if (idst == DMA_DEST_MAX_NUM) { 2702 if (idst == dst_cnt) { 2703 set_bit(PPC440SPE_DESC_QCHECK, 2704 &iter->flags); 2705 } else { 2706 set_bit(PPC440SPE_DESC_PCHECK, 2707 &iter->flags); 2708 } 2709 } else { 2710 if (qdest) { 2711 set_bit(PPC440SPE_DESC_QCHECK, 2712 &iter->flags); 2713 } else { 2714 set_bit(PPC440SPE_DESC_PCHECK, 2715 &iter->flags); 2716 } 2717 } 2718 iter->xor_check_result = pqres; 2719 2720 /* 2721 * set it to zero, if check fail then result will 2722 * be updated 2723 */ 2724 *iter->xor_check_result = 0; 2725 ppc440spe_desc_set_dcheck(iter, ppc440spe_chan, 2726 ppc440spe_qword); 2727 2728 if (!(--dst_cnt)) 2729 break; 2730 } 2731 2732 /* Setup sources and mults for P/Q ops */ 2733 list_for_each_entry_continue_reverse(iter, &sw_desc->group_list, 2734 chain_node) { 2735 struct ppc440spe_adma_chan *chan; 2736 u32 mult_dst; 2737 2738 chan = to_ppc440spe_adma_chan(iter->async_tx.chan); 2739 ppc440spe_desc_set_src_addr(iter, chan, 0, 2740 DMA_CUED_XOR_HB, 2741 src[src_cnt - 1]); 2742 if (qdest) { 2743 mult_dst = (dst_cnt - 1) ? DMA_CDB_SG_DST2 : 2744 DMA_CDB_SG_DST1; 2745 ppc440spe_desc_set_src_mult(iter, chan, 2746 DMA_CUED_MULT1_OFF, 2747 mult_dst, 2748 scf[src_cnt - 1]); 2749 } 2750 if (!(--src_cnt)) 2751 break; 2752 } 2753 } 2754 spin_unlock_bh(&ppc440spe_chan->lock); 2755 return sw_desc ? &sw_desc->async_tx : NULL; 2756 } 2757 2758 /** 2759 * ppc440spe_adma_prep_dma_xor_zero_sum - prepare CDB group for 2760 * XOR ZERO_SUM operation 2761 */ 2762 static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_xor_zero_sum( 2763 struct dma_chan *chan, dma_addr_t *src, unsigned int src_cnt, 2764 size_t len, enum sum_check_flags *result, unsigned long flags) 2765 { 2766 struct dma_async_tx_descriptor *tx; 2767 dma_addr_t pq[2]; 2768 2769 /* validate P, disable Q */ 2770 pq[0] = src[0]; 2771 pq[1] = 0; 2772 flags |= DMA_PREP_PQ_DISABLE_Q; 2773 2774 tx = ppc440spe_adma_prep_dma_pqzero_sum(chan, pq, &src[1], 2775 src_cnt - 1, 0, len, 2776 result, flags); 2777 return tx; 2778 } 2779 2780 /** 2781 * ppc440spe_adma_set_dest - set destination address into descriptor 2782 */ 2783 static void ppc440spe_adma_set_dest(struct ppc440spe_adma_desc_slot *sw_desc, 2784 dma_addr_t addr, int index) 2785 { 2786 struct ppc440spe_adma_chan *chan; 2787 2788 BUG_ON(index >= sw_desc->dst_cnt); 2789 2790 chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan); 2791 2792 switch (chan->device->id) { 2793 case PPC440SPE_DMA0_ID: 2794 case PPC440SPE_DMA1_ID: 2795 /* to do: support transfers lengths > 2796 * PPC440SPE_ADMA_DMA/XOR_MAX_BYTE_COUNT 2797 */ 2798 ppc440spe_desc_set_dest_addr(sw_desc->group_head, 2799 chan, 0, addr, index); 2800 break; 2801 case PPC440SPE_XOR_ID: 2802 sw_desc = ppc440spe_get_group_entry(sw_desc, index); 2803 ppc440spe_desc_set_dest_addr(sw_desc, 2804 chan, 0, addr, index); 2805 break; 2806 } 2807 } 2808 2809 static void ppc440spe_adma_pq_zero_op(struct ppc440spe_adma_desc_slot *iter, 2810 struct ppc440spe_adma_chan *chan, dma_addr_t addr) 2811 { 2812 /* To clear destinations update the descriptor 2813 * (P or Q depending on index) as follows: 2814 * addr is destination (0 corresponds to SG2): 2815 */ 2816 ppc440spe_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE, addr, 0); 2817 2818 /* ... and the addr is source: */ 2819 ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB, addr); 2820 2821 /* addr is always SG2 then the mult is always DST1 */ 2822 ppc440spe_desc_set_src_mult(iter, chan, DMA_CUED_MULT1_OFF, 2823 DMA_CDB_SG_DST1, 1); 2824 } 2825 2826 /** 2827 * ppc440spe_adma_pq_set_dest - set destination address into descriptor 2828 * for the PQXOR operation 2829 */ 2830 static void ppc440spe_adma_pq_set_dest(struct ppc440spe_adma_desc_slot *sw_desc, 2831 dma_addr_t *addrs, unsigned long flags) 2832 { 2833 struct ppc440spe_adma_desc_slot *iter; 2834 struct ppc440spe_adma_chan *chan; 2835 dma_addr_t paddr, qaddr; 2836 dma_addr_t addr = 0, ppath, qpath; 2837 int index = 0, i; 2838 2839 chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan); 2840 2841 if (flags & DMA_PREP_PQ_DISABLE_P) 2842 paddr = 0; 2843 else 2844 paddr = addrs[0]; 2845 2846 if (flags & DMA_PREP_PQ_DISABLE_Q) 2847 qaddr = 0; 2848 else 2849 qaddr = addrs[1]; 2850 2851 if (!paddr || !qaddr) 2852 addr = paddr ? paddr : qaddr; 2853 2854 switch (chan->device->id) { 2855 case PPC440SPE_DMA0_ID: 2856 case PPC440SPE_DMA1_ID: 2857 /* walk through the WXOR source list and set P/Q-destinations 2858 * for each slot: 2859 */ 2860 if (!test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags)) { 2861 /* This is WXOR-only chain; may have 1/2 zero descs */ 2862 if (test_bit(PPC440SPE_ZERO_P, &sw_desc->flags)) 2863 index++; 2864 if (test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags)) 2865 index++; 2866 2867 iter = ppc440spe_get_group_entry(sw_desc, index); 2868 if (addr) { 2869 /* one destination */ 2870 list_for_each_entry_from(iter, 2871 &sw_desc->group_list, chain_node) 2872 ppc440spe_desc_set_dest_addr(iter, chan, 2873 DMA_CUED_XOR_BASE, addr, 0); 2874 } else { 2875 /* two destinations */ 2876 list_for_each_entry_from(iter, 2877 &sw_desc->group_list, chain_node) { 2878 ppc440spe_desc_set_dest_addr(iter, chan, 2879 DMA_CUED_XOR_BASE, paddr, 0); 2880 ppc440spe_desc_set_dest_addr(iter, chan, 2881 DMA_CUED_XOR_BASE, qaddr, 1); 2882 } 2883 } 2884 2885 if (index) { 2886 /* To clear destinations update the descriptor 2887 * (1st,2nd, or both depending on flags) 2888 */ 2889 index = 0; 2890 if (test_bit(PPC440SPE_ZERO_P, 2891 &sw_desc->flags)) { 2892 iter = ppc440spe_get_group_entry( 2893 sw_desc, index++); 2894 ppc440spe_adma_pq_zero_op(iter, chan, 2895 paddr); 2896 } 2897 2898 if (test_bit(PPC440SPE_ZERO_Q, 2899 &sw_desc->flags)) { 2900 iter = ppc440spe_get_group_entry( 2901 sw_desc, index++); 2902 ppc440spe_adma_pq_zero_op(iter, chan, 2903 qaddr); 2904 } 2905 2906 return; 2907 } 2908 } else { 2909 /* This is RXOR-only or RXOR/WXOR mixed chain */ 2910 2911 /* If we want to include destination into calculations, 2912 * then make dest addresses cued with mult=1 (XOR). 2913 */ 2914 ppath = test_bit(PPC440SPE_ZERO_P, &sw_desc->flags) ? 2915 DMA_CUED_XOR_HB : 2916 DMA_CUED_XOR_BASE | 2917 (1 << DMA_CUED_MULT1_OFF); 2918 qpath = test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags) ? 2919 DMA_CUED_XOR_HB : 2920 DMA_CUED_XOR_BASE | 2921 (1 << DMA_CUED_MULT1_OFF); 2922 2923 /* Setup destination(s) in RXOR slot(s) */ 2924 iter = ppc440spe_get_group_entry(sw_desc, index++); 2925 ppc440spe_desc_set_dest_addr(iter, chan, 2926 paddr ? ppath : qpath, 2927 paddr ? paddr : qaddr, 0); 2928 if (!addr) { 2929 /* two destinations */ 2930 iter = ppc440spe_get_group_entry(sw_desc, 2931 index++); 2932 ppc440spe_desc_set_dest_addr(iter, chan, 2933 qpath, qaddr, 0); 2934 } 2935 2936 if (test_bit(PPC440SPE_DESC_WXOR, &sw_desc->flags)) { 2937 /* Setup destination(s) in remaining WXOR 2938 * slots 2939 */ 2940 iter = ppc440spe_get_group_entry(sw_desc, 2941 index); 2942 if (addr) { 2943 /* one destination */ 2944 list_for_each_entry_from(iter, 2945 &sw_desc->group_list, 2946 chain_node) 2947 ppc440spe_desc_set_dest_addr( 2948 iter, chan, 2949 DMA_CUED_XOR_BASE, 2950 addr, 0); 2951 2952 } else { 2953 /* two destinations */ 2954 list_for_each_entry_from(iter, 2955 &sw_desc->group_list, 2956 chain_node) { 2957 ppc440spe_desc_set_dest_addr( 2958 iter, chan, 2959 DMA_CUED_XOR_BASE, 2960 paddr, 0); 2961 ppc440spe_desc_set_dest_addr( 2962 iter, chan, 2963 DMA_CUED_XOR_BASE, 2964 qaddr, 1); 2965 } 2966 } 2967 } 2968 2969 } 2970 break; 2971 2972 case PPC440SPE_XOR_ID: 2973 /* DMA2 descriptors have only 1 destination, so there are 2974 * two chains - one for each dest. 2975 * If we want to include destination into calculations, 2976 * then make dest addresses cued with mult=1 (XOR). 2977 */ 2978 ppath = test_bit(PPC440SPE_ZERO_P, &sw_desc->flags) ? 2979 DMA_CUED_XOR_HB : 2980 DMA_CUED_XOR_BASE | 2981 (1 << DMA_CUED_MULT1_OFF); 2982 2983 qpath = test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags) ? 2984 DMA_CUED_XOR_HB : 2985 DMA_CUED_XOR_BASE | 2986 (1 << DMA_CUED_MULT1_OFF); 2987 2988 iter = ppc440spe_get_group_entry(sw_desc, 0); 2989 for (i = 0; i < sw_desc->descs_per_op; i++) { 2990 ppc440spe_desc_set_dest_addr(iter, chan, 2991 paddr ? ppath : qpath, 2992 paddr ? paddr : qaddr, 0); 2993 iter = list_entry(iter->chain_node.next, 2994 struct ppc440spe_adma_desc_slot, 2995 chain_node); 2996 } 2997 2998 if (!addr) { 2999 /* Two destinations; setup Q here */ 3000 iter = ppc440spe_get_group_entry(sw_desc, 3001 sw_desc->descs_per_op); 3002 for (i = 0; i < sw_desc->descs_per_op; i++) { 3003 ppc440spe_desc_set_dest_addr(iter, 3004 chan, qpath, qaddr, 0); 3005 iter = list_entry(iter->chain_node.next, 3006 struct ppc440spe_adma_desc_slot, 3007 chain_node); 3008 } 3009 } 3010 3011 break; 3012 } 3013 } 3014 3015 /** 3016 * ppc440spe_adma_pq_zero_sum_set_dest - set destination address into descriptor 3017 * for the PQ_ZERO_SUM operation 3018 */ 3019 static void ppc440spe_adma_pqzero_sum_set_dest( 3020 struct ppc440spe_adma_desc_slot *sw_desc, 3021 dma_addr_t paddr, dma_addr_t qaddr) 3022 { 3023 struct ppc440spe_adma_desc_slot *iter, *end; 3024 struct ppc440spe_adma_chan *chan; 3025 dma_addr_t addr = 0; 3026 int idx; 3027 3028 chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan); 3029 3030 /* walk through the WXOR source list and set P/Q-destinations 3031 * for each slot 3032 */ 3033 idx = (paddr && qaddr) ? 2 : 1; 3034 /* set end */ 3035 list_for_each_entry_reverse(end, &sw_desc->group_list, 3036 chain_node) { 3037 if (!(--idx)) 3038 break; 3039 } 3040 /* set start */ 3041 idx = (paddr && qaddr) ? 2 : 1; 3042 iter = ppc440spe_get_group_entry(sw_desc, idx); 3043 3044 if (paddr && qaddr) { 3045 /* two destinations */ 3046 list_for_each_entry_from(iter, &sw_desc->group_list, 3047 chain_node) { 3048 if (unlikely(iter == end)) 3049 break; 3050 ppc440spe_desc_set_dest_addr(iter, chan, 3051 DMA_CUED_XOR_BASE, paddr, 0); 3052 ppc440spe_desc_set_dest_addr(iter, chan, 3053 DMA_CUED_XOR_BASE, qaddr, 1); 3054 } 3055 } else { 3056 /* one destination */ 3057 addr = paddr ? paddr : qaddr; 3058 list_for_each_entry_from(iter, &sw_desc->group_list, 3059 chain_node) { 3060 if (unlikely(iter == end)) 3061 break; 3062 ppc440spe_desc_set_dest_addr(iter, chan, 3063 DMA_CUED_XOR_BASE, addr, 0); 3064 } 3065 } 3066 3067 /* The remaining descriptors are DATACHECK. These have no need in 3068 * destination. Actually, these destinations are used there 3069 * as sources for check operation. So, set addr as source. 3070 */ 3071 ppc440spe_desc_set_src_addr(end, chan, 0, 0, addr ? addr : paddr); 3072 3073 if (!addr) { 3074 end = list_entry(end->chain_node.next, 3075 struct ppc440spe_adma_desc_slot, chain_node); 3076 ppc440spe_desc_set_src_addr(end, chan, 0, 0, qaddr); 3077 } 3078 } 3079 3080 /** 3081 * ppc440spe_desc_set_xor_src_cnt - set source count into descriptor 3082 */ 3083 static inline void ppc440spe_desc_set_xor_src_cnt( 3084 struct ppc440spe_adma_desc_slot *desc, 3085 int src_cnt) 3086 { 3087 struct xor_cb *hw_desc = desc->hw_desc; 3088 3089 hw_desc->cbc &= ~XOR_CDCR_OAC_MSK; 3090 hw_desc->cbc |= src_cnt; 3091 } 3092 3093 /** 3094 * ppc440spe_adma_pq_set_src - set source address into descriptor 3095 */ 3096 static void ppc440spe_adma_pq_set_src(struct ppc440spe_adma_desc_slot *sw_desc, 3097 dma_addr_t addr, int index) 3098 { 3099 struct ppc440spe_adma_chan *chan; 3100 dma_addr_t haddr = 0; 3101 struct ppc440spe_adma_desc_slot *iter = NULL; 3102 3103 chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan); 3104 3105 switch (chan->device->id) { 3106 case PPC440SPE_DMA0_ID: 3107 case PPC440SPE_DMA1_ID: 3108 /* DMA0,1 may do: WXOR, RXOR, RXOR+WXORs chain 3109 */ 3110 if (test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags)) { 3111 /* RXOR-only or RXOR/WXOR operation */ 3112 int iskip = test_bit(PPC440SPE_DESC_RXOR12, 3113 &sw_desc->flags) ? 2 : 3; 3114 3115 if (index == 0) { 3116 /* 1st slot (RXOR) */ 3117 /* setup sources region (R1-2-3, R1-2-4, 3118 * or R1-2-5) 3119 */ 3120 if (test_bit(PPC440SPE_DESC_RXOR12, 3121 &sw_desc->flags)) 3122 haddr = DMA_RXOR12 << 3123 DMA_CUED_REGION_OFF; 3124 else if (test_bit(PPC440SPE_DESC_RXOR123, 3125 &sw_desc->flags)) 3126 haddr = DMA_RXOR123 << 3127 DMA_CUED_REGION_OFF; 3128 else if (test_bit(PPC440SPE_DESC_RXOR124, 3129 &sw_desc->flags)) 3130 haddr = DMA_RXOR124 << 3131 DMA_CUED_REGION_OFF; 3132 else if (test_bit(PPC440SPE_DESC_RXOR125, 3133 &sw_desc->flags)) 3134 haddr = DMA_RXOR125 << 3135 DMA_CUED_REGION_OFF; 3136 else 3137 BUG(); 3138 haddr |= DMA_CUED_XOR_BASE; 3139 iter = ppc440spe_get_group_entry(sw_desc, 0); 3140 } else if (index < iskip) { 3141 /* 1st slot (RXOR) 3142 * shall actually set source address only once 3143 * instead of first <iskip> 3144 */ 3145 iter = NULL; 3146 } else { 3147 /* 2nd/3d and next slots (WXOR); 3148 * skip first slot with RXOR 3149 */ 3150 haddr = DMA_CUED_XOR_HB; 3151 iter = ppc440spe_get_group_entry(sw_desc, 3152 index - iskip + sw_desc->dst_cnt); 3153 } 3154 } else { 3155 int znum = 0; 3156 3157 /* WXOR-only operation; skip first slots with 3158 * zeroing destinations 3159 */ 3160 if (test_bit(PPC440SPE_ZERO_P, &sw_desc->flags)) 3161 znum++; 3162 if (test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags)) 3163 znum++; 3164 3165 haddr = DMA_CUED_XOR_HB; 3166 iter = ppc440spe_get_group_entry(sw_desc, 3167 index + znum); 3168 } 3169 3170 if (likely(iter)) { 3171 ppc440spe_desc_set_src_addr(iter, chan, 0, haddr, addr); 3172 3173 if (!index && 3174 test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags) && 3175 sw_desc->dst_cnt == 2) { 3176 /* if we have two destinations for RXOR, then 3177 * setup source in the second descr too 3178 */ 3179 iter = ppc440spe_get_group_entry(sw_desc, 1); 3180 ppc440spe_desc_set_src_addr(iter, chan, 0, 3181 haddr, addr); 3182 } 3183 } 3184 break; 3185 3186 case PPC440SPE_XOR_ID: 3187 /* DMA2 may do Biskup */ 3188 iter = sw_desc->group_head; 3189 if (iter->dst_cnt == 2) { 3190 /* both P & Q calculations required; set P src here */ 3191 ppc440spe_adma_dma2rxor_set_src(iter, index, addr); 3192 3193 /* this is for Q */ 3194 iter = ppc440spe_get_group_entry(sw_desc, 3195 sw_desc->descs_per_op); 3196 } 3197 ppc440spe_adma_dma2rxor_set_src(iter, index, addr); 3198 break; 3199 } 3200 } 3201 3202 /** 3203 * ppc440spe_adma_memcpy_xor_set_src - set source address into descriptor 3204 */ 3205 static void ppc440spe_adma_memcpy_xor_set_src( 3206 struct ppc440spe_adma_desc_slot *sw_desc, 3207 dma_addr_t addr, int index) 3208 { 3209 struct ppc440spe_adma_chan *chan; 3210 3211 chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan); 3212 sw_desc = sw_desc->group_head; 3213 3214 if (likely(sw_desc)) 3215 ppc440spe_desc_set_src_addr(sw_desc, chan, index, 0, addr); 3216 } 3217 3218 /** 3219 * ppc440spe_adma_dma2rxor_inc_addr - 3220 */ 3221 static void ppc440spe_adma_dma2rxor_inc_addr( 3222 struct ppc440spe_adma_desc_slot *desc, 3223 struct ppc440spe_rxor *cursor, int index, int src_cnt) 3224 { 3225 cursor->addr_count++; 3226 if (index == src_cnt - 1) { 3227 ppc440spe_desc_set_xor_src_cnt(desc, cursor->addr_count); 3228 } else if (cursor->addr_count == XOR_MAX_OPS) { 3229 ppc440spe_desc_set_xor_src_cnt(desc, cursor->addr_count); 3230 cursor->addr_count = 0; 3231 cursor->desc_count++; 3232 } 3233 } 3234 3235 /** 3236 * ppc440spe_adma_dma2rxor_prep_src - setup RXOR types in DMA2 CDB 3237 */ 3238 static int ppc440spe_adma_dma2rxor_prep_src( 3239 struct ppc440spe_adma_desc_slot *hdesc, 3240 struct ppc440spe_rxor *cursor, int index, 3241 int src_cnt, u32 addr) 3242 { 3243 u32 sign; 3244 struct ppc440spe_adma_desc_slot *desc = hdesc; 3245 int i; 3246 3247 for (i = 0; i < cursor->desc_count; i++) { 3248 desc = list_entry(hdesc->chain_node.next, 3249 struct ppc440spe_adma_desc_slot, 3250 chain_node); 3251 } 3252 3253 switch (cursor->state) { 3254 case 0: 3255 if (addr == cursor->addrl + cursor->len) { 3256 /* direct RXOR */ 3257 cursor->state = 1; 3258 cursor->xor_count++; 3259 if (index == src_cnt-1) { 3260 ppc440spe_rxor_set_region(desc, 3261 cursor->addr_count, 3262 DMA_RXOR12 << DMA_CUED_REGION_OFF); 3263 ppc440spe_adma_dma2rxor_inc_addr( 3264 desc, cursor, index, src_cnt); 3265 } 3266 } else if (cursor->addrl == addr + cursor->len) { 3267 /* reverse RXOR */ 3268 cursor->state = 1; 3269 cursor->xor_count++; 3270 set_bit(cursor->addr_count, &desc->reverse_flags[0]); 3271 if (index == src_cnt-1) { 3272 ppc440spe_rxor_set_region(desc, 3273 cursor->addr_count, 3274 DMA_RXOR12 << DMA_CUED_REGION_OFF); 3275 ppc440spe_adma_dma2rxor_inc_addr( 3276 desc, cursor, index, src_cnt); 3277 } 3278 } else { 3279 printk(KERN_ERR "Cannot build " 3280 "DMA2 RXOR command block.\n"); 3281 BUG(); 3282 } 3283 break; 3284 case 1: 3285 sign = test_bit(cursor->addr_count, 3286 desc->reverse_flags) 3287 ? -1 : 1; 3288 if (index == src_cnt-2 || (sign == -1 3289 && addr != cursor->addrl - 2*cursor->len)) { 3290 cursor->state = 0; 3291 cursor->xor_count = 1; 3292 cursor->addrl = addr; 3293 ppc440spe_rxor_set_region(desc, 3294 cursor->addr_count, 3295 DMA_RXOR12 << DMA_CUED_REGION_OFF); 3296 ppc440spe_adma_dma2rxor_inc_addr( 3297 desc, cursor, index, src_cnt); 3298 } else if (addr == cursor->addrl + 2*sign*cursor->len) { 3299 cursor->state = 2; 3300 cursor->xor_count = 0; 3301 ppc440spe_rxor_set_region(desc, 3302 cursor->addr_count, 3303 DMA_RXOR123 << DMA_CUED_REGION_OFF); 3304 if (index == src_cnt-1) { 3305 ppc440spe_adma_dma2rxor_inc_addr( 3306 desc, cursor, index, src_cnt); 3307 } 3308 } else if (addr == cursor->addrl + 3*cursor->len) { 3309 cursor->state = 2; 3310 cursor->xor_count = 0; 3311 ppc440spe_rxor_set_region(desc, 3312 cursor->addr_count, 3313 DMA_RXOR124 << DMA_CUED_REGION_OFF); 3314 if (index == src_cnt-1) { 3315 ppc440spe_adma_dma2rxor_inc_addr( 3316 desc, cursor, index, src_cnt); 3317 } 3318 } else if (addr == cursor->addrl + 4*cursor->len) { 3319 cursor->state = 2; 3320 cursor->xor_count = 0; 3321 ppc440spe_rxor_set_region(desc, 3322 cursor->addr_count, 3323 DMA_RXOR125 << DMA_CUED_REGION_OFF); 3324 if (index == src_cnt-1) { 3325 ppc440spe_adma_dma2rxor_inc_addr( 3326 desc, cursor, index, src_cnt); 3327 } 3328 } else { 3329 cursor->state = 0; 3330 cursor->xor_count = 1; 3331 cursor->addrl = addr; 3332 ppc440spe_rxor_set_region(desc, 3333 cursor->addr_count, 3334 DMA_RXOR12 << DMA_CUED_REGION_OFF); 3335 ppc440spe_adma_dma2rxor_inc_addr( 3336 desc, cursor, index, src_cnt); 3337 } 3338 break; 3339 case 2: 3340 cursor->state = 0; 3341 cursor->addrl = addr; 3342 cursor->xor_count++; 3343 if (index) { 3344 ppc440spe_adma_dma2rxor_inc_addr( 3345 desc, cursor, index, src_cnt); 3346 } 3347 break; 3348 } 3349 3350 return 0; 3351 } 3352 3353 /** 3354 * ppc440spe_adma_dma2rxor_set_src - set RXOR source address; it's assumed that 3355 * ppc440spe_adma_dma2rxor_prep_src() has already done prior this call 3356 */ 3357 static void ppc440spe_adma_dma2rxor_set_src( 3358 struct ppc440spe_adma_desc_slot *desc, 3359 int index, dma_addr_t addr) 3360 { 3361 struct xor_cb *xcb = desc->hw_desc; 3362 int k = 0, op = 0, lop = 0; 3363 3364 /* get the RXOR operand which corresponds to index addr */ 3365 while (op <= index) { 3366 lop = op; 3367 if (k == XOR_MAX_OPS) { 3368 k = 0; 3369 desc = list_entry(desc->chain_node.next, 3370 struct ppc440spe_adma_desc_slot, chain_node); 3371 xcb = desc->hw_desc; 3372 3373 } 3374 if ((xcb->ops[k++].h & (DMA_RXOR12 << DMA_CUED_REGION_OFF)) == 3375 (DMA_RXOR12 << DMA_CUED_REGION_OFF)) 3376 op += 2; 3377 else 3378 op += 3; 3379 } 3380 3381 BUG_ON(k < 1); 3382 3383 if (test_bit(k-1, desc->reverse_flags)) { 3384 /* reverse operand order; put last op in RXOR group */ 3385 if (index == op - 1) 3386 ppc440spe_rxor_set_src(desc, k - 1, addr); 3387 } else { 3388 /* direct operand order; put first op in RXOR group */ 3389 if (index == lop) 3390 ppc440spe_rxor_set_src(desc, k - 1, addr); 3391 } 3392 } 3393 3394 /** 3395 * ppc440spe_adma_dma2rxor_set_mult - set RXOR multipliers; it's assumed that 3396 * ppc440spe_adma_dma2rxor_prep_src() has already done prior this call 3397 */ 3398 static void ppc440spe_adma_dma2rxor_set_mult( 3399 struct ppc440spe_adma_desc_slot *desc, 3400 int index, u8 mult) 3401 { 3402 struct xor_cb *xcb = desc->hw_desc; 3403 int k = 0, op = 0, lop = 0; 3404 3405 /* get the RXOR operand which corresponds to index mult */ 3406 while (op <= index) { 3407 lop = op; 3408 if (k == XOR_MAX_OPS) { 3409 k = 0; 3410 desc = list_entry(desc->chain_node.next, 3411 struct ppc440spe_adma_desc_slot, 3412 chain_node); 3413 xcb = desc->hw_desc; 3414 3415 } 3416 if ((xcb->ops[k++].h & (DMA_RXOR12 << DMA_CUED_REGION_OFF)) == 3417 (DMA_RXOR12 << DMA_CUED_REGION_OFF)) 3418 op += 2; 3419 else 3420 op += 3; 3421 } 3422 3423 BUG_ON(k < 1); 3424 if (test_bit(k-1, desc->reverse_flags)) { 3425 /* reverse order */ 3426 ppc440spe_rxor_set_mult(desc, k - 1, op - index - 1, mult); 3427 } else { 3428 /* direct order */ 3429 ppc440spe_rxor_set_mult(desc, k - 1, index - lop, mult); 3430 } 3431 } 3432 3433 /** 3434 * ppc440spe_init_rxor_cursor - 3435 */ 3436 static void ppc440spe_init_rxor_cursor(struct ppc440spe_rxor *cursor) 3437 { 3438 memset(cursor, 0, sizeof(struct ppc440spe_rxor)); 3439 cursor->state = 2; 3440 } 3441 3442 /** 3443 * ppc440spe_adma_pq_set_src_mult - set multiplication coefficient into 3444 * descriptor for the PQXOR operation 3445 */ 3446 static void ppc440spe_adma_pq_set_src_mult( 3447 struct ppc440spe_adma_desc_slot *sw_desc, 3448 unsigned char mult, int index, int dst_pos) 3449 { 3450 struct ppc440spe_adma_chan *chan; 3451 u32 mult_idx, mult_dst; 3452 struct ppc440spe_adma_desc_slot *iter = NULL, *iter1 = NULL; 3453 3454 chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan); 3455 3456 switch (chan->device->id) { 3457 case PPC440SPE_DMA0_ID: 3458 case PPC440SPE_DMA1_ID: 3459 if (test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags)) { 3460 int region = test_bit(PPC440SPE_DESC_RXOR12, 3461 &sw_desc->flags) ? 2 : 3; 3462 3463 if (index < region) { 3464 /* RXOR multipliers */ 3465 iter = ppc440spe_get_group_entry(sw_desc, 3466 sw_desc->dst_cnt - 1); 3467 if (sw_desc->dst_cnt == 2) 3468 iter1 = ppc440spe_get_group_entry( 3469 sw_desc, 0); 3470 3471 mult_idx = DMA_CUED_MULT1_OFF + (index << 3); 3472 mult_dst = DMA_CDB_SG_SRC; 3473 } else { 3474 /* WXOR multiplier */ 3475 iter = ppc440spe_get_group_entry(sw_desc, 3476 index - region + 3477 sw_desc->dst_cnt); 3478 mult_idx = DMA_CUED_MULT1_OFF; 3479 mult_dst = dst_pos ? DMA_CDB_SG_DST2 : 3480 DMA_CDB_SG_DST1; 3481 } 3482 } else { 3483 int znum = 0; 3484 3485 /* WXOR-only; 3486 * skip first slots with destinations (if ZERO_DST has 3487 * place) 3488 */ 3489 if (test_bit(PPC440SPE_ZERO_P, &sw_desc->flags)) 3490 znum++; 3491 if (test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags)) 3492 znum++; 3493 3494 iter = ppc440spe_get_group_entry(sw_desc, index + znum); 3495 mult_idx = DMA_CUED_MULT1_OFF; 3496 mult_dst = dst_pos ? DMA_CDB_SG_DST2 : DMA_CDB_SG_DST1; 3497 } 3498 3499 if (likely(iter)) { 3500 ppc440spe_desc_set_src_mult(iter, chan, 3501 mult_idx, mult_dst, mult); 3502 3503 if (unlikely(iter1)) { 3504 /* if we have two destinations for RXOR, then 3505 * we've just set Q mult. Set-up P now. 3506 */ 3507 ppc440spe_desc_set_src_mult(iter1, chan, 3508 mult_idx, mult_dst, 1); 3509 } 3510 3511 } 3512 break; 3513 3514 case PPC440SPE_XOR_ID: 3515 iter = sw_desc->group_head; 3516 if (sw_desc->dst_cnt == 2) { 3517 /* both P & Q calculations required; set P mult here */ 3518 ppc440spe_adma_dma2rxor_set_mult(iter, index, 1); 3519 3520 /* and then set Q mult */ 3521 iter = ppc440spe_get_group_entry(sw_desc, 3522 sw_desc->descs_per_op); 3523 } 3524 ppc440spe_adma_dma2rxor_set_mult(iter, index, mult); 3525 break; 3526 } 3527 } 3528 3529 /** 3530 * ppc440spe_adma_free_chan_resources - free the resources allocated 3531 */ 3532 static void ppc440spe_adma_free_chan_resources(struct dma_chan *chan) 3533 { 3534 struct ppc440spe_adma_chan *ppc440spe_chan; 3535 struct ppc440spe_adma_desc_slot *iter, *_iter; 3536 int in_use_descs = 0; 3537 3538 ppc440spe_chan = to_ppc440spe_adma_chan(chan); 3539 ppc440spe_adma_slot_cleanup(ppc440spe_chan); 3540 3541 spin_lock_bh(&ppc440spe_chan->lock); 3542 list_for_each_entry_safe(iter, _iter, &ppc440spe_chan->chain, 3543 chain_node) { 3544 in_use_descs++; 3545 list_del(&iter->chain_node); 3546 } 3547 list_for_each_entry_safe_reverse(iter, _iter, 3548 &ppc440spe_chan->all_slots, slot_node) { 3549 list_del(&iter->slot_node); 3550 kfree(iter); 3551 ppc440spe_chan->slots_allocated--; 3552 } 3553 ppc440spe_chan->last_used = NULL; 3554 3555 dev_dbg(ppc440spe_chan->device->common.dev, 3556 "ppc440spe adma%d %s slots_allocated %d\n", 3557 ppc440spe_chan->device->id, 3558 __func__, ppc440spe_chan->slots_allocated); 3559 spin_unlock_bh(&ppc440spe_chan->lock); 3560 3561 /* one is ok since we left it on there on purpose */ 3562 if (in_use_descs > 1) 3563 printk(KERN_ERR "SPE: Freeing %d in use descriptors!\n", 3564 in_use_descs - 1); 3565 } 3566 3567 /** 3568 * ppc440spe_adma_tx_status - poll the status of an ADMA transaction 3569 * @chan: ADMA channel handle 3570 * @cookie: ADMA transaction identifier 3571 * @txstate: a holder for the current state of the channel 3572 */ 3573 static enum dma_status ppc440spe_adma_tx_status(struct dma_chan *chan, 3574 dma_cookie_t cookie, struct dma_tx_state *txstate) 3575 { 3576 struct ppc440spe_adma_chan *ppc440spe_chan; 3577 enum dma_status ret; 3578 3579 ppc440spe_chan = to_ppc440spe_adma_chan(chan); 3580 ret = dma_cookie_status(chan, cookie, txstate); 3581 if (ret == DMA_COMPLETE) 3582 return ret; 3583 3584 ppc440spe_adma_slot_cleanup(ppc440spe_chan); 3585 3586 return dma_cookie_status(chan, cookie, txstate); 3587 } 3588 3589 /** 3590 * ppc440spe_adma_eot_handler - end of transfer interrupt handler 3591 */ 3592 static irqreturn_t ppc440spe_adma_eot_handler(int irq, void *data) 3593 { 3594 struct ppc440spe_adma_chan *chan = data; 3595 3596 dev_dbg(chan->device->common.dev, 3597 "ppc440spe adma%d: %s\n", chan->device->id, __func__); 3598 3599 tasklet_schedule(&chan->irq_tasklet); 3600 ppc440spe_adma_device_clear_eot_status(chan); 3601 3602 return IRQ_HANDLED; 3603 } 3604 3605 /** 3606 * ppc440spe_adma_err_handler - DMA error interrupt handler; 3607 * do the same things as a eot handler 3608 */ 3609 static irqreturn_t ppc440spe_adma_err_handler(int irq, void *data) 3610 { 3611 struct ppc440spe_adma_chan *chan = data; 3612 3613 dev_dbg(chan->device->common.dev, 3614 "ppc440spe adma%d: %s\n", chan->device->id, __func__); 3615 3616 tasklet_schedule(&chan->irq_tasklet); 3617 ppc440spe_adma_device_clear_eot_status(chan); 3618 3619 return IRQ_HANDLED; 3620 } 3621 3622 /** 3623 * ppc440spe_test_callback - called when test operation has been done 3624 */ 3625 static void ppc440spe_test_callback(void *unused) 3626 { 3627 complete(&ppc440spe_r6_test_comp); 3628 } 3629 3630 /** 3631 * ppc440spe_adma_issue_pending - flush all pending descriptors to h/w 3632 */ 3633 static void ppc440spe_adma_issue_pending(struct dma_chan *chan) 3634 { 3635 struct ppc440spe_adma_chan *ppc440spe_chan; 3636 3637 ppc440spe_chan = to_ppc440spe_adma_chan(chan); 3638 dev_dbg(ppc440spe_chan->device->common.dev, 3639 "ppc440spe adma%d: %s %d \n", ppc440spe_chan->device->id, 3640 __func__, ppc440spe_chan->pending); 3641 3642 if (ppc440spe_chan->pending) { 3643 ppc440spe_chan->pending = 0; 3644 ppc440spe_chan_append(ppc440spe_chan); 3645 } 3646 } 3647 3648 /** 3649 * ppc440spe_chan_start_null_xor - initiate the first XOR operation (DMA engines 3650 * use FIFOs (as opposite to chains used in XOR) so this is a XOR 3651 * specific operation) 3652 */ 3653 static void ppc440spe_chan_start_null_xor(struct ppc440spe_adma_chan *chan) 3654 { 3655 struct ppc440spe_adma_desc_slot *sw_desc, *group_start; 3656 dma_cookie_t cookie; 3657 int slot_cnt, slots_per_op; 3658 3659 dev_dbg(chan->device->common.dev, 3660 "ppc440spe adma%d: %s\n", chan->device->id, __func__); 3661 3662 spin_lock_bh(&chan->lock); 3663 slot_cnt = ppc440spe_chan_xor_slot_count(0, 2, &slots_per_op); 3664 sw_desc = ppc440spe_adma_alloc_slots(chan, slot_cnt, slots_per_op); 3665 if (sw_desc) { 3666 group_start = sw_desc->group_head; 3667 list_splice_init(&sw_desc->group_list, &chan->chain); 3668 async_tx_ack(&sw_desc->async_tx); 3669 ppc440spe_desc_init_null_xor(group_start); 3670 3671 cookie = dma_cookie_assign(&sw_desc->async_tx); 3672 3673 /* initialize the completed cookie to be less than 3674 * the most recently used cookie 3675 */ 3676 chan->common.completed_cookie = cookie - 1; 3677 3678 /* channel should not be busy */ 3679 BUG_ON(ppc440spe_chan_is_busy(chan)); 3680 3681 /* set the descriptor address */ 3682 ppc440spe_chan_set_first_xor_descriptor(chan, sw_desc); 3683 3684 /* run the descriptor */ 3685 ppc440spe_chan_run(chan); 3686 } else 3687 printk(KERN_ERR "ppc440spe adma%d" 3688 " failed to allocate null descriptor\n", 3689 chan->device->id); 3690 spin_unlock_bh(&chan->lock); 3691 } 3692 3693 /** 3694 * ppc440spe_test_raid6 - test are RAID-6 capabilities enabled successfully. 3695 * For this we just perform one WXOR operation with the same source 3696 * and destination addresses, the GF-multiplier is 1; so if RAID-6 3697 * capabilities are enabled then we'll get src/dst filled with zero. 3698 */ 3699 static int ppc440spe_test_raid6(struct ppc440spe_adma_chan *chan) 3700 { 3701 struct ppc440spe_adma_desc_slot *sw_desc, *iter; 3702 struct page *pg; 3703 char *a; 3704 dma_addr_t dma_addr, addrs[2]; 3705 unsigned long op = 0; 3706 int rval = 0; 3707 3708 set_bit(PPC440SPE_DESC_WXOR, &op); 3709 3710 pg = alloc_page(GFP_KERNEL); 3711 if (!pg) 3712 return -ENOMEM; 3713 3714 spin_lock_bh(&chan->lock); 3715 sw_desc = ppc440spe_adma_alloc_slots(chan, 1, 1); 3716 if (sw_desc) { 3717 /* 1 src, 1 dsr, int_ena, WXOR */ 3718 ppc440spe_desc_init_dma01pq(sw_desc, 1, 1, 1, op); 3719 list_for_each_entry(iter, &sw_desc->group_list, chain_node) { 3720 ppc440spe_desc_set_byte_count(iter, chan, PAGE_SIZE); 3721 iter->unmap_len = PAGE_SIZE; 3722 } 3723 } else { 3724 rval = -EFAULT; 3725 spin_unlock_bh(&chan->lock); 3726 goto exit; 3727 } 3728 spin_unlock_bh(&chan->lock); 3729 3730 /* Fill the test page with ones */ 3731 memset(page_address(pg), 0xFF, PAGE_SIZE); 3732 dma_addr = dma_map_page(chan->device->dev, pg, 0, 3733 PAGE_SIZE, DMA_BIDIRECTIONAL); 3734 3735 /* Setup addresses */ 3736 ppc440spe_adma_pq_set_src(sw_desc, dma_addr, 0); 3737 ppc440spe_adma_pq_set_src_mult(sw_desc, 1, 0, 0); 3738 addrs[0] = dma_addr; 3739 addrs[1] = 0; 3740 ppc440spe_adma_pq_set_dest(sw_desc, addrs, DMA_PREP_PQ_DISABLE_Q); 3741 3742 async_tx_ack(&sw_desc->async_tx); 3743 sw_desc->async_tx.callback = ppc440spe_test_callback; 3744 sw_desc->async_tx.callback_param = NULL; 3745 3746 init_completion(&ppc440spe_r6_test_comp); 3747 3748 ppc440spe_adma_tx_submit(&sw_desc->async_tx); 3749 ppc440spe_adma_issue_pending(&chan->common); 3750 3751 wait_for_completion(&ppc440spe_r6_test_comp); 3752 3753 /* Now check if the test page is zeroed */ 3754 a = page_address(pg); 3755 if ((*(u32 *)a) == 0 && memcmp(a, a+4, PAGE_SIZE-4) == 0) { 3756 /* page is zero - RAID-6 enabled */ 3757 rval = 0; 3758 } else { 3759 /* RAID-6 was not enabled */ 3760 rval = -EINVAL; 3761 } 3762 exit: 3763 __free_page(pg); 3764 return rval; 3765 } 3766 3767 static void ppc440spe_adma_init_capabilities(struct ppc440spe_adma_device *adev) 3768 { 3769 switch (adev->id) { 3770 case PPC440SPE_DMA0_ID: 3771 case PPC440SPE_DMA1_ID: 3772 dma_cap_set(DMA_MEMCPY, adev->common.cap_mask); 3773 dma_cap_set(DMA_INTERRUPT, adev->common.cap_mask); 3774 dma_cap_set(DMA_PQ, adev->common.cap_mask); 3775 dma_cap_set(DMA_PQ_VAL, adev->common.cap_mask); 3776 dma_cap_set(DMA_XOR_VAL, adev->common.cap_mask); 3777 break; 3778 case PPC440SPE_XOR_ID: 3779 dma_cap_set(DMA_XOR, adev->common.cap_mask); 3780 dma_cap_set(DMA_PQ, adev->common.cap_mask); 3781 dma_cap_set(DMA_INTERRUPT, adev->common.cap_mask); 3782 adev->common.cap_mask = adev->common.cap_mask; 3783 break; 3784 } 3785 3786 /* Set base routines */ 3787 adev->common.device_alloc_chan_resources = 3788 ppc440spe_adma_alloc_chan_resources; 3789 adev->common.device_free_chan_resources = 3790 ppc440spe_adma_free_chan_resources; 3791 adev->common.device_tx_status = ppc440spe_adma_tx_status; 3792 adev->common.device_issue_pending = ppc440spe_adma_issue_pending; 3793 3794 /* Set prep routines based on capability */ 3795 if (dma_has_cap(DMA_MEMCPY, adev->common.cap_mask)) { 3796 adev->common.device_prep_dma_memcpy = 3797 ppc440spe_adma_prep_dma_memcpy; 3798 } 3799 if (dma_has_cap(DMA_XOR, adev->common.cap_mask)) { 3800 adev->common.max_xor = XOR_MAX_OPS; 3801 adev->common.device_prep_dma_xor = 3802 ppc440spe_adma_prep_dma_xor; 3803 } 3804 if (dma_has_cap(DMA_PQ, adev->common.cap_mask)) { 3805 switch (adev->id) { 3806 case PPC440SPE_DMA0_ID: 3807 dma_set_maxpq(&adev->common, 3808 DMA0_FIFO_SIZE / sizeof(struct dma_cdb), 0); 3809 break; 3810 case PPC440SPE_DMA1_ID: 3811 dma_set_maxpq(&adev->common, 3812 DMA1_FIFO_SIZE / sizeof(struct dma_cdb), 0); 3813 break; 3814 case PPC440SPE_XOR_ID: 3815 adev->common.max_pq = XOR_MAX_OPS * 3; 3816 break; 3817 } 3818 adev->common.device_prep_dma_pq = 3819 ppc440spe_adma_prep_dma_pq; 3820 } 3821 if (dma_has_cap(DMA_PQ_VAL, adev->common.cap_mask)) { 3822 switch (adev->id) { 3823 case PPC440SPE_DMA0_ID: 3824 adev->common.max_pq = DMA0_FIFO_SIZE / 3825 sizeof(struct dma_cdb); 3826 break; 3827 case PPC440SPE_DMA1_ID: 3828 adev->common.max_pq = DMA1_FIFO_SIZE / 3829 sizeof(struct dma_cdb); 3830 break; 3831 } 3832 adev->common.device_prep_dma_pq_val = 3833 ppc440spe_adma_prep_dma_pqzero_sum; 3834 } 3835 if (dma_has_cap(DMA_XOR_VAL, adev->common.cap_mask)) { 3836 switch (adev->id) { 3837 case PPC440SPE_DMA0_ID: 3838 adev->common.max_xor = DMA0_FIFO_SIZE / 3839 sizeof(struct dma_cdb); 3840 break; 3841 case PPC440SPE_DMA1_ID: 3842 adev->common.max_xor = DMA1_FIFO_SIZE / 3843 sizeof(struct dma_cdb); 3844 break; 3845 } 3846 adev->common.device_prep_dma_xor_val = 3847 ppc440spe_adma_prep_dma_xor_zero_sum; 3848 } 3849 if (dma_has_cap(DMA_INTERRUPT, adev->common.cap_mask)) { 3850 adev->common.device_prep_dma_interrupt = 3851 ppc440spe_adma_prep_dma_interrupt; 3852 } 3853 pr_info("%s: AMCC(R) PPC440SP(E) ADMA Engine: " 3854 "( %s%s%s%s%s%s)\n", 3855 dev_name(adev->dev), 3856 dma_has_cap(DMA_PQ, adev->common.cap_mask) ? "pq " : "", 3857 dma_has_cap(DMA_PQ_VAL, adev->common.cap_mask) ? "pq_val " : "", 3858 dma_has_cap(DMA_XOR, adev->common.cap_mask) ? "xor " : "", 3859 dma_has_cap(DMA_XOR_VAL, adev->common.cap_mask) ? "xor_val " : "", 3860 dma_has_cap(DMA_MEMCPY, adev->common.cap_mask) ? "memcpy " : "", 3861 dma_has_cap(DMA_INTERRUPT, adev->common.cap_mask) ? "intr " : ""); 3862 } 3863 3864 static int ppc440spe_adma_setup_irqs(struct ppc440spe_adma_device *adev, 3865 struct ppc440spe_adma_chan *chan, 3866 int *initcode) 3867 { 3868 struct platform_device *ofdev; 3869 struct device_node *np; 3870 int ret; 3871 3872 ofdev = container_of(adev->dev, struct platform_device, dev); 3873 np = ofdev->dev.of_node; 3874 if (adev->id != PPC440SPE_XOR_ID) { 3875 adev->err_irq = irq_of_parse_and_map(np, 1); 3876 if (!adev->err_irq) { 3877 dev_warn(adev->dev, "no err irq resource?\n"); 3878 *initcode = PPC_ADMA_INIT_IRQ2; 3879 adev->err_irq = -ENXIO; 3880 } else 3881 atomic_inc(&ppc440spe_adma_err_irq_ref); 3882 } else { 3883 adev->err_irq = -ENXIO; 3884 } 3885 3886 adev->irq = irq_of_parse_and_map(np, 0); 3887 if (!adev->irq) { 3888 dev_err(adev->dev, "no irq resource\n"); 3889 *initcode = PPC_ADMA_INIT_IRQ1; 3890 ret = -ENXIO; 3891 goto err_irq_map; 3892 } 3893 dev_dbg(adev->dev, "irq %d, err irq %d\n", 3894 adev->irq, adev->err_irq); 3895 3896 ret = request_irq(adev->irq, ppc440spe_adma_eot_handler, 3897 0, dev_driver_string(adev->dev), chan); 3898 if (ret) { 3899 dev_err(adev->dev, "can't request irq %d\n", 3900 adev->irq); 3901 *initcode = PPC_ADMA_INIT_IRQ1; 3902 ret = -EIO; 3903 goto err_req1; 3904 } 3905 3906 /* only DMA engines have a separate error IRQ 3907 * so it's Ok if err_irq < 0 in XOR engine case. 3908 */ 3909 if (adev->err_irq > 0) { 3910 /* both DMA engines share common error IRQ */ 3911 ret = request_irq(adev->err_irq, 3912 ppc440spe_adma_err_handler, 3913 IRQF_SHARED, 3914 dev_driver_string(adev->dev), 3915 chan); 3916 if (ret) { 3917 dev_err(adev->dev, "can't request irq %d\n", 3918 adev->err_irq); 3919 *initcode = PPC_ADMA_INIT_IRQ2; 3920 ret = -EIO; 3921 goto err_req2; 3922 } 3923 } 3924 3925 if (adev->id == PPC440SPE_XOR_ID) { 3926 /* enable XOR engine interrupts */ 3927 iowrite32be(XOR_IE_CBCIE_BIT | XOR_IE_ICBIE_BIT | 3928 XOR_IE_ICIE_BIT | XOR_IE_RPTIE_BIT, 3929 &adev->xor_reg->ier); 3930 } else { 3931 u32 mask, enable; 3932 3933 np = of_find_compatible_node(NULL, NULL, "ibm,i2o-440spe"); 3934 if (!np) { 3935 pr_err("%s: can't find I2O device tree node\n", 3936 __func__); 3937 ret = -ENODEV; 3938 goto err_req2; 3939 } 3940 adev->i2o_reg = of_iomap(np, 0); 3941 if (!adev->i2o_reg) { 3942 pr_err("%s: failed to map I2O registers\n", __func__); 3943 of_node_put(np); 3944 ret = -EINVAL; 3945 goto err_req2; 3946 } 3947 of_node_put(np); 3948 /* Unmask 'CS FIFO Attention' interrupts and 3949 * enable generating interrupts on errors 3950 */ 3951 enable = (adev->id == PPC440SPE_DMA0_ID) ? 3952 ~(I2O_IOPIM_P0SNE | I2O_IOPIM_P0EM) : 3953 ~(I2O_IOPIM_P1SNE | I2O_IOPIM_P1EM); 3954 mask = ioread32(&adev->i2o_reg->iopim) & enable; 3955 iowrite32(mask, &adev->i2o_reg->iopim); 3956 } 3957 return 0; 3958 3959 err_req2: 3960 free_irq(adev->irq, chan); 3961 err_req1: 3962 irq_dispose_mapping(adev->irq); 3963 err_irq_map: 3964 if (adev->err_irq > 0) { 3965 if (atomic_dec_and_test(&ppc440spe_adma_err_irq_ref)) 3966 irq_dispose_mapping(adev->err_irq); 3967 } 3968 return ret; 3969 } 3970 3971 static void ppc440spe_adma_release_irqs(struct ppc440spe_adma_device *adev, 3972 struct ppc440spe_adma_chan *chan) 3973 { 3974 u32 mask, disable; 3975 3976 if (adev->id == PPC440SPE_XOR_ID) { 3977 /* disable XOR engine interrupts */ 3978 mask = ioread32be(&adev->xor_reg->ier); 3979 mask &= ~(XOR_IE_CBCIE_BIT | XOR_IE_ICBIE_BIT | 3980 XOR_IE_ICIE_BIT | XOR_IE_RPTIE_BIT); 3981 iowrite32be(mask, &adev->xor_reg->ier); 3982 } else { 3983 /* disable DMAx engine interrupts */ 3984 disable = (adev->id == PPC440SPE_DMA0_ID) ? 3985 (I2O_IOPIM_P0SNE | I2O_IOPIM_P0EM) : 3986 (I2O_IOPIM_P1SNE | I2O_IOPIM_P1EM); 3987 mask = ioread32(&adev->i2o_reg->iopim) | disable; 3988 iowrite32(mask, &adev->i2o_reg->iopim); 3989 } 3990 free_irq(adev->irq, chan); 3991 irq_dispose_mapping(adev->irq); 3992 if (adev->err_irq > 0) { 3993 free_irq(adev->err_irq, chan); 3994 if (atomic_dec_and_test(&ppc440spe_adma_err_irq_ref)) { 3995 irq_dispose_mapping(adev->err_irq); 3996 iounmap(adev->i2o_reg); 3997 } 3998 } 3999 } 4000 4001 /** 4002 * ppc440spe_adma_probe - probe the asynch device 4003 */ 4004 static int ppc440spe_adma_probe(struct platform_device *ofdev) 4005 { 4006 struct device_node *np = ofdev->dev.of_node; 4007 struct resource res; 4008 struct ppc440spe_adma_device *adev; 4009 struct ppc440spe_adma_chan *chan; 4010 struct ppc_dma_chan_ref *ref, *_ref; 4011 int ret = 0, initcode = PPC_ADMA_INIT_OK; 4012 const u32 *idx; 4013 int len; 4014 void *regs; 4015 u32 id, pool_size; 4016 4017 if (of_device_is_compatible(np, "amcc,xor-accelerator")) { 4018 id = PPC440SPE_XOR_ID; 4019 /* As far as the XOR engine is concerned, it does not 4020 * use FIFOs but uses linked list. So there is no dependency 4021 * between pool size to allocate and the engine configuration. 4022 */ 4023 pool_size = PAGE_SIZE << 1; 4024 } else { 4025 /* it is DMA0 or DMA1 */ 4026 idx = of_get_property(np, "cell-index", &len); 4027 if (!idx || (len != sizeof(u32))) { 4028 dev_err(&ofdev->dev, "Device node %pOF has missing " 4029 "or invalid cell-index property\n", 4030 np); 4031 return -EINVAL; 4032 } 4033 id = *idx; 4034 /* DMA0,1 engines use FIFO to maintain CDBs, so we 4035 * should allocate the pool accordingly to size of this 4036 * FIFO. Thus, the pool size depends on the FIFO depth: 4037 * how much CDBs pointers the FIFO may contain then so 4038 * much CDBs we should provide in the pool. 4039 * That is 4040 * CDB size = 32B; 4041 * CDBs number = (DMA0_FIFO_SIZE >> 3); 4042 * Pool size = CDBs number * CDB size = 4043 * = (DMA0_FIFO_SIZE >> 3) << 5 = DMA0_FIFO_SIZE << 2. 4044 */ 4045 pool_size = (id == PPC440SPE_DMA0_ID) ? 4046 DMA0_FIFO_SIZE : DMA1_FIFO_SIZE; 4047 pool_size <<= 2; 4048 } 4049 4050 if (of_address_to_resource(np, 0, &res)) { 4051 dev_err(&ofdev->dev, "failed to get memory resource\n"); 4052 initcode = PPC_ADMA_INIT_MEMRES; 4053 ret = -ENODEV; 4054 goto out; 4055 } 4056 4057 if (!request_mem_region(res.start, resource_size(&res), 4058 dev_driver_string(&ofdev->dev))) { 4059 dev_err(&ofdev->dev, "failed to request memory region %pR\n", 4060 &res); 4061 initcode = PPC_ADMA_INIT_MEMREG; 4062 ret = -EBUSY; 4063 goto out; 4064 } 4065 4066 /* create a device */ 4067 adev = kzalloc(sizeof(*adev), GFP_KERNEL); 4068 if (!adev) { 4069 initcode = PPC_ADMA_INIT_ALLOC; 4070 ret = -ENOMEM; 4071 goto err_adev_alloc; 4072 } 4073 4074 adev->id = id; 4075 adev->pool_size = pool_size; 4076 /* allocate coherent memory for hardware descriptors */ 4077 adev->dma_desc_pool_virt = dma_alloc_coherent(&ofdev->dev, 4078 adev->pool_size, &adev->dma_desc_pool, 4079 GFP_KERNEL); 4080 if (adev->dma_desc_pool_virt == NULL) { 4081 dev_err(&ofdev->dev, "failed to allocate %d bytes of coherent " 4082 "memory for hardware descriptors\n", 4083 adev->pool_size); 4084 initcode = PPC_ADMA_INIT_COHERENT; 4085 ret = -ENOMEM; 4086 goto err_dma_alloc; 4087 } 4088 dev_dbg(&ofdev->dev, "allocated descriptor pool virt 0x%p phys 0x%llx\n", 4089 adev->dma_desc_pool_virt, (u64)adev->dma_desc_pool); 4090 4091 regs = ioremap(res.start, resource_size(&res)); 4092 if (!regs) { 4093 dev_err(&ofdev->dev, "failed to ioremap regs!\n"); 4094 ret = -ENOMEM; 4095 goto err_regs_alloc; 4096 } 4097 4098 if (adev->id == PPC440SPE_XOR_ID) { 4099 adev->xor_reg = regs; 4100 /* Reset XOR */ 4101 iowrite32be(XOR_CRSR_XASR_BIT, &adev->xor_reg->crsr); 4102 iowrite32be(XOR_CRSR_64BA_BIT, &adev->xor_reg->crrr); 4103 } else { 4104 size_t fifo_size = (adev->id == PPC440SPE_DMA0_ID) ? 4105 DMA0_FIFO_SIZE : DMA1_FIFO_SIZE; 4106 adev->dma_reg = regs; 4107 /* DMAx_FIFO_SIZE is defined in bytes, 4108 * <fsiz> - is defined in number of CDB pointers (8byte). 4109 * DMA FIFO Length = CSlength + CPlength, where 4110 * CSlength = CPlength = (fsiz + 1) * 8. 4111 */ 4112 iowrite32(DMA_FIFO_ENABLE | ((fifo_size >> 3) - 2), 4113 &adev->dma_reg->fsiz); 4114 /* Configure DMA engine */ 4115 iowrite32(DMA_CFG_DXEPR_HP | DMA_CFG_DFMPP_HP | DMA_CFG_FALGN, 4116 &adev->dma_reg->cfg); 4117 /* Clear Status */ 4118 iowrite32(~0, &adev->dma_reg->dsts); 4119 } 4120 4121 adev->dev = &ofdev->dev; 4122 adev->common.dev = &ofdev->dev; 4123 INIT_LIST_HEAD(&adev->common.channels); 4124 platform_set_drvdata(ofdev, adev); 4125 4126 /* create a channel */ 4127 chan = kzalloc(sizeof(*chan), GFP_KERNEL); 4128 if (!chan) { 4129 initcode = PPC_ADMA_INIT_CHANNEL; 4130 ret = -ENOMEM; 4131 goto err_chan_alloc; 4132 } 4133 4134 spin_lock_init(&chan->lock); 4135 INIT_LIST_HEAD(&chan->chain); 4136 INIT_LIST_HEAD(&chan->all_slots); 4137 chan->device = adev; 4138 chan->common.device = &adev->common; 4139 dma_cookie_init(&chan->common); 4140 list_add_tail(&chan->common.device_node, &adev->common.channels); 4141 tasklet_setup(&chan->irq_tasklet, ppc440spe_adma_tasklet); 4142 4143 /* allocate and map helper pages for async validation or 4144 * async_mult/async_sum_product operations on DMA0/1. 4145 */ 4146 if (adev->id != PPC440SPE_XOR_ID) { 4147 chan->pdest_page = alloc_page(GFP_KERNEL); 4148 chan->qdest_page = alloc_page(GFP_KERNEL); 4149 if (!chan->pdest_page || 4150 !chan->qdest_page) { 4151 if (chan->pdest_page) 4152 __free_page(chan->pdest_page); 4153 if (chan->qdest_page) 4154 __free_page(chan->qdest_page); 4155 ret = -ENOMEM; 4156 goto err_page_alloc; 4157 } 4158 chan->pdest = dma_map_page(&ofdev->dev, chan->pdest_page, 0, 4159 PAGE_SIZE, DMA_BIDIRECTIONAL); 4160 chan->qdest = dma_map_page(&ofdev->dev, chan->qdest_page, 0, 4161 PAGE_SIZE, DMA_BIDIRECTIONAL); 4162 } 4163 4164 ref = kmalloc(sizeof(*ref), GFP_KERNEL); 4165 if (ref) { 4166 ref->chan = &chan->common; 4167 INIT_LIST_HEAD(&ref->node); 4168 list_add_tail(&ref->node, &ppc440spe_adma_chan_list); 4169 } else { 4170 dev_err(&ofdev->dev, "failed to allocate channel reference!\n"); 4171 ret = -ENOMEM; 4172 goto err_ref_alloc; 4173 } 4174 4175 ret = ppc440spe_adma_setup_irqs(adev, chan, &initcode); 4176 if (ret) 4177 goto err_irq; 4178 4179 ppc440spe_adma_init_capabilities(adev); 4180 4181 ret = dma_async_device_register(&adev->common); 4182 if (ret) { 4183 initcode = PPC_ADMA_INIT_REGISTER; 4184 dev_err(&ofdev->dev, "failed to register dma device\n"); 4185 goto err_dev_reg; 4186 } 4187 4188 goto out; 4189 4190 err_dev_reg: 4191 ppc440spe_adma_release_irqs(adev, chan); 4192 err_irq: 4193 list_for_each_entry_safe(ref, _ref, &ppc440spe_adma_chan_list, node) { 4194 if (chan == to_ppc440spe_adma_chan(ref->chan)) { 4195 list_del(&ref->node); 4196 kfree(ref); 4197 } 4198 } 4199 err_ref_alloc: 4200 if (adev->id != PPC440SPE_XOR_ID) { 4201 dma_unmap_page(&ofdev->dev, chan->pdest, 4202 PAGE_SIZE, DMA_BIDIRECTIONAL); 4203 dma_unmap_page(&ofdev->dev, chan->qdest, 4204 PAGE_SIZE, DMA_BIDIRECTIONAL); 4205 __free_page(chan->pdest_page); 4206 __free_page(chan->qdest_page); 4207 } 4208 err_page_alloc: 4209 kfree(chan); 4210 err_chan_alloc: 4211 if (adev->id == PPC440SPE_XOR_ID) 4212 iounmap(adev->xor_reg); 4213 else 4214 iounmap(adev->dma_reg); 4215 err_regs_alloc: 4216 dma_free_coherent(adev->dev, adev->pool_size, 4217 adev->dma_desc_pool_virt, 4218 adev->dma_desc_pool); 4219 err_dma_alloc: 4220 kfree(adev); 4221 err_adev_alloc: 4222 release_mem_region(res.start, resource_size(&res)); 4223 out: 4224 if (id < PPC440SPE_ADMA_ENGINES_NUM) 4225 ppc440spe_adma_devices[id] = initcode; 4226 4227 return ret; 4228 } 4229 4230 /** 4231 * ppc440spe_adma_remove - remove the asynch device 4232 */ 4233 static int ppc440spe_adma_remove(struct platform_device *ofdev) 4234 { 4235 struct ppc440spe_adma_device *adev = platform_get_drvdata(ofdev); 4236 struct device_node *np = ofdev->dev.of_node; 4237 struct resource res; 4238 struct dma_chan *chan, *_chan; 4239 struct ppc_dma_chan_ref *ref, *_ref; 4240 struct ppc440spe_adma_chan *ppc440spe_chan; 4241 4242 if (adev->id < PPC440SPE_ADMA_ENGINES_NUM) 4243 ppc440spe_adma_devices[adev->id] = -1; 4244 4245 dma_async_device_unregister(&adev->common); 4246 4247 list_for_each_entry_safe(chan, _chan, &adev->common.channels, 4248 device_node) { 4249 ppc440spe_chan = to_ppc440spe_adma_chan(chan); 4250 ppc440spe_adma_release_irqs(adev, ppc440spe_chan); 4251 tasklet_kill(&ppc440spe_chan->irq_tasklet); 4252 if (adev->id != PPC440SPE_XOR_ID) { 4253 dma_unmap_page(&ofdev->dev, ppc440spe_chan->pdest, 4254 PAGE_SIZE, DMA_BIDIRECTIONAL); 4255 dma_unmap_page(&ofdev->dev, ppc440spe_chan->qdest, 4256 PAGE_SIZE, DMA_BIDIRECTIONAL); 4257 __free_page(ppc440spe_chan->pdest_page); 4258 __free_page(ppc440spe_chan->qdest_page); 4259 } 4260 list_for_each_entry_safe(ref, _ref, &ppc440spe_adma_chan_list, 4261 node) { 4262 if (ppc440spe_chan == 4263 to_ppc440spe_adma_chan(ref->chan)) { 4264 list_del(&ref->node); 4265 kfree(ref); 4266 } 4267 } 4268 list_del(&chan->device_node); 4269 kfree(ppc440spe_chan); 4270 } 4271 4272 dma_free_coherent(adev->dev, adev->pool_size, 4273 adev->dma_desc_pool_virt, adev->dma_desc_pool); 4274 if (adev->id == PPC440SPE_XOR_ID) 4275 iounmap(adev->xor_reg); 4276 else 4277 iounmap(adev->dma_reg); 4278 of_address_to_resource(np, 0, &res); 4279 release_mem_region(res.start, resource_size(&res)); 4280 kfree(adev); 4281 return 0; 4282 } 4283 4284 /* 4285 * /sys driver interface to enable h/w RAID-6 capabilities 4286 * Files created in e.g. /sys/devices/plb.0/400100100.dma0/driver/ 4287 * directory are "devices", "enable" and "poly". 4288 * "devices" shows available engines. 4289 * "enable" is used to enable RAID-6 capabilities or to check 4290 * whether these has been activated. 4291 * "poly" allows setting/checking used polynomial (for PPC440SPe only). 4292 */ 4293 4294 static ssize_t devices_show(struct device_driver *dev, char *buf) 4295 { 4296 ssize_t size = 0; 4297 int i; 4298 4299 for (i = 0; i < PPC440SPE_ADMA_ENGINES_NUM; i++) { 4300 if (ppc440spe_adma_devices[i] == -1) 4301 continue; 4302 size += scnprintf(buf + size, PAGE_SIZE - size, 4303 "PPC440SP(E)-ADMA.%d: %s\n", i, 4304 ppc_adma_errors[ppc440spe_adma_devices[i]]); 4305 } 4306 return size; 4307 } 4308 static DRIVER_ATTR_RO(devices); 4309 4310 static ssize_t enable_show(struct device_driver *dev, char *buf) 4311 { 4312 return snprintf(buf, PAGE_SIZE, 4313 "PPC440SP(e) RAID-6 capabilities are %sABLED.\n", 4314 ppc440spe_r6_enabled ? "EN" : "DIS"); 4315 } 4316 4317 static ssize_t enable_store(struct device_driver *dev, const char *buf, 4318 size_t count) 4319 { 4320 unsigned long val; 4321 int err; 4322 4323 if (!count || count > 11) 4324 return -EINVAL; 4325 4326 if (!ppc440spe_r6_tchan) 4327 return -EFAULT; 4328 4329 /* Write a key */ 4330 err = kstrtoul(buf, 16, &val); 4331 if (err) 4332 return err; 4333 4334 dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_XORBA, val); 4335 isync(); 4336 4337 /* Verify whether it really works now */ 4338 if (ppc440spe_test_raid6(ppc440spe_r6_tchan) == 0) { 4339 pr_info("PPC440SP(e) RAID-6 has been activated " 4340 "successfully\n"); 4341 ppc440spe_r6_enabled = 1; 4342 } else { 4343 pr_info("PPC440SP(e) RAID-6 hasn't been activated!" 4344 " Error key ?\n"); 4345 ppc440spe_r6_enabled = 0; 4346 } 4347 return count; 4348 } 4349 static DRIVER_ATTR_RW(enable); 4350 4351 static ssize_t poly_show(struct device_driver *dev, char *buf) 4352 { 4353 ssize_t size = 0; 4354 u32 reg; 4355 4356 #ifdef CONFIG_440SP 4357 /* 440SP has fixed polynomial */ 4358 reg = 0x4d; 4359 #else 4360 reg = dcr_read(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL); 4361 reg >>= MQ0_CFBHL_POLY; 4362 reg &= 0xFF; 4363 #endif 4364 4365 size = snprintf(buf, PAGE_SIZE, "PPC440SP(e) RAID-6 driver " 4366 "uses 0x1%02x polynomial.\n", reg); 4367 return size; 4368 } 4369 4370 static ssize_t poly_store(struct device_driver *dev, const char *buf, 4371 size_t count) 4372 { 4373 unsigned long reg, val; 4374 int err; 4375 #ifdef CONFIG_440SP 4376 /* 440SP uses default 0x14D polynomial only */ 4377 return -EINVAL; 4378 #endif 4379 4380 if (!count || count > 6) 4381 return -EINVAL; 4382 4383 /* e.g., 0x14D or 0x11D */ 4384 err = kstrtoul(buf, 16, &val); 4385 if (err) 4386 return err; 4387 4388 if (val & ~0x1FF) 4389 return -EINVAL; 4390 4391 val &= 0xFF; 4392 reg = dcr_read(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL); 4393 reg &= ~(0xFF << MQ0_CFBHL_POLY); 4394 reg |= val << MQ0_CFBHL_POLY; 4395 dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL, reg); 4396 4397 return count; 4398 } 4399 static DRIVER_ATTR_RW(poly); 4400 4401 /* 4402 * Common initialisation for RAID engines; allocate memory for 4403 * DMAx FIFOs, perform configuration common for all DMA engines. 4404 * Further DMA engine specific configuration is done at probe time. 4405 */ 4406 static int ppc440spe_configure_raid_devices(void) 4407 { 4408 struct device_node *np; 4409 struct resource i2o_res; 4410 struct i2o_regs __iomem *i2o_reg; 4411 dcr_host_t i2o_dcr_host; 4412 unsigned int dcr_base, dcr_len; 4413 int i, ret; 4414 4415 np = of_find_compatible_node(NULL, NULL, "ibm,i2o-440spe"); 4416 if (!np) { 4417 pr_err("%s: can't find I2O device tree node\n", 4418 __func__); 4419 return -ENODEV; 4420 } 4421 4422 if (of_address_to_resource(np, 0, &i2o_res)) { 4423 of_node_put(np); 4424 return -EINVAL; 4425 } 4426 4427 i2o_reg = of_iomap(np, 0); 4428 if (!i2o_reg) { 4429 pr_err("%s: failed to map I2O registers\n", __func__); 4430 of_node_put(np); 4431 return -EINVAL; 4432 } 4433 4434 /* Get I2O DCRs base */ 4435 dcr_base = dcr_resource_start(np, 0); 4436 dcr_len = dcr_resource_len(np, 0); 4437 if (!dcr_base && !dcr_len) { 4438 pr_err("%pOF: can't get DCR registers base/len!\n", np); 4439 of_node_put(np); 4440 iounmap(i2o_reg); 4441 return -ENODEV; 4442 } 4443 4444 i2o_dcr_host = dcr_map(np, dcr_base, dcr_len); 4445 if (!DCR_MAP_OK(i2o_dcr_host)) { 4446 pr_err("%pOF: failed to map DCRs!\n", np); 4447 of_node_put(np); 4448 iounmap(i2o_reg); 4449 return -ENODEV; 4450 } 4451 of_node_put(np); 4452 4453 /* Provide memory regions for DMA's FIFOs: I2O, DMA0 and DMA1 share 4454 * the base address of FIFO memory space. 4455 * Actually we need twice more physical memory than programmed in the 4456 * <fsiz> register (because there are two FIFOs for each DMA: CP and CS) 4457 */ 4458 ppc440spe_dma_fifo_buf = kmalloc((DMA0_FIFO_SIZE + DMA1_FIFO_SIZE) << 1, 4459 GFP_KERNEL); 4460 if (!ppc440spe_dma_fifo_buf) { 4461 pr_err("%s: DMA FIFO buffer allocation failed.\n", __func__); 4462 iounmap(i2o_reg); 4463 dcr_unmap(i2o_dcr_host, dcr_len); 4464 return -ENOMEM; 4465 } 4466 4467 /* 4468 * Configure h/w 4469 */ 4470 /* Reset I2O/DMA */ 4471 mtdcri(SDR0, DCRN_SDR0_SRST, DCRN_SDR0_SRST_I2ODMA); 4472 mtdcri(SDR0, DCRN_SDR0_SRST, 0); 4473 4474 /* Setup the base address of mmaped registers */ 4475 dcr_write(i2o_dcr_host, DCRN_I2O0_IBAH, (u32)(i2o_res.start >> 32)); 4476 dcr_write(i2o_dcr_host, DCRN_I2O0_IBAL, (u32)(i2o_res.start) | 4477 I2O_REG_ENABLE); 4478 dcr_unmap(i2o_dcr_host, dcr_len); 4479 4480 /* Setup FIFO memory space base address */ 4481 iowrite32(0, &i2o_reg->ifbah); 4482 iowrite32(((u32)__pa(ppc440spe_dma_fifo_buf)), &i2o_reg->ifbal); 4483 4484 /* set zero FIFO size for I2O, so the whole 4485 * ppc440spe_dma_fifo_buf is used by DMAs. 4486 * DMAx_FIFOs will be configured while probe. 4487 */ 4488 iowrite32(0, &i2o_reg->ifsiz); 4489 iounmap(i2o_reg); 4490 4491 /* To prepare WXOR/RXOR functionality we need access to 4492 * Memory Queue Module DCRs (finally it will be enabled 4493 * via /sys interface of the ppc440spe ADMA driver). 4494 */ 4495 np = of_find_compatible_node(NULL, NULL, "ibm,mq-440spe"); 4496 if (!np) { 4497 pr_err("%s: can't find MQ device tree node\n", 4498 __func__); 4499 ret = -ENODEV; 4500 goto out_free; 4501 } 4502 4503 /* Get MQ DCRs base */ 4504 dcr_base = dcr_resource_start(np, 0); 4505 dcr_len = dcr_resource_len(np, 0); 4506 if (!dcr_base && !dcr_len) { 4507 pr_err("%pOF: can't get DCR registers base/len!\n", np); 4508 ret = -ENODEV; 4509 goto out_mq; 4510 } 4511 4512 ppc440spe_mq_dcr_host = dcr_map(np, dcr_base, dcr_len); 4513 if (!DCR_MAP_OK(ppc440spe_mq_dcr_host)) { 4514 pr_err("%pOF: failed to map DCRs!\n", np); 4515 ret = -ENODEV; 4516 goto out_mq; 4517 } 4518 of_node_put(np); 4519 ppc440spe_mq_dcr_len = dcr_len; 4520 4521 /* Set HB alias */ 4522 dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_BAUH, DMA_CUED_XOR_HB); 4523 4524 /* Set: 4525 * - LL transaction passing limit to 1; 4526 * - Memory controller cycle limit to 1; 4527 * - Galois Polynomial to 0x14d (default) 4528 */ 4529 dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL, 4530 (1 << MQ0_CFBHL_TPLM) | (1 << MQ0_CFBHL_HBCL) | 4531 (PPC440SPE_DEFAULT_POLY << MQ0_CFBHL_POLY)); 4532 4533 atomic_set(&ppc440spe_adma_err_irq_ref, 0); 4534 for (i = 0; i < PPC440SPE_ADMA_ENGINES_NUM; i++) 4535 ppc440spe_adma_devices[i] = -1; 4536 4537 return 0; 4538 4539 out_mq: 4540 of_node_put(np); 4541 out_free: 4542 kfree(ppc440spe_dma_fifo_buf); 4543 return ret; 4544 } 4545 4546 static const struct of_device_id ppc440spe_adma_of_match[] = { 4547 { .compatible = "ibm,dma-440spe", }, 4548 { .compatible = "amcc,xor-accelerator", }, 4549 {}, 4550 }; 4551 MODULE_DEVICE_TABLE(of, ppc440spe_adma_of_match); 4552 4553 static struct platform_driver ppc440spe_adma_driver = { 4554 .probe = ppc440spe_adma_probe, 4555 .remove = ppc440spe_adma_remove, 4556 .driver = { 4557 .name = "PPC440SP(E)-ADMA", 4558 .of_match_table = ppc440spe_adma_of_match, 4559 }, 4560 }; 4561 4562 static __init int ppc440spe_adma_init(void) 4563 { 4564 int ret; 4565 4566 ret = ppc440spe_configure_raid_devices(); 4567 if (ret) 4568 return ret; 4569 4570 ret = platform_driver_register(&ppc440spe_adma_driver); 4571 if (ret) { 4572 pr_err("%s: failed to register platform driver\n", 4573 __func__); 4574 goto out_reg; 4575 } 4576 4577 /* Initialization status */ 4578 ret = driver_create_file(&ppc440spe_adma_driver.driver, 4579 &driver_attr_devices); 4580 if (ret) 4581 goto out_dev; 4582 4583 /* RAID-6 h/w enable entry */ 4584 ret = driver_create_file(&ppc440spe_adma_driver.driver, 4585 &driver_attr_enable); 4586 if (ret) 4587 goto out_en; 4588 4589 /* GF polynomial to use */ 4590 ret = driver_create_file(&ppc440spe_adma_driver.driver, 4591 &driver_attr_poly); 4592 if (!ret) 4593 return ret; 4594 4595 driver_remove_file(&ppc440spe_adma_driver.driver, 4596 &driver_attr_enable); 4597 out_en: 4598 driver_remove_file(&ppc440spe_adma_driver.driver, 4599 &driver_attr_devices); 4600 out_dev: 4601 /* User will not be able to enable h/w RAID-6 */ 4602 pr_err("%s: failed to create RAID-6 driver interface\n", 4603 __func__); 4604 platform_driver_unregister(&ppc440spe_adma_driver); 4605 out_reg: 4606 dcr_unmap(ppc440spe_mq_dcr_host, ppc440spe_mq_dcr_len); 4607 kfree(ppc440spe_dma_fifo_buf); 4608 return ret; 4609 } 4610 4611 static void __exit ppc440spe_adma_exit(void) 4612 { 4613 driver_remove_file(&ppc440spe_adma_driver.driver, 4614 &driver_attr_poly); 4615 driver_remove_file(&ppc440spe_adma_driver.driver, 4616 &driver_attr_enable); 4617 driver_remove_file(&ppc440spe_adma_driver.driver, 4618 &driver_attr_devices); 4619 platform_driver_unregister(&ppc440spe_adma_driver); 4620 dcr_unmap(ppc440spe_mq_dcr_host, ppc440spe_mq_dcr_len); 4621 kfree(ppc440spe_dma_fifo_buf); 4622 } 4623 4624 arch_initcall(ppc440spe_adma_init); 4625 module_exit(ppc440spe_adma_exit); 4626 4627 MODULE_AUTHOR("Yuri Tikhonov <yur@emcraft.com>"); 4628 MODULE_DESCRIPTION("PPC440SPE ADMA Engine Driver"); 4629 MODULE_LICENSE("GPL"); 4630