1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Copyright (C) 2006-2009 DENX Software Engineering. 4 * 5 * Author: Yuri Tikhonov <yur@emcraft.com> 6 * 7 * Further porting to arch/powerpc by 8 * Anatolij Gustschin <agust@denx.de> 9 */ 10 11 /* 12 * This driver supports the asynchrounous DMA copy and RAID engines available 13 * on the AMCC PPC440SPe Processors. 14 * Based on the Intel Xscale(R) family of I/O Processors (IOP 32x, 33x, 134x) 15 * ADMA driver written by D.Williams. 16 */ 17 18 #include <linux/init.h> 19 #include <linux/module.h> 20 #include <linux/async_tx.h> 21 #include <linux/delay.h> 22 #include <linux/dma-mapping.h> 23 #include <linux/spinlock.h> 24 #include <linux/interrupt.h> 25 #include <linux/slab.h> 26 #include <linux/uaccess.h> 27 #include <linux/proc_fs.h> 28 #include <linux/of.h> 29 #include <linux/of_address.h> 30 #include <linux/of_irq.h> 31 #include <linux/of_platform.h> 32 #include <asm/dcr.h> 33 #include <asm/dcr-regs.h> 34 #include "adma.h" 35 #include "../dmaengine.h" 36 37 enum ppc_adma_init_code { 38 PPC_ADMA_INIT_OK = 0, 39 PPC_ADMA_INIT_MEMRES, 40 PPC_ADMA_INIT_MEMREG, 41 PPC_ADMA_INIT_ALLOC, 42 PPC_ADMA_INIT_COHERENT, 43 PPC_ADMA_INIT_CHANNEL, 44 PPC_ADMA_INIT_IRQ1, 45 PPC_ADMA_INIT_IRQ2, 46 PPC_ADMA_INIT_REGISTER 47 }; 48 49 static char *ppc_adma_errors[] = { 50 [PPC_ADMA_INIT_OK] = "ok", 51 [PPC_ADMA_INIT_MEMRES] = "failed to get memory resource", 52 [PPC_ADMA_INIT_MEMREG] = "failed to request memory region", 53 [PPC_ADMA_INIT_ALLOC] = "failed to allocate memory for adev " 54 "structure", 55 [PPC_ADMA_INIT_COHERENT] = "failed to allocate coherent memory for " 56 "hardware descriptors", 57 [PPC_ADMA_INIT_CHANNEL] = "failed to allocate memory for channel", 58 [PPC_ADMA_INIT_IRQ1] = "failed to request first irq", 59 [PPC_ADMA_INIT_IRQ2] = "failed to request second irq", 60 [PPC_ADMA_INIT_REGISTER] = "failed to register dma async device", 61 }; 62 63 static enum ppc_adma_init_code 64 ppc440spe_adma_devices[PPC440SPE_ADMA_ENGINES_NUM]; 65 66 struct ppc_dma_chan_ref { 67 struct dma_chan *chan; 68 struct list_head node; 69 }; 70 71 /* The list of channels exported by ppc440spe ADMA */ 72 static struct list_head 73 ppc440spe_adma_chan_list = LIST_HEAD_INIT(ppc440spe_adma_chan_list); 74 75 /* This flag is set when want to refetch the xor chain in the interrupt 76 * handler 77 */ 78 static u32 do_xor_refetch; 79 80 /* Pointer to DMA0, DMA1 CP/CS FIFO */ 81 static void *ppc440spe_dma_fifo_buf; 82 83 /* Pointers to last submitted to DMA0, DMA1 CDBs */ 84 static struct ppc440spe_adma_desc_slot *chan_last_sub[3]; 85 static struct ppc440spe_adma_desc_slot *chan_first_cdb[3]; 86 87 /* Pointer to last linked and submitted xor CB */ 88 static struct ppc440spe_adma_desc_slot *xor_last_linked; 89 static struct ppc440spe_adma_desc_slot *xor_last_submit; 90 91 /* This array is used in data-check operations for storing a pattern */ 92 static char ppc440spe_qword[16]; 93 94 static atomic_t ppc440spe_adma_err_irq_ref; 95 static dcr_host_t ppc440spe_mq_dcr_host; 96 static unsigned int ppc440spe_mq_dcr_len; 97 98 /* Since RXOR operations use the common register (MQ0_CF2H) for setting-up 99 * the block size in transactions, then we do not allow to activate more than 100 * only one RXOR transactions simultaneously. So use this var to store 101 * the information about is RXOR currently active (PPC440SPE_RXOR_RUN bit is 102 * set) or not (PPC440SPE_RXOR_RUN is clear). 103 */ 104 static unsigned long ppc440spe_rxor_state; 105 106 /* These are used in enable & check routines 107 */ 108 static u32 ppc440spe_r6_enabled; 109 static struct ppc440spe_adma_chan *ppc440spe_r6_tchan; 110 static struct completion ppc440spe_r6_test_comp; 111 112 static int ppc440spe_adma_dma2rxor_prep_src( 113 struct ppc440spe_adma_desc_slot *desc, 114 struct ppc440spe_rxor *cursor, int index, 115 int src_cnt, u32 addr); 116 static void ppc440spe_adma_dma2rxor_set_src( 117 struct ppc440spe_adma_desc_slot *desc, 118 int index, dma_addr_t addr); 119 static void ppc440spe_adma_dma2rxor_set_mult( 120 struct ppc440spe_adma_desc_slot *desc, 121 int index, u8 mult); 122 123 #ifdef ADMA_LL_DEBUG 124 #define ADMA_LL_DBG(x) ({ if (1) x; 0; }) 125 #else 126 #define ADMA_LL_DBG(x) ({ if (0) x; 0; }) 127 #endif 128 129 static void print_cb(struct ppc440spe_adma_chan *chan, void *block) 130 { 131 struct dma_cdb *cdb; 132 struct xor_cb *cb; 133 int i; 134 135 switch (chan->device->id) { 136 case 0: 137 case 1: 138 cdb = block; 139 140 pr_debug("CDB at %p [%d]:\n" 141 "\t attr 0x%02x opc 0x%02x cnt 0x%08x\n" 142 "\t sg1u 0x%08x sg1l 0x%08x\n" 143 "\t sg2u 0x%08x sg2l 0x%08x\n" 144 "\t sg3u 0x%08x sg3l 0x%08x\n", 145 cdb, chan->device->id, 146 cdb->attr, cdb->opc, le32_to_cpu(cdb->cnt), 147 le32_to_cpu(cdb->sg1u), le32_to_cpu(cdb->sg1l), 148 le32_to_cpu(cdb->sg2u), le32_to_cpu(cdb->sg2l), 149 le32_to_cpu(cdb->sg3u), le32_to_cpu(cdb->sg3l) 150 ); 151 break; 152 case 2: 153 cb = block; 154 155 pr_debug("CB at %p [%d]:\n" 156 "\t cbc 0x%08x cbbc 0x%08x cbs 0x%08x\n" 157 "\t cbtah 0x%08x cbtal 0x%08x\n" 158 "\t cblah 0x%08x cblal 0x%08x\n", 159 cb, chan->device->id, 160 cb->cbc, cb->cbbc, cb->cbs, 161 cb->cbtah, cb->cbtal, 162 cb->cblah, cb->cblal); 163 for (i = 0; i < 16; i++) { 164 if (i && !cb->ops[i].h && !cb->ops[i].l) 165 continue; 166 pr_debug("\t ops[%2d]: h 0x%08x l 0x%08x\n", 167 i, cb->ops[i].h, cb->ops[i].l); 168 } 169 break; 170 } 171 } 172 173 static void print_cb_list(struct ppc440spe_adma_chan *chan, 174 struct ppc440spe_adma_desc_slot *iter) 175 { 176 for (; iter; iter = iter->hw_next) 177 print_cb(chan, iter->hw_desc); 178 } 179 180 static void prep_dma_xor_dbg(int id, dma_addr_t dst, dma_addr_t *src, 181 unsigned int src_cnt) 182 { 183 int i; 184 185 pr_debug("\n%s(%d):\nsrc: ", __func__, id); 186 for (i = 0; i < src_cnt; i++) 187 pr_debug("\t0x%016llx ", src[i]); 188 pr_debug("dst:\n\t0x%016llx\n", dst); 189 } 190 191 static void prep_dma_pq_dbg(int id, dma_addr_t *dst, dma_addr_t *src, 192 unsigned int src_cnt) 193 { 194 int i; 195 196 pr_debug("\n%s(%d):\nsrc: ", __func__, id); 197 for (i = 0; i < src_cnt; i++) 198 pr_debug("\t0x%016llx ", src[i]); 199 pr_debug("dst: "); 200 for (i = 0; i < 2; i++) 201 pr_debug("\t0x%016llx ", dst[i]); 202 } 203 204 static void prep_dma_pqzero_sum_dbg(int id, dma_addr_t *src, 205 unsigned int src_cnt, 206 const unsigned char *scf) 207 { 208 int i; 209 210 pr_debug("\n%s(%d):\nsrc(coef): ", __func__, id); 211 if (scf) { 212 for (i = 0; i < src_cnt; i++) 213 pr_debug("\t0x%016llx(0x%02x) ", src[i], scf[i]); 214 } else { 215 for (i = 0; i < src_cnt; i++) 216 pr_debug("\t0x%016llx(no) ", src[i]); 217 } 218 219 pr_debug("dst: "); 220 for (i = 0; i < 2; i++) 221 pr_debug("\t0x%016llx ", src[src_cnt + i]); 222 } 223 224 /****************************************************************************** 225 * Command (Descriptor) Blocks low-level routines 226 ******************************************************************************/ 227 /** 228 * ppc440spe_desc_init_interrupt - initialize the descriptor for INTERRUPT 229 * pseudo operation 230 */ 231 static void ppc440spe_desc_init_interrupt(struct ppc440spe_adma_desc_slot *desc, 232 struct ppc440spe_adma_chan *chan) 233 { 234 struct xor_cb *p; 235 236 switch (chan->device->id) { 237 case PPC440SPE_XOR_ID: 238 p = desc->hw_desc; 239 memset(desc->hw_desc, 0, sizeof(struct xor_cb)); 240 /* NOP with Command Block Complete Enable */ 241 p->cbc = XOR_CBCR_CBCE_BIT; 242 break; 243 case PPC440SPE_DMA0_ID: 244 case PPC440SPE_DMA1_ID: 245 memset(desc->hw_desc, 0, sizeof(struct dma_cdb)); 246 /* NOP with interrupt */ 247 set_bit(PPC440SPE_DESC_INT, &desc->flags); 248 break; 249 default: 250 printk(KERN_ERR "Unsupported id %d in %s\n", chan->device->id, 251 __func__); 252 break; 253 } 254 } 255 256 /** 257 * ppc440spe_desc_init_null_xor - initialize the descriptor for NULL XOR 258 * pseudo operation 259 */ 260 static void ppc440spe_desc_init_null_xor(struct ppc440spe_adma_desc_slot *desc) 261 { 262 memset(desc->hw_desc, 0, sizeof(struct xor_cb)); 263 desc->hw_next = NULL; 264 desc->src_cnt = 0; 265 desc->dst_cnt = 1; 266 } 267 268 /** 269 * ppc440spe_desc_init_xor - initialize the descriptor for XOR operation 270 */ 271 static void ppc440spe_desc_init_xor(struct ppc440spe_adma_desc_slot *desc, 272 int src_cnt, unsigned long flags) 273 { 274 struct xor_cb *hw_desc = desc->hw_desc; 275 276 memset(desc->hw_desc, 0, sizeof(struct xor_cb)); 277 desc->hw_next = NULL; 278 desc->src_cnt = src_cnt; 279 desc->dst_cnt = 1; 280 281 hw_desc->cbc = XOR_CBCR_TGT_BIT | src_cnt; 282 if (flags & DMA_PREP_INTERRUPT) 283 /* Enable interrupt on completion */ 284 hw_desc->cbc |= XOR_CBCR_CBCE_BIT; 285 } 286 287 /** 288 * ppc440spe_desc_init_dma2pq - initialize the descriptor for PQ 289 * operation in DMA2 controller 290 */ 291 static void ppc440spe_desc_init_dma2pq(struct ppc440spe_adma_desc_slot *desc, 292 int dst_cnt, int src_cnt, unsigned long flags) 293 { 294 struct xor_cb *hw_desc = desc->hw_desc; 295 296 memset(desc->hw_desc, 0, sizeof(struct xor_cb)); 297 desc->hw_next = NULL; 298 desc->src_cnt = src_cnt; 299 desc->dst_cnt = dst_cnt; 300 memset(desc->reverse_flags, 0, sizeof(desc->reverse_flags)); 301 desc->descs_per_op = 0; 302 303 hw_desc->cbc = XOR_CBCR_TGT_BIT; 304 if (flags & DMA_PREP_INTERRUPT) 305 /* Enable interrupt on completion */ 306 hw_desc->cbc |= XOR_CBCR_CBCE_BIT; 307 } 308 309 #define DMA_CTRL_FLAGS_LAST DMA_PREP_FENCE 310 #define DMA_PREP_ZERO_P (DMA_CTRL_FLAGS_LAST << 1) 311 #define DMA_PREP_ZERO_Q (DMA_PREP_ZERO_P << 1) 312 313 /** 314 * ppc440spe_desc_init_dma01pq - initialize the descriptors for PQ operation 315 * with DMA0/1 316 */ 317 static void ppc440spe_desc_init_dma01pq(struct ppc440spe_adma_desc_slot *desc, 318 int dst_cnt, int src_cnt, unsigned long flags, 319 unsigned long op) 320 { 321 struct dma_cdb *hw_desc; 322 struct ppc440spe_adma_desc_slot *iter; 323 u8 dopc; 324 325 /* Common initialization of a PQ descriptors chain */ 326 set_bits(op, &desc->flags); 327 desc->src_cnt = src_cnt; 328 desc->dst_cnt = dst_cnt; 329 330 /* WXOR MULTICAST if both P and Q are being computed 331 * MV_SG1_SG2 if Q only 332 */ 333 dopc = (desc->dst_cnt == DMA_DEST_MAX_NUM) ? 334 DMA_CDB_OPC_MULTICAST : DMA_CDB_OPC_MV_SG1_SG2; 335 336 list_for_each_entry(iter, &desc->group_list, chain_node) { 337 hw_desc = iter->hw_desc; 338 memset(iter->hw_desc, 0, sizeof(struct dma_cdb)); 339 340 if (likely(!list_is_last(&iter->chain_node, 341 &desc->group_list))) { 342 /* set 'next' pointer */ 343 iter->hw_next = list_entry(iter->chain_node.next, 344 struct ppc440spe_adma_desc_slot, chain_node); 345 clear_bit(PPC440SPE_DESC_INT, &iter->flags); 346 } else { 347 /* this is the last descriptor. 348 * this slot will be pasted from ADMA level 349 * each time it wants to configure parameters 350 * of the transaction (src, dst, ...) 351 */ 352 iter->hw_next = NULL; 353 if (flags & DMA_PREP_INTERRUPT) 354 set_bit(PPC440SPE_DESC_INT, &iter->flags); 355 else 356 clear_bit(PPC440SPE_DESC_INT, &iter->flags); 357 } 358 } 359 360 /* Set OPS depending on WXOR/RXOR type of operation */ 361 if (!test_bit(PPC440SPE_DESC_RXOR, &desc->flags)) { 362 /* This is a WXOR only chain: 363 * - first descriptors are for zeroing destinations 364 * if PPC440SPE_ZERO_P/Q set; 365 * - descriptors remained are for GF-XOR operations. 366 */ 367 iter = list_first_entry(&desc->group_list, 368 struct ppc440spe_adma_desc_slot, 369 chain_node); 370 371 if (test_bit(PPC440SPE_ZERO_P, &desc->flags)) { 372 hw_desc = iter->hw_desc; 373 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; 374 iter = list_first_entry(&iter->chain_node, 375 struct ppc440spe_adma_desc_slot, 376 chain_node); 377 } 378 379 if (test_bit(PPC440SPE_ZERO_Q, &desc->flags)) { 380 hw_desc = iter->hw_desc; 381 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; 382 iter = list_first_entry(&iter->chain_node, 383 struct ppc440spe_adma_desc_slot, 384 chain_node); 385 } 386 387 list_for_each_entry_from(iter, &desc->group_list, chain_node) { 388 hw_desc = iter->hw_desc; 389 hw_desc->opc = dopc; 390 } 391 } else { 392 /* This is either RXOR-only or mixed RXOR/WXOR */ 393 394 /* The first 1 or 2 slots in chain are always RXOR, 395 * if need to calculate P & Q, then there are two 396 * RXOR slots; if only P or only Q, then there is one 397 */ 398 iter = list_first_entry(&desc->group_list, 399 struct ppc440spe_adma_desc_slot, 400 chain_node); 401 hw_desc = iter->hw_desc; 402 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; 403 404 if (desc->dst_cnt == DMA_DEST_MAX_NUM) { 405 iter = list_first_entry(&iter->chain_node, 406 struct ppc440spe_adma_desc_slot, 407 chain_node); 408 hw_desc = iter->hw_desc; 409 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; 410 } 411 412 /* The remaining descs (if any) are WXORs */ 413 if (test_bit(PPC440SPE_DESC_WXOR, &desc->flags)) { 414 iter = list_first_entry(&iter->chain_node, 415 struct ppc440spe_adma_desc_slot, 416 chain_node); 417 list_for_each_entry_from(iter, &desc->group_list, 418 chain_node) { 419 hw_desc = iter->hw_desc; 420 hw_desc->opc = dopc; 421 } 422 } 423 } 424 } 425 426 /** 427 * ppc440spe_desc_init_dma01pqzero_sum - initialize the descriptor 428 * for PQ_ZERO_SUM operation 429 */ 430 static void ppc440spe_desc_init_dma01pqzero_sum( 431 struct ppc440spe_adma_desc_slot *desc, 432 int dst_cnt, int src_cnt) 433 { 434 struct dma_cdb *hw_desc; 435 struct ppc440spe_adma_desc_slot *iter; 436 int i = 0; 437 u8 dopc = (dst_cnt == 2) ? DMA_CDB_OPC_MULTICAST : 438 DMA_CDB_OPC_MV_SG1_SG2; 439 /* 440 * Initialize starting from 2nd or 3rd descriptor dependent 441 * on dst_cnt. First one or two slots are for cloning P 442 * and/or Q to chan->pdest and/or chan->qdest as we have 443 * to preserve original P/Q. 444 */ 445 iter = list_first_entry(&desc->group_list, 446 struct ppc440spe_adma_desc_slot, chain_node); 447 iter = list_entry(iter->chain_node.next, 448 struct ppc440spe_adma_desc_slot, chain_node); 449 450 if (dst_cnt > 1) { 451 iter = list_entry(iter->chain_node.next, 452 struct ppc440spe_adma_desc_slot, chain_node); 453 } 454 /* initialize each source descriptor in chain */ 455 list_for_each_entry_from(iter, &desc->group_list, chain_node) { 456 hw_desc = iter->hw_desc; 457 memset(iter->hw_desc, 0, sizeof(struct dma_cdb)); 458 iter->src_cnt = 0; 459 iter->dst_cnt = 0; 460 461 /* This is a ZERO_SUM operation: 462 * - <src_cnt> descriptors starting from 2nd or 3rd 463 * descriptor are for GF-XOR operations; 464 * - remaining <dst_cnt> descriptors are for checking the result 465 */ 466 if (i++ < src_cnt) 467 /* MV_SG1_SG2 if only Q is being verified 468 * MULTICAST if both P and Q are being verified 469 */ 470 hw_desc->opc = dopc; 471 else 472 /* DMA_CDB_OPC_DCHECK128 operation */ 473 hw_desc->opc = DMA_CDB_OPC_DCHECK128; 474 475 if (likely(!list_is_last(&iter->chain_node, 476 &desc->group_list))) { 477 /* set 'next' pointer */ 478 iter->hw_next = list_entry(iter->chain_node.next, 479 struct ppc440spe_adma_desc_slot, 480 chain_node); 481 } else { 482 /* this is the last descriptor. 483 * this slot will be pasted from ADMA level 484 * each time it wants to configure parameters 485 * of the transaction (src, dst, ...) 486 */ 487 iter->hw_next = NULL; 488 /* always enable interrupt generation since we get 489 * the status of pqzero from the handler 490 */ 491 set_bit(PPC440SPE_DESC_INT, &iter->flags); 492 } 493 } 494 desc->src_cnt = src_cnt; 495 desc->dst_cnt = dst_cnt; 496 } 497 498 /** 499 * ppc440spe_desc_init_memcpy - initialize the descriptor for MEMCPY operation 500 */ 501 static void ppc440spe_desc_init_memcpy(struct ppc440spe_adma_desc_slot *desc, 502 unsigned long flags) 503 { 504 struct dma_cdb *hw_desc = desc->hw_desc; 505 506 memset(desc->hw_desc, 0, sizeof(struct dma_cdb)); 507 desc->hw_next = NULL; 508 desc->src_cnt = 1; 509 desc->dst_cnt = 1; 510 511 if (flags & DMA_PREP_INTERRUPT) 512 set_bit(PPC440SPE_DESC_INT, &desc->flags); 513 else 514 clear_bit(PPC440SPE_DESC_INT, &desc->flags); 515 516 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; 517 } 518 519 /** 520 * ppc440spe_desc_set_src_addr - set source address into the descriptor 521 */ 522 static void ppc440spe_desc_set_src_addr(struct ppc440spe_adma_desc_slot *desc, 523 struct ppc440spe_adma_chan *chan, 524 int src_idx, dma_addr_t addrh, 525 dma_addr_t addrl) 526 { 527 struct dma_cdb *dma_hw_desc; 528 struct xor_cb *xor_hw_desc; 529 phys_addr_t addr64, tmplow, tmphi; 530 531 switch (chan->device->id) { 532 case PPC440SPE_DMA0_ID: 533 case PPC440SPE_DMA1_ID: 534 if (!addrh) { 535 addr64 = addrl; 536 tmphi = (addr64 >> 32); 537 tmplow = (addr64 & 0xFFFFFFFF); 538 } else { 539 tmphi = addrh; 540 tmplow = addrl; 541 } 542 dma_hw_desc = desc->hw_desc; 543 dma_hw_desc->sg1l = cpu_to_le32((u32)tmplow); 544 dma_hw_desc->sg1u |= cpu_to_le32((u32)tmphi); 545 break; 546 case PPC440SPE_XOR_ID: 547 xor_hw_desc = desc->hw_desc; 548 xor_hw_desc->ops[src_idx].l = addrl; 549 xor_hw_desc->ops[src_idx].h |= addrh; 550 break; 551 } 552 } 553 554 /** 555 * ppc440spe_desc_set_src_mult - set source address mult into the descriptor 556 */ 557 static void ppc440spe_desc_set_src_mult(struct ppc440spe_adma_desc_slot *desc, 558 struct ppc440spe_adma_chan *chan, u32 mult_index, 559 int sg_index, unsigned char mult_value) 560 { 561 struct dma_cdb *dma_hw_desc; 562 u32 *psgu; 563 564 switch (chan->device->id) { 565 case PPC440SPE_DMA0_ID: 566 case PPC440SPE_DMA1_ID: 567 dma_hw_desc = desc->hw_desc; 568 569 switch (sg_index) { 570 /* for RXOR operations set multiplier 571 * into source cued address 572 */ 573 case DMA_CDB_SG_SRC: 574 psgu = &dma_hw_desc->sg1u; 575 break; 576 /* for WXOR operations set multiplier 577 * into destination cued address(es) 578 */ 579 case DMA_CDB_SG_DST1: 580 psgu = &dma_hw_desc->sg2u; 581 break; 582 case DMA_CDB_SG_DST2: 583 psgu = &dma_hw_desc->sg3u; 584 break; 585 default: 586 BUG(); 587 } 588 589 *psgu |= cpu_to_le32(mult_value << mult_index); 590 break; 591 case PPC440SPE_XOR_ID: 592 break; 593 default: 594 BUG(); 595 } 596 } 597 598 /** 599 * ppc440spe_desc_set_dest_addr - set destination address into the descriptor 600 */ 601 static void ppc440spe_desc_set_dest_addr(struct ppc440spe_adma_desc_slot *desc, 602 struct ppc440spe_adma_chan *chan, 603 dma_addr_t addrh, dma_addr_t addrl, 604 u32 dst_idx) 605 { 606 struct dma_cdb *dma_hw_desc; 607 struct xor_cb *xor_hw_desc; 608 phys_addr_t addr64, tmphi, tmplow; 609 u32 *psgu, *psgl; 610 611 switch (chan->device->id) { 612 case PPC440SPE_DMA0_ID: 613 case PPC440SPE_DMA1_ID: 614 if (!addrh) { 615 addr64 = addrl; 616 tmphi = (addr64 >> 32); 617 tmplow = (addr64 & 0xFFFFFFFF); 618 } else { 619 tmphi = addrh; 620 tmplow = addrl; 621 } 622 dma_hw_desc = desc->hw_desc; 623 624 psgu = dst_idx ? &dma_hw_desc->sg3u : &dma_hw_desc->sg2u; 625 psgl = dst_idx ? &dma_hw_desc->sg3l : &dma_hw_desc->sg2l; 626 627 *psgl = cpu_to_le32((u32)tmplow); 628 *psgu |= cpu_to_le32((u32)tmphi); 629 break; 630 case PPC440SPE_XOR_ID: 631 xor_hw_desc = desc->hw_desc; 632 xor_hw_desc->cbtal = addrl; 633 xor_hw_desc->cbtah |= addrh; 634 break; 635 } 636 } 637 638 /** 639 * ppc440spe_desc_set_byte_count - set number of data bytes involved 640 * into the operation 641 */ 642 static void ppc440spe_desc_set_byte_count(struct ppc440spe_adma_desc_slot *desc, 643 struct ppc440spe_adma_chan *chan, 644 u32 byte_count) 645 { 646 struct dma_cdb *dma_hw_desc; 647 struct xor_cb *xor_hw_desc; 648 649 switch (chan->device->id) { 650 case PPC440SPE_DMA0_ID: 651 case PPC440SPE_DMA1_ID: 652 dma_hw_desc = desc->hw_desc; 653 dma_hw_desc->cnt = cpu_to_le32(byte_count); 654 break; 655 case PPC440SPE_XOR_ID: 656 xor_hw_desc = desc->hw_desc; 657 xor_hw_desc->cbbc = byte_count; 658 break; 659 } 660 } 661 662 /** 663 * ppc440spe_desc_set_rxor_block_size - set RXOR block size 664 */ 665 static inline void ppc440spe_desc_set_rxor_block_size(u32 byte_count) 666 { 667 /* assume that byte_count is aligned on the 512-boundary; 668 * thus write it directly to the register (bits 23:31 are 669 * reserved there). 670 */ 671 dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_CF2H, byte_count); 672 } 673 674 /** 675 * ppc440spe_desc_set_dcheck - set CHECK pattern 676 */ 677 static void ppc440spe_desc_set_dcheck(struct ppc440spe_adma_desc_slot *desc, 678 struct ppc440spe_adma_chan *chan, u8 *qword) 679 { 680 struct dma_cdb *dma_hw_desc; 681 682 switch (chan->device->id) { 683 case PPC440SPE_DMA0_ID: 684 case PPC440SPE_DMA1_ID: 685 dma_hw_desc = desc->hw_desc; 686 iowrite32(qword[0], &dma_hw_desc->sg3l); 687 iowrite32(qword[4], &dma_hw_desc->sg3u); 688 iowrite32(qword[8], &dma_hw_desc->sg2l); 689 iowrite32(qword[12], &dma_hw_desc->sg2u); 690 break; 691 default: 692 BUG(); 693 } 694 } 695 696 /** 697 * ppc440spe_xor_set_link - set link address in xor CB 698 */ 699 static void ppc440spe_xor_set_link(struct ppc440spe_adma_desc_slot *prev_desc, 700 struct ppc440spe_adma_desc_slot *next_desc) 701 { 702 struct xor_cb *xor_hw_desc = prev_desc->hw_desc; 703 704 if (unlikely(!next_desc || !(next_desc->phys))) { 705 printk(KERN_ERR "%s: next_desc=0x%p; next_desc->phys=0x%llx\n", 706 __func__, next_desc, 707 next_desc ? next_desc->phys : 0); 708 BUG(); 709 } 710 711 xor_hw_desc->cbs = 0; 712 xor_hw_desc->cblal = next_desc->phys; 713 xor_hw_desc->cblah = 0; 714 xor_hw_desc->cbc |= XOR_CBCR_LNK_BIT; 715 } 716 717 /** 718 * ppc440spe_desc_set_link - set the address of descriptor following this 719 * descriptor in chain 720 */ 721 static void ppc440spe_desc_set_link(struct ppc440spe_adma_chan *chan, 722 struct ppc440spe_adma_desc_slot *prev_desc, 723 struct ppc440spe_adma_desc_slot *next_desc) 724 { 725 unsigned long flags; 726 struct ppc440spe_adma_desc_slot *tail = next_desc; 727 728 if (unlikely(!prev_desc || !next_desc || 729 (prev_desc->hw_next && prev_desc->hw_next != next_desc))) { 730 /* If previous next is overwritten something is wrong. 731 * though we may refetch from append to initiate list 732 * processing; in this case - it's ok. 733 */ 734 printk(KERN_ERR "%s: prev_desc=0x%p; next_desc=0x%p; " 735 "prev->hw_next=0x%p\n", __func__, prev_desc, 736 next_desc, prev_desc ? prev_desc->hw_next : 0); 737 BUG(); 738 } 739 740 local_irq_save(flags); 741 742 /* do s/w chaining both for DMA and XOR descriptors */ 743 prev_desc->hw_next = next_desc; 744 745 switch (chan->device->id) { 746 case PPC440SPE_DMA0_ID: 747 case PPC440SPE_DMA1_ID: 748 break; 749 case PPC440SPE_XOR_ID: 750 /* bind descriptor to the chain */ 751 while (tail->hw_next) 752 tail = tail->hw_next; 753 xor_last_linked = tail; 754 755 if (prev_desc == xor_last_submit) 756 /* do not link to the last submitted CB */ 757 break; 758 ppc440spe_xor_set_link(prev_desc, next_desc); 759 break; 760 } 761 762 local_irq_restore(flags); 763 } 764 765 /** 766 * ppc440spe_desc_get_link - get the address of the descriptor that 767 * follows this one 768 */ 769 static inline u32 ppc440spe_desc_get_link(struct ppc440spe_adma_desc_slot *desc, 770 struct ppc440spe_adma_chan *chan) 771 { 772 if (!desc->hw_next) 773 return 0; 774 775 return desc->hw_next->phys; 776 } 777 778 /** 779 * ppc440spe_desc_is_aligned - check alignment 780 */ 781 static inline int ppc440spe_desc_is_aligned( 782 struct ppc440spe_adma_desc_slot *desc, int num_slots) 783 { 784 return (desc->idx & (num_slots - 1)) ? 0 : 1; 785 } 786 787 /** 788 * ppc440spe_chan_xor_slot_count - get the number of slots necessary for 789 * XOR operation 790 */ 791 static int ppc440spe_chan_xor_slot_count(size_t len, int src_cnt, 792 int *slots_per_op) 793 { 794 int slot_cnt; 795 796 /* each XOR descriptor provides up to 16 source operands */ 797 slot_cnt = *slots_per_op = (src_cnt + XOR_MAX_OPS - 1)/XOR_MAX_OPS; 798 799 if (likely(len <= PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT)) 800 return slot_cnt; 801 802 printk(KERN_ERR "%s: len %d > max %d !!\n", 803 __func__, len, PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT); 804 BUG(); 805 return slot_cnt; 806 } 807 808 /** 809 * ppc440spe_dma2_pq_slot_count - get the number of slots necessary for 810 * DMA2 PQ operation 811 */ 812 static int ppc440spe_dma2_pq_slot_count(dma_addr_t *srcs, 813 int src_cnt, size_t len) 814 { 815 signed long long order = 0; 816 int state = 0; 817 int addr_count = 0; 818 int i; 819 for (i = 1; i < src_cnt; i++) { 820 dma_addr_t cur_addr = srcs[i]; 821 dma_addr_t old_addr = srcs[i-1]; 822 switch (state) { 823 case 0: 824 if (cur_addr == old_addr + len) { 825 /* direct RXOR */ 826 order = 1; 827 state = 1; 828 if (i == src_cnt-1) 829 addr_count++; 830 } else if (old_addr == cur_addr + len) { 831 /* reverse RXOR */ 832 order = -1; 833 state = 1; 834 if (i == src_cnt-1) 835 addr_count++; 836 } else { 837 state = 3; 838 } 839 break; 840 case 1: 841 if (i == src_cnt-2 || (order == -1 842 && cur_addr != old_addr - len)) { 843 order = 0; 844 state = 0; 845 addr_count++; 846 } else if (cur_addr == old_addr + len*order) { 847 state = 2; 848 if (i == src_cnt-1) 849 addr_count++; 850 } else if (cur_addr == old_addr + 2*len) { 851 state = 2; 852 if (i == src_cnt-1) 853 addr_count++; 854 } else if (cur_addr == old_addr + 3*len) { 855 state = 2; 856 if (i == src_cnt-1) 857 addr_count++; 858 } else { 859 order = 0; 860 state = 0; 861 addr_count++; 862 } 863 break; 864 case 2: 865 order = 0; 866 state = 0; 867 addr_count++; 868 break; 869 } 870 if (state == 3) 871 break; 872 } 873 if (src_cnt <= 1 || (state != 1 && state != 2)) { 874 pr_err("%s: src_cnt=%d, state=%d, addr_count=%d, order=%lld\n", 875 __func__, src_cnt, state, addr_count, order); 876 for (i = 0; i < src_cnt; i++) 877 pr_err("\t[%d] 0x%llx \n", i, srcs[i]); 878 BUG(); 879 } 880 881 return (addr_count + XOR_MAX_OPS - 1) / XOR_MAX_OPS; 882 } 883 884 885 /****************************************************************************** 886 * ADMA channel low-level routines 887 ******************************************************************************/ 888 889 static u32 890 ppc440spe_chan_get_current_descriptor(struct ppc440spe_adma_chan *chan); 891 static void ppc440spe_chan_append(struct ppc440spe_adma_chan *chan); 892 893 /** 894 * ppc440spe_adma_device_clear_eot_status - interrupt ack to XOR or DMA engine 895 */ 896 static void ppc440spe_adma_device_clear_eot_status( 897 struct ppc440spe_adma_chan *chan) 898 { 899 struct dma_regs *dma_reg; 900 struct xor_regs *xor_reg; 901 u8 *p = chan->device->dma_desc_pool_virt; 902 struct dma_cdb *cdb; 903 u32 rv, i; 904 905 switch (chan->device->id) { 906 case PPC440SPE_DMA0_ID: 907 case PPC440SPE_DMA1_ID: 908 /* read FIFO to ack */ 909 dma_reg = chan->device->dma_reg; 910 while ((rv = ioread32(&dma_reg->csfpl))) { 911 i = rv & DMA_CDB_ADDR_MSK; 912 cdb = (struct dma_cdb *)&p[i - 913 (u32)chan->device->dma_desc_pool]; 914 915 /* Clear opcode to ack. This is necessary for 916 * ZeroSum operations only 917 */ 918 cdb->opc = 0; 919 920 if (test_bit(PPC440SPE_RXOR_RUN, 921 &ppc440spe_rxor_state)) { 922 /* probably this is a completed RXOR op, 923 * get pointer to CDB using the fact that 924 * physical and virtual addresses of CDB 925 * in pools have the same offsets 926 */ 927 if (le32_to_cpu(cdb->sg1u) & 928 DMA_CUED_XOR_BASE) { 929 /* this is a RXOR */ 930 clear_bit(PPC440SPE_RXOR_RUN, 931 &ppc440spe_rxor_state); 932 } 933 } 934 935 if (rv & DMA_CDB_STATUS_MSK) { 936 /* ZeroSum check failed 937 */ 938 struct ppc440spe_adma_desc_slot *iter; 939 dma_addr_t phys = rv & ~DMA_CDB_MSK; 940 941 /* 942 * Update the status of corresponding 943 * descriptor. 944 */ 945 list_for_each_entry(iter, &chan->chain, 946 chain_node) { 947 if (iter->phys == phys) 948 break; 949 } 950 /* 951 * if cannot find the corresponding 952 * slot it's a bug 953 */ 954 BUG_ON(&iter->chain_node == &chan->chain); 955 956 if (iter->xor_check_result) { 957 if (test_bit(PPC440SPE_DESC_PCHECK, 958 &iter->flags)) { 959 *iter->xor_check_result |= 960 SUM_CHECK_P_RESULT; 961 } else 962 if (test_bit(PPC440SPE_DESC_QCHECK, 963 &iter->flags)) { 964 *iter->xor_check_result |= 965 SUM_CHECK_Q_RESULT; 966 } else 967 BUG(); 968 } 969 } 970 } 971 972 rv = ioread32(&dma_reg->dsts); 973 if (rv) { 974 pr_err("DMA%d err status: 0x%x\n", 975 chan->device->id, rv); 976 /* write back to clear */ 977 iowrite32(rv, &dma_reg->dsts); 978 } 979 break; 980 case PPC440SPE_XOR_ID: 981 /* reset status bits to ack */ 982 xor_reg = chan->device->xor_reg; 983 rv = ioread32be(&xor_reg->sr); 984 iowrite32be(rv, &xor_reg->sr); 985 986 if (rv & (XOR_IE_ICBIE_BIT|XOR_IE_ICIE_BIT|XOR_IE_RPTIE_BIT)) { 987 if (rv & XOR_IE_RPTIE_BIT) { 988 /* Read PLB Timeout Error. 989 * Try to resubmit the CB 990 */ 991 u32 val = ioread32be(&xor_reg->ccbalr); 992 993 iowrite32be(val, &xor_reg->cblalr); 994 995 val = ioread32be(&xor_reg->crsr); 996 iowrite32be(val | XOR_CRSR_XAE_BIT, 997 &xor_reg->crsr); 998 } else 999 pr_err("XOR ERR 0x%x status\n", rv); 1000 break; 1001 } 1002 1003 /* if the XORcore is idle, but there are unprocessed CBs 1004 * then refetch the s/w chain here 1005 */ 1006 if (!(ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT) && 1007 do_xor_refetch) 1008 ppc440spe_chan_append(chan); 1009 break; 1010 } 1011 } 1012 1013 /** 1014 * ppc440spe_chan_is_busy - get the channel status 1015 */ 1016 static int ppc440spe_chan_is_busy(struct ppc440spe_adma_chan *chan) 1017 { 1018 struct dma_regs *dma_reg; 1019 struct xor_regs *xor_reg; 1020 int busy = 0; 1021 1022 switch (chan->device->id) { 1023 case PPC440SPE_DMA0_ID: 1024 case PPC440SPE_DMA1_ID: 1025 dma_reg = chan->device->dma_reg; 1026 /* if command FIFO's head and tail pointers are equal and 1027 * status tail is the same as command, then channel is free 1028 */ 1029 if (ioread16(&dma_reg->cpfhp) != ioread16(&dma_reg->cpftp) || 1030 ioread16(&dma_reg->cpftp) != ioread16(&dma_reg->csftp)) 1031 busy = 1; 1032 break; 1033 case PPC440SPE_XOR_ID: 1034 /* use the special status bit for the XORcore 1035 */ 1036 xor_reg = chan->device->xor_reg; 1037 busy = (ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT) ? 1 : 0; 1038 break; 1039 } 1040 1041 return busy; 1042 } 1043 1044 /** 1045 * ppc440spe_chan_set_first_xor_descriptor - init XORcore chain 1046 */ 1047 static void ppc440spe_chan_set_first_xor_descriptor( 1048 struct ppc440spe_adma_chan *chan, 1049 struct ppc440spe_adma_desc_slot *next_desc) 1050 { 1051 struct xor_regs *xor_reg = chan->device->xor_reg; 1052 1053 if (ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT) 1054 printk(KERN_INFO "%s: Warn: XORcore is running " 1055 "when try to set the first CDB!\n", 1056 __func__); 1057 1058 xor_last_submit = xor_last_linked = next_desc; 1059 1060 iowrite32be(XOR_CRSR_64BA_BIT, &xor_reg->crsr); 1061 1062 iowrite32be(next_desc->phys, &xor_reg->cblalr); 1063 iowrite32be(0, &xor_reg->cblahr); 1064 iowrite32be(ioread32be(&xor_reg->cbcr) | XOR_CBCR_LNK_BIT, 1065 &xor_reg->cbcr); 1066 1067 chan->hw_chain_inited = 1; 1068 } 1069 1070 /** 1071 * ppc440spe_dma_put_desc - put DMA0,1 descriptor to FIFO. 1072 * called with irqs disabled 1073 */ 1074 static void ppc440spe_dma_put_desc(struct ppc440spe_adma_chan *chan, 1075 struct ppc440spe_adma_desc_slot *desc) 1076 { 1077 u32 pcdb; 1078 struct dma_regs *dma_reg = chan->device->dma_reg; 1079 1080 pcdb = desc->phys; 1081 if (!test_bit(PPC440SPE_DESC_INT, &desc->flags)) 1082 pcdb |= DMA_CDB_NO_INT; 1083 1084 chan_last_sub[chan->device->id] = desc; 1085 1086 ADMA_LL_DBG(print_cb(chan, desc->hw_desc)); 1087 1088 iowrite32(pcdb, &dma_reg->cpfpl); 1089 } 1090 1091 /** 1092 * ppc440spe_chan_append - update the h/w chain in the channel 1093 */ 1094 static void ppc440spe_chan_append(struct ppc440spe_adma_chan *chan) 1095 { 1096 struct xor_regs *xor_reg; 1097 struct ppc440spe_adma_desc_slot *iter; 1098 struct xor_cb *xcb; 1099 u32 cur_desc; 1100 unsigned long flags; 1101 1102 local_irq_save(flags); 1103 1104 switch (chan->device->id) { 1105 case PPC440SPE_DMA0_ID: 1106 case PPC440SPE_DMA1_ID: 1107 cur_desc = ppc440spe_chan_get_current_descriptor(chan); 1108 1109 if (likely(cur_desc)) { 1110 iter = chan_last_sub[chan->device->id]; 1111 BUG_ON(!iter); 1112 } else { 1113 /* first peer */ 1114 iter = chan_first_cdb[chan->device->id]; 1115 BUG_ON(!iter); 1116 ppc440spe_dma_put_desc(chan, iter); 1117 chan->hw_chain_inited = 1; 1118 } 1119 1120 /* is there something new to append */ 1121 if (!iter->hw_next) 1122 break; 1123 1124 /* flush descriptors from the s/w queue to fifo */ 1125 list_for_each_entry_continue(iter, &chan->chain, chain_node) { 1126 ppc440spe_dma_put_desc(chan, iter); 1127 if (!iter->hw_next) 1128 break; 1129 } 1130 break; 1131 case PPC440SPE_XOR_ID: 1132 /* update h/w links and refetch */ 1133 if (!xor_last_submit->hw_next) 1134 break; 1135 1136 xor_reg = chan->device->xor_reg; 1137 /* the last linked CDB has to generate an interrupt 1138 * that we'd be able to append the next lists to h/w 1139 * regardless of the XOR engine state at the moment of 1140 * appending of these next lists 1141 */ 1142 xcb = xor_last_linked->hw_desc; 1143 xcb->cbc |= XOR_CBCR_CBCE_BIT; 1144 1145 if (!(ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT)) { 1146 /* XORcore is idle. Refetch now */ 1147 do_xor_refetch = 0; 1148 ppc440spe_xor_set_link(xor_last_submit, 1149 xor_last_submit->hw_next); 1150 1151 ADMA_LL_DBG(print_cb_list(chan, 1152 xor_last_submit->hw_next)); 1153 1154 xor_last_submit = xor_last_linked; 1155 iowrite32be(ioread32be(&xor_reg->crsr) | 1156 XOR_CRSR_RCBE_BIT | XOR_CRSR_64BA_BIT, 1157 &xor_reg->crsr); 1158 } else { 1159 /* XORcore is running. Refetch later in the handler */ 1160 do_xor_refetch = 1; 1161 } 1162 1163 break; 1164 } 1165 1166 local_irq_restore(flags); 1167 } 1168 1169 /** 1170 * ppc440spe_chan_get_current_descriptor - get the currently executed descriptor 1171 */ 1172 static u32 1173 ppc440spe_chan_get_current_descriptor(struct ppc440spe_adma_chan *chan) 1174 { 1175 struct dma_regs *dma_reg; 1176 struct xor_regs *xor_reg; 1177 1178 if (unlikely(!chan->hw_chain_inited)) 1179 /* h/w descriptor chain is not initialized yet */ 1180 return 0; 1181 1182 switch (chan->device->id) { 1183 case PPC440SPE_DMA0_ID: 1184 case PPC440SPE_DMA1_ID: 1185 dma_reg = chan->device->dma_reg; 1186 return ioread32(&dma_reg->acpl) & (~DMA_CDB_MSK); 1187 case PPC440SPE_XOR_ID: 1188 xor_reg = chan->device->xor_reg; 1189 return ioread32be(&xor_reg->ccbalr); 1190 } 1191 return 0; 1192 } 1193 1194 /** 1195 * ppc440spe_chan_run - enable the channel 1196 */ 1197 static void ppc440spe_chan_run(struct ppc440spe_adma_chan *chan) 1198 { 1199 struct xor_regs *xor_reg; 1200 1201 switch (chan->device->id) { 1202 case PPC440SPE_DMA0_ID: 1203 case PPC440SPE_DMA1_ID: 1204 /* DMAs are always enabled, do nothing */ 1205 break; 1206 case PPC440SPE_XOR_ID: 1207 /* drain write buffer */ 1208 xor_reg = chan->device->xor_reg; 1209 1210 /* fetch descriptor pointed to in <link> */ 1211 iowrite32be(XOR_CRSR_64BA_BIT | XOR_CRSR_XAE_BIT, 1212 &xor_reg->crsr); 1213 break; 1214 } 1215 } 1216 1217 /****************************************************************************** 1218 * ADMA device level 1219 ******************************************************************************/ 1220 1221 static void ppc440spe_chan_start_null_xor(struct ppc440spe_adma_chan *chan); 1222 static int ppc440spe_adma_alloc_chan_resources(struct dma_chan *chan); 1223 1224 static dma_cookie_t 1225 ppc440spe_adma_tx_submit(struct dma_async_tx_descriptor *tx); 1226 1227 static void ppc440spe_adma_set_dest(struct ppc440spe_adma_desc_slot *tx, 1228 dma_addr_t addr, int index); 1229 static void 1230 ppc440spe_adma_memcpy_xor_set_src(struct ppc440spe_adma_desc_slot *tx, 1231 dma_addr_t addr, int index); 1232 1233 static void 1234 ppc440spe_adma_pq_set_dest(struct ppc440spe_adma_desc_slot *tx, 1235 dma_addr_t *paddr, unsigned long flags); 1236 static void 1237 ppc440spe_adma_pq_set_src(struct ppc440spe_adma_desc_slot *tx, 1238 dma_addr_t addr, int index); 1239 static void 1240 ppc440spe_adma_pq_set_src_mult(struct ppc440spe_adma_desc_slot *tx, 1241 unsigned char mult, int index, int dst_pos); 1242 static void 1243 ppc440spe_adma_pqzero_sum_set_dest(struct ppc440spe_adma_desc_slot *tx, 1244 dma_addr_t paddr, dma_addr_t qaddr); 1245 1246 static struct page *ppc440spe_rxor_srcs[32]; 1247 1248 /** 1249 * ppc440spe_can_rxor - check if the operands may be processed with RXOR 1250 */ 1251 static int ppc440spe_can_rxor(struct page **srcs, int src_cnt, size_t len) 1252 { 1253 int i, order = 0, state = 0; 1254 int idx = 0; 1255 1256 if (unlikely(!(src_cnt > 1))) 1257 return 0; 1258 1259 BUG_ON(src_cnt > ARRAY_SIZE(ppc440spe_rxor_srcs)); 1260 1261 /* Skip holes in the source list before checking */ 1262 for (i = 0; i < src_cnt; i++) { 1263 if (!srcs[i]) 1264 continue; 1265 ppc440spe_rxor_srcs[idx++] = srcs[i]; 1266 } 1267 src_cnt = idx; 1268 1269 for (i = 1; i < src_cnt; i++) { 1270 char *cur_addr = page_address(ppc440spe_rxor_srcs[i]); 1271 char *old_addr = page_address(ppc440spe_rxor_srcs[i - 1]); 1272 1273 switch (state) { 1274 case 0: 1275 if (cur_addr == old_addr + len) { 1276 /* direct RXOR */ 1277 order = 1; 1278 state = 1; 1279 } else if (old_addr == cur_addr + len) { 1280 /* reverse RXOR */ 1281 order = -1; 1282 state = 1; 1283 } else 1284 goto out; 1285 break; 1286 case 1: 1287 if ((i == src_cnt - 2) || 1288 (order == -1 && cur_addr != old_addr - len)) { 1289 order = 0; 1290 state = 0; 1291 } else if ((cur_addr == old_addr + len * order) || 1292 (cur_addr == old_addr + 2 * len) || 1293 (cur_addr == old_addr + 3 * len)) { 1294 state = 2; 1295 } else { 1296 order = 0; 1297 state = 0; 1298 } 1299 break; 1300 case 2: 1301 order = 0; 1302 state = 0; 1303 break; 1304 } 1305 } 1306 1307 out: 1308 if (state == 1 || state == 2) 1309 return 1; 1310 1311 return 0; 1312 } 1313 1314 /** 1315 * ppc440spe_adma_device_estimate - estimate the efficiency of processing 1316 * the operation given on this channel. It's assumed that 'chan' is 1317 * capable to process 'cap' type of operation. 1318 * @chan: channel to use 1319 * @cap: type of transaction 1320 * @dst_lst: array of destination pointers 1321 * @dst_cnt: number of destination operands 1322 * @src_lst: array of source pointers 1323 * @src_cnt: number of source operands 1324 * @src_sz: size of each source operand 1325 */ 1326 static int ppc440spe_adma_estimate(struct dma_chan *chan, 1327 enum dma_transaction_type cap, struct page **dst_lst, int dst_cnt, 1328 struct page **src_lst, int src_cnt, size_t src_sz) 1329 { 1330 int ef = 1; 1331 1332 if (cap == DMA_PQ || cap == DMA_PQ_VAL) { 1333 /* If RAID-6 capabilities were not activated don't try 1334 * to use them 1335 */ 1336 if (unlikely(!ppc440spe_r6_enabled)) 1337 return -1; 1338 } 1339 /* In the current implementation of ppc440spe ADMA driver it 1340 * makes sense to pick out only pq case, because it may be 1341 * processed: 1342 * (1) either using Biskup method on DMA2; 1343 * (2) or on DMA0/1. 1344 * Thus we give a favour to (1) if the sources are suitable; 1345 * else let it be processed on one of the DMA0/1 engines. 1346 * In the sum_product case where destination is also the 1347 * source process it on DMA0/1 only. 1348 */ 1349 if (cap == DMA_PQ && chan->chan_id == PPC440SPE_XOR_ID) { 1350 1351 if (dst_cnt == 1 && src_cnt == 2 && dst_lst[0] == src_lst[1]) 1352 ef = 0; /* sum_product case, process on DMA0/1 */ 1353 else if (ppc440spe_can_rxor(src_lst, src_cnt, src_sz)) 1354 ef = 3; /* override (DMA0/1 + idle) */ 1355 else 1356 ef = 0; /* can't process on DMA2 if !rxor */ 1357 } 1358 1359 /* channel idleness increases the priority */ 1360 if (likely(ef) && 1361 !ppc440spe_chan_is_busy(to_ppc440spe_adma_chan(chan))) 1362 ef++; 1363 1364 return ef; 1365 } 1366 1367 struct dma_chan * 1368 ppc440spe_async_tx_find_best_channel(enum dma_transaction_type cap, 1369 struct page **dst_lst, int dst_cnt, struct page **src_lst, 1370 int src_cnt, size_t src_sz) 1371 { 1372 struct dma_chan *best_chan = NULL; 1373 struct ppc_dma_chan_ref *ref; 1374 int best_rank = -1; 1375 1376 if (unlikely(!src_sz)) 1377 return NULL; 1378 if (src_sz > PAGE_SIZE) { 1379 /* 1380 * should a user of the api ever pass > PAGE_SIZE requests 1381 * we sort out cases where temporary page-sized buffers 1382 * are used. 1383 */ 1384 switch (cap) { 1385 case DMA_PQ: 1386 if (src_cnt == 1 && dst_lst[1] == src_lst[0]) 1387 return NULL; 1388 if (src_cnt == 2 && dst_lst[1] == src_lst[1]) 1389 return NULL; 1390 break; 1391 case DMA_PQ_VAL: 1392 case DMA_XOR_VAL: 1393 return NULL; 1394 default: 1395 break; 1396 } 1397 } 1398 1399 list_for_each_entry(ref, &ppc440spe_adma_chan_list, node) { 1400 if (dma_has_cap(cap, ref->chan->device->cap_mask)) { 1401 int rank; 1402 1403 rank = ppc440spe_adma_estimate(ref->chan, cap, dst_lst, 1404 dst_cnt, src_lst, src_cnt, src_sz); 1405 if (rank > best_rank) { 1406 best_rank = rank; 1407 best_chan = ref->chan; 1408 } 1409 } 1410 } 1411 1412 return best_chan; 1413 } 1414 EXPORT_SYMBOL_GPL(ppc440spe_async_tx_find_best_channel); 1415 1416 /** 1417 * ppc440spe_get_group_entry - get group entry with index idx 1418 * @tdesc: is the last allocated slot in the group. 1419 */ 1420 static struct ppc440spe_adma_desc_slot * 1421 ppc440spe_get_group_entry(struct ppc440spe_adma_desc_slot *tdesc, u32 entry_idx) 1422 { 1423 struct ppc440spe_adma_desc_slot *iter = tdesc->group_head; 1424 int i = 0; 1425 1426 if (entry_idx < 0 || entry_idx >= (tdesc->src_cnt + tdesc->dst_cnt)) { 1427 printk("%s: entry_idx %d, src_cnt %d, dst_cnt %d\n", 1428 __func__, entry_idx, tdesc->src_cnt, tdesc->dst_cnt); 1429 BUG(); 1430 } 1431 1432 list_for_each_entry(iter, &tdesc->group_list, chain_node) { 1433 if (i++ == entry_idx) 1434 break; 1435 } 1436 return iter; 1437 } 1438 1439 /** 1440 * ppc440spe_adma_free_slots - flags descriptor slots for reuse 1441 * @slot: Slot to free 1442 * Caller must hold &ppc440spe_chan->lock while calling this function 1443 */ 1444 static void ppc440spe_adma_free_slots(struct ppc440spe_adma_desc_slot *slot, 1445 struct ppc440spe_adma_chan *chan) 1446 { 1447 int stride = slot->slots_per_op; 1448 1449 while (stride--) { 1450 slot->slots_per_op = 0; 1451 slot = list_entry(slot->slot_node.next, 1452 struct ppc440spe_adma_desc_slot, 1453 slot_node); 1454 } 1455 } 1456 1457 /** 1458 * ppc440spe_adma_run_tx_complete_actions - call functions to be called 1459 * upon completion 1460 */ 1461 static dma_cookie_t ppc440spe_adma_run_tx_complete_actions( 1462 struct ppc440spe_adma_desc_slot *desc, 1463 struct ppc440spe_adma_chan *chan, 1464 dma_cookie_t cookie) 1465 { 1466 BUG_ON(desc->async_tx.cookie < 0); 1467 if (desc->async_tx.cookie > 0) { 1468 cookie = desc->async_tx.cookie; 1469 desc->async_tx.cookie = 0; 1470 1471 dma_descriptor_unmap(&desc->async_tx); 1472 /* call the callback (must not sleep or submit new 1473 * operations to this channel) 1474 */ 1475 dmaengine_desc_get_callback_invoke(&desc->async_tx, NULL); 1476 } 1477 1478 /* run dependent operations */ 1479 dma_run_dependencies(&desc->async_tx); 1480 1481 return cookie; 1482 } 1483 1484 /** 1485 * ppc440spe_adma_clean_slot - clean up CDB slot (if ack is set) 1486 */ 1487 static int ppc440spe_adma_clean_slot(struct ppc440spe_adma_desc_slot *desc, 1488 struct ppc440spe_adma_chan *chan) 1489 { 1490 /* the client is allowed to attach dependent operations 1491 * until 'ack' is set 1492 */ 1493 if (!async_tx_test_ack(&desc->async_tx)) 1494 return 0; 1495 1496 /* leave the last descriptor in the chain 1497 * so we can append to it 1498 */ 1499 if (list_is_last(&desc->chain_node, &chan->chain) || 1500 desc->phys == ppc440spe_chan_get_current_descriptor(chan)) 1501 return 1; 1502 1503 if (chan->device->id != PPC440SPE_XOR_ID) { 1504 /* our DMA interrupt handler clears opc field of 1505 * each processed descriptor. For all types of 1506 * operations except for ZeroSum we do not actually 1507 * need ack from the interrupt handler. ZeroSum is a 1508 * special case since the result of this operation 1509 * is available from the handler only, so if we see 1510 * such type of descriptor (which is unprocessed yet) 1511 * then leave it in chain. 1512 */ 1513 struct dma_cdb *cdb = desc->hw_desc; 1514 if (cdb->opc == DMA_CDB_OPC_DCHECK128) 1515 return 1; 1516 } 1517 1518 dev_dbg(chan->device->common.dev, "\tfree slot %llx: %d stride: %d\n", 1519 desc->phys, desc->idx, desc->slots_per_op); 1520 1521 list_del(&desc->chain_node); 1522 ppc440spe_adma_free_slots(desc, chan); 1523 return 0; 1524 } 1525 1526 /** 1527 * __ppc440spe_adma_slot_cleanup - this is the common clean-up routine 1528 * which runs through the channel CDBs list until reach the descriptor 1529 * currently processed. When routine determines that all CDBs of group 1530 * are completed then corresponding callbacks (if any) are called and slots 1531 * are freed. 1532 */ 1533 static void __ppc440spe_adma_slot_cleanup(struct ppc440spe_adma_chan *chan) 1534 { 1535 struct ppc440spe_adma_desc_slot *iter, *_iter, *group_start = NULL; 1536 dma_cookie_t cookie = 0; 1537 u32 current_desc = ppc440spe_chan_get_current_descriptor(chan); 1538 int busy = ppc440spe_chan_is_busy(chan); 1539 int seen_current = 0, slot_cnt = 0, slots_per_op = 0; 1540 1541 dev_dbg(chan->device->common.dev, "ppc440spe adma%d: %s\n", 1542 chan->device->id, __func__); 1543 1544 if (!current_desc) { 1545 /* There were no transactions yet, so 1546 * nothing to clean 1547 */ 1548 return; 1549 } 1550 1551 /* free completed slots from the chain starting with 1552 * the oldest descriptor 1553 */ 1554 list_for_each_entry_safe(iter, _iter, &chan->chain, 1555 chain_node) { 1556 dev_dbg(chan->device->common.dev, "\tcookie: %d slot: %d " 1557 "busy: %d this_desc: %#llx next_desc: %#x " 1558 "cur: %#x ack: %d\n", 1559 iter->async_tx.cookie, iter->idx, busy, iter->phys, 1560 ppc440spe_desc_get_link(iter, chan), current_desc, 1561 async_tx_test_ack(&iter->async_tx)); 1562 prefetch(_iter); 1563 prefetch(&_iter->async_tx); 1564 1565 /* do not advance past the current descriptor loaded into the 1566 * hardware channel,subsequent descriptors are either in process 1567 * or have not been submitted 1568 */ 1569 if (seen_current) 1570 break; 1571 1572 /* stop the search if we reach the current descriptor and the 1573 * channel is busy, or if it appears that the current descriptor 1574 * needs to be re-read (i.e. has been appended to) 1575 */ 1576 if (iter->phys == current_desc) { 1577 BUG_ON(seen_current++); 1578 if (busy || ppc440spe_desc_get_link(iter, chan)) { 1579 /* not all descriptors of the group have 1580 * been completed; exit. 1581 */ 1582 break; 1583 } 1584 } 1585 1586 /* detect the start of a group transaction */ 1587 if (!slot_cnt && !slots_per_op) { 1588 slot_cnt = iter->slot_cnt; 1589 slots_per_op = iter->slots_per_op; 1590 if (slot_cnt <= slots_per_op) { 1591 slot_cnt = 0; 1592 slots_per_op = 0; 1593 } 1594 } 1595 1596 if (slot_cnt) { 1597 if (!group_start) 1598 group_start = iter; 1599 slot_cnt -= slots_per_op; 1600 } 1601 1602 /* all the members of a group are complete */ 1603 if (slots_per_op != 0 && slot_cnt == 0) { 1604 struct ppc440spe_adma_desc_slot *grp_iter, *_grp_iter; 1605 int end_of_chain = 0; 1606 1607 /* clean up the group */ 1608 slot_cnt = group_start->slot_cnt; 1609 grp_iter = group_start; 1610 list_for_each_entry_safe_from(grp_iter, _grp_iter, 1611 &chan->chain, chain_node) { 1612 1613 cookie = ppc440spe_adma_run_tx_complete_actions( 1614 grp_iter, chan, cookie); 1615 1616 slot_cnt -= slots_per_op; 1617 end_of_chain = ppc440spe_adma_clean_slot( 1618 grp_iter, chan); 1619 if (end_of_chain && slot_cnt) { 1620 /* Should wait for ZeroSum completion */ 1621 if (cookie > 0) 1622 chan->common.completed_cookie = cookie; 1623 return; 1624 } 1625 1626 if (slot_cnt == 0 || end_of_chain) 1627 break; 1628 } 1629 1630 /* the group should be complete at this point */ 1631 BUG_ON(slot_cnt); 1632 1633 slots_per_op = 0; 1634 group_start = NULL; 1635 if (end_of_chain) 1636 break; 1637 else 1638 continue; 1639 } else if (slots_per_op) /* wait for group completion */ 1640 continue; 1641 1642 cookie = ppc440spe_adma_run_tx_complete_actions(iter, chan, 1643 cookie); 1644 1645 if (ppc440spe_adma_clean_slot(iter, chan)) 1646 break; 1647 } 1648 1649 BUG_ON(!seen_current); 1650 1651 if (cookie > 0) { 1652 chan->common.completed_cookie = cookie; 1653 pr_debug("\tcompleted cookie %d\n", cookie); 1654 } 1655 1656 } 1657 1658 /** 1659 * ppc440spe_adma_tasklet - clean up watch-dog initiator 1660 */ 1661 static void ppc440spe_adma_tasklet(struct tasklet_struct *t) 1662 { 1663 struct ppc440spe_adma_chan *chan = from_tasklet(chan, t, irq_tasklet); 1664 1665 spin_lock_nested(&chan->lock, SINGLE_DEPTH_NESTING); 1666 __ppc440spe_adma_slot_cleanup(chan); 1667 spin_unlock(&chan->lock); 1668 } 1669 1670 /** 1671 * ppc440spe_adma_slot_cleanup - clean up scheduled initiator 1672 */ 1673 static void ppc440spe_adma_slot_cleanup(struct ppc440spe_adma_chan *chan) 1674 { 1675 spin_lock_bh(&chan->lock); 1676 __ppc440spe_adma_slot_cleanup(chan); 1677 spin_unlock_bh(&chan->lock); 1678 } 1679 1680 /** 1681 * ppc440spe_adma_alloc_slots - allocate free slots (if any) 1682 */ 1683 static struct ppc440spe_adma_desc_slot *ppc440spe_adma_alloc_slots( 1684 struct ppc440spe_adma_chan *chan, int num_slots, 1685 int slots_per_op) 1686 { 1687 struct ppc440spe_adma_desc_slot *iter = NULL, *_iter; 1688 struct ppc440spe_adma_desc_slot *alloc_start = NULL; 1689 struct list_head chain = LIST_HEAD_INIT(chain); 1690 int slots_found, retry = 0; 1691 1692 1693 BUG_ON(!num_slots || !slots_per_op); 1694 /* start search from the last allocated descrtiptor 1695 * if a contiguous allocation can not be found start searching 1696 * from the beginning of the list 1697 */ 1698 retry: 1699 slots_found = 0; 1700 if (retry == 0) 1701 iter = chan->last_used; 1702 else 1703 iter = list_entry(&chan->all_slots, 1704 struct ppc440spe_adma_desc_slot, 1705 slot_node); 1706 list_for_each_entry_safe_continue(iter, _iter, &chan->all_slots, 1707 slot_node) { 1708 prefetch(_iter); 1709 prefetch(&_iter->async_tx); 1710 if (iter->slots_per_op) { 1711 slots_found = 0; 1712 continue; 1713 } 1714 1715 /* start the allocation if the slot is correctly aligned */ 1716 if (!slots_found++) 1717 alloc_start = iter; 1718 1719 if (slots_found == num_slots) { 1720 struct ppc440spe_adma_desc_slot *alloc_tail = NULL; 1721 struct ppc440spe_adma_desc_slot *last_used = NULL; 1722 1723 iter = alloc_start; 1724 while (num_slots) { 1725 int i; 1726 /* pre-ack all but the last descriptor */ 1727 if (num_slots != slots_per_op) 1728 async_tx_ack(&iter->async_tx); 1729 1730 list_add_tail(&iter->chain_node, &chain); 1731 alloc_tail = iter; 1732 iter->async_tx.cookie = 0; 1733 iter->hw_next = NULL; 1734 iter->flags = 0; 1735 iter->slot_cnt = num_slots; 1736 iter->xor_check_result = NULL; 1737 for (i = 0; i < slots_per_op; i++) { 1738 iter->slots_per_op = slots_per_op - i; 1739 last_used = iter; 1740 iter = list_entry(iter->slot_node.next, 1741 struct ppc440spe_adma_desc_slot, 1742 slot_node); 1743 } 1744 num_slots -= slots_per_op; 1745 } 1746 alloc_tail->group_head = alloc_start; 1747 alloc_tail->async_tx.cookie = -EBUSY; 1748 list_splice(&chain, &alloc_tail->group_list); 1749 chan->last_used = last_used; 1750 return alloc_tail; 1751 } 1752 } 1753 if (!retry++) 1754 goto retry; 1755 1756 /* try to free some slots if the allocation fails */ 1757 tasklet_schedule(&chan->irq_tasklet); 1758 return NULL; 1759 } 1760 1761 /** 1762 * ppc440spe_adma_alloc_chan_resources - allocate pools for CDB slots 1763 */ 1764 static int ppc440spe_adma_alloc_chan_resources(struct dma_chan *chan) 1765 { 1766 struct ppc440spe_adma_chan *ppc440spe_chan; 1767 struct ppc440spe_adma_desc_slot *slot = NULL; 1768 char *hw_desc; 1769 int i, db_sz; 1770 int init; 1771 1772 ppc440spe_chan = to_ppc440spe_adma_chan(chan); 1773 init = ppc440spe_chan->slots_allocated ? 0 : 1; 1774 chan->chan_id = ppc440spe_chan->device->id; 1775 1776 /* Allocate descriptor slots */ 1777 i = ppc440spe_chan->slots_allocated; 1778 if (ppc440spe_chan->device->id != PPC440SPE_XOR_ID) 1779 db_sz = sizeof(struct dma_cdb); 1780 else 1781 db_sz = sizeof(struct xor_cb); 1782 1783 for (; i < (ppc440spe_chan->device->pool_size / db_sz); i++) { 1784 slot = kzalloc(sizeof(struct ppc440spe_adma_desc_slot), 1785 GFP_KERNEL); 1786 if (!slot) { 1787 printk(KERN_INFO "SPE ADMA Channel only initialized" 1788 " %d descriptor slots", i--); 1789 break; 1790 } 1791 1792 hw_desc = (char *) ppc440spe_chan->device->dma_desc_pool_virt; 1793 slot->hw_desc = (void *) &hw_desc[i * db_sz]; 1794 dma_async_tx_descriptor_init(&slot->async_tx, chan); 1795 slot->async_tx.tx_submit = ppc440spe_adma_tx_submit; 1796 INIT_LIST_HEAD(&slot->chain_node); 1797 INIT_LIST_HEAD(&slot->slot_node); 1798 INIT_LIST_HEAD(&slot->group_list); 1799 slot->phys = ppc440spe_chan->device->dma_desc_pool + i * db_sz; 1800 slot->idx = i; 1801 1802 spin_lock_bh(&ppc440spe_chan->lock); 1803 ppc440spe_chan->slots_allocated++; 1804 list_add_tail(&slot->slot_node, &ppc440spe_chan->all_slots); 1805 spin_unlock_bh(&ppc440spe_chan->lock); 1806 } 1807 1808 if (i && !ppc440spe_chan->last_used) { 1809 ppc440spe_chan->last_used = 1810 list_entry(ppc440spe_chan->all_slots.next, 1811 struct ppc440spe_adma_desc_slot, 1812 slot_node); 1813 } 1814 1815 dev_dbg(ppc440spe_chan->device->common.dev, 1816 "ppc440spe adma%d: allocated %d descriptor slots\n", 1817 ppc440spe_chan->device->id, i); 1818 1819 /* initialize the channel and the chain with a null operation */ 1820 if (init) { 1821 switch (ppc440spe_chan->device->id) { 1822 case PPC440SPE_DMA0_ID: 1823 case PPC440SPE_DMA1_ID: 1824 ppc440spe_chan->hw_chain_inited = 0; 1825 /* Use WXOR for self-testing */ 1826 if (!ppc440spe_r6_tchan) 1827 ppc440spe_r6_tchan = ppc440spe_chan; 1828 break; 1829 case PPC440SPE_XOR_ID: 1830 ppc440spe_chan_start_null_xor(ppc440spe_chan); 1831 break; 1832 default: 1833 BUG(); 1834 } 1835 ppc440spe_chan->needs_unmap = 1; 1836 } 1837 1838 return (i > 0) ? i : -ENOMEM; 1839 } 1840 1841 /** 1842 * ppc440spe_rxor_set_region_data - 1843 */ 1844 static void ppc440spe_rxor_set_region(struct ppc440spe_adma_desc_slot *desc, 1845 u8 xor_arg_no, u32 mask) 1846 { 1847 struct xor_cb *xcb = desc->hw_desc; 1848 1849 xcb->ops[xor_arg_no].h |= mask; 1850 } 1851 1852 /** 1853 * ppc440spe_rxor_set_src - 1854 */ 1855 static void ppc440spe_rxor_set_src(struct ppc440spe_adma_desc_slot *desc, 1856 u8 xor_arg_no, dma_addr_t addr) 1857 { 1858 struct xor_cb *xcb = desc->hw_desc; 1859 1860 xcb->ops[xor_arg_no].h |= DMA_CUED_XOR_BASE; 1861 xcb->ops[xor_arg_no].l = addr; 1862 } 1863 1864 /** 1865 * ppc440spe_rxor_set_mult - 1866 */ 1867 static void ppc440spe_rxor_set_mult(struct ppc440spe_adma_desc_slot *desc, 1868 u8 xor_arg_no, u8 idx, u8 mult) 1869 { 1870 struct xor_cb *xcb = desc->hw_desc; 1871 1872 xcb->ops[xor_arg_no].h |= mult << (DMA_CUED_MULT1_OFF + idx * 8); 1873 } 1874 1875 /** 1876 * ppc440spe_adma_check_threshold - append CDBs to h/w chain if threshold 1877 * has been achieved 1878 */ 1879 static void ppc440spe_adma_check_threshold(struct ppc440spe_adma_chan *chan) 1880 { 1881 dev_dbg(chan->device->common.dev, "ppc440spe adma%d: pending: %d\n", 1882 chan->device->id, chan->pending); 1883 1884 if (chan->pending >= PPC440SPE_ADMA_THRESHOLD) { 1885 chan->pending = 0; 1886 ppc440spe_chan_append(chan); 1887 } 1888 } 1889 1890 /** 1891 * ppc440spe_adma_tx_submit - submit new descriptor group to the channel 1892 * (it's not necessary that descriptors will be submitted to the h/w 1893 * chains too right now) 1894 */ 1895 static dma_cookie_t ppc440spe_adma_tx_submit(struct dma_async_tx_descriptor *tx) 1896 { 1897 struct ppc440spe_adma_desc_slot *sw_desc; 1898 struct ppc440spe_adma_chan *chan = to_ppc440spe_adma_chan(tx->chan); 1899 struct ppc440spe_adma_desc_slot *group_start, *old_chain_tail; 1900 int slot_cnt; 1901 int slots_per_op; 1902 dma_cookie_t cookie; 1903 1904 sw_desc = tx_to_ppc440spe_adma_slot(tx); 1905 1906 group_start = sw_desc->group_head; 1907 slot_cnt = group_start->slot_cnt; 1908 slots_per_op = group_start->slots_per_op; 1909 1910 spin_lock_bh(&chan->lock); 1911 cookie = dma_cookie_assign(tx); 1912 1913 if (unlikely(list_empty(&chan->chain))) { 1914 /* first peer */ 1915 list_splice_init(&sw_desc->group_list, &chan->chain); 1916 chan_first_cdb[chan->device->id] = group_start; 1917 } else { 1918 /* isn't first peer, bind CDBs to chain */ 1919 old_chain_tail = list_entry(chan->chain.prev, 1920 struct ppc440spe_adma_desc_slot, 1921 chain_node); 1922 list_splice_init(&sw_desc->group_list, 1923 &old_chain_tail->chain_node); 1924 /* fix up the hardware chain */ 1925 ppc440spe_desc_set_link(chan, old_chain_tail, group_start); 1926 } 1927 1928 /* increment the pending count by the number of operations */ 1929 chan->pending += slot_cnt / slots_per_op; 1930 ppc440spe_adma_check_threshold(chan); 1931 spin_unlock_bh(&chan->lock); 1932 1933 dev_dbg(chan->device->common.dev, 1934 "ppc440spe adma%d: %s cookie: %d slot: %d tx %p\n", 1935 chan->device->id, __func__, 1936 sw_desc->async_tx.cookie, sw_desc->idx, sw_desc); 1937 1938 return cookie; 1939 } 1940 1941 /** 1942 * ppc440spe_adma_prep_dma_interrupt - prepare CDB for a pseudo DMA operation 1943 */ 1944 static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_interrupt( 1945 struct dma_chan *chan, unsigned long flags) 1946 { 1947 struct ppc440spe_adma_chan *ppc440spe_chan; 1948 struct ppc440spe_adma_desc_slot *sw_desc, *group_start; 1949 int slot_cnt, slots_per_op; 1950 1951 ppc440spe_chan = to_ppc440spe_adma_chan(chan); 1952 1953 dev_dbg(ppc440spe_chan->device->common.dev, 1954 "ppc440spe adma%d: %s\n", ppc440spe_chan->device->id, 1955 __func__); 1956 1957 spin_lock_bh(&ppc440spe_chan->lock); 1958 slot_cnt = slots_per_op = 1; 1959 sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1960 slots_per_op); 1961 if (sw_desc) { 1962 group_start = sw_desc->group_head; 1963 ppc440spe_desc_init_interrupt(group_start, ppc440spe_chan); 1964 group_start->unmap_len = 0; 1965 sw_desc->async_tx.flags = flags; 1966 } 1967 spin_unlock_bh(&ppc440spe_chan->lock); 1968 1969 return sw_desc ? &sw_desc->async_tx : NULL; 1970 } 1971 1972 /** 1973 * ppc440spe_adma_prep_dma_memcpy - prepare CDB for a MEMCPY operation 1974 */ 1975 static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_memcpy( 1976 struct dma_chan *chan, dma_addr_t dma_dest, 1977 dma_addr_t dma_src, size_t len, unsigned long flags) 1978 { 1979 struct ppc440spe_adma_chan *ppc440spe_chan; 1980 struct ppc440spe_adma_desc_slot *sw_desc, *group_start; 1981 int slot_cnt, slots_per_op; 1982 1983 ppc440spe_chan = to_ppc440spe_adma_chan(chan); 1984 1985 if (unlikely(!len)) 1986 return NULL; 1987 1988 BUG_ON(len > PPC440SPE_ADMA_DMA_MAX_BYTE_COUNT); 1989 1990 spin_lock_bh(&ppc440spe_chan->lock); 1991 1992 dev_dbg(ppc440spe_chan->device->common.dev, 1993 "ppc440spe adma%d: %s len: %u int_en %d\n", 1994 ppc440spe_chan->device->id, __func__, len, 1995 flags & DMA_PREP_INTERRUPT ? 1 : 0); 1996 slot_cnt = slots_per_op = 1; 1997 sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1998 slots_per_op); 1999 if (sw_desc) { 2000 group_start = sw_desc->group_head; 2001 ppc440spe_desc_init_memcpy(group_start, flags); 2002 ppc440spe_adma_set_dest(group_start, dma_dest, 0); 2003 ppc440spe_adma_memcpy_xor_set_src(group_start, dma_src, 0); 2004 ppc440spe_desc_set_byte_count(group_start, ppc440spe_chan, len); 2005 sw_desc->unmap_len = len; 2006 sw_desc->async_tx.flags = flags; 2007 } 2008 spin_unlock_bh(&ppc440spe_chan->lock); 2009 2010 return sw_desc ? &sw_desc->async_tx : NULL; 2011 } 2012 2013 /** 2014 * ppc440spe_adma_prep_dma_xor - prepare CDB for a XOR operation 2015 */ 2016 static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_xor( 2017 struct dma_chan *chan, dma_addr_t dma_dest, 2018 dma_addr_t *dma_src, u32 src_cnt, size_t len, 2019 unsigned long flags) 2020 { 2021 struct ppc440spe_adma_chan *ppc440spe_chan; 2022 struct ppc440spe_adma_desc_slot *sw_desc, *group_start; 2023 int slot_cnt, slots_per_op; 2024 2025 ppc440spe_chan = to_ppc440spe_adma_chan(chan); 2026 2027 ADMA_LL_DBG(prep_dma_xor_dbg(ppc440spe_chan->device->id, 2028 dma_dest, dma_src, src_cnt)); 2029 if (unlikely(!len)) 2030 return NULL; 2031 BUG_ON(len > PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT); 2032 2033 dev_dbg(ppc440spe_chan->device->common.dev, 2034 "ppc440spe adma%d: %s src_cnt: %d len: %u int_en: %d\n", 2035 ppc440spe_chan->device->id, __func__, src_cnt, len, 2036 flags & DMA_PREP_INTERRUPT ? 1 : 0); 2037 2038 spin_lock_bh(&ppc440spe_chan->lock); 2039 slot_cnt = ppc440spe_chan_xor_slot_count(len, src_cnt, &slots_per_op); 2040 sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 2041 slots_per_op); 2042 if (sw_desc) { 2043 group_start = sw_desc->group_head; 2044 ppc440spe_desc_init_xor(group_start, src_cnt, flags); 2045 ppc440spe_adma_set_dest(group_start, dma_dest, 0); 2046 while (src_cnt--) 2047 ppc440spe_adma_memcpy_xor_set_src(group_start, 2048 dma_src[src_cnt], src_cnt); 2049 ppc440spe_desc_set_byte_count(group_start, ppc440spe_chan, len); 2050 sw_desc->unmap_len = len; 2051 sw_desc->async_tx.flags = flags; 2052 } 2053 spin_unlock_bh(&ppc440spe_chan->lock); 2054 2055 return sw_desc ? &sw_desc->async_tx : NULL; 2056 } 2057 2058 static inline void 2059 ppc440spe_desc_set_xor_src_cnt(struct ppc440spe_adma_desc_slot *desc, 2060 int src_cnt); 2061 static void ppc440spe_init_rxor_cursor(struct ppc440spe_rxor *cursor); 2062 2063 /** 2064 * ppc440spe_adma_init_dma2rxor_slot - 2065 */ 2066 static void ppc440spe_adma_init_dma2rxor_slot( 2067 struct ppc440spe_adma_desc_slot *desc, 2068 dma_addr_t *src, int src_cnt) 2069 { 2070 int i; 2071 2072 /* initialize CDB */ 2073 for (i = 0; i < src_cnt; i++) { 2074 ppc440spe_adma_dma2rxor_prep_src(desc, &desc->rxor_cursor, i, 2075 desc->src_cnt, (u32)src[i]); 2076 } 2077 } 2078 2079 /** 2080 * ppc440spe_dma01_prep_mult - 2081 * for Q operation where destination is also the source 2082 */ 2083 static struct ppc440spe_adma_desc_slot *ppc440spe_dma01_prep_mult( 2084 struct ppc440spe_adma_chan *ppc440spe_chan, 2085 dma_addr_t *dst, int dst_cnt, dma_addr_t *src, int src_cnt, 2086 const unsigned char *scf, size_t len, unsigned long flags) 2087 { 2088 struct ppc440spe_adma_desc_slot *sw_desc = NULL; 2089 unsigned long op = 0; 2090 int slot_cnt; 2091 2092 set_bit(PPC440SPE_DESC_WXOR, &op); 2093 slot_cnt = 2; 2094 2095 spin_lock_bh(&ppc440spe_chan->lock); 2096 2097 /* use WXOR, each descriptor occupies one slot */ 2098 sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1); 2099 if (sw_desc) { 2100 struct ppc440spe_adma_chan *chan; 2101 struct ppc440spe_adma_desc_slot *iter; 2102 struct dma_cdb *hw_desc; 2103 2104 chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan); 2105 set_bits(op, &sw_desc->flags); 2106 sw_desc->src_cnt = src_cnt; 2107 sw_desc->dst_cnt = dst_cnt; 2108 /* First descriptor, zero data in the destination and copy it 2109 * to q page using MULTICAST transfer. 2110 */ 2111 iter = list_first_entry(&sw_desc->group_list, 2112 struct ppc440spe_adma_desc_slot, 2113 chain_node); 2114 memset(iter->hw_desc, 0, sizeof(struct dma_cdb)); 2115 /* set 'next' pointer */ 2116 iter->hw_next = list_entry(iter->chain_node.next, 2117 struct ppc440spe_adma_desc_slot, 2118 chain_node); 2119 clear_bit(PPC440SPE_DESC_INT, &iter->flags); 2120 hw_desc = iter->hw_desc; 2121 hw_desc->opc = DMA_CDB_OPC_MULTICAST; 2122 2123 ppc440spe_desc_set_dest_addr(iter, chan, 2124 DMA_CUED_XOR_BASE, dst[0], 0); 2125 ppc440spe_desc_set_dest_addr(iter, chan, 0, dst[1], 1); 2126 ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB, 2127 src[0]); 2128 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len); 2129 iter->unmap_len = len; 2130 2131 /* 2132 * Second descriptor, multiply data from the q page 2133 * and store the result in real destination. 2134 */ 2135 iter = list_first_entry(&iter->chain_node, 2136 struct ppc440spe_adma_desc_slot, 2137 chain_node); 2138 memset(iter->hw_desc, 0, sizeof(struct dma_cdb)); 2139 iter->hw_next = NULL; 2140 if (flags & DMA_PREP_INTERRUPT) 2141 set_bit(PPC440SPE_DESC_INT, &iter->flags); 2142 else 2143 clear_bit(PPC440SPE_DESC_INT, &iter->flags); 2144 2145 hw_desc = iter->hw_desc; 2146 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; 2147 ppc440spe_desc_set_src_addr(iter, chan, 0, 2148 DMA_CUED_XOR_HB, dst[1]); 2149 ppc440spe_desc_set_dest_addr(iter, chan, 2150 DMA_CUED_XOR_BASE, dst[0], 0); 2151 2152 ppc440spe_desc_set_src_mult(iter, chan, DMA_CUED_MULT1_OFF, 2153 DMA_CDB_SG_DST1, scf[0]); 2154 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len); 2155 iter->unmap_len = len; 2156 sw_desc->async_tx.flags = flags; 2157 } 2158 2159 spin_unlock_bh(&ppc440spe_chan->lock); 2160 2161 return sw_desc; 2162 } 2163 2164 /** 2165 * ppc440spe_dma01_prep_sum_product - 2166 * Dx = A*(P+Pxy) + B*(Q+Qxy) operation where destination is also 2167 * the source. 2168 */ 2169 static struct ppc440spe_adma_desc_slot *ppc440spe_dma01_prep_sum_product( 2170 struct ppc440spe_adma_chan *ppc440spe_chan, 2171 dma_addr_t *dst, dma_addr_t *src, int src_cnt, 2172 const unsigned char *scf, size_t len, unsigned long flags) 2173 { 2174 struct ppc440spe_adma_desc_slot *sw_desc = NULL; 2175 unsigned long op = 0; 2176 int slot_cnt; 2177 2178 set_bit(PPC440SPE_DESC_WXOR, &op); 2179 slot_cnt = 3; 2180 2181 spin_lock_bh(&ppc440spe_chan->lock); 2182 2183 /* WXOR, each descriptor occupies one slot */ 2184 sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1); 2185 if (sw_desc) { 2186 struct ppc440spe_adma_chan *chan; 2187 struct ppc440spe_adma_desc_slot *iter; 2188 struct dma_cdb *hw_desc; 2189 2190 chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan); 2191 set_bits(op, &sw_desc->flags); 2192 sw_desc->src_cnt = src_cnt; 2193 sw_desc->dst_cnt = 1; 2194 /* 1st descriptor, src[1] data to q page and zero destination */ 2195 iter = list_first_entry(&sw_desc->group_list, 2196 struct ppc440spe_adma_desc_slot, 2197 chain_node); 2198 memset(iter->hw_desc, 0, sizeof(struct dma_cdb)); 2199 iter->hw_next = list_entry(iter->chain_node.next, 2200 struct ppc440spe_adma_desc_slot, 2201 chain_node); 2202 clear_bit(PPC440SPE_DESC_INT, &iter->flags); 2203 hw_desc = iter->hw_desc; 2204 hw_desc->opc = DMA_CDB_OPC_MULTICAST; 2205 2206 ppc440spe_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE, 2207 *dst, 0); 2208 ppc440spe_desc_set_dest_addr(iter, chan, 0, 2209 ppc440spe_chan->qdest, 1); 2210 ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB, 2211 src[1]); 2212 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len); 2213 iter->unmap_len = len; 2214 2215 /* 2nd descriptor, multiply src[1] data and store the 2216 * result in destination */ 2217 iter = list_first_entry(&iter->chain_node, 2218 struct ppc440spe_adma_desc_slot, 2219 chain_node); 2220 memset(iter->hw_desc, 0, sizeof(struct dma_cdb)); 2221 /* set 'next' pointer */ 2222 iter->hw_next = list_entry(iter->chain_node.next, 2223 struct ppc440spe_adma_desc_slot, 2224 chain_node); 2225 if (flags & DMA_PREP_INTERRUPT) 2226 set_bit(PPC440SPE_DESC_INT, &iter->flags); 2227 else 2228 clear_bit(PPC440SPE_DESC_INT, &iter->flags); 2229 2230 hw_desc = iter->hw_desc; 2231 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; 2232 ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB, 2233 ppc440spe_chan->qdest); 2234 ppc440spe_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE, 2235 *dst, 0); 2236 ppc440spe_desc_set_src_mult(iter, chan, DMA_CUED_MULT1_OFF, 2237 DMA_CDB_SG_DST1, scf[1]); 2238 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len); 2239 iter->unmap_len = len; 2240 2241 /* 2242 * 3rd descriptor, multiply src[0] data and xor it 2243 * with destination 2244 */ 2245 iter = list_first_entry(&iter->chain_node, 2246 struct ppc440spe_adma_desc_slot, 2247 chain_node); 2248 memset(iter->hw_desc, 0, sizeof(struct dma_cdb)); 2249 iter->hw_next = NULL; 2250 if (flags & DMA_PREP_INTERRUPT) 2251 set_bit(PPC440SPE_DESC_INT, &iter->flags); 2252 else 2253 clear_bit(PPC440SPE_DESC_INT, &iter->flags); 2254 2255 hw_desc = iter->hw_desc; 2256 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; 2257 ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB, 2258 src[0]); 2259 ppc440spe_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE, 2260 *dst, 0); 2261 ppc440spe_desc_set_src_mult(iter, chan, DMA_CUED_MULT1_OFF, 2262 DMA_CDB_SG_DST1, scf[0]); 2263 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len); 2264 iter->unmap_len = len; 2265 sw_desc->async_tx.flags = flags; 2266 } 2267 2268 spin_unlock_bh(&ppc440spe_chan->lock); 2269 2270 return sw_desc; 2271 } 2272 2273 static struct ppc440spe_adma_desc_slot *ppc440spe_dma01_prep_pq( 2274 struct ppc440spe_adma_chan *ppc440spe_chan, 2275 dma_addr_t *dst, int dst_cnt, dma_addr_t *src, int src_cnt, 2276 const unsigned char *scf, size_t len, unsigned long flags) 2277 { 2278 int slot_cnt; 2279 struct ppc440spe_adma_desc_slot *sw_desc = NULL, *iter; 2280 unsigned long op = 0; 2281 unsigned char mult = 1; 2282 2283 pr_debug("%s: dst_cnt %d, src_cnt %d, len %d\n", 2284 __func__, dst_cnt, src_cnt, len); 2285 /* select operations WXOR/RXOR depending on the 2286 * source addresses of operators and the number 2287 * of destinations (RXOR support only Q-parity calculations) 2288 */ 2289 set_bit(PPC440SPE_DESC_WXOR, &op); 2290 if (!test_and_set_bit(PPC440SPE_RXOR_RUN, &ppc440spe_rxor_state)) { 2291 /* no active RXOR; 2292 * do RXOR if: 2293 * - there are more than 1 source, 2294 * - len is aligned on 512-byte boundary, 2295 * - source addresses fit to one of 4 possible regions. 2296 */ 2297 if (src_cnt > 1 && 2298 !(len & MQ0_CF2H_RXOR_BS_MASK) && 2299 (src[0] + len) == src[1]) { 2300 /* may do RXOR R1 R2 */ 2301 set_bit(PPC440SPE_DESC_RXOR, &op); 2302 if (src_cnt != 2) { 2303 /* may try to enhance region of RXOR */ 2304 if ((src[1] + len) == src[2]) { 2305 /* do RXOR R1 R2 R3 */ 2306 set_bit(PPC440SPE_DESC_RXOR123, 2307 &op); 2308 } else if ((src[1] + len * 2) == src[2]) { 2309 /* do RXOR R1 R2 R4 */ 2310 set_bit(PPC440SPE_DESC_RXOR124, &op); 2311 } else if ((src[1] + len * 3) == src[2]) { 2312 /* do RXOR R1 R2 R5 */ 2313 set_bit(PPC440SPE_DESC_RXOR125, 2314 &op); 2315 } else { 2316 /* do RXOR R1 R2 */ 2317 set_bit(PPC440SPE_DESC_RXOR12, 2318 &op); 2319 } 2320 } else { 2321 /* do RXOR R1 R2 */ 2322 set_bit(PPC440SPE_DESC_RXOR12, &op); 2323 } 2324 } 2325 2326 if (!test_bit(PPC440SPE_DESC_RXOR, &op)) { 2327 /* can not do this operation with RXOR */ 2328 clear_bit(PPC440SPE_RXOR_RUN, 2329 &ppc440spe_rxor_state); 2330 } else { 2331 /* can do; set block size right now */ 2332 ppc440spe_desc_set_rxor_block_size(len); 2333 } 2334 } 2335 2336 /* Number of necessary slots depends on operation type selected */ 2337 if (!test_bit(PPC440SPE_DESC_RXOR, &op)) { 2338 /* This is a WXOR only chain. Need descriptors for each 2339 * source to GF-XOR them with WXOR, and need descriptors 2340 * for each destination to zero them with WXOR 2341 */ 2342 slot_cnt = src_cnt; 2343 2344 if (flags & DMA_PREP_ZERO_P) { 2345 slot_cnt++; 2346 set_bit(PPC440SPE_ZERO_P, &op); 2347 } 2348 if (flags & DMA_PREP_ZERO_Q) { 2349 slot_cnt++; 2350 set_bit(PPC440SPE_ZERO_Q, &op); 2351 } 2352 } else { 2353 /* Need 1/2 descriptor for RXOR operation, and 2354 * need (src_cnt - (2 or 3)) for WXOR of sources 2355 * remained (if any) 2356 */ 2357 slot_cnt = dst_cnt; 2358 2359 if (flags & DMA_PREP_ZERO_P) 2360 set_bit(PPC440SPE_ZERO_P, &op); 2361 if (flags & DMA_PREP_ZERO_Q) 2362 set_bit(PPC440SPE_ZERO_Q, &op); 2363 2364 if (test_bit(PPC440SPE_DESC_RXOR12, &op)) 2365 slot_cnt += src_cnt - 2; 2366 else 2367 slot_cnt += src_cnt - 3; 2368 2369 /* Thus we have either RXOR only chain or 2370 * mixed RXOR/WXOR 2371 */ 2372 if (slot_cnt == dst_cnt) 2373 /* RXOR only chain */ 2374 clear_bit(PPC440SPE_DESC_WXOR, &op); 2375 } 2376 2377 spin_lock_bh(&ppc440spe_chan->lock); 2378 /* for both RXOR/WXOR each descriptor occupies one slot */ 2379 sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1); 2380 if (sw_desc) { 2381 ppc440spe_desc_init_dma01pq(sw_desc, dst_cnt, src_cnt, 2382 flags, op); 2383 2384 /* setup dst/src/mult */ 2385 pr_debug("%s: set dst descriptor 0, 1: 0x%016llx, 0x%016llx\n", 2386 __func__, dst[0], dst[1]); 2387 ppc440spe_adma_pq_set_dest(sw_desc, dst, flags); 2388 while (src_cnt--) { 2389 ppc440spe_adma_pq_set_src(sw_desc, src[src_cnt], 2390 src_cnt); 2391 2392 /* NOTE: "Multi = 0 is equivalent to = 1" as it 2393 * stated in 440SPSPe_RAID6_Addendum_UM_1_17.pdf 2394 * doesn't work for RXOR with DMA0/1! Instead, multi=0 2395 * leads to zeroing source data after RXOR. 2396 * So, for P case set-up mult=1 explicitly. 2397 */ 2398 if (!(flags & DMA_PREP_PQ_DISABLE_Q)) 2399 mult = scf[src_cnt]; 2400 ppc440spe_adma_pq_set_src_mult(sw_desc, 2401 mult, src_cnt, dst_cnt - 1); 2402 } 2403 2404 /* Setup byte count foreach slot just allocated */ 2405 sw_desc->async_tx.flags = flags; 2406 list_for_each_entry(iter, &sw_desc->group_list, 2407 chain_node) { 2408 ppc440spe_desc_set_byte_count(iter, 2409 ppc440spe_chan, len); 2410 iter->unmap_len = len; 2411 } 2412 } 2413 spin_unlock_bh(&ppc440spe_chan->lock); 2414 2415 return sw_desc; 2416 } 2417 2418 static struct ppc440spe_adma_desc_slot *ppc440spe_dma2_prep_pq( 2419 struct ppc440spe_adma_chan *ppc440spe_chan, 2420 dma_addr_t *dst, int dst_cnt, dma_addr_t *src, int src_cnt, 2421 const unsigned char *scf, size_t len, unsigned long flags) 2422 { 2423 int slot_cnt, descs_per_op; 2424 struct ppc440spe_adma_desc_slot *sw_desc = NULL, *iter; 2425 unsigned long op = 0; 2426 unsigned char mult = 1; 2427 2428 BUG_ON(!dst_cnt); 2429 /*pr_debug("%s: dst_cnt %d, src_cnt %d, len %d\n", 2430 __func__, dst_cnt, src_cnt, len);*/ 2431 2432 spin_lock_bh(&ppc440spe_chan->lock); 2433 descs_per_op = ppc440spe_dma2_pq_slot_count(src, src_cnt, len); 2434 if (descs_per_op < 0) { 2435 spin_unlock_bh(&ppc440spe_chan->lock); 2436 return NULL; 2437 } 2438 2439 /* depending on number of sources we have 1 or 2 RXOR chains */ 2440 slot_cnt = descs_per_op * dst_cnt; 2441 2442 sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1); 2443 if (sw_desc) { 2444 op = slot_cnt; 2445 sw_desc->async_tx.flags = flags; 2446 list_for_each_entry(iter, &sw_desc->group_list, chain_node) { 2447 ppc440spe_desc_init_dma2pq(iter, dst_cnt, src_cnt, 2448 --op ? 0 : flags); 2449 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, 2450 len); 2451 iter->unmap_len = len; 2452 2453 ppc440spe_init_rxor_cursor(&(iter->rxor_cursor)); 2454 iter->rxor_cursor.len = len; 2455 iter->descs_per_op = descs_per_op; 2456 } 2457 op = 0; 2458 list_for_each_entry(iter, &sw_desc->group_list, chain_node) { 2459 op++; 2460 if (op % descs_per_op == 0) 2461 ppc440spe_adma_init_dma2rxor_slot(iter, src, 2462 src_cnt); 2463 if (likely(!list_is_last(&iter->chain_node, 2464 &sw_desc->group_list))) { 2465 /* set 'next' pointer */ 2466 iter->hw_next = 2467 list_entry(iter->chain_node.next, 2468 struct ppc440spe_adma_desc_slot, 2469 chain_node); 2470 ppc440spe_xor_set_link(iter, iter->hw_next); 2471 } else { 2472 /* this is the last descriptor. */ 2473 iter->hw_next = NULL; 2474 } 2475 } 2476 2477 /* fixup head descriptor */ 2478 sw_desc->dst_cnt = dst_cnt; 2479 if (flags & DMA_PREP_ZERO_P) 2480 set_bit(PPC440SPE_ZERO_P, &sw_desc->flags); 2481 if (flags & DMA_PREP_ZERO_Q) 2482 set_bit(PPC440SPE_ZERO_Q, &sw_desc->flags); 2483 2484 /* setup dst/src/mult */ 2485 ppc440spe_adma_pq_set_dest(sw_desc, dst, flags); 2486 2487 while (src_cnt--) { 2488 /* handle descriptors (if dst_cnt == 2) inside 2489 * the ppc440spe_adma_pq_set_srcxxx() functions 2490 */ 2491 ppc440spe_adma_pq_set_src(sw_desc, src[src_cnt], 2492 src_cnt); 2493 if (!(flags & DMA_PREP_PQ_DISABLE_Q)) 2494 mult = scf[src_cnt]; 2495 ppc440spe_adma_pq_set_src_mult(sw_desc, 2496 mult, src_cnt, dst_cnt - 1); 2497 } 2498 } 2499 spin_unlock_bh(&ppc440spe_chan->lock); 2500 ppc440spe_desc_set_rxor_block_size(len); 2501 return sw_desc; 2502 } 2503 2504 /** 2505 * ppc440spe_adma_prep_dma_pq - prepare CDB (group) for a GF-XOR operation 2506 */ 2507 static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_pq( 2508 struct dma_chan *chan, dma_addr_t *dst, dma_addr_t *src, 2509 unsigned int src_cnt, const unsigned char *scf, 2510 size_t len, unsigned long flags) 2511 { 2512 struct ppc440spe_adma_chan *ppc440spe_chan; 2513 struct ppc440spe_adma_desc_slot *sw_desc = NULL; 2514 int dst_cnt = 0; 2515 2516 ppc440spe_chan = to_ppc440spe_adma_chan(chan); 2517 2518 ADMA_LL_DBG(prep_dma_pq_dbg(ppc440spe_chan->device->id, 2519 dst, src, src_cnt)); 2520 BUG_ON(!len); 2521 BUG_ON(len > PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT); 2522 BUG_ON(!src_cnt); 2523 2524 if (src_cnt == 1 && dst[1] == src[0]) { 2525 dma_addr_t dest[2]; 2526 2527 /* dst[1] is real destination (Q) */ 2528 dest[0] = dst[1]; 2529 /* this is the page to multicast source data to */ 2530 dest[1] = ppc440spe_chan->qdest; 2531 sw_desc = ppc440spe_dma01_prep_mult(ppc440spe_chan, 2532 dest, 2, src, src_cnt, scf, len, flags); 2533 return sw_desc ? &sw_desc->async_tx : NULL; 2534 } 2535 2536 if (src_cnt == 2 && dst[1] == src[1]) { 2537 sw_desc = ppc440spe_dma01_prep_sum_product(ppc440spe_chan, 2538 &dst[1], src, 2, scf, len, flags); 2539 return sw_desc ? &sw_desc->async_tx : NULL; 2540 } 2541 2542 if (!(flags & DMA_PREP_PQ_DISABLE_P)) { 2543 BUG_ON(!dst[0]); 2544 dst_cnt++; 2545 flags |= DMA_PREP_ZERO_P; 2546 } 2547 2548 if (!(flags & DMA_PREP_PQ_DISABLE_Q)) { 2549 BUG_ON(!dst[1]); 2550 dst_cnt++; 2551 flags |= DMA_PREP_ZERO_Q; 2552 } 2553 2554 BUG_ON(!dst_cnt); 2555 2556 dev_dbg(ppc440spe_chan->device->common.dev, 2557 "ppc440spe adma%d: %s src_cnt: %d len: %u int_en: %d\n", 2558 ppc440spe_chan->device->id, __func__, src_cnt, len, 2559 flags & DMA_PREP_INTERRUPT ? 1 : 0); 2560 2561 switch (ppc440spe_chan->device->id) { 2562 case PPC440SPE_DMA0_ID: 2563 case PPC440SPE_DMA1_ID: 2564 sw_desc = ppc440spe_dma01_prep_pq(ppc440spe_chan, 2565 dst, dst_cnt, src, src_cnt, scf, 2566 len, flags); 2567 break; 2568 2569 case PPC440SPE_XOR_ID: 2570 sw_desc = ppc440spe_dma2_prep_pq(ppc440spe_chan, 2571 dst, dst_cnt, src, src_cnt, scf, 2572 len, flags); 2573 break; 2574 } 2575 2576 return sw_desc ? &sw_desc->async_tx : NULL; 2577 } 2578 2579 /** 2580 * ppc440spe_adma_prep_dma_pqzero_sum - prepare CDB group for 2581 * a PQ_ZERO_SUM operation 2582 */ 2583 static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_pqzero_sum( 2584 struct dma_chan *chan, dma_addr_t *pq, dma_addr_t *src, 2585 unsigned int src_cnt, const unsigned char *scf, size_t len, 2586 enum sum_check_flags *pqres, unsigned long flags) 2587 { 2588 struct ppc440spe_adma_chan *ppc440spe_chan; 2589 struct ppc440spe_adma_desc_slot *sw_desc, *iter; 2590 dma_addr_t pdest, qdest; 2591 int slot_cnt, slots_per_op, idst, dst_cnt; 2592 2593 ppc440spe_chan = to_ppc440spe_adma_chan(chan); 2594 2595 if (flags & DMA_PREP_PQ_DISABLE_P) 2596 pdest = 0; 2597 else 2598 pdest = pq[0]; 2599 2600 if (flags & DMA_PREP_PQ_DISABLE_Q) 2601 qdest = 0; 2602 else 2603 qdest = pq[1]; 2604 2605 ADMA_LL_DBG(prep_dma_pqzero_sum_dbg(ppc440spe_chan->device->id, 2606 src, src_cnt, scf)); 2607 2608 /* Always use WXOR for P/Q calculations (two destinations). 2609 * Need 1 or 2 extra slots to verify results are zero. 2610 */ 2611 idst = dst_cnt = (pdest && qdest) ? 2 : 1; 2612 2613 /* One additional slot per destination to clone P/Q 2614 * before calculation (we have to preserve destinations). 2615 */ 2616 slot_cnt = src_cnt + dst_cnt * 2; 2617 slots_per_op = 1; 2618 2619 spin_lock_bh(&ppc440spe_chan->lock); 2620 sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 2621 slots_per_op); 2622 if (sw_desc) { 2623 ppc440spe_desc_init_dma01pqzero_sum(sw_desc, dst_cnt, src_cnt); 2624 2625 /* Setup byte count for each slot just allocated */ 2626 sw_desc->async_tx.flags = flags; 2627 list_for_each_entry(iter, &sw_desc->group_list, chain_node) { 2628 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, 2629 len); 2630 iter->unmap_len = len; 2631 } 2632 2633 if (pdest) { 2634 struct dma_cdb *hw_desc; 2635 struct ppc440spe_adma_chan *chan; 2636 2637 iter = sw_desc->group_head; 2638 chan = to_ppc440spe_adma_chan(iter->async_tx.chan); 2639 memset(iter->hw_desc, 0, sizeof(struct dma_cdb)); 2640 iter->hw_next = list_entry(iter->chain_node.next, 2641 struct ppc440spe_adma_desc_slot, 2642 chain_node); 2643 hw_desc = iter->hw_desc; 2644 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; 2645 iter->src_cnt = 0; 2646 iter->dst_cnt = 0; 2647 ppc440spe_desc_set_dest_addr(iter, chan, 0, 2648 ppc440spe_chan->pdest, 0); 2649 ppc440spe_desc_set_src_addr(iter, chan, 0, 0, pdest); 2650 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, 2651 len); 2652 iter->unmap_len = 0; 2653 /* override pdest to preserve original P */ 2654 pdest = ppc440spe_chan->pdest; 2655 } 2656 if (qdest) { 2657 struct dma_cdb *hw_desc; 2658 struct ppc440spe_adma_chan *chan; 2659 2660 iter = list_first_entry(&sw_desc->group_list, 2661 struct ppc440spe_adma_desc_slot, 2662 chain_node); 2663 chan = to_ppc440spe_adma_chan(iter->async_tx.chan); 2664 2665 if (pdest) { 2666 iter = list_entry(iter->chain_node.next, 2667 struct ppc440spe_adma_desc_slot, 2668 chain_node); 2669 } 2670 2671 memset(iter->hw_desc, 0, sizeof(struct dma_cdb)); 2672 iter->hw_next = list_entry(iter->chain_node.next, 2673 struct ppc440spe_adma_desc_slot, 2674 chain_node); 2675 hw_desc = iter->hw_desc; 2676 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; 2677 iter->src_cnt = 0; 2678 iter->dst_cnt = 0; 2679 ppc440spe_desc_set_dest_addr(iter, chan, 0, 2680 ppc440spe_chan->qdest, 0); 2681 ppc440spe_desc_set_src_addr(iter, chan, 0, 0, qdest); 2682 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, 2683 len); 2684 iter->unmap_len = 0; 2685 /* override qdest to preserve original Q */ 2686 qdest = ppc440spe_chan->qdest; 2687 } 2688 2689 /* Setup destinations for P/Q ops */ 2690 ppc440spe_adma_pqzero_sum_set_dest(sw_desc, pdest, qdest); 2691 2692 /* Setup zero QWORDs into DCHECK CDBs */ 2693 idst = dst_cnt; 2694 list_for_each_entry_reverse(iter, &sw_desc->group_list, 2695 chain_node) { 2696 /* 2697 * The last CDB corresponds to Q-parity check, 2698 * the one before last CDB corresponds 2699 * P-parity check 2700 */ 2701 if (idst == DMA_DEST_MAX_NUM) { 2702 if (idst == dst_cnt) { 2703 set_bit(PPC440SPE_DESC_QCHECK, 2704 &iter->flags); 2705 } else { 2706 set_bit(PPC440SPE_DESC_PCHECK, 2707 &iter->flags); 2708 } 2709 } else { 2710 if (qdest) { 2711 set_bit(PPC440SPE_DESC_QCHECK, 2712 &iter->flags); 2713 } else { 2714 set_bit(PPC440SPE_DESC_PCHECK, 2715 &iter->flags); 2716 } 2717 } 2718 iter->xor_check_result = pqres; 2719 2720 /* 2721 * set it to zero, if check fail then result will 2722 * be updated 2723 */ 2724 *iter->xor_check_result = 0; 2725 ppc440spe_desc_set_dcheck(iter, ppc440spe_chan, 2726 ppc440spe_qword); 2727 2728 if (!(--dst_cnt)) 2729 break; 2730 } 2731 2732 /* Setup sources and mults for P/Q ops */ 2733 list_for_each_entry_continue_reverse(iter, &sw_desc->group_list, 2734 chain_node) { 2735 struct ppc440spe_adma_chan *chan; 2736 u32 mult_dst; 2737 2738 chan = to_ppc440spe_adma_chan(iter->async_tx.chan); 2739 ppc440spe_desc_set_src_addr(iter, chan, 0, 2740 DMA_CUED_XOR_HB, 2741 src[src_cnt - 1]); 2742 if (qdest) { 2743 mult_dst = (dst_cnt - 1) ? DMA_CDB_SG_DST2 : 2744 DMA_CDB_SG_DST1; 2745 ppc440spe_desc_set_src_mult(iter, chan, 2746 DMA_CUED_MULT1_OFF, 2747 mult_dst, 2748 scf[src_cnt - 1]); 2749 } 2750 if (!(--src_cnt)) 2751 break; 2752 } 2753 } 2754 spin_unlock_bh(&ppc440spe_chan->lock); 2755 return sw_desc ? &sw_desc->async_tx : NULL; 2756 } 2757 2758 /** 2759 * ppc440spe_adma_prep_dma_xor_zero_sum - prepare CDB group for 2760 * XOR ZERO_SUM operation 2761 */ 2762 static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_xor_zero_sum( 2763 struct dma_chan *chan, dma_addr_t *src, unsigned int src_cnt, 2764 size_t len, enum sum_check_flags *result, unsigned long flags) 2765 { 2766 struct dma_async_tx_descriptor *tx; 2767 dma_addr_t pq[2]; 2768 2769 /* validate P, disable Q */ 2770 pq[0] = src[0]; 2771 pq[1] = 0; 2772 flags |= DMA_PREP_PQ_DISABLE_Q; 2773 2774 tx = ppc440spe_adma_prep_dma_pqzero_sum(chan, pq, &src[1], 2775 src_cnt - 1, 0, len, 2776 result, flags); 2777 return tx; 2778 } 2779 2780 /** 2781 * ppc440spe_adma_set_dest - set destination address into descriptor 2782 */ 2783 static void ppc440spe_adma_set_dest(struct ppc440spe_adma_desc_slot *sw_desc, 2784 dma_addr_t addr, int index) 2785 { 2786 struct ppc440spe_adma_chan *chan; 2787 2788 BUG_ON(index >= sw_desc->dst_cnt); 2789 2790 chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan); 2791 2792 switch (chan->device->id) { 2793 case PPC440SPE_DMA0_ID: 2794 case PPC440SPE_DMA1_ID: 2795 /* to do: support transfers lengths > 2796 * PPC440SPE_ADMA_DMA/XOR_MAX_BYTE_COUNT 2797 */ 2798 ppc440spe_desc_set_dest_addr(sw_desc->group_head, 2799 chan, 0, addr, index); 2800 break; 2801 case PPC440SPE_XOR_ID: 2802 sw_desc = ppc440spe_get_group_entry(sw_desc, index); 2803 ppc440spe_desc_set_dest_addr(sw_desc, 2804 chan, 0, addr, index); 2805 break; 2806 } 2807 } 2808 2809 static void ppc440spe_adma_pq_zero_op(struct ppc440spe_adma_desc_slot *iter, 2810 struct ppc440spe_adma_chan *chan, dma_addr_t addr) 2811 { 2812 /* To clear destinations update the descriptor 2813 * (P or Q depending on index) as follows: 2814 * addr is destination (0 corresponds to SG2): 2815 */ 2816 ppc440spe_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE, addr, 0); 2817 2818 /* ... and the addr is source: */ 2819 ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB, addr); 2820 2821 /* addr is always SG2 then the mult is always DST1 */ 2822 ppc440spe_desc_set_src_mult(iter, chan, DMA_CUED_MULT1_OFF, 2823 DMA_CDB_SG_DST1, 1); 2824 } 2825 2826 /** 2827 * ppc440spe_adma_pq_set_dest - set destination address into descriptor 2828 * for the PQXOR operation 2829 */ 2830 static void ppc440spe_adma_pq_set_dest(struct ppc440spe_adma_desc_slot *sw_desc, 2831 dma_addr_t *addrs, unsigned long flags) 2832 { 2833 struct ppc440spe_adma_desc_slot *iter; 2834 struct ppc440spe_adma_chan *chan; 2835 dma_addr_t paddr, qaddr; 2836 dma_addr_t addr = 0, ppath, qpath; 2837 int index = 0, i; 2838 2839 chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan); 2840 2841 if (flags & DMA_PREP_PQ_DISABLE_P) 2842 paddr = 0; 2843 else 2844 paddr = addrs[0]; 2845 2846 if (flags & DMA_PREP_PQ_DISABLE_Q) 2847 qaddr = 0; 2848 else 2849 qaddr = addrs[1]; 2850 2851 if (!paddr || !qaddr) 2852 addr = paddr ? paddr : qaddr; 2853 2854 switch (chan->device->id) { 2855 case PPC440SPE_DMA0_ID: 2856 case PPC440SPE_DMA1_ID: 2857 /* walk through the WXOR source list and set P/Q-destinations 2858 * for each slot: 2859 */ 2860 if (!test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags)) { 2861 /* This is WXOR-only chain; may have 1/2 zero descs */ 2862 if (test_bit(PPC440SPE_ZERO_P, &sw_desc->flags)) 2863 index++; 2864 if (test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags)) 2865 index++; 2866 2867 iter = ppc440spe_get_group_entry(sw_desc, index); 2868 if (addr) { 2869 /* one destination */ 2870 list_for_each_entry_from(iter, 2871 &sw_desc->group_list, chain_node) 2872 ppc440spe_desc_set_dest_addr(iter, chan, 2873 DMA_CUED_XOR_BASE, addr, 0); 2874 } else { 2875 /* two destinations */ 2876 list_for_each_entry_from(iter, 2877 &sw_desc->group_list, chain_node) { 2878 ppc440spe_desc_set_dest_addr(iter, chan, 2879 DMA_CUED_XOR_BASE, paddr, 0); 2880 ppc440spe_desc_set_dest_addr(iter, chan, 2881 DMA_CUED_XOR_BASE, qaddr, 1); 2882 } 2883 } 2884 2885 if (index) { 2886 /* To clear destinations update the descriptor 2887 * (1st,2nd, or both depending on flags) 2888 */ 2889 index = 0; 2890 if (test_bit(PPC440SPE_ZERO_P, 2891 &sw_desc->flags)) { 2892 iter = ppc440spe_get_group_entry( 2893 sw_desc, index++); 2894 ppc440spe_adma_pq_zero_op(iter, chan, 2895 paddr); 2896 } 2897 2898 if (test_bit(PPC440SPE_ZERO_Q, 2899 &sw_desc->flags)) { 2900 iter = ppc440spe_get_group_entry( 2901 sw_desc, index++); 2902 ppc440spe_adma_pq_zero_op(iter, chan, 2903 qaddr); 2904 } 2905 2906 return; 2907 } 2908 } else { 2909 /* This is RXOR-only or RXOR/WXOR mixed chain */ 2910 2911 /* If we want to include destination into calculations, 2912 * then make dest addresses cued with mult=1 (XOR). 2913 */ 2914 ppath = test_bit(PPC440SPE_ZERO_P, &sw_desc->flags) ? 2915 DMA_CUED_XOR_HB : 2916 DMA_CUED_XOR_BASE | 2917 (1 << DMA_CUED_MULT1_OFF); 2918 qpath = test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags) ? 2919 DMA_CUED_XOR_HB : 2920 DMA_CUED_XOR_BASE | 2921 (1 << DMA_CUED_MULT1_OFF); 2922 2923 /* Setup destination(s) in RXOR slot(s) */ 2924 iter = ppc440spe_get_group_entry(sw_desc, index++); 2925 ppc440spe_desc_set_dest_addr(iter, chan, 2926 paddr ? ppath : qpath, 2927 paddr ? paddr : qaddr, 0); 2928 if (!addr) { 2929 /* two destinations */ 2930 iter = ppc440spe_get_group_entry(sw_desc, 2931 index++); 2932 ppc440spe_desc_set_dest_addr(iter, chan, 2933 qpath, qaddr, 0); 2934 } 2935 2936 if (test_bit(PPC440SPE_DESC_WXOR, &sw_desc->flags)) { 2937 /* Setup destination(s) in remaining WXOR 2938 * slots 2939 */ 2940 iter = ppc440spe_get_group_entry(sw_desc, 2941 index); 2942 if (addr) { 2943 /* one destination */ 2944 list_for_each_entry_from(iter, 2945 &sw_desc->group_list, 2946 chain_node) 2947 ppc440spe_desc_set_dest_addr( 2948 iter, chan, 2949 DMA_CUED_XOR_BASE, 2950 addr, 0); 2951 2952 } else { 2953 /* two destinations */ 2954 list_for_each_entry_from(iter, 2955 &sw_desc->group_list, 2956 chain_node) { 2957 ppc440spe_desc_set_dest_addr( 2958 iter, chan, 2959 DMA_CUED_XOR_BASE, 2960 paddr, 0); 2961 ppc440spe_desc_set_dest_addr( 2962 iter, chan, 2963 DMA_CUED_XOR_BASE, 2964 qaddr, 1); 2965 } 2966 } 2967 } 2968 2969 } 2970 break; 2971 2972 case PPC440SPE_XOR_ID: 2973 /* DMA2 descriptors have only 1 destination, so there are 2974 * two chains - one for each dest. 2975 * If we want to include destination into calculations, 2976 * then make dest addresses cued with mult=1 (XOR). 2977 */ 2978 ppath = test_bit(PPC440SPE_ZERO_P, &sw_desc->flags) ? 2979 DMA_CUED_XOR_HB : 2980 DMA_CUED_XOR_BASE | 2981 (1 << DMA_CUED_MULT1_OFF); 2982 2983 qpath = test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags) ? 2984 DMA_CUED_XOR_HB : 2985 DMA_CUED_XOR_BASE | 2986 (1 << DMA_CUED_MULT1_OFF); 2987 2988 iter = ppc440spe_get_group_entry(sw_desc, 0); 2989 for (i = 0; i < sw_desc->descs_per_op; i++) { 2990 ppc440spe_desc_set_dest_addr(iter, chan, 2991 paddr ? ppath : qpath, 2992 paddr ? paddr : qaddr, 0); 2993 iter = list_entry(iter->chain_node.next, 2994 struct ppc440spe_adma_desc_slot, 2995 chain_node); 2996 } 2997 2998 if (!addr) { 2999 /* Two destinations; setup Q here */ 3000 iter = ppc440spe_get_group_entry(sw_desc, 3001 sw_desc->descs_per_op); 3002 for (i = 0; i < sw_desc->descs_per_op; i++) { 3003 ppc440spe_desc_set_dest_addr(iter, 3004 chan, qpath, qaddr, 0); 3005 iter = list_entry(iter->chain_node.next, 3006 struct ppc440spe_adma_desc_slot, 3007 chain_node); 3008 } 3009 } 3010 3011 break; 3012 } 3013 } 3014 3015 /** 3016 * ppc440spe_adma_pq_zero_sum_set_dest - set destination address into descriptor 3017 * for the PQ_ZERO_SUM operation 3018 */ 3019 static void ppc440spe_adma_pqzero_sum_set_dest( 3020 struct ppc440spe_adma_desc_slot *sw_desc, 3021 dma_addr_t paddr, dma_addr_t qaddr) 3022 { 3023 struct ppc440spe_adma_desc_slot *iter, *end; 3024 struct ppc440spe_adma_chan *chan; 3025 dma_addr_t addr = 0; 3026 int idx; 3027 3028 chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan); 3029 3030 /* walk through the WXOR source list and set P/Q-destinations 3031 * for each slot 3032 */ 3033 idx = (paddr && qaddr) ? 2 : 1; 3034 /* set end */ 3035 list_for_each_entry_reverse(end, &sw_desc->group_list, 3036 chain_node) { 3037 if (!(--idx)) 3038 break; 3039 } 3040 /* set start */ 3041 idx = (paddr && qaddr) ? 2 : 1; 3042 iter = ppc440spe_get_group_entry(sw_desc, idx); 3043 3044 if (paddr && qaddr) { 3045 /* two destinations */ 3046 list_for_each_entry_from(iter, &sw_desc->group_list, 3047 chain_node) { 3048 if (unlikely(iter == end)) 3049 break; 3050 ppc440spe_desc_set_dest_addr(iter, chan, 3051 DMA_CUED_XOR_BASE, paddr, 0); 3052 ppc440spe_desc_set_dest_addr(iter, chan, 3053 DMA_CUED_XOR_BASE, qaddr, 1); 3054 } 3055 } else { 3056 /* one destination */ 3057 addr = paddr ? paddr : qaddr; 3058 list_for_each_entry_from(iter, &sw_desc->group_list, 3059 chain_node) { 3060 if (unlikely(iter == end)) 3061 break; 3062 ppc440spe_desc_set_dest_addr(iter, chan, 3063 DMA_CUED_XOR_BASE, addr, 0); 3064 } 3065 } 3066 3067 /* The remaining descriptors are DATACHECK. These have no need in 3068 * destination. Actually, these destinations are used there 3069 * as sources for check operation. So, set addr as source. 3070 */ 3071 ppc440spe_desc_set_src_addr(end, chan, 0, 0, addr ? addr : paddr); 3072 3073 if (!addr) { 3074 end = list_entry(end->chain_node.next, 3075 struct ppc440spe_adma_desc_slot, chain_node); 3076 ppc440spe_desc_set_src_addr(end, chan, 0, 0, qaddr); 3077 } 3078 } 3079 3080 /** 3081 * ppc440spe_desc_set_xor_src_cnt - set source count into descriptor 3082 */ 3083 static inline void ppc440spe_desc_set_xor_src_cnt( 3084 struct ppc440spe_adma_desc_slot *desc, 3085 int src_cnt) 3086 { 3087 struct xor_cb *hw_desc = desc->hw_desc; 3088 3089 hw_desc->cbc &= ~XOR_CDCR_OAC_MSK; 3090 hw_desc->cbc |= src_cnt; 3091 } 3092 3093 /** 3094 * ppc440spe_adma_pq_set_src - set source address into descriptor 3095 */ 3096 static void ppc440spe_adma_pq_set_src(struct ppc440spe_adma_desc_slot *sw_desc, 3097 dma_addr_t addr, int index) 3098 { 3099 struct ppc440spe_adma_chan *chan; 3100 dma_addr_t haddr = 0; 3101 struct ppc440spe_adma_desc_slot *iter = NULL; 3102 3103 chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan); 3104 3105 switch (chan->device->id) { 3106 case PPC440SPE_DMA0_ID: 3107 case PPC440SPE_DMA1_ID: 3108 /* DMA0,1 may do: WXOR, RXOR, RXOR+WXORs chain 3109 */ 3110 if (test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags)) { 3111 /* RXOR-only or RXOR/WXOR operation */ 3112 int iskip = test_bit(PPC440SPE_DESC_RXOR12, 3113 &sw_desc->flags) ? 2 : 3; 3114 3115 if (index == 0) { 3116 /* 1st slot (RXOR) */ 3117 /* setup sources region (R1-2-3, R1-2-4, 3118 * or R1-2-5) 3119 */ 3120 if (test_bit(PPC440SPE_DESC_RXOR12, 3121 &sw_desc->flags)) 3122 haddr = DMA_RXOR12 << 3123 DMA_CUED_REGION_OFF; 3124 else if (test_bit(PPC440SPE_DESC_RXOR123, 3125 &sw_desc->flags)) 3126 haddr = DMA_RXOR123 << 3127 DMA_CUED_REGION_OFF; 3128 else if (test_bit(PPC440SPE_DESC_RXOR124, 3129 &sw_desc->flags)) 3130 haddr = DMA_RXOR124 << 3131 DMA_CUED_REGION_OFF; 3132 else if (test_bit(PPC440SPE_DESC_RXOR125, 3133 &sw_desc->flags)) 3134 haddr = DMA_RXOR125 << 3135 DMA_CUED_REGION_OFF; 3136 else 3137 BUG(); 3138 haddr |= DMA_CUED_XOR_BASE; 3139 iter = ppc440spe_get_group_entry(sw_desc, 0); 3140 } else if (index < iskip) { 3141 /* 1st slot (RXOR) 3142 * shall actually set source address only once 3143 * instead of first <iskip> 3144 */ 3145 iter = NULL; 3146 } else { 3147 /* 2nd/3d and next slots (WXOR); 3148 * skip first slot with RXOR 3149 */ 3150 haddr = DMA_CUED_XOR_HB; 3151 iter = ppc440spe_get_group_entry(sw_desc, 3152 index - iskip + sw_desc->dst_cnt); 3153 } 3154 } else { 3155 int znum = 0; 3156 3157 /* WXOR-only operation; skip first slots with 3158 * zeroing destinations 3159 */ 3160 if (test_bit(PPC440SPE_ZERO_P, &sw_desc->flags)) 3161 znum++; 3162 if (test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags)) 3163 znum++; 3164 3165 haddr = DMA_CUED_XOR_HB; 3166 iter = ppc440spe_get_group_entry(sw_desc, 3167 index + znum); 3168 } 3169 3170 if (likely(iter)) { 3171 ppc440spe_desc_set_src_addr(iter, chan, 0, haddr, addr); 3172 3173 if (!index && 3174 test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags) && 3175 sw_desc->dst_cnt == 2) { 3176 /* if we have two destinations for RXOR, then 3177 * setup source in the second descr too 3178 */ 3179 iter = ppc440spe_get_group_entry(sw_desc, 1); 3180 ppc440spe_desc_set_src_addr(iter, chan, 0, 3181 haddr, addr); 3182 } 3183 } 3184 break; 3185 3186 case PPC440SPE_XOR_ID: 3187 /* DMA2 may do Biskup */ 3188 iter = sw_desc->group_head; 3189 if (iter->dst_cnt == 2) { 3190 /* both P & Q calculations required; set P src here */ 3191 ppc440spe_adma_dma2rxor_set_src(iter, index, addr); 3192 3193 /* this is for Q */ 3194 iter = ppc440spe_get_group_entry(sw_desc, 3195 sw_desc->descs_per_op); 3196 } 3197 ppc440spe_adma_dma2rxor_set_src(iter, index, addr); 3198 break; 3199 } 3200 } 3201 3202 /** 3203 * ppc440spe_adma_memcpy_xor_set_src - set source address into descriptor 3204 */ 3205 static void ppc440spe_adma_memcpy_xor_set_src( 3206 struct ppc440spe_adma_desc_slot *sw_desc, 3207 dma_addr_t addr, int index) 3208 { 3209 struct ppc440spe_adma_chan *chan; 3210 3211 chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan); 3212 sw_desc = sw_desc->group_head; 3213 3214 if (likely(sw_desc)) 3215 ppc440spe_desc_set_src_addr(sw_desc, chan, index, 0, addr); 3216 } 3217 3218 /** 3219 * ppc440spe_adma_dma2rxor_inc_addr - 3220 */ 3221 static void ppc440spe_adma_dma2rxor_inc_addr( 3222 struct ppc440spe_adma_desc_slot *desc, 3223 struct ppc440spe_rxor *cursor, int index, int src_cnt) 3224 { 3225 cursor->addr_count++; 3226 if (index == src_cnt - 1) { 3227 ppc440spe_desc_set_xor_src_cnt(desc, cursor->addr_count); 3228 } else if (cursor->addr_count == XOR_MAX_OPS) { 3229 ppc440spe_desc_set_xor_src_cnt(desc, cursor->addr_count); 3230 cursor->addr_count = 0; 3231 cursor->desc_count++; 3232 } 3233 } 3234 3235 /** 3236 * ppc440spe_adma_dma2rxor_prep_src - setup RXOR types in DMA2 CDB 3237 */ 3238 static int ppc440spe_adma_dma2rxor_prep_src( 3239 struct ppc440spe_adma_desc_slot *hdesc, 3240 struct ppc440spe_rxor *cursor, int index, 3241 int src_cnt, u32 addr) 3242 { 3243 int rval = 0; 3244 u32 sign; 3245 struct ppc440spe_adma_desc_slot *desc = hdesc; 3246 int i; 3247 3248 for (i = 0; i < cursor->desc_count; i++) { 3249 desc = list_entry(hdesc->chain_node.next, 3250 struct ppc440spe_adma_desc_slot, 3251 chain_node); 3252 } 3253 3254 switch (cursor->state) { 3255 case 0: 3256 if (addr == cursor->addrl + cursor->len) { 3257 /* direct RXOR */ 3258 cursor->state = 1; 3259 cursor->xor_count++; 3260 if (index == src_cnt-1) { 3261 ppc440spe_rxor_set_region(desc, 3262 cursor->addr_count, 3263 DMA_RXOR12 << DMA_CUED_REGION_OFF); 3264 ppc440spe_adma_dma2rxor_inc_addr( 3265 desc, cursor, index, src_cnt); 3266 } 3267 } else if (cursor->addrl == addr + cursor->len) { 3268 /* reverse RXOR */ 3269 cursor->state = 1; 3270 cursor->xor_count++; 3271 set_bit(cursor->addr_count, &desc->reverse_flags[0]); 3272 if (index == src_cnt-1) { 3273 ppc440spe_rxor_set_region(desc, 3274 cursor->addr_count, 3275 DMA_RXOR12 << DMA_CUED_REGION_OFF); 3276 ppc440spe_adma_dma2rxor_inc_addr( 3277 desc, cursor, index, src_cnt); 3278 } 3279 } else { 3280 printk(KERN_ERR "Cannot build " 3281 "DMA2 RXOR command block.\n"); 3282 BUG(); 3283 } 3284 break; 3285 case 1: 3286 sign = test_bit(cursor->addr_count, 3287 desc->reverse_flags) 3288 ? -1 : 1; 3289 if (index == src_cnt-2 || (sign == -1 3290 && addr != cursor->addrl - 2*cursor->len)) { 3291 cursor->state = 0; 3292 cursor->xor_count = 1; 3293 cursor->addrl = addr; 3294 ppc440spe_rxor_set_region(desc, 3295 cursor->addr_count, 3296 DMA_RXOR12 << DMA_CUED_REGION_OFF); 3297 ppc440spe_adma_dma2rxor_inc_addr( 3298 desc, cursor, index, src_cnt); 3299 } else if (addr == cursor->addrl + 2*sign*cursor->len) { 3300 cursor->state = 2; 3301 cursor->xor_count = 0; 3302 ppc440spe_rxor_set_region(desc, 3303 cursor->addr_count, 3304 DMA_RXOR123 << DMA_CUED_REGION_OFF); 3305 if (index == src_cnt-1) { 3306 ppc440spe_adma_dma2rxor_inc_addr( 3307 desc, cursor, index, src_cnt); 3308 } 3309 } else if (addr == cursor->addrl + 3*cursor->len) { 3310 cursor->state = 2; 3311 cursor->xor_count = 0; 3312 ppc440spe_rxor_set_region(desc, 3313 cursor->addr_count, 3314 DMA_RXOR124 << DMA_CUED_REGION_OFF); 3315 if (index == src_cnt-1) { 3316 ppc440spe_adma_dma2rxor_inc_addr( 3317 desc, cursor, index, src_cnt); 3318 } 3319 } else if (addr == cursor->addrl + 4*cursor->len) { 3320 cursor->state = 2; 3321 cursor->xor_count = 0; 3322 ppc440spe_rxor_set_region(desc, 3323 cursor->addr_count, 3324 DMA_RXOR125 << DMA_CUED_REGION_OFF); 3325 if (index == src_cnt-1) { 3326 ppc440spe_adma_dma2rxor_inc_addr( 3327 desc, cursor, index, src_cnt); 3328 } 3329 } else { 3330 cursor->state = 0; 3331 cursor->xor_count = 1; 3332 cursor->addrl = addr; 3333 ppc440spe_rxor_set_region(desc, 3334 cursor->addr_count, 3335 DMA_RXOR12 << DMA_CUED_REGION_OFF); 3336 ppc440spe_adma_dma2rxor_inc_addr( 3337 desc, cursor, index, src_cnt); 3338 } 3339 break; 3340 case 2: 3341 cursor->state = 0; 3342 cursor->addrl = addr; 3343 cursor->xor_count++; 3344 if (index) { 3345 ppc440spe_adma_dma2rxor_inc_addr( 3346 desc, cursor, index, src_cnt); 3347 } 3348 break; 3349 } 3350 3351 return rval; 3352 } 3353 3354 /** 3355 * ppc440spe_adma_dma2rxor_set_src - set RXOR source address; it's assumed that 3356 * ppc440spe_adma_dma2rxor_prep_src() has already done prior this call 3357 */ 3358 static void ppc440spe_adma_dma2rxor_set_src( 3359 struct ppc440spe_adma_desc_slot *desc, 3360 int index, dma_addr_t addr) 3361 { 3362 struct xor_cb *xcb = desc->hw_desc; 3363 int k = 0, op = 0, lop = 0; 3364 3365 /* get the RXOR operand which corresponds to index addr */ 3366 while (op <= index) { 3367 lop = op; 3368 if (k == XOR_MAX_OPS) { 3369 k = 0; 3370 desc = list_entry(desc->chain_node.next, 3371 struct ppc440spe_adma_desc_slot, chain_node); 3372 xcb = desc->hw_desc; 3373 3374 } 3375 if ((xcb->ops[k++].h & (DMA_RXOR12 << DMA_CUED_REGION_OFF)) == 3376 (DMA_RXOR12 << DMA_CUED_REGION_OFF)) 3377 op += 2; 3378 else 3379 op += 3; 3380 } 3381 3382 BUG_ON(k < 1); 3383 3384 if (test_bit(k-1, desc->reverse_flags)) { 3385 /* reverse operand order; put last op in RXOR group */ 3386 if (index == op - 1) 3387 ppc440spe_rxor_set_src(desc, k - 1, addr); 3388 } else { 3389 /* direct operand order; put first op in RXOR group */ 3390 if (index == lop) 3391 ppc440spe_rxor_set_src(desc, k - 1, addr); 3392 } 3393 } 3394 3395 /** 3396 * ppc440spe_adma_dma2rxor_set_mult - set RXOR multipliers; it's assumed that 3397 * ppc440spe_adma_dma2rxor_prep_src() has already done prior this call 3398 */ 3399 static void ppc440spe_adma_dma2rxor_set_mult( 3400 struct ppc440spe_adma_desc_slot *desc, 3401 int index, u8 mult) 3402 { 3403 struct xor_cb *xcb = desc->hw_desc; 3404 int k = 0, op = 0, lop = 0; 3405 3406 /* get the RXOR operand which corresponds to index mult */ 3407 while (op <= index) { 3408 lop = op; 3409 if (k == XOR_MAX_OPS) { 3410 k = 0; 3411 desc = list_entry(desc->chain_node.next, 3412 struct ppc440spe_adma_desc_slot, 3413 chain_node); 3414 xcb = desc->hw_desc; 3415 3416 } 3417 if ((xcb->ops[k++].h & (DMA_RXOR12 << DMA_CUED_REGION_OFF)) == 3418 (DMA_RXOR12 << DMA_CUED_REGION_OFF)) 3419 op += 2; 3420 else 3421 op += 3; 3422 } 3423 3424 BUG_ON(k < 1); 3425 if (test_bit(k-1, desc->reverse_flags)) { 3426 /* reverse order */ 3427 ppc440spe_rxor_set_mult(desc, k - 1, op - index - 1, mult); 3428 } else { 3429 /* direct order */ 3430 ppc440spe_rxor_set_mult(desc, k - 1, index - lop, mult); 3431 } 3432 } 3433 3434 /** 3435 * ppc440spe_init_rxor_cursor - 3436 */ 3437 static void ppc440spe_init_rxor_cursor(struct ppc440spe_rxor *cursor) 3438 { 3439 memset(cursor, 0, sizeof(struct ppc440spe_rxor)); 3440 cursor->state = 2; 3441 } 3442 3443 /** 3444 * ppc440spe_adma_pq_set_src_mult - set multiplication coefficient into 3445 * descriptor for the PQXOR operation 3446 */ 3447 static void ppc440spe_adma_pq_set_src_mult( 3448 struct ppc440spe_adma_desc_slot *sw_desc, 3449 unsigned char mult, int index, int dst_pos) 3450 { 3451 struct ppc440spe_adma_chan *chan; 3452 u32 mult_idx, mult_dst; 3453 struct ppc440spe_adma_desc_slot *iter = NULL, *iter1 = NULL; 3454 3455 chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan); 3456 3457 switch (chan->device->id) { 3458 case PPC440SPE_DMA0_ID: 3459 case PPC440SPE_DMA1_ID: 3460 if (test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags)) { 3461 int region = test_bit(PPC440SPE_DESC_RXOR12, 3462 &sw_desc->flags) ? 2 : 3; 3463 3464 if (index < region) { 3465 /* RXOR multipliers */ 3466 iter = ppc440spe_get_group_entry(sw_desc, 3467 sw_desc->dst_cnt - 1); 3468 if (sw_desc->dst_cnt == 2) 3469 iter1 = ppc440spe_get_group_entry( 3470 sw_desc, 0); 3471 3472 mult_idx = DMA_CUED_MULT1_OFF + (index << 3); 3473 mult_dst = DMA_CDB_SG_SRC; 3474 } else { 3475 /* WXOR multiplier */ 3476 iter = ppc440spe_get_group_entry(sw_desc, 3477 index - region + 3478 sw_desc->dst_cnt); 3479 mult_idx = DMA_CUED_MULT1_OFF; 3480 mult_dst = dst_pos ? DMA_CDB_SG_DST2 : 3481 DMA_CDB_SG_DST1; 3482 } 3483 } else { 3484 int znum = 0; 3485 3486 /* WXOR-only; 3487 * skip first slots with destinations (if ZERO_DST has 3488 * place) 3489 */ 3490 if (test_bit(PPC440SPE_ZERO_P, &sw_desc->flags)) 3491 znum++; 3492 if (test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags)) 3493 znum++; 3494 3495 iter = ppc440spe_get_group_entry(sw_desc, index + znum); 3496 mult_idx = DMA_CUED_MULT1_OFF; 3497 mult_dst = dst_pos ? DMA_CDB_SG_DST2 : DMA_CDB_SG_DST1; 3498 } 3499 3500 if (likely(iter)) { 3501 ppc440spe_desc_set_src_mult(iter, chan, 3502 mult_idx, mult_dst, mult); 3503 3504 if (unlikely(iter1)) { 3505 /* if we have two destinations for RXOR, then 3506 * we've just set Q mult. Set-up P now. 3507 */ 3508 ppc440spe_desc_set_src_mult(iter1, chan, 3509 mult_idx, mult_dst, 1); 3510 } 3511 3512 } 3513 break; 3514 3515 case PPC440SPE_XOR_ID: 3516 iter = sw_desc->group_head; 3517 if (sw_desc->dst_cnt == 2) { 3518 /* both P & Q calculations required; set P mult here */ 3519 ppc440spe_adma_dma2rxor_set_mult(iter, index, 1); 3520 3521 /* and then set Q mult */ 3522 iter = ppc440spe_get_group_entry(sw_desc, 3523 sw_desc->descs_per_op); 3524 } 3525 ppc440spe_adma_dma2rxor_set_mult(iter, index, mult); 3526 break; 3527 } 3528 } 3529 3530 /** 3531 * ppc440spe_adma_free_chan_resources - free the resources allocated 3532 */ 3533 static void ppc440spe_adma_free_chan_resources(struct dma_chan *chan) 3534 { 3535 struct ppc440spe_adma_chan *ppc440spe_chan; 3536 struct ppc440spe_adma_desc_slot *iter, *_iter; 3537 int in_use_descs = 0; 3538 3539 ppc440spe_chan = to_ppc440spe_adma_chan(chan); 3540 ppc440spe_adma_slot_cleanup(ppc440spe_chan); 3541 3542 spin_lock_bh(&ppc440spe_chan->lock); 3543 list_for_each_entry_safe(iter, _iter, &ppc440spe_chan->chain, 3544 chain_node) { 3545 in_use_descs++; 3546 list_del(&iter->chain_node); 3547 } 3548 list_for_each_entry_safe_reverse(iter, _iter, 3549 &ppc440spe_chan->all_slots, slot_node) { 3550 list_del(&iter->slot_node); 3551 kfree(iter); 3552 ppc440spe_chan->slots_allocated--; 3553 } 3554 ppc440spe_chan->last_used = NULL; 3555 3556 dev_dbg(ppc440spe_chan->device->common.dev, 3557 "ppc440spe adma%d %s slots_allocated %d\n", 3558 ppc440spe_chan->device->id, 3559 __func__, ppc440spe_chan->slots_allocated); 3560 spin_unlock_bh(&ppc440spe_chan->lock); 3561 3562 /* one is ok since we left it on there on purpose */ 3563 if (in_use_descs > 1) 3564 printk(KERN_ERR "SPE: Freeing %d in use descriptors!\n", 3565 in_use_descs - 1); 3566 } 3567 3568 /** 3569 * ppc440spe_adma_tx_status - poll the status of an ADMA transaction 3570 * @chan: ADMA channel handle 3571 * @cookie: ADMA transaction identifier 3572 * @txstate: a holder for the current state of the channel 3573 */ 3574 static enum dma_status ppc440spe_adma_tx_status(struct dma_chan *chan, 3575 dma_cookie_t cookie, struct dma_tx_state *txstate) 3576 { 3577 struct ppc440spe_adma_chan *ppc440spe_chan; 3578 enum dma_status ret; 3579 3580 ppc440spe_chan = to_ppc440spe_adma_chan(chan); 3581 ret = dma_cookie_status(chan, cookie, txstate); 3582 if (ret == DMA_COMPLETE) 3583 return ret; 3584 3585 ppc440spe_adma_slot_cleanup(ppc440spe_chan); 3586 3587 return dma_cookie_status(chan, cookie, txstate); 3588 } 3589 3590 /** 3591 * ppc440spe_adma_eot_handler - end of transfer interrupt handler 3592 */ 3593 static irqreturn_t ppc440spe_adma_eot_handler(int irq, void *data) 3594 { 3595 struct ppc440spe_adma_chan *chan = data; 3596 3597 dev_dbg(chan->device->common.dev, 3598 "ppc440spe adma%d: %s\n", chan->device->id, __func__); 3599 3600 tasklet_schedule(&chan->irq_tasklet); 3601 ppc440spe_adma_device_clear_eot_status(chan); 3602 3603 return IRQ_HANDLED; 3604 } 3605 3606 /** 3607 * ppc440spe_adma_err_handler - DMA error interrupt handler; 3608 * do the same things as a eot handler 3609 */ 3610 static irqreturn_t ppc440spe_adma_err_handler(int irq, void *data) 3611 { 3612 struct ppc440spe_adma_chan *chan = data; 3613 3614 dev_dbg(chan->device->common.dev, 3615 "ppc440spe adma%d: %s\n", chan->device->id, __func__); 3616 3617 tasklet_schedule(&chan->irq_tasklet); 3618 ppc440spe_adma_device_clear_eot_status(chan); 3619 3620 return IRQ_HANDLED; 3621 } 3622 3623 /** 3624 * ppc440spe_test_callback - called when test operation has been done 3625 */ 3626 static void ppc440spe_test_callback(void *unused) 3627 { 3628 complete(&ppc440spe_r6_test_comp); 3629 } 3630 3631 /** 3632 * ppc440spe_adma_issue_pending - flush all pending descriptors to h/w 3633 */ 3634 static void ppc440spe_adma_issue_pending(struct dma_chan *chan) 3635 { 3636 struct ppc440spe_adma_chan *ppc440spe_chan; 3637 3638 ppc440spe_chan = to_ppc440spe_adma_chan(chan); 3639 dev_dbg(ppc440spe_chan->device->common.dev, 3640 "ppc440spe adma%d: %s %d \n", ppc440spe_chan->device->id, 3641 __func__, ppc440spe_chan->pending); 3642 3643 if (ppc440spe_chan->pending) { 3644 ppc440spe_chan->pending = 0; 3645 ppc440spe_chan_append(ppc440spe_chan); 3646 } 3647 } 3648 3649 /** 3650 * ppc440spe_chan_start_null_xor - initiate the first XOR operation (DMA engines 3651 * use FIFOs (as opposite to chains used in XOR) so this is a XOR 3652 * specific operation) 3653 */ 3654 static void ppc440spe_chan_start_null_xor(struct ppc440spe_adma_chan *chan) 3655 { 3656 struct ppc440spe_adma_desc_slot *sw_desc, *group_start; 3657 dma_cookie_t cookie; 3658 int slot_cnt, slots_per_op; 3659 3660 dev_dbg(chan->device->common.dev, 3661 "ppc440spe adma%d: %s\n", chan->device->id, __func__); 3662 3663 spin_lock_bh(&chan->lock); 3664 slot_cnt = ppc440spe_chan_xor_slot_count(0, 2, &slots_per_op); 3665 sw_desc = ppc440spe_adma_alloc_slots(chan, slot_cnt, slots_per_op); 3666 if (sw_desc) { 3667 group_start = sw_desc->group_head; 3668 list_splice_init(&sw_desc->group_list, &chan->chain); 3669 async_tx_ack(&sw_desc->async_tx); 3670 ppc440spe_desc_init_null_xor(group_start); 3671 3672 cookie = dma_cookie_assign(&sw_desc->async_tx); 3673 3674 /* initialize the completed cookie to be less than 3675 * the most recently used cookie 3676 */ 3677 chan->common.completed_cookie = cookie - 1; 3678 3679 /* channel should not be busy */ 3680 BUG_ON(ppc440spe_chan_is_busy(chan)); 3681 3682 /* set the descriptor address */ 3683 ppc440spe_chan_set_first_xor_descriptor(chan, sw_desc); 3684 3685 /* run the descriptor */ 3686 ppc440spe_chan_run(chan); 3687 } else 3688 printk(KERN_ERR "ppc440spe adma%d" 3689 " failed to allocate null descriptor\n", 3690 chan->device->id); 3691 spin_unlock_bh(&chan->lock); 3692 } 3693 3694 /** 3695 * ppc440spe_test_raid6 - test are RAID-6 capabilities enabled successfully. 3696 * For this we just perform one WXOR operation with the same source 3697 * and destination addresses, the GF-multiplier is 1; so if RAID-6 3698 * capabilities are enabled then we'll get src/dst filled with zero. 3699 */ 3700 static int ppc440spe_test_raid6(struct ppc440spe_adma_chan *chan) 3701 { 3702 struct ppc440spe_adma_desc_slot *sw_desc, *iter; 3703 struct page *pg; 3704 char *a; 3705 dma_addr_t dma_addr, addrs[2]; 3706 unsigned long op = 0; 3707 int rval = 0; 3708 3709 set_bit(PPC440SPE_DESC_WXOR, &op); 3710 3711 pg = alloc_page(GFP_KERNEL); 3712 if (!pg) 3713 return -ENOMEM; 3714 3715 spin_lock_bh(&chan->lock); 3716 sw_desc = ppc440spe_adma_alloc_slots(chan, 1, 1); 3717 if (sw_desc) { 3718 /* 1 src, 1 dsr, int_ena, WXOR */ 3719 ppc440spe_desc_init_dma01pq(sw_desc, 1, 1, 1, op); 3720 list_for_each_entry(iter, &sw_desc->group_list, chain_node) { 3721 ppc440spe_desc_set_byte_count(iter, chan, PAGE_SIZE); 3722 iter->unmap_len = PAGE_SIZE; 3723 } 3724 } else { 3725 rval = -EFAULT; 3726 spin_unlock_bh(&chan->lock); 3727 goto exit; 3728 } 3729 spin_unlock_bh(&chan->lock); 3730 3731 /* Fill the test page with ones */ 3732 memset(page_address(pg), 0xFF, PAGE_SIZE); 3733 dma_addr = dma_map_page(chan->device->dev, pg, 0, 3734 PAGE_SIZE, DMA_BIDIRECTIONAL); 3735 3736 /* Setup addresses */ 3737 ppc440spe_adma_pq_set_src(sw_desc, dma_addr, 0); 3738 ppc440spe_adma_pq_set_src_mult(sw_desc, 1, 0, 0); 3739 addrs[0] = dma_addr; 3740 addrs[1] = 0; 3741 ppc440spe_adma_pq_set_dest(sw_desc, addrs, DMA_PREP_PQ_DISABLE_Q); 3742 3743 async_tx_ack(&sw_desc->async_tx); 3744 sw_desc->async_tx.callback = ppc440spe_test_callback; 3745 sw_desc->async_tx.callback_param = NULL; 3746 3747 init_completion(&ppc440spe_r6_test_comp); 3748 3749 ppc440spe_adma_tx_submit(&sw_desc->async_tx); 3750 ppc440spe_adma_issue_pending(&chan->common); 3751 3752 wait_for_completion(&ppc440spe_r6_test_comp); 3753 3754 /* Now check if the test page is zeroed */ 3755 a = page_address(pg); 3756 if ((*(u32 *)a) == 0 && memcmp(a, a+4, PAGE_SIZE-4) == 0) { 3757 /* page is zero - RAID-6 enabled */ 3758 rval = 0; 3759 } else { 3760 /* RAID-6 was not enabled */ 3761 rval = -EINVAL; 3762 } 3763 exit: 3764 __free_page(pg); 3765 return rval; 3766 } 3767 3768 static void ppc440spe_adma_init_capabilities(struct ppc440spe_adma_device *adev) 3769 { 3770 switch (adev->id) { 3771 case PPC440SPE_DMA0_ID: 3772 case PPC440SPE_DMA1_ID: 3773 dma_cap_set(DMA_MEMCPY, adev->common.cap_mask); 3774 dma_cap_set(DMA_INTERRUPT, adev->common.cap_mask); 3775 dma_cap_set(DMA_PQ, adev->common.cap_mask); 3776 dma_cap_set(DMA_PQ_VAL, adev->common.cap_mask); 3777 dma_cap_set(DMA_XOR_VAL, adev->common.cap_mask); 3778 break; 3779 case PPC440SPE_XOR_ID: 3780 dma_cap_set(DMA_XOR, adev->common.cap_mask); 3781 dma_cap_set(DMA_PQ, adev->common.cap_mask); 3782 dma_cap_set(DMA_INTERRUPT, adev->common.cap_mask); 3783 adev->common.cap_mask = adev->common.cap_mask; 3784 break; 3785 } 3786 3787 /* Set base routines */ 3788 adev->common.device_alloc_chan_resources = 3789 ppc440spe_adma_alloc_chan_resources; 3790 adev->common.device_free_chan_resources = 3791 ppc440spe_adma_free_chan_resources; 3792 adev->common.device_tx_status = ppc440spe_adma_tx_status; 3793 adev->common.device_issue_pending = ppc440spe_adma_issue_pending; 3794 3795 /* Set prep routines based on capability */ 3796 if (dma_has_cap(DMA_MEMCPY, adev->common.cap_mask)) { 3797 adev->common.device_prep_dma_memcpy = 3798 ppc440spe_adma_prep_dma_memcpy; 3799 } 3800 if (dma_has_cap(DMA_XOR, adev->common.cap_mask)) { 3801 adev->common.max_xor = XOR_MAX_OPS; 3802 adev->common.device_prep_dma_xor = 3803 ppc440spe_adma_prep_dma_xor; 3804 } 3805 if (dma_has_cap(DMA_PQ, adev->common.cap_mask)) { 3806 switch (adev->id) { 3807 case PPC440SPE_DMA0_ID: 3808 dma_set_maxpq(&adev->common, 3809 DMA0_FIFO_SIZE / sizeof(struct dma_cdb), 0); 3810 break; 3811 case PPC440SPE_DMA1_ID: 3812 dma_set_maxpq(&adev->common, 3813 DMA1_FIFO_SIZE / sizeof(struct dma_cdb), 0); 3814 break; 3815 case PPC440SPE_XOR_ID: 3816 adev->common.max_pq = XOR_MAX_OPS * 3; 3817 break; 3818 } 3819 adev->common.device_prep_dma_pq = 3820 ppc440spe_adma_prep_dma_pq; 3821 } 3822 if (dma_has_cap(DMA_PQ_VAL, adev->common.cap_mask)) { 3823 switch (adev->id) { 3824 case PPC440SPE_DMA0_ID: 3825 adev->common.max_pq = DMA0_FIFO_SIZE / 3826 sizeof(struct dma_cdb); 3827 break; 3828 case PPC440SPE_DMA1_ID: 3829 adev->common.max_pq = DMA1_FIFO_SIZE / 3830 sizeof(struct dma_cdb); 3831 break; 3832 } 3833 adev->common.device_prep_dma_pq_val = 3834 ppc440spe_adma_prep_dma_pqzero_sum; 3835 } 3836 if (dma_has_cap(DMA_XOR_VAL, adev->common.cap_mask)) { 3837 switch (adev->id) { 3838 case PPC440SPE_DMA0_ID: 3839 adev->common.max_xor = DMA0_FIFO_SIZE / 3840 sizeof(struct dma_cdb); 3841 break; 3842 case PPC440SPE_DMA1_ID: 3843 adev->common.max_xor = DMA1_FIFO_SIZE / 3844 sizeof(struct dma_cdb); 3845 break; 3846 } 3847 adev->common.device_prep_dma_xor_val = 3848 ppc440spe_adma_prep_dma_xor_zero_sum; 3849 } 3850 if (dma_has_cap(DMA_INTERRUPT, adev->common.cap_mask)) { 3851 adev->common.device_prep_dma_interrupt = 3852 ppc440spe_adma_prep_dma_interrupt; 3853 } 3854 pr_info("%s: AMCC(R) PPC440SP(E) ADMA Engine: " 3855 "( %s%s%s%s%s%s)\n", 3856 dev_name(adev->dev), 3857 dma_has_cap(DMA_PQ, adev->common.cap_mask) ? "pq " : "", 3858 dma_has_cap(DMA_PQ_VAL, adev->common.cap_mask) ? "pq_val " : "", 3859 dma_has_cap(DMA_XOR, adev->common.cap_mask) ? "xor " : "", 3860 dma_has_cap(DMA_XOR_VAL, adev->common.cap_mask) ? "xor_val " : "", 3861 dma_has_cap(DMA_MEMCPY, adev->common.cap_mask) ? "memcpy " : "", 3862 dma_has_cap(DMA_INTERRUPT, adev->common.cap_mask) ? "intr " : ""); 3863 } 3864 3865 static int ppc440spe_adma_setup_irqs(struct ppc440spe_adma_device *adev, 3866 struct ppc440spe_adma_chan *chan, 3867 int *initcode) 3868 { 3869 struct platform_device *ofdev; 3870 struct device_node *np; 3871 int ret; 3872 3873 ofdev = container_of(adev->dev, struct platform_device, dev); 3874 np = ofdev->dev.of_node; 3875 if (adev->id != PPC440SPE_XOR_ID) { 3876 adev->err_irq = irq_of_parse_and_map(np, 1); 3877 if (!adev->err_irq) { 3878 dev_warn(adev->dev, "no err irq resource?\n"); 3879 *initcode = PPC_ADMA_INIT_IRQ2; 3880 adev->err_irq = -ENXIO; 3881 } else 3882 atomic_inc(&ppc440spe_adma_err_irq_ref); 3883 } else { 3884 adev->err_irq = -ENXIO; 3885 } 3886 3887 adev->irq = irq_of_parse_and_map(np, 0); 3888 if (!adev->irq) { 3889 dev_err(adev->dev, "no irq resource\n"); 3890 *initcode = PPC_ADMA_INIT_IRQ1; 3891 ret = -ENXIO; 3892 goto err_irq_map; 3893 } 3894 dev_dbg(adev->dev, "irq %d, err irq %d\n", 3895 adev->irq, adev->err_irq); 3896 3897 ret = request_irq(adev->irq, ppc440spe_adma_eot_handler, 3898 0, dev_driver_string(adev->dev), chan); 3899 if (ret) { 3900 dev_err(adev->dev, "can't request irq %d\n", 3901 adev->irq); 3902 *initcode = PPC_ADMA_INIT_IRQ1; 3903 ret = -EIO; 3904 goto err_req1; 3905 } 3906 3907 /* only DMA engines have a separate error IRQ 3908 * so it's Ok if err_irq < 0 in XOR engine case. 3909 */ 3910 if (adev->err_irq > 0) { 3911 /* both DMA engines share common error IRQ */ 3912 ret = request_irq(adev->err_irq, 3913 ppc440spe_adma_err_handler, 3914 IRQF_SHARED, 3915 dev_driver_string(adev->dev), 3916 chan); 3917 if (ret) { 3918 dev_err(adev->dev, "can't request irq %d\n", 3919 adev->err_irq); 3920 *initcode = PPC_ADMA_INIT_IRQ2; 3921 ret = -EIO; 3922 goto err_req2; 3923 } 3924 } 3925 3926 if (adev->id == PPC440SPE_XOR_ID) { 3927 /* enable XOR engine interrupts */ 3928 iowrite32be(XOR_IE_CBCIE_BIT | XOR_IE_ICBIE_BIT | 3929 XOR_IE_ICIE_BIT | XOR_IE_RPTIE_BIT, 3930 &adev->xor_reg->ier); 3931 } else { 3932 u32 mask, enable; 3933 3934 np = of_find_compatible_node(NULL, NULL, "ibm,i2o-440spe"); 3935 if (!np) { 3936 pr_err("%s: can't find I2O device tree node\n", 3937 __func__); 3938 ret = -ENODEV; 3939 goto err_req2; 3940 } 3941 adev->i2o_reg = of_iomap(np, 0); 3942 if (!adev->i2o_reg) { 3943 pr_err("%s: failed to map I2O registers\n", __func__); 3944 of_node_put(np); 3945 ret = -EINVAL; 3946 goto err_req2; 3947 } 3948 of_node_put(np); 3949 /* Unmask 'CS FIFO Attention' interrupts and 3950 * enable generating interrupts on errors 3951 */ 3952 enable = (adev->id == PPC440SPE_DMA0_ID) ? 3953 ~(I2O_IOPIM_P0SNE | I2O_IOPIM_P0EM) : 3954 ~(I2O_IOPIM_P1SNE | I2O_IOPIM_P1EM); 3955 mask = ioread32(&adev->i2o_reg->iopim) & enable; 3956 iowrite32(mask, &adev->i2o_reg->iopim); 3957 } 3958 return 0; 3959 3960 err_req2: 3961 free_irq(adev->irq, chan); 3962 err_req1: 3963 irq_dispose_mapping(adev->irq); 3964 err_irq_map: 3965 if (adev->err_irq > 0) { 3966 if (atomic_dec_and_test(&ppc440spe_adma_err_irq_ref)) 3967 irq_dispose_mapping(adev->err_irq); 3968 } 3969 return ret; 3970 } 3971 3972 static void ppc440spe_adma_release_irqs(struct ppc440spe_adma_device *adev, 3973 struct ppc440spe_adma_chan *chan) 3974 { 3975 u32 mask, disable; 3976 3977 if (adev->id == PPC440SPE_XOR_ID) { 3978 /* disable XOR engine interrupts */ 3979 mask = ioread32be(&adev->xor_reg->ier); 3980 mask &= ~(XOR_IE_CBCIE_BIT | XOR_IE_ICBIE_BIT | 3981 XOR_IE_ICIE_BIT | XOR_IE_RPTIE_BIT); 3982 iowrite32be(mask, &adev->xor_reg->ier); 3983 } else { 3984 /* disable DMAx engine interrupts */ 3985 disable = (adev->id == PPC440SPE_DMA0_ID) ? 3986 (I2O_IOPIM_P0SNE | I2O_IOPIM_P0EM) : 3987 (I2O_IOPIM_P1SNE | I2O_IOPIM_P1EM); 3988 mask = ioread32(&adev->i2o_reg->iopim) | disable; 3989 iowrite32(mask, &adev->i2o_reg->iopim); 3990 } 3991 free_irq(adev->irq, chan); 3992 irq_dispose_mapping(adev->irq); 3993 if (adev->err_irq > 0) { 3994 free_irq(adev->err_irq, chan); 3995 if (atomic_dec_and_test(&ppc440spe_adma_err_irq_ref)) { 3996 irq_dispose_mapping(adev->err_irq); 3997 iounmap(adev->i2o_reg); 3998 } 3999 } 4000 } 4001 4002 /** 4003 * ppc440spe_adma_probe - probe the asynch device 4004 */ 4005 static int ppc440spe_adma_probe(struct platform_device *ofdev) 4006 { 4007 struct device_node *np = ofdev->dev.of_node; 4008 struct resource res; 4009 struct ppc440spe_adma_device *adev; 4010 struct ppc440spe_adma_chan *chan; 4011 struct ppc_dma_chan_ref *ref, *_ref; 4012 int ret = 0, initcode = PPC_ADMA_INIT_OK; 4013 const u32 *idx; 4014 int len; 4015 void *regs; 4016 u32 id, pool_size; 4017 4018 if (of_device_is_compatible(np, "amcc,xor-accelerator")) { 4019 id = PPC440SPE_XOR_ID; 4020 /* As far as the XOR engine is concerned, it does not 4021 * use FIFOs but uses linked list. So there is no dependency 4022 * between pool size to allocate and the engine configuration. 4023 */ 4024 pool_size = PAGE_SIZE << 1; 4025 } else { 4026 /* it is DMA0 or DMA1 */ 4027 idx = of_get_property(np, "cell-index", &len); 4028 if (!idx || (len != sizeof(u32))) { 4029 dev_err(&ofdev->dev, "Device node %pOF has missing " 4030 "or invalid cell-index property\n", 4031 np); 4032 return -EINVAL; 4033 } 4034 id = *idx; 4035 /* DMA0,1 engines use FIFO to maintain CDBs, so we 4036 * should allocate the pool accordingly to size of this 4037 * FIFO. Thus, the pool size depends on the FIFO depth: 4038 * how much CDBs pointers the FIFO may contain then so 4039 * much CDBs we should provide in the pool. 4040 * That is 4041 * CDB size = 32B; 4042 * CDBs number = (DMA0_FIFO_SIZE >> 3); 4043 * Pool size = CDBs number * CDB size = 4044 * = (DMA0_FIFO_SIZE >> 3) << 5 = DMA0_FIFO_SIZE << 2. 4045 */ 4046 pool_size = (id == PPC440SPE_DMA0_ID) ? 4047 DMA0_FIFO_SIZE : DMA1_FIFO_SIZE; 4048 pool_size <<= 2; 4049 } 4050 4051 if (of_address_to_resource(np, 0, &res)) { 4052 dev_err(&ofdev->dev, "failed to get memory resource\n"); 4053 initcode = PPC_ADMA_INIT_MEMRES; 4054 ret = -ENODEV; 4055 goto out; 4056 } 4057 4058 if (!request_mem_region(res.start, resource_size(&res), 4059 dev_driver_string(&ofdev->dev))) { 4060 dev_err(&ofdev->dev, "failed to request memory region %pR\n", 4061 &res); 4062 initcode = PPC_ADMA_INIT_MEMREG; 4063 ret = -EBUSY; 4064 goto out; 4065 } 4066 4067 /* create a device */ 4068 adev = kzalloc(sizeof(*adev), GFP_KERNEL); 4069 if (!adev) { 4070 initcode = PPC_ADMA_INIT_ALLOC; 4071 ret = -ENOMEM; 4072 goto err_adev_alloc; 4073 } 4074 4075 adev->id = id; 4076 adev->pool_size = pool_size; 4077 /* allocate coherent memory for hardware descriptors */ 4078 adev->dma_desc_pool_virt = dma_alloc_coherent(&ofdev->dev, 4079 adev->pool_size, &adev->dma_desc_pool, 4080 GFP_KERNEL); 4081 if (adev->dma_desc_pool_virt == NULL) { 4082 dev_err(&ofdev->dev, "failed to allocate %d bytes of coherent " 4083 "memory for hardware descriptors\n", 4084 adev->pool_size); 4085 initcode = PPC_ADMA_INIT_COHERENT; 4086 ret = -ENOMEM; 4087 goto err_dma_alloc; 4088 } 4089 dev_dbg(&ofdev->dev, "allocated descriptor pool virt 0x%p phys 0x%llx\n", 4090 adev->dma_desc_pool_virt, (u64)adev->dma_desc_pool); 4091 4092 regs = ioremap(res.start, resource_size(&res)); 4093 if (!regs) { 4094 dev_err(&ofdev->dev, "failed to ioremap regs!\n"); 4095 ret = -ENOMEM; 4096 goto err_regs_alloc; 4097 } 4098 4099 if (adev->id == PPC440SPE_XOR_ID) { 4100 adev->xor_reg = regs; 4101 /* Reset XOR */ 4102 iowrite32be(XOR_CRSR_XASR_BIT, &adev->xor_reg->crsr); 4103 iowrite32be(XOR_CRSR_64BA_BIT, &adev->xor_reg->crrr); 4104 } else { 4105 size_t fifo_size = (adev->id == PPC440SPE_DMA0_ID) ? 4106 DMA0_FIFO_SIZE : DMA1_FIFO_SIZE; 4107 adev->dma_reg = regs; 4108 /* DMAx_FIFO_SIZE is defined in bytes, 4109 * <fsiz> - is defined in number of CDB pointers (8byte). 4110 * DMA FIFO Length = CSlength + CPlength, where 4111 * CSlength = CPlength = (fsiz + 1) * 8. 4112 */ 4113 iowrite32(DMA_FIFO_ENABLE | ((fifo_size >> 3) - 2), 4114 &adev->dma_reg->fsiz); 4115 /* Configure DMA engine */ 4116 iowrite32(DMA_CFG_DXEPR_HP | DMA_CFG_DFMPP_HP | DMA_CFG_FALGN, 4117 &adev->dma_reg->cfg); 4118 /* Clear Status */ 4119 iowrite32(~0, &adev->dma_reg->dsts); 4120 } 4121 4122 adev->dev = &ofdev->dev; 4123 adev->common.dev = &ofdev->dev; 4124 INIT_LIST_HEAD(&adev->common.channels); 4125 platform_set_drvdata(ofdev, adev); 4126 4127 /* create a channel */ 4128 chan = kzalloc(sizeof(*chan), GFP_KERNEL); 4129 if (!chan) { 4130 initcode = PPC_ADMA_INIT_CHANNEL; 4131 ret = -ENOMEM; 4132 goto err_chan_alloc; 4133 } 4134 4135 spin_lock_init(&chan->lock); 4136 INIT_LIST_HEAD(&chan->chain); 4137 INIT_LIST_HEAD(&chan->all_slots); 4138 chan->device = adev; 4139 chan->common.device = &adev->common; 4140 dma_cookie_init(&chan->common); 4141 list_add_tail(&chan->common.device_node, &adev->common.channels); 4142 tasklet_setup(&chan->irq_tasklet, ppc440spe_adma_tasklet); 4143 4144 /* allocate and map helper pages for async validation or 4145 * async_mult/async_sum_product operations on DMA0/1. 4146 */ 4147 if (adev->id != PPC440SPE_XOR_ID) { 4148 chan->pdest_page = alloc_page(GFP_KERNEL); 4149 chan->qdest_page = alloc_page(GFP_KERNEL); 4150 if (!chan->pdest_page || 4151 !chan->qdest_page) { 4152 if (chan->pdest_page) 4153 __free_page(chan->pdest_page); 4154 if (chan->qdest_page) 4155 __free_page(chan->qdest_page); 4156 ret = -ENOMEM; 4157 goto err_page_alloc; 4158 } 4159 chan->pdest = dma_map_page(&ofdev->dev, chan->pdest_page, 0, 4160 PAGE_SIZE, DMA_BIDIRECTIONAL); 4161 chan->qdest = dma_map_page(&ofdev->dev, chan->qdest_page, 0, 4162 PAGE_SIZE, DMA_BIDIRECTIONAL); 4163 } 4164 4165 ref = kmalloc(sizeof(*ref), GFP_KERNEL); 4166 if (ref) { 4167 ref->chan = &chan->common; 4168 INIT_LIST_HEAD(&ref->node); 4169 list_add_tail(&ref->node, &ppc440spe_adma_chan_list); 4170 } else { 4171 dev_err(&ofdev->dev, "failed to allocate channel reference!\n"); 4172 ret = -ENOMEM; 4173 goto err_ref_alloc; 4174 } 4175 4176 ret = ppc440spe_adma_setup_irqs(adev, chan, &initcode); 4177 if (ret) 4178 goto err_irq; 4179 4180 ppc440spe_adma_init_capabilities(adev); 4181 4182 ret = dma_async_device_register(&adev->common); 4183 if (ret) { 4184 initcode = PPC_ADMA_INIT_REGISTER; 4185 dev_err(&ofdev->dev, "failed to register dma device\n"); 4186 goto err_dev_reg; 4187 } 4188 4189 goto out; 4190 4191 err_dev_reg: 4192 ppc440spe_adma_release_irqs(adev, chan); 4193 err_irq: 4194 list_for_each_entry_safe(ref, _ref, &ppc440spe_adma_chan_list, node) { 4195 if (chan == to_ppc440spe_adma_chan(ref->chan)) { 4196 list_del(&ref->node); 4197 kfree(ref); 4198 } 4199 } 4200 err_ref_alloc: 4201 if (adev->id != PPC440SPE_XOR_ID) { 4202 dma_unmap_page(&ofdev->dev, chan->pdest, 4203 PAGE_SIZE, DMA_BIDIRECTIONAL); 4204 dma_unmap_page(&ofdev->dev, chan->qdest, 4205 PAGE_SIZE, DMA_BIDIRECTIONAL); 4206 __free_page(chan->pdest_page); 4207 __free_page(chan->qdest_page); 4208 } 4209 err_page_alloc: 4210 kfree(chan); 4211 err_chan_alloc: 4212 if (adev->id == PPC440SPE_XOR_ID) 4213 iounmap(adev->xor_reg); 4214 else 4215 iounmap(adev->dma_reg); 4216 err_regs_alloc: 4217 dma_free_coherent(adev->dev, adev->pool_size, 4218 adev->dma_desc_pool_virt, 4219 adev->dma_desc_pool); 4220 err_dma_alloc: 4221 kfree(adev); 4222 err_adev_alloc: 4223 release_mem_region(res.start, resource_size(&res)); 4224 out: 4225 if (id < PPC440SPE_ADMA_ENGINES_NUM) 4226 ppc440spe_adma_devices[id] = initcode; 4227 4228 return ret; 4229 } 4230 4231 /** 4232 * ppc440spe_adma_remove - remove the asynch device 4233 */ 4234 static int ppc440spe_adma_remove(struct platform_device *ofdev) 4235 { 4236 struct ppc440spe_adma_device *adev = platform_get_drvdata(ofdev); 4237 struct device_node *np = ofdev->dev.of_node; 4238 struct resource res; 4239 struct dma_chan *chan, *_chan; 4240 struct ppc_dma_chan_ref *ref, *_ref; 4241 struct ppc440spe_adma_chan *ppc440spe_chan; 4242 4243 if (adev->id < PPC440SPE_ADMA_ENGINES_NUM) 4244 ppc440spe_adma_devices[adev->id] = -1; 4245 4246 dma_async_device_unregister(&adev->common); 4247 4248 list_for_each_entry_safe(chan, _chan, &adev->common.channels, 4249 device_node) { 4250 ppc440spe_chan = to_ppc440spe_adma_chan(chan); 4251 ppc440spe_adma_release_irqs(adev, ppc440spe_chan); 4252 tasklet_kill(&ppc440spe_chan->irq_tasklet); 4253 if (adev->id != PPC440SPE_XOR_ID) { 4254 dma_unmap_page(&ofdev->dev, ppc440spe_chan->pdest, 4255 PAGE_SIZE, DMA_BIDIRECTIONAL); 4256 dma_unmap_page(&ofdev->dev, ppc440spe_chan->qdest, 4257 PAGE_SIZE, DMA_BIDIRECTIONAL); 4258 __free_page(ppc440spe_chan->pdest_page); 4259 __free_page(ppc440spe_chan->qdest_page); 4260 } 4261 list_for_each_entry_safe(ref, _ref, &ppc440spe_adma_chan_list, 4262 node) { 4263 if (ppc440spe_chan == 4264 to_ppc440spe_adma_chan(ref->chan)) { 4265 list_del(&ref->node); 4266 kfree(ref); 4267 } 4268 } 4269 list_del(&chan->device_node); 4270 kfree(ppc440spe_chan); 4271 } 4272 4273 dma_free_coherent(adev->dev, adev->pool_size, 4274 adev->dma_desc_pool_virt, adev->dma_desc_pool); 4275 if (adev->id == PPC440SPE_XOR_ID) 4276 iounmap(adev->xor_reg); 4277 else 4278 iounmap(adev->dma_reg); 4279 of_address_to_resource(np, 0, &res); 4280 release_mem_region(res.start, resource_size(&res)); 4281 kfree(adev); 4282 return 0; 4283 } 4284 4285 /* 4286 * /sys driver interface to enable h/w RAID-6 capabilities 4287 * Files created in e.g. /sys/devices/plb.0/400100100.dma0/driver/ 4288 * directory are "devices", "enable" and "poly". 4289 * "devices" shows available engines. 4290 * "enable" is used to enable RAID-6 capabilities or to check 4291 * whether these has been activated. 4292 * "poly" allows setting/checking used polynomial (for PPC440SPe only). 4293 */ 4294 4295 static ssize_t devices_show(struct device_driver *dev, char *buf) 4296 { 4297 ssize_t size = 0; 4298 int i; 4299 4300 for (i = 0; i < PPC440SPE_ADMA_ENGINES_NUM; i++) { 4301 if (ppc440spe_adma_devices[i] == -1) 4302 continue; 4303 size += scnprintf(buf + size, PAGE_SIZE - size, 4304 "PPC440SP(E)-ADMA.%d: %s\n", i, 4305 ppc_adma_errors[ppc440spe_adma_devices[i]]); 4306 } 4307 return size; 4308 } 4309 static DRIVER_ATTR_RO(devices); 4310 4311 static ssize_t enable_show(struct device_driver *dev, char *buf) 4312 { 4313 return snprintf(buf, PAGE_SIZE, 4314 "PPC440SP(e) RAID-6 capabilities are %sABLED.\n", 4315 ppc440spe_r6_enabled ? "EN" : "DIS"); 4316 } 4317 4318 static ssize_t enable_store(struct device_driver *dev, const char *buf, 4319 size_t count) 4320 { 4321 unsigned long val; 4322 4323 if (!count || count > 11) 4324 return -EINVAL; 4325 4326 if (!ppc440spe_r6_tchan) 4327 return -EFAULT; 4328 4329 /* Write a key */ 4330 sscanf(buf, "%lx", &val); 4331 dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_XORBA, val); 4332 isync(); 4333 4334 /* Verify whether it really works now */ 4335 if (ppc440spe_test_raid6(ppc440spe_r6_tchan) == 0) { 4336 pr_info("PPC440SP(e) RAID-6 has been activated " 4337 "successfully\n"); 4338 ppc440spe_r6_enabled = 1; 4339 } else { 4340 pr_info("PPC440SP(e) RAID-6 hasn't been activated!" 4341 " Error key ?\n"); 4342 ppc440spe_r6_enabled = 0; 4343 } 4344 return count; 4345 } 4346 static DRIVER_ATTR_RW(enable); 4347 4348 static ssize_t poly_show(struct device_driver *dev, char *buf) 4349 { 4350 ssize_t size = 0; 4351 u32 reg; 4352 4353 #ifdef CONFIG_440SP 4354 /* 440SP has fixed polynomial */ 4355 reg = 0x4d; 4356 #else 4357 reg = dcr_read(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL); 4358 reg >>= MQ0_CFBHL_POLY; 4359 reg &= 0xFF; 4360 #endif 4361 4362 size = snprintf(buf, PAGE_SIZE, "PPC440SP(e) RAID-6 driver " 4363 "uses 0x1%02x polynomial.\n", reg); 4364 return size; 4365 } 4366 4367 static ssize_t poly_store(struct device_driver *dev, const char *buf, 4368 size_t count) 4369 { 4370 unsigned long reg, val; 4371 4372 #ifdef CONFIG_440SP 4373 /* 440SP uses default 0x14D polynomial only */ 4374 return -EINVAL; 4375 #endif 4376 4377 if (!count || count > 6) 4378 return -EINVAL; 4379 4380 /* e.g., 0x14D or 0x11D */ 4381 sscanf(buf, "%lx", &val); 4382 4383 if (val & ~0x1FF) 4384 return -EINVAL; 4385 4386 val &= 0xFF; 4387 reg = dcr_read(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL); 4388 reg &= ~(0xFF << MQ0_CFBHL_POLY); 4389 reg |= val << MQ0_CFBHL_POLY; 4390 dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL, reg); 4391 4392 return count; 4393 } 4394 static DRIVER_ATTR_RW(poly); 4395 4396 /* 4397 * Common initialisation for RAID engines; allocate memory for 4398 * DMAx FIFOs, perform configuration common for all DMA engines. 4399 * Further DMA engine specific configuration is done at probe time. 4400 */ 4401 static int ppc440spe_configure_raid_devices(void) 4402 { 4403 struct device_node *np; 4404 struct resource i2o_res; 4405 struct i2o_regs __iomem *i2o_reg; 4406 dcr_host_t i2o_dcr_host; 4407 unsigned int dcr_base, dcr_len; 4408 int i, ret; 4409 4410 np = of_find_compatible_node(NULL, NULL, "ibm,i2o-440spe"); 4411 if (!np) { 4412 pr_err("%s: can't find I2O device tree node\n", 4413 __func__); 4414 return -ENODEV; 4415 } 4416 4417 if (of_address_to_resource(np, 0, &i2o_res)) { 4418 of_node_put(np); 4419 return -EINVAL; 4420 } 4421 4422 i2o_reg = of_iomap(np, 0); 4423 if (!i2o_reg) { 4424 pr_err("%s: failed to map I2O registers\n", __func__); 4425 of_node_put(np); 4426 return -EINVAL; 4427 } 4428 4429 /* Get I2O DCRs base */ 4430 dcr_base = dcr_resource_start(np, 0); 4431 dcr_len = dcr_resource_len(np, 0); 4432 if (!dcr_base && !dcr_len) { 4433 pr_err("%pOF: can't get DCR registers base/len!\n", np); 4434 of_node_put(np); 4435 iounmap(i2o_reg); 4436 return -ENODEV; 4437 } 4438 4439 i2o_dcr_host = dcr_map(np, dcr_base, dcr_len); 4440 if (!DCR_MAP_OK(i2o_dcr_host)) { 4441 pr_err("%pOF: failed to map DCRs!\n", np); 4442 of_node_put(np); 4443 iounmap(i2o_reg); 4444 return -ENODEV; 4445 } 4446 of_node_put(np); 4447 4448 /* Provide memory regions for DMA's FIFOs: I2O, DMA0 and DMA1 share 4449 * the base address of FIFO memory space. 4450 * Actually we need twice more physical memory than programmed in the 4451 * <fsiz> register (because there are two FIFOs for each DMA: CP and CS) 4452 */ 4453 ppc440spe_dma_fifo_buf = kmalloc((DMA0_FIFO_SIZE + DMA1_FIFO_SIZE) << 1, 4454 GFP_KERNEL); 4455 if (!ppc440spe_dma_fifo_buf) { 4456 pr_err("%s: DMA FIFO buffer allocation failed.\n", __func__); 4457 iounmap(i2o_reg); 4458 dcr_unmap(i2o_dcr_host, dcr_len); 4459 return -ENOMEM; 4460 } 4461 4462 /* 4463 * Configure h/w 4464 */ 4465 /* Reset I2O/DMA */ 4466 mtdcri(SDR0, DCRN_SDR0_SRST, DCRN_SDR0_SRST_I2ODMA); 4467 mtdcri(SDR0, DCRN_SDR0_SRST, 0); 4468 4469 /* Setup the base address of mmaped registers */ 4470 dcr_write(i2o_dcr_host, DCRN_I2O0_IBAH, (u32)(i2o_res.start >> 32)); 4471 dcr_write(i2o_dcr_host, DCRN_I2O0_IBAL, (u32)(i2o_res.start) | 4472 I2O_REG_ENABLE); 4473 dcr_unmap(i2o_dcr_host, dcr_len); 4474 4475 /* Setup FIFO memory space base address */ 4476 iowrite32(0, &i2o_reg->ifbah); 4477 iowrite32(((u32)__pa(ppc440spe_dma_fifo_buf)), &i2o_reg->ifbal); 4478 4479 /* set zero FIFO size for I2O, so the whole 4480 * ppc440spe_dma_fifo_buf is used by DMAs. 4481 * DMAx_FIFOs will be configured while probe. 4482 */ 4483 iowrite32(0, &i2o_reg->ifsiz); 4484 iounmap(i2o_reg); 4485 4486 /* To prepare WXOR/RXOR functionality we need access to 4487 * Memory Queue Module DCRs (finally it will be enabled 4488 * via /sys interface of the ppc440spe ADMA driver). 4489 */ 4490 np = of_find_compatible_node(NULL, NULL, "ibm,mq-440spe"); 4491 if (!np) { 4492 pr_err("%s: can't find MQ device tree node\n", 4493 __func__); 4494 ret = -ENODEV; 4495 goto out_free; 4496 } 4497 4498 /* Get MQ DCRs base */ 4499 dcr_base = dcr_resource_start(np, 0); 4500 dcr_len = dcr_resource_len(np, 0); 4501 if (!dcr_base && !dcr_len) { 4502 pr_err("%pOF: can't get DCR registers base/len!\n", np); 4503 ret = -ENODEV; 4504 goto out_mq; 4505 } 4506 4507 ppc440spe_mq_dcr_host = dcr_map(np, dcr_base, dcr_len); 4508 if (!DCR_MAP_OK(ppc440spe_mq_dcr_host)) { 4509 pr_err("%pOF: failed to map DCRs!\n", np); 4510 ret = -ENODEV; 4511 goto out_mq; 4512 } 4513 of_node_put(np); 4514 ppc440spe_mq_dcr_len = dcr_len; 4515 4516 /* Set HB alias */ 4517 dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_BAUH, DMA_CUED_XOR_HB); 4518 4519 /* Set: 4520 * - LL transaction passing limit to 1; 4521 * - Memory controller cycle limit to 1; 4522 * - Galois Polynomial to 0x14d (default) 4523 */ 4524 dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL, 4525 (1 << MQ0_CFBHL_TPLM) | (1 << MQ0_CFBHL_HBCL) | 4526 (PPC440SPE_DEFAULT_POLY << MQ0_CFBHL_POLY)); 4527 4528 atomic_set(&ppc440spe_adma_err_irq_ref, 0); 4529 for (i = 0; i < PPC440SPE_ADMA_ENGINES_NUM; i++) 4530 ppc440spe_adma_devices[i] = -1; 4531 4532 return 0; 4533 4534 out_mq: 4535 of_node_put(np); 4536 out_free: 4537 kfree(ppc440spe_dma_fifo_buf); 4538 return ret; 4539 } 4540 4541 static const struct of_device_id ppc440spe_adma_of_match[] = { 4542 { .compatible = "ibm,dma-440spe", }, 4543 { .compatible = "amcc,xor-accelerator", }, 4544 {}, 4545 }; 4546 MODULE_DEVICE_TABLE(of, ppc440spe_adma_of_match); 4547 4548 static struct platform_driver ppc440spe_adma_driver = { 4549 .probe = ppc440spe_adma_probe, 4550 .remove = ppc440spe_adma_remove, 4551 .driver = { 4552 .name = "PPC440SP(E)-ADMA", 4553 .of_match_table = ppc440spe_adma_of_match, 4554 }, 4555 }; 4556 4557 static __init int ppc440spe_adma_init(void) 4558 { 4559 int ret; 4560 4561 ret = ppc440spe_configure_raid_devices(); 4562 if (ret) 4563 return ret; 4564 4565 ret = platform_driver_register(&ppc440spe_adma_driver); 4566 if (ret) { 4567 pr_err("%s: failed to register platform driver\n", 4568 __func__); 4569 goto out_reg; 4570 } 4571 4572 /* Initialization status */ 4573 ret = driver_create_file(&ppc440spe_adma_driver.driver, 4574 &driver_attr_devices); 4575 if (ret) 4576 goto out_dev; 4577 4578 /* RAID-6 h/w enable entry */ 4579 ret = driver_create_file(&ppc440spe_adma_driver.driver, 4580 &driver_attr_enable); 4581 if (ret) 4582 goto out_en; 4583 4584 /* GF polynomial to use */ 4585 ret = driver_create_file(&ppc440spe_adma_driver.driver, 4586 &driver_attr_poly); 4587 if (!ret) 4588 return ret; 4589 4590 driver_remove_file(&ppc440spe_adma_driver.driver, 4591 &driver_attr_enable); 4592 out_en: 4593 driver_remove_file(&ppc440spe_adma_driver.driver, 4594 &driver_attr_devices); 4595 out_dev: 4596 /* User will not be able to enable h/w RAID-6 */ 4597 pr_err("%s: failed to create RAID-6 driver interface\n", 4598 __func__); 4599 platform_driver_unregister(&ppc440spe_adma_driver); 4600 out_reg: 4601 dcr_unmap(ppc440spe_mq_dcr_host, ppc440spe_mq_dcr_len); 4602 kfree(ppc440spe_dma_fifo_buf); 4603 return ret; 4604 } 4605 4606 static void __exit ppc440spe_adma_exit(void) 4607 { 4608 driver_remove_file(&ppc440spe_adma_driver.driver, 4609 &driver_attr_poly); 4610 driver_remove_file(&ppc440spe_adma_driver.driver, 4611 &driver_attr_enable); 4612 driver_remove_file(&ppc440spe_adma_driver.driver, 4613 &driver_attr_devices); 4614 platform_driver_unregister(&ppc440spe_adma_driver); 4615 dcr_unmap(ppc440spe_mq_dcr_host, ppc440spe_mq_dcr_len); 4616 kfree(ppc440spe_dma_fifo_buf); 4617 } 4618 4619 arch_initcall(ppc440spe_adma_init); 4620 module_exit(ppc440spe_adma_exit); 4621 4622 MODULE_AUTHOR("Yuri Tikhonov <yur@emcraft.com>"); 4623 MODULE_DESCRIPTION("PPC440SPE ADMA Engine Driver"); 4624 MODULE_LICENSE("GPL"); 4625