1 /* 2 * Copyright (C) 2006-2009 DENX Software Engineering. 3 * 4 * Author: Yuri Tikhonov <yur@emcraft.com> 5 * 6 * Further porting to arch/powerpc by 7 * Anatolij Gustschin <agust@denx.de> 8 * 9 * This program is free software; you can redistribute it and/or modify it 10 * under the terms of the GNU General Public License as published by the Free 11 * Software Foundation; either version 2 of the License, or (at your option) 12 * any later version. 13 * 14 * This program is distributed in the hope that it will be useful, but WITHOUT 15 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 16 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 17 * more details. 18 * 19 * The full GNU General Public License is included in this distribution in the 20 * file called COPYING. 21 */ 22 23 /* 24 * This driver supports the asynchrounous DMA copy and RAID engines available 25 * on the AMCC PPC440SPe Processors. 26 * Based on the Intel Xscale(R) family of I/O Processors (IOP 32x, 33x, 134x) 27 * ADMA driver written by D.Williams. 28 */ 29 30 #include <linux/init.h> 31 #include <linux/module.h> 32 #include <linux/async_tx.h> 33 #include <linux/delay.h> 34 #include <linux/dma-mapping.h> 35 #include <linux/spinlock.h> 36 #include <linux/interrupt.h> 37 #include <linux/slab.h> 38 #include <linux/uaccess.h> 39 #include <linux/proc_fs.h> 40 #include <linux/of.h> 41 #include <linux/of_address.h> 42 #include <linux/of_irq.h> 43 #include <linux/of_platform.h> 44 #include <asm/dcr.h> 45 #include <asm/dcr-regs.h> 46 #include "adma.h" 47 #include "../dmaengine.h" 48 49 enum ppc_adma_init_code { 50 PPC_ADMA_INIT_OK = 0, 51 PPC_ADMA_INIT_MEMRES, 52 PPC_ADMA_INIT_MEMREG, 53 PPC_ADMA_INIT_ALLOC, 54 PPC_ADMA_INIT_COHERENT, 55 PPC_ADMA_INIT_CHANNEL, 56 PPC_ADMA_INIT_IRQ1, 57 PPC_ADMA_INIT_IRQ2, 58 PPC_ADMA_INIT_REGISTER 59 }; 60 61 static char *ppc_adma_errors[] = { 62 [PPC_ADMA_INIT_OK] = "ok", 63 [PPC_ADMA_INIT_MEMRES] = "failed to get memory resource", 64 [PPC_ADMA_INIT_MEMREG] = "failed to request memory region", 65 [PPC_ADMA_INIT_ALLOC] = "failed to allocate memory for adev " 66 "structure", 67 [PPC_ADMA_INIT_COHERENT] = "failed to allocate coherent memory for " 68 "hardware descriptors", 69 [PPC_ADMA_INIT_CHANNEL] = "failed to allocate memory for channel", 70 [PPC_ADMA_INIT_IRQ1] = "failed to request first irq", 71 [PPC_ADMA_INIT_IRQ2] = "failed to request second irq", 72 [PPC_ADMA_INIT_REGISTER] = "failed to register dma async device", 73 }; 74 75 static enum ppc_adma_init_code 76 ppc440spe_adma_devices[PPC440SPE_ADMA_ENGINES_NUM]; 77 78 struct ppc_dma_chan_ref { 79 struct dma_chan *chan; 80 struct list_head node; 81 }; 82 83 /* The list of channels exported by ppc440spe ADMA */ 84 struct list_head 85 ppc440spe_adma_chan_list = LIST_HEAD_INIT(ppc440spe_adma_chan_list); 86 87 /* This flag is set when want to refetch the xor chain in the interrupt 88 * handler 89 */ 90 static u32 do_xor_refetch; 91 92 /* Pointer to DMA0, DMA1 CP/CS FIFO */ 93 static void *ppc440spe_dma_fifo_buf; 94 95 /* Pointers to last submitted to DMA0, DMA1 CDBs */ 96 static struct ppc440spe_adma_desc_slot *chan_last_sub[3]; 97 static struct ppc440spe_adma_desc_slot *chan_first_cdb[3]; 98 99 /* Pointer to last linked and submitted xor CB */ 100 static struct ppc440spe_adma_desc_slot *xor_last_linked; 101 static struct ppc440spe_adma_desc_slot *xor_last_submit; 102 103 /* This array is used in data-check operations for storing a pattern */ 104 static char ppc440spe_qword[16]; 105 106 static atomic_t ppc440spe_adma_err_irq_ref; 107 static dcr_host_t ppc440spe_mq_dcr_host; 108 static unsigned int ppc440spe_mq_dcr_len; 109 110 /* Since RXOR operations use the common register (MQ0_CF2H) for setting-up 111 * the block size in transactions, then we do not allow to activate more than 112 * only one RXOR transactions simultaneously. So use this var to store 113 * the information about is RXOR currently active (PPC440SPE_RXOR_RUN bit is 114 * set) or not (PPC440SPE_RXOR_RUN is clear). 115 */ 116 static unsigned long ppc440spe_rxor_state; 117 118 /* These are used in enable & check routines 119 */ 120 static u32 ppc440spe_r6_enabled; 121 static struct ppc440spe_adma_chan *ppc440spe_r6_tchan; 122 static struct completion ppc440spe_r6_test_comp; 123 124 static int ppc440spe_adma_dma2rxor_prep_src( 125 struct ppc440spe_adma_desc_slot *desc, 126 struct ppc440spe_rxor *cursor, int index, 127 int src_cnt, u32 addr); 128 static void ppc440spe_adma_dma2rxor_set_src( 129 struct ppc440spe_adma_desc_slot *desc, 130 int index, dma_addr_t addr); 131 static void ppc440spe_adma_dma2rxor_set_mult( 132 struct ppc440spe_adma_desc_slot *desc, 133 int index, u8 mult); 134 135 #ifdef ADMA_LL_DEBUG 136 #define ADMA_LL_DBG(x) ({ if (1) x; 0; }) 137 #else 138 #define ADMA_LL_DBG(x) ({ if (0) x; 0; }) 139 #endif 140 141 static void print_cb(struct ppc440spe_adma_chan *chan, void *block) 142 { 143 struct dma_cdb *cdb; 144 struct xor_cb *cb; 145 int i; 146 147 switch (chan->device->id) { 148 case 0: 149 case 1: 150 cdb = block; 151 152 pr_debug("CDB at %p [%d]:\n" 153 "\t attr 0x%02x opc 0x%02x cnt 0x%08x\n" 154 "\t sg1u 0x%08x sg1l 0x%08x\n" 155 "\t sg2u 0x%08x sg2l 0x%08x\n" 156 "\t sg3u 0x%08x sg3l 0x%08x\n", 157 cdb, chan->device->id, 158 cdb->attr, cdb->opc, le32_to_cpu(cdb->cnt), 159 le32_to_cpu(cdb->sg1u), le32_to_cpu(cdb->sg1l), 160 le32_to_cpu(cdb->sg2u), le32_to_cpu(cdb->sg2l), 161 le32_to_cpu(cdb->sg3u), le32_to_cpu(cdb->sg3l) 162 ); 163 break; 164 case 2: 165 cb = block; 166 167 pr_debug("CB at %p [%d]:\n" 168 "\t cbc 0x%08x cbbc 0x%08x cbs 0x%08x\n" 169 "\t cbtah 0x%08x cbtal 0x%08x\n" 170 "\t cblah 0x%08x cblal 0x%08x\n", 171 cb, chan->device->id, 172 cb->cbc, cb->cbbc, cb->cbs, 173 cb->cbtah, cb->cbtal, 174 cb->cblah, cb->cblal); 175 for (i = 0; i < 16; i++) { 176 if (i && !cb->ops[i].h && !cb->ops[i].l) 177 continue; 178 pr_debug("\t ops[%2d]: h 0x%08x l 0x%08x\n", 179 i, cb->ops[i].h, cb->ops[i].l); 180 } 181 break; 182 } 183 } 184 185 static void print_cb_list(struct ppc440spe_adma_chan *chan, 186 struct ppc440spe_adma_desc_slot *iter) 187 { 188 for (; iter; iter = iter->hw_next) 189 print_cb(chan, iter->hw_desc); 190 } 191 192 static void prep_dma_xor_dbg(int id, dma_addr_t dst, dma_addr_t *src, 193 unsigned int src_cnt) 194 { 195 int i; 196 197 pr_debug("\n%s(%d):\nsrc: ", __func__, id); 198 for (i = 0; i < src_cnt; i++) 199 pr_debug("\t0x%016llx ", src[i]); 200 pr_debug("dst:\n\t0x%016llx\n", dst); 201 } 202 203 static void prep_dma_pq_dbg(int id, dma_addr_t *dst, dma_addr_t *src, 204 unsigned int src_cnt) 205 { 206 int i; 207 208 pr_debug("\n%s(%d):\nsrc: ", __func__, id); 209 for (i = 0; i < src_cnt; i++) 210 pr_debug("\t0x%016llx ", src[i]); 211 pr_debug("dst: "); 212 for (i = 0; i < 2; i++) 213 pr_debug("\t0x%016llx ", dst[i]); 214 } 215 216 static void prep_dma_pqzero_sum_dbg(int id, dma_addr_t *src, 217 unsigned int src_cnt, 218 const unsigned char *scf) 219 { 220 int i; 221 222 pr_debug("\n%s(%d):\nsrc(coef): ", __func__, id); 223 if (scf) { 224 for (i = 0; i < src_cnt; i++) 225 pr_debug("\t0x%016llx(0x%02x) ", src[i], scf[i]); 226 } else { 227 for (i = 0; i < src_cnt; i++) 228 pr_debug("\t0x%016llx(no) ", src[i]); 229 } 230 231 pr_debug("dst: "); 232 for (i = 0; i < 2; i++) 233 pr_debug("\t0x%016llx ", src[src_cnt + i]); 234 } 235 236 /****************************************************************************** 237 * Command (Descriptor) Blocks low-level routines 238 ******************************************************************************/ 239 /** 240 * ppc440spe_desc_init_interrupt - initialize the descriptor for INTERRUPT 241 * pseudo operation 242 */ 243 static void ppc440spe_desc_init_interrupt(struct ppc440spe_adma_desc_slot *desc, 244 struct ppc440spe_adma_chan *chan) 245 { 246 struct xor_cb *p; 247 248 switch (chan->device->id) { 249 case PPC440SPE_XOR_ID: 250 p = desc->hw_desc; 251 memset(desc->hw_desc, 0, sizeof(struct xor_cb)); 252 /* NOP with Command Block Complete Enable */ 253 p->cbc = XOR_CBCR_CBCE_BIT; 254 break; 255 case PPC440SPE_DMA0_ID: 256 case PPC440SPE_DMA1_ID: 257 memset(desc->hw_desc, 0, sizeof(struct dma_cdb)); 258 /* NOP with interrupt */ 259 set_bit(PPC440SPE_DESC_INT, &desc->flags); 260 break; 261 default: 262 printk(KERN_ERR "Unsupported id %d in %s\n", chan->device->id, 263 __func__); 264 break; 265 } 266 } 267 268 /** 269 * ppc440spe_desc_init_null_xor - initialize the descriptor for NULL XOR 270 * pseudo operation 271 */ 272 static void ppc440spe_desc_init_null_xor(struct ppc440spe_adma_desc_slot *desc) 273 { 274 memset(desc->hw_desc, 0, sizeof(struct xor_cb)); 275 desc->hw_next = NULL; 276 desc->src_cnt = 0; 277 desc->dst_cnt = 1; 278 } 279 280 /** 281 * ppc440spe_desc_init_xor - initialize the descriptor for XOR operation 282 */ 283 static void ppc440spe_desc_init_xor(struct ppc440spe_adma_desc_slot *desc, 284 int src_cnt, unsigned long flags) 285 { 286 struct xor_cb *hw_desc = desc->hw_desc; 287 288 memset(desc->hw_desc, 0, sizeof(struct xor_cb)); 289 desc->hw_next = NULL; 290 desc->src_cnt = src_cnt; 291 desc->dst_cnt = 1; 292 293 hw_desc->cbc = XOR_CBCR_TGT_BIT | src_cnt; 294 if (flags & DMA_PREP_INTERRUPT) 295 /* Enable interrupt on completion */ 296 hw_desc->cbc |= XOR_CBCR_CBCE_BIT; 297 } 298 299 /** 300 * ppc440spe_desc_init_dma2pq - initialize the descriptor for PQ 301 * operation in DMA2 controller 302 */ 303 static void ppc440spe_desc_init_dma2pq(struct ppc440spe_adma_desc_slot *desc, 304 int dst_cnt, int src_cnt, unsigned long flags) 305 { 306 struct xor_cb *hw_desc = desc->hw_desc; 307 308 memset(desc->hw_desc, 0, sizeof(struct xor_cb)); 309 desc->hw_next = NULL; 310 desc->src_cnt = src_cnt; 311 desc->dst_cnt = dst_cnt; 312 memset(desc->reverse_flags, 0, sizeof(desc->reverse_flags)); 313 desc->descs_per_op = 0; 314 315 hw_desc->cbc = XOR_CBCR_TGT_BIT; 316 if (flags & DMA_PREP_INTERRUPT) 317 /* Enable interrupt on completion */ 318 hw_desc->cbc |= XOR_CBCR_CBCE_BIT; 319 } 320 321 #define DMA_CTRL_FLAGS_LAST DMA_PREP_FENCE 322 #define DMA_PREP_ZERO_P (DMA_CTRL_FLAGS_LAST << 1) 323 #define DMA_PREP_ZERO_Q (DMA_PREP_ZERO_P << 1) 324 325 /** 326 * ppc440spe_desc_init_dma01pq - initialize the descriptors for PQ operation 327 * with DMA0/1 328 */ 329 static void ppc440spe_desc_init_dma01pq(struct ppc440spe_adma_desc_slot *desc, 330 int dst_cnt, int src_cnt, unsigned long flags, 331 unsigned long op) 332 { 333 struct dma_cdb *hw_desc; 334 struct ppc440spe_adma_desc_slot *iter; 335 u8 dopc; 336 337 /* Common initialization of a PQ descriptors chain */ 338 set_bits(op, &desc->flags); 339 desc->src_cnt = src_cnt; 340 desc->dst_cnt = dst_cnt; 341 342 /* WXOR MULTICAST if both P and Q are being computed 343 * MV_SG1_SG2 if Q only 344 */ 345 dopc = (desc->dst_cnt == DMA_DEST_MAX_NUM) ? 346 DMA_CDB_OPC_MULTICAST : DMA_CDB_OPC_MV_SG1_SG2; 347 348 list_for_each_entry(iter, &desc->group_list, chain_node) { 349 hw_desc = iter->hw_desc; 350 memset(iter->hw_desc, 0, sizeof(struct dma_cdb)); 351 352 if (likely(!list_is_last(&iter->chain_node, 353 &desc->group_list))) { 354 /* set 'next' pointer */ 355 iter->hw_next = list_entry(iter->chain_node.next, 356 struct ppc440spe_adma_desc_slot, chain_node); 357 clear_bit(PPC440SPE_DESC_INT, &iter->flags); 358 } else { 359 /* this is the last descriptor. 360 * this slot will be pasted from ADMA level 361 * each time it wants to configure parameters 362 * of the transaction (src, dst, ...) 363 */ 364 iter->hw_next = NULL; 365 if (flags & DMA_PREP_INTERRUPT) 366 set_bit(PPC440SPE_DESC_INT, &iter->flags); 367 else 368 clear_bit(PPC440SPE_DESC_INT, &iter->flags); 369 } 370 } 371 372 /* Set OPS depending on WXOR/RXOR type of operation */ 373 if (!test_bit(PPC440SPE_DESC_RXOR, &desc->flags)) { 374 /* This is a WXOR only chain: 375 * - first descriptors are for zeroing destinations 376 * if PPC440SPE_ZERO_P/Q set; 377 * - descriptors remained are for GF-XOR operations. 378 */ 379 iter = list_first_entry(&desc->group_list, 380 struct ppc440spe_adma_desc_slot, 381 chain_node); 382 383 if (test_bit(PPC440SPE_ZERO_P, &desc->flags)) { 384 hw_desc = iter->hw_desc; 385 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; 386 iter = list_first_entry(&iter->chain_node, 387 struct ppc440spe_adma_desc_slot, 388 chain_node); 389 } 390 391 if (test_bit(PPC440SPE_ZERO_Q, &desc->flags)) { 392 hw_desc = iter->hw_desc; 393 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; 394 iter = list_first_entry(&iter->chain_node, 395 struct ppc440spe_adma_desc_slot, 396 chain_node); 397 } 398 399 list_for_each_entry_from(iter, &desc->group_list, chain_node) { 400 hw_desc = iter->hw_desc; 401 hw_desc->opc = dopc; 402 } 403 } else { 404 /* This is either RXOR-only or mixed RXOR/WXOR */ 405 406 /* The first 1 or 2 slots in chain are always RXOR, 407 * if need to calculate P & Q, then there are two 408 * RXOR slots; if only P or only Q, then there is one 409 */ 410 iter = list_first_entry(&desc->group_list, 411 struct ppc440spe_adma_desc_slot, 412 chain_node); 413 hw_desc = iter->hw_desc; 414 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; 415 416 if (desc->dst_cnt == DMA_DEST_MAX_NUM) { 417 iter = list_first_entry(&iter->chain_node, 418 struct ppc440spe_adma_desc_slot, 419 chain_node); 420 hw_desc = iter->hw_desc; 421 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; 422 } 423 424 /* The remaining descs (if any) are WXORs */ 425 if (test_bit(PPC440SPE_DESC_WXOR, &desc->flags)) { 426 iter = list_first_entry(&iter->chain_node, 427 struct ppc440spe_adma_desc_slot, 428 chain_node); 429 list_for_each_entry_from(iter, &desc->group_list, 430 chain_node) { 431 hw_desc = iter->hw_desc; 432 hw_desc->opc = dopc; 433 } 434 } 435 } 436 } 437 438 /** 439 * ppc440spe_desc_init_dma01pqzero_sum - initialize the descriptor 440 * for PQ_ZERO_SUM operation 441 */ 442 static void ppc440spe_desc_init_dma01pqzero_sum( 443 struct ppc440spe_adma_desc_slot *desc, 444 int dst_cnt, int src_cnt) 445 { 446 struct dma_cdb *hw_desc; 447 struct ppc440spe_adma_desc_slot *iter; 448 int i = 0; 449 u8 dopc = (dst_cnt == 2) ? DMA_CDB_OPC_MULTICAST : 450 DMA_CDB_OPC_MV_SG1_SG2; 451 /* 452 * Initialize starting from 2nd or 3rd descriptor dependent 453 * on dst_cnt. First one or two slots are for cloning P 454 * and/or Q to chan->pdest and/or chan->qdest as we have 455 * to preserve original P/Q. 456 */ 457 iter = list_first_entry(&desc->group_list, 458 struct ppc440spe_adma_desc_slot, chain_node); 459 iter = list_entry(iter->chain_node.next, 460 struct ppc440spe_adma_desc_slot, chain_node); 461 462 if (dst_cnt > 1) { 463 iter = list_entry(iter->chain_node.next, 464 struct ppc440spe_adma_desc_slot, chain_node); 465 } 466 /* initialize each source descriptor in chain */ 467 list_for_each_entry_from(iter, &desc->group_list, chain_node) { 468 hw_desc = iter->hw_desc; 469 memset(iter->hw_desc, 0, sizeof(struct dma_cdb)); 470 iter->src_cnt = 0; 471 iter->dst_cnt = 0; 472 473 /* This is a ZERO_SUM operation: 474 * - <src_cnt> descriptors starting from 2nd or 3rd 475 * descriptor are for GF-XOR operations; 476 * - remaining <dst_cnt> descriptors are for checking the result 477 */ 478 if (i++ < src_cnt) 479 /* MV_SG1_SG2 if only Q is being verified 480 * MULTICAST if both P and Q are being verified 481 */ 482 hw_desc->opc = dopc; 483 else 484 /* DMA_CDB_OPC_DCHECK128 operation */ 485 hw_desc->opc = DMA_CDB_OPC_DCHECK128; 486 487 if (likely(!list_is_last(&iter->chain_node, 488 &desc->group_list))) { 489 /* set 'next' pointer */ 490 iter->hw_next = list_entry(iter->chain_node.next, 491 struct ppc440spe_adma_desc_slot, 492 chain_node); 493 } else { 494 /* this is the last descriptor. 495 * this slot will be pasted from ADMA level 496 * each time it wants to configure parameters 497 * of the transaction (src, dst, ...) 498 */ 499 iter->hw_next = NULL; 500 /* always enable interrupt generation since we get 501 * the status of pqzero from the handler 502 */ 503 set_bit(PPC440SPE_DESC_INT, &iter->flags); 504 } 505 } 506 desc->src_cnt = src_cnt; 507 desc->dst_cnt = dst_cnt; 508 } 509 510 /** 511 * ppc440spe_desc_init_memcpy - initialize the descriptor for MEMCPY operation 512 */ 513 static void ppc440spe_desc_init_memcpy(struct ppc440spe_adma_desc_slot *desc, 514 unsigned long flags) 515 { 516 struct dma_cdb *hw_desc = desc->hw_desc; 517 518 memset(desc->hw_desc, 0, sizeof(struct dma_cdb)); 519 desc->hw_next = NULL; 520 desc->src_cnt = 1; 521 desc->dst_cnt = 1; 522 523 if (flags & DMA_PREP_INTERRUPT) 524 set_bit(PPC440SPE_DESC_INT, &desc->flags); 525 else 526 clear_bit(PPC440SPE_DESC_INT, &desc->flags); 527 528 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; 529 } 530 531 /** 532 * ppc440spe_desc_set_src_addr - set source address into the descriptor 533 */ 534 static void ppc440spe_desc_set_src_addr(struct ppc440spe_adma_desc_slot *desc, 535 struct ppc440spe_adma_chan *chan, 536 int src_idx, dma_addr_t addrh, 537 dma_addr_t addrl) 538 { 539 struct dma_cdb *dma_hw_desc; 540 struct xor_cb *xor_hw_desc; 541 phys_addr_t addr64, tmplow, tmphi; 542 543 switch (chan->device->id) { 544 case PPC440SPE_DMA0_ID: 545 case PPC440SPE_DMA1_ID: 546 if (!addrh) { 547 addr64 = addrl; 548 tmphi = (addr64 >> 32); 549 tmplow = (addr64 & 0xFFFFFFFF); 550 } else { 551 tmphi = addrh; 552 tmplow = addrl; 553 } 554 dma_hw_desc = desc->hw_desc; 555 dma_hw_desc->sg1l = cpu_to_le32((u32)tmplow); 556 dma_hw_desc->sg1u |= cpu_to_le32((u32)tmphi); 557 break; 558 case PPC440SPE_XOR_ID: 559 xor_hw_desc = desc->hw_desc; 560 xor_hw_desc->ops[src_idx].l = addrl; 561 xor_hw_desc->ops[src_idx].h |= addrh; 562 break; 563 } 564 } 565 566 /** 567 * ppc440spe_desc_set_src_mult - set source address mult into the descriptor 568 */ 569 static void ppc440spe_desc_set_src_mult(struct ppc440spe_adma_desc_slot *desc, 570 struct ppc440spe_adma_chan *chan, u32 mult_index, 571 int sg_index, unsigned char mult_value) 572 { 573 struct dma_cdb *dma_hw_desc; 574 struct xor_cb *xor_hw_desc; 575 u32 *psgu; 576 577 switch (chan->device->id) { 578 case PPC440SPE_DMA0_ID: 579 case PPC440SPE_DMA1_ID: 580 dma_hw_desc = desc->hw_desc; 581 582 switch (sg_index) { 583 /* for RXOR operations set multiplier 584 * into source cued address 585 */ 586 case DMA_CDB_SG_SRC: 587 psgu = &dma_hw_desc->sg1u; 588 break; 589 /* for WXOR operations set multiplier 590 * into destination cued address(es) 591 */ 592 case DMA_CDB_SG_DST1: 593 psgu = &dma_hw_desc->sg2u; 594 break; 595 case DMA_CDB_SG_DST2: 596 psgu = &dma_hw_desc->sg3u; 597 break; 598 default: 599 BUG(); 600 } 601 602 *psgu |= cpu_to_le32(mult_value << mult_index); 603 break; 604 case PPC440SPE_XOR_ID: 605 xor_hw_desc = desc->hw_desc; 606 break; 607 default: 608 BUG(); 609 } 610 } 611 612 /** 613 * ppc440spe_desc_set_dest_addr - set destination address into the descriptor 614 */ 615 static void ppc440spe_desc_set_dest_addr(struct ppc440spe_adma_desc_slot *desc, 616 struct ppc440spe_adma_chan *chan, 617 dma_addr_t addrh, dma_addr_t addrl, 618 u32 dst_idx) 619 { 620 struct dma_cdb *dma_hw_desc; 621 struct xor_cb *xor_hw_desc; 622 phys_addr_t addr64, tmphi, tmplow; 623 u32 *psgu, *psgl; 624 625 switch (chan->device->id) { 626 case PPC440SPE_DMA0_ID: 627 case PPC440SPE_DMA1_ID: 628 if (!addrh) { 629 addr64 = addrl; 630 tmphi = (addr64 >> 32); 631 tmplow = (addr64 & 0xFFFFFFFF); 632 } else { 633 tmphi = addrh; 634 tmplow = addrl; 635 } 636 dma_hw_desc = desc->hw_desc; 637 638 psgu = dst_idx ? &dma_hw_desc->sg3u : &dma_hw_desc->sg2u; 639 psgl = dst_idx ? &dma_hw_desc->sg3l : &dma_hw_desc->sg2l; 640 641 *psgl = cpu_to_le32((u32)tmplow); 642 *psgu |= cpu_to_le32((u32)tmphi); 643 break; 644 case PPC440SPE_XOR_ID: 645 xor_hw_desc = desc->hw_desc; 646 xor_hw_desc->cbtal = addrl; 647 xor_hw_desc->cbtah |= addrh; 648 break; 649 } 650 } 651 652 /** 653 * ppc440spe_desc_set_byte_count - set number of data bytes involved 654 * into the operation 655 */ 656 static void ppc440spe_desc_set_byte_count(struct ppc440spe_adma_desc_slot *desc, 657 struct ppc440spe_adma_chan *chan, 658 u32 byte_count) 659 { 660 struct dma_cdb *dma_hw_desc; 661 struct xor_cb *xor_hw_desc; 662 663 switch (chan->device->id) { 664 case PPC440SPE_DMA0_ID: 665 case PPC440SPE_DMA1_ID: 666 dma_hw_desc = desc->hw_desc; 667 dma_hw_desc->cnt = cpu_to_le32(byte_count); 668 break; 669 case PPC440SPE_XOR_ID: 670 xor_hw_desc = desc->hw_desc; 671 xor_hw_desc->cbbc = byte_count; 672 break; 673 } 674 } 675 676 /** 677 * ppc440spe_desc_set_rxor_block_size - set RXOR block size 678 */ 679 static inline void ppc440spe_desc_set_rxor_block_size(u32 byte_count) 680 { 681 /* assume that byte_count is aligned on the 512-boundary; 682 * thus write it directly to the register (bits 23:31 are 683 * reserved there). 684 */ 685 dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_CF2H, byte_count); 686 } 687 688 /** 689 * ppc440spe_desc_set_dcheck - set CHECK pattern 690 */ 691 static void ppc440spe_desc_set_dcheck(struct ppc440spe_adma_desc_slot *desc, 692 struct ppc440spe_adma_chan *chan, u8 *qword) 693 { 694 struct dma_cdb *dma_hw_desc; 695 696 switch (chan->device->id) { 697 case PPC440SPE_DMA0_ID: 698 case PPC440SPE_DMA1_ID: 699 dma_hw_desc = desc->hw_desc; 700 iowrite32(qword[0], &dma_hw_desc->sg3l); 701 iowrite32(qword[4], &dma_hw_desc->sg3u); 702 iowrite32(qword[8], &dma_hw_desc->sg2l); 703 iowrite32(qword[12], &dma_hw_desc->sg2u); 704 break; 705 default: 706 BUG(); 707 } 708 } 709 710 /** 711 * ppc440spe_xor_set_link - set link address in xor CB 712 */ 713 static void ppc440spe_xor_set_link(struct ppc440spe_adma_desc_slot *prev_desc, 714 struct ppc440spe_adma_desc_slot *next_desc) 715 { 716 struct xor_cb *xor_hw_desc = prev_desc->hw_desc; 717 718 if (unlikely(!next_desc || !(next_desc->phys))) { 719 printk(KERN_ERR "%s: next_desc=0x%p; next_desc->phys=0x%llx\n", 720 __func__, next_desc, 721 next_desc ? next_desc->phys : 0); 722 BUG(); 723 } 724 725 xor_hw_desc->cbs = 0; 726 xor_hw_desc->cblal = next_desc->phys; 727 xor_hw_desc->cblah = 0; 728 xor_hw_desc->cbc |= XOR_CBCR_LNK_BIT; 729 } 730 731 /** 732 * ppc440spe_desc_set_link - set the address of descriptor following this 733 * descriptor in chain 734 */ 735 static void ppc440spe_desc_set_link(struct ppc440spe_adma_chan *chan, 736 struct ppc440spe_adma_desc_slot *prev_desc, 737 struct ppc440spe_adma_desc_slot *next_desc) 738 { 739 unsigned long flags; 740 struct ppc440spe_adma_desc_slot *tail = next_desc; 741 742 if (unlikely(!prev_desc || !next_desc || 743 (prev_desc->hw_next && prev_desc->hw_next != next_desc))) { 744 /* If previous next is overwritten something is wrong. 745 * though we may refetch from append to initiate list 746 * processing; in this case - it's ok. 747 */ 748 printk(KERN_ERR "%s: prev_desc=0x%p; next_desc=0x%p; " 749 "prev->hw_next=0x%p\n", __func__, prev_desc, 750 next_desc, prev_desc ? prev_desc->hw_next : 0); 751 BUG(); 752 } 753 754 local_irq_save(flags); 755 756 /* do s/w chaining both for DMA and XOR descriptors */ 757 prev_desc->hw_next = next_desc; 758 759 switch (chan->device->id) { 760 case PPC440SPE_DMA0_ID: 761 case PPC440SPE_DMA1_ID: 762 break; 763 case PPC440SPE_XOR_ID: 764 /* bind descriptor to the chain */ 765 while (tail->hw_next) 766 tail = tail->hw_next; 767 xor_last_linked = tail; 768 769 if (prev_desc == xor_last_submit) 770 /* do not link to the last submitted CB */ 771 break; 772 ppc440spe_xor_set_link(prev_desc, next_desc); 773 break; 774 } 775 776 local_irq_restore(flags); 777 } 778 779 /** 780 * ppc440spe_desc_get_link - get the address of the descriptor that 781 * follows this one 782 */ 783 static inline u32 ppc440spe_desc_get_link(struct ppc440spe_adma_desc_slot *desc, 784 struct ppc440spe_adma_chan *chan) 785 { 786 if (!desc->hw_next) 787 return 0; 788 789 return desc->hw_next->phys; 790 } 791 792 /** 793 * ppc440spe_desc_is_aligned - check alignment 794 */ 795 static inline int ppc440spe_desc_is_aligned( 796 struct ppc440spe_adma_desc_slot *desc, int num_slots) 797 { 798 return (desc->idx & (num_slots - 1)) ? 0 : 1; 799 } 800 801 /** 802 * ppc440spe_chan_xor_slot_count - get the number of slots necessary for 803 * XOR operation 804 */ 805 static int ppc440spe_chan_xor_slot_count(size_t len, int src_cnt, 806 int *slots_per_op) 807 { 808 int slot_cnt; 809 810 /* each XOR descriptor provides up to 16 source operands */ 811 slot_cnt = *slots_per_op = (src_cnt + XOR_MAX_OPS - 1)/XOR_MAX_OPS; 812 813 if (likely(len <= PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT)) 814 return slot_cnt; 815 816 printk(KERN_ERR "%s: len %d > max %d !!\n", 817 __func__, len, PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT); 818 BUG(); 819 return slot_cnt; 820 } 821 822 /** 823 * ppc440spe_dma2_pq_slot_count - get the number of slots necessary for 824 * DMA2 PQ operation 825 */ 826 static int ppc440spe_dma2_pq_slot_count(dma_addr_t *srcs, 827 int src_cnt, size_t len) 828 { 829 signed long long order = 0; 830 int state = 0; 831 int addr_count = 0; 832 int i; 833 for (i = 1; i < src_cnt; i++) { 834 dma_addr_t cur_addr = srcs[i]; 835 dma_addr_t old_addr = srcs[i-1]; 836 switch (state) { 837 case 0: 838 if (cur_addr == old_addr + len) { 839 /* direct RXOR */ 840 order = 1; 841 state = 1; 842 if (i == src_cnt-1) 843 addr_count++; 844 } else if (old_addr == cur_addr + len) { 845 /* reverse RXOR */ 846 order = -1; 847 state = 1; 848 if (i == src_cnt-1) 849 addr_count++; 850 } else { 851 state = 3; 852 } 853 break; 854 case 1: 855 if (i == src_cnt-2 || (order == -1 856 && cur_addr != old_addr - len)) { 857 order = 0; 858 state = 0; 859 addr_count++; 860 } else if (cur_addr == old_addr + len*order) { 861 state = 2; 862 if (i == src_cnt-1) 863 addr_count++; 864 } else if (cur_addr == old_addr + 2*len) { 865 state = 2; 866 if (i == src_cnt-1) 867 addr_count++; 868 } else if (cur_addr == old_addr + 3*len) { 869 state = 2; 870 if (i == src_cnt-1) 871 addr_count++; 872 } else { 873 order = 0; 874 state = 0; 875 addr_count++; 876 } 877 break; 878 case 2: 879 order = 0; 880 state = 0; 881 addr_count++; 882 break; 883 } 884 if (state == 3) 885 break; 886 } 887 if (src_cnt <= 1 || (state != 1 && state != 2)) { 888 pr_err("%s: src_cnt=%d, state=%d, addr_count=%d, order=%lld\n", 889 __func__, src_cnt, state, addr_count, order); 890 for (i = 0; i < src_cnt; i++) 891 pr_err("\t[%d] 0x%llx \n", i, srcs[i]); 892 BUG(); 893 } 894 895 return (addr_count + XOR_MAX_OPS - 1) / XOR_MAX_OPS; 896 } 897 898 899 /****************************************************************************** 900 * ADMA channel low-level routines 901 ******************************************************************************/ 902 903 static u32 904 ppc440spe_chan_get_current_descriptor(struct ppc440spe_adma_chan *chan); 905 static void ppc440spe_chan_append(struct ppc440spe_adma_chan *chan); 906 907 /** 908 * ppc440spe_adma_device_clear_eot_status - interrupt ack to XOR or DMA engine 909 */ 910 static void ppc440spe_adma_device_clear_eot_status( 911 struct ppc440spe_adma_chan *chan) 912 { 913 struct dma_regs *dma_reg; 914 struct xor_regs *xor_reg; 915 u8 *p = chan->device->dma_desc_pool_virt; 916 struct dma_cdb *cdb; 917 u32 rv, i; 918 919 switch (chan->device->id) { 920 case PPC440SPE_DMA0_ID: 921 case PPC440SPE_DMA1_ID: 922 /* read FIFO to ack */ 923 dma_reg = chan->device->dma_reg; 924 while ((rv = ioread32(&dma_reg->csfpl))) { 925 i = rv & DMA_CDB_ADDR_MSK; 926 cdb = (struct dma_cdb *)&p[i - 927 (u32)chan->device->dma_desc_pool]; 928 929 /* Clear opcode to ack. This is necessary for 930 * ZeroSum operations only 931 */ 932 cdb->opc = 0; 933 934 if (test_bit(PPC440SPE_RXOR_RUN, 935 &ppc440spe_rxor_state)) { 936 /* probably this is a completed RXOR op, 937 * get pointer to CDB using the fact that 938 * physical and virtual addresses of CDB 939 * in pools have the same offsets 940 */ 941 if (le32_to_cpu(cdb->sg1u) & 942 DMA_CUED_XOR_BASE) { 943 /* this is a RXOR */ 944 clear_bit(PPC440SPE_RXOR_RUN, 945 &ppc440spe_rxor_state); 946 } 947 } 948 949 if (rv & DMA_CDB_STATUS_MSK) { 950 /* ZeroSum check failed 951 */ 952 struct ppc440spe_adma_desc_slot *iter; 953 dma_addr_t phys = rv & ~DMA_CDB_MSK; 954 955 /* 956 * Update the status of corresponding 957 * descriptor. 958 */ 959 list_for_each_entry(iter, &chan->chain, 960 chain_node) { 961 if (iter->phys == phys) 962 break; 963 } 964 /* 965 * if cannot find the corresponding 966 * slot it's a bug 967 */ 968 BUG_ON(&iter->chain_node == &chan->chain); 969 970 if (iter->xor_check_result) { 971 if (test_bit(PPC440SPE_DESC_PCHECK, 972 &iter->flags)) { 973 *iter->xor_check_result |= 974 SUM_CHECK_P_RESULT; 975 } else 976 if (test_bit(PPC440SPE_DESC_QCHECK, 977 &iter->flags)) { 978 *iter->xor_check_result |= 979 SUM_CHECK_Q_RESULT; 980 } else 981 BUG(); 982 } 983 } 984 } 985 986 rv = ioread32(&dma_reg->dsts); 987 if (rv) { 988 pr_err("DMA%d err status: 0x%x\n", 989 chan->device->id, rv); 990 /* write back to clear */ 991 iowrite32(rv, &dma_reg->dsts); 992 } 993 break; 994 case PPC440SPE_XOR_ID: 995 /* reset status bits to ack */ 996 xor_reg = chan->device->xor_reg; 997 rv = ioread32be(&xor_reg->sr); 998 iowrite32be(rv, &xor_reg->sr); 999 1000 if (rv & (XOR_IE_ICBIE_BIT|XOR_IE_ICIE_BIT|XOR_IE_RPTIE_BIT)) { 1001 if (rv & XOR_IE_RPTIE_BIT) { 1002 /* Read PLB Timeout Error. 1003 * Try to resubmit the CB 1004 */ 1005 u32 val = ioread32be(&xor_reg->ccbalr); 1006 1007 iowrite32be(val, &xor_reg->cblalr); 1008 1009 val = ioread32be(&xor_reg->crsr); 1010 iowrite32be(val | XOR_CRSR_XAE_BIT, 1011 &xor_reg->crsr); 1012 } else 1013 pr_err("XOR ERR 0x%x status\n", rv); 1014 break; 1015 } 1016 1017 /* if the XORcore is idle, but there are unprocessed CBs 1018 * then refetch the s/w chain here 1019 */ 1020 if (!(ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT) && 1021 do_xor_refetch) 1022 ppc440spe_chan_append(chan); 1023 break; 1024 } 1025 } 1026 1027 /** 1028 * ppc440spe_chan_is_busy - get the channel status 1029 */ 1030 static int ppc440spe_chan_is_busy(struct ppc440spe_adma_chan *chan) 1031 { 1032 struct dma_regs *dma_reg; 1033 struct xor_regs *xor_reg; 1034 int busy = 0; 1035 1036 switch (chan->device->id) { 1037 case PPC440SPE_DMA0_ID: 1038 case PPC440SPE_DMA1_ID: 1039 dma_reg = chan->device->dma_reg; 1040 /* if command FIFO's head and tail pointers are equal and 1041 * status tail is the same as command, then channel is free 1042 */ 1043 if (ioread16(&dma_reg->cpfhp) != ioread16(&dma_reg->cpftp) || 1044 ioread16(&dma_reg->cpftp) != ioread16(&dma_reg->csftp)) 1045 busy = 1; 1046 break; 1047 case PPC440SPE_XOR_ID: 1048 /* use the special status bit for the XORcore 1049 */ 1050 xor_reg = chan->device->xor_reg; 1051 busy = (ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT) ? 1 : 0; 1052 break; 1053 } 1054 1055 return busy; 1056 } 1057 1058 /** 1059 * ppc440spe_chan_set_first_xor_descriptor - init XORcore chain 1060 */ 1061 static void ppc440spe_chan_set_first_xor_descriptor( 1062 struct ppc440spe_adma_chan *chan, 1063 struct ppc440spe_adma_desc_slot *next_desc) 1064 { 1065 struct xor_regs *xor_reg = chan->device->xor_reg; 1066 1067 if (ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT) 1068 printk(KERN_INFO "%s: Warn: XORcore is running " 1069 "when try to set the first CDB!\n", 1070 __func__); 1071 1072 xor_last_submit = xor_last_linked = next_desc; 1073 1074 iowrite32be(XOR_CRSR_64BA_BIT, &xor_reg->crsr); 1075 1076 iowrite32be(next_desc->phys, &xor_reg->cblalr); 1077 iowrite32be(0, &xor_reg->cblahr); 1078 iowrite32be(ioread32be(&xor_reg->cbcr) | XOR_CBCR_LNK_BIT, 1079 &xor_reg->cbcr); 1080 1081 chan->hw_chain_inited = 1; 1082 } 1083 1084 /** 1085 * ppc440spe_dma_put_desc - put DMA0,1 descriptor to FIFO. 1086 * called with irqs disabled 1087 */ 1088 static void ppc440spe_dma_put_desc(struct ppc440spe_adma_chan *chan, 1089 struct ppc440spe_adma_desc_slot *desc) 1090 { 1091 u32 pcdb; 1092 struct dma_regs *dma_reg = chan->device->dma_reg; 1093 1094 pcdb = desc->phys; 1095 if (!test_bit(PPC440SPE_DESC_INT, &desc->flags)) 1096 pcdb |= DMA_CDB_NO_INT; 1097 1098 chan_last_sub[chan->device->id] = desc; 1099 1100 ADMA_LL_DBG(print_cb(chan, desc->hw_desc)); 1101 1102 iowrite32(pcdb, &dma_reg->cpfpl); 1103 } 1104 1105 /** 1106 * ppc440spe_chan_append - update the h/w chain in the channel 1107 */ 1108 static void ppc440spe_chan_append(struct ppc440spe_adma_chan *chan) 1109 { 1110 struct xor_regs *xor_reg; 1111 struct ppc440spe_adma_desc_slot *iter; 1112 struct xor_cb *xcb; 1113 u32 cur_desc; 1114 unsigned long flags; 1115 1116 local_irq_save(flags); 1117 1118 switch (chan->device->id) { 1119 case PPC440SPE_DMA0_ID: 1120 case PPC440SPE_DMA1_ID: 1121 cur_desc = ppc440spe_chan_get_current_descriptor(chan); 1122 1123 if (likely(cur_desc)) { 1124 iter = chan_last_sub[chan->device->id]; 1125 BUG_ON(!iter); 1126 } else { 1127 /* first peer */ 1128 iter = chan_first_cdb[chan->device->id]; 1129 BUG_ON(!iter); 1130 ppc440spe_dma_put_desc(chan, iter); 1131 chan->hw_chain_inited = 1; 1132 } 1133 1134 /* is there something new to append */ 1135 if (!iter->hw_next) 1136 break; 1137 1138 /* flush descriptors from the s/w queue to fifo */ 1139 list_for_each_entry_continue(iter, &chan->chain, chain_node) { 1140 ppc440spe_dma_put_desc(chan, iter); 1141 if (!iter->hw_next) 1142 break; 1143 } 1144 break; 1145 case PPC440SPE_XOR_ID: 1146 /* update h/w links and refetch */ 1147 if (!xor_last_submit->hw_next) 1148 break; 1149 1150 xor_reg = chan->device->xor_reg; 1151 /* the last linked CDB has to generate an interrupt 1152 * that we'd be able to append the next lists to h/w 1153 * regardless of the XOR engine state at the moment of 1154 * appending of these next lists 1155 */ 1156 xcb = xor_last_linked->hw_desc; 1157 xcb->cbc |= XOR_CBCR_CBCE_BIT; 1158 1159 if (!(ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT)) { 1160 /* XORcore is idle. Refetch now */ 1161 do_xor_refetch = 0; 1162 ppc440spe_xor_set_link(xor_last_submit, 1163 xor_last_submit->hw_next); 1164 1165 ADMA_LL_DBG(print_cb_list(chan, 1166 xor_last_submit->hw_next)); 1167 1168 xor_last_submit = xor_last_linked; 1169 iowrite32be(ioread32be(&xor_reg->crsr) | 1170 XOR_CRSR_RCBE_BIT | XOR_CRSR_64BA_BIT, 1171 &xor_reg->crsr); 1172 } else { 1173 /* XORcore is running. Refetch later in the handler */ 1174 do_xor_refetch = 1; 1175 } 1176 1177 break; 1178 } 1179 1180 local_irq_restore(flags); 1181 } 1182 1183 /** 1184 * ppc440spe_chan_get_current_descriptor - get the currently executed descriptor 1185 */ 1186 static u32 1187 ppc440spe_chan_get_current_descriptor(struct ppc440spe_adma_chan *chan) 1188 { 1189 struct dma_regs *dma_reg; 1190 struct xor_regs *xor_reg; 1191 1192 if (unlikely(!chan->hw_chain_inited)) 1193 /* h/w descriptor chain is not initialized yet */ 1194 return 0; 1195 1196 switch (chan->device->id) { 1197 case PPC440SPE_DMA0_ID: 1198 case PPC440SPE_DMA1_ID: 1199 dma_reg = chan->device->dma_reg; 1200 return ioread32(&dma_reg->acpl) & (~DMA_CDB_MSK); 1201 case PPC440SPE_XOR_ID: 1202 xor_reg = chan->device->xor_reg; 1203 return ioread32be(&xor_reg->ccbalr); 1204 } 1205 return 0; 1206 } 1207 1208 /** 1209 * ppc440spe_chan_run - enable the channel 1210 */ 1211 static void ppc440spe_chan_run(struct ppc440spe_adma_chan *chan) 1212 { 1213 struct xor_regs *xor_reg; 1214 1215 switch (chan->device->id) { 1216 case PPC440SPE_DMA0_ID: 1217 case PPC440SPE_DMA1_ID: 1218 /* DMAs are always enabled, do nothing */ 1219 break; 1220 case PPC440SPE_XOR_ID: 1221 /* drain write buffer */ 1222 xor_reg = chan->device->xor_reg; 1223 1224 /* fetch descriptor pointed to in <link> */ 1225 iowrite32be(XOR_CRSR_64BA_BIT | XOR_CRSR_XAE_BIT, 1226 &xor_reg->crsr); 1227 break; 1228 } 1229 } 1230 1231 /****************************************************************************** 1232 * ADMA device level 1233 ******************************************************************************/ 1234 1235 static void ppc440spe_chan_start_null_xor(struct ppc440spe_adma_chan *chan); 1236 static int ppc440spe_adma_alloc_chan_resources(struct dma_chan *chan); 1237 1238 static dma_cookie_t 1239 ppc440spe_adma_tx_submit(struct dma_async_tx_descriptor *tx); 1240 1241 static void ppc440spe_adma_set_dest(struct ppc440spe_adma_desc_slot *tx, 1242 dma_addr_t addr, int index); 1243 static void 1244 ppc440spe_adma_memcpy_xor_set_src(struct ppc440spe_adma_desc_slot *tx, 1245 dma_addr_t addr, int index); 1246 1247 static void 1248 ppc440spe_adma_pq_set_dest(struct ppc440spe_adma_desc_slot *tx, 1249 dma_addr_t *paddr, unsigned long flags); 1250 static void 1251 ppc440spe_adma_pq_set_src(struct ppc440spe_adma_desc_slot *tx, 1252 dma_addr_t addr, int index); 1253 static void 1254 ppc440spe_adma_pq_set_src_mult(struct ppc440spe_adma_desc_slot *tx, 1255 unsigned char mult, int index, int dst_pos); 1256 static void 1257 ppc440spe_adma_pqzero_sum_set_dest(struct ppc440spe_adma_desc_slot *tx, 1258 dma_addr_t paddr, dma_addr_t qaddr); 1259 1260 static struct page *ppc440spe_rxor_srcs[32]; 1261 1262 /** 1263 * ppc440spe_can_rxor - check if the operands may be processed with RXOR 1264 */ 1265 static int ppc440spe_can_rxor(struct page **srcs, int src_cnt, size_t len) 1266 { 1267 int i, order = 0, state = 0; 1268 int idx = 0; 1269 1270 if (unlikely(!(src_cnt > 1))) 1271 return 0; 1272 1273 BUG_ON(src_cnt > ARRAY_SIZE(ppc440spe_rxor_srcs)); 1274 1275 /* Skip holes in the source list before checking */ 1276 for (i = 0; i < src_cnt; i++) { 1277 if (!srcs[i]) 1278 continue; 1279 ppc440spe_rxor_srcs[idx++] = srcs[i]; 1280 } 1281 src_cnt = idx; 1282 1283 for (i = 1; i < src_cnt; i++) { 1284 char *cur_addr = page_address(ppc440spe_rxor_srcs[i]); 1285 char *old_addr = page_address(ppc440spe_rxor_srcs[i - 1]); 1286 1287 switch (state) { 1288 case 0: 1289 if (cur_addr == old_addr + len) { 1290 /* direct RXOR */ 1291 order = 1; 1292 state = 1; 1293 } else if (old_addr == cur_addr + len) { 1294 /* reverse RXOR */ 1295 order = -1; 1296 state = 1; 1297 } else 1298 goto out; 1299 break; 1300 case 1: 1301 if ((i == src_cnt - 2) || 1302 (order == -1 && cur_addr != old_addr - len)) { 1303 order = 0; 1304 state = 0; 1305 } else if ((cur_addr == old_addr + len * order) || 1306 (cur_addr == old_addr + 2 * len) || 1307 (cur_addr == old_addr + 3 * len)) { 1308 state = 2; 1309 } else { 1310 order = 0; 1311 state = 0; 1312 } 1313 break; 1314 case 2: 1315 order = 0; 1316 state = 0; 1317 break; 1318 } 1319 } 1320 1321 out: 1322 if (state == 1 || state == 2) 1323 return 1; 1324 1325 return 0; 1326 } 1327 1328 /** 1329 * ppc440spe_adma_device_estimate - estimate the efficiency of processing 1330 * the operation given on this channel. It's assumed that 'chan' is 1331 * capable to process 'cap' type of operation. 1332 * @chan: channel to use 1333 * @cap: type of transaction 1334 * @dst_lst: array of destination pointers 1335 * @dst_cnt: number of destination operands 1336 * @src_lst: array of source pointers 1337 * @src_cnt: number of source operands 1338 * @src_sz: size of each source operand 1339 */ 1340 static int ppc440spe_adma_estimate(struct dma_chan *chan, 1341 enum dma_transaction_type cap, struct page **dst_lst, int dst_cnt, 1342 struct page **src_lst, int src_cnt, size_t src_sz) 1343 { 1344 int ef = 1; 1345 1346 if (cap == DMA_PQ || cap == DMA_PQ_VAL) { 1347 /* If RAID-6 capabilities were not activated don't try 1348 * to use them 1349 */ 1350 if (unlikely(!ppc440spe_r6_enabled)) 1351 return -1; 1352 } 1353 /* In the current implementation of ppc440spe ADMA driver it 1354 * makes sense to pick out only pq case, because it may be 1355 * processed: 1356 * (1) either using Biskup method on DMA2; 1357 * (2) or on DMA0/1. 1358 * Thus we give a favour to (1) if the sources are suitable; 1359 * else let it be processed on one of the DMA0/1 engines. 1360 * In the sum_product case where destination is also the 1361 * source process it on DMA0/1 only. 1362 */ 1363 if (cap == DMA_PQ && chan->chan_id == PPC440SPE_XOR_ID) { 1364 1365 if (dst_cnt == 1 && src_cnt == 2 && dst_lst[0] == src_lst[1]) 1366 ef = 0; /* sum_product case, process on DMA0/1 */ 1367 else if (ppc440spe_can_rxor(src_lst, src_cnt, src_sz)) 1368 ef = 3; /* override (DMA0/1 + idle) */ 1369 else 1370 ef = 0; /* can't process on DMA2 if !rxor */ 1371 } 1372 1373 /* channel idleness increases the priority */ 1374 if (likely(ef) && 1375 !ppc440spe_chan_is_busy(to_ppc440spe_adma_chan(chan))) 1376 ef++; 1377 1378 return ef; 1379 } 1380 1381 struct dma_chan * 1382 ppc440spe_async_tx_find_best_channel(enum dma_transaction_type cap, 1383 struct page **dst_lst, int dst_cnt, struct page **src_lst, 1384 int src_cnt, size_t src_sz) 1385 { 1386 struct dma_chan *best_chan = NULL; 1387 struct ppc_dma_chan_ref *ref; 1388 int best_rank = -1; 1389 1390 if (unlikely(!src_sz)) 1391 return NULL; 1392 if (src_sz > PAGE_SIZE) { 1393 /* 1394 * should a user of the api ever pass > PAGE_SIZE requests 1395 * we sort out cases where temporary page-sized buffers 1396 * are used. 1397 */ 1398 switch (cap) { 1399 case DMA_PQ: 1400 if (src_cnt == 1 && dst_lst[1] == src_lst[0]) 1401 return NULL; 1402 if (src_cnt == 2 && dst_lst[1] == src_lst[1]) 1403 return NULL; 1404 break; 1405 case DMA_PQ_VAL: 1406 case DMA_XOR_VAL: 1407 return NULL; 1408 default: 1409 break; 1410 } 1411 } 1412 1413 list_for_each_entry(ref, &ppc440spe_adma_chan_list, node) { 1414 if (dma_has_cap(cap, ref->chan->device->cap_mask)) { 1415 int rank; 1416 1417 rank = ppc440spe_adma_estimate(ref->chan, cap, dst_lst, 1418 dst_cnt, src_lst, src_cnt, src_sz); 1419 if (rank > best_rank) { 1420 best_rank = rank; 1421 best_chan = ref->chan; 1422 } 1423 } 1424 } 1425 1426 return best_chan; 1427 } 1428 EXPORT_SYMBOL_GPL(ppc440spe_async_tx_find_best_channel); 1429 1430 /** 1431 * ppc440spe_get_group_entry - get group entry with index idx 1432 * @tdesc: is the last allocated slot in the group. 1433 */ 1434 static struct ppc440spe_adma_desc_slot * 1435 ppc440spe_get_group_entry(struct ppc440spe_adma_desc_slot *tdesc, u32 entry_idx) 1436 { 1437 struct ppc440spe_adma_desc_slot *iter = tdesc->group_head; 1438 int i = 0; 1439 1440 if (entry_idx < 0 || entry_idx >= (tdesc->src_cnt + tdesc->dst_cnt)) { 1441 printk("%s: entry_idx %d, src_cnt %d, dst_cnt %d\n", 1442 __func__, entry_idx, tdesc->src_cnt, tdesc->dst_cnt); 1443 BUG(); 1444 } 1445 1446 list_for_each_entry(iter, &tdesc->group_list, chain_node) { 1447 if (i++ == entry_idx) 1448 break; 1449 } 1450 return iter; 1451 } 1452 1453 /** 1454 * ppc440spe_adma_free_slots - flags descriptor slots for reuse 1455 * @slot: Slot to free 1456 * Caller must hold &ppc440spe_chan->lock while calling this function 1457 */ 1458 static void ppc440spe_adma_free_slots(struct ppc440spe_adma_desc_slot *slot, 1459 struct ppc440spe_adma_chan *chan) 1460 { 1461 int stride = slot->slots_per_op; 1462 1463 while (stride--) { 1464 slot->slots_per_op = 0; 1465 slot = list_entry(slot->slot_node.next, 1466 struct ppc440spe_adma_desc_slot, 1467 slot_node); 1468 } 1469 } 1470 1471 /** 1472 * ppc440spe_adma_run_tx_complete_actions - call functions to be called 1473 * upon completion 1474 */ 1475 static dma_cookie_t ppc440spe_adma_run_tx_complete_actions( 1476 struct ppc440spe_adma_desc_slot *desc, 1477 struct ppc440spe_adma_chan *chan, 1478 dma_cookie_t cookie) 1479 { 1480 BUG_ON(desc->async_tx.cookie < 0); 1481 if (desc->async_tx.cookie > 0) { 1482 cookie = desc->async_tx.cookie; 1483 desc->async_tx.cookie = 0; 1484 1485 dma_descriptor_unmap(&desc->async_tx); 1486 /* call the callback (must not sleep or submit new 1487 * operations to this channel) 1488 */ 1489 dmaengine_desc_get_callback_invoke(&desc->async_tx, NULL); 1490 } 1491 1492 /* run dependent operations */ 1493 dma_run_dependencies(&desc->async_tx); 1494 1495 return cookie; 1496 } 1497 1498 /** 1499 * ppc440spe_adma_clean_slot - clean up CDB slot (if ack is set) 1500 */ 1501 static int ppc440spe_adma_clean_slot(struct ppc440spe_adma_desc_slot *desc, 1502 struct ppc440spe_adma_chan *chan) 1503 { 1504 /* the client is allowed to attach dependent operations 1505 * until 'ack' is set 1506 */ 1507 if (!async_tx_test_ack(&desc->async_tx)) 1508 return 0; 1509 1510 /* leave the last descriptor in the chain 1511 * so we can append to it 1512 */ 1513 if (list_is_last(&desc->chain_node, &chan->chain) || 1514 desc->phys == ppc440spe_chan_get_current_descriptor(chan)) 1515 return 1; 1516 1517 if (chan->device->id != PPC440SPE_XOR_ID) { 1518 /* our DMA interrupt handler clears opc field of 1519 * each processed descriptor. For all types of 1520 * operations except for ZeroSum we do not actually 1521 * need ack from the interrupt handler. ZeroSum is a 1522 * special case since the result of this operation 1523 * is available from the handler only, so if we see 1524 * such type of descriptor (which is unprocessed yet) 1525 * then leave it in chain. 1526 */ 1527 struct dma_cdb *cdb = desc->hw_desc; 1528 if (cdb->opc == DMA_CDB_OPC_DCHECK128) 1529 return 1; 1530 } 1531 1532 dev_dbg(chan->device->common.dev, "\tfree slot %llx: %d stride: %d\n", 1533 desc->phys, desc->idx, desc->slots_per_op); 1534 1535 list_del(&desc->chain_node); 1536 ppc440spe_adma_free_slots(desc, chan); 1537 return 0; 1538 } 1539 1540 /** 1541 * __ppc440spe_adma_slot_cleanup - this is the common clean-up routine 1542 * which runs through the channel CDBs list until reach the descriptor 1543 * currently processed. When routine determines that all CDBs of group 1544 * are completed then corresponding callbacks (if any) are called and slots 1545 * are freed. 1546 */ 1547 static void __ppc440spe_adma_slot_cleanup(struct ppc440spe_adma_chan *chan) 1548 { 1549 struct ppc440spe_adma_desc_slot *iter, *_iter, *group_start = NULL; 1550 dma_cookie_t cookie = 0; 1551 u32 current_desc = ppc440spe_chan_get_current_descriptor(chan); 1552 int busy = ppc440spe_chan_is_busy(chan); 1553 int seen_current = 0, slot_cnt = 0, slots_per_op = 0; 1554 1555 dev_dbg(chan->device->common.dev, "ppc440spe adma%d: %s\n", 1556 chan->device->id, __func__); 1557 1558 if (!current_desc) { 1559 /* There were no transactions yet, so 1560 * nothing to clean 1561 */ 1562 return; 1563 } 1564 1565 /* free completed slots from the chain starting with 1566 * the oldest descriptor 1567 */ 1568 list_for_each_entry_safe(iter, _iter, &chan->chain, 1569 chain_node) { 1570 dev_dbg(chan->device->common.dev, "\tcookie: %d slot: %d " 1571 "busy: %d this_desc: %#llx next_desc: %#x " 1572 "cur: %#x ack: %d\n", 1573 iter->async_tx.cookie, iter->idx, busy, iter->phys, 1574 ppc440spe_desc_get_link(iter, chan), current_desc, 1575 async_tx_test_ack(&iter->async_tx)); 1576 prefetch(_iter); 1577 prefetch(&_iter->async_tx); 1578 1579 /* do not advance past the current descriptor loaded into the 1580 * hardware channel,subsequent descriptors are either in process 1581 * or have not been submitted 1582 */ 1583 if (seen_current) 1584 break; 1585 1586 /* stop the search if we reach the current descriptor and the 1587 * channel is busy, or if it appears that the current descriptor 1588 * needs to be re-read (i.e. has been appended to) 1589 */ 1590 if (iter->phys == current_desc) { 1591 BUG_ON(seen_current++); 1592 if (busy || ppc440spe_desc_get_link(iter, chan)) { 1593 /* not all descriptors of the group have 1594 * been completed; exit. 1595 */ 1596 break; 1597 } 1598 } 1599 1600 /* detect the start of a group transaction */ 1601 if (!slot_cnt && !slots_per_op) { 1602 slot_cnt = iter->slot_cnt; 1603 slots_per_op = iter->slots_per_op; 1604 if (slot_cnt <= slots_per_op) { 1605 slot_cnt = 0; 1606 slots_per_op = 0; 1607 } 1608 } 1609 1610 if (slot_cnt) { 1611 if (!group_start) 1612 group_start = iter; 1613 slot_cnt -= slots_per_op; 1614 } 1615 1616 /* all the members of a group are complete */ 1617 if (slots_per_op != 0 && slot_cnt == 0) { 1618 struct ppc440spe_adma_desc_slot *grp_iter, *_grp_iter; 1619 int end_of_chain = 0; 1620 1621 /* clean up the group */ 1622 slot_cnt = group_start->slot_cnt; 1623 grp_iter = group_start; 1624 list_for_each_entry_safe_from(grp_iter, _grp_iter, 1625 &chan->chain, chain_node) { 1626 1627 cookie = ppc440spe_adma_run_tx_complete_actions( 1628 grp_iter, chan, cookie); 1629 1630 slot_cnt -= slots_per_op; 1631 end_of_chain = ppc440spe_adma_clean_slot( 1632 grp_iter, chan); 1633 if (end_of_chain && slot_cnt) { 1634 /* Should wait for ZeroSum completion */ 1635 if (cookie > 0) 1636 chan->common.completed_cookie = cookie; 1637 return; 1638 } 1639 1640 if (slot_cnt == 0 || end_of_chain) 1641 break; 1642 } 1643 1644 /* the group should be complete at this point */ 1645 BUG_ON(slot_cnt); 1646 1647 slots_per_op = 0; 1648 group_start = NULL; 1649 if (end_of_chain) 1650 break; 1651 else 1652 continue; 1653 } else if (slots_per_op) /* wait for group completion */ 1654 continue; 1655 1656 cookie = ppc440spe_adma_run_tx_complete_actions(iter, chan, 1657 cookie); 1658 1659 if (ppc440spe_adma_clean_slot(iter, chan)) 1660 break; 1661 } 1662 1663 BUG_ON(!seen_current); 1664 1665 if (cookie > 0) { 1666 chan->common.completed_cookie = cookie; 1667 pr_debug("\tcompleted cookie %d\n", cookie); 1668 } 1669 1670 } 1671 1672 /** 1673 * ppc440spe_adma_tasklet - clean up watch-dog initiator 1674 */ 1675 static void ppc440spe_adma_tasklet(unsigned long data) 1676 { 1677 struct ppc440spe_adma_chan *chan = (struct ppc440spe_adma_chan *) data; 1678 1679 spin_lock_nested(&chan->lock, SINGLE_DEPTH_NESTING); 1680 __ppc440spe_adma_slot_cleanup(chan); 1681 spin_unlock(&chan->lock); 1682 } 1683 1684 /** 1685 * ppc440spe_adma_slot_cleanup - clean up scheduled initiator 1686 */ 1687 static void ppc440spe_adma_slot_cleanup(struct ppc440spe_adma_chan *chan) 1688 { 1689 spin_lock_bh(&chan->lock); 1690 __ppc440spe_adma_slot_cleanup(chan); 1691 spin_unlock_bh(&chan->lock); 1692 } 1693 1694 /** 1695 * ppc440spe_adma_alloc_slots - allocate free slots (if any) 1696 */ 1697 static struct ppc440spe_adma_desc_slot *ppc440spe_adma_alloc_slots( 1698 struct ppc440spe_adma_chan *chan, int num_slots, 1699 int slots_per_op) 1700 { 1701 struct ppc440spe_adma_desc_slot *iter = NULL, *_iter; 1702 struct ppc440spe_adma_desc_slot *alloc_start = NULL; 1703 struct list_head chain = LIST_HEAD_INIT(chain); 1704 int slots_found, retry = 0; 1705 1706 1707 BUG_ON(!num_slots || !slots_per_op); 1708 /* start search from the last allocated descrtiptor 1709 * if a contiguous allocation can not be found start searching 1710 * from the beginning of the list 1711 */ 1712 retry: 1713 slots_found = 0; 1714 if (retry == 0) 1715 iter = chan->last_used; 1716 else 1717 iter = list_entry(&chan->all_slots, 1718 struct ppc440spe_adma_desc_slot, 1719 slot_node); 1720 list_for_each_entry_safe_continue(iter, _iter, &chan->all_slots, 1721 slot_node) { 1722 prefetch(_iter); 1723 prefetch(&_iter->async_tx); 1724 if (iter->slots_per_op) { 1725 slots_found = 0; 1726 continue; 1727 } 1728 1729 /* start the allocation if the slot is correctly aligned */ 1730 if (!slots_found++) 1731 alloc_start = iter; 1732 1733 if (slots_found == num_slots) { 1734 struct ppc440spe_adma_desc_slot *alloc_tail = NULL; 1735 struct ppc440spe_adma_desc_slot *last_used = NULL; 1736 1737 iter = alloc_start; 1738 while (num_slots) { 1739 int i; 1740 /* pre-ack all but the last descriptor */ 1741 if (num_slots != slots_per_op) 1742 async_tx_ack(&iter->async_tx); 1743 1744 list_add_tail(&iter->chain_node, &chain); 1745 alloc_tail = iter; 1746 iter->async_tx.cookie = 0; 1747 iter->hw_next = NULL; 1748 iter->flags = 0; 1749 iter->slot_cnt = num_slots; 1750 iter->xor_check_result = NULL; 1751 for (i = 0; i < slots_per_op; i++) { 1752 iter->slots_per_op = slots_per_op - i; 1753 last_used = iter; 1754 iter = list_entry(iter->slot_node.next, 1755 struct ppc440spe_adma_desc_slot, 1756 slot_node); 1757 } 1758 num_slots -= slots_per_op; 1759 } 1760 alloc_tail->group_head = alloc_start; 1761 alloc_tail->async_tx.cookie = -EBUSY; 1762 list_splice(&chain, &alloc_tail->group_list); 1763 chan->last_used = last_used; 1764 return alloc_tail; 1765 } 1766 } 1767 if (!retry++) 1768 goto retry; 1769 1770 /* try to free some slots if the allocation fails */ 1771 tasklet_schedule(&chan->irq_tasklet); 1772 return NULL; 1773 } 1774 1775 /** 1776 * ppc440spe_adma_alloc_chan_resources - allocate pools for CDB slots 1777 */ 1778 static int ppc440spe_adma_alloc_chan_resources(struct dma_chan *chan) 1779 { 1780 struct ppc440spe_adma_chan *ppc440spe_chan; 1781 struct ppc440spe_adma_desc_slot *slot = NULL; 1782 char *hw_desc; 1783 int i, db_sz; 1784 int init; 1785 1786 ppc440spe_chan = to_ppc440spe_adma_chan(chan); 1787 init = ppc440spe_chan->slots_allocated ? 0 : 1; 1788 chan->chan_id = ppc440spe_chan->device->id; 1789 1790 /* Allocate descriptor slots */ 1791 i = ppc440spe_chan->slots_allocated; 1792 if (ppc440spe_chan->device->id != PPC440SPE_XOR_ID) 1793 db_sz = sizeof(struct dma_cdb); 1794 else 1795 db_sz = sizeof(struct xor_cb); 1796 1797 for (; i < (ppc440spe_chan->device->pool_size / db_sz); i++) { 1798 slot = kzalloc(sizeof(struct ppc440spe_adma_desc_slot), 1799 GFP_KERNEL); 1800 if (!slot) { 1801 printk(KERN_INFO "SPE ADMA Channel only initialized" 1802 " %d descriptor slots", i--); 1803 break; 1804 } 1805 1806 hw_desc = (char *) ppc440spe_chan->device->dma_desc_pool_virt; 1807 slot->hw_desc = (void *) &hw_desc[i * db_sz]; 1808 dma_async_tx_descriptor_init(&slot->async_tx, chan); 1809 slot->async_tx.tx_submit = ppc440spe_adma_tx_submit; 1810 INIT_LIST_HEAD(&slot->chain_node); 1811 INIT_LIST_HEAD(&slot->slot_node); 1812 INIT_LIST_HEAD(&slot->group_list); 1813 slot->phys = ppc440spe_chan->device->dma_desc_pool + i * db_sz; 1814 slot->idx = i; 1815 1816 spin_lock_bh(&ppc440spe_chan->lock); 1817 ppc440spe_chan->slots_allocated++; 1818 list_add_tail(&slot->slot_node, &ppc440spe_chan->all_slots); 1819 spin_unlock_bh(&ppc440spe_chan->lock); 1820 } 1821 1822 if (i && !ppc440spe_chan->last_used) { 1823 ppc440spe_chan->last_used = 1824 list_entry(ppc440spe_chan->all_slots.next, 1825 struct ppc440spe_adma_desc_slot, 1826 slot_node); 1827 } 1828 1829 dev_dbg(ppc440spe_chan->device->common.dev, 1830 "ppc440spe adma%d: allocated %d descriptor slots\n", 1831 ppc440spe_chan->device->id, i); 1832 1833 /* initialize the channel and the chain with a null operation */ 1834 if (init) { 1835 switch (ppc440spe_chan->device->id) { 1836 case PPC440SPE_DMA0_ID: 1837 case PPC440SPE_DMA1_ID: 1838 ppc440spe_chan->hw_chain_inited = 0; 1839 /* Use WXOR for self-testing */ 1840 if (!ppc440spe_r6_tchan) 1841 ppc440spe_r6_tchan = ppc440spe_chan; 1842 break; 1843 case PPC440SPE_XOR_ID: 1844 ppc440spe_chan_start_null_xor(ppc440spe_chan); 1845 break; 1846 default: 1847 BUG(); 1848 } 1849 ppc440spe_chan->needs_unmap = 1; 1850 } 1851 1852 return (i > 0) ? i : -ENOMEM; 1853 } 1854 1855 /** 1856 * ppc440spe_rxor_set_region_data - 1857 */ 1858 static void ppc440spe_rxor_set_region(struct ppc440spe_adma_desc_slot *desc, 1859 u8 xor_arg_no, u32 mask) 1860 { 1861 struct xor_cb *xcb = desc->hw_desc; 1862 1863 xcb->ops[xor_arg_no].h |= mask; 1864 } 1865 1866 /** 1867 * ppc440spe_rxor_set_src - 1868 */ 1869 static void ppc440spe_rxor_set_src(struct ppc440spe_adma_desc_slot *desc, 1870 u8 xor_arg_no, dma_addr_t addr) 1871 { 1872 struct xor_cb *xcb = desc->hw_desc; 1873 1874 xcb->ops[xor_arg_no].h |= DMA_CUED_XOR_BASE; 1875 xcb->ops[xor_arg_no].l = addr; 1876 } 1877 1878 /** 1879 * ppc440spe_rxor_set_mult - 1880 */ 1881 static void ppc440spe_rxor_set_mult(struct ppc440spe_adma_desc_slot *desc, 1882 u8 xor_arg_no, u8 idx, u8 mult) 1883 { 1884 struct xor_cb *xcb = desc->hw_desc; 1885 1886 xcb->ops[xor_arg_no].h |= mult << (DMA_CUED_MULT1_OFF + idx * 8); 1887 } 1888 1889 /** 1890 * ppc440spe_adma_check_threshold - append CDBs to h/w chain if threshold 1891 * has been achieved 1892 */ 1893 static void ppc440spe_adma_check_threshold(struct ppc440spe_adma_chan *chan) 1894 { 1895 dev_dbg(chan->device->common.dev, "ppc440spe adma%d: pending: %d\n", 1896 chan->device->id, chan->pending); 1897 1898 if (chan->pending >= PPC440SPE_ADMA_THRESHOLD) { 1899 chan->pending = 0; 1900 ppc440spe_chan_append(chan); 1901 } 1902 } 1903 1904 /** 1905 * ppc440spe_adma_tx_submit - submit new descriptor group to the channel 1906 * (it's not necessary that descriptors will be submitted to the h/w 1907 * chains too right now) 1908 */ 1909 static dma_cookie_t ppc440spe_adma_tx_submit(struct dma_async_tx_descriptor *tx) 1910 { 1911 struct ppc440spe_adma_desc_slot *sw_desc; 1912 struct ppc440spe_adma_chan *chan = to_ppc440spe_adma_chan(tx->chan); 1913 struct ppc440spe_adma_desc_slot *group_start, *old_chain_tail; 1914 int slot_cnt; 1915 int slots_per_op; 1916 dma_cookie_t cookie; 1917 1918 sw_desc = tx_to_ppc440spe_adma_slot(tx); 1919 1920 group_start = sw_desc->group_head; 1921 slot_cnt = group_start->slot_cnt; 1922 slots_per_op = group_start->slots_per_op; 1923 1924 spin_lock_bh(&chan->lock); 1925 cookie = dma_cookie_assign(tx); 1926 1927 if (unlikely(list_empty(&chan->chain))) { 1928 /* first peer */ 1929 list_splice_init(&sw_desc->group_list, &chan->chain); 1930 chan_first_cdb[chan->device->id] = group_start; 1931 } else { 1932 /* isn't first peer, bind CDBs to chain */ 1933 old_chain_tail = list_entry(chan->chain.prev, 1934 struct ppc440spe_adma_desc_slot, 1935 chain_node); 1936 list_splice_init(&sw_desc->group_list, 1937 &old_chain_tail->chain_node); 1938 /* fix up the hardware chain */ 1939 ppc440spe_desc_set_link(chan, old_chain_tail, group_start); 1940 } 1941 1942 /* increment the pending count by the number of operations */ 1943 chan->pending += slot_cnt / slots_per_op; 1944 ppc440spe_adma_check_threshold(chan); 1945 spin_unlock_bh(&chan->lock); 1946 1947 dev_dbg(chan->device->common.dev, 1948 "ppc440spe adma%d: %s cookie: %d slot: %d tx %p\n", 1949 chan->device->id, __func__, 1950 sw_desc->async_tx.cookie, sw_desc->idx, sw_desc); 1951 1952 return cookie; 1953 } 1954 1955 /** 1956 * ppc440spe_adma_prep_dma_interrupt - prepare CDB for a pseudo DMA operation 1957 */ 1958 static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_interrupt( 1959 struct dma_chan *chan, unsigned long flags) 1960 { 1961 struct ppc440spe_adma_chan *ppc440spe_chan; 1962 struct ppc440spe_adma_desc_slot *sw_desc, *group_start; 1963 int slot_cnt, slots_per_op; 1964 1965 ppc440spe_chan = to_ppc440spe_adma_chan(chan); 1966 1967 dev_dbg(ppc440spe_chan->device->common.dev, 1968 "ppc440spe adma%d: %s\n", ppc440spe_chan->device->id, 1969 __func__); 1970 1971 spin_lock_bh(&ppc440spe_chan->lock); 1972 slot_cnt = slots_per_op = 1; 1973 sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1974 slots_per_op); 1975 if (sw_desc) { 1976 group_start = sw_desc->group_head; 1977 ppc440spe_desc_init_interrupt(group_start, ppc440spe_chan); 1978 group_start->unmap_len = 0; 1979 sw_desc->async_tx.flags = flags; 1980 } 1981 spin_unlock_bh(&ppc440spe_chan->lock); 1982 1983 return sw_desc ? &sw_desc->async_tx : NULL; 1984 } 1985 1986 /** 1987 * ppc440spe_adma_prep_dma_memcpy - prepare CDB for a MEMCPY operation 1988 */ 1989 static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_memcpy( 1990 struct dma_chan *chan, dma_addr_t dma_dest, 1991 dma_addr_t dma_src, size_t len, unsigned long flags) 1992 { 1993 struct ppc440spe_adma_chan *ppc440spe_chan; 1994 struct ppc440spe_adma_desc_slot *sw_desc, *group_start; 1995 int slot_cnt, slots_per_op; 1996 1997 ppc440spe_chan = to_ppc440spe_adma_chan(chan); 1998 1999 if (unlikely(!len)) 2000 return NULL; 2001 2002 BUG_ON(len > PPC440SPE_ADMA_DMA_MAX_BYTE_COUNT); 2003 2004 spin_lock_bh(&ppc440spe_chan->lock); 2005 2006 dev_dbg(ppc440spe_chan->device->common.dev, 2007 "ppc440spe adma%d: %s len: %u int_en %d\n", 2008 ppc440spe_chan->device->id, __func__, len, 2009 flags & DMA_PREP_INTERRUPT ? 1 : 0); 2010 slot_cnt = slots_per_op = 1; 2011 sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 2012 slots_per_op); 2013 if (sw_desc) { 2014 group_start = sw_desc->group_head; 2015 ppc440spe_desc_init_memcpy(group_start, flags); 2016 ppc440spe_adma_set_dest(group_start, dma_dest, 0); 2017 ppc440spe_adma_memcpy_xor_set_src(group_start, dma_src, 0); 2018 ppc440spe_desc_set_byte_count(group_start, ppc440spe_chan, len); 2019 sw_desc->unmap_len = len; 2020 sw_desc->async_tx.flags = flags; 2021 } 2022 spin_unlock_bh(&ppc440spe_chan->lock); 2023 2024 return sw_desc ? &sw_desc->async_tx : NULL; 2025 } 2026 2027 /** 2028 * ppc440spe_adma_prep_dma_xor - prepare CDB for a XOR operation 2029 */ 2030 static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_xor( 2031 struct dma_chan *chan, dma_addr_t dma_dest, 2032 dma_addr_t *dma_src, u32 src_cnt, size_t len, 2033 unsigned long flags) 2034 { 2035 struct ppc440spe_adma_chan *ppc440spe_chan; 2036 struct ppc440spe_adma_desc_slot *sw_desc, *group_start; 2037 int slot_cnt, slots_per_op; 2038 2039 ppc440spe_chan = to_ppc440spe_adma_chan(chan); 2040 2041 ADMA_LL_DBG(prep_dma_xor_dbg(ppc440spe_chan->device->id, 2042 dma_dest, dma_src, src_cnt)); 2043 if (unlikely(!len)) 2044 return NULL; 2045 BUG_ON(len > PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT); 2046 2047 dev_dbg(ppc440spe_chan->device->common.dev, 2048 "ppc440spe adma%d: %s src_cnt: %d len: %u int_en: %d\n", 2049 ppc440spe_chan->device->id, __func__, src_cnt, len, 2050 flags & DMA_PREP_INTERRUPT ? 1 : 0); 2051 2052 spin_lock_bh(&ppc440spe_chan->lock); 2053 slot_cnt = ppc440spe_chan_xor_slot_count(len, src_cnt, &slots_per_op); 2054 sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 2055 slots_per_op); 2056 if (sw_desc) { 2057 group_start = sw_desc->group_head; 2058 ppc440spe_desc_init_xor(group_start, src_cnt, flags); 2059 ppc440spe_adma_set_dest(group_start, dma_dest, 0); 2060 while (src_cnt--) 2061 ppc440spe_adma_memcpy_xor_set_src(group_start, 2062 dma_src[src_cnt], src_cnt); 2063 ppc440spe_desc_set_byte_count(group_start, ppc440spe_chan, len); 2064 sw_desc->unmap_len = len; 2065 sw_desc->async_tx.flags = flags; 2066 } 2067 spin_unlock_bh(&ppc440spe_chan->lock); 2068 2069 return sw_desc ? &sw_desc->async_tx : NULL; 2070 } 2071 2072 static inline void 2073 ppc440spe_desc_set_xor_src_cnt(struct ppc440spe_adma_desc_slot *desc, 2074 int src_cnt); 2075 static void ppc440spe_init_rxor_cursor(struct ppc440spe_rxor *cursor); 2076 2077 /** 2078 * ppc440spe_adma_init_dma2rxor_slot - 2079 */ 2080 static void ppc440spe_adma_init_dma2rxor_slot( 2081 struct ppc440spe_adma_desc_slot *desc, 2082 dma_addr_t *src, int src_cnt) 2083 { 2084 int i; 2085 2086 /* initialize CDB */ 2087 for (i = 0; i < src_cnt; i++) { 2088 ppc440spe_adma_dma2rxor_prep_src(desc, &desc->rxor_cursor, i, 2089 desc->src_cnt, (u32)src[i]); 2090 } 2091 } 2092 2093 /** 2094 * ppc440spe_dma01_prep_mult - 2095 * for Q operation where destination is also the source 2096 */ 2097 static struct ppc440spe_adma_desc_slot *ppc440spe_dma01_prep_mult( 2098 struct ppc440spe_adma_chan *ppc440spe_chan, 2099 dma_addr_t *dst, int dst_cnt, dma_addr_t *src, int src_cnt, 2100 const unsigned char *scf, size_t len, unsigned long flags) 2101 { 2102 struct ppc440spe_adma_desc_slot *sw_desc = NULL; 2103 unsigned long op = 0; 2104 int slot_cnt; 2105 2106 set_bit(PPC440SPE_DESC_WXOR, &op); 2107 slot_cnt = 2; 2108 2109 spin_lock_bh(&ppc440spe_chan->lock); 2110 2111 /* use WXOR, each descriptor occupies one slot */ 2112 sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1); 2113 if (sw_desc) { 2114 struct ppc440spe_adma_chan *chan; 2115 struct ppc440spe_adma_desc_slot *iter; 2116 struct dma_cdb *hw_desc; 2117 2118 chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan); 2119 set_bits(op, &sw_desc->flags); 2120 sw_desc->src_cnt = src_cnt; 2121 sw_desc->dst_cnt = dst_cnt; 2122 /* First descriptor, zero data in the destination and copy it 2123 * to q page using MULTICAST transfer. 2124 */ 2125 iter = list_first_entry(&sw_desc->group_list, 2126 struct ppc440spe_adma_desc_slot, 2127 chain_node); 2128 memset(iter->hw_desc, 0, sizeof(struct dma_cdb)); 2129 /* set 'next' pointer */ 2130 iter->hw_next = list_entry(iter->chain_node.next, 2131 struct ppc440spe_adma_desc_slot, 2132 chain_node); 2133 clear_bit(PPC440SPE_DESC_INT, &iter->flags); 2134 hw_desc = iter->hw_desc; 2135 hw_desc->opc = DMA_CDB_OPC_MULTICAST; 2136 2137 ppc440spe_desc_set_dest_addr(iter, chan, 2138 DMA_CUED_XOR_BASE, dst[0], 0); 2139 ppc440spe_desc_set_dest_addr(iter, chan, 0, dst[1], 1); 2140 ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB, 2141 src[0]); 2142 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len); 2143 iter->unmap_len = len; 2144 2145 /* 2146 * Second descriptor, multiply data from the q page 2147 * and store the result in real destination. 2148 */ 2149 iter = list_first_entry(&iter->chain_node, 2150 struct ppc440spe_adma_desc_slot, 2151 chain_node); 2152 memset(iter->hw_desc, 0, sizeof(struct dma_cdb)); 2153 iter->hw_next = NULL; 2154 if (flags & DMA_PREP_INTERRUPT) 2155 set_bit(PPC440SPE_DESC_INT, &iter->flags); 2156 else 2157 clear_bit(PPC440SPE_DESC_INT, &iter->flags); 2158 2159 hw_desc = iter->hw_desc; 2160 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; 2161 ppc440spe_desc_set_src_addr(iter, chan, 0, 2162 DMA_CUED_XOR_HB, dst[1]); 2163 ppc440spe_desc_set_dest_addr(iter, chan, 2164 DMA_CUED_XOR_BASE, dst[0], 0); 2165 2166 ppc440spe_desc_set_src_mult(iter, chan, DMA_CUED_MULT1_OFF, 2167 DMA_CDB_SG_DST1, scf[0]); 2168 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len); 2169 iter->unmap_len = len; 2170 sw_desc->async_tx.flags = flags; 2171 } 2172 2173 spin_unlock_bh(&ppc440spe_chan->lock); 2174 2175 return sw_desc; 2176 } 2177 2178 /** 2179 * ppc440spe_dma01_prep_sum_product - 2180 * Dx = A*(P+Pxy) + B*(Q+Qxy) operation where destination is also 2181 * the source. 2182 */ 2183 static struct ppc440spe_adma_desc_slot *ppc440spe_dma01_prep_sum_product( 2184 struct ppc440spe_adma_chan *ppc440spe_chan, 2185 dma_addr_t *dst, dma_addr_t *src, int src_cnt, 2186 const unsigned char *scf, size_t len, unsigned long flags) 2187 { 2188 struct ppc440spe_adma_desc_slot *sw_desc = NULL; 2189 unsigned long op = 0; 2190 int slot_cnt; 2191 2192 set_bit(PPC440SPE_DESC_WXOR, &op); 2193 slot_cnt = 3; 2194 2195 spin_lock_bh(&ppc440spe_chan->lock); 2196 2197 /* WXOR, each descriptor occupies one slot */ 2198 sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1); 2199 if (sw_desc) { 2200 struct ppc440spe_adma_chan *chan; 2201 struct ppc440spe_adma_desc_slot *iter; 2202 struct dma_cdb *hw_desc; 2203 2204 chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan); 2205 set_bits(op, &sw_desc->flags); 2206 sw_desc->src_cnt = src_cnt; 2207 sw_desc->dst_cnt = 1; 2208 /* 1st descriptor, src[1] data to q page and zero destination */ 2209 iter = list_first_entry(&sw_desc->group_list, 2210 struct ppc440spe_adma_desc_slot, 2211 chain_node); 2212 memset(iter->hw_desc, 0, sizeof(struct dma_cdb)); 2213 iter->hw_next = list_entry(iter->chain_node.next, 2214 struct ppc440spe_adma_desc_slot, 2215 chain_node); 2216 clear_bit(PPC440SPE_DESC_INT, &iter->flags); 2217 hw_desc = iter->hw_desc; 2218 hw_desc->opc = DMA_CDB_OPC_MULTICAST; 2219 2220 ppc440spe_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE, 2221 *dst, 0); 2222 ppc440spe_desc_set_dest_addr(iter, chan, 0, 2223 ppc440spe_chan->qdest, 1); 2224 ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB, 2225 src[1]); 2226 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len); 2227 iter->unmap_len = len; 2228 2229 /* 2nd descriptor, multiply src[1] data and store the 2230 * result in destination */ 2231 iter = list_first_entry(&iter->chain_node, 2232 struct ppc440spe_adma_desc_slot, 2233 chain_node); 2234 memset(iter->hw_desc, 0, sizeof(struct dma_cdb)); 2235 /* set 'next' pointer */ 2236 iter->hw_next = list_entry(iter->chain_node.next, 2237 struct ppc440spe_adma_desc_slot, 2238 chain_node); 2239 if (flags & DMA_PREP_INTERRUPT) 2240 set_bit(PPC440SPE_DESC_INT, &iter->flags); 2241 else 2242 clear_bit(PPC440SPE_DESC_INT, &iter->flags); 2243 2244 hw_desc = iter->hw_desc; 2245 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; 2246 ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB, 2247 ppc440spe_chan->qdest); 2248 ppc440spe_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE, 2249 *dst, 0); 2250 ppc440spe_desc_set_src_mult(iter, chan, DMA_CUED_MULT1_OFF, 2251 DMA_CDB_SG_DST1, scf[1]); 2252 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len); 2253 iter->unmap_len = len; 2254 2255 /* 2256 * 3rd descriptor, multiply src[0] data and xor it 2257 * with destination 2258 */ 2259 iter = list_first_entry(&iter->chain_node, 2260 struct ppc440spe_adma_desc_slot, 2261 chain_node); 2262 memset(iter->hw_desc, 0, sizeof(struct dma_cdb)); 2263 iter->hw_next = NULL; 2264 if (flags & DMA_PREP_INTERRUPT) 2265 set_bit(PPC440SPE_DESC_INT, &iter->flags); 2266 else 2267 clear_bit(PPC440SPE_DESC_INT, &iter->flags); 2268 2269 hw_desc = iter->hw_desc; 2270 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; 2271 ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB, 2272 src[0]); 2273 ppc440spe_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE, 2274 *dst, 0); 2275 ppc440spe_desc_set_src_mult(iter, chan, DMA_CUED_MULT1_OFF, 2276 DMA_CDB_SG_DST1, scf[0]); 2277 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len); 2278 iter->unmap_len = len; 2279 sw_desc->async_tx.flags = flags; 2280 } 2281 2282 spin_unlock_bh(&ppc440spe_chan->lock); 2283 2284 return sw_desc; 2285 } 2286 2287 static struct ppc440spe_adma_desc_slot *ppc440spe_dma01_prep_pq( 2288 struct ppc440spe_adma_chan *ppc440spe_chan, 2289 dma_addr_t *dst, int dst_cnt, dma_addr_t *src, int src_cnt, 2290 const unsigned char *scf, size_t len, unsigned long flags) 2291 { 2292 int slot_cnt; 2293 struct ppc440spe_adma_desc_slot *sw_desc = NULL, *iter; 2294 unsigned long op = 0; 2295 unsigned char mult = 1; 2296 2297 pr_debug("%s: dst_cnt %d, src_cnt %d, len %d\n", 2298 __func__, dst_cnt, src_cnt, len); 2299 /* select operations WXOR/RXOR depending on the 2300 * source addresses of operators and the number 2301 * of destinations (RXOR support only Q-parity calculations) 2302 */ 2303 set_bit(PPC440SPE_DESC_WXOR, &op); 2304 if (!test_and_set_bit(PPC440SPE_RXOR_RUN, &ppc440spe_rxor_state)) { 2305 /* no active RXOR; 2306 * do RXOR if: 2307 * - there are more than 1 source, 2308 * - len is aligned on 512-byte boundary, 2309 * - source addresses fit to one of 4 possible regions. 2310 */ 2311 if (src_cnt > 1 && 2312 !(len & MQ0_CF2H_RXOR_BS_MASK) && 2313 (src[0] + len) == src[1]) { 2314 /* may do RXOR R1 R2 */ 2315 set_bit(PPC440SPE_DESC_RXOR, &op); 2316 if (src_cnt != 2) { 2317 /* may try to enhance region of RXOR */ 2318 if ((src[1] + len) == src[2]) { 2319 /* do RXOR R1 R2 R3 */ 2320 set_bit(PPC440SPE_DESC_RXOR123, 2321 &op); 2322 } else if ((src[1] + len * 2) == src[2]) { 2323 /* do RXOR R1 R2 R4 */ 2324 set_bit(PPC440SPE_DESC_RXOR124, &op); 2325 } else if ((src[1] + len * 3) == src[2]) { 2326 /* do RXOR R1 R2 R5 */ 2327 set_bit(PPC440SPE_DESC_RXOR125, 2328 &op); 2329 } else { 2330 /* do RXOR R1 R2 */ 2331 set_bit(PPC440SPE_DESC_RXOR12, 2332 &op); 2333 } 2334 } else { 2335 /* do RXOR R1 R2 */ 2336 set_bit(PPC440SPE_DESC_RXOR12, &op); 2337 } 2338 } 2339 2340 if (!test_bit(PPC440SPE_DESC_RXOR, &op)) { 2341 /* can not do this operation with RXOR */ 2342 clear_bit(PPC440SPE_RXOR_RUN, 2343 &ppc440spe_rxor_state); 2344 } else { 2345 /* can do; set block size right now */ 2346 ppc440spe_desc_set_rxor_block_size(len); 2347 } 2348 } 2349 2350 /* Number of necessary slots depends on operation type selected */ 2351 if (!test_bit(PPC440SPE_DESC_RXOR, &op)) { 2352 /* This is a WXOR only chain. Need descriptors for each 2353 * source to GF-XOR them with WXOR, and need descriptors 2354 * for each destination to zero them with WXOR 2355 */ 2356 slot_cnt = src_cnt; 2357 2358 if (flags & DMA_PREP_ZERO_P) { 2359 slot_cnt++; 2360 set_bit(PPC440SPE_ZERO_P, &op); 2361 } 2362 if (flags & DMA_PREP_ZERO_Q) { 2363 slot_cnt++; 2364 set_bit(PPC440SPE_ZERO_Q, &op); 2365 } 2366 } else { 2367 /* Need 1/2 descriptor for RXOR operation, and 2368 * need (src_cnt - (2 or 3)) for WXOR of sources 2369 * remained (if any) 2370 */ 2371 slot_cnt = dst_cnt; 2372 2373 if (flags & DMA_PREP_ZERO_P) 2374 set_bit(PPC440SPE_ZERO_P, &op); 2375 if (flags & DMA_PREP_ZERO_Q) 2376 set_bit(PPC440SPE_ZERO_Q, &op); 2377 2378 if (test_bit(PPC440SPE_DESC_RXOR12, &op)) 2379 slot_cnt += src_cnt - 2; 2380 else 2381 slot_cnt += src_cnt - 3; 2382 2383 /* Thus we have either RXOR only chain or 2384 * mixed RXOR/WXOR 2385 */ 2386 if (slot_cnt == dst_cnt) 2387 /* RXOR only chain */ 2388 clear_bit(PPC440SPE_DESC_WXOR, &op); 2389 } 2390 2391 spin_lock_bh(&ppc440spe_chan->lock); 2392 /* for both RXOR/WXOR each descriptor occupies one slot */ 2393 sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1); 2394 if (sw_desc) { 2395 ppc440spe_desc_init_dma01pq(sw_desc, dst_cnt, src_cnt, 2396 flags, op); 2397 2398 /* setup dst/src/mult */ 2399 pr_debug("%s: set dst descriptor 0, 1: 0x%016llx, 0x%016llx\n", 2400 __func__, dst[0], dst[1]); 2401 ppc440spe_adma_pq_set_dest(sw_desc, dst, flags); 2402 while (src_cnt--) { 2403 ppc440spe_adma_pq_set_src(sw_desc, src[src_cnt], 2404 src_cnt); 2405 2406 /* NOTE: "Multi = 0 is equivalent to = 1" as it 2407 * stated in 440SPSPe_RAID6_Addendum_UM_1_17.pdf 2408 * doesn't work for RXOR with DMA0/1! Instead, multi=0 2409 * leads to zeroing source data after RXOR. 2410 * So, for P case set-up mult=1 explicitly. 2411 */ 2412 if (!(flags & DMA_PREP_PQ_DISABLE_Q)) 2413 mult = scf[src_cnt]; 2414 ppc440spe_adma_pq_set_src_mult(sw_desc, 2415 mult, src_cnt, dst_cnt - 1); 2416 } 2417 2418 /* Setup byte count foreach slot just allocated */ 2419 sw_desc->async_tx.flags = flags; 2420 list_for_each_entry(iter, &sw_desc->group_list, 2421 chain_node) { 2422 ppc440spe_desc_set_byte_count(iter, 2423 ppc440spe_chan, len); 2424 iter->unmap_len = len; 2425 } 2426 } 2427 spin_unlock_bh(&ppc440spe_chan->lock); 2428 2429 return sw_desc; 2430 } 2431 2432 static struct ppc440spe_adma_desc_slot *ppc440spe_dma2_prep_pq( 2433 struct ppc440spe_adma_chan *ppc440spe_chan, 2434 dma_addr_t *dst, int dst_cnt, dma_addr_t *src, int src_cnt, 2435 const unsigned char *scf, size_t len, unsigned long flags) 2436 { 2437 int slot_cnt, descs_per_op; 2438 struct ppc440spe_adma_desc_slot *sw_desc = NULL, *iter; 2439 unsigned long op = 0; 2440 unsigned char mult = 1; 2441 2442 BUG_ON(!dst_cnt); 2443 /*pr_debug("%s: dst_cnt %d, src_cnt %d, len %d\n", 2444 __func__, dst_cnt, src_cnt, len);*/ 2445 2446 spin_lock_bh(&ppc440spe_chan->lock); 2447 descs_per_op = ppc440spe_dma2_pq_slot_count(src, src_cnt, len); 2448 if (descs_per_op < 0) { 2449 spin_unlock_bh(&ppc440spe_chan->lock); 2450 return NULL; 2451 } 2452 2453 /* depending on number of sources we have 1 or 2 RXOR chains */ 2454 slot_cnt = descs_per_op * dst_cnt; 2455 2456 sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1); 2457 if (sw_desc) { 2458 op = slot_cnt; 2459 sw_desc->async_tx.flags = flags; 2460 list_for_each_entry(iter, &sw_desc->group_list, chain_node) { 2461 ppc440spe_desc_init_dma2pq(iter, dst_cnt, src_cnt, 2462 --op ? 0 : flags); 2463 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, 2464 len); 2465 iter->unmap_len = len; 2466 2467 ppc440spe_init_rxor_cursor(&(iter->rxor_cursor)); 2468 iter->rxor_cursor.len = len; 2469 iter->descs_per_op = descs_per_op; 2470 } 2471 op = 0; 2472 list_for_each_entry(iter, &sw_desc->group_list, chain_node) { 2473 op++; 2474 if (op % descs_per_op == 0) 2475 ppc440spe_adma_init_dma2rxor_slot(iter, src, 2476 src_cnt); 2477 if (likely(!list_is_last(&iter->chain_node, 2478 &sw_desc->group_list))) { 2479 /* set 'next' pointer */ 2480 iter->hw_next = 2481 list_entry(iter->chain_node.next, 2482 struct ppc440spe_adma_desc_slot, 2483 chain_node); 2484 ppc440spe_xor_set_link(iter, iter->hw_next); 2485 } else { 2486 /* this is the last descriptor. */ 2487 iter->hw_next = NULL; 2488 } 2489 } 2490 2491 /* fixup head descriptor */ 2492 sw_desc->dst_cnt = dst_cnt; 2493 if (flags & DMA_PREP_ZERO_P) 2494 set_bit(PPC440SPE_ZERO_P, &sw_desc->flags); 2495 if (flags & DMA_PREP_ZERO_Q) 2496 set_bit(PPC440SPE_ZERO_Q, &sw_desc->flags); 2497 2498 /* setup dst/src/mult */ 2499 ppc440spe_adma_pq_set_dest(sw_desc, dst, flags); 2500 2501 while (src_cnt--) { 2502 /* handle descriptors (if dst_cnt == 2) inside 2503 * the ppc440spe_adma_pq_set_srcxxx() functions 2504 */ 2505 ppc440spe_adma_pq_set_src(sw_desc, src[src_cnt], 2506 src_cnt); 2507 if (!(flags & DMA_PREP_PQ_DISABLE_Q)) 2508 mult = scf[src_cnt]; 2509 ppc440spe_adma_pq_set_src_mult(sw_desc, 2510 mult, src_cnt, dst_cnt - 1); 2511 } 2512 } 2513 spin_unlock_bh(&ppc440spe_chan->lock); 2514 ppc440spe_desc_set_rxor_block_size(len); 2515 return sw_desc; 2516 } 2517 2518 /** 2519 * ppc440spe_adma_prep_dma_pq - prepare CDB (group) for a GF-XOR operation 2520 */ 2521 static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_pq( 2522 struct dma_chan *chan, dma_addr_t *dst, dma_addr_t *src, 2523 unsigned int src_cnt, const unsigned char *scf, 2524 size_t len, unsigned long flags) 2525 { 2526 struct ppc440spe_adma_chan *ppc440spe_chan; 2527 struct ppc440spe_adma_desc_slot *sw_desc = NULL; 2528 int dst_cnt = 0; 2529 2530 ppc440spe_chan = to_ppc440spe_adma_chan(chan); 2531 2532 ADMA_LL_DBG(prep_dma_pq_dbg(ppc440spe_chan->device->id, 2533 dst, src, src_cnt)); 2534 BUG_ON(!len); 2535 BUG_ON(len > PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT); 2536 BUG_ON(!src_cnt); 2537 2538 if (src_cnt == 1 && dst[1] == src[0]) { 2539 dma_addr_t dest[2]; 2540 2541 /* dst[1] is real destination (Q) */ 2542 dest[0] = dst[1]; 2543 /* this is the page to multicast source data to */ 2544 dest[1] = ppc440spe_chan->qdest; 2545 sw_desc = ppc440spe_dma01_prep_mult(ppc440spe_chan, 2546 dest, 2, src, src_cnt, scf, len, flags); 2547 return sw_desc ? &sw_desc->async_tx : NULL; 2548 } 2549 2550 if (src_cnt == 2 && dst[1] == src[1]) { 2551 sw_desc = ppc440spe_dma01_prep_sum_product(ppc440spe_chan, 2552 &dst[1], src, 2, scf, len, flags); 2553 return sw_desc ? &sw_desc->async_tx : NULL; 2554 } 2555 2556 if (!(flags & DMA_PREP_PQ_DISABLE_P)) { 2557 BUG_ON(!dst[0]); 2558 dst_cnt++; 2559 flags |= DMA_PREP_ZERO_P; 2560 } 2561 2562 if (!(flags & DMA_PREP_PQ_DISABLE_Q)) { 2563 BUG_ON(!dst[1]); 2564 dst_cnt++; 2565 flags |= DMA_PREP_ZERO_Q; 2566 } 2567 2568 BUG_ON(!dst_cnt); 2569 2570 dev_dbg(ppc440spe_chan->device->common.dev, 2571 "ppc440spe adma%d: %s src_cnt: %d len: %u int_en: %d\n", 2572 ppc440spe_chan->device->id, __func__, src_cnt, len, 2573 flags & DMA_PREP_INTERRUPT ? 1 : 0); 2574 2575 switch (ppc440spe_chan->device->id) { 2576 case PPC440SPE_DMA0_ID: 2577 case PPC440SPE_DMA1_ID: 2578 sw_desc = ppc440spe_dma01_prep_pq(ppc440spe_chan, 2579 dst, dst_cnt, src, src_cnt, scf, 2580 len, flags); 2581 break; 2582 2583 case PPC440SPE_XOR_ID: 2584 sw_desc = ppc440spe_dma2_prep_pq(ppc440spe_chan, 2585 dst, dst_cnt, src, src_cnt, scf, 2586 len, flags); 2587 break; 2588 } 2589 2590 return sw_desc ? &sw_desc->async_tx : NULL; 2591 } 2592 2593 /** 2594 * ppc440spe_adma_prep_dma_pqzero_sum - prepare CDB group for 2595 * a PQ_ZERO_SUM operation 2596 */ 2597 static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_pqzero_sum( 2598 struct dma_chan *chan, dma_addr_t *pq, dma_addr_t *src, 2599 unsigned int src_cnt, const unsigned char *scf, size_t len, 2600 enum sum_check_flags *pqres, unsigned long flags) 2601 { 2602 struct ppc440spe_adma_chan *ppc440spe_chan; 2603 struct ppc440spe_adma_desc_slot *sw_desc, *iter; 2604 dma_addr_t pdest, qdest; 2605 int slot_cnt, slots_per_op, idst, dst_cnt; 2606 2607 ppc440spe_chan = to_ppc440spe_adma_chan(chan); 2608 2609 if (flags & DMA_PREP_PQ_DISABLE_P) 2610 pdest = 0; 2611 else 2612 pdest = pq[0]; 2613 2614 if (flags & DMA_PREP_PQ_DISABLE_Q) 2615 qdest = 0; 2616 else 2617 qdest = pq[1]; 2618 2619 ADMA_LL_DBG(prep_dma_pqzero_sum_dbg(ppc440spe_chan->device->id, 2620 src, src_cnt, scf)); 2621 2622 /* Always use WXOR for P/Q calculations (two destinations). 2623 * Need 1 or 2 extra slots to verify results are zero. 2624 */ 2625 idst = dst_cnt = (pdest && qdest) ? 2 : 1; 2626 2627 /* One additional slot per destination to clone P/Q 2628 * before calculation (we have to preserve destinations). 2629 */ 2630 slot_cnt = src_cnt + dst_cnt * 2; 2631 slots_per_op = 1; 2632 2633 spin_lock_bh(&ppc440spe_chan->lock); 2634 sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 2635 slots_per_op); 2636 if (sw_desc) { 2637 ppc440spe_desc_init_dma01pqzero_sum(sw_desc, dst_cnt, src_cnt); 2638 2639 /* Setup byte count for each slot just allocated */ 2640 sw_desc->async_tx.flags = flags; 2641 list_for_each_entry(iter, &sw_desc->group_list, chain_node) { 2642 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, 2643 len); 2644 iter->unmap_len = len; 2645 } 2646 2647 if (pdest) { 2648 struct dma_cdb *hw_desc; 2649 struct ppc440spe_adma_chan *chan; 2650 2651 iter = sw_desc->group_head; 2652 chan = to_ppc440spe_adma_chan(iter->async_tx.chan); 2653 memset(iter->hw_desc, 0, sizeof(struct dma_cdb)); 2654 iter->hw_next = list_entry(iter->chain_node.next, 2655 struct ppc440spe_adma_desc_slot, 2656 chain_node); 2657 hw_desc = iter->hw_desc; 2658 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; 2659 iter->src_cnt = 0; 2660 iter->dst_cnt = 0; 2661 ppc440spe_desc_set_dest_addr(iter, chan, 0, 2662 ppc440spe_chan->pdest, 0); 2663 ppc440spe_desc_set_src_addr(iter, chan, 0, 0, pdest); 2664 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, 2665 len); 2666 iter->unmap_len = 0; 2667 /* override pdest to preserve original P */ 2668 pdest = ppc440spe_chan->pdest; 2669 } 2670 if (qdest) { 2671 struct dma_cdb *hw_desc; 2672 struct ppc440spe_adma_chan *chan; 2673 2674 iter = list_first_entry(&sw_desc->group_list, 2675 struct ppc440spe_adma_desc_slot, 2676 chain_node); 2677 chan = to_ppc440spe_adma_chan(iter->async_tx.chan); 2678 2679 if (pdest) { 2680 iter = list_entry(iter->chain_node.next, 2681 struct ppc440spe_adma_desc_slot, 2682 chain_node); 2683 } 2684 2685 memset(iter->hw_desc, 0, sizeof(struct dma_cdb)); 2686 iter->hw_next = list_entry(iter->chain_node.next, 2687 struct ppc440spe_adma_desc_slot, 2688 chain_node); 2689 hw_desc = iter->hw_desc; 2690 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; 2691 iter->src_cnt = 0; 2692 iter->dst_cnt = 0; 2693 ppc440spe_desc_set_dest_addr(iter, chan, 0, 2694 ppc440spe_chan->qdest, 0); 2695 ppc440spe_desc_set_src_addr(iter, chan, 0, 0, qdest); 2696 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, 2697 len); 2698 iter->unmap_len = 0; 2699 /* override qdest to preserve original Q */ 2700 qdest = ppc440spe_chan->qdest; 2701 } 2702 2703 /* Setup destinations for P/Q ops */ 2704 ppc440spe_adma_pqzero_sum_set_dest(sw_desc, pdest, qdest); 2705 2706 /* Setup zero QWORDs into DCHECK CDBs */ 2707 idst = dst_cnt; 2708 list_for_each_entry_reverse(iter, &sw_desc->group_list, 2709 chain_node) { 2710 /* 2711 * The last CDB corresponds to Q-parity check, 2712 * the one before last CDB corresponds 2713 * P-parity check 2714 */ 2715 if (idst == DMA_DEST_MAX_NUM) { 2716 if (idst == dst_cnt) { 2717 set_bit(PPC440SPE_DESC_QCHECK, 2718 &iter->flags); 2719 } else { 2720 set_bit(PPC440SPE_DESC_PCHECK, 2721 &iter->flags); 2722 } 2723 } else { 2724 if (qdest) { 2725 set_bit(PPC440SPE_DESC_QCHECK, 2726 &iter->flags); 2727 } else { 2728 set_bit(PPC440SPE_DESC_PCHECK, 2729 &iter->flags); 2730 } 2731 } 2732 iter->xor_check_result = pqres; 2733 2734 /* 2735 * set it to zero, if check fail then result will 2736 * be updated 2737 */ 2738 *iter->xor_check_result = 0; 2739 ppc440spe_desc_set_dcheck(iter, ppc440spe_chan, 2740 ppc440spe_qword); 2741 2742 if (!(--dst_cnt)) 2743 break; 2744 } 2745 2746 /* Setup sources and mults for P/Q ops */ 2747 list_for_each_entry_continue_reverse(iter, &sw_desc->group_list, 2748 chain_node) { 2749 struct ppc440spe_adma_chan *chan; 2750 u32 mult_dst; 2751 2752 chan = to_ppc440spe_adma_chan(iter->async_tx.chan); 2753 ppc440spe_desc_set_src_addr(iter, chan, 0, 2754 DMA_CUED_XOR_HB, 2755 src[src_cnt - 1]); 2756 if (qdest) { 2757 mult_dst = (dst_cnt - 1) ? DMA_CDB_SG_DST2 : 2758 DMA_CDB_SG_DST1; 2759 ppc440spe_desc_set_src_mult(iter, chan, 2760 DMA_CUED_MULT1_OFF, 2761 mult_dst, 2762 scf[src_cnt - 1]); 2763 } 2764 if (!(--src_cnt)) 2765 break; 2766 } 2767 } 2768 spin_unlock_bh(&ppc440spe_chan->lock); 2769 return sw_desc ? &sw_desc->async_tx : NULL; 2770 } 2771 2772 /** 2773 * ppc440spe_adma_prep_dma_xor_zero_sum - prepare CDB group for 2774 * XOR ZERO_SUM operation 2775 */ 2776 static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_xor_zero_sum( 2777 struct dma_chan *chan, dma_addr_t *src, unsigned int src_cnt, 2778 size_t len, enum sum_check_flags *result, unsigned long flags) 2779 { 2780 struct dma_async_tx_descriptor *tx; 2781 dma_addr_t pq[2]; 2782 2783 /* validate P, disable Q */ 2784 pq[0] = src[0]; 2785 pq[1] = 0; 2786 flags |= DMA_PREP_PQ_DISABLE_Q; 2787 2788 tx = ppc440spe_adma_prep_dma_pqzero_sum(chan, pq, &src[1], 2789 src_cnt - 1, 0, len, 2790 result, flags); 2791 return tx; 2792 } 2793 2794 /** 2795 * ppc440spe_adma_set_dest - set destination address into descriptor 2796 */ 2797 static void ppc440spe_adma_set_dest(struct ppc440spe_adma_desc_slot *sw_desc, 2798 dma_addr_t addr, int index) 2799 { 2800 struct ppc440spe_adma_chan *chan; 2801 2802 BUG_ON(index >= sw_desc->dst_cnt); 2803 2804 chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan); 2805 2806 switch (chan->device->id) { 2807 case PPC440SPE_DMA0_ID: 2808 case PPC440SPE_DMA1_ID: 2809 /* to do: support transfers lengths > 2810 * PPC440SPE_ADMA_DMA/XOR_MAX_BYTE_COUNT 2811 */ 2812 ppc440spe_desc_set_dest_addr(sw_desc->group_head, 2813 chan, 0, addr, index); 2814 break; 2815 case PPC440SPE_XOR_ID: 2816 sw_desc = ppc440spe_get_group_entry(sw_desc, index); 2817 ppc440spe_desc_set_dest_addr(sw_desc, 2818 chan, 0, addr, index); 2819 break; 2820 } 2821 } 2822 2823 static void ppc440spe_adma_pq_zero_op(struct ppc440spe_adma_desc_slot *iter, 2824 struct ppc440spe_adma_chan *chan, dma_addr_t addr) 2825 { 2826 /* To clear destinations update the descriptor 2827 * (P or Q depending on index) as follows: 2828 * addr is destination (0 corresponds to SG2): 2829 */ 2830 ppc440spe_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE, addr, 0); 2831 2832 /* ... and the addr is source: */ 2833 ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB, addr); 2834 2835 /* addr is always SG2 then the mult is always DST1 */ 2836 ppc440spe_desc_set_src_mult(iter, chan, DMA_CUED_MULT1_OFF, 2837 DMA_CDB_SG_DST1, 1); 2838 } 2839 2840 /** 2841 * ppc440spe_adma_pq_set_dest - set destination address into descriptor 2842 * for the PQXOR operation 2843 */ 2844 static void ppc440spe_adma_pq_set_dest(struct ppc440spe_adma_desc_slot *sw_desc, 2845 dma_addr_t *addrs, unsigned long flags) 2846 { 2847 struct ppc440spe_adma_desc_slot *iter; 2848 struct ppc440spe_adma_chan *chan; 2849 dma_addr_t paddr, qaddr; 2850 dma_addr_t addr = 0, ppath, qpath; 2851 int index = 0, i; 2852 2853 chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan); 2854 2855 if (flags & DMA_PREP_PQ_DISABLE_P) 2856 paddr = 0; 2857 else 2858 paddr = addrs[0]; 2859 2860 if (flags & DMA_PREP_PQ_DISABLE_Q) 2861 qaddr = 0; 2862 else 2863 qaddr = addrs[1]; 2864 2865 if (!paddr || !qaddr) 2866 addr = paddr ? paddr : qaddr; 2867 2868 switch (chan->device->id) { 2869 case PPC440SPE_DMA0_ID: 2870 case PPC440SPE_DMA1_ID: 2871 /* walk through the WXOR source list and set P/Q-destinations 2872 * for each slot: 2873 */ 2874 if (!test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags)) { 2875 /* This is WXOR-only chain; may have 1/2 zero descs */ 2876 if (test_bit(PPC440SPE_ZERO_P, &sw_desc->flags)) 2877 index++; 2878 if (test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags)) 2879 index++; 2880 2881 iter = ppc440spe_get_group_entry(sw_desc, index); 2882 if (addr) { 2883 /* one destination */ 2884 list_for_each_entry_from(iter, 2885 &sw_desc->group_list, chain_node) 2886 ppc440spe_desc_set_dest_addr(iter, chan, 2887 DMA_CUED_XOR_BASE, addr, 0); 2888 } else { 2889 /* two destinations */ 2890 list_for_each_entry_from(iter, 2891 &sw_desc->group_list, chain_node) { 2892 ppc440spe_desc_set_dest_addr(iter, chan, 2893 DMA_CUED_XOR_BASE, paddr, 0); 2894 ppc440spe_desc_set_dest_addr(iter, chan, 2895 DMA_CUED_XOR_BASE, qaddr, 1); 2896 } 2897 } 2898 2899 if (index) { 2900 /* To clear destinations update the descriptor 2901 * (1st,2nd, or both depending on flags) 2902 */ 2903 index = 0; 2904 if (test_bit(PPC440SPE_ZERO_P, 2905 &sw_desc->flags)) { 2906 iter = ppc440spe_get_group_entry( 2907 sw_desc, index++); 2908 ppc440spe_adma_pq_zero_op(iter, chan, 2909 paddr); 2910 } 2911 2912 if (test_bit(PPC440SPE_ZERO_Q, 2913 &sw_desc->flags)) { 2914 iter = ppc440spe_get_group_entry( 2915 sw_desc, index++); 2916 ppc440spe_adma_pq_zero_op(iter, chan, 2917 qaddr); 2918 } 2919 2920 return; 2921 } 2922 } else { 2923 /* This is RXOR-only or RXOR/WXOR mixed chain */ 2924 2925 /* If we want to include destination into calculations, 2926 * then make dest addresses cued with mult=1 (XOR). 2927 */ 2928 ppath = test_bit(PPC440SPE_ZERO_P, &sw_desc->flags) ? 2929 DMA_CUED_XOR_HB : 2930 DMA_CUED_XOR_BASE | 2931 (1 << DMA_CUED_MULT1_OFF); 2932 qpath = test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags) ? 2933 DMA_CUED_XOR_HB : 2934 DMA_CUED_XOR_BASE | 2935 (1 << DMA_CUED_MULT1_OFF); 2936 2937 /* Setup destination(s) in RXOR slot(s) */ 2938 iter = ppc440spe_get_group_entry(sw_desc, index++); 2939 ppc440spe_desc_set_dest_addr(iter, chan, 2940 paddr ? ppath : qpath, 2941 paddr ? paddr : qaddr, 0); 2942 if (!addr) { 2943 /* two destinations */ 2944 iter = ppc440spe_get_group_entry(sw_desc, 2945 index++); 2946 ppc440spe_desc_set_dest_addr(iter, chan, 2947 qpath, qaddr, 0); 2948 } 2949 2950 if (test_bit(PPC440SPE_DESC_WXOR, &sw_desc->flags)) { 2951 /* Setup destination(s) in remaining WXOR 2952 * slots 2953 */ 2954 iter = ppc440spe_get_group_entry(sw_desc, 2955 index); 2956 if (addr) { 2957 /* one destination */ 2958 list_for_each_entry_from(iter, 2959 &sw_desc->group_list, 2960 chain_node) 2961 ppc440spe_desc_set_dest_addr( 2962 iter, chan, 2963 DMA_CUED_XOR_BASE, 2964 addr, 0); 2965 2966 } else { 2967 /* two destinations */ 2968 list_for_each_entry_from(iter, 2969 &sw_desc->group_list, 2970 chain_node) { 2971 ppc440spe_desc_set_dest_addr( 2972 iter, chan, 2973 DMA_CUED_XOR_BASE, 2974 paddr, 0); 2975 ppc440spe_desc_set_dest_addr( 2976 iter, chan, 2977 DMA_CUED_XOR_BASE, 2978 qaddr, 1); 2979 } 2980 } 2981 } 2982 2983 } 2984 break; 2985 2986 case PPC440SPE_XOR_ID: 2987 /* DMA2 descriptors have only 1 destination, so there are 2988 * two chains - one for each dest. 2989 * If we want to include destination into calculations, 2990 * then make dest addresses cued with mult=1 (XOR). 2991 */ 2992 ppath = test_bit(PPC440SPE_ZERO_P, &sw_desc->flags) ? 2993 DMA_CUED_XOR_HB : 2994 DMA_CUED_XOR_BASE | 2995 (1 << DMA_CUED_MULT1_OFF); 2996 2997 qpath = test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags) ? 2998 DMA_CUED_XOR_HB : 2999 DMA_CUED_XOR_BASE | 3000 (1 << DMA_CUED_MULT1_OFF); 3001 3002 iter = ppc440spe_get_group_entry(sw_desc, 0); 3003 for (i = 0; i < sw_desc->descs_per_op; i++) { 3004 ppc440spe_desc_set_dest_addr(iter, chan, 3005 paddr ? ppath : qpath, 3006 paddr ? paddr : qaddr, 0); 3007 iter = list_entry(iter->chain_node.next, 3008 struct ppc440spe_adma_desc_slot, 3009 chain_node); 3010 } 3011 3012 if (!addr) { 3013 /* Two destinations; setup Q here */ 3014 iter = ppc440spe_get_group_entry(sw_desc, 3015 sw_desc->descs_per_op); 3016 for (i = 0; i < sw_desc->descs_per_op; i++) { 3017 ppc440spe_desc_set_dest_addr(iter, 3018 chan, qpath, qaddr, 0); 3019 iter = list_entry(iter->chain_node.next, 3020 struct ppc440spe_adma_desc_slot, 3021 chain_node); 3022 } 3023 } 3024 3025 break; 3026 } 3027 } 3028 3029 /** 3030 * ppc440spe_adma_pq_zero_sum_set_dest - set destination address into descriptor 3031 * for the PQ_ZERO_SUM operation 3032 */ 3033 static void ppc440spe_adma_pqzero_sum_set_dest( 3034 struct ppc440spe_adma_desc_slot *sw_desc, 3035 dma_addr_t paddr, dma_addr_t qaddr) 3036 { 3037 struct ppc440spe_adma_desc_slot *iter, *end; 3038 struct ppc440spe_adma_chan *chan; 3039 dma_addr_t addr = 0; 3040 int idx; 3041 3042 chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan); 3043 3044 /* walk through the WXOR source list and set P/Q-destinations 3045 * for each slot 3046 */ 3047 idx = (paddr && qaddr) ? 2 : 1; 3048 /* set end */ 3049 list_for_each_entry_reverse(end, &sw_desc->group_list, 3050 chain_node) { 3051 if (!(--idx)) 3052 break; 3053 } 3054 /* set start */ 3055 idx = (paddr && qaddr) ? 2 : 1; 3056 iter = ppc440spe_get_group_entry(sw_desc, idx); 3057 3058 if (paddr && qaddr) { 3059 /* two destinations */ 3060 list_for_each_entry_from(iter, &sw_desc->group_list, 3061 chain_node) { 3062 if (unlikely(iter == end)) 3063 break; 3064 ppc440spe_desc_set_dest_addr(iter, chan, 3065 DMA_CUED_XOR_BASE, paddr, 0); 3066 ppc440spe_desc_set_dest_addr(iter, chan, 3067 DMA_CUED_XOR_BASE, qaddr, 1); 3068 } 3069 } else { 3070 /* one destination */ 3071 addr = paddr ? paddr : qaddr; 3072 list_for_each_entry_from(iter, &sw_desc->group_list, 3073 chain_node) { 3074 if (unlikely(iter == end)) 3075 break; 3076 ppc440spe_desc_set_dest_addr(iter, chan, 3077 DMA_CUED_XOR_BASE, addr, 0); 3078 } 3079 } 3080 3081 /* The remaining descriptors are DATACHECK. These have no need in 3082 * destination. Actually, these destinations are used there 3083 * as sources for check operation. So, set addr as source. 3084 */ 3085 ppc440spe_desc_set_src_addr(end, chan, 0, 0, addr ? addr : paddr); 3086 3087 if (!addr) { 3088 end = list_entry(end->chain_node.next, 3089 struct ppc440spe_adma_desc_slot, chain_node); 3090 ppc440spe_desc_set_src_addr(end, chan, 0, 0, qaddr); 3091 } 3092 } 3093 3094 /** 3095 * ppc440spe_desc_set_xor_src_cnt - set source count into descriptor 3096 */ 3097 static inline void ppc440spe_desc_set_xor_src_cnt( 3098 struct ppc440spe_adma_desc_slot *desc, 3099 int src_cnt) 3100 { 3101 struct xor_cb *hw_desc = desc->hw_desc; 3102 3103 hw_desc->cbc &= ~XOR_CDCR_OAC_MSK; 3104 hw_desc->cbc |= src_cnt; 3105 } 3106 3107 /** 3108 * ppc440spe_adma_pq_set_src - set source address into descriptor 3109 */ 3110 static void ppc440spe_adma_pq_set_src(struct ppc440spe_adma_desc_slot *sw_desc, 3111 dma_addr_t addr, int index) 3112 { 3113 struct ppc440spe_adma_chan *chan; 3114 dma_addr_t haddr = 0; 3115 struct ppc440spe_adma_desc_slot *iter = NULL; 3116 3117 chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan); 3118 3119 switch (chan->device->id) { 3120 case PPC440SPE_DMA0_ID: 3121 case PPC440SPE_DMA1_ID: 3122 /* DMA0,1 may do: WXOR, RXOR, RXOR+WXORs chain 3123 */ 3124 if (test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags)) { 3125 /* RXOR-only or RXOR/WXOR operation */ 3126 int iskip = test_bit(PPC440SPE_DESC_RXOR12, 3127 &sw_desc->flags) ? 2 : 3; 3128 3129 if (index == 0) { 3130 /* 1st slot (RXOR) */ 3131 /* setup sources region (R1-2-3, R1-2-4, 3132 * or R1-2-5) 3133 */ 3134 if (test_bit(PPC440SPE_DESC_RXOR12, 3135 &sw_desc->flags)) 3136 haddr = DMA_RXOR12 << 3137 DMA_CUED_REGION_OFF; 3138 else if (test_bit(PPC440SPE_DESC_RXOR123, 3139 &sw_desc->flags)) 3140 haddr = DMA_RXOR123 << 3141 DMA_CUED_REGION_OFF; 3142 else if (test_bit(PPC440SPE_DESC_RXOR124, 3143 &sw_desc->flags)) 3144 haddr = DMA_RXOR124 << 3145 DMA_CUED_REGION_OFF; 3146 else if (test_bit(PPC440SPE_DESC_RXOR125, 3147 &sw_desc->flags)) 3148 haddr = DMA_RXOR125 << 3149 DMA_CUED_REGION_OFF; 3150 else 3151 BUG(); 3152 haddr |= DMA_CUED_XOR_BASE; 3153 iter = ppc440spe_get_group_entry(sw_desc, 0); 3154 } else if (index < iskip) { 3155 /* 1st slot (RXOR) 3156 * shall actually set source address only once 3157 * instead of first <iskip> 3158 */ 3159 iter = NULL; 3160 } else { 3161 /* 2nd/3d and next slots (WXOR); 3162 * skip first slot with RXOR 3163 */ 3164 haddr = DMA_CUED_XOR_HB; 3165 iter = ppc440spe_get_group_entry(sw_desc, 3166 index - iskip + sw_desc->dst_cnt); 3167 } 3168 } else { 3169 int znum = 0; 3170 3171 /* WXOR-only operation; skip first slots with 3172 * zeroing destinations 3173 */ 3174 if (test_bit(PPC440SPE_ZERO_P, &sw_desc->flags)) 3175 znum++; 3176 if (test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags)) 3177 znum++; 3178 3179 haddr = DMA_CUED_XOR_HB; 3180 iter = ppc440spe_get_group_entry(sw_desc, 3181 index + znum); 3182 } 3183 3184 if (likely(iter)) { 3185 ppc440spe_desc_set_src_addr(iter, chan, 0, haddr, addr); 3186 3187 if (!index && 3188 test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags) && 3189 sw_desc->dst_cnt == 2) { 3190 /* if we have two destinations for RXOR, then 3191 * setup source in the second descr too 3192 */ 3193 iter = ppc440spe_get_group_entry(sw_desc, 1); 3194 ppc440spe_desc_set_src_addr(iter, chan, 0, 3195 haddr, addr); 3196 } 3197 } 3198 break; 3199 3200 case PPC440SPE_XOR_ID: 3201 /* DMA2 may do Biskup */ 3202 iter = sw_desc->group_head; 3203 if (iter->dst_cnt == 2) { 3204 /* both P & Q calculations required; set P src here */ 3205 ppc440spe_adma_dma2rxor_set_src(iter, index, addr); 3206 3207 /* this is for Q */ 3208 iter = ppc440spe_get_group_entry(sw_desc, 3209 sw_desc->descs_per_op); 3210 } 3211 ppc440spe_adma_dma2rxor_set_src(iter, index, addr); 3212 break; 3213 } 3214 } 3215 3216 /** 3217 * ppc440spe_adma_memcpy_xor_set_src - set source address into descriptor 3218 */ 3219 static void ppc440spe_adma_memcpy_xor_set_src( 3220 struct ppc440spe_adma_desc_slot *sw_desc, 3221 dma_addr_t addr, int index) 3222 { 3223 struct ppc440spe_adma_chan *chan; 3224 3225 chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan); 3226 sw_desc = sw_desc->group_head; 3227 3228 if (likely(sw_desc)) 3229 ppc440spe_desc_set_src_addr(sw_desc, chan, index, 0, addr); 3230 } 3231 3232 /** 3233 * ppc440spe_adma_dma2rxor_inc_addr - 3234 */ 3235 static void ppc440spe_adma_dma2rxor_inc_addr( 3236 struct ppc440spe_adma_desc_slot *desc, 3237 struct ppc440spe_rxor *cursor, int index, int src_cnt) 3238 { 3239 cursor->addr_count++; 3240 if (index == src_cnt - 1) { 3241 ppc440spe_desc_set_xor_src_cnt(desc, cursor->addr_count); 3242 } else if (cursor->addr_count == XOR_MAX_OPS) { 3243 ppc440spe_desc_set_xor_src_cnt(desc, cursor->addr_count); 3244 cursor->addr_count = 0; 3245 cursor->desc_count++; 3246 } 3247 } 3248 3249 /** 3250 * ppc440spe_adma_dma2rxor_prep_src - setup RXOR types in DMA2 CDB 3251 */ 3252 static int ppc440spe_adma_dma2rxor_prep_src( 3253 struct ppc440spe_adma_desc_slot *hdesc, 3254 struct ppc440spe_rxor *cursor, int index, 3255 int src_cnt, u32 addr) 3256 { 3257 int rval = 0; 3258 u32 sign; 3259 struct ppc440spe_adma_desc_slot *desc = hdesc; 3260 int i; 3261 3262 for (i = 0; i < cursor->desc_count; i++) { 3263 desc = list_entry(hdesc->chain_node.next, 3264 struct ppc440spe_adma_desc_slot, 3265 chain_node); 3266 } 3267 3268 switch (cursor->state) { 3269 case 0: 3270 if (addr == cursor->addrl + cursor->len) { 3271 /* direct RXOR */ 3272 cursor->state = 1; 3273 cursor->xor_count++; 3274 if (index == src_cnt-1) { 3275 ppc440spe_rxor_set_region(desc, 3276 cursor->addr_count, 3277 DMA_RXOR12 << DMA_CUED_REGION_OFF); 3278 ppc440spe_adma_dma2rxor_inc_addr( 3279 desc, cursor, index, src_cnt); 3280 } 3281 } else if (cursor->addrl == addr + cursor->len) { 3282 /* reverse RXOR */ 3283 cursor->state = 1; 3284 cursor->xor_count++; 3285 set_bit(cursor->addr_count, &desc->reverse_flags[0]); 3286 if (index == src_cnt-1) { 3287 ppc440spe_rxor_set_region(desc, 3288 cursor->addr_count, 3289 DMA_RXOR12 << DMA_CUED_REGION_OFF); 3290 ppc440spe_adma_dma2rxor_inc_addr( 3291 desc, cursor, index, src_cnt); 3292 } 3293 } else { 3294 printk(KERN_ERR "Cannot build " 3295 "DMA2 RXOR command block.\n"); 3296 BUG(); 3297 } 3298 break; 3299 case 1: 3300 sign = test_bit(cursor->addr_count, 3301 desc->reverse_flags) 3302 ? -1 : 1; 3303 if (index == src_cnt-2 || (sign == -1 3304 && addr != cursor->addrl - 2*cursor->len)) { 3305 cursor->state = 0; 3306 cursor->xor_count = 1; 3307 cursor->addrl = addr; 3308 ppc440spe_rxor_set_region(desc, 3309 cursor->addr_count, 3310 DMA_RXOR12 << DMA_CUED_REGION_OFF); 3311 ppc440spe_adma_dma2rxor_inc_addr( 3312 desc, cursor, index, src_cnt); 3313 } else if (addr == cursor->addrl + 2*sign*cursor->len) { 3314 cursor->state = 2; 3315 cursor->xor_count = 0; 3316 ppc440spe_rxor_set_region(desc, 3317 cursor->addr_count, 3318 DMA_RXOR123 << DMA_CUED_REGION_OFF); 3319 if (index == src_cnt-1) { 3320 ppc440spe_adma_dma2rxor_inc_addr( 3321 desc, cursor, index, src_cnt); 3322 } 3323 } else if (addr == cursor->addrl + 3*cursor->len) { 3324 cursor->state = 2; 3325 cursor->xor_count = 0; 3326 ppc440spe_rxor_set_region(desc, 3327 cursor->addr_count, 3328 DMA_RXOR124 << DMA_CUED_REGION_OFF); 3329 if (index == src_cnt-1) { 3330 ppc440spe_adma_dma2rxor_inc_addr( 3331 desc, cursor, index, src_cnt); 3332 } 3333 } else if (addr == cursor->addrl + 4*cursor->len) { 3334 cursor->state = 2; 3335 cursor->xor_count = 0; 3336 ppc440spe_rxor_set_region(desc, 3337 cursor->addr_count, 3338 DMA_RXOR125 << DMA_CUED_REGION_OFF); 3339 if (index == src_cnt-1) { 3340 ppc440spe_adma_dma2rxor_inc_addr( 3341 desc, cursor, index, src_cnt); 3342 } 3343 } else { 3344 cursor->state = 0; 3345 cursor->xor_count = 1; 3346 cursor->addrl = addr; 3347 ppc440spe_rxor_set_region(desc, 3348 cursor->addr_count, 3349 DMA_RXOR12 << DMA_CUED_REGION_OFF); 3350 ppc440spe_adma_dma2rxor_inc_addr( 3351 desc, cursor, index, src_cnt); 3352 } 3353 break; 3354 case 2: 3355 cursor->state = 0; 3356 cursor->addrl = addr; 3357 cursor->xor_count++; 3358 if (index) { 3359 ppc440spe_adma_dma2rxor_inc_addr( 3360 desc, cursor, index, src_cnt); 3361 } 3362 break; 3363 } 3364 3365 return rval; 3366 } 3367 3368 /** 3369 * ppc440spe_adma_dma2rxor_set_src - set RXOR source address; it's assumed that 3370 * ppc440spe_adma_dma2rxor_prep_src() has already done prior this call 3371 */ 3372 static void ppc440spe_adma_dma2rxor_set_src( 3373 struct ppc440spe_adma_desc_slot *desc, 3374 int index, dma_addr_t addr) 3375 { 3376 struct xor_cb *xcb = desc->hw_desc; 3377 int k = 0, op = 0, lop = 0; 3378 3379 /* get the RXOR operand which corresponds to index addr */ 3380 while (op <= index) { 3381 lop = op; 3382 if (k == XOR_MAX_OPS) { 3383 k = 0; 3384 desc = list_entry(desc->chain_node.next, 3385 struct ppc440spe_adma_desc_slot, chain_node); 3386 xcb = desc->hw_desc; 3387 3388 } 3389 if ((xcb->ops[k++].h & (DMA_RXOR12 << DMA_CUED_REGION_OFF)) == 3390 (DMA_RXOR12 << DMA_CUED_REGION_OFF)) 3391 op += 2; 3392 else 3393 op += 3; 3394 } 3395 3396 BUG_ON(k < 1); 3397 3398 if (test_bit(k-1, desc->reverse_flags)) { 3399 /* reverse operand order; put last op in RXOR group */ 3400 if (index == op - 1) 3401 ppc440spe_rxor_set_src(desc, k - 1, addr); 3402 } else { 3403 /* direct operand order; put first op in RXOR group */ 3404 if (index == lop) 3405 ppc440spe_rxor_set_src(desc, k - 1, addr); 3406 } 3407 } 3408 3409 /** 3410 * ppc440spe_adma_dma2rxor_set_mult - set RXOR multipliers; it's assumed that 3411 * ppc440spe_adma_dma2rxor_prep_src() has already done prior this call 3412 */ 3413 static void ppc440spe_adma_dma2rxor_set_mult( 3414 struct ppc440spe_adma_desc_slot *desc, 3415 int index, u8 mult) 3416 { 3417 struct xor_cb *xcb = desc->hw_desc; 3418 int k = 0, op = 0, lop = 0; 3419 3420 /* get the RXOR operand which corresponds to index mult */ 3421 while (op <= index) { 3422 lop = op; 3423 if (k == XOR_MAX_OPS) { 3424 k = 0; 3425 desc = list_entry(desc->chain_node.next, 3426 struct ppc440spe_adma_desc_slot, 3427 chain_node); 3428 xcb = desc->hw_desc; 3429 3430 } 3431 if ((xcb->ops[k++].h & (DMA_RXOR12 << DMA_CUED_REGION_OFF)) == 3432 (DMA_RXOR12 << DMA_CUED_REGION_OFF)) 3433 op += 2; 3434 else 3435 op += 3; 3436 } 3437 3438 BUG_ON(k < 1); 3439 if (test_bit(k-1, desc->reverse_flags)) { 3440 /* reverse order */ 3441 ppc440spe_rxor_set_mult(desc, k - 1, op - index - 1, mult); 3442 } else { 3443 /* direct order */ 3444 ppc440spe_rxor_set_mult(desc, k - 1, index - lop, mult); 3445 } 3446 } 3447 3448 /** 3449 * ppc440spe_init_rxor_cursor - 3450 */ 3451 static void ppc440spe_init_rxor_cursor(struct ppc440spe_rxor *cursor) 3452 { 3453 memset(cursor, 0, sizeof(struct ppc440spe_rxor)); 3454 cursor->state = 2; 3455 } 3456 3457 /** 3458 * ppc440spe_adma_pq_set_src_mult - set multiplication coefficient into 3459 * descriptor for the PQXOR operation 3460 */ 3461 static void ppc440spe_adma_pq_set_src_mult( 3462 struct ppc440spe_adma_desc_slot *sw_desc, 3463 unsigned char mult, int index, int dst_pos) 3464 { 3465 struct ppc440spe_adma_chan *chan; 3466 u32 mult_idx, mult_dst; 3467 struct ppc440spe_adma_desc_slot *iter = NULL, *iter1 = NULL; 3468 3469 chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan); 3470 3471 switch (chan->device->id) { 3472 case PPC440SPE_DMA0_ID: 3473 case PPC440SPE_DMA1_ID: 3474 if (test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags)) { 3475 int region = test_bit(PPC440SPE_DESC_RXOR12, 3476 &sw_desc->flags) ? 2 : 3; 3477 3478 if (index < region) { 3479 /* RXOR multipliers */ 3480 iter = ppc440spe_get_group_entry(sw_desc, 3481 sw_desc->dst_cnt - 1); 3482 if (sw_desc->dst_cnt == 2) 3483 iter1 = ppc440spe_get_group_entry( 3484 sw_desc, 0); 3485 3486 mult_idx = DMA_CUED_MULT1_OFF + (index << 3); 3487 mult_dst = DMA_CDB_SG_SRC; 3488 } else { 3489 /* WXOR multiplier */ 3490 iter = ppc440spe_get_group_entry(sw_desc, 3491 index - region + 3492 sw_desc->dst_cnt); 3493 mult_idx = DMA_CUED_MULT1_OFF; 3494 mult_dst = dst_pos ? DMA_CDB_SG_DST2 : 3495 DMA_CDB_SG_DST1; 3496 } 3497 } else { 3498 int znum = 0; 3499 3500 /* WXOR-only; 3501 * skip first slots with destinations (if ZERO_DST has 3502 * place) 3503 */ 3504 if (test_bit(PPC440SPE_ZERO_P, &sw_desc->flags)) 3505 znum++; 3506 if (test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags)) 3507 znum++; 3508 3509 iter = ppc440spe_get_group_entry(sw_desc, index + znum); 3510 mult_idx = DMA_CUED_MULT1_OFF; 3511 mult_dst = dst_pos ? DMA_CDB_SG_DST2 : DMA_CDB_SG_DST1; 3512 } 3513 3514 if (likely(iter)) { 3515 ppc440spe_desc_set_src_mult(iter, chan, 3516 mult_idx, mult_dst, mult); 3517 3518 if (unlikely(iter1)) { 3519 /* if we have two destinations for RXOR, then 3520 * we've just set Q mult. Set-up P now. 3521 */ 3522 ppc440spe_desc_set_src_mult(iter1, chan, 3523 mult_idx, mult_dst, 1); 3524 } 3525 3526 } 3527 break; 3528 3529 case PPC440SPE_XOR_ID: 3530 iter = sw_desc->group_head; 3531 if (sw_desc->dst_cnt == 2) { 3532 /* both P & Q calculations required; set P mult here */ 3533 ppc440spe_adma_dma2rxor_set_mult(iter, index, 1); 3534 3535 /* and then set Q mult */ 3536 iter = ppc440spe_get_group_entry(sw_desc, 3537 sw_desc->descs_per_op); 3538 } 3539 ppc440spe_adma_dma2rxor_set_mult(iter, index, mult); 3540 break; 3541 } 3542 } 3543 3544 /** 3545 * ppc440spe_adma_free_chan_resources - free the resources allocated 3546 */ 3547 static void ppc440spe_adma_free_chan_resources(struct dma_chan *chan) 3548 { 3549 struct ppc440spe_adma_chan *ppc440spe_chan; 3550 struct ppc440spe_adma_desc_slot *iter, *_iter; 3551 int in_use_descs = 0; 3552 3553 ppc440spe_chan = to_ppc440spe_adma_chan(chan); 3554 ppc440spe_adma_slot_cleanup(ppc440spe_chan); 3555 3556 spin_lock_bh(&ppc440spe_chan->lock); 3557 list_for_each_entry_safe(iter, _iter, &ppc440spe_chan->chain, 3558 chain_node) { 3559 in_use_descs++; 3560 list_del(&iter->chain_node); 3561 } 3562 list_for_each_entry_safe_reverse(iter, _iter, 3563 &ppc440spe_chan->all_slots, slot_node) { 3564 list_del(&iter->slot_node); 3565 kfree(iter); 3566 ppc440spe_chan->slots_allocated--; 3567 } 3568 ppc440spe_chan->last_used = NULL; 3569 3570 dev_dbg(ppc440spe_chan->device->common.dev, 3571 "ppc440spe adma%d %s slots_allocated %d\n", 3572 ppc440spe_chan->device->id, 3573 __func__, ppc440spe_chan->slots_allocated); 3574 spin_unlock_bh(&ppc440spe_chan->lock); 3575 3576 /* one is ok since we left it on there on purpose */ 3577 if (in_use_descs > 1) 3578 printk(KERN_ERR "SPE: Freeing %d in use descriptors!\n", 3579 in_use_descs - 1); 3580 } 3581 3582 /** 3583 * ppc440spe_adma_tx_status - poll the status of an ADMA transaction 3584 * @chan: ADMA channel handle 3585 * @cookie: ADMA transaction identifier 3586 * @txstate: a holder for the current state of the channel 3587 */ 3588 static enum dma_status ppc440spe_adma_tx_status(struct dma_chan *chan, 3589 dma_cookie_t cookie, struct dma_tx_state *txstate) 3590 { 3591 struct ppc440spe_adma_chan *ppc440spe_chan; 3592 enum dma_status ret; 3593 3594 ppc440spe_chan = to_ppc440spe_adma_chan(chan); 3595 ret = dma_cookie_status(chan, cookie, txstate); 3596 if (ret == DMA_COMPLETE) 3597 return ret; 3598 3599 ppc440spe_adma_slot_cleanup(ppc440spe_chan); 3600 3601 return dma_cookie_status(chan, cookie, txstate); 3602 } 3603 3604 /** 3605 * ppc440spe_adma_eot_handler - end of transfer interrupt handler 3606 */ 3607 static irqreturn_t ppc440spe_adma_eot_handler(int irq, void *data) 3608 { 3609 struct ppc440spe_adma_chan *chan = data; 3610 3611 dev_dbg(chan->device->common.dev, 3612 "ppc440spe adma%d: %s\n", chan->device->id, __func__); 3613 3614 tasklet_schedule(&chan->irq_tasklet); 3615 ppc440spe_adma_device_clear_eot_status(chan); 3616 3617 return IRQ_HANDLED; 3618 } 3619 3620 /** 3621 * ppc440spe_adma_err_handler - DMA error interrupt handler; 3622 * do the same things as a eot handler 3623 */ 3624 static irqreturn_t ppc440spe_adma_err_handler(int irq, void *data) 3625 { 3626 struct ppc440spe_adma_chan *chan = data; 3627 3628 dev_dbg(chan->device->common.dev, 3629 "ppc440spe adma%d: %s\n", chan->device->id, __func__); 3630 3631 tasklet_schedule(&chan->irq_tasklet); 3632 ppc440spe_adma_device_clear_eot_status(chan); 3633 3634 return IRQ_HANDLED; 3635 } 3636 3637 /** 3638 * ppc440spe_test_callback - called when test operation has been done 3639 */ 3640 static void ppc440spe_test_callback(void *unused) 3641 { 3642 complete(&ppc440spe_r6_test_comp); 3643 } 3644 3645 /** 3646 * ppc440spe_adma_issue_pending - flush all pending descriptors to h/w 3647 */ 3648 static void ppc440spe_adma_issue_pending(struct dma_chan *chan) 3649 { 3650 struct ppc440spe_adma_chan *ppc440spe_chan; 3651 3652 ppc440spe_chan = to_ppc440spe_adma_chan(chan); 3653 dev_dbg(ppc440spe_chan->device->common.dev, 3654 "ppc440spe adma%d: %s %d \n", ppc440spe_chan->device->id, 3655 __func__, ppc440spe_chan->pending); 3656 3657 if (ppc440spe_chan->pending) { 3658 ppc440spe_chan->pending = 0; 3659 ppc440spe_chan_append(ppc440spe_chan); 3660 } 3661 } 3662 3663 /** 3664 * ppc440spe_chan_start_null_xor - initiate the first XOR operation (DMA engines 3665 * use FIFOs (as opposite to chains used in XOR) so this is a XOR 3666 * specific operation) 3667 */ 3668 static void ppc440spe_chan_start_null_xor(struct ppc440spe_adma_chan *chan) 3669 { 3670 struct ppc440spe_adma_desc_slot *sw_desc, *group_start; 3671 dma_cookie_t cookie; 3672 int slot_cnt, slots_per_op; 3673 3674 dev_dbg(chan->device->common.dev, 3675 "ppc440spe adma%d: %s\n", chan->device->id, __func__); 3676 3677 spin_lock_bh(&chan->lock); 3678 slot_cnt = ppc440spe_chan_xor_slot_count(0, 2, &slots_per_op); 3679 sw_desc = ppc440spe_adma_alloc_slots(chan, slot_cnt, slots_per_op); 3680 if (sw_desc) { 3681 group_start = sw_desc->group_head; 3682 list_splice_init(&sw_desc->group_list, &chan->chain); 3683 async_tx_ack(&sw_desc->async_tx); 3684 ppc440spe_desc_init_null_xor(group_start); 3685 3686 cookie = dma_cookie_assign(&sw_desc->async_tx); 3687 3688 /* initialize the completed cookie to be less than 3689 * the most recently used cookie 3690 */ 3691 chan->common.completed_cookie = cookie - 1; 3692 3693 /* channel should not be busy */ 3694 BUG_ON(ppc440spe_chan_is_busy(chan)); 3695 3696 /* set the descriptor address */ 3697 ppc440spe_chan_set_first_xor_descriptor(chan, sw_desc); 3698 3699 /* run the descriptor */ 3700 ppc440spe_chan_run(chan); 3701 } else 3702 printk(KERN_ERR "ppc440spe adma%d" 3703 " failed to allocate null descriptor\n", 3704 chan->device->id); 3705 spin_unlock_bh(&chan->lock); 3706 } 3707 3708 /** 3709 * ppc440spe_test_raid6 - test are RAID-6 capabilities enabled successfully. 3710 * For this we just perform one WXOR operation with the same source 3711 * and destination addresses, the GF-multiplier is 1; so if RAID-6 3712 * capabilities are enabled then we'll get src/dst filled with zero. 3713 */ 3714 static int ppc440spe_test_raid6(struct ppc440spe_adma_chan *chan) 3715 { 3716 struct ppc440spe_adma_desc_slot *sw_desc, *iter; 3717 struct page *pg; 3718 char *a; 3719 dma_addr_t dma_addr, addrs[2]; 3720 unsigned long op = 0; 3721 int rval = 0; 3722 3723 set_bit(PPC440SPE_DESC_WXOR, &op); 3724 3725 pg = alloc_page(GFP_KERNEL); 3726 if (!pg) 3727 return -ENOMEM; 3728 3729 spin_lock_bh(&chan->lock); 3730 sw_desc = ppc440spe_adma_alloc_slots(chan, 1, 1); 3731 if (sw_desc) { 3732 /* 1 src, 1 dsr, int_ena, WXOR */ 3733 ppc440spe_desc_init_dma01pq(sw_desc, 1, 1, 1, op); 3734 list_for_each_entry(iter, &sw_desc->group_list, chain_node) { 3735 ppc440spe_desc_set_byte_count(iter, chan, PAGE_SIZE); 3736 iter->unmap_len = PAGE_SIZE; 3737 } 3738 } else { 3739 rval = -EFAULT; 3740 spin_unlock_bh(&chan->lock); 3741 goto exit; 3742 } 3743 spin_unlock_bh(&chan->lock); 3744 3745 /* Fill the test page with ones */ 3746 memset(page_address(pg), 0xFF, PAGE_SIZE); 3747 dma_addr = dma_map_page(chan->device->dev, pg, 0, 3748 PAGE_SIZE, DMA_BIDIRECTIONAL); 3749 3750 /* Setup addresses */ 3751 ppc440spe_adma_pq_set_src(sw_desc, dma_addr, 0); 3752 ppc440spe_adma_pq_set_src_mult(sw_desc, 1, 0, 0); 3753 addrs[0] = dma_addr; 3754 addrs[1] = 0; 3755 ppc440spe_adma_pq_set_dest(sw_desc, addrs, DMA_PREP_PQ_DISABLE_Q); 3756 3757 async_tx_ack(&sw_desc->async_tx); 3758 sw_desc->async_tx.callback = ppc440spe_test_callback; 3759 sw_desc->async_tx.callback_param = NULL; 3760 3761 init_completion(&ppc440spe_r6_test_comp); 3762 3763 ppc440spe_adma_tx_submit(&sw_desc->async_tx); 3764 ppc440spe_adma_issue_pending(&chan->common); 3765 3766 wait_for_completion(&ppc440spe_r6_test_comp); 3767 3768 /* Now check if the test page is zeroed */ 3769 a = page_address(pg); 3770 if ((*(u32 *)a) == 0 && memcmp(a, a+4, PAGE_SIZE-4) == 0) { 3771 /* page is zero - RAID-6 enabled */ 3772 rval = 0; 3773 } else { 3774 /* RAID-6 was not enabled */ 3775 rval = -EINVAL; 3776 } 3777 exit: 3778 __free_page(pg); 3779 return rval; 3780 } 3781 3782 static void ppc440spe_adma_init_capabilities(struct ppc440spe_adma_device *adev) 3783 { 3784 switch (adev->id) { 3785 case PPC440SPE_DMA0_ID: 3786 case PPC440SPE_DMA1_ID: 3787 dma_cap_set(DMA_MEMCPY, adev->common.cap_mask); 3788 dma_cap_set(DMA_INTERRUPT, adev->common.cap_mask); 3789 dma_cap_set(DMA_PQ, adev->common.cap_mask); 3790 dma_cap_set(DMA_PQ_VAL, adev->common.cap_mask); 3791 dma_cap_set(DMA_XOR_VAL, adev->common.cap_mask); 3792 break; 3793 case PPC440SPE_XOR_ID: 3794 dma_cap_set(DMA_XOR, adev->common.cap_mask); 3795 dma_cap_set(DMA_PQ, adev->common.cap_mask); 3796 dma_cap_set(DMA_INTERRUPT, adev->common.cap_mask); 3797 adev->common.cap_mask = adev->common.cap_mask; 3798 break; 3799 } 3800 3801 /* Set base routines */ 3802 adev->common.device_alloc_chan_resources = 3803 ppc440spe_adma_alloc_chan_resources; 3804 adev->common.device_free_chan_resources = 3805 ppc440spe_adma_free_chan_resources; 3806 adev->common.device_tx_status = ppc440spe_adma_tx_status; 3807 adev->common.device_issue_pending = ppc440spe_adma_issue_pending; 3808 3809 /* Set prep routines based on capability */ 3810 if (dma_has_cap(DMA_MEMCPY, adev->common.cap_mask)) { 3811 adev->common.device_prep_dma_memcpy = 3812 ppc440spe_adma_prep_dma_memcpy; 3813 } 3814 if (dma_has_cap(DMA_XOR, adev->common.cap_mask)) { 3815 adev->common.max_xor = XOR_MAX_OPS; 3816 adev->common.device_prep_dma_xor = 3817 ppc440spe_adma_prep_dma_xor; 3818 } 3819 if (dma_has_cap(DMA_PQ, adev->common.cap_mask)) { 3820 switch (adev->id) { 3821 case PPC440SPE_DMA0_ID: 3822 dma_set_maxpq(&adev->common, 3823 DMA0_FIFO_SIZE / sizeof(struct dma_cdb), 0); 3824 break; 3825 case PPC440SPE_DMA1_ID: 3826 dma_set_maxpq(&adev->common, 3827 DMA1_FIFO_SIZE / sizeof(struct dma_cdb), 0); 3828 break; 3829 case PPC440SPE_XOR_ID: 3830 adev->common.max_pq = XOR_MAX_OPS * 3; 3831 break; 3832 } 3833 adev->common.device_prep_dma_pq = 3834 ppc440spe_adma_prep_dma_pq; 3835 } 3836 if (dma_has_cap(DMA_PQ_VAL, adev->common.cap_mask)) { 3837 switch (adev->id) { 3838 case PPC440SPE_DMA0_ID: 3839 adev->common.max_pq = DMA0_FIFO_SIZE / 3840 sizeof(struct dma_cdb); 3841 break; 3842 case PPC440SPE_DMA1_ID: 3843 adev->common.max_pq = DMA1_FIFO_SIZE / 3844 sizeof(struct dma_cdb); 3845 break; 3846 } 3847 adev->common.device_prep_dma_pq_val = 3848 ppc440spe_adma_prep_dma_pqzero_sum; 3849 } 3850 if (dma_has_cap(DMA_XOR_VAL, adev->common.cap_mask)) { 3851 switch (adev->id) { 3852 case PPC440SPE_DMA0_ID: 3853 adev->common.max_xor = DMA0_FIFO_SIZE / 3854 sizeof(struct dma_cdb); 3855 break; 3856 case PPC440SPE_DMA1_ID: 3857 adev->common.max_xor = DMA1_FIFO_SIZE / 3858 sizeof(struct dma_cdb); 3859 break; 3860 } 3861 adev->common.device_prep_dma_xor_val = 3862 ppc440spe_adma_prep_dma_xor_zero_sum; 3863 } 3864 if (dma_has_cap(DMA_INTERRUPT, adev->common.cap_mask)) { 3865 adev->common.device_prep_dma_interrupt = 3866 ppc440spe_adma_prep_dma_interrupt; 3867 } 3868 pr_info("%s: AMCC(R) PPC440SP(E) ADMA Engine: " 3869 "( %s%s%s%s%s%s)\n", 3870 dev_name(adev->dev), 3871 dma_has_cap(DMA_PQ, adev->common.cap_mask) ? "pq " : "", 3872 dma_has_cap(DMA_PQ_VAL, adev->common.cap_mask) ? "pq_val " : "", 3873 dma_has_cap(DMA_XOR, adev->common.cap_mask) ? "xor " : "", 3874 dma_has_cap(DMA_XOR_VAL, adev->common.cap_mask) ? "xor_val " : "", 3875 dma_has_cap(DMA_MEMCPY, adev->common.cap_mask) ? "memcpy " : "", 3876 dma_has_cap(DMA_INTERRUPT, adev->common.cap_mask) ? "intr " : ""); 3877 } 3878 3879 static int ppc440spe_adma_setup_irqs(struct ppc440spe_adma_device *adev, 3880 struct ppc440spe_adma_chan *chan, 3881 int *initcode) 3882 { 3883 struct platform_device *ofdev; 3884 struct device_node *np; 3885 int ret; 3886 3887 ofdev = container_of(adev->dev, struct platform_device, dev); 3888 np = ofdev->dev.of_node; 3889 if (adev->id != PPC440SPE_XOR_ID) { 3890 adev->err_irq = irq_of_parse_and_map(np, 1); 3891 if (!adev->err_irq) { 3892 dev_warn(adev->dev, "no err irq resource?\n"); 3893 *initcode = PPC_ADMA_INIT_IRQ2; 3894 adev->err_irq = -ENXIO; 3895 } else 3896 atomic_inc(&ppc440spe_adma_err_irq_ref); 3897 } else { 3898 adev->err_irq = -ENXIO; 3899 } 3900 3901 adev->irq = irq_of_parse_and_map(np, 0); 3902 if (!adev->irq) { 3903 dev_err(adev->dev, "no irq resource\n"); 3904 *initcode = PPC_ADMA_INIT_IRQ1; 3905 ret = -ENXIO; 3906 goto err_irq_map; 3907 } 3908 dev_dbg(adev->dev, "irq %d, err irq %d\n", 3909 adev->irq, adev->err_irq); 3910 3911 ret = request_irq(adev->irq, ppc440spe_adma_eot_handler, 3912 0, dev_driver_string(adev->dev), chan); 3913 if (ret) { 3914 dev_err(adev->dev, "can't request irq %d\n", 3915 adev->irq); 3916 *initcode = PPC_ADMA_INIT_IRQ1; 3917 ret = -EIO; 3918 goto err_req1; 3919 } 3920 3921 /* only DMA engines have a separate error IRQ 3922 * so it's Ok if err_irq < 0 in XOR engine case. 3923 */ 3924 if (adev->err_irq > 0) { 3925 /* both DMA engines share common error IRQ */ 3926 ret = request_irq(adev->err_irq, 3927 ppc440spe_adma_err_handler, 3928 IRQF_SHARED, 3929 dev_driver_string(adev->dev), 3930 chan); 3931 if (ret) { 3932 dev_err(adev->dev, "can't request irq %d\n", 3933 adev->err_irq); 3934 *initcode = PPC_ADMA_INIT_IRQ2; 3935 ret = -EIO; 3936 goto err_req2; 3937 } 3938 } 3939 3940 if (adev->id == PPC440SPE_XOR_ID) { 3941 /* enable XOR engine interrupts */ 3942 iowrite32be(XOR_IE_CBCIE_BIT | XOR_IE_ICBIE_BIT | 3943 XOR_IE_ICIE_BIT | XOR_IE_RPTIE_BIT, 3944 &adev->xor_reg->ier); 3945 } else { 3946 u32 mask, enable; 3947 3948 np = of_find_compatible_node(NULL, NULL, "ibm,i2o-440spe"); 3949 if (!np) { 3950 pr_err("%s: can't find I2O device tree node\n", 3951 __func__); 3952 ret = -ENODEV; 3953 goto err_req2; 3954 } 3955 adev->i2o_reg = of_iomap(np, 0); 3956 if (!adev->i2o_reg) { 3957 pr_err("%s: failed to map I2O registers\n", __func__); 3958 of_node_put(np); 3959 ret = -EINVAL; 3960 goto err_req2; 3961 } 3962 of_node_put(np); 3963 /* Unmask 'CS FIFO Attention' interrupts and 3964 * enable generating interrupts on errors 3965 */ 3966 enable = (adev->id == PPC440SPE_DMA0_ID) ? 3967 ~(I2O_IOPIM_P0SNE | I2O_IOPIM_P0EM) : 3968 ~(I2O_IOPIM_P1SNE | I2O_IOPIM_P1EM); 3969 mask = ioread32(&adev->i2o_reg->iopim) & enable; 3970 iowrite32(mask, &adev->i2o_reg->iopim); 3971 } 3972 return 0; 3973 3974 err_req2: 3975 free_irq(adev->irq, chan); 3976 err_req1: 3977 irq_dispose_mapping(adev->irq); 3978 err_irq_map: 3979 if (adev->err_irq > 0) { 3980 if (atomic_dec_and_test(&ppc440spe_adma_err_irq_ref)) 3981 irq_dispose_mapping(adev->err_irq); 3982 } 3983 return ret; 3984 } 3985 3986 static void ppc440spe_adma_release_irqs(struct ppc440spe_adma_device *adev, 3987 struct ppc440spe_adma_chan *chan) 3988 { 3989 u32 mask, disable; 3990 3991 if (adev->id == PPC440SPE_XOR_ID) { 3992 /* disable XOR engine interrupts */ 3993 mask = ioread32be(&adev->xor_reg->ier); 3994 mask &= ~(XOR_IE_CBCIE_BIT | XOR_IE_ICBIE_BIT | 3995 XOR_IE_ICIE_BIT | XOR_IE_RPTIE_BIT); 3996 iowrite32be(mask, &adev->xor_reg->ier); 3997 } else { 3998 /* disable DMAx engine interrupts */ 3999 disable = (adev->id == PPC440SPE_DMA0_ID) ? 4000 (I2O_IOPIM_P0SNE | I2O_IOPIM_P0EM) : 4001 (I2O_IOPIM_P1SNE | I2O_IOPIM_P1EM); 4002 mask = ioread32(&adev->i2o_reg->iopim) | disable; 4003 iowrite32(mask, &adev->i2o_reg->iopim); 4004 } 4005 free_irq(adev->irq, chan); 4006 irq_dispose_mapping(adev->irq); 4007 if (adev->err_irq > 0) { 4008 free_irq(adev->err_irq, chan); 4009 if (atomic_dec_and_test(&ppc440spe_adma_err_irq_ref)) { 4010 irq_dispose_mapping(adev->err_irq); 4011 iounmap(adev->i2o_reg); 4012 } 4013 } 4014 } 4015 4016 /** 4017 * ppc440spe_adma_probe - probe the asynch device 4018 */ 4019 static int ppc440spe_adma_probe(struct platform_device *ofdev) 4020 { 4021 struct device_node *np = ofdev->dev.of_node; 4022 struct resource res; 4023 struct ppc440spe_adma_device *adev; 4024 struct ppc440spe_adma_chan *chan; 4025 struct ppc_dma_chan_ref *ref, *_ref; 4026 int ret = 0, initcode = PPC_ADMA_INIT_OK; 4027 const u32 *idx; 4028 int len; 4029 void *regs; 4030 u32 id, pool_size; 4031 4032 if (of_device_is_compatible(np, "amcc,xor-accelerator")) { 4033 id = PPC440SPE_XOR_ID; 4034 /* As far as the XOR engine is concerned, it does not 4035 * use FIFOs but uses linked list. So there is no dependency 4036 * between pool size to allocate and the engine configuration. 4037 */ 4038 pool_size = PAGE_SIZE << 1; 4039 } else { 4040 /* it is DMA0 or DMA1 */ 4041 idx = of_get_property(np, "cell-index", &len); 4042 if (!idx || (len != sizeof(u32))) { 4043 dev_err(&ofdev->dev, "Device node %pOF has missing " 4044 "or invalid cell-index property\n", 4045 np); 4046 return -EINVAL; 4047 } 4048 id = *idx; 4049 /* DMA0,1 engines use FIFO to maintain CDBs, so we 4050 * should allocate the pool accordingly to size of this 4051 * FIFO. Thus, the pool size depends on the FIFO depth: 4052 * how much CDBs pointers the FIFO may contain then so 4053 * much CDBs we should provide in the pool. 4054 * That is 4055 * CDB size = 32B; 4056 * CDBs number = (DMA0_FIFO_SIZE >> 3); 4057 * Pool size = CDBs number * CDB size = 4058 * = (DMA0_FIFO_SIZE >> 3) << 5 = DMA0_FIFO_SIZE << 2. 4059 */ 4060 pool_size = (id == PPC440SPE_DMA0_ID) ? 4061 DMA0_FIFO_SIZE : DMA1_FIFO_SIZE; 4062 pool_size <<= 2; 4063 } 4064 4065 if (of_address_to_resource(np, 0, &res)) { 4066 dev_err(&ofdev->dev, "failed to get memory resource\n"); 4067 initcode = PPC_ADMA_INIT_MEMRES; 4068 ret = -ENODEV; 4069 goto out; 4070 } 4071 4072 if (!request_mem_region(res.start, resource_size(&res), 4073 dev_driver_string(&ofdev->dev))) { 4074 dev_err(&ofdev->dev, "failed to request memory region %pR\n", 4075 &res); 4076 initcode = PPC_ADMA_INIT_MEMREG; 4077 ret = -EBUSY; 4078 goto out; 4079 } 4080 4081 /* create a device */ 4082 adev = kzalloc(sizeof(*adev), GFP_KERNEL); 4083 if (!adev) { 4084 initcode = PPC_ADMA_INIT_ALLOC; 4085 ret = -ENOMEM; 4086 goto err_adev_alloc; 4087 } 4088 4089 adev->id = id; 4090 adev->pool_size = pool_size; 4091 /* allocate coherent memory for hardware descriptors */ 4092 adev->dma_desc_pool_virt = dma_alloc_coherent(&ofdev->dev, 4093 adev->pool_size, &adev->dma_desc_pool, 4094 GFP_KERNEL); 4095 if (adev->dma_desc_pool_virt == NULL) { 4096 dev_err(&ofdev->dev, "failed to allocate %d bytes of coherent " 4097 "memory for hardware descriptors\n", 4098 adev->pool_size); 4099 initcode = PPC_ADMA_INIT_COHERENT; 4100 ret = -ENOMEM; 4101 goto err_dma_alloc; 4102 } 4103 dev_dbg(&ofdev->dev, "allocated descriptor pool virt 0x%p phys 0x%llx\n", 4104 adev->dma_desc_pool_virt, (u64)adev->dma_desc_pool); 4105 4106 regs = ioremap(res.start, resource_size(&res)); 4107 if (!regs) { 4108 dev_err(&ofdev->dev, "failed to ioremap regs!\n"); 4109 ret = -ENOMEM; 4110 goto err_regs_alloc; 4111 } 4112 4113 if (adev->id == PPC440SPE_XOR_ID) { 4114 adev->xor_reg = regs; 4115 /* Reset XOR */ 4116 iowrite32be(XOR_CRSR_XASR_BIT, &adev->xor_reg->crsr); 4117 iowrite32be(XOR_CRSR_64BA_BIT, &adev->xor_reg->crrr); 4118 } else { 4119 size_t fifo_size = (adev->id == PPC440SPE_DMA0_ID) ? 4120 DMA0_FIFO_SIZE : DMA1_FIFO_SIZE; 4121 adev->dma_reg = regs; 4122 /* DMAx_FIFO_SIZE is defined in bytes, 4123 * <fsiz> - is defined in number of CDB pointers (8byte). 4124 * DMA FIFO Length = CSlength + CPlength, where 4125 * CSlength = CPlength = (fsiz + 1) * 8. 4126 */ 4127 iowrite32(DMA_FIFO_ENABLE | ((fifo_size >> 3) - 2), 4128 &adev->dma_reg->fsiz); 4129 /* Configure DMA engine */ 4130 iowrite32(DMA_CFG_DXEPR_HP | DMA_CFG_DFMPP_HP | DMA_CFG_FALGN, 4131 &adev->dma_reg->cfg); 4132 /* Clear Status */ 4133 iowrite32(~0, &adev->dma_reg->dsts); 4134 } 4135 4136 adev->dev = &ofdev->dev; 4137 adev->common.dev = &ofdev->dev; 4138 INIT_LIST_HEAD(&adev->common.channels); 4139 platform_set_drvdata(ofdev, adev); 4140 4141 /* create a channel */ 4142 chan = kzalloc(sizeof(*chan), GFP_KERNEL); 4143 if (!chan) { 4144 initcode = PPC_ADMA_INIT_CHANNEL; 4145 ret = -ENOMEM; 4146 goto err_chan_alloc; 4147 } 4148 4149 spin_lock_init(&chan->lock); 4150 INIT_LIST_HEAD(&chan->chain); 4151 INIT_LIST_HEAD(&chan->all_slots); 4152 chan->device = adev; 4153 chan->common.device = &adev->common; 4154 dma_cookie_init(&chan->common); 4155 list_add_tail(&chan->common.device_node, &adev->common.channels); 4156 tasklet_init(&chan->irq_tasklet, ppc440spe_adma_tasklet, 4157 (unsigned long)chan); 4158 4159 /* allocate and map helper pages for async validation or 4160 * async_mult/async_sum_product operations on DMA0/1. 4161 */ 4162 if (adev->id != PPC440SPE_XOR_ID) { 4163 chan->pdest_page = alloc_page(GFP_KERNEL); 4164 chan->qdest_page = alloc_page(GFP_KERNEL); 4165 if (!chan->pdest_page || 4166 !chan->qdest_page) { 4167 if (chan->pdest_page) 4168 __free_page(chan->pdest_page); 4169 if (chan->qdest_page) 4170 __free_page(chan->qdest_page); 4171 ret = -ENOMEM; 4172 goto err_page_alloc; 4173 } 4174 chan->pdest = dma_map_page(&ofdev->dev, chan->pdest_page, 0, 4175 PAGE_SIZE, DMA_BIDIRECTIONAL); 4176 chan->qdest = dma_map_page(&ofdev->dev, chan->qdest_page, 0, 4177 PAGE_SIZE, DMA_BIDIRECTIONAL); 4178 } 4179 4180 ref = kmalloc(sizeof(*ref), GFP_KERNEL); 4181 if (ref) { 4182 ref->chan = &chan->common; 4183 INIT_LIST_HEAD(&ref->node); 4184 list_add_tail(&ref->node, &ppc440spe_adma_chan_list); 4185 } else { 4186 dev_err(&ofdev->dev, "failed to allocate channel reference!\n"); 4187 ret = -ENOMEM; 4188 goto err_ref_alloc; 4189 } 4190 4191 ret = ppc440spe_adma_setup_irqs(adev, chan, &initcode); 4192 if (ret) 4193 goto err_irq; 4194 4195 ppc440spe_adma_init_capabilities(adev); 4196 4197 ret = dma_async_device_register(&adev->common); 4198 if (ret) { 4199 initcode = PPC_ADMA_INIT_REGISTER; 4200 dev_err(&ofdev->dev, "failed to register dma device\n"); 4201 goto err_dev_reg; 4202 } 4203 4204 goto out; 4205 4206 err_dev_reg: 4207 ppc440spe_adma_release_irqs(adev, chan); 4208 err_irq: 4209 list_for_each_entry_safe(ref, _ref, &ppc440spe_adma_chan_list, node) { 4210 if (chan == to_ppc440spe_adma_chan(ref->chan)) { 4211 list_del(&ref->node); 4212 kfree(ref); 4213 } 4214 } 4215 err_ref_alloc: 4216 if (adev->id != PPC440SPE_XOR_ID) { 4217 dma_unmap_page(&ofdev->dev, chan->pdest, 4218 PAGE_SIZE, DMA_BIDIRECTIONAL); 4219 dma_unmap_page(&ofdev->dev, chan->qdest, 4220 PAGE_SIZE, DMA_BIDIRECTIONAL); 4221 __free_page(chan->pdest_page); 4222 __free_page(chan->qdest_page); 4223 } 4224 err_page_alloc: 4225 kfree(chan); 4226 err_chan_alloc: 4227 if (adev->id == PPC440SPE_XOR_ID) 4228 iounmap(adev->xor_reg); 4229 else 4230 iounmap(adev->dma_reg); 4231 err_regs_alloc: 4232 dma_free_coherent(adev->dev, adev->pool_size, 4233 adev->dma_desc_pool_virt, 4234 adev->dma_desc_pool); 4235 err_dma_alloc: 4236 kfree(adev); 4237 err_adev_alloc: 4238 release_mem_region(res.start, resource_size(&res)); 4239 out: 4240 if (id < PPC440SPE_ADMA_ENGINES_NUM) 4241 ppc440spe_adma_devices[id] = initcode; 4242 4243 return ret; 4244 } 4245 4246 /** 4247 * ppc440spe_adma_remove - remove the asynch device 4248 */ 4249 static int ppc440spe_adma_remove(struct platform_device *ofdev) 4250 { 4251 struct ppc440spe_adma_device *adev = platform_get_drvdata(ofdev); 4252 struct device_node *np = ofdev->dev.of_node; 4253 struct resource res; 4254 struct dma_chan *chan, *_chan; 4255 struct ppc_dma_chan_ref *ref, *_ref; 4256 struct ppc440spe_adma_chan *ppc440spe_chan; 4257 4258 if (adev->id < PPC440SPE_ADMA_ENGINES_NUM) 4259 ppc440spe_adma_devices[adev->id] = -1; 4260 4261 dma_async_device_unregister(&adev->common); 4262 4263 list_for_each_entry_safe(chan, _chan, &adev->common.channels, 4264 device_node) { 4265 ppc440spe_chan = to_ppc440spe_adma_chan(chan); 4266 ppc440spe_adma_release_irqs(adev, ppc440spe_chan); 4267 tasklet_kill(&ppc440spe_chan->irq_tasklet); 4268 if (adev->id != PPC440SPE_XOR_ID) { 4269 dma_unmap_page(&ofdev->dev, ppc440spe_chan->pdest, 4270 PAGE_SIZE, DMA_BIDIRECTIONAL); 4271 dma_unmap_page(&ofdev->dev, ppc440spe_chan->qdest, 4272 PAGE_SIZE, DMA_BIDIRECTIONAL); 4273 __free_page(ppc440spe_chan->pdest_page); 4274 __free_page(ppc440spe_chan->qdest_page); 4275 } 4276 list_for_each_entry_safe(ref, _ref, &ppc440spe_adma_chan_list, 4277 node) { 4278 if (ppc440spe_chan == 4279 to_ppc440spe_adma_chan(ref->chan)) { 4280 list_del(&ref->node); 4281 kfree(ref); 4282 } 4283 } 4284 list_del(&chan->device_node); 4285 kfree(ppc440spe_chan); 4286 } 4287 4288 dma_free_coherent(adev->dev, adev->pool_size, 4289 adev->dma_desc_pool_virt, adev->dma_desc_pool); 4290 if (adev->id == PPC440SPE_XOR_ID) 4291 iounmap(adev->xor_reg); 4292 else 4293 iounmap(adev->dma_reg); 4294 of_address_to_resource(np, 0, &res); 4295 release_mem_region(res.start, resource_size(&res)); 4296 kfree(adev); 4297 return 0; 4298 } 4299 4300 /* 4301 * /sys driver interface to enable h/w RAID-6 capabilities 4302 * Files created in e.g. /sys/devices/plb.0/400100100.dma0/driver/ 4303 * directory are "devices", "enable" and "poly". 4304 * "devices" shows available engines. 4305 * "enable" is used to enable RAID-6 capabilities or to check 4306 * whether these has been activated. 4307 * "poly" allows setting/checking used polynomial (for PPC440SPe only). 4308 */ 4309 4310 static ssize_t devices_show(struct device_driver *dev, char *buf) 4311 { 4312 ssize_t size = 0; 4313 int i; 4314 4315 for (i = 0; i < PPC440SPE_ADMA_ENGINES_NUM; i++) { 4316 if (ppc440spe_adma_devices[i] == -1) 4317 continue; 4318 size += snprintf(buf + size, PAGE_SIZE - size, 4319 "PPC440SP(E)-ADMA.%d: %s\n", i, 4320 ppc_adma_errors[ppc440spe_adma_devices[i]]); 4321 } 4322 return size; 4323 } 4324 static DRIVER_ATTR_RO(devices); 4325 4326 static ssize_t enable_show(struct device_driver *dev, char *buf) 4327 { 4328 return snprintf(buf, PAGE_SIZE, 4329 "PPC440SP(e) RAID-6 capabilities are %sABLED.\n", 4330 ppc440spe_r6_enabled ? "EN" : "DIS"); 4331 } 4332 4333 static ssize_t enable_store(struct device_driver *dev, const char *buf, 4334 size_t count) 4335 { 4336 unsigned long val; 4337 4338 if (!count || count > 11) 4339 return -EINVAL; 4340 4341 if (!ppc440spe_r6_tchan) 4342 return -EFAULT; 4343 4344 /* Write a key */ 4345 sscanf(buf, "%lx", &val); 4346 dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_XORBA, val); 4347 isync(); 4348 4349 /* Verify whether it really works now */ 4350 if (ppc440spe_test_raid6(ppc440spe_r6_tchan) == 0) { 4351 pr_info("PPC440SP(e) RAID-6 has been activated " 4352 "successfully\n"); 4353 ppc440spe_r6_enabled = 1; 4354 } else { 4355 pr_info("PPC440SP(e) RAID-6 hasn't been activated!" 4356 " Error key ?\n"); 4357 ppc440spe_r6_enabled = 0; 4358 } 4359 return count; 4360 } 4361 static DRIVER_ATTR_RW(enable); 4362 4363 static ssize_t poly_store(struct device_driver *dev, char *buf) 4364 { 4365 ssize_t size = 0; 4366 u32 reg; 4367 4368 #ifdef CONFIG_440SP 4369 /* 440SP has fixed polynomial */ 4370 reg = 0x4d; 4371 #else 4372 reg = dcr_read(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL); 4373 reg >>= MQ0_CFBHL_POLY; 4374 reg &= 0xFF; 4375 #endif 4376 4377 size = snprintf(buf, PAGE_SIZE, "PPC440SP(e) RAID-6 driver " 4378 "uses 0x1%02x polynomial.\n", reg); 4379 return size; 4380 } 4381 4382 static ssize_t poly_store(struct device_driver *dev, const char *buf, 4383 size_t count) 4384 { 4385 unsigned long reg, val; 4386 4387 #ifdef CONFIG_440SP 4388 /* 440SP uses default 0x14D polynomial only */ 4389 return -EINVAL; 4390 #endif 4391 4392 if (!count || count > 6) 4393 return -EINVAL; 4394 4395 /* e.g., 0x14D or 0x11D */ 4396 sscanf(buf, "%lx", &val); 4397 4398 if (val & ~0x1FF) 4399 return -EINVAL; 4400 4401 val &= 0xFF; 4402 reg = dcr_read(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL); 4403 reg &= ~(0xFF << MQ0_CFBHL_POLY); 4404 reg |= val << MQ0_CFBHL_POLY; 4405 dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL, reg); 4406 4407 return count; 4408 } 4409 static DRIVER_ATTR_RW(poly); 4410 4411 /* 4412 * Common initialisation for RAID engines; allocate memory for 4413 * DMAx FIFOs, perform configuration common for all DMA engines. 4414 * Further DMA engine specific configuration is done at probe time. 4415 */ 4416 static int ppc440spe_configure_raid_devices(void) 4417 { 4418 struct device_node *np; 4419 struct resource i2o_res; 4420 struct i2o_regs __iomem *i2o_reg; 4421 dcr_host_t i2o_dcr_host; 4422 unsigned int dcr_base, dcr_len; 4423 int i, ret; 4424 4425 np = of_find_compatible_node(NULL, NULL, "ibm,i2o-440spe"); 4426 if (!np) { 4427 pr_err("%s: can't find I2O device tree node\n", 4428 __func__); 4429 return -ENODEV; 4430 } 4431 4432 if (of_address_to_resource(np, 0, &i2o_res)) { 4433 of_node_put(np); 4434 return -EINVAL; 4435 } 4436 4437 i2o_reg = of_iomap(np, 0); 4438 if (!i2o_reg) { 4439 pr_err("%s: failed to map I2O registers\n", __func__); 4440 of_node_put(np); 4441 return -EINVAL; 4442 } 4443 4444 /* Get I2O DCRs base */ 4445 dcr_base = dcr_resource_start(np, 0); 4446 dcr_len = dcr_resource_len(np, 0); 4447 if (!dcr_base && !dcr_len) { 4448 pr_err("%pOF: can't get DCR registers base/len!\n", np); 4449 of_node_put(np); 4450 iounmap(i2o_reg); 4451 return -ENODEV; 4452 } 4453 4454 i2o_dcr_host = dcr_map(np, dcr_base, dcr_len); 4455 if (!DCR_MAP_OK(i2o_dcr_host)) { 4456 pr_err("%pOF: failed to map DCRs!\n", np); 4457 of_node_put(np); 4458 iounmap(i2o_reg); 4459 return -ENODEV; 4460 } 4461 of_node_put(np); 4462 4463 /* Provide memory regions for DMA's FIFOs: I2O, DMA0 and DMA1 share 4464 * the base address of FIFO memory space. 4465 * Actually we need twice more physical memory than programmed in the 4466 * <fsiz> register (because there are two FIFOs for each DMA: CP and CS) 4467 */ 4468 ppc440spe_dma_fifo_buf = kmalloc((DMA0_FIFO_SIZE + DMA1_FIFO_SIZE) << 1, 4469 GFP_KERNEL); 4470 if (!ppc440spe_dma_fifo_buf) { 4471 pr_err("%s: DMA FIFO buffer allocation failed.\n", __func__); 4472 iounmap(i2o_reg); 4473 dcr_unmap(i2o_dcr_host, dcr_len); 4474 return -ENOMEM; 4475 } 4476 4477 /* 4478 * Configure h/w 4479 */ 4480 /* Reset I2O/DMA */ 4481 mtdcri(SDR0, DCRN_SDR0_SRST, DCRN_SDR0_SRST_I2ODMA); 4482 mtdcri(SDR0, DCRN_SDR0_SRST, 0); 4483 4484 /* Setup the base address of mmaped registers */ 4485 dcr_write(i2o_dcr_host, DCRN_I2O0_IBAH, (u32)(i2o_res.start >> 32)); 4486 dcr_write(i2o_dcr_host, DCRN_I2O0_IBAL, (u32)(i2o_res.start) | 4487 I2O_REG_ENABLE); 4488 dcr_unmap(i2o_dcr_host, dcr_len); 4489 4490 /* Setup FIFO memory space base address */ 4491 iowrite32(0, &i2o_reg->ifbah); 4492 iowrite32(((u32)__pa(ppc440spe_dma_fifo_buf)), &i2o_reg->ifbal); 4493 4494 /* set zero FIFO size for I2O, so the whole 4495 * ppc440spe_dma_fifo_buf is used by DMAs. 4496 * DMAx_FIFOs will be configured while probe. 4497 */ 4498 iowrite32(0, &i2o_reg->ifsiz); 4499 iounmap(i2o_reg); 4500 4501 /* To prepare WXOR/RXOR functionality we need access to 4502 * Memory Queue Module DCRs (finally it will be enabled 4503 * via /sys interface of the ppc440spe ADMA driver). 4504 */ 4505 np = of_find_compatible_node(NULL, NULL, "ibm,mq-440spe"); 4506 if (!np) { 4507 pr_err("%s: can't find MQ device tree node\n", 4508 __func__); 4509 ret = -ENODEV; 4510 goto out_free; 4511 } 4512 4513 /* Get MQ DCRs base */ 4514 dcr_base = dcr_resource_start(np, 0); 4515 dcr_len = dcr_resource_len(np, 0); 4516 if (!dcr_base && !dcr_len) { 4517 pr_err("%pOF: can't get DCR registers base/len!\n", np); 4518 ret = -ENODEV; 4519 goto out_mq; 4520 } 4521 4522 ppc440spe_mq_dcr_host = dcr_map(np, dcr_base, dcr_len); 4523 if (!DCR_MAP_OK(ppc440spe_mq_dcr_host)) { 4524 pr_err("%pOF: failed to map DCRs!\n", np); 4525 ret = -ENODEV; 4526 goto out_mq; 4527 } 4528 of_node_put(np); 4529 ppc440spe_mq_dcr_len = dcr_len; 4530 4531 /* Set HB alias */ 4532 dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_BAUH, DMA_CUED_XOR_HB); 4533 4534 /* Set: 4535 * - LL transaction passing limit to 1; 4536 * - Memory controller cycle limit to 1; 4537 * - Galois Polynomial to 0x14d (default) 4538 */ 4539 dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL, 4540 (1 << MQ0_CFBHL_TPLM) | (1 << MQ0_CFBHL_HBCL) | 4541 (PPC440SPE_DEFAULT_POLY << MQ0_CFBHL_POLY)); 4542 4543 atomic_set(&ppc440spe_adma_err_irq_ref, 0); 4544 for (i = 0; i < PPC440SPE_ADMA_ENGINES_NUM; i++) 4545 ppc440spe_adma_devices[i] = -1; 4546 4547 return 0; 4548 4549 out_mq: 4550 of_node_put(np); 4551 out_free: 4552 kfree(ppc440spe_dma_fifo_buf); 4553 return ret; 4554 } 4555 4556 static const struct of_device_id ppc440spe_adma_of_match[] = { 4557 { .compatible = "ibm,dma-440spe", }, 4558 { .compatible = "amcc,xor-accelerator", }, 4559 {}, 4560 }; 4561 MODULE_DEVICE_TABLE(of, ppc440spe_adma_of_match); 4562 4563 static struct platform_driver ppc440spe_adma_driver = { 4564 .probe = ppc440spe_adma_probe, 4565 .remove = ppc440spe_adma_remove, 4566 .driver = { 4567 .name = "PPC440SP(E)-ADMA", 4568 .of_match_table = ppc440spe_adma_of_match, 4569 }, 4570 }; 4571 4572 static __init int ppc440spe_adma_init(void) 4573 { 4574 int ret; 4575 4576 ret = ppc440spe_configure_raid_devices(); 4577 if (ret) 4578 return ret; 4579 4580 ret = platform_driver_register(&ppc440spe_adma_driver); 4581 if (ret) { 4582 pr_err("%s: failed to register platform driver\n", 4583 __func__); 4584 goto out_reg; 4585 } 4586 4587 /* Initialization status */ 4588 ret = driver_create_file(&ppc440spe_adma_driver.driver, 4589 &driver_attr_devices); 4590 if (ret) 4591 goto out_dev; 4592 4593 /* RAID-6 h/w enable entry */ 4594 ret = driver_create_file(&ppc440spe_adma_driver.driver, 4595 &driver_attr_enable); 4596 if (ret) 4597 goto out_en; 4598 4599 /* GF polynomial to use */ 4600 ret = driver_create_file(&ppc440spe_adma_driver.driver, 4601 &driver_attr_poly); 4602 if (!ret) 4603 return ret; 4604 4605 driver_remove_file(&ppc440spe_adma_driver.driver, 4606 &driver_attr_enable); 4607 out_en: 4608 driver_remove_file(&ppc440spe_adma_driver.driver, 4609 &driver_attr_devices); 4610 out_dev: 4611 /* User will not be able to enable h/w RAID-6 */ 4612 pr_err("%s: failed to create RAID-6 driver interface\n", 4613 __func__); 4614 platform_driver_unregister(&ppc440spe_adma_driver); 4615 out_reg: 4616 dcr_unmap(ppc440spe_mq_dcr_host, ppc440spe_mq_dcr_len); 4617 kfree(ppc440spe_dma_fifo_buf); 4618 return ret; 4619 } 4620 4621 static void __exit ppc440spe_adma_exit(void) 4622 { 4623 driver_remove_file(&ppc440spe_adma_driver.driver, 4624 &driver_attr_poly); 4625 driver_remove_file(&ppc440spe_adma_driver.driver, 4626 &driver_attr_enable); 4627 driver_remove_file(&ppc440spe_adma_driver.driver, 4628 &driver_attr_devices); 4629 platform_driver_unregister(&ppc440spe_adma_driver); 4630 dcr_unmap(ppc440spe_mq_dcr_host, ppc440spe_mq_dcr_len); 4631 kfree(ppc440spe_dma_fifo_buf); 4632 } 4633 4634 arch_initcall(ppc440spe_adma_init); 4635 module_exit(ppc440spe_adma_exit); 4636 4637 MODULE_AUTHOR("Yuri Tikhonov <yur@emcraft.com>"); 4638 MODULE_DESCRIPTION("PPC440SPE ADMA Engine Driver"); 4639 MODULE_LICENSE("GPL"); 4640