1 /* 2 * Copyright (C) 2006-2009 DENX Software Engineering. 3 * 4 * Author: Yuri Tikhonov <yur@emcraft.com> 5 * 6 * Further porting to arch/powerpc by 7 * Anatolij Gustschin <agust@denx.de> 8 * 9 * This program is free software; you can redistribute it and/or modify it 10 * under the terms of the GNU General Public License as published by the Free 11 * Software Foundation; either version 2 of the License, or (at your option) 12 * any later version. 13 * 14 * This program is distributed in the hope that it will be useful, but WITHOUT 15 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 16 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 17 * more details. 18 * 19 * You should have received a copy of the GNU General Public License along with 20 * this program; if not, write to the Free Software Foundation, Inc., 59 21 * Temple Place - Suite 330, Boston, MA 02111-1307, USA. 22 * 23 * The full GNU General Public License is included in this distribution in the 24 * file called COPYING. 25 */ 26 27 /* 28 * This driver supports the asynchrounous DMA copy and RAID engines available 29 * on the AMCC PPC440SPe Processors. 30 * Based on the Intel Xscale(R) family of I/O Processors (IOP 32x, 33x, 134x) 31 * ADMA driver written by D.Williams. 32 */ 33 34 #include <linux/init.h> 35 #include <linux/module.h> 36 #include <linux/async_tx.h> 37 #include <linux/delay.h> 38 #include <linux/dma-mapping.h> 39 #include <linux/spinlock.h> 40 #include <linux/interrupt.h> 41 #include <linux/slab.h> 42 #include <linux/uaccess.h> 43 #include <linux/proc_fs.h> 44 #include <linux/of.h> 45 #include <linux/of_platform.h> 46 #include <asm/dcr.h> 47 #include <asm/dcr-regs.h> 48 #include "adma.h" 49 #include "../dmaengine.h" 50 51 enum ppc_adma_init_code { 52 PPC_ADMA_INIT_OK = 0, 53 PPC_ADMA_INIT_MEMRES, 54 PPC_ADMA_INIT_MEMREG, 55 PPC_ADMA_INIT_ALLOC, 56 PPC_ADMA_INIT_COHERENT, 57 PPC_ADMA_INIT_CHANNEL, 58 PPC_ADMA_INIT_IRQ1, 59 PPC_ADMA_INIT_IRQ2, 60 PPC_ADMA_INIT_REGISTER 61 }; 62 63 static char *ppc_adma_errors[] = { 64 [PPC_ADMA_INIT_OK] = "ok", 65 [PPC_ADMA_INIT_MEMRES] = "failed to get memory resource", 66 [PPC_ADMA_INIT_MEMREG] = "failed to request memory region", 67 [PPC_ADMA_INIT_ALLOC] = "failed to allocate memory for adev " 68 "structure", 69 [PPC_ADMA_INIT_COHERENT] = "failed to allocate coherent memory for " 70 "hardware descriptors", 71 [PPC_ADMA_INIT_CHANNEL] = "failed to allocate memory for channel", 72 [PPC_ADMA_INIT_IRQ1] = "failed to request first irq", 73 [PPC_ADMA_INIT_IRQ2] = "failed to request second irq", 74 [PPC_ADMA_INIT_REGISTER] = "failed to register dma async device", 75 }; 76 77 static enum ppc_adma_init_code 78 ppc440spe_adma_devices[PPC440SPE_ADMA_ENGINES_NUM]; 79 80 struct ppc_dma_chan_ref { 81 struct dma_chan *chan; 82 struct list_head node; 83 }; 84 85 /* The list of channels exported by ppc440spe ADMA */ 86 struct list_head 87 ppc440spe_adma_chan_list = LIST_HEAD_INIT(ppc440spe_adma_chan_list); 88 89 /* This flag is set when want to refetch the xor chain in the interrupt 90 * handler 91 */ 92 static u32 do_xor_refetch; 93 94 /* Pointer to DMA0, DMA1 CP/CS FIFO */ 95 static void *ppc440spe_dma_fifo_buf; 96 97 /* Pointers to last submitted to DMA0, DMA1 CDBs */ 98 static struct ppc440spe_adma_desc_slot *chan_last_sub[3]; 99 static struct ppc440spe_adma_desc_slot *chan_first_cdb[3]; 100 101 /* Pointer to last linked and submitted xor CB */ 102 static struct ppc440spe_adma_desc_slot *xor_last_linked; 103 static struct ppc440spe_adma_desc_slot *xor_last_submit; 104 105 /* This array is used in data-check operations for storing a pattern */ 106 static char ppc440spe_qword[16]; 107 108 static atomic_t ppc440spe_adma_err_irq_ref; 109 static dcr_host_t ppc440spe_mq_dcr_host; 110 static unsigned int ppc440spe_mq_dcr_len; 111 112 /* Since RXOR operations use the common register (MQ0_CF2H) for setting-up 113 * the block size in transactions, then we do not allow to activate more than 114 * only one RXOR transactions simultaneously. So use this var to store 115 * the information about is RXOR currently active (PPC440SPE_RXOR_RUN bit is 116 * set) or not (PPC440SPE_RXOR_RUN is clear). 117 */ 118 static unsigned long ppc440spe_rxor_state; 119 120 /* These are used in enable & check routines 121 */ 122 static u32 ppc440spe_r6_enabled; 123 static struct ppc440spe_adma_chan *ppc440spe_r6_tchan; 124 static struct completion ppc440spe_r6_test_comp; 125 126 static int ppc440spe_adma_dma2rxor_prep_src( 127 struct ppc440spe_adma_desc_slot *desc, 128 struct ppc440spe_rxor *cursor, int index, 129 int src_cnt, u32 addr); 130 static void ppc440spe_adma_dma2rxor_set_src( 131 struct ppc440spe_adma_desc_slot *desc, 132 int index, dma_addr_t addr); 133 static void ppc440spe_adma_dma2rxor_set_mult( 134 struct ppc440spe_adma_desc_slot *desc, 135 int index, u8 mult); 136 137 #ifdef ADMA_LL_DEBUG 138 #define ADMA_LL_DBG(x) ({ if (1) x; 0; }) 139 #else 140 #define ADMA_LL_DBG(x) ({ if (0) x; 0; }) 141 #endif 142 143 static void print_cb(struct ppc440spe_adma_chan *chan, void *block) 144 { 145 struct dma_cdb *cdb; 146 struct xor_cb *cb; 147 int i; 148 149 switch (chan->device->id) { 150 case 0: 151 case 1: 152 cdb = block; 153 154 pr_debug("CDB at %p [%d]:\n" 155 "\t attr 0x%02x opc 0x%02x cnt 0x%08x\n" 156 "\t sg1u 0x%08x sg1l 0x%08x\n" 157 "\t sg2u 0x%08x sg2l 0x%08x\n" 158 "\t sg3u 0x%08x sg3l 0x%08x\n", 159 cdb, chan->device->id, 160 cdb->attr, cdb->opc, le32_to_cpu(cdb->cnt), 161 le32_to_cpu(cdb->sg1u), le32_to_cpu(cdb->sg1l), 162 le32_to_cpu(cdb->sg2u), le32_to_cpu(cdb->sg2l), 163 le32_to_cpu(cdb->sg3u), le32_to_cpu(cdb->sg3l) 164 ); 165 break; 166 case 2: 167 cb = block; 168 169 pr_debug("CB at %p [%d]:\n" 170 "\t cbc 0x%08x cbbc 0x%08x cbs 0x%08x\n" 171 "\t cbtah 0x%08x cbtal 0x%08x\n" 172 "\t cblah 0x%08x cblal 0x%08x\n", 173 cb, chan->device->id, 174 cb->cbc, cb->cbbc, cb->cbs, 175 cb->cbtah, cb->cbtal, 176 cb->cblah, cb->cblal); 177 for (i = 0; i < 16; i++) { 178 if (i && !cb->ops[i].h && !cb->ops[i].l) 179 continue; 180 pr_debug("\t ops[%2d]: h 0x%08x l 0x%08x\n", 181 i, cb->ops[i].h, cb->ops[i].l); 182 } 183 break; 184 } 185 } 186 187 static void print_cb_list(struct ppc440spe_adma_chan *chan, 188 struct ppc440spe_adma_desc_slot *iter) 189 { 190 for (; iter; iter = iter->hw_next) 191 print_cb(chan, iter->hw_desc); 192 } 193 194 static void prep_dma_xor_dbg(int id, dma_addr_t dst, dma_addr_t *src, 195 unsigned int src_cnt) 196 { 197 int i; 198 199 pr_debug("\n%s(%d):\nsrc: ", __func__, id); 200 for (i = 0; i < src_cnt; i++) 201 pr_debug("\t0x%016llx ", src[i]); 202 pr_debug("dst:\n\t0x%016llx\n", dst); 203 } 204 205 static void prep_dma_pq_dbg(int id, dma_addr_t *dst, dma_addr_t *src, 206 unsigned int src_cnt) 207 { 208 int i; 209 210 pr_debug("\n%s(%d):\nsrc: ", __func__, id); 211 for (i = 0; i < src_cnt; i++) 212 pr_debug("\t0x%016llx ", src[i]); 213 pr_debug("dst: "); 214 for (i = 0; i < 2; i++) 215 pr_debug("\t0x%016llx ", dst[i]); 216 } 217 218 static void prep_dma_pqzero_sum_dbg(int id, dma_addr_t *src, 219 unsigned int src_cnt, 220 const unsigned char *scf) 221 { 222 int i; 223 224 pr_debug("\n%s(%d):\nsrc(coef): ", __func__, id); 225 if (scf) { 226 for (i = 0; i < src_cnt; i++) 227 pr_debug("\t0x%016llx(0x%02x) ", src[i], scf[i]); 228 } else { 229 for (i = 0; i < src_cnt; i++) 230 pr_debug("\t0x%016llx(no) ", src[i]); 231 } 232 233 pr_debug("dst: "); 234 for (i = 0; i < 2; i++) 235 pr_debug("\t0x%016llx ", src[src_cnt + i]); 236 } 237 238 /****************************************************************************** 239 * Command (Descriptor) Blocks low-level routines 240 ******************************************************************************/ 241 /** 242 * ppc440spe_desc_init_interrupt - initialize the descriptor for INTERRUPT 243 * pseudo operation 244 */ 245 static void ppc440spe_desc_init_interrupt(struct ppc440spe_adma_desc_slot *desc, 246 struct ppc440spe_adma_chan *chan) 247 { 248 struct xor_cb *p; 249 250 switch (chan->device->id) { 251 case PPC440SPE_XOR_ID: 252 p = desc->hw_desc; 253 memset(desc->hw_desc, 0, sizeof(struct xor_cb)); 254 /* NOP with Command Block Complete Enable */ 255 p->cbc = XOR_CBCR_CBCE_BIT; 256 break; 257 case PPC440SPE_DMA0_ID: 258 case PPC440SPE_DMA1_ID: 259 memset(desc->hw_desc, 0, sizeof(struct dma_cdb)); 260 /* NOP with interrupt */ 261 set_bit(PPC440SPE_DESC_INT, &desc->flags); 262 break; 263 default: 264 printk(KERN_ERR "Unsupported id %d in %s\n", chan->device->id, 265 __func__); 266 break; 267 } 268 } 269 270 /** 271 * ppc440spe_desc_init_null_xor - initialize the descriptor for NULL XOR 272 * pseudo operation 273 */ 274 static void ppc440spe_desc_init_null_xor(struct ppc440spe_adma_desc_slot *desc) 275 { 276 memset(desc->hw_desc, 0, sizeof(struct xor_cb)); 277 desc->hw_next = NULL; 278 desc->src_cnt = 0; 279 desc->dst_cnt = 1; 280 } 281 282 /** 283 * ppc440spe_desc_init_xor - initialize the descriptor for XOR operation 284 */ 285 static void ppc440spe_desc_init_xor(struct ppc440spe_adma_desc_slot *desc, 286 int src_cnt, unsigned long flags) 287 { 288 struct xor_cb *hw_desc = desc->hw_desc; 289 290 memset(desc->hw_desc, 0, sizeof(struct xor_cb)); 291 desc->hw_next = NULL; 292 desc->src_cnt = src_cnt; 293 desc->dst_cnt = 1; 294 295 hw_desc->cbc = XOR_CBCR_TGT_BIT | src_cnt; 296 if (flags & DMA_PREP_INTERRUPT) 297 /* Enable interrupt on completion */ 298 hw_desc->cbc |= XOR_CBCR_CBCE_BIT; 299 } 300 301 /** 302 * ppc440spe_desc_init_dma2pq - initialize the descriptor for PQ 303 * operation in DMA2 controller 304 */ 305 static void ppc440spe_desc_init_dma2pq(struct ppc440spe_adma_desc_slot *desc, 306 int dst_cnt, int src_cnt, unsigned long flags) 307 { 308 struct xor_cb *hw_desc = desc->hw_desc; 309 310 memset(desc->hw_desc, 0, sizeof(struct xor_cb)); 311 desc->hw_next = NULL; 312 desc->src_cnt = src_cnt; 313 desc->dst_cnt = dst_cnt; 314 memset(desc->reverse_flags, 0, sizeof(desc->reverse_flags)); 315 desc->descs_per_op = 0; 316 317 hw_desc->cbc = XOR_CBCR_TGT_BIT; 318 if (flags & DMA_PREP_INTERRUPT) 319 /* Enable interrupt on completion */ 320 hw_desc->cbc |= XOR_CBCR_CBCE_BIT; 321 } 322 323 #define DMA_CTRL_FLAGS_LAST DMA_PREP_FENCE 324 #define DMA_PREP_ZERO_P (DMA_CTRL_FLAGS_LAST << 1) 325 #define DMA_PREP_ZERO_Q (DMA_PREP_ZERO_P << 1) 326 327 /** 328 * ppc440spe_desc_init_dma01pq - initialize the descriptors for PQ operation 329 * with DMA0/1 330 */ 331 static void ppc440spe_desc_init_dma01pq(struct ppc440spe_adma_desc_slot *desc, 332 int dst_cnt, int src_cnt, unsigned long flags, 333 unsigned long op) 334 { 335 struct dma_cdb *hw_desc; 336 struct ppc440spe_adma_desc_slot *iter; 337 u8 dopc; 338 339 /* Common initialization of a PQ descriptors chain */ 340 set_bits(op, &desc->flags); 341 desc->src_cnt = src_cnt; 342 desc->dst_cnt = dst_cnt; 343 344 /* WXOR MULTICAST if both P and Q are being computed 345 * MV_SG1_SG2 if Q only 346 */ 347 dopc = (desc->dst_cnt == DMA_DEST_MAX_NUM) ? 348 DMA_CDB_OPC_MULTICAST : DMA_CDB_OPC_MV_SG1_SG2; 349 350 list_for_each_entry(iter, &desc->group_list, chain_node) { 351 hw_desc = iter->hw_desc; 352 memset(iter->hw_desc, 0, sizeof(struct dma_cdb)); 353 354 if (likely(!list_is_last(&iter->chain_node, 355 &desc->group_list))) { 356 /* set 'next' pointer */ 357 iter->hw_next = list_entry(iter->chain_node.next, 358 struct ppc440spe_adma_desc_slot, chain_node); 359 clear_bit(PPC440SPE_DESC_INT, &iter->flags); 360 } else { 361 /* this is the last descriptor. 362 * this slot will be pasted from ADMA level 363 * each time it wants to configure parameters 364 * of the transaction (src, dst, ...) 365 */ 366 iter->hw_next = NULL; 367 if (flags & DMA_PREP_INTERRUPT) 368 set_bit(PPC440SPE_DESC_INT, &iter->flags); 369 else 370 clear_bit(PPC440SPE_DESC_INT, &iter->flags); 371 } 372 } 373 374 /* Set OPS depending on WXOR/RXOR type of operation */ 375 if (!test_bit(PPC440SPE_DESC_RXOR, &desc->flags)) { 376 /* This is a WXOR only chain: 377 * - first descriptors are for zeroing destinations 378 * if PPC440SPE_ZERO_P/Q set; 379 * - descriptors remained are for GF-XOR operations. 380 */ 381 iter = list_first_entry(&desc->group_list, 382 struct ppc440spe_adma_desc_slot, 383 chain_node); 384 385 if (test_bit(PPC440SPE_ZERO_P, &desc->flags)) { 386 hw_desc = iter->hw_desc; 387 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; 388 iter = list_first_entry(&iter->chain_node, 389 struct ppc440spe_adma_desc_slot, 390 chain_node); 391 } 392 393 if (test_bit(PPC440SPE_ZERO_Q, &desc->flags)) { 394 hw_desc = iter->hw_desc; 395 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; 396 iter = list_first_entry(&iter->chain_node, 397 struct ppc440spe_adma_desc_slot, 398 chain_node); 399 } 400 401 list_for_each_entry_from(iter, &desc->group_list, chain_node) { 402 hw_desc = iter->hw_desc; 403 hw_desc->opc = dopc; 404 } 405 } else { 406 /* This is either RXOR-only or mixed RXOR/WXOR */ 407 408 /* The first 1 or 2 slots in chain are always RXOR, 409 * if need to calculate P & Q, then there are two 410 * RXOR slots; if only P or only Q, then there is one 411 */ 412 iter = list_first_entry(&desc->group_list, 413 struct ppc440spe_adma_desc_slot, 414 chain_node); 415 hw_desc = iter->hw_desc; 416 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; 417 418 if (desc->dst_cnt == DMA_DEST_MAX_NUM) { 419 iter = list_first_entry(&iter->chain_node, 420 struct ppc440spe_adma_desc_slot, 421 chain_node); 422 hw_desc = iter->hw_desc; 423 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; 424 } 425 426 /* The remaining descs (if any) are WXORs */ 427 if (test_bit(PPC440SPE_DESC_WXOR, &desc->flags)) { 428 iter = list_first_entry(&iter->chain_node, 429 struct ppc440spe_adma_desc_slot, 430 chain_node); 431 list_for_each_entry_from(iter, &desc->group_list, 432 chain_node) { 433 hw_desc = iter->hw_desc; 434 hw_desc->opc = dopc; 435 } 436 } 437 } 438 } 439 440 /** 441 * ppc440spe_desc_init_dma01pqzero_sum - initialize the descriptor 442 * for PQ_ZERO_SUM operation 443 */ 444 static void ppc440spe_desc_init_dma01pqzero_sum( 445 struct ppc440spe_adma_desc_slot *desc, 446 int dst_cnt, int src_cnt) 447 { 448 struct dma_cdb *hw_desc; 449 struct ppc440spe_adma_desc_slot *iter; 450 int i = 0; 451 u8 dopc = (dst_cnt == 2) ? DMA_CDB_OPC_MULTICAST : 452 DMA_CDB_OPC_MV_SG1_SG2; 453 /* 454 * Initialize starting from 2nd or 3rd descriptor dependent 455 * on dst_cnt. First one or two slots are for cloning P 456 * and/or Q to chan->pdest and/or chan->qdest as we have 457 * to preserve original P/Q. 458 */ 459 iter = list_first_entry(&desc->group_list, 460 struct ppc440spe_adma_desc_slot, chain_node); 461 iter = list_entry(iter->chain_node.next, 462 struct ppc440spe_adma_desc_slot, chain_node); 463 464 if (dst_cnt > 1) { 465 iter = list_entry(iter->chain_node.next, 466 struct ppc440spe_adma_desc_slot, chain_node); 467 } 468 /* initialize each source descriptor in chain */ 469 list_for_each_entry_from(iter, &desc->group_list, chain_node) { 470 hw_desc = iter->hw_desc; 471 memset(iter->hw_desc, 0, sizeof(struct dma_cdb)); 472 iter->src_cnt = 0; 473 iter->dst_cnt = 0; 474 475 /* This is a ZERO_SUM operation: 476 * - <src_cnt> descriptors starting from 2nd or 3rd 477 * descriptor are for GF-XOR operations; 478 * - remaining <dst_cnt> descriptors are for checking the result 479 */ 480 if (i++ < src_cnt) 481 /* MV_SG1_SG2 if only Q is being verified 482 * MULTICAST if both P and Q are being verified 483 */ 484 hw_desc->opc = dopc; 485 else 486 /* DMA_CDB_OPC_DCHECK128 operation */ 487 hw_desc->opc = DMA_CDB_OPC_DCHECK128; 488 489 if (likely(!list_is_last(&iter->chain_node, 490 &desc->group_list))) { 491 /* set 'next' pointer */ 492 iter->hw_next = list_entry(iter->chain_node.next, 493 struct ppc440spe_adma_desc_slot, 494 chain_node); 495 } else { 496 /* this is the last descriptor. 497 * this slot will be pasted from ADMA level 498 * each time it wants to configure parameters 499 * of the transaction (src, dst, ...) 500 */ 501 iter->hw_next = NULL; 502 /* always enable interrupt generation since we get 503 * the status of pqzero from the handler 504 */ 505 set_bit(PPC440SPE_DESC_INT, &iter->flags); 506 } 507 } 508 desc->src_cnt = src_cnt; 509 desc->dst_cnt = dst_cnt; 510 } 511 512 /** 513 * ppc440spe_desc_init_memcpy - initialize the descriptor for MEMCPY operation 514 */ 515 static void ppc440spe_desc_init_memcpy(struct ppc440spe_adma_desc_slot *desc, 516 unsigned long flags) 517 { 518 struct dma_cdb *hw_desc = desc->hw_desc; 519 520 memset(desc->hw_desc, 0, sizeof(struct dma_cdb)); 521 desc->hw_next = NULL; 522 desc->src_cnt = 1; 523 desc->dst_cnt = 1; 524 525 if (flags & DMA_PREP_INTERRUPT) 526 set_bit(PPC440SPE_DESC_INT, &desc->flags); 527 else 528 clear_bit(PPC440SPE_DESC_INT, &desc->flags); 529 530 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; 531 } 532 533 /** 534 * ppc440spe_desc_init_memset - initialize the descriptor for MEMSET operation 535 */ 536 static void ppc440spe_desc_init_memset(struct ppc440spe_adma_desc_slot *desc, 537 int value, unsigned long flags) 538 { 539 struct dma_cdb *hw_desc = desc->hw_desc; 540 541 memset(desc->hw_desc, 0, sizeof(struct dma_cdb)); 542 desc->hw_next = NULL; 543 desc->src_cnt = 1; 544 desc->dst_cnt = 1; 545 546 if (flags & DMA_PREP_INTERRUPT) 547 set_bit(PPC440SPE_DESC_INT, &desc->flags); 548 else 549 clear_bit(PPC440SPE_DESC_INT, &desc->flags); 550 551 hw_desc->sg1u = hw_desc->sg1l = cpu_to_le32((u32)value); 552 hw_desc->sg3u = hw_desc->sg3l = cpu_to_le32((u32)value); 553 hw_desc->opc = DMA_CDB_OPC_DFILL128; 554 } 555 556 /** 557 * ppc440spe_desc_set_src_addr - set source address into the descriptor 558 */ 559 static void ppc440spe_desc_set_src_addr(struct ppc440spe_adma_desc_slot *desc, 560 struct ppc440spe_adma_chan *chan, 561 int src_idx, dma_addr_t addrh, 562 dma_addr_t addrl) 563 { 564 struct dma_cdb *dma_hw_desc; 565 struct xor_cb *xor_hw_desc; 566 phys_addr_t addr64, tmplow, tmphi; 567 568 switch (chan->device->id) { 569 case PPC440SPE_DMA0_ID: 570 case PPC440SPE_DMA1_ID: 571 if (!addrh) { 572 addr64 = addrl; 573 tmphi = (addr64 >> 32); 574 tmplow = (addr64 & 0xFFFFFFFF); 575 } else { 576 tmphi = addrh; 577 tmplow = addrl; 578 } 579 dma_hw_desc = desc->hw_desc; 580 dma_hw_desc->sg1l = cpu_to_le32((u32)tmplow); 581 dma_hw_desc->sg1u |= cpu_to_le32((u32)tmphi); 582 break; 583 case PPC440SPE_XOR_ID: 584 xor_hw_desc = desc->hw_desc; 585 xor_hw_desc->ops[src_idx].l = addrl; 586 xor_hw_desc->ops[src_idx].h |= addrh; 587 break; 588 } 589 } 590 591 /** 592 * ppc440spe_desc_set_src_mult - set source address mult into the descriptor 593 */ 594 static void ppc440spe_desc_set_src_mult(struct ppc440spe_adma_desc_slot *desc, 595 struct ppc440spe_adma_chan *chan, u32 mult_index, 596 int sg_index, unsigned char mult_value) 597 { 598 struct dma_cdb *dma_hw_desc; 599 struct xor_cb *xor_hw_desc; 600 u32 *psgu; 601 602 switch (chan->device->id) { 603 case PPC440SPE_DMA0_ID: 604 case PPC440SPE_DMA1_ID: 605 dma_hw_desc = desc->hw_desc; 606 607 switch (sg_index) { 608 /* for RXOR operations set multiplier 609 * into source cued address 610 */ 611 case DMA_CDB_SG_SRC: 612 psgu = &dma_hw_desc->sg1u; 613 break; 614 /* for WXOR operations set multiplier 615 * into destination cued address(es) 616 */ 617 case DMA_CDB_SG_DST1: 618 psgu = &dma_hw_desc->sg2u; 619 break; 620 case DMA_CDB_SG_DST2: 621 psgu = &dma_hw_desc->sg3u; 622 break; 623 default: 624 BUG(); 625 } 626 627 *psgu |= cpu_to_le32(mult_value << mult_index); 628 break; 629 case PPC440SPE_XOR_ID: 630 xor_hw_desc = desc->hw_desc; 631 break; 632 default: 633 BUG(); 634 } 635 } 636 637 /** 638 * ppc440spe_desc_set_dest_addr - set destination address into the descriptor 639 */ 640 static void ppc440spe_desc_set_dest_addr(struct ppc440spe_adma_desc_slot *desc, 641 struct ppc440spe_adma_chan *chan, 642 dma_addr_t addrh, dma_addr_t addrl, 643 u32 dst_idx) 644 { 645 struct dma_cdb *dma_hw_desc; 646 struct xor_cb *xor_hw_desc; 647 phys_addr_t addr64, tmphi, tmplow; 648 u32 *psgu, *psgl; 649 650 switch (chan->device->id) { 651 case PPC440SPE_DMA0_ID: 652 case PPC440SPE_DMA1_ID: 653 if (!addrh) { 654 addr64 = addrl; 655 tmphi = (addr64 >> 32); 656 tmplow = (addr64 & 0xFFFFFFFF); 657 } else { 658 tmphi = addrh; 659 tmplow = addrl; 660 } 661 dma_hw_desc = desc->hw_desc; 662 663 psgu = dst_idx ? &dma_hw_desc->sg3u : &dma_hw_desc->sg2u; 664 psgl = dst_idx ? &dma_hw_desc->sg3l : &dma_hw_desc->sg2l; 665 666 *psgl = cpu_to_le32((u32)tmplow); 667 *psgu |= cpu_to_le32((u32)tmphi); 668 break; 669 case PPC440SPE_XOR_ID: 670 xor_hw_desc = desc->hw_desc; 671 xor_hw_desc->cbtal = addrl; 672 xor_hw_desc->cbtah |= addrh; 673 break; 674 } 675 } 676 677 /** 678 * ppc440spe_desc_set_byte_count - set number of data bytes involved 679 * into the operation 680 */ 681 static void ppc440spe_desc_set_byte_count(struct ppc440spe_adma_desc_slot *desc, 682 struct ppc440spe_adma_chan *chan, 683 u32 byte_count) 684 { 685 struct dma_cdb *dma_hw_desc; 686 struct xor_cb *xor_hw_desc; 687 688 switch (chan->device->id) { 689 case PPC440SPE_DMA0_ID: 690 case PPC440SPE_DMA1_ID: 691 dma_hw_desc = desc->hw_desc; 692 dma_hw_desc->cnt = cpu_to_le32(byte_count); 693 break; 694 case PPC440SPE_XOR_ID: 695 xor_hw_desc = desc->hw_desc; 696 xor_hw_desc->cbbc = byte_count; 697 break; 698 } 699 } 700 701 /** 702 * ppc440spe_desc_set_rxor_block_size - set RXOR block size 703 */ 704 static inline void ppc440spe_desc_set_rxor_block_size(u32 byte_count) 705 { 706 /* assume that byte_count is aligned on the 512-boundary; 707 * thus write it directly to the register (bits 23:31 are 708 * reserved there). 709 */ 710 dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_CF2H, byte_count); 711 } 712 713 /** 714 * ppc440spe_desc_set_dcheck - set CHECK pattern 715 */ 716 static void ppc440spe_desc_set_dcheck(struct ppc440spe_adma_desc_slot *desc, 717 struct ppc440spe_adma_chan *chan, u8 *qword) 718 { 719 struct dma_cdb *dma_hw_desc; 720 721 switch (chan->device->id) { 722 case PPC440SPE_DMA0_ID: 723 case PPC440SPE_DMA1_ID: 724 dma_hw_desc = desc->hw_desc; 725 iowrite32(qword[0], &dma_hw_desc->sg3l); 726 iowrite32(qword[4], &dma_hw_desc->sg3u); 727 iowrite32(qword[8], &dma_hw_desc->sg2l); 728 iowrite32(qword[12], &dma_hw_desc->sg2u); 729 break; 730 default: 731 BUG(); 732 } 733 } 734 735 /** 736 * ppc440spe_xor_set_link - set link address in xor CB 737 */ 738 static void ppc440spe_xor_set_link(struct ppc440spe_adma_desc_slot *prev_desc, 739 struct ppc440spe_adma_desc_slot *next_desc) 740 { 741 struct xor_cb *xor_hw_desc = prev_desc->hw_desc; 742 743 if (unlikely(!next_desc || !(next_desc->phys))) { 744 printk(KERN_ERR "%s: next_desc=0x%p; next_desc->phys=0x%llx\n", 745 __func__, next_desc, 746 next_desc ? next_desc->phys : 0); 747 BUG(); 748 } 749 750 xor_hw_desc->cbs = 0; 751 xor_hw_desc->cblal = next_desc->phys; 752 xor_hw_desc->cblah = 0; 753 xor_hw_desc->cbc |= XOR_CBCR_LNK_BIT; 754 } 755 756 /** 757 * ppc440spe_desc_set_link - set the address of descriptor following this 758 * descriptor in chain 759 */ 760 static void ppc440spe_desc_set_link(struct ppc440spe_adma_chan *chan, 761 struct ppc440spe_adma_desc_slot *prev_desc, 762 struct ppc440spe_adma_desc_slot *next_desc) 763 { 764 unsigned long flags; 765 struct ppc440spe_adma_desc_slot *tail = next_desc; 766 767 if (unlikely(!prev_desc || !next_desc || 768 (prev_desc->hw_next && prev_desc->hw_next != next_desc))) { 769 /* If previous next is overwritten something is wrong. 770 * though we may refetch from append to initiate list 771 * processing; in this case - it's ok. 772 */ 773 printk(KERN_ERR "%s: prev_desc=0x%p; next_desc=0x%p; " 774 "prev->hw_next=0x%p\n", __func__, prev_desc, 775 next_desc, prev_desc ? prev_desc->hw_next : 0); 776 BUG(); 777 } 778 779 local_irq_save(flags); 780 781 /* do s/w chaining both for DMA and XOR descriptors */ 782 prev_desc->hw_next = next_desc; 783 784 switch (chan->device->id) { 785 case PPC440SPE_DMA0_ID: 786 case PPC440SPE_DMA1_ID: 787 break; 788 case PPC440SPE_XOR_ID: 789 /* bind descriptor to the chain */ 790 while (tail->hw_next) 791 tail = tail->hw_next; 792 xor_last_linked = tail; 793 794 if (prev_desc == xor_last_submit) 795 /* do not link to the last submitted CB */ 796 break; 797 ppc440spe_xor_set_link(prev_desc, next_desc); 798 break; 799 } 800 801 local_irq_restore(flags); 802 } 803 804 /** 805 * ppc440spe_desc_get_src_addr - extract the source address from the descriptor 806 */ 807 static u32 ppc440spe_desc_get_src_addr(struct ppc440spe_adma_desc_slot *desc, 808 struct ppc440spe_adma_chan *chan, int src_idx) 809 { 810 struct dma_cdb *dma_hw_desc; 811 struct xor_cb *xor_hw_desc; 812 813 switch (chan->device->id) { 814 case PPC440SPE_DMA0_ID: 815 case PPC440SPE_DMA1_ID: 816 dma_hw_desc = desc->hw_desc; 817 /* May have 0, 1, 2, or 3 sources */ 818 switch (dma_hw_desc->opc) { 819 case DMA_CDB_OPC_NO_OP: 820 case DMA_CDB_OPC_DFILL128: 821 return 0; 822 case DMA_CDB_OPC_DCHECK128: 823 if (unlikely(src_idx)) { 824 printk(KERN_ERR "%s: try to get %d source for" 825 " DCHECK128\n", __func__, src_idx); 826 BUG(); 827 } 828 return le32_to_cpu(dma_hw_desc->sg1l); 829 case DMA_CDB_OPC_MULTICAST: 830 case DMA_CDB_OPC_MV_SG1_SG2: 831 if (unlikely(src_idx > 2)) { 832 printk(KERN_ERR "%s: try to get %d source from" 833 " DMA descr\n", __func__, src_idx); 834 BUG(); 835 } 836 if (src_idx) { 837 if (le32_to_cpu(dma_hw_desc->sg1u) & 838 DMA_CUED_XOR_WIN_MSK) { 839 u8 region; 840 841 if (src_idx == 1) 842 return le32_to_cpu( 843 dma_hw_desc->sg1l) + 844 desc->unmap_len; 845 846 region = (le32_to_cpu( 847 dma_hw_desc->sg1u)) >> 848 DMA_CUED_REGION_OFF; 849 850 region &= DMA_CUED_REGION_MSK; 851 switch (region) { 852 case DMA_RXOR123: 853 return le32_to_cpu( 854 dma_hw_desc->sg1l) + 855 (desc->unmap_len << 1); 856 case DMA_RXOR124: 857 return le32_to_cpu( 858 dma_hw_desc->sg1l) + 859 (desc->unmap_len * 3); 860 case DMA_RXOR125: 861 return le32_to_cpu( 862 dma_hw_desc->sg1l) + 863 (desc->unmap_len << 2); 864 default: 865 printk(KERN_ERR 866 "%s: try to" 867 " get src3 for region %02x" 868 "PPC440SPE_DESC_RXOR12?\n", 869 __func__, region); 870 BUG(); 871 } 872 } else { 873 printk(KERN_ERR 874 "%s: try to get %d" 875 " source for non-cued descr\n", 876 __func__, src_idx); 877 BUG(); 878 } 879 } 880 return le32_to_cpu(dma_hw_desc->sg1l); 881 default: 882 printk(KERN_ERR "%s: unknown OPC 0x%02x\n", 883 __func__, dma_hw_desc->opc); 884 BUG(); 885 } 886 return le32_to_cpu(dma_hw_desc->sg1l); 887 case PPC440SPE_XOR_ID: 888 /* May have up to 16 sources */ 889 xor_hw_desc = desc->hw_desc; 890 return xor_hw_desc->ops[src_idx].l; 891 } 892 return 0; 893 } 894 895 /** 896 * ppc440spe_desc_get_dest_addr - extract the destination address from the 897 * descriptor 898 */ 899 static u32 ppc440spe_desc_get_dest_addr(struct ppc440spe_adma_desc_slot *desc, 900 struct ppc440spe_adma_chan *chan, int idx) 901 { 902 struct dma_cdb *dma_hw_desc; 903 struct xor_cb *xor_hw_desc; 904 905 switch (chan->device->id) { 906 case PPC440SPE_DMA0_ID: 907 case PPC440SPE_DMA1_ID: 908 dma_hw_desc = desc->hw_desc; 909 910 if (likely(!idx)) 911 return le32_to_cpu(dma_hw_desc->sg2l); 912 return le32_to_cpu(dma_hw_desc->sg3l); 913 case PPC440SPE_XOR_ID: 914 xor_hw_desc = desc->hw_desc; 915 return xor_hw_desc->cbtal; 916 } 917 return 0; 918 } 919 920 /** 921 * ppc440spe_desc_get_src_num - extract the number of source addresses from 922 * the descriptor 923 */ 924 static u32 ppc440spe_desc_get_src_num(struct ppc440spe_adma_desc_slot *desc, 925 struct ppc440spe_adma_chan *chan) 926 { 927 struct dma_cdb *dma_hw_desc; 928 struct xor_cb *xor_hw_desc; 929 930 switch (chan->device->id) { 931 case PPC440SPE_DMA0_ID: 932 case PPC440SPE_DMA1_ID: 933 dma_hw_desc = desc->hw_desc; 934 935 switch (dma_hw_desc->opc) { 936 case DMA_CDB_OPC_NO_OP: 937 case DMA_CDB_OPC_DFILL128: 938 return 0; 939 case DMA_CDB_OPC_DCHECK128: 940 return 1; 941 case DMA_CDB_OPC_MV_SG1_SG2: 942 case DMA_CDB_OPC_MULTICAST: 943 /* 944 * Only for RXOR operations we have more than 945 * one source 946 */ 947 if (le32_to_cpu(dma_hw_desc->sg1u) & 948 DMA_CUED_XOR_WIN_MSK) { 949 /* RXOR op, there are 2 or 3 sources */ 950 if (((le32_to_cpu(dma_hw_desc->sg1u) >> 951 DMA_CUED_REGION_OFF) & 952 DMA_CUED_REGION_MSK) == DMA_RXOR12) { 953 /* RXOR 1-2 */ 954 return 2; 955 } else { 956 /* RXOR 1-2-3/1-2-4/1-2-5 */ 957 return 3; 958 } 959 } 960 return 1; 961 default: 962 printk(KERN_ERR "%s: unknown OPC 0x%02x\n", 963 __func__, dma_hw_desc->opc); 964 BUG(); 965 } 966 case PPC440SPE_XOR_ID: 967 /* up to 16 sources */ 968 xor_hw_desc = desc->hw_desc; 969 return xor_hw_desc->cbc & XOR_CDCR_OAC_MSK; 970 default: 971 BUG(); 972 } 973 return 0; 974 } 975 976 /** 977 * ppc440spe_desc_get_dst_num - get the number of destination addresses in 978 * this descriptor 979 */ 980 static u32 ppc440spe_desc_get_dst_num(struct ppc440spe_adma_desc_slot *desc, 981 struct ppc440spe_adma_chan *chan) 982 { 983 struct dma_cdb *dma_hw_desc; 984 985 switch (chan->device->id) { 986 case PPC440SPE_DMA0_ID: 987 case PPC440SPE_DMA1_ID: 988 /* May be 1 or 2 destinations */ 989 dma_hw_desc = desc->hw_desc; 990 switch (dma_hw_desc->opc) { 991 case DMA_CDB_OPC_NO_OP: 992 case DMA_CDB_OPC_DCHECK128: 993 return 0; 994 case DMA_CDB_OPC_MV_SG1_SG2: 995 case DMA_CDB_OPC_DFILL128: 996 return 1; 997 case DMA_CDB_OPC_MULTICAST: 998 if (desc->dst_cnt == 2) 999 return 2; 1000 else 1001 return 1; 1002 default: 1003 printk(KERN_ERR "%s: unknown OPC 0x%02x\n", 1004 __func__, dma_hw_desc->opc); 1005 BUG(); 1006 } 1007 case PPC440SPE_XOR_ID: 1008 /* Always only 1 destination */ 1009 return 1; 1010 default: 1011 BUG(); 1012 } 1013 return 0; 1014 } 1015 1016 /** 1017 * ppc440spe_desc_get_link - get the address of the descriptor that 1018 * follows this one 1019 */ 1020 static inline u32 ppc440spe_desc_get_link(struct ppc440spe_adma_desc_slot *desc, 1021 struct ppc440spe_adma_chan *chan) 1022 { 1023 if (!desc->hw_next) 1024 return 0; 1025 1026 return desc->hw_next->phys; 1027 } 1028 1029 /** 1030 * ppc440spe_desc_is_aligned - check alignment 1031 */ 1032 static inline int ppc440spe_desc_is_aligned( 1033 struct ppc440spe_adma_desc_slot *desc, int num_slots) 1034 { 1035 return (desc->idx & (num_slots - 1)) ? 0 : 1; 1036 } 1037 1038 /** 1039 * ppc440spe_chan_xor_slot_count - get the number of slots necessary for 1040 * XOR operation 1041 */ 1042 static int ppc440spe_chan_xor_slot_count(size_t len, int src_cnt, 1043 int *slots_per_op) 1044 { 1045 int slot_cnt; 1046 1047 /* each XOR descriptor provides up to 16 source operands */ 1048 slot_cnt = *slots_per_op = (src_cnt + XOR_MAX_OPS - 1)/XOR_MAX_OPS; 1049 1050 if (likely(len <= PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT)) 1051 return slot_cnt; 1052 1053 printk(KERN_ERR "%s: len %d > max %d !!\n", 1054 __func__, len, PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT); 1055 BUG(); 1056 return slot_cnt; 1057 } 1058 1059 /** 1060 * ppc440spe_dma2_pq_slot_count - get the number of slots necessary for 1061 * DMA2 PQ operation 1062 */ 1063 static int ppc440spe_dma2_pq_slot_count(dma_addr_t *srcs, 1064 int src_cnt, size_t len) 1065 { 1066 signed long long order = 0; 1067 int state = 0; 1068 int addr_count = 0; 1069 int i; 1070 for (i = 1; i < src_cnt; i++) { 1071 dma_addr_t cur_addr = srcs[i]; 1072 dma_addr_t old_addr = srcs[i-1]; 1073 switch (state) { 1074 case 0: 1075 if (cur_addr == old_addr + len) { 1076 /* direct RXOR */ 1077 order = 1; 1078 state = 1; 1079 if (i == src_cnt-1) 1080 addr_count++; 1081 } else if (old_addr == cur_addr + len) { 1082 /* reverse RXOR */ 1083 order = -1; 1084 state = 1; 1085 if (i == src_cnt-1) 1086 addr_count++; 1087 } else { 1088 state = 3; 1089 } 1090 break; 1091 case 1: 1092 if (i == src_cnt-2 || (order == -1 1093 && cur_addr != old_addr - len)) { 1094 order = 0; 1095 state = 0; 1096 addr_count++; 1097 } else if (cur_addr == old_addr + len*order) { 1098 state = 2; 1099 if (i == src_cnt-1) 1100 addr_count++; 1101 } else if (cur_addr == old_addr + 2*len) { 1102 state = 2; 1103 if (i == src_cnt-1) 1104 addr_count++; 1105 } else if (cur_addr == old_addr + 3*len) { 1106 state = 2; 1107 if (i == src_cnt-1) 1108 addr_count++; 1109 } else { 1110 order = 0; 1111 state = 0; 1112 addr_count++; 1113 } 1114 break; 1115 case 2: 1116 order = 0; 1117 state = 0; 1118 addr_count++; 1119 break; 1120 } 1121 if (state == 3) 1122 break; 1123 } 1124 if (src_cnt <= 1 || (state != 1 && state != 2)) { 1125 pr_err("%s: src_cnt=%d, state=%d, addr_count=%d, order=%lld\n", 1126 __func__, src_cnt, state, addr_count, order); 1127 for (i = 0; i < src_cnt; i++) 1128 pr_err("\t[%d] 0x%llx \n", i, srcs[i]); 1129 BUG(); 1130 } 1131 1132 return (addr_count + XOR_MAX_OPS - 1) / XOR_MAX_OPS; 1133 } 1134 1135 1136 /****************************************************************************** 1137 * ADMA channel low-level routines 1138 ******************************************************************************/ 1139 1140 static u32 1141 ppc440spe_chan_get_current_descriptor(struct ppc440spe_adma_chan *chan); 1142 static void ppc440spe_chan_append(struct ppc440spe_adma_chan *chan); 1143 1144 /** 1145 * ppc440spe_adma_device_clear_eot_status - interrupt ack to XOR or DMA engine 1146 */ 1147 static void ppc440spe_adma_device_clear_eot_status( 1148 struct ppc440spe_adma_chan *chan) 1149 { 1150 struct dma_regs *dma_reg; 1151 struct xor_regs *xor_reg; 1152 u8 *p = chan->device->dma_desc_pool_virt; 1153 struct dma_cdb *cdb; 1154 u32 rv, i; 1155 1156 switch (chan->device->id) { 1157 case PPC440SPE_DMA0_ID: 1158 case PPC440SPE_DMA1_ID: 1159 /* read FIFO to ack */ 1160 dma_reg = chan->device->dma_reg; 1161 while ((rv = ioread32(&dma_reg->csfpl))) { 1162 i = rv & DMA_CDB_ADDR_MSK; 1163 cdb = (struct dma_cdb *)&p[i - 1164 (u32)chan->device->dma_desc_pool]; 1165 1166 /* Clear opcode to ack. This is necessary for 1167 * ZeroSum operations only 1168 */ 1169 cdb->opc = 0; 1170 1171 if (test_bit(PPC440SPE_RXOR_RUN, 1172 &ppc440spe_rxor_state)) { 1173 /* probably this is a completed RXOR op, 1174 * get pointer to CDB using the fact that 1175 * physical and virtual addresses of CDB 1176 * in pools have the same offsets 1177 */ 1178 if (le32_to_cpu(cdb->sg1u) & 1179 DMA_CUED_XOR_BASE) { 1180 /* this is a RXOR */ 1181 clear_bit(PPC440SPE_RXOR_RUN, 1182 &ppc440spe_rxor_state); 1183 } 1184 } 1185 1186 if (rv & DMA_CDB_STATUS_MSK) { 1187 /* ZeroSum check failed 1188 */ 1189 struct ppc440spe_adma_desc_slot *iter; 1190 dma_addr_t phys = rv & ~DMA_CDB_MSK; 1191 1192 /* 1193 * Update the status of corresponding 1194 * descriptor. 1195 */ 1196 list_for_each_entry(iter, &chan->chain, 1197 chain_node) { 1198 if (iter->phys == phys) 1199 break; 1200 } 1201 /* 1202 * if cannot find the corresponding 1203 * slot it's a bug 1204 */ 1205 BUG_ON(&iter->chain_node == &chan->chain); 1206 1207 if (iter->xor_check_result) { 1208 if (test_bit(PPC440SPE_DESC_PCHECK, 1209 &iter->flags)) { 1210 *iter->xor_check_result |= 1211 SUM_CHECK_P_RESULT; 1212 } else 1213 if (test_bit(PPC440SPE_DESC_QCHECK, 1214 &iter->flags)) { 1215 *iter->xor_check_result |= 1216 SUM_CHECK_Q_RESULT; 1217 } else 1218 BUG(); 1219 } 1220 } 1221 } 1222 1223 rv = ioread32(&dma_reg->dsts); 1224 if (rv) { 1225 pr_err("DMA%d err status: 0x%x\n", 1226 chan->device->id, rv); 1227 /* write back to clear */ 1228 iowrite32(rv, &dma_reg->dsts); 1229 } 1230 break; 1231 case PPC440SPE_XOR_ID: 1232 /* reset status bits to ack */ 1233 xor_reg = chan->device->xor_reg; 1234 rv = ioread32be(&xor_reg->sr); 1235 iowrite32be(rv, &xor_reg->sr); 1236 1237 if (rv & (XOR_IE_ICBIE_BIT|XOR_IE_ICIE_BIT|XOR_IE_RPTIE_BIT)) { 1238 if (rv & XOR_IE_RPTIE_BIT) { 1239 /* Read PLB Timeout Error. 1240 * Try to resubmit the CB 1241 */ 1242 u32 val = ioread32be(&xor_reg->ccbalr); 1243 1244 iowrite32be(val, &xor_reg->cblalr); 1245 1246 val = ioread32be(&xor_reg->crsr); 1247 iowrite32be(val | XOR_CRSR_XAE_BIT, 1248 &xor_reg->crsr); 1249 } else 1250 pr_err("XOR ERR 0x%x status\n", rv); 1251 break; 1252 } 1253 1254 /* if the XORcore is idle, but there are unprocessed CBs 1255 * then refetch the s/w chain here 1256 */ 1257 if (!(ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT) && 1258 do_xor_refetch) 1259 ppc440spe_chan_append(chan); 1260 break; 1261 } 1262 } 1263 1264 /** 1265 * ppc440spe_chan_is_busy - get the channel status 1266 */ 1267 static int ppc440spe_chan_is_busy(struct ppc440spe_adma_chan *chan) 1268 { 1269 struct dma_regs *dma_reg; 1270 struct xor_regs *xor_reg; 1271 int busy = 0; 1272 1273 switch (chan->device->id) { 1274 case PPC440SPE_DMA0_ID: 1275 case PPC440SPE_DMA1_ID: 1276 dma_reg = chan->device->dma_reg; 1277 /* if command FIFO's head and tail pointers are equal and 1278 * status tail is the same as command, then channel is free 1279 */ 1280 if (ioread16(&dma_reg->cpfhp) != ioread16(&dma_reg->cpftp) || 1281 ioread16(&dma_reg->cpftp) != ioread16(&dma_reg->csftp)) 1282 busy = 1; 1283 break; 1284 case PPC440SPE_XOR_ID: 1285 /* use the special status bit for the XORcore 1286 */ 1287 xor_reg = chan->device->xor_reg; 1288 busy = (ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT) ? 1 : 0; 1289 break; 1290 } 1291 1292 return busy; 1293 } 1294 1295 /** 1296 * ppc440spe_chan_set_first_xor_descriptor - init XORcore chain 1297 */ 1298 static void ppc440spe_chan_set_first_xor_descriptor( 1299 struct ppc440spe_adma_chan *chan, 1300 struct ppc440spe_adma_desc_slot *next_desc) 1301 { 1302 struct xor_regs *xor_reg = chan->device->xor_reg; 1303 1304 if (ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT) 1305 printk(KERN_INFO "%s: Warn: XORcore is running " 1306 "when try to set the first CDB!\n", 1307 __func__); 1308 1309 xor_last_submit = xor_last_linked = next_desc; 1310 1311 iowrite32be(XOR_CRSR_64BA_BIT, &xor_reg->crsr); 1312 1313 iowrite32be(next_desc->phys, &xor_reg->cblalr); 1314 iowrite32be(0, &xor_reg->cblahr); 1315 iowrite32be(ioread32be(&xor_reg->cbcr) | XOR_CBCR_LNK_BIT, 1316 &xor_reg->cbcr); 1317 1318 chan->hw_chain_inited = 1; 1319 } 1320 1321 /** 1322 * ppc440spe_dma_put_desc - put DMA0,1 descriptor to FIFO. 1323 * called with irqs disabled 1324 */ 1325 static void ppc440spe_dma_put_desc(struct ppc440spe_adma_chan *chan, 1326 struct ppc440spe_adma_desc_slot *desc) 1327 { 1328 u32 pcdb; 1329 struct dma_regs *dma_reg = chan->device->dma_reg; 1330 1331 pcdb = desc->phys; 1332 if (!test_bit(PPC440SPE_DESC_INT, &desc->flags)) 1333 pcdb |= DMA_CDB_NO_INT; 1334 1335 chan_last_sub[chan->device->id] = desc; 1336 1337 ADMA_LL_DBG(print_cb(chan, desc->hw_desc)); 1338 1339 iowrite32(pcdb, &dma_reg->cpfpl); 1340 } 1341 1342 /** 1343 * ppc440spe_chan_append - update the h/w chain in the channel 1344 */ 1345 static void ppc440spe_chan_append(struct ppc440spe_adma_chan *chan) 1346 { 1347 struct xor_regs *xor_reg; 1348 struct ppc440spe_adma_desc_slot *iter; 1349 struct xor_cb *xcb; 1350 u32 cur_desc; 1351 unsigned long flags; 1352 1353 local_irq_save(flags); 1354 1355 switch (chan->device->id) { 1356 case PPC440SPE_DMA0_ID: 1357 case PPC440SPE_DMA1_ID: 1358 cur_desc = ppc440spe_chan_get_current_descriptor(chan); 1359 1360 if (likely(cur_desc)) { 1361 iter = chan_last_sub[chan->device->id]; 1362 BUG_ON(!iter); 1363 } else { 1364 /* first peer */ 1365 iter = chan_first_cdb[chan->device->id]; 1366 BUG_ON(!iter); 1367 ppc440spe_dma_put_desc(chan, iter); 1368 chan->hw_chain_inited = 1; 1369 } 1370 1371 /* is there something new to append */ 1372 if (!iter->hw_next) 1373 break; 1374 1375 /* flush descriptors from the s/w queue to fifo */ 1376 list_for_each_entry_continue(iter, &chan->chain, chain_node) { 1377 ppc440spe_dma_put_desc(chan, iter); 1378 if (!iter->hw_next) 1379 break; 1380 } 1381 break; 1382 case PPC440SPE_XOR_ID: 1383 /* update h/w links and refetch */ 1384 if (!xor_last_submit->hw_next) 1385 break; 1386 1387 xor_reg = chan->device->xor_reg; 1388 /* the last linked CDB has to generate an interrupt 1389 * that we'd be able to append the next lists to h/w 1390 * regardless of the XOR engine state at the moment of 1391 * appending of these next lists 1392 */ 1393 xcb = xor_last_linked->hw_desc; 1394 xcb->cbc |= XOR_CBCR_CBCE_BIT; 1395 1396 if (!(ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT)) { 1397 /* XORcore is idle. Refetch now */ 1398 do_xor_refetch = 0; 1399 ppc440spe_xor_set_link(xor_last_submit, 1400 xor_last_submit->hw_next); 1401 1402 ADMA_LL_DBG(print_cb_list(chan, 1403 xor_last_submit->hw_next)); 1404 1405 xor_last_submit = xor_last_linked; 1406 iowrite32be(ioread32be(&xor_reg->crsr) | 1407 XOR_CRSR_RCBE_BIT | XOR_CRSR_64BA_BIT, 1408 &xor_reg->crsr); 1409 } else { 1410 /* XORcore is running. Refetch later in the handler */ 1411 do_xor_refetch = 1; 1412 } 1413 1414 break; 1415 } 1416 1417 local_irq_restore(flags); 1418 } 1419 1420 /** 1421 * ppc440spe_chan_get_current_descriptor - get the currently executed descriptor 1422 */ 1423 static u32 1424 ppc440spe_chan_get_current_descriptor(struct ppc440spe_adma_chan *chan) 1425 { 1426 struct dma_regs *dma_reg; 1427 struct xor_regs *xor_reg; 1428 1429 if (unlikely(!chan->hw_chain_inited)) 1430 /* h/w descriptor chain is not initialized yet */ 1431 return 0; 1432 1433 switch (chan->device->id) { 1434 case PPC440SPE_DMA0_ID: 1435 case PPC440SPE_DMA1_ID: 1436 dma_reg = chan->device->dma_reg; 1437 return ioread32(&dma_reg->acpl) & (~DMA_CDB_MSK); 1438 case PPC440SPE_XOR_ID: 1439 xor_reg = chan->device->xor_reg; 1440 return ioread32be(&xor_reg->ccbalr); 1441 } 1442 return 0; 1443 } 1444 1445 /** 1446 * ppc440spe_chan_run - enable the channel 1447 */ 1448 static void ppc440spe_chan_run(struct ppc440spe_adma_chan *chan) 1449 { 1450 struct xor_regs *xor_reg; 1451 1452 switch (chan->device->id) { 1453 case PPC440SPE_DMA0_ID: 1454 case PPC440SPE_DMA1_ID: 1455 /* DMAs are always enabled, do nothing */ 1456 break; 1457 case PPC440SPE_XOR_ID: 1458 /* drain write buffer */ 1459 xor_reg = chan->device->xor_reg; 1460 1461 /* fetch descriptor pointed to in <link> */ 1462 iowrite32be(XOR_CRSR_64BA_BIT | XOR_CRSR_XAE_BIT, 1463 &xor_reg->crsr); 1464 break; 1465 } 1466 } 1467 1468 /****************************************************************************** 1469 * ADMA device level 1470 ******************************************************************************/ 1471 1472 static void ppc440spe_chan_start_null_xor(struct ppc440spe_adma_chan *chan); 1473 static int ppc440spe_adma_alloc_chan_resources(struct dma_chan *chan); 1474 1475 static dma_cookie_t 1476 ppc440spe_adma_tx_submit(struct dma_async_tx_descriptor *tx); 1477 1478 static void ppc440spe_adma_set_dest(struct ppc440spe_adma_desc_slot *tx, 1479 dma_addr_t addr, int index); 1480 static void 1481 ppc440spe_adma_memcpy_xor_set_src(struct ppc440spe_adma_desc_slot *tx, 1482 dma_addr_t addr, int index); 1483 1484 static void 1485 ppc440spe_adma_pq_set_dest(struct ppc440spe_adma_desc_slot *tx, 1486 dma_addr_t *paddr, unsigned long flags); 1487 static void 1488 ppc440spe_adma_pq_set_src(struct ppc440spe_adma_desc_slot *tx, 1489 dma_addr_t addr, int index); 1490 static void 1491 ppc440spe_adma_pq_set_src_mult(struct ppc440spe_adma_desc_slot *tx, 1492 unsigned char mult, int index, int dst_pos); 1493 static void 1494 ppc440spe_adma_pqzero_sum_set_dest(struct ppc440spe_adma_desc_slot *tx, 1495 dma_addr_t paddr, dma_addr_t qaddr); 1496 1497 static struct page *ppc440spe_rxor_srcs[32]; 1498 1499 /** 1500 * ppc440spe_can_rxor - check if the operands may be processed with RXOR 1501 */ 1502 static int ppc440spe_can_rxor(struct page **srcs, int src_cnt, size_t len) 1503 { 1504 int i, order = 0, state = 0; 1505 int idx = 0; 1506 1507 if (unlikely(!(src_cnt > 1))) 1508 return 0; 1509 1510 BUG_ON(src_cnt > ARRAY_SIZE(ppc440spe_rxor_srcs)); 1511 1512 /* Skip holes in the source list before checking */ 1513 for (i = 0; i < src_cnt; i++) { 1514 if (!srcs[i]) 1515 continue; 1516 ppc440spe_rxor_srcs[idx++] = srcs[i]; 1517 } 1518 src_cnt = idx; 1519 1520 for (i = 1; i < src_cnt; i++) { 1521 char *cur_addr = page_address(ppc440spe_rxor_srcs[i]); 1522 char *old_addr = page_address(ppc440spe_rxor_srcs[i - 1]); 1523 1524 switch (state) { 1525 case 0: 1526 if (cur_addr == old_addr + len) { 1527 /* direct RXOR */ 1528 order = 1; 1529 state = 1; 1530 } else if (old_addr == cur_addr + len) { 1531 /* reverse RXOR */ 1532 order = -1; 1533 state = 1; 1534 } else 1535 goto out; 1536 break; 1537 case 1: 1538 if ((i == src_cnt - 2) || 1539 (order == -1 && cur_addr != old_addr - len)) { 1540 order = 0; 1541 state = 0; 1542 } else if ((cur_addr == old_addr + len * order) || 1543 (cur_addr == old_addr + 2 * len) || 1544 (cur_addr == old_addr + 3 * len)) { 1545 state = 2; 1546 } else { 1547 order = 0; 1548 state = 0; 1549 } 1550 break; 1551 case 2: 1552 order = 0; 1553 state = 0; 1554 break; 1555 } 1556 } 1557 1558 out: 1559 if (state == 1 || state == 2) 1560 return 1; 1561 1562 return 0; 1563 } 1564 1565 /** 1566 * ppc440spe_adma_device_estimate - estimate the efficiency of processing 1567 * the operation given on this channel. It's assumed that 'chan' is 1568 * capable to process 'cap' type of operation. 1569 * @chan: channel to use 1570 * @cap: type of transaction 1571 * @dst_lst: array of destination pointers 1572 * @dst_cnt: number of destination operands 1573 * @src_lst: array of source pointers 1574 * @src_cnt: number of source operands 1575 * @src_sz: size of each source operand 1576 */ 1577 static int ppc440spe_adma_estimate(struct dma_chan *chan, 1578 enum dma_transaction_type cap, struct page **dst_lst, int dst_cnt, 1579 struct page **src_lst, int src_cnt, size_t src_sz) 1580 { 1581 int ef = 1; 1582 1583 if (cap == DMA_PQ || cap == DMA_PQ_VAL) { 1584 /* If RAID-6 capabilities were not activated don't try 1585 * to use them 1586 */ 1587 if (unlikely(!ppc440spe_r6_enabled)) 1588 return -1; 1589 } 1590 /* In the current implementation of ppc440spe ADMA driver it 1591 * makes sense to pick out only pq case, because it may be 1592 * processed: 1593 * (1) either using Biskup method on DMA2; 1594 * (2) or on DMA0/1. 1595 * Thus we give a favour to (1) if the sources are suitable; 1596 * else let it be processed on one of the DMA0/1 engines. 1597 * In the sum_product case where destination is also the 1598 * source process it on DMA0/1 only. 1599 */ 1600 if (cap == DMA_PQ && chan->chan_id == PPC440SPE_XOR_ID) { 1601 1602 if (dst_cnt == 1 && src_cnt == 2 && dst_lst[0] == src_lst[1]) 1603 ef = 0; /* sum_product case, process on DMA0/1 */ 1604 else if (ppc440spe_can_rxor(src_lst, src_cnt, src_sz)) 1605 ef = 3; /* override (DMA0/1 + idle) */ 1606 else 1607 ef = 0; /* can't process on DMA2 if !rxor */ 1608 } 1609 1610 /* channel idleness increases the priority */ 1611 if (likely(ef) && 1612 !ppc440spe_chan_is_busy(to_ppc440spe_adma_chan(chan))) 1613 ef++; 1614 1615 return ef; 1616 } 1617 1618 struct dma_chan * 1619 ppc440spe_async_tx_find_best_channel(enum dma_transaction_type cap, 1620 struct page **dst_lst, int dst_cnt, struct page **src_lst, 1621 int src_cnt, size_t src_sz) 1622 { 1623 struct dma_chan *best_chan = NULL; 1624 struct ppc_dma_chan_ref *ref; 1625 int best_rank = -1; 1626 1627 if (unlikely(!src_sz)) 1628 return NULL; 1629 if (src_sz > PAGE_SIZE) { 1630 /* 1631 * should a user of the api ever pass > PAGE_SIZE requests 1632 * we sort out cases where temporary page-sized buffers 1633 * are used. 1634 */ 1635 switch (cap) { 1636 case DMA_PQ: 1637 if (src_cnt == 1 && dst_lst[1] == src_lst[0]) 1638 return NULL; 1639 if (src_cnt == 2 && dst_lst[1] == src_lst[1]) 1640 return NULL; 1641 break; 1642 case DMA_PQ_VAL: 1643 case DMA_XOR_VAL: 1644 return NULL; 1645 default: 1646 break; 1647 } 1648 } 1649 1650 list_for_each_entry(ref, &ppc440spe_adma_chan_list, node) { 1651 if (dma_has_cap(cap, ref->chan->device->cap_mask)) { 1652 int rank; 1653 1654 rank = ppc440spe_adma_estimate(ref->chan, cap, dst_lst, 1655 dst_cnt, src_lst, src_cnt, src_sz); 1656 if (rank > best_rank) { 1657 best_rank = rank; 1658 best_chan = ref->chan; 1659 } 1660 } 1661 } 1662 1663 return best_chan; 1664 } 1665 EXPORT_SYMBOL_GPL(ppc440spe_async_tx_find_best_channel); 1666 1667 /** 1668 * ppc440spe_get_group_entry - get group entry with index idx 1669 * @tdesc: is the last allocated slot in the group. 1670 */ 1671 static struct ppc440spe_adma_desc_slot * 1672 ppc440spe_get_group_entry(struct ppc440spe_adma_desc_slot *tdesc, u32 entry_idx) 1673 { 1674 struct ppc440spe_adma_desc_slot *iter = tdesc->group_head; 1675 int i = 0; 1676 1677 if (entry_idx < 0 || entry_idx >= (tdesc->src_cnt + tdesc->dst_cnt)) { 1678 printk("%s: entry_idx %d, src_cnt %d, dst_cnt %d\n", 1679 __func__, entry_idx, tdesc->src_cnt, tdesc->dst_cnt); 1680 BUG(); 1681 } 1682 1683 list_for_each_entry(iter, &tdesc->group_list, chain_node) { 1684 if (i++ == entry_idx) 1685 break; 1686 } 1687 return iter; 1688 } 1689 1690 /** 1691 * ppc440spe_adma_free_slots - flags descriptor slots for reuse 1692 * @slot: Slot to free 1693 * Caller must hold &ppc440spe_chan->lock while calling this function 1694 */ 1695 static void ppc440spe_adma_free_slots(struct ppc440spe_adma_desc_slot *slot, 1696 struct ppc440spe_adma_chan *chan) 1697 { 1698 int stride = slot->slots_per_op; 1699 1700 while (stride--) { 1701 slot->slots_per_op = 0; 1702 slot = list_entry(slot->slot_node.next, 1703 struct ppc440spe_adma_desc_slot, 1704 slot_node); 1705 } 1706 } 1707 1708 static void ppc440spe_adma_unmap(struct ppc440spe_adma_chan *chan, 1709 struct ppc440spe_adma_desc_slot *desc) 1710 { 1711 u32 src_cnt, dst_cnt; 1712 dma_addr_t addr; 1713 1714 /* 1715 * get the number of sources & destination 1716 * included in this descriptor and unmap 1717 * them all 1718 */ 1719 src_cnt = ppc440spe_desc_get_src_num(desc, chan); 1720 dst_cnt = ppc440spe_desc_get_dst_num(desc, chan); 1721 1722 /* unmap destinations */ 1723 if (!(desc->async_tx.flags & DMA_COMPL_SKIP_DEST_UNMAP)) { 1724 while (dst_cnt--) { 1725 addr = ppc440spe_desc_get_dest_addr( 1726 desc, chan, dst_cnt); 1727 dma_unmap_page(chan->device->dev, 1728 addr, desc->unmap_len, 1729 DMA_FROM_DEVICE); 1730 } 1731 } 1732 1733 /* unmap sources */ 1734 if (!(desc->async_tx.flags & DMA_COMPL_SKIP_SRC_UNMAP)) { 1735 while (src_cnt--) { 1736 addr = ppc440spe_desc_get_src_addr( 1737 desc, chan, src_cnt); 1738 dma_unmap_page(chan->device->dev, 1739 addr, desc->unmap_len, 1740 DMA_TO_DEVICE); 1741 } 1742 } 1743 } 1744 1745 /** 1746 * ppc440spe_adma_run_tx_complete_actions - call functions to be called 1747 * upon completion 1748 */ 1749 static dma_cookie_t ppc440spe_adma_run_tx_complete_actions( 1750 struct ppc440spe_adma_desc_slot *desc, 1751 struct ppc440spe_adma_chan *chan, 1752 dma_cookie_t cookie) 1753 { 1754 int i; 1755 1756 BUG_ON(desc->async_tx.cookie < 0); 1757 if (desc->async_tx.cookie > 0) { 1758 cookie = desc->async_tx.cookie; 1759 desc->async_tx.cookie = 0; 1760 1761 /* call the callback (must not sleep or submit new 1762 * operations to this channel) 1763 */ 1764 if (desc->async_tx.callback) 1765 desc->async_tx.callback( 1766 desc->async_tx.callback_param); 1767 1768 /* unmap dma addresses 1769 * (unmap_single vs unmap_page?) 1770 * 1771 * actually, ppc's dma_unmap_page() functions are empty, so 1772 * the following code is just for the sake of completeness 1773 */ 1774 if (chan && chan->needs_unmap && desc->group_head && 1775 desc->unmap_len) { 1776 struct ppc440spe_adma_desc_slot *unmap = 1777 desc->group_head; 1778 /* assume 1 slot per op always */ 1779 u32 slot_count = unmap->slot_cnt; 1780 1781 /* Run through the group list and unmap addresses */ 1782 for (i = 0; i < slot_count; i++) { 1783 BUG_ON(!unmap); 1784 ppc440spe_adma_unmap(chan, unmap); 1785 unmap = unmap->hw_next; 1786 } 1787 } 1788 } 1789 1790 /* run dependent operations */ 1791 dma_run_dependencies(&desc->async_tx); 1792 1793 return cookie; 1794 } 1795 1796 /** 1797 * ppc440spe_adma_clean_slot - clean up CDB slot (if ack is set) 1798 */ 1799 static int ppc440spe_adma_clean_slot(struct ppc440spe_adma_desc_slot *desc, 1800 struct ppc440spe_adma_chan *chan) 1801 { 1802 /* the client is allowed to attach dependent operations 1803 * until 'ack' is set 1804 */ 1805 if (!async_tx_test_ack(&desc->async_tx)) 1806 return 0; 1807 1808 /* leave the last descriptor in the chain 1809 * so we can append to it 1810 */ 1811 if (list_is_last(&desc->chain_node, &chan->chain) || 1812 desc->phys == ppc440spe_chan_get_current_descriptor(chan)) 1813 return 1; 1814 1815 if (chan->device->id != PPC440SPE_XOR_ID) { 1816 /* our DMA interrupt handler clears opc field of 1817 * each processed descriptor. For all types of 1818 * operations except for ZeroSum we do not actually 1819 * need ack from the interrupt handler. ZeroSum is a 1820 * special case since the result of this operation 1821 * is available from the handler only, so if we see 1822 * such type of descriptor (which is unprocessed yet) 1823 * then leave it in chain. 1824 */ 1825 struct dma_cdb *cdb = desc->hw_desc; 1826 if (cdb->opc == DMA_CDB_OPC_DCHECK128) 1827 return 1; 1828 } 1829 1830 dev_dbg(chan->device->common.dev, "\tfree slot %llx: %d stride: %d\n", 1831 desc->phys, desc->idx, desc->slots_per_op); 1832 1833 list_del(&desc->chain_node); 1834 ppc440spe_adma_free_slots(desc, chan); 1835 return 0; 1836 } 1837 1838 /** 1839 * __ppc440spe_adma_slot_cleanup - this is the common clean-up routine 1840 * which runs through the channel CDBs list until reach the descriptor 1841 * currently processed. When routine determines that all CDBs of group 1842 * are completed then corresponding callbacks (if any) are called and slots 1843 * are freed. 1844 */ 1845 static void __ppc440spe_adma_slot_cleanup(struct ppc440spe_adma_chan *chan) 1846 { 1847 struct ppc440spe_adma_desc_slot *iter, *_iter, *group_start = NULL; 1848 dma_cookie_t cookie = 0; 1849 u32 current_desc = ppc440spe_chan_get_current_descriptor(chan); 1850 int busy = ppc440spe_chan_is_busy(chan); 1851 int seen_current = 0, slot_cnt = 0, slots_per_op = 0; 1852 1853 dev_dbg(chan->device->common.dev, "ppc440spe adma%d: %s\n", 1854 chan->device->id, __func__); 1855 1856 if (!current_desc) { 1857 /* There were no transactions yet, so 1858 * nothing to clean 1859 */ 1860 return; 1861 } 1862 1863 /* free completed slots from the chain starting with 1864 * the oldest descriptor 1865 */ 1866 list_for_each_entry_safe(iter, _iter, &chan->chain, 1867 chain_node) { 1868 dev_dbg(chan->device->common.dev, "\tcookie: %d slot: %d " 1869 "busy: %d this_desc: %#llx next_desc: %#x " 1870 "cur: %#x ack: %d\n", 1871 iter->async_tx.cookie, iter->idx, busy, iter->phys, 1872 ppc440spe_desc_get_link(iter, chan), current_desc, 1873 async_tx_test_ack(&iter->async_tx)); 1874 prefetch(_iter); 1875 prefetch(&_iter->async_tx); 1876 1877 /* do not advance past the current descriptor loaded into the 1878 * hardware channel,subsequent descriptors are either in process 1879 * or have not been submitted 1880 */ 1881 if (seen_current) 1882 break; 1883 1884 /* stop the search if we reach the current descriptor and the 1885 * channel is busy, or if it appears that the current descriptor 1886 * needs to be re-read (i.e. has been appended to) 1887 */ 1888 if (iter->phys == current_desc) { 1889 BUG_ON(seen_current++); 1890 if (busy || ppc440spe_desc_get_link(iter, chan)) { 1891 /* not all descriptors of the group have 1892 * been completed; exit. 1893 */ 1894 break; 1895 } 1896 } 1897 1898 /* detect the start of a group transaction */ 1899 if (!slot_cnt && !slots_per_op) { 1900 slot_cnt = iter->slot_cnt; 1901 slots_per_op = iter->slots_per_op; 1902 if (slot_cnt <= slots_per_op) { 1903 slot_cnt = 0; 1904 slots_per_op = 0; 1905 } 1906 } 1907 1908 if (slot_cnt) { 1909 if (!group_start) 1910 group_start = iter; 1911 slot_cnt -= slots_per_op; 1912 } 1913 1914 /* all the members of a group are complete */ 1915 if (slots_per_op != 0 && slot_cnt == 0) { 1916 struct ppc440spe_adma_desc_slot *grp_iter, *_grp_iter; 1917 int end_of_chain = 0; 1918 1919 /* clean up the group */ 1920 slot_cnt = group_start->slot_cnt; 1921 grp_iter = group_start; 1922 list_for_each_entry_safe_from(grp_iter, _grp_iter, 1923 &chan->chain, chain_node) { 1924 1925 cookie = ppc440spe_adma_run_tx_complete_actions( 1926 grp_iter, chan, cookie); 1927 1928 slot_cnt -= slots_per_op; 1929 end_of_chain = ppc440spe_adma_clean_slot( 1930 grp_iter, chan); 1931 if (end_of_chain && slot_cnt) { 1932 /* Should wait for ZeroSum completion */ 1933 if (cookie > 0) 1934 chan->common.completed_cookie = cookie; 1935 return; 1936 } 1937 1938 if (slot_cnt == 0 || end_of_chain) 1939 break; 1940 } 1941 1942 /* the group should be complete at this point */ 1943 BUG_ON(slot_cnt); 1944 1945 slots_per_op = 0; 1946 group_start = NULL; 1947 if (end_of_chain) 1948 break; 1949 else 1950 continue; 1951 } else if (slots_per_op) /* wait for group completion */ 1952 continue; 1953 1954 cookie = ppc440spe_adma_run_tx_complete_actions(iter, chan, 1955 cookie); 1956 1957 if (ppc440spe_adma_clean_slot(iter, chan)) 1958 break; 1959 } 1960 1961 BUG_ON(!seen_current); 1962 1963 if (cookie > 0) { 1964 chan->common.completed_cookie = cookie; 1965 pr_debug("\tcompleted cookie %d\n", cookie); 1966 } 1967 1968 } 1969 1970 /** 1971 * ppc440spe_adma_tasklet - clean up watch-dog initiator 1972 */ 1973 static void ppc440spe_adma_tasklet(unsigned long data) 1974 { 1975 struct ppc440spe_adma_chan *chan = (struct ppc440spe_adma_chan *) data; 1976 1977 spin_lock_nested(&chan->lock, SINGLE_DEPTH_NESTING); 1978 __ppc440spe_adma_slot_cleanup(chan); 1979 spin_unlock(&chan->lock); 1980 } 1981 1982 /** 1983 * ppc440spe_adma_slot_cleanup - clean up scheduled initiator 1984 */ 1985 static void ppc440spe_adma_slot_cleanup(struct ppc440spe_adma_chan *chan) 1986 { 1987 spin_lock_bh(&chan->lock); 1988 __ppc440spe_adma_slot_cleanup(chan); 1989 spin_unlock_bh(&chan->lock); 1990 } 1991 1992 /** 1993 * ppc440spe_adma_alloc_slots - allocate free slots (if any) 1994 */ 1995 static struct ppc440spe_adma_desc_slot *ppc440spe_adma_alloc_slots( 1996 struct ppc440spe_adma_chan *chan, int num_slots, 1997 int slots_per_op) 1998 { 1999 struct ppc440spe_adma_desc_slot *iter = NULL, *_iter; 2000 struct ppc440spe_adma_desc_slot *alloc_start = NULL; 2001 struct list_head chain = LIST_HEAD_INIT(chain); 2002 int slots_found, retry = 0; 2003 2004 2005 BUG_ON(!num_slots || !slots_per_op); 2006 /* start search from the last allocated descrtiptor 2007 * if a contiguous allocation can not be found start searching 2008 * from the beginning of the list 2009 */ 2010 retry: 2011 slots_found = 0; 2012 if (retry == 0) 2013 iter = chan->last_used; 2014 else 2015 iter = list_entry(&chan->all_slots, 2016 struct ppc440spe_adma_desc_slot, 2017 slot_node); 2018 list_for_each_entry_safe_continue(iter, _iter, &chan->all_slots, 2019 slot_node) { 2020 prefetch(_iter); 2021 prefetch(&_iter->async_tx); 2022 if (iter->slots_per_op) { 2023 slots_found = 0; 2024 continue; 2025 } 2026 2027 /* start the allocation if the slot is correctly aligned */ 2028 if (!slots_found++) 2029 alloc_start = iter; 2030 2031 if (slots_found == num_slots) { 2032 struct ppc440spe_adma_desc_slot *alloc_tail = NULL; 2033 struct ppc440spe_adma_desc_slot *last_used = NULL; 2034 2035 iter = alloc_start; 2036 while (num_slots) { 2037 int i; 2038 /* pre-ack all but the last descriptor */ 2039 if (num_slots != slots_per_op) 2040 async_tx_ack(&iter->async_tx); 2041 2042 list_add_tail(&iter->chain_node, &chain); 2043 alloc_tail = iter; 2044 iter->async_tx.cookie = 0; 2045 iter->hw_next = NULL; 2046 iter->flags = 0; 2047 iter->slot_cnt = num_slots; 2048 iter->xor_check_result = NULL; 2049 for (i = 0; i < slots_per_op; i++) { 2050 iter->slots_per_op = slots_per_op - i; 2051 last_used = iter; 2052 iter = list_entry(iter->slot_node.next, 2053 struct ppc440spe_adma_desc_slot, 2054 slot_node); 2055 } 2056 num_slots -= slots_per_op; 2057 } 2058 alloc_tail->group_head = alloc_start; 2059 alloc_tail->async_tx.cookie = -EBUSY; 2060 list_splice(&chain, &alloc_tail->group_list); 2061 chan->last_used = last_used; 2062 return alloc_tail; 2063 } 2064 } 2065 if (!retry++) 2066 goto retry; 2067 2068 /* try to free some slots if the allocation fails */ 2069 tasklet_schedule(&chan->irq_tasklet); 2070 return NULL; 2071 } 2072 2073 /** 2074 * ppc440spe_adma_alloc_chan_resources - allocate pools for CDB slots 2075 */ 2076 static int ppc440spe_adma_alloc_chan_resources(struct dma_chan *chan) 2077 { 2078 struct ppc440spe_adma_chan *ppc440spe_chan; 2079 struct ppc440spe_adma_desc_slot *slot = NULL; 2080 char *hw_desc; 2081 int i, db_sz; 2082 int init; 2083 2084 ppc440spe_chan = to_ppc440spe_adma_chan(chan); 2085 init = ppc440spe_chan->slots_allocated ? 0 : 1; 2086 chan->chan_id = ppc440spe_chan->device->id; 2087 2088 /* Allocate descriptor slots */ 2089 i = ppc440spe_chan->slots_allocated; 2090 if (ppc440spe_chan->device->id != PPC440SPE_XOR_ID) 2091 db_sz = sizeof(struct dma_cdb); 2092 else 2093 db_sz = sizeof(struct xor_cb); 2094 2095 for (; i < (ppc440spe_chan->device->pool_size / db_sz); i++) { 2096 slot = kzalloc(sizeof(struct ppc440spe_adma_desc_slot), 2097 GFP_KERNEL); 2098 if (!slot) { 2099 printk(KERN_INFO "SPE ADMA Channel only initialized" 2100 " %d descriptor slots", i--); 2101 break; 2102 } 2103 2104 hw_desc = (char *) ppc440spe_chan->device->dma_desc_pool_virt; 2105 slot->hw_desc = (void *) &hw_desc[i * db_sz]; 2106 dma_async_tx_descriptor_init(&slot->async_tx, chan); 2107 slot->async_tx.tx_submit = ppc440spe_adma_tx_submit; 2108 INIT_LIST_HEAD(&slot->chain_node); 2109 INIT_LIST_HEAD(&slot->slot_node); 2110 INIT_LIST_HEAD(&slot->group_list); 2111 slot->phys = ppc440spe_chan->device->dma_desc_pool + i * db_sz; 2112 slot->idx = i; 2113 2114 spin_lock_bh(&ppc440spe_chan->lock); 2115 ppc440spe_chan->slots_allocated++; 2116 list_add_tail(&slot->slot_node, &ppc440spe_chan->all_slots); 2117 spin_unlock_bh(&ppc440spe_chan->lock); 2118 } 2119 2120 if (i && !ppc440spe_chan->last_used) { 2121 ppc440spe_chan->last_used = 2122 list_entry(ppc440spe_chan->all_slots.next, 2123 struct ppc440spe_adma_desc_slot, 2124 slot_node); 2125 } 2126 2127 dev_dbg(ppc440spe_chan->device->common.dev, 2128 "ppc440spe adma%d: allocated %d descriptor slots\n", 2129 ppc440spe_chan->device->id, i); 2130 2131 /* initialize the channel and the chain with a null operation */ 2132 if (init) { 2133 switch (ppc440spe_chan->device->id) { 2134 case PPC440SPE_DMA0_ID: 2135 case PPC440SPE_DMA1_ID: 2136 ppc440spe_chan->hw_chain_inited = 0; 2137 /* Use WXOR for self-testing */ 2138 if (!ppc440spe_r6_tchan) 2139 ppc440spe_r6_tchan = ppc440spe_chan; 2140 break; 2141 case PPC440SPE_XOR_ID: 2142 ppc440spe_chan_start_null_xor(ppc440spe_chan); 2143 break; 2144 default: 2145 BUG(); 2146 } 2147 ppc440spe_chan->needs_unmap = 1; 2148 } 2149 2150 return (i > 0) ? i : -ENOMEM; 2151 } 2152 2153 /** 2154 * ppc440spe_rxor_set_region_data - 2155 */ 2156 static void ppc440spe_rxor_set_region(struct ppc440spe_adma_desc_slot *desc, 2157 u8 xor_arg_no, u32 mask) 2158 { 2159 struct xor_cb *xcb = desc->hw_desc; 2160 2161 xcb->ops[xor_arg_no].h |= mask; 2162 } 2163 2164 /** 2165 * ppc440spe_rxor_set_src - 2166 */ 2167 static void ppc440spe_rxor_set_src(struct ppc440spe_adma_desc_slot *desc, 2168 u8 xor_arg_no, dma_addr_t addr) 2169 { 2170 struct xor_cb *xcb = desc->hw_desc; 2171 2172 xcb->ops[xor_arg_no].h |= DMA_CUED_XOR_BASE; 2173 xcb->ops[xor_arg_no].l = addr; 2174 } 2175 2176 /** 2177 * ppc440spe_rxor_set_mult - 2178 */ 2179 static void ppc440spe_rxor_set_mult(struct ppc440spe_adma_desc_slot *desc, 2180 u8 xor_arg_no, u8 idx, u8 mult) 2181 { 2182 struct xor_cb *xcb = desc->hw_desc; 2183 2184 xcb->ops[xor_arg_no].h |= mult << (DMA_CUED_MULT1_OFF + idx * 8); 2185 } 2186 2187 /** 2188 * ppc440spe_adma_check_threshold - append CDBs to h/w chain if threshold 2189 * has been achieved 2190 */ 2191 static void ppc440spe_adma_check_threshold(struct ppc440spe_adma_chan *chan) 2192 { 2193 dev_dbg(chan->device->common.dev, "ppc440spe adma%d: pending: %d\n", 2194 chan->device->id, chan->pending); 2195 2196 if (chan->pending >= PPC440SPE_ADMA_THRESHOLD) { 2197 chan->pending = 0; 2198 ppc440spe_chan_append(chan); 2199 } 2200 } 2201 2202 /** 2203 * ppc440spe_adma_tx_submit - submit new descriptor group to the channel 2204 * (it's not necessary that descriptors will be submitted to the h/w 2205 * chains too right now) 2206 */ 2207 static dma_cookie_t ppc440spe_adma_tx_submit(struct dma_async_tx_descriptor *tx) 2208 { 2209 struct ppc440spe_adma_desc_slot *sw_desc; 2210 struct ppc440spe_adma_chan *chan = to_ppc440spe_adma_chan(tx->chan); 2211 struct ppc440spe_adma_desc_slot *group_start, *old_chain_tail; 2212 int slot_cnt; 2213 int slots_per_op; 2214 dma_cookie_t cookie; 2215 2216 sw_desc = tx_to_ppc440spe_adma_slot(tx); 2217 2218 group_start = sw_desc->group_head; 2219 slot_cnt = group_start->slot_cnt; 2220 slots_per_op = group_start->slots_per_op; 2221 2222 spin_lock_bh(&chan->lock); 2223 cookie = dma_cookie_assign(tx); 2224 2225 if (unlikely(list_empty(&chan->chain))) { 2226 /* first peer */ 2227 list_splice_init(&sw_desc->group_list, &chan->chain); 2228 chan_first_cdb[chan->device->id] = group_start; 2229 } else { 2230 /* isn't first peer, bind CDBs to chain */ 2231 old_chain_tail = list_entry(chan->chain.prev, 2232 struct ppc440spe_adma_desc_slot, 2233 chain_node); 2234 list_splice_init(&sw_desc->group_list, 2235 &old_chain_tail->chain_node); 2236 /* fix up the hardware chain */ 2237 ppc440spe_desc_set_link(chan, old_chain_tail, group_start); 2238 } 2239 2240 /* increment the pending count by the number of operations */ 2241 chan->pending += slot_cnt / slots_per_op; 2242 ppc440spe_adma_check_threshold(chan); 2243 spin_unlock_bh(&chan->lock); 2244 2245 dev_dbg(chan->device->common.dev, 2246 "ppc440spe adma%d: %s cookie: %d slot: %d tx %p\n", 2247 chan->device->id, __func__, 2248 sw_desc->async_tx.cookie, sw_desc->idx, sw_desc); 2249 2250 return cookie; 2251 } 2252 2253 /** 2254 * ppc440spe_adma_prep_dma_interrupt - prepare CDB for a pseudo DMA operation 2255 */ 2256 static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_interrupt( 2257 struct dma_chan *chan, unsigned long flags) 2258 { 2259 struct ppc440spe_adma_chan *ppc440spe_chan; 2260 struct ppc440spe_adma_desc_slot *sw_desc, *group_start; 2261 int slot_cnt, slots_per_op; 2262 2263 ppc440spe_chan = to_ppc440spe_adma_chan(chan); 2264 2265 dev_dbg(ppc440spe_chan->device->common.dev, 2266 "ppc440spe adma%d: %s\n", ppc440spe_chan->device->id, 2267 __func__); 2268 2269 spin_lock_bh(&ppc440spe_chan->lock); 2270 slot_cnt = slots_per_op = 1; 2271 sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 2272 slots_per_op); 2273 if (sw_desc) { 2274 group_start = sw_desc->group_head; 2275 ppc440spe_desc_init_interrupt(group_start, ppc440spe_chan); 2276 group_start->unmap_len = 0; 2277 sw_desc->async_tx.flags = flags; 2278 } 2279 spin_unlock_bh(&ppc440spe_chan->lock); 2280 2281 return sw_desc ? &sw_desc->async_tx : NULL; 2282 } 2283 2284 /** 2285 * ppc440spe_adma_prep_dma_memcpy - prepare CDB for a MEMCPY operation 2286 */ 2287 static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_memcpy( 2288 struct dma_chan *chan, dma_addr_t dma_dest, 2289 dma_addr_t dma_src, size_t len, unsigned long flags) 2290 { 2291 struct ppc440spe_adma_chan *ppc440spe_chan; 2292 struct ppc440spe_adma_desc_slot *sw_desc, *group_start; 2293 int slot_cnt, slots_per_op; 2294 2295 ppc440spe_chan = to_ppc440spe_adma_chan(chan); 2296 2297 if (unlikely(!len)) 2298 return NULL; 2299 2300 BUG_ON(len > PPC440SPE_ADMA_DMA_MAX_BYTE_COUNT); 2301 2302 spin_lock_bh(&ppc440spe_chan->lock); 2303 2304 dev_dbg(ppc440spe_chan->device->common.dev, 2305 "ppc440spe adma%d: %s len: %u int_en %d\n", 2306 ppc440spe_chan->device->id, __func__, len, 2307 flags & DMA_PREP_INTERRUPT ? 1 : 0); 2308 slot_cnt = slots_per_op = 1; 2309 sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 2310 slots_per_op); 2311 if (sw_desc) { 2312 group_start = sw_desc->group_head; 2313 ppc440spe_desc_init_memcpy(group_start, flags); 2314 ppc440spe_adma_set_dest(group_start, dma_dest, 0); 2315 ppc440spe_adma_memcpy_xor_set_src(group_start, dma_src, 0); 2316 ppc440spe_desc_set_byte_count(group_start, ppc440spe_chan, len); 2317 sw_desc->unmap_len = len; 2318 sw_desc->async_tx.flags = flags; 2319 } 2320 spin_unlock_bh(&ppc440spe_chan->lock); 2321 2322 return sw_desc ? &sw_desc->async_tx : NULL; 2323 } 2324 2325 /** 2326 * ppc440spe_adma_prep_dma_xor - prepare CDB for a XOR operation 2327 */ 2328 static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_xor( 2329 struct dma_chan *chan, dma_addr_t dma_dest, 2330 dma_addr_t *dma_src, u32 src_cnt, size_t len, 2331 unsigned long flags) 2332 { 2333 struct ppc440spe_adma_chan *ppc440spe_chan; 2334 struct ppc440spe_adma_desc_slot *sw_desc, *group_start; 2335 int slot_cnt, slots_per_op; 2336 2337 ppc440spe_chan = to_ppc440spe_adma_chan(chan); 2338 2339 ADMA_LL_DBG(prep_dma_xor_dbg(ppc440spe_chan->device->id, 2340 dma_dest, dma_src, src_cnt)); 2341 if (unlikely(!len)) 2342 return NULL; 2343 BUG_ON(len > PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT); 2344 2345 dev_dbg(ppc440spe_chan->device->common.dev, 2346 "ppc440spe adma%d: %s src_cnt: %d len: %u int_en: %d\n", 2347 ppc440spe_chan->device->id, __func__, src_cnt, len, 2348 flags & DMA_PREP_INTERRUPT ? 1 : 0); 2349 2350 spin_lock_bh(&ppc440spe_chan->lock); 2351 slot_cnt = ppc440spe_chan_xor_slot_count(len, src_cnt, &slots_per_op); 2352 sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 2353 slots_per_op); 2354 if (sw_desc) { 2355 group_start = sw_desc->group_head; 2356 ppc440spe_desc_init_xor(group_start, src_cnt, flags); 2357 ppc440spe_adma_set_dest(group_start, dma_dest, 0); 2358 while (src_cnt--) 2359 ppc440spe_adma_memcpy_xor_set_src(group_start, 2360 dma_src[src_cnt], src_cnt); 2361 ppc440spe_desc_set_byte_count(group_start, ppc440spe_chan, len); 2362 sw_desc->unmap_len = len; 2363 sw_desc->async_tx.flags = flags; 2364 } 2365 spin_unlock_bh(&ppc440spe_chan->lock); 2366 2367 return sw_desc ? &sw_desc->async_tx : NULL; 2368 } 2369 2370 static inline void 2371 ppc440spe_desc_set_xor_src_cnt(struct ppc440spe_adma_desc_slot *desc, 2372 int src_cnt); 2373 static void ppc440spe_init_rxor_cursor(struct ppc440spe_rxor *cursor); 2374 2375 /** 2376 * ppc440spe_adma_init_dma2rxor_slot - 2377 */ 2378 static void ppc440spe_adma_init_dma2rxor_slot( 2379 struct ppc440spe_adma_desc_slot *desc, 2380 dma_addr_t *src, int src_cnt) 2381 { 2382 int i; 2383 2384 /* initialize CDB */ 2385 for (i = 0; i < src_cnt; i++) { 2386 ppc440spe_adma_dma2rxor_prep_src(desc, &desc->rxor_cursor, i, 2387 desc->src_cnt, (u32)src[i]); 2388 } 2389 } 2390 2391 /** 2392 * ppc440spe_dma01_prep_mult - 2393 * for Q operation where destination is also the source 2394 */ 2395 static struct ppc440spe_adma_desc_slot *ppc440spe_dma01_prep_mult( 2396 struct ppc440spe_adma_chan *ppc440spe_chan, 2397 dma_addr_t *dst, int dst_cnt, dma_addr_t *src, int src_cnt, 2398 const unsigned char *scf, size_t len, unsigned long flags) 2399 { 2400 struct ppc440spe_adma_desc_slot *sw_desc = NULL; 2401 unsigned long op = 0; 2402 int slot_cnt; 2403 2404 set_bit(PPC440SPE_DESC_WXOR, &op); 2405 slot_cnt = 2; 2406 2407 spin_lock_bh(&ppc440spe_chan->lock); 2408 2409 /* use WXOR, each descriptor occupies one slot */ 2410 sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1); 2411 if (sw_desc) { 2412 struct ppc440spe_adma_chan *chan; 2413 struct ppc440spe_adma_desc_slot *iter; 2414 struct dma_cdb *hw_desc; 2415 2416 chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan); 2417 set_bits(op, &sw_desc->flags); 2418 sw_desc->src_cnt = src_cnt; 2419 sw_desc->dst_cnt = dst_cnt; 2420 /* First descriptor, zero data in the destination and copy it 2421 * to q page using MULTICAST transfer. 2422 */ 2423 iter = list_first_entry(&sw_desc->group_list, 2424 struct ppc440spe_adma_desc_slot, 2425 chain_node); 2426 memset(iter->hw_desc, 0, sizeof(struct dma_cdb)); 2427 /* set 'next' pointer */ 2428 iter->hw_next = list_entry(iter->chain_node.next, 2429 struct ppc440spe_adma_desc_slot, 2430 chain_node); 2431 clear_bit(PPC440SPE_DESC_INT, &iter->flags); 2432 hw_desc = iter->hw_desc; 2433 hw_desc->opc = DMA_CDB_OPC_MULTICAST; 2434 2435 ppc440spe_desc_set_dest_addr(iter, chan, 2436 DMA_CUED_XOR_BASE, dst[0], 0); 2437 ppc440spe_desc_set_dest_addr(iter, chan, 0, dst[1], 1); 2438 ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB, 2439 src[0]); 2440 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len); 2441 iter->unmap_len = len; 2442 2443 /* 2444 * Second descriptor, multiply data from the q page 2445 * and store the result in real destination. 2446 */ 2447 iter = list_first_entry(&iter->chain_node, 2448 struct ppc440spe_adma_desc_slot, 2449 chain_node); 2450 memset(iter->hw_desc, 0, sizeof(struct dma_cdb)); 2451 iter->hw_next = NULL; 2452 if (flags & DMA_PREP_INTERRUPT) 2453 set_bit(PPC440SPE_DESC_INT, &iter->flags); 2454 else 2455 clear_bit(PPC440SPE_DESC_INT, &iter->flags); 2456 2457 hw_desc = iter->hw_desc; 2458 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; 2459 ppc440spe_desc_set_src_addr(iter, chan, 0, 2460 DMA_CUED_XOR_HB, dst[1]); 2461 ppc440spe_desc_set_dest_addr(iter, chan, 2462 DMA_CUED_XOR_BASE, dst[0], 0); 2463 2464 ppc440spe_desc_set_src_mult(iter, chan, DMA_CUED_MULT1_OFF, 2465 DMA_CDB_SG_DST1, scf[0]); 2466 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len); 2467 iter->unmap_len = len; 2468 sw_desc->async_tx.flags = flags; 2469 } 2470 2471 spin_unlock_bh(&ppc440spe_chan->lock); 2472 2473 return sw_desc; 2474 } 2475 2476 /** 2477 * ppc440spe_dma01_prep_sum_product - 2478 * Dx = A*(P+Pxy) + B*(Q+Qxy) operation where destination is also 2479 * the source. 2480 */ 2481 static struct ppc440spe_adma_desc_slot *ppc440spe_dma01_prep_sum_product( 2482 struct ppc440spe_adma_chan *ppc440spe_chan, 2483 dma_addr_t *dst, dma_addr_t *src, int src_cnt, 2484 const unsigned char *scf, size_t len, unsigned long flags) 2485 { 2486 struct ppc440spe_adma_desc_slot *sw_desc = NULL; 2487 unsigned long op = 0; 2488 int slot_cnt; 2489 2490 set_bit(PPC440SPE_DESC_WXOR, &op); 2491 slot_cnt = 3; 2492 2493 spin_lock_bh(&ppc440spe_chan->lock); 2494 2495 /* WXOR, each descriptor occupies one slot */ 2496 sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1); 2497 if (sw_desc) { 2498 struct ppc440spe_adma_chan *chan; 2499 struct ppc440spe_adma_desc_slot *iter; 2500 struct dma_cdb *hw_desc; 2501 2502 chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan); 2503 set_bits(op, &sw_desc->flags); 2504 sw_desc->src_cnt = src_cnt; 2505 sw_desc->dst_cnt = 1; 2506 /* 1st descriptor, src[1] data to q page and zero destination */ 2507 iter = list_first_entry(&sw_desc->group_list, 2508 struct ppc440spe_adma_desc_slot, 2509 chain_node); 2510 memset(iter->hw_desc, 0, sizeof(struct dma_cdb)); 2511 iter->hw_next = list_entry(iter->chain_node.next, 2512 struct ppc440spe_adma_desc_slot, 2513 chain_node); 2514 clear_bit(PPC440SPE_DESC_INT, &iter->flags); 2515 hw_desc = iter->hw_desc; 2516 hw_desc->opc = DMA_CDB_OPC_MULTICAST; 2517 2518 ppc440spe_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE, 2519 *dst, 0); 2520 ppc440spe_desc_set_dest_addr(iter, chan, 0, 2521 ppc440spe_chan->qdest, 1); 2522 ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB, 2523 src[1]); 2524 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len); 2525 iter->unmap_len = len; 2526 2527 /* 2nd descriptor, multiply src[1] data and store the 2528 * result in destination */ 2529 iter = list_first_entry(&iter->chain_node, 2530 struct ppc440spe_adma_desc_slot, 2531 chain_node); 2532 memset(iter->hw_desc, 0, sizeof(struct dma_cdb)); 2533 /* set 'next' pointer */ 2534 iter->hw_next = list_entry(iter->chain_node.next, 2535 struct ppc440spe_adma_desc_slot, 2536 chain_node); 2537 if (flags & DMA_PREP_INTERRUPT) 2538 set_bit(PPC440SPE_DESC_INT, &iter->flags); 2539 else 2540 clear_bit(PPC440SPE_DESC_INT, &iter->flags); 2541 2542 hw_desc = iter->hw_desc; 2543 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; 2544 ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB, 2545 ppc440spe_chan->qdest); 2546 ppc440spe_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE, 2547 *dst, 0); 2548 ppc440spe_desc_set_src_mult(iter, chan, DMA_CUED_MULT1_OFF, 2549 DMA_CDB_SG_DST1, scf[1]); 2550 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len); 2551 iter->unmap_len = len; 2552 2553 /* 2554 * 3rd descriptor, multiply src[0] data and xor it 2555 * with destination 2556 */ 2557 iter = list_first_entry(&iter->chain_node, 2558 struct ppc440spe_adma_desc_slot, 2559 chain_node); 2560 memset(iter->hw_desc, 0, sizeof(struct dma_cdb)); 2561 iter->hw_next = NULL; 2562 if (flags & DMA_PREP_INTERRUPT) 2563 set_bit(PPC440SPE_DESC_INT, &iter->flags); 2564 else 2565 clear_bit(PPC440SPE_DESC_INT, &iter->flags); 2566 2567 hw_desc = iter->hw_desc; 2568 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; 2569 ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB, 2570 src[0]); 2571 ppc440spe_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE, 2572 *dst, 0); 2573 ppc440spe_desc_set_src_mult(iter, chan, DMA_CUED_MULT1_OFF, 2574 DMA_CDB_SG_DST1, scf[0]); 2575 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len); 2576 iter->unmap_len = len; 2577 sw_desc->async_tx.flags = flags; 2578 } 2579 2580 spin_unlock_bh(&ppc440spe_chan->lock); 2581 2582 return sw_desc; 2583 } 2584 2585 static struct ppc440spe_adma_desc_slot *ppc440spe_dma01_prep_pq( 2586 struct ppc440spe_adma_chan *ppc440spe_chan, 2587 dma_addr_t *dst, int dst_cnt, dma_addr_t *src, int src_cnt, 2588 const unsigned char *scf, size_t len, unsigned long flags) 2589 { 2590 int slot_cnt; 2591 struct ppc440spe_adma_desc_slot *sw_desc = NULL, *iter; 2592 unsigned long op = 0; 2593 unsigned char mult = 1; 2594 2595 pr_debug("%s: dst_cnt %d, src_cnt %d, len %d\n", 2596 __func__, dst_cnt, src_cnt, len); 2597 /* select operations WXOR/RXOR depending on the 2598 * source addresses of operators and the number 2599 * of destinations (RXOR support only Q-parity calculations) 2600 */ 2601 set_bit(PPC440SPE_DESC_WXOR, &op); 2602 if (!test_and_set_bit(PPC440SPE_RXOR_RUN, &ppc440spe_rxor_state)) { 2603 /* no active RXOR; 2604 * do RXOR if: 2605 * - there are more than 1 source, 2606 * - len is aligned on 512-byte boundary, 2607 * - source addresses fit to one of 4 possible regions. 2608 */ 2609 if (src_cnt > 1 && 2610 !(len & MQ0_CF2H_RXOR_BS_MASK) && 2611 (src[0] + len) == src[1]) { 2612 /* may do RXOR R1 R2 */ 2613 set_bit(PPC440SPE_DESC_RXOR, &op); 2614 if (src_cnt != 2) { 2615 /* may try to enhance region of RXOR */ 2616 if ((src[1] + len) == src[2]) { 2617 /* do RXOR R1 R2 R3 */ 2618 set_bit(PPC440SPE_DESC_RXOR123, 2619 &op); 2620 } else if ((src[1] + len * 2) == src[2]) { 2621 /* do RXOR R1 R2 R4 */ 2622 set_bit(PPC440SPE_DESC_RXOR124, &op); 2623 } else if ((src[1] + len * 3) == src[2]) { 2624 /* do RXOR R1 R2 R5 */ 2625 set_bit(PPC440SPE_DESC_RXOR125, 2626 &op); 2627 } else { 2628 /* do RXOR R1 R2 */ 2629 set_bit(PPC440SPE_DESC_RXOR12, 2630 &op); 2631 } 2632 } else { 2633 /* do RXOR R1 R2 */ 2634 set_bit(PPC440SPE_DESC_RXOR12, &op); 2635 } 2636 } 2637 2638 if (!test_bit(PPC440SPE_DESC_RXOR, &op)) { 2639 /* can not do this operation with RXOR */ 2640 clear_bit(PPC440SPE_RXOR_RUN, 2641 &ppc440spe_rxor_state); 2642 } else { 2643 /* can do; set block size right now */ 2644 ppc440spe_desc_set_rxor_block_size(len); 2645 } 2646 } 2647 2648 /* Number of necessary slots depends on operation type selected */ 2649 if (!test_bit(PPC440SPE_DESC_RXOR, &op)) { 2650 /* This is a WXOR only chain. Need descriptors for each 2651 * source to GF-XOR them with WXOR, and need descriptors 2652 * for each destination to zero them with WXOR 2653 */ 2654 slot_cnt = src_cnt; 2655 2656 if (flags & DMA_PREP_ZERO_P) { 2657 slot_cnt++; 2658 set_bit(PPC440SPE_ZERO_P, &op); 2659 } 2660 if (flags & DMA_PREP_ZERO_Q) { 2661 slot_cnt++; 2662 set_bit(PPC440SPE_ZERO_Q, &op); 2663 } 2664 } else { 2665 /* Need 1/2 descriptor for RXOR operation, and 2666 * need (src_cnt - (2 or 3)) for WXOR of sources 2667 * remained (if any) 2668 */ 2669 slot_cnt = dst_cnt; 2670 2671 if (flags & DMA_PREP_ZERO_P) 2672 set_bit(PPC440SPE_ZERO_P, &op); 2673 if (flags & DMA_PREP_ZERO_Q) 2674 set_bit(PPC440SPE_ZERO_Q, &op); 2675 2676 if (test_bit(PPC440SPE_DESC_RXOR12, &op)) 2677 slot_cnt += src_cnt - 2; 2678 else 2679 slot_cnt += src_cnt - 3; 2680 2681 /* Thus we have either RXOR only chain or 2682 * mixed RXOR/WXOR 2683 */ 2684 if (slot_cnt == dst_cnt) 2685 /* RXOR only chain */ 2686 clear_bit(PPC440SPE_DESC_WXOR, &op); 2687 } 2688 2689 spin_lock_bh(&ppc440spe_chan->lock); 2690 /* for both RXOR/WXOR each descriptor occupies one slot */ 2691 sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1); 2692 if (sw_desc) { 2693 ppc440spe_desc_init_dma01pq(sw_desc, dst_cnt, src_cnt, 2694 flags, op); 2695 2696 /* setup dst/src/mult */ 2697 pr_debug("%s: set dst descriptor 0, 1: 0x%016llx, 0x%016llx\n", 2698 __func__, dst[0], dst[1]); 2699 ppc440spe_adma_pq_set_dest(sw_desc, dst, flags); 2700 while (src_cnt--) { 2701 ppc440spe_adma_pq_set_src(sw_desc, src[src_cnt], 2702 src_cnt); 2703 2704 /* NOTE: "Multi = 0 is equivalent to = 1" as it 2705 * stated in 440SPSPe_RAID6_Addendum_UM_1_17.pdf 2706 * doesn't work for RXOR with DMA0/1! Instead, multi=0 2707 * leads to zeroing source data after RXOR. 2708 * So, for P case set-up mult=1 explicitly. 2709 */ 2710 if (!(flags & DMA_PREP_PQ_DISABLE_Q)) 2711 mult = scf[src_cnt]; 2712 ppc440spe_adma_pq_set_src_mult(sw_desc, 2713 mult, src_cnt, dst_cnt - 1); 2714 } 2715 2716 /* Setup byte count foreach slot just allocated */ 2717 sw_desc->async_tx.flags = flags; 2718 list_for_each_entry(iter, &sw_desc->group_list, 2719 chain_node) { 2720 ppc440spe_desc_set_byte_count(iter, 2721 ppc440spe_chan, len); 2722 iter->unmap_len = len; 2723 } 2724 } 2725 spin_unlock_bh(&ppc440spe_chan->lock); 2726 2727 return sw_desc; 2728 } 2729 2730 static struct ppc440spe_adma_desc_slot *ppc440spe_dma2_prep_pq( 2731 struct ppc440spe_adma_chan *ppc440spe_chan, 2732 dma_addr_t *dst, int dst_cnt, dma_addr_t *src, int src_cnt, 2733 const unsigned char *scf, size_t len, unsigned long flags) 2734 { 2735 int slot_cnt, descs_per_op; 2736 struct ppc440spe_adma_desc_slot *sw_desc = NULL, *iter; 2737 unsigned long op = 0; 2738 unsigned char mult = 1; 2739 2740 BUG_ON(!dst_cnt); 2741 /*pr_debug("%s: dst_cnt %d, src_cnt %d, len %d\n", 2742 __func__, dst_cnt, src_cnt, len);*/ 2743 2744 spin_lock_bh(&ppc440spe_chan->lock); 2745 descs_per_op = ppc440spe_dma2_pq_slot_count(src, src_cnt, len); 2746 if (descs_per_op < 0) { 2747 spin_unlock_bh(&ppc440spe_chan->lock); 2748 return NULL; 2749 } 2750 2751 /* depending on number of sources we have 1 or 2 RXOR chains */ 2752 slot_cnt = descs_per_op * dst_cnt; 2753 2754 sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1); 2755 if (sw_desc) { 2756 op = slot_cnt; 2757 sw_desc->async_tx.flags = flags; 2758 list_for_each_entry(iter, &sw_desc->group_list, chain_node) { 2759 ppc440spe_desc_init_dma2pq(iter, dst_cnt, src_cnt, 2760 --op ? 0 : flags); 2761 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, 2762 len); 2763 iter->unmap_len = len; 2764 2765 ppc440spe_init_rxor_cursor(&(iter->rxor_cursor)); 2766 iter->rxor_cursor.len = len; 2767 iter->descs_per_op = descs_per_op; 2768 } 2769 op = 0; 2770 list_for_each_entry(iter, &sw_desc->group_list, chain_node) { 2771 op++; 2772 if (op % descs_per_op == 0) 2773 ppc440spe_adma_init_dma2rxor_slot(iter, src, 2774 src_cnt); 2775 if (likely(!list_is_last(&iter->chain_node, 2776 &sw_desc->group_list))) { 2777 /* set 'next' pointer */ 2778 iter->hw_next = 2779 list_entry(iter->chain_node.next, 2780 struct ppc440spe_adma_desc_slot, 2781 chain_node); 2782 ppc440spe_xor_set_link(iter, iter->hw_next); 2783 } else { 2784 /* this is the last descriptor. */ 2785 iter->hw_next = NULL; 2786 } 2787 } 2788 2789 /* fixup head descriptor */ 2790 sw_desc->dst_cnt = dst_cnt; 2791 if (flags & DMA_PREP_ZERO_P) 2792 set_bit(PPC440SPE_ZERO_P, &sw_desc->flags); 2793 if (flags & DMA_PREP_ZERO_Q) 2794 set_bit(PPC440SPE_ZERO_Q, &sw_desc->flags); 2795 2796 /* setup dst/src/mult */ 2797 ppc440spe_adma_pq_set_dest(sw_desc, dst, flags); 2798 2799 while (src_cnt--) { 2800 /* handle descriptors (if dst_cnt == 2) inside 2801 * the ppc440spe_adma_pq_set_srcxxx() functions 2802 */ 2803 ppc440spe_adma_pq_set_src(sw_desc, src[src_cnt], 2804 src_cnt); 2805 if (!(flags & DMA_PREP_PQ_DISABLE_Q)) 2806 mult = scf[src_cnt]; 2807 ppc440spe_adma_pq_set_src_mult(sw_desc, 2808 mult, src_cnt, dst_cnt - 1); 2809 } 2810 } 2811 spin_unlock_bh(&ppc440spe_chan->lock); 2812 ppc440spe_desc_set_rxor_block_size(len); 2813 return sw_desc; 2814 } 2815 2816 /** 2817 * ppc440spe_adma_prep_dma_pq - prepare CDB (group) for a GF-XOR operation 2818 */ 2819 static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_pq( 2820 struct dma_chan *chan, dma_addr_t *dst, dma_addr_t *src, 2821 unsigned int src_cnt, const unsigned char *scf, 2822 size_t len, unsigned long flags) 2823 { 2824 struct ppc440spe_adma_chan *ppc440spe_chan; 2825 struct ppc440spe_adma_desc_slot *sw_desc = NULL; 2826 int dst_cnt = 0; 2827 2828 ppc440spe_chan = to_ppc440spe_adma_chan(chan); 2829 2830 ADMA_LL_DBG(prep_dma_pq_dbg(ppc440spe_chan->device->id, 2831 dst, src, src_cnt)); 2832 BUG_ON(!len); 2833 BUG_ON(len > PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT); 2834 BUG_ON(!src_cnt); 2835 2836 if (src_cnt == 1 && dst[1] == src[0]) { 2837 dma_addr_t dest[2]; 2838 2839 /* dst[1] is real destination (Q) */ 2840 dest[0] = dst[1]; 2841 /* this is the page to multicast source data to */ 2842 dest[1] = ppc440spe_chan->qdest; 2843 sw_desc = ppc440spe_dma01_prep_mult(ppc440spe_chan, 2844 dest, 2, src, src_cnt, scf, len, flags); 2845 return sw_desc ? &sw_desc->async_tx : NULL; 2846 } 2847 2848 if (src_cnt == 2 && dst[1] == src[1]) { 2849 sw_desc = ppc440spe_dma01_prep_sum_product(ppc440spe_chan, 2850 &dst[1], src, 2, scf, len, flags); 2851 return sw_desc ? &sw_desc->async_tx : NULL; 2852 } 2853 2854 if (!(flags & DMA_PREP_PQ_DISABLE_P)) { 2855 BUG_ON(!dst[0]); 2856 dst_cnt++; 2857 flags |= DMA_PREP_ZERO_P; 2858 } 2859 2860 if (!(flags & DMA_PREP_PQ_DISABLE_Q)) { 2861 BUG_ON(!dst[1]); 2862 dst_cnt++; 2863 flags |= DMA_PREP_ZERO_Q; 2864 } 2865 2866 BUG_ON(!dst_cnt); 2867 2868 dev_dbg(ppc440spe_chan->device->common.dev, 2869 "ppc440spe adma%d: %s src_cnt: %d len: %u int_en: %d\n", 2870 ppc440spe_chan->device->id, __func__, src_cnt, len, 2871 flags & DMA_PREP_INTERRUPT ? 1 : 0); 2872 2873 switch (ppc440spe_chan->device->id) { 2874 case PPC440SPE_DMA0_ID: 2875 case PPC440SPE_DMA1_ID: 2876 sw_desc = ppc440spe_dma01_prep_pq(ppc440spe_chan, 2877 dst, dst_cnt, src, src_cnt, scf, 2878 len, flags); 2879 break; 2880 2881 case PPC440SPE_XOR_ID: 2882 sw_desc = ppc440spe_dma2_prep_pq(ppc440spe_chan, 2883 dst, dst_cnt, src, src_cnt, scf, 2884 len, flags); 2885 break; 2886 } 2887 2888 return sw_desc ? &sw_desc->async_tx : NULL; 2889 } 2890 2891 /** 2892 * ppc440spe_adma_prep_dma_pqzero_sum - prepare CDB group for 2893 * a PQ_ZERO_SUM operation 2894 */ 2895 static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_pqzero_sum( 2896 struct dma_chan *chan, dma_addr_t *pq, dma_addr_t *src, 2897 unsigned int src_cnt, const unsigned char *scf, size_t len, 2898 enum sum_check_flags *pqres, unsigned long flags) 2899 { 2900 struct ppc440spe_adma_chan *ppc440spe_chan; 2901 struct ppc440spe_adma_desc_slot *sw_desc, *iter; 2902 dma_addr_t pdest, qdest; 2903 int slot_cnt, slots_per_op, idst, dst_cnt; 2904 2905 ppc440spe_chan = to_ppc440spe_adma_chan(chan); 2906 2907 if (flags & DMA_PREP_PQ_DISABLE_P) 2908 pdest = 0; 2909 else 2910 pdest = pq[0]; 2911 2912 if (flags & DMA_PREP_PQ_DISABLE_Q) 2913 qdest = 0; 2914 else 2915 qdest = pq[1]; 2916 2917 ADMA_LL_DBG(prep_dma_pqzero_sum_dbg(ppc440spe_chan->device->id, 2918 src, src_cnt, scf)); 2919 2920 /* Always use WXOR for P/Q calculations (two destinations). 2921 * Need 1 or 2 extra slots to verify results are zero. 2922 */ 2923 idst = dst_cnt = (pdest && qdest) ? 2 : 1; 2924 2925 /* One additional slot per destination to clone P/Q 2926 * before calculation (we have to preserve destinations). 2927 */ 2928 slot_cnt = src_cnt + dst_cnt * 2; 2929 slots_per_op = 1; 2930 2931 spin_lock_bh(&ppc440spe_chan->lock); 2932 sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 2933 slots_per_op); 2934 if (sw_desc) { 2935 ppc440spe_desc_init_dma01pqzero_sum(sw_desc, dst_cnt, src_cnt); 2936 2937 /* Setup byte count for each slot just allocated */ 2938 sw_desc->async_tx.flags = flags; 2939 list_for_each_entry(iter, &sw_desc->group_list, chain_node) { 2940 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, 2941 len); 2942 iter->unmap_len = len; 2943 } 2944 2945 if (pdest) { 2946 struct dma_cdb *hw_desc; 2947 struct ppc440spe_adma_chan *chan; 2948 2949 iter = sw_desc->group_head; 2950 chan = to_ppc440spe_adma_chan(iter->async_tx.chan); 2951 memset(iter->hw_desc, 0, sizeof(struct dma_cdb)); 2952 iter->hw_next = list_entry(iter->chain_node.next, 2953 struct ppc440spe_adma_desc_slot, 2954 chain_node); 2955 hw_desc = iter->hw_desc; 2956 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; 2957 iter->src_cnt = 0; 2958 iter->dst_cnt = 0; 2959 ppc440spe_desc_set_dest_addr(iter, chan, 0, 2960 ppc440spe_chan->pdest, 0); 2961 ppc440spe_desc_set_src_addr(iter, chan, 0, 0, pdest); 2962 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, 2963 len); 2964 iter->unmap_len = 0; 2965 /* override pdest to preserve original P */ 2966 pdest = ppc440spe_chan->pdest; 2967 } 2968 if (qdest) { 2969 struct dma_cdb *hw_desc; 2970 struct ppc440spe_adma_chan *chan; 2971 2972 iter = list_first_entry(&sw_desc->group_list, 2973 struct ppc440spe_adma_desc_slot, 2974 chain_node); 2975 chan = to_ppc440spe_adma_chan(iter->async_tx.chan); 2976 2977 if (pdest) { 2978 iter = list_entry(iter->chain_node.next, 2979 struct ppc440spe_adma_desc_slot, 2980 chain_node); 2981 } 2982 2983 memset(iter->hw_desc, 0, sizeof(struct dma_cdb)); 2984 iter->hw_next = list_entry(iter->chain_node.next, 2985 struct ppc440spe_adma_desc_slot, 2986 chain_node); 2987 hw_desc = iter->hw_desc; 2988 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; 2989 iter->src_cnt = 0; 2990 iter->dst_cnt = 0; 2991 ppc440spe_desc_set_dest_addr(iter, chan, 0, 2992 ppc440spe_chan->qdest, 0); 2993 ppc440spe_desc_set_src_addr(iter, chan, 0, 0, qdest); 2994 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, 2995 len); 2996 iter->unmap_len = 0; 2997 /* override qdest to preserve original Q */ 2998 qdest = ppc440spe_chan->qdest; 2999 } 3000 3001 /* Setup destinations for P/Q ops */ 3002 ppc440spe_adma_pqzero_sum_set_dest(sw_desc, pdest, qdest); 3003 3004 /* Setup zero QWORDs into DCHECK CDBs */ 3005 idst = dst_cnt; 3006 list_for_each_entry_reverse(iter, &sw_desc->group_list, 3007 chain_node) { 3008 /* 3009 * The last CDB corresponds to Q-parity check, 3010 * the one before last CDB corresponds 3011 * P-parity check 3012 */ 3013 if (idst == DMA_DEST_MAX_NUM) { 3014 if (idst == dst_cnt) { 3015 set_bit(PPC440SPE_DESC_QCHECK, 3016 &iter->flags); 3017 } else { 3018 set_bit(PPC440SPE_DESC_PCHECK, 3019 &iter->flags); 3020 } 3021 } else { 3022 if (qdest) { 3023 set_bit(PPC440SPE_DESC_QCHECK, 3024 &iter->flags); 3025 } else { 3026 set_bit(PPC440SPE_DESC_PCHECK, 3027 &iter->flags); 3028 } 3029 } 3030 iter->xor_check_result = pqres; 3031 3032 /* 3033 * set it to zero, if check fail then result will 3034 * be updated 3035 */ 3036 *iter->xor_check_result = 0; 3037 ppc440spe_desc_set_dcheck(iter, ppc440spe_chan, 3038 ppc440spe_qword); 3039 3040 if (!(--dst_cnt)) 3041 break; 3042 } 3043 3044 /* Setup sources and mults for P/Q ops */ 3045 list_for_each_entry_continue_reverse(iter, &sw_desc->group_list, 3046 chain_node) { 3047 struct ppc440spe_adma_chan *chan; 3048 u32 mult_dst; 3049 3050 chan = to_ppc440spe_adma_chan(iter->async_tx.chan); 3051 ppc440spe_desc_set_src_addr(iter, chan, 0, 3052 DMA_CUED_XOR_HB, 3053 src[src_cnt - 1]); 3054 if (qdest) { 3055 mult_dst = (dst_cnt - 1) ? DMA_CDB_SG_DST2 : 3056 DMA_CDB_SG_DST1; 3057 ppc440spe_desc_set_src_mult(iter, chan, 3058 DMA_CUED_MULT1_OFF, 3059 mult_dst, 3060 scf[src_cnt - 1]); 3061 } 3062 if (!(--src_cnt)) 3063 break; 3064 } 3065 } 3066 spin_unlock_bh(&ppc440spe_chan->lock); 3067 return sw_desc ? &sw_desc->async_tx : NULL; 3068 } 3069 3070 /** 3071 * ppc440spe_adma_prep_dma_xor_zero_sum - prepare CDB group for 3072 * XOR ZERO_SUM operation 3073 */ 3074 static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_xor_zero_sum( 3075 struct dma_chan *chan, dma_addr_t *src, unsigned int src_cnt, 3076 size_t len, enum sum_check_flags *result, unsigned long flags) 3077 { 3078 struct dma_async_tx_descriptor *tx; 3079 dma_addr_t pq[2]; 3080 3081 /* validate P, disable Q */ 3082 pq[0] = src[0]; 3083 pq[1] = 0; 3084 flags |= DMA_PREP_PQ_DISABLE_Q; 3085 3086 tx = ppc440spe_adma_prep_dma_pqzero_sum(chan, pq, &src[1], 3087 src_cnt - 1, 0, len, 3088 result, flags); 3089 return tx; 3090 } 3091 3092 /** 3093 * ppc440spe_adma_set_dest - set destination address into descriptor 3094 */ 3095 static void ppc440spe_adma_set_dest(struct ppc440spe_adma_desc_slot *sw_desc, 3096 dma_addr_t addr, int index) 3097 { 3098 struct ppc440spe_adma_chan *chan; 3099 3100 BUG_ON(index >= sw_desc->dst_cnt); 3101 3102 chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan); 3103 3104 switch (chan->device->id) { 3105 case PPC440SPE_DMA0_ID: 3106 case PPC440SPE_DMA1_ID: 3107 /* to do: support transfers lengths > 3108 * PPC440SPE_ADMA_DMA/XOR_MAX_BYTE_COUNT 3109 */ 3110 ppc440spe_desc_set_dest_addr(sw_desc->group_head, 3111 chan, 0, addr, index); 3112 break; 3113 case PPC440SPE_XOR_ID: 3114 sw_desc = ppc440spe_get_group_entry(sw_desc, index); 3115 ppc440spe_desc_set_dest_addr(sw_desc, 3116 chan, 0, addr, index); 3117 break; 3118 } 3119 } 3120 3121 static void ppc440spe_adma_pq_zero_op(struct ppc440spe_adma_desc_slot *iter, 3122 struct ppc440spe_adma_chan *chan, dma_addr_t addr) 3123 { 3124 /* To clear destinations update the descriptor 3125 * (P or Q depending on index) as follows: 3126 * addr is destination (0 corresponds to SG2): 3127 */ 3128 ppc440spe_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE, addr, 0); 3129 3130 /* ... and the addr is source: */ 3131 ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB, addr); 3132 3133 /* addr is always SG2 then the mult is always DST1 */ 3134 ppc440spe_desc_set_src_mult(iter, chan, DMA_CUED_MULT1_OFF, 3135 DMA_CDB_SG_DST1, 1); 3136 } 3137 3138 /** 3139 * ppc440spe_adma_pq_set_dest - set destination address into descriptor 3140 * for the PQXOR operation 3141 */ 3142 static void ppc440spe_adma_pq_set_dest(struct ppc440spe_adma_desc_slot *sw_desc, 3143 dma_addr_t *addrs, unsigned long flags) 3144 { 3145 struct ppc440spe_adma_desc_slot *iter; 3146 struct ppc440spe_adma_chan *chan; 3147 dma_addr_t paddr, qaddr; 3148 dma_addr_t addr = 0, ppath, qpath; 3149 int index = 0, i; 3150 3151 chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan); 3152 3153 if (flags & DMA_PREP_PQ_DISABLE_P) 3154 paddr = 0; 3155 else 3156 paddr = addrs[0]; 3157 3158 if (flags & DMA_PREP_PQ_DISABLE_Q) 3159 qaddr = 0; 3160 else 3161 qaddr = addrs[1]; 3162 3163 if (!paddr || !qaddr) 3164 addr = paddr ? paddr : qaddr; 3165 3166 switch (chan->device->id) { 3167 case PPC440SPE_DMA0_ID: 3168 case PPC440SPE_DMA1_ID: 3169 /* walk through the WXOR source list and set P/Q-destinations 3170 * for each slot: 3171 */ 3172 if (!test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags)) { 3173 /* This is WXOR-only chain; may have 1/2 zero descs */ 3174 if (test_bit(PPC440SPE_ZERO_P, &sw_desc->flags)) 3175 index++; 3176 if (test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags)) 3177 index++; 3178 3179 iter = ppc440spe_get_group_entry(sw_desc, index); 3180 if (addr) { 3181 /* one destination */ 3182 list_for_each_entry_from(iter, 3183 &sw_desc->group_list, chain_node) 3184 ppc440spe_desc_set_dest_addr(iter, chan, 3185 DMA_CUED_XOR_BASE, addr, 0); 3186 } else { 3187 /* two destinations */ 3188 list_for_each_entry_from(iter, 3189 &sw_desc->group_list, chain_node) { 3190 ppc440spe_desc_set_dest_addr(iter, chan, 3191 DMA_CUED_XOR_BASE, paddr, 0); 3192 ppc440spe_desc_set_dest_addr(iter, chan, 3193 DMA_CUED_XOR_BASE, qaddr, 1); 3194 } 3195 } 3196 3197 if (index) { 3198 /* To clear destinations update the descriptor 3199 * (1st,2nd, or both depending on flags) 3200 */ 3201 index = 0; 3202 if (test_bit(PPC440SPE_ZERO_P, 3203 &sw_desc->flags)) { 3204 iter = ppc440spe_get_group_entry( 3205 sw_desc, index++); 3206 ppc440spe_adma_pq_zero_op(iter, chan, 3207 paddr); 3208 } 3209 3210 if (test_bit(PPC440SPE_ZERO_Q, 3211 &sw_desc->flags)) { 3212 iter = ppc440spe_get_group_entry( 3213 sw_desc, index++); 3214 ppc440spe_adma_pq_zero_op(iter, chan, 3215 qaddr); 3216 } 3217 3218 return; 3219 } 3220 } else { 3221 /* This is RXOR-only or RXOR/WXOR mixed chain */ 3222 3223 /* If we want to include destination into calculations, 3224 * then make dest addresses cued with mult=1 (XOR). 3225 */ 3226 ppath = test_bit(PPC440SPE_ZERO_P, &sw_desc->flags) ? 3227 DMA_CUED_XOR_HB : 3228 DMA_CUED_XOR_BASE | 3229 (1 << DMA_CUED_MULT1_OFF); 3230 qpath = test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags) ? 3231 DMA_CUED_XOR_HB : 3232 DMA_CUED_XOR_BASE | 3233 (1 << DMA_CUED_MULT1_OFF); 3234 3235 /* Setup destination(s) in RXOR slot(s) */ 3236 iter = ppc440spe_get_group_entry(sw_desc, index++); 3237 ppc440spe_desc_set_dest_addr(iter, chan, 3238 paddr ? ppath : qpath, 3239 paddr ? paddr : qaddr, 0); 3240 if (!addr) { 3241 /* two destinations */ 3242 iter = ppc440spe_get_group_entry(sw_desc, 3243 index++); 3244 ppc440spe_desc_set_dest_addr(iter, chan, 3245 qpath, qaddr, 0); 3246 } 3247 3248 if (test_bit(PPC440SPE_DESC_WXOR, &sw_desc->flags)) { 3249 /* Setup destination(s) in remaining WXOR 3250 * slots 3251 */ 3252 iter = ppc440spe_get_group_entry(sw_desc, 3253 index); 3254 if (addr) { 3255 /* one destination */ 3256 list_for_each_entry_from(iter, 3257 &sw_desc->group_list, 3258 chain_node) 3259 ppc440spe_desc_set_dest_addr( 3260 iter, chan, 3261 DMA_CUED_XOR_BASE, 3262 addr, 0); 3263 3264 } else { 3265 /* two destinations */ 3266 list_for_each_entry_from(iter, 3267 &sw_desc->group_list, 3268 chain_node) { 3269 ppc440spe_desc_set_dest_addr( 3270 iter, chan, 3271 DMA_CUED_XOR_BASE, 3272 paddr, 0); 3273 ppc440spe_desc_set_dest_addr( 3274 iter, chan, 3275 DMA_CUED_XOR_BASE, 3276 qaddr, 1); 3277 } 3278 } 3279 } 3280 3281 } 3282 break; 3283 3284 case PPC440SPE_XOR_ID: 3285 /* DMA2 descriptors have only 1 destination, so there are 3286 * two chains - one for each dest. 3287 * If we want to include destination into calculations, 3288 * then make dest addresses cued with mult=1 (XOR). 3289 */ 3290 ppath = test_bit(PPC440SPE_ZERO_P, &sw_desc->flags) ? 3291 DMA_CUED_XOR_HB : 3292 DMA_CUED_XOR_BASE | 3293 (1 << DMA_CUED_MULT1_OFF); 3294 3295 qpath = test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags) ? 3296 DMA_CUED_XOR_HB : 3297 DMA_CUED_XOR_BASE | 3298 (1 << DMA_CUED_MULT1_OFF); 3299 3300 iter = ppc440spe_get_group_entry(sw_desc, 0); 3301 for (i = 0; i < sw_desc->descs_per_op; i++) { 3302 ppc440spe_desc_set_dest_addr(iter, chan, 3303 paddr ? ppath : qpath, 3304 paddr ? paddr : qaddr, 0); 3305 iter = list_entry(iter->chain_node.next, 3306 struct ppc440spe_adma_desc_slot, 3307 chain_node); 3308 } 3309 3310 if (!addr) { 3311 /* Two destinations; setup Q here */ 3312 iter = ppc440spe_get_group_entry(sw_desc, 3313 sw_desc->descs_per_op); 3314 for (i = 0; i < sw_desc->descs_per_op; i++) { 3315 ppc440spe_desc_set_dest_addr(iter, 3316 chan, qpath, qaddr, 0); 3317 iter = list_entry(iter->chain_node.next, 3318 struct ppc440spe_adma_desc_slot, 3319 chain_node); 3320 } 3321 } 3322 3323 break; 3324 } 3325 } 3326 3327 /** 3328 * ppc440spe_adma_pq_zero_sum_set_dest - set destination address into descriptor 3329 * for the PQ_ZERO_SUM operation 3330 */ 3331 static void ppc440spe_adma_pqzero_sum_set_dest( 3332 struct ppc440spe_adma_desc_slot *sw_desc, 3333 dma_addr_t paddr, dma_addr_t qaddr) 3334 { 3335 struct ppc440spe_adma_desc_slot *iter, *end; 3336 struct ppc440spe_adma_chan *chan; 3337 dma_addr_t addr = 0; 3338 int idx; 3339 3340 chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan); 3341 3342 /* walk through the WXOR source list and set P/Q-destinations 3343 * for each slot 3344 */ 3345 idx = (paddr && qaddr) ? 2 : 1; 3346 /* set end */ 3347 list_for_each_entry_reverse(end, &sw_desc->group_list, 3348 chain_node) { 3349 if (!(--idx)) 3350 break; 3351 } 3352 /* set start */ 3353 idx = (paddr && qaddr) ? 2 : 1; 3354 iter = ppc440spe_get_group_entry(sw_desc, idx); 3355 3356 if (paddr && qaddr) { 3357 /* two destinations */ 3358 list_for_each_entry_from(iter, &sw_desc->group_list, 3359 chain_node) { 3360 if (unlikely(iter == end)) 3361 break; 3362 ppc440spe_desc_set_dest_addr(iter, chan, 3363 DMA_CUED_XOR_BASE, paddr, 0); 3364 ppc440spe_desc_set_dest_addr(iter, chan, 3365 DMA_CUED_XOR_BASE, qaddr, 1); 3366 } 3367 } else { 3368 /* one destination */ 3369 addr = paddr ? paddr : qaddr; 3370 list_for_each_entry_from(iter, &sw_desc->group_list, 3371 chain_node) { 3372 if (unlikely(iter == end)) 3373 break; 3374 ppc440spe_desc_set_dest_addr(iter, chan, 3375 DMA_CUED_XOR_BASE, addr, 0); 3376 } 3377 } 3378 3379 /* The remaining descriptors are DATACHECK. These have no need in 3380 * destination. Actually, these destinations are used there 3381 * as sources for check operation. So, set addr as source. 3382 */ 3383 ppc440spe_desc_set_src_addr(end, chan, 0, 0, addr ? addr : paddr); 3384 3385 if (!addr) { 3386 end = list_entry(end->chain_node.next, 3387 struct ppc440spe_adma_desc_slot, chain_node); 3388 ppc440spe_desc_set_src_addr(end, chan, 0, 0, qaddr); 3389 } 3390 } 3391 3392 /** 3393 * ppc440spe_desc_set_xor_src_cnt - set source count into descriptor 3394 */ 3395 static inline void ppc440spe_desc_set_xor_src_cnt( 3396 struct ppc440spe_adma_desc_slot *desc, 3397 int src_cnt) 3398 { 3399 struct xor_cb *hw_desc = desc->hw_desc; 3400 3401 hw_desc->cbc &= ~XOR_CDCR_OAC_MSK; 3402 hw_desc->cbc |= src_cnt; 3403 } 3404 3405 /** 3406 * ppc440spe_adma_pq_set_src - set source address into descriptor 3407 */ 3408 static void ppc440spe_adma_pq_set_src(struct ppc440spe_adma_desc_slot *sw_desc, 3409 dma_addr_t addr, int index) 3410 { 3411 struct ppc440spe_adma_chan *chan; 3412 dma_addr_t haddr = 0; 3413 struct ppc440spe_adma_desc_slot *iter = NULL; 3414 3415 chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan); 3416 3417 switch (chan->device->id) { 3418 case PPC440SPE_DMA0_ID: 3419 case PPC440SPE_DMA1_ID: 3420 /* DMA0,1 may do: WXOR, RXOR, RXOR+WXORs chain 3421 */ 3422 if (test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags)) { 3423 /* RXOR-only or RXOR/WXOR operation */ 3424 int iskip = test_bit(PPC440SPE_DESC_RXOR12, 3425 &sw_desc->flags) ? 2 : 3; 3426 3427 if (index == 0) { 3428 /* 1st slot (RXOR) */ 3429 /* setup sources region (R1-2-3, R1-2-4, 3430 * or R1-2-5) 3431 */ 3432 if (test_bit(PPC440SPE_DESC_RXOR12, 3433 &sw_desc->flags)) 3434 haddr = DMA_RXOR12 << 3435 DMA_CUED_REGION_OFF; 3436 else if (test_bit(PPC440SPE_DESC_RXOR123, 3437 &sw_desc->flags)) 3438 haddr = DMA_RXOR123 << 3439 DMA_CUED_REGION_OFF; 3440 else if (test_bit(PPC440SPE_DESC_RXOR124, 3441 &sw_desc->flags)) 3442 haddr = DMA_RXOR124 << 3443 DMA_CUED_REGION_OFF; 3444 else if (test_bit(PPC440SPE_DESC_RXOR125, 3445 &sw_desc->flags)) 3446 haddr = DMA_RXOR125 << 3447 DMA_CUED_REGION_OFF; 3448 else 3449 BUG(); 3450 haddr |= DMA_CUED_XOR_BASE; 3451 iter = ppc440spe_get_group_entry(sw_desc, 0); 3452 } else if (index < iskip) { 3453 /* 1st slot (RXOR) 3454 * shall actually set source address only once 3455 * instead of first <iskip> 3456 */ 3457 iter = NULL; 3458 } else { 3459 /* 2nd/3d and next slots (WXOR); 3460 * skip first slot with RXOR 3461 */ 3462 haddr = DMA_CUED_XOR_HB; 3463 iter = ppc440spe_get_group_entry(sw_desc, 3464 index - iskip + sw_desc->dst_cnt); 3465 } 3466 } else { 3467 int znum = 0; 3468 3469 /* WXOR-only operation; skip first slots with 3470 * zeroing destinations 3471 */ 3472 if (test_bit(PPC440SPE_ZERO_P, &sw_desc->flags)) 3473 znum++; 3474 if (test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags)) 3475 znum++; 3476 3477 haddr = DMA_CUED_XOR_HB; 3478 iter = ppc440spe_get_group_entry(sw_desc, 3479 index + znum); 3480 } 3481 3482 if (likely(iter)) { 3483 ppc440spe_desc_set_src_addr(iter, chan, 0, haddr, addr); 3484 3485 if (!index && 3486 test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags) && 3487 sw_desc->dst_cnt == 2) { 3488 /* if we have two destinations for RXOR, then 3489 * setup source in the second descr too 3490 */ 3491 iter = ppc440spe_get_group_entry(sw_desc, 1); 3492 ppc440spe_desc_set_src_addr(iter, chan, 0, 3493 haddr, addr); 3494 } 3495 } 3496 break; 3497 3498 case PPC440SPE_XOR_ID: 3499 /* DMA2 may do Biskup */ 3500 iter = sw_desc->group_head; 3501 if (iter->dst_cnt == 2) { 3502 /* both P & Q calculations required; set P src here */ 3503 ppc440spe_adma_dma2rxor_set_src(iter, index, addr); 3504 3505 /* this is for Q */ 3506 iter = ppc440spe_get_group_entry(sw_desc, 3507 sw_desc->descs_per_op); 3508 } 3509 ppc440spe_adma_dma2rxor_set_src(iter, index, addr); 3510 break; 3511 } 3512 } 3513 3514 /** 3515 * ppc440spe_adma_memcpy_xor_set_src - set source address into descriptor 3516 */ 3517 static void ppc440spe_adma_memcpy_xor_set_src( 3518 struct ppc440spe_adma_desc_slot *sw_desc, 3519 dma_addr_t addr, int index) 3520 { 3521 struct ppc440spe_adma_chan *chan; 3522 3523 chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan); 3524 sw_desc = sw_desc->group_head; 3525 3526 if (likely(sw_desc)) 3527 ppc440spe_desc_set_src_addr(sw_desc, chan, index, 0, addr); 3528 } 3529 3530 /** 3531 * ppc440spe_adma_dma2rxor_inc_addr - 3532 */ 3533 static void ppc440spe_adma_dma2rxor_inc_addr( 3534 struct ppc440spe_adma_desc_slot *desc, 3535 struct ppc440spe_rxor *cursor, int index, int src_cnt) 3536 { 3537 cursor->addr_count++; 3538 if (index == src_cnt - 1) { 3539 ppc440spe_desc_set_xor_src_cnt(desc, cursor->addr_count); 3540 } else if (cursor->addr_count == XOR_MAX_OPS) { 3541 ppc440spe_desc_set_xor_src_cnt(desc, cursor->addr_count); 3542 cursor->addr_count = 0; 3543 cursor->desc_count++; 3544 } 3545 } 3546 3547 /** 3548 * ppc440spe_adma_dma2rxor_prep_src - setup RXOR types in DMA2 CDB 3549 */ 3550 static int ppc440spe_adma_dma2rxor_prep_src( 3551 struct ppc440spe_adma_desc_slot *hdesc, 3552 struct ppc440spe_rxor *cursor, int index, 3553 int src_cnt, u32 addr) 3554 { 3555 int rval = 0; 3556 u32 sign; 3557 struct ppc440spe_adma_desc_slot *desc = hdesc; 3558 int i; 3559 3560 for (i = 0; i < cursor->desc_count; i++) { 3561 desc = list_entry(hdesc->chain_node.next, 3562 struct ppc440spe_adma_desc_slot, 3563 chain_node); 3564 } 3565 3566 switch (cursor->state) { 3567 case 0: 3568 if (addr == cursor->addrl + cursor->len) { 3569 /* direct RXOR */ 3570 cursor->state = 1; 3571 cursor->xor_count++; 3572 if (index == src_cnt-1) { 3573 ppc440spe_rxor_set_region(desc, 3574 cursor->addr_count, 3575 DMA_RXOR12 << DMA_CUED_REGION_OFF); 3576 ppc440spe_adma_dma2rxor_inc_addr( 3577 desc, cursor, index, src_cnt); 3578 } 3579 } else if (cursor->addrl == addr + cursor->len) { 3580 /* reverse RXOR */ 3581 cursor->state = 1; 3582 cursor->xor_count++; 3583 set_bit(cursor->addr_count, &desc->reverse_flags[0]); 3584 if (index == src_cnt-1) { 3585 ppc440spe_rxor_set_region(desc, 3586 cursor->addr_count, 3587 DMA_RXOR12 << DMA_CUED_REGION_OFF); 3588 ppc440spe_adma_dma2rxor_inc_addr( 3589 desc, cursor, index, src_cnt); 3590 } 3591 } else { 3592 printk(KERN_ERR "Cannot build " 3593 "DMA2 RXOR command block.\n"); 3594 BUG(); 3595 } 3596 break; 3597 case 1: 3598 sign = test_bit(cursor->addr_count, 3599 desc->reverse_flags) 3600 ? -1 : 1; 3601 if (index == src_cnt-2 || (sign == -1 3602 && addr != cursor->addrl - 2*cursor->len)) { 3603 cursor->state = 0; 3604 cursor->xor_count = 1; 3605 cursor->addrl = addr; 3606 ppc440spe_rxor_set_region(desc, 3607 cursor->addr_count, 3608 DMA_RXOR12 << DMA_CUED_REGION_OFF); 3609 ppc440spe_adma_dma2rxor_inc_addr( 3610 desc, cursor, index, src_cnt); 3611 } else if (addr == cursor->addrl + 2*sign*cursor->len) { 3612 cursor->state = 2; 3613 cursor->xor_count = 0; 3614 ppc440spe_rxor_set_region(desc, 3615 cursor->addr_count, 3616 DMA_RXOR123 << DMA_CUED_REGION_OFF); 3617 if (index == src_cnt-1) { 3618 ppc440spe_adma_dma2rxor_inc_addr( 3619 desc, cursor, index, src_cnt); 3620 } 3621 } else if (addr == cursor->addrl + 3*cursor->len) { 3622 cursor->state = 2; 3623 cursor->xor_count = 0; 3624 ppc440spe_rxor_set_region(desc, 3625 cursor->addr_count, 3626 DMA_RXOR124 << DMA_CUED_REGION_OFF); 3627 if (index == src_cnt-1) { 3628 ppc440spe_adma_dma2rxor_inc_addr( 3629 desc, cursor, index, src_cnt); 3630 } 3631 } else if (addr == cursor->addrl + 4*cursor->len) { 3632 cursor->state = 2; 3633 cursor->xor_count = 0; 3634 ppc440spe_rxor_set_region(desc, 3635 cursor->addr_count, 3636 DMA_RXOR125 << DMA_CUED_REGION_OFF); 3637 if (index == src_cnt-1) { 3638 ppc440spe_adma_dma2rxor_inc_addr( 3639 desc, cursor, index, src_cnt); 3640 } 3641 } else { 3642 cursor->state = 0; 3643 cursor->xor_count = 1; 3644 cursor->addrl = addr; 3645 ppc440spe_rxor_set_region(desc, 3646 cursor->addr_count, 3647 DMA_RXOR12 << DMA_CUED_REGION_OFF); 3648 ppc440spe_adma_dma2rxor_inc_addr( 3649 desc, cursor, index, src_cnt); 3650 } 3651 break; 3652 case 2: 3653 cursor->state = 0; 3654 cursor->addrl = addr; 3655 cursor->xor_count++; 3656 if (index) { 3657 ppc440spe_adma_dma2rxor_inc_addr( 3658 desc, cursor, index, src_cnt); 3659 } 3660 break; 3661 } 3662 3663 return rval; 3664 } 3665 3666 /** 3667 * ppc440spe_adma_dma2rxor_set_src - set RXOR source address; it's assumed that 3668 * ppc440spe_adma_dma2rxor_prep_src() has already done prior this call 3669 */ 3670 static void ppc440spe_adma_dma2rxor_set_src( 3671 struct ppc440spe_adma_desc_slot *desc, 3672 int index, dma_addr_t addr) 3673 { 3674 struct xor_cb *xcb = desc->hw_desc; 3675 int k = 0, op = 0, lop = 0; 3676 3677 /* get the RXOR operand which corresponds to index addr */ 3678 while (op <= index) { 3679 lop = op; 3680 if (k == XOR_MAX_OPS) { 3681 k = 0; 3682 desc = list_entry(desc->chain_node.next, 3683 struct ppc440spe_adma_desc_slot, chain_node); 3684 xcb = desc->hw_desc; 3685 3686 } 3687 if ((xcb->ops[k++].h & (DMA_RXOR12 << DMA_CUED_REGION_OFF)) == 3688 (DMA_RXOR12 << DMA_CUED_REGION_OFF)) 3689 op += 2; 3690 else 3691 op += 3; 3692 } 3693 3694 BUG_ON(k < 1); 3695 3696 if (test_bit(k-1, desc->reverse_flags)) { 3697 /* reverse operand order; put last op in RXOR group */ 3698 if (index == op - 1) 3699 ppc440spe_rxor_set_src(desc, k - 1, addr); 3700 } else { 3701 /* direct operand order; put first op in RXOR group */ 3702 if (index == lop) 3703 ppc440spe_rxor_set_src(desc, k - 1, addr); 3704 } 3705 } 3706 3707 /** 3708 * ppc440spe_adma_dma2rxor_set_mult - set RXOR multipliers; it's assumed that 3709 * ppc440spe_adma_dma2rxor_prep_src() has already done prior this call 3710 */ 3711 static void ppc440spe_adma_dma2rxor_set_mult( 3712 struct ppc440spe_adma_desc_slot *desc, 3713 int index, u8 mult) 3714 { 3715 struct xor_cb *xcb = desc->hw_desc; 3716 int k = 0, op = 0, lop = 0; 3717 3718 /* get the RXOR operand which corresponds to index mult */ 3719 while (op <= index) { 3720 lop = op; 3721 if (k == XOR_MAX_OPS) { 3722 k = 0; 3723 desc = list_entry(desc->chain_node.next, 3724 struct ppc440spe_adma_desc_slot, 3725 chain_node); 3726 xcb = desc->hw_desc; 3727 3728 } 3729 if ((xcb->ops[k++].h & (DMA_RXOR12 << DMA_CUED_REGION_OFF)) == 3730 (DMA_RXOR12 << DMA_CUED_REGION_OFF)) 3731 op += 2; 3732 else 3733 op += 3; 3734 } 3735 3736 BUG_ON(k < 1); 3737 if (test_bit(k-1, desc->reverse_flags)) { 3738 /* reverse order */ 3739 ppc440spe_rxor_set_mult(desc, k - 1, op - index - 1, mult); 3740 } else { 3741 /* direct order */ 3742 ppc440spe_rxor_set_mult(desc, k - 1, index - lop, mult); 3743 } 3744 } 3745 3746 /** 3747 * ppc440spe_init_rxor_cursor - 3748 */ 3749 static void ppc440spe_init_rxor_cursor(struct ppc440spe_rxor *cursor) 3750 { 3751 memset(cursor, 0, sizeof(struct ppc440spe_rxor)); 3752 cursor->state = 2; 3753 } 3754 3755 /** 3756 * ppc440spe_adma_pq_set_src_mult - set multiplication coefficient into 3757 * descriptor for the PQXOR operation 3758 */ 3759 static void ppc440spe_adma_pq_set_src_mult( 3760 struct ppc440spe_adma_desc_slot *sw_desc, 3761 unsigned char mult, int index, int dst_pos) 3762 { 3763 struct ppc440spe_adma_chan *chan; 3764 u32 mult_idx, mult_dst; 3765 struct ppc440spe_adma_desc_slot *iter = NULL, *iter1 = NULL; 3766 3767 chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan); 3768 3769 switch (chan->device->id) { 3770 case PPC440SPE_DMA0_ID: 3771 case PPC440SPE_DMA1_ID: 3772 if (test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags)) { 3773 int region = test_bit(PPC440SPE_DESC_RXOR12, 3774 &sw_desc->flags) ? 2 : 3; 3775 3776 if (index < region) { 3777 /* RXOR multipliers */ 3778 iter = ppc440spe_get_group_entry(sw_desc, 3779 sw_desc->dst_cnt - 1); 3780 if (sw_desc->dst_cnt == 2) 3781 iter1 = ppc440spe_get_group_entry( 3782 sw_desc, 0); 3783 3784 mult_idx = DMA_CUED_MULT1_OFF + (index << 3); 3785 mult_dst = DMA_CDB_SG_SRC; 3786 } else { 3787 /* WXOR multiplier */ 3788 iter = ppc440spe_get_group_entry(sw_desc, 3789 index - region + 3790 sw_desc->dst_cnt); 3791 mult_idx = DMA_CUED_MULT1_OFF; 3792 mult_dst = dst_pos ? DMA_CDB_SG_DST2 : 3793 DMA_CDB_SG_DST1; 3794 } 3795 } else { 3796 int znum = 0; 3797 3798 /* WXOR-only; 3799 * skip first slots with destinations (if ZERO_DST has 3800 * place) 3801 */ 3802 if (test_bit(PPC440SPE_ZERO_P, &sw_desc->flags)) 3803 znum++; 3804 if (test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags)) 3805 znum++; 3806 3807 iter = ppc440spe_get_group_entry(sw_desc, index + znum); 3808 mult_idx = DMA_CUED_MULT1_OFF; 3809 mult_dst = dst_pos ? DMA_CDB_SG_DST2 : DMA_CDB_SG_DST1; 3810 } 3811 3812 if (likely(iter)) { 3813 ppc440spe_desc_set_src_mult(iter, chan, 3814 mult_idx, mult_dst, mult); 3815 3816 if (unlikely(iter1)) { 3817 /* if we have two destinations for RXOR, then 3818 * we've just set Q mult. Set-up P now. 3819 */ 3820 ppc440spe_desc_set_src_mult(iter1, chan, 3821 mult_idx, mult_dst, 1); 3822 } 3823 3824 } 3825 break; 3826 3827 case PPC440SPE_XOR_ID: 3828 iter = sw_desc->group_head; 3829 if (sw_desc->dst_cnt == 2) { 3830 /* both P & Q calculations required; set P mult here */ 3831 ppc440spe_adma_dma2rxor_set_mult(iter, index, 1); 3832 3833 /* and then set Q mult */ 3834 iter = ppc440spe_get_group_entry(sw_desc, 3835 sw_desc->descs_per_op); 3836 } 3837 ppc440spe_adma_dma2rxor_set_mult(iter, index, mult); 3838 break; 3839 } 3840 } 3841 3842 /** 3843 * ppc440spe_adma_free_chan_resources - free the resources allocated 3844 */ 3845 static void ppc440spe_adma_free_chan_resources(struct dma_chan *chan) 3846 { 3847 struct ppc440spe_adma_chan *ppc440spe_chan; 3848 struct ppc440spe_adma_desc_slot *iter, *_iter; 3849 int in_use_descs = 0; 3850 3851 ppc440spe_chan = to_ppc440spe_adma_chan(chan); 3852 ppc440spe_adma_slot_cleanup(ppc440spe_chan); 3853 3854 spin_lock_bh(&ppc440spe_chan->lock); 3855 list_for_each_entry_safe(iter, _iter, &ppc440spe_chan->chain, 3856 chain_node) { 3857 in_use_descs++; 3858 list_del(&iter->chain_node); 3859 } 3860 list_for_each_entry_safe_reverse(iter, _iter, 3861 &ppc440spe_chan->all_slots, slot_node) { 3862 list_del(&iter->slot_node); 3863 kfree(iter); 3864 ppc440spe_chan->slots_allocated--; 3865 } 3866 ppc440spe_chan->last_used = NULL; 3867 3868 dev_dbg(ppc440spe_chan->device->common.dev, 3869 "ppc440spe adma%d %s slots_allocated %d\n", 3870 ppc440spe_chan->device->id, 3871 __func__, ppc440spe_chan->slots_allocated); 3872 spin_unlock_bh(&ppc440spe_chan->lock); 3873 3874 /* one is ok since we left it on there on purpose */ 3875 if (in_use_descs > 1) 3876 printk(KERN_ERR "SPE: Freeing %d in use descriptors!\n", 3877 in_use_descs - 1); 3878 } 3879 3880 /** 3881 * ppc440spe_adma_tx_status - poll the status of an ADMA transaction 3882 * @chan: ADMA channel handle 3883 * @cookie: ADMA transaction identifier 3884 * @txstate: a holder for the current state of the channel 3885 */ 3886 static enum dma_status ppc440spe_adma_tx_status(struct dma_chan *chan, 3887 dma_cookie_t cookie, struct dma_tx_state *txstate) 3888 { 3889 struct ppc440spe_adma_chan *ppc440spe_chan; 3890 enum dma_status ret; 3891 3892 ppc440spe_chan = to_ppc440spe_adma_chan(chan); 3893 ret = dma_cookie_status(chan, cookie, txstate); 3894 if (ret == DMA_SUCCESS) 3895 return ret; 3896 3897 ppc440spe_adma_slot_cleanup(ppc440spe_chan); 3898 3899 return dma_cookie_status(chan, cookie, txstate); 3900 } 3901 3902 /** 3903 * ppc440spe_adma_eot_handler - end of transfer interrupt handler 3904 */ 3905 static irqreturn_t ppc440spe_adma_eot_handler(int irq, void *data) 3906 { 3907 struct ppc440spe_adma_chan *chan = data; 3908 3909 dev_dbg(chan->device->common.dev, 3910 "ppc440spe adma%d: %s\n", chan->device->id, __func__); 3911 3912 tasklet_schedule(&chan->irq_tasklet); 3913 ppc440spe_adma_device_clear_eot_status(chan); 3914 3915 return IRQ_HANDLED; 3916 } 3917 3918 /** 3919 * ppc440spe_adma_err_handler - DMA error interrupt handler; 3920 * do the same things as a eot handler 3921 */ 3922 static irqreturn_t ppc440spe_adma_err_handler(int irq, void *data) 3923 { 3924 struct ppc440spe_adma_chan *chan = data; 3925 3926 dev_dbg(chan->device->common.dev, 3927 "ppc440spe adma%d: %s\n", chan->device->id, __func__); 3928 3929 tasklet_schedule(&chan->irq_tasklet); 3930 ppc440spe_adma_device_clear_eot_status(chan); 3931 3932 return IRQ_HANDLED; 3933 } 3934 3935 /** 3936 * ppc440spe_test_callback - called when test operation has been done 3937 */ 3938 static void ppc440spe_test_callback(void *unused) 3939 { 3940 complete(&ppc440spe_r6_test_comp); 3941 } 3942 3943 /** 3944 * ppc440spe_adma_issue_pending - flush all pending descriptors to h/w 3945 */ 3946 static void ppc440spe_adma_issue_pending(struct dma_chan *chan) 3947 { 3948 struct ppc440spe_adma_chan *ppc440spe_chan; 3949 3950 ppc440spe_chan = to_ppc440spe_adma_chan(chan); 3951 dev_dbg(ppc440spe_chan->device->common.dev, 3952 "ppc440spe adma%d: %s %d \n", ppc440spe_chan->device->id, 3953 __func__, ppc440spe_chan->pending); 3954 3955 if (ppc440spe_chan->pending) { 3956 ppc440spe_chan->pending = 0; 3957 ppc440spe_chan_append(ppc440spe_chan); 3958 } 3959 } 3960 3961 /** 3962 * ppc440spe_chan_start_null_xor - initiate the first XOR operation (DMA engines 3963 * use FIFOs (as opposite to chains used in XOR) so this is a XOR 3964 * specific operation) 3965 */ 3966 static void ppc440spe_chan_start_null_xor(struct ppc440spe_adma_chan *chan) 3967 { 3968 struct ppc440spe_adma_desc_slot *sw_desc, *group_start; 3969 dma_cookie_t cookie; 3970 int slot_cnt, slots_per_op; 3971 3972 dev_dbg(chan->device->common.dev, 3973 "ppc440spe adma%d: %s\n", chan->device->id, __func__); 3974 3975 spin_lock_bh(&chan->lock); 3976 slot_cnt = ppc440spe_chan_xor_slot_count(0, 2, &slots_per_op); 3977 sw_desc = ppc440spe_adma_alloc_slots(chan, slot_cnt, slots_per_op); 3978 if (sw_desc) { 3979 group_start = sw_desc->group_head; 3980 list_splice_init(&sw_desc->group_list, &chan->chain); 3981 async_tx_ack(&sw_desc->async_tx); 3982 ppc440spe_desc_init_null_xor(group_start); 3983 3984 cookie = dma_cookie_assign(&sw_desc->async_tx); 3985 3986 /* initialize the completed cookie to be less than 3987 * the most recently used cookie 3988 */ 3989 chan->common.completed_cookie = cookie - 1; 3990 3991 /* channel should not be busy */ 3992 BUG_ON(ppc440spe_chan_is_busy(chan)); 3993 3994 /* set the descriptor address */ 3995 ppc440spe_chan_set_first_xor_descriptor(chan, sw_desc); 3996 3997 /* run the descriptor */ 3998 ppc440spe_chan_run(chan); 3999 } else 4000 printk(KERN_ERR "ppc440spe adma%d" 4001 " failed to allocate null descriptor\n", 4002 chan->device->id); 4003 spin_unlock_bh(&chan->lock); 4004 } 4005 4006 /** 4007 * ppc440spe_test_raid6 - test are RAID-6 capabilities enabled successfully. 4008 * For this we just perform one WXOR operation with the same source 4009 * and destination addresses, the GF-multiplier is 1; so if RAID-6 4010 * capabilities are enabled then we'll get src/dst filled with zero. 4011 */ 4012 static int ppc440spe_test_raid6(struct ppc440spe_adma_chan *chan) 4013 { 4014 struct ppc440spe_adma_desc_slot *sw_desc, *iter; 4015 struct page *pg; 4016 char *a; 4017 dma_addr_t dma_addr, addrs[2]; 4018 unsigned long op = 0; 4019 int rval = 0; 4020 4021 set_bit(PPC440SPE_DESC_WXOR, &op); 4022 4023 pg = alloc_page(GFP_KERNEL); 4024 if (!pg) 4025 return -ENOMEM; 4026 4027 spin_lock_bh(&chan->lock); 4028 sw_desc = ppc440spe_adma_alloc_slots(chan, 1, 1); 4029 if (sw_desc) { 4030 /* 1 src, 1 dsr, int_ena, WXOR */ 4031 ppc440spe_desc_init_dma01pq(sw_desc, 1, 1, 1, op); 4032 list_for_each_entry(iter, &sw_desc->group_list, chain_node) { 4033 ppc440spe_desc_set_byte_count(iter, chan, PAGE_SIZE); 4034 iter->unmap_len = PAGE_SIZE; 4035 } 4036 } else { 4037 rval = -EFAULT; 4038 spin_unlock_bh(&chan->lock); 4039 goto exit; 4040 } 4041 spin_unlock_bh(&chan->lock); 4042 4043 /* Fill the test page with ones */ 4044 memset(page_address(pg), 0xFF, PAGE_SIZE); 4045 dma_addr = dma_map_page(chan->device->dev, pg, 0, 4046 PAGE_SIZE, DMA_BIDIRECTIONAL); 4047 4048 /* Setup addresses */ 4049 ppc440spe_adma_pq_set_src(sw_desc, dma_addr, 0); 4050 ppc440spe_adma_pq_set_src_mult(sw_desc, 1, 0, 0); 4051 addrs[0] = dma_addr; 4052 addrs[1] = 0; 4053 ppc440spe_adma_pq_set_dest(sw_desc, addrs, DMA_PREP_PQ_DISABLE_Q); 4054 4055 async_tx_ack(&sw_desc->async_tx); 4056 sw_desc->async_tx.callback = ppc440spe_test_callback; 4057 sw_desc->async_tx.callback_param = NULL; 4058 4059 init_completion(&ppc440spe_r6_test_comp); 4060 4061 ppc440spe_adma_tx_submit(&sw_desc->async_tx); 4062 ppc440spe_adma_issue_pending(&chan->common); 4063 4064 wait_for_completion(&ppc440spe_r6_test_comp); 4065 4066 /* Now check if the test page is zeroed */ 4067 a = page_address(pg); 4068 if ((*(u32 *)a) == 0 && memcmp(a, a+4, PAGE_SIZE-4) == 0) { 4069 /* page is zero - RAID-6 enabled */ 4070 rval = 0; 4071 } else { 4072 /* RAID-6 was not enabled */ 4073 rval = -EINVAL; 4074 } 4075 exit: 4076 __free_page(pg); 4077 return rval; 4078 } 4079 4080 static void ppc440spe_adma_init_capabilities(struct ppc440spe_adma_device *adev) 4081 { 4082 switch (adev->id) { 4083 case PPC440SPE_DMA0_ID: 4084 case PPC440SPE_DMA1_ID: 4085 dma_cap_set(DMA_MEMCPY, adev->common.cap_mask); 4086 dma_cap_set(DMA_INTERRUPT, adev->common.cap_mask); 4087 dma_cap_set(DMA_PQ, adev->common.cap_mask); 4088 dma_cap_set(DMA_PQ_VAL, adev->common.cap_mask); 4089 dma_cap_set(DMA_XOR_VAL, adev->common.cap_mask); 4090 break; 4091 case PPC440SPE_XOR_ID: 4092 dma_cap_set(DMA_XOR, adev->common.cap_mask); 4093 dma_cap_set(DMA_PQ, adev->common.cap_mask); 4094 dma_cap_set(DMA_INTERRUPT, adev->common.cap_mask); 4095 adev->common.cap_mask = adev->common.cap_mask; 4096 break; 4097 } 4098 4099 /* Set base routines */ 4100 adev->common.device_alloc_chan_resources = 4101 ppc440spe_adma_alloc_chan_resources; 4102 adev->common.device_free_chan_resources = 4103 ppc440spe_adma_free_chan_resources; 4104 adev->common.device_tx_status = ppc440spe_adma_tx_status; 4105 adev->common.device_issue_pending = ppc440spe_adma_issue_pending; 4106 4107 /* Set prep routines based on capability */ 4108 if (dma_has_cap(DMA_MEMCPY, adev->common.cap_mask)) { 4109 adev->common.device_prep_dma_memcpy = 4110 ppc440spe_adma_prep_dma_memcpy; 4111 } 4112 if (dma_has_cap(DMA_XOR, adev->common.cap_mask)) { 4113 adev->common.max_xor = XOR_MAX_OPS; 4114 adev->common.device_prep_dma_xor = 4115 ppc440spe_adma_prep_dma_xor; 4116 } 4117 if (dma_has_cap(DMA_PQ, adev->common.cap_mask)) { 4118 switch (adev->id) { 4119 case PPC440SPE_DMA0_ID: 4120 dma_set_maxpq(&adev->common, 4121 DMA0_FIFO_SIZE / sizeof(struct dma_cdb), 0); 4122 break; 4123 case PPC440SPE_DMA1_ID: 4124 dma_set_maxpq(&adev->common, 4125 DMA1_FIFO_SIZE / sizeof(struct dma_cdb), 0); 4126 break; 4127 case PPC440SPE_XOR_ID: 4128 adev->common.max_pq = XOR_MAX_OPS * 3; 4129 break; 4130 } 4131 adev->common.device_prep_dma_pq = 4132 ppc440spe_adma_prep_dma_pq; 4133 } 4134 if (dma_has_cap(DMA_PQ_VAL, adev->common.cap_mask)) { 4135 switch (adev->id) { 4136 case PPC440SPE_DMA0_ID: 4137 adev->common.max_pq = DMA0_FIFO_SIZE / 4138 sizeof(struct dma_cdb); 4139 break; 4140 case PPC440SPE_DMA1_ID: 4141 adev->common.max_pq = DMA1_FIFO_SIZE / 4142 sizeof(struct dma_cdb); 4143 break; 4144 } 4145 adev->common.device_prep_dma_pq_val = 4146 ppc440spe_adma_prep_dma_pqzero_sum; 4147 } 4148 if (dma_has_cap(DMA_XOR_VAL, adev->common.cap_mask)) { 4149 switch (adev->id) { 4150 case PPC440SPE_DMA0_ID: 4151 adev->common.max_xor = DMA0_FIFO_SIZE / 4152 sizeof(struct dma_cdb); 4153 break; 4154 case PPC440SPE_DMA1_ID: 4155 adev->common.max_xor = DMA1_FIFO_SIZE / 4156 sizeof(struct dma_cdb); 4157 break; 4158 } 4159 adev->common.device_prep_dma_xor_val = 4160 ppc440spe_adma_prep_dma_xor_zero_sum; 4161 } 4162 if (dma_has_cap(DMA_INTERRUPT, adev->common.cap_mask)) { 4163 adev->common.device_prep_dma_interrupt = 4164 ppc440spe_adma_prep_dma_interrupt; 4165 } 4166 pr_info("%s: AMCC(R) PPC440SP(E) ADMA Engine: " 4167 "( %s%s%s%s%s%s%s)\n", 4168 dev_name(adev->dev), 4169 dma_has_cap(DMA_PQ, adev->common.cap_mask) ? "pq " : "", 4170 dma_has_cap(DMA_PQ_VAL, adev->common.cap_mask) ? "pq_val " : "", 4171 dma_has_cap(DMA_XOR, adev->common.cap_mask) ? "xor " : "", 4172 dma_has_cap(DMA_XOR_VAL, adev->common.cap_mask) ? "xor_val " : "", 4173 dma_has_cap(DMA_MEMCPY, adev->common.cap_mask) ? "memcpy " : "", 4174 dma_has_cap(DMA_INTERRUPT, adev->common.cap_mask) ? "intr " : ""); 4175 } 4176 4177 static int ppc440spe_adma_setup_irqs(struct ppc440spe_adma_device *adev, 4178 struct ppc440spe_adma_chan *chan, 4179 int *initcode) 4180 { 4181 struct platform_device *ofdev; 4182 struct device_node *np; 4183 int ret; 4184 4185 ofdev = container_of(adev->dev, struct platform_device, dev); 4186 np = ofdev->dev.of_node; 4187 if (adev->id != PPC440SPE_XOR_ID) { 4188 adev->err_irq = irq_of_parse_and_map(np, 1); 4189 if (adev->err_irq == NO_IRQ) { 4190 dev_warn(adev->dev, "no err irq resource?\n"); 4191 *initcode = PPC_ADMA_INIT_IRQ2; 4192 adev->err_irq = -ENXIO; 4193 } else 4194 atomic_inc(&ppc440spe_adma_err_irq_ref); 4195 } else { 4196 adev->err_irq = -ENXIO; 4197 } 4198 4199 adev->irq = irq_of_parse_and_map(np, 0); 4200 if (adev->irq == NO_IRQ) { 4201 dev_err(adev->dev, "no irq resource\n"); 4202 *initcode = PPC_ADMA_INIT_IRQ1; 4203 ret = -ENXIO; 4204 goto err_irq_map; 4205 } 4206 dev_dbg(adev->dev, "irq %d, err irq %d\n", 4207 adev->irq, adev->err_irq); 4208 4209 ret = request_irq(adev->irq, ppc440spe_adma_eot_handler, 4210 0, dev_driver_string(adev->dev), chan); 4211 if (ret) { 4212 dev_err(adev->dev, "can't request irq %d\n", 4213 adev->irq); 4214 *initcode = PPC_ADMA_INIT_IRQ1; 4215 ret = -EIO; 4216 goto err_req1; 4217 } 4218 4219 /* only DMA engines have a separate error IRQ 4220 * so it's Ok if err_irq < 0 in XOR engine case. 4221 */ 4222 if (adev->err_irq > 0) { 4223 /* both DMA engines share common error IRQ */ 4224 ret = request_irq(adev->err_irq, 4225 ppc440spe_adma_err_handler, 4226 IRQF_SHARED, 4227 dev_driver_string(adev->dev), 4228 chan); 4229 if (ret) { 4230 dev_err(adev->dev, "can't request irq %d\n", 4231 adev->err_irq); 4232 *initcode = PPC_ADMA_INIT_IRQ2; 4233 ret = -EIO; 4234 goto err_req2; 4235 } 4236 } 4237 4238 if (adev->id == PPC440SPE_XOR_ID) { 4239 /* enable XOR engine interrupts */ 4240 iowrite32be(XOR_IE_CBCIE_BIT | XOR_IE_ICBIE_BIT | 4241 XOR_IE_ICIE_BIT | XOR_IE_RPTIE_BIT, 4242 &adev->xor_reg->ier); 4243 } else { 4244 u32 mask, enable; 4245 4246 np = of_find_compatible_node(NULL, NULL, "ibm,i2o-440spe"); 4247 if (!np) { 4248 pr_err("%s: can't find I2O device tree node\n", 4249 __func__); 4250 ret = -ENODEV; 4251 goto err_req2; 4252 } 4253 adev->i2o_reg = of_iomap(np, 0); 4254 if (!adev->i2o_reg) { 4255 pr_err("%s: failed to map I2O registers\n", __func__); 4256 of_node_put(np); 4257 ret = -EINVAL; 4258 goto err_req2; 4259 } 4260 of_node_put(np); 4261 /* Unmask 'CS FIFO Attention' interrupts and 4262 * enable generating interrupts on errors 4263 */ 4264 enable = (adev->id == PPC440SPE_DMA0_ID) ? 4265 ~(I2O_IOPIM_P0SNE | I2O_IOPIM_P0EM) : 4266 ~(I2O_IOPIM_P1SNE | I2O_IOPIM_P1EM); 4267 mask = ioread32(&adev->i2o_reg->iopim) & enable; 4268 iowrite32(mask, &adev->i2o_reg->iopim); 4269 } 4270 return 0; 4271 4272 err_req2: 4273 free_irq(adev->irq, chan); 4274 err_req1: 4275 irq_dispose_mapping(adev->irq); 4276 err_irq_map: 4277 if (adev->err_irq > 0) { 4278 if (atomic_dec_and_test(&ppc440spe_adma_err_irq_ref)) 4279 irq_dispose_mapping(adev->err_irq); 4280 } 4281 return ret; 4282 } 4283 4284 static void ppc440spe_adma_release_irqs(struct ppc440spe_adma_device *adev, 4285 struct ppc440spe_adma_chan *chan) 4286 { 4287 u32 mask, disable; 4288 4289 if (adev->id == PPC440SPE_XOR_ID) { 4290 /* disable XOR engine interrupts */ 4291 mask = ioread32be(&adev->xor_reg->ier); 4292 mask &= ~(XOR_IE_CBCIE_BIT | XOR_IE_ICBIE_BIT | 4293 XOR_IE_ICIE_BIT | XOR_IE_RPTIE_BIT); 4294 iowrite32be(mask, &adev->xor_reg->ier); 4295 } else { 4296 /* disable DMAx engine interrupts */ 4297 disable = (adev->id == PPC440SPE_DMA0_ID) ? 4298 (I2O_IOPIM_P0SNE | I2O_IOPIM_P0EM) : 4299 (I2O_IOPIM_P1SNE | I2O_IOPIM_P1EM); 4300 mask = ioread32(&adev->i2o_reg->iopim) | disable; 4301 iowrite32(mask, &adev->i2o_reg->iopim); 4302 } 4303 free_irq(adev->irq, chan); 4304 irq_dispose_mapping(adev->irq); 4305 if (adev->err_irq > 0) { 4306 free_irq(adev->err_irq, chan); 4307 if (atomic_dec_and_test(&ppc440spe_adma_err_irq_ref)) { 4308 irq_dispose_mapping(adev->err_irq); 4309 iounmap(adev->i2o_reg); 4310 } 4311 } 4312 } 4313 4314 /** 4315 * ppc440spe_adma_probe - probe the asynch device 4316 */ 4317 static int ppc440spe_adma_probe(struct platform_device *ofdev) 4318 { 4319 struct device_node *np = ofdev->dev.of_node; 4320 struct resource res; 4321 struct ppc440spe_adma_device *adev; 4322 struct ppc440spe_adma_chan *chan; 4323 struct ppc_dma_chan_ref *ref, *_ref; 4324 int ret = 0, initcode = PPC_ADMA_INIT_OK; 4325 const u32 *idx; 4326 int len; 4327 void *regs; 4328 u32 id, pool_size; 4329 4330 if (of_device_is_compatible(np, "amcc,xor-accelerator")) { 4331 id = PPC440SPE_XOR_ID; 4332 /* As far as the XOR engine is concerned, it does not 4333 * use FIFOs but uses linked list. So there is no dependency 4334 * between pool size to allocate and the engine configuration. 4335 */ 4336 pool_size = PAGE_SIZE << 1; 4337 } else { 4338 /* it is DMA0 or DMA1 */ 4339 idx = of_get_property(np, "cell-index", &len); 4340 if (!idx || (len != sizeof(u32))) { 4341 dev_err(&ofdev->dev, "Device node %s has missing " 4342 "or invalid cell-index property\n", 4343 np->full_name); 4344 return -EINVAL; 4345 } 4346 id = *idx; 4347 /* DMA0,1 engines use FIFO to maintain CDBs, so we 4348 * should allocate the pool accordingly to size of this 4349 * FIFO. Thus, the pool size depends on the FIFO depth: 4350 * how much CDBs pointers the FIFO may contain then so 4351 * much CDBs we should provide in the pool. 4352 * That is 4353 * CDB size = 32B; 4354 * CDBs number = (DMA0_FIFO_SIZE >> 3); 4355 * Pool size = CDBs number * CDB size = 4356 * = (DMA0_FIFO_SIZE >> 3) << 5 = DMA0_FIFO_SIZE << 2. 4357 */ 4358 pool_size = (id == PPC440SPE_DMA0_ID) ? 4359 DMA0_FIFO_SIZE : DMA1_FIFO_SIZE; 4360 pool_size <<= 2; 4361 } 4362 4363 if (of_address_to_resource(np, 0, &res)) { 4364 dev_err(&ofdev->dev, "failed to get memory resource\n"); 4365 initcode = PPC_ADMA_INIT_MEMRES; 4366 ret = -ENODEV; 4367 goto out; 4368 } 4369 4370 if (!request_mem_region(res.start, resource_size(&res), 4371 dev_driver_string(&ofdev->dev))) { 4372 dev_err(&ofdev->dev, "failed to request memory region %pR\n", 4373 &res); 4374 initcode = PPC_ADMA_INIT_MEMREG; 4375 ret = -EBUSY; 4376 goto out; 4377 } 4378 4379 /* create a device */ 4380 adev = kzalloc(sizeof(*adev), GFP_KERNEL); 4381 if (!adev) { 4382 dev_err(&ofdev->dev, "failed to allocate device\n"); 4383 initcode = PPC_ADMA_INIT_ALLOC; 4384 ret = -ENOMEM; 4385 goto err_adev_alloc; 4386 } 4387 4388 adev->id = id; 4389 adev->pool_size = pool_size; 4390 /* allocate coherent memory for hardware descriptors */ 4391 adev->dma_desc_pool_virt = dma_alloc_coherent(&ofdev->dev, 4392 adev->pool_size, &adev->dma_desc_pool, 4393 GFP_KERNEL); 4394 if (adev->dma_desc_pool_virt == NULL) { 4395 dev_err(&ofdev->dev, "failed to allocate %d bytes of coherent " 4396 "memory for hardware descriptors\n", 4397 adev->pool_size); 4398 initcode = PPC_ADMA_INIT_COHERENT; 4399 ret = -ENOMEM; 4400 goto err_dma_alloc; 4401 } 4402 dev_dbg(&ofdev->dev, "allocated descriptor pool virt 0x%p phys 0x%llx\n", 4403 adev->dma_desc_pool_virt, (u64)adev->dma_desc_pool); 4404 4405 regs = ioremap(res.start, resource_size(&res)); 4406 if (!regs) { 4407 dev_err(&ofdev->dev, "failed to ioremap regs!\n"); 4408 goto err_regs_alloc; 4409 } 4410 4411 if (adev->id == PPC440SPE_XOR_ID) { 4412 adev->xor_reg = regs; 4413 /* Reset XOR */ 4414 iowrite32be(XOR_CRSR_XASR_BIT, &adev->xor_reg->crsr); 4415 iowrite32be(XOR_CRSR_64BA_BIT, &adev->xor_reg->crrr); 4416 } else { 4417 size_t fifo_size = (adev->id == PPC440SPE_DMA0_ID) ? 4418 DMA0_FIFO_SIZE : DMA1_FIFO_SIZE; 4419 adev->dma_reg = regs; 4420 /* DMAx_FIFO_SIZE is defined in bytes, 4421 * <fsiz> - is defined in number of CDB pointers (8byte). 4422 * DMA FIFO Length = CSlength + CPlength, where 4423 * CSlength = CPlength = (fsiz + 1) * 8. 4424 */ 4425 iowrite32(DMA_FIFO_ENABLE | ((fifo_size >> 3) - 2), 4426 &adev->dma_reg->fsiz); 4427 /* Configure DMA engine */ 4428 iowrite32(DMA_CFG_DXEPR_HP | DMA_CFG_DFMPP_HP | DMA_CFG_FALGN, 4429 &adev->dma_reg->cfg); 4430 /* Clear Status */ 4431 iowrite32(~0, &adev->dma_reg->dsts); 4432 } 4433 4434 adev->dev = &ofdev->dev; 4435 adev->common.dev = &ofdev->dev; 4436 INIT_LIST_HEAD(&adev->common.channels); 4437 platform_set_drvdata(ofdev, adev); 4438 4439 /* create a channel */ 4440 chan = kzalloc(sizeof(*chan), GFP_KERNEL); 4441 if (!chan) { 4442 dev_err(&ofdev->dev, "can't allocate channel structure\n"); 4443 initcode = PPC_ADMA_INIT_CHANNEL; 4444 ret = -ENOMEM; 4445 goto err_chan_alloc; 4446 } 4447 4448 spin_lock_init(&chan->lock); 4449 INIT_LIST_HEAD(&chan->chain); 4450 INIT_LIST_HEAD(&chan->all_slots); 4451 chan->device = adev; 4452 chan->common.device = &adev->common; 4453 dma_cookie_init(&chan->common); 4454 list_add_tail(&chan->common.device_node, &adev->common.channels); 4455 tasklet_init(&chan->irq_tasklet, ppc440spe_adma_tasklet, 4456 (unsigned long)chan); 4457 4458 /* allocate and map helper pages for async validation or 4459 * async_mult/async_sum_product operations on DMA0/1. 4460 */ 4461 if (adev->id != PPC440SPE_XOR_ID) { 4462 chan->pdest_page = alloc_page(GFP_KERNEL); 4463 chan->qdest_page = alloc_page(GFP_KERNEL); 4464 if (!chan->pdest_page || 4465 !chan->qdest_page) { 4466 if (chan->pdest_page) 4467 __free_page(chan->pdest_page); 4468 if (chan->qdest_page) 4469 __free_page(chan->qdest_page); 4470 ret = -ENOMEM; 4471 goto err_page_alloc; 4472 } 4473 chan->pdest = dma_map_page(&ofdev->dev, chan->pdest_page, 0, 4474 PAGE_SIZE, DMA_BIDIRECTIONAL); 4475 chan->qdest = dma_map_page(&ofdev->dev, chan->qdest_page, 0, 4476 PAGE_SIZE, DMA_BIDIRECTIONAL); 4477 } 4478 4479 ref = kmalloc(sizeof(*ref), GFP_KERNEL); 4480 if (ref) { 4481 ref->chan = &chan->common; 4482 INIT_LIST_HEAD(&ref->node); 4483 list_add_tail(&ref->node, &ppc440spe_adma_chan_list); 4484 } else { 4485 dev_err(&ofdev->dev, "failed to allocate channel reference!\n"); 4486 ret = -ENOMEM; 4487 goto err_ref_alloc; 4488 } 4489 4490 ret = ppc440spe_adma_setup_irqs(adev, chan, &initcode); 4491 if (ret) 4492 goto err_irq; 4493 4494 ppc440spe_adma_init_capabilities(adev); 4495 4496 ret = dma_async_device_register(&adev->common); 4497 if (ret) { 4498 initcode = PPC_ADMA_INIT_REGISTER; 4499 dev_err(&ofdev->dev, "failed to register dma device\n"); 4500 goto err_dev_reg; 4501 } 4502 4503 goto out; 4504 4505 err_dev_reg: 4506 ppc440spe_adma_release_irqs(adev, chan); 4507 err_irq: 4508 list_for_each_entry_safe(ref, _ref, &ppc440spe_adma_chan_list, node) { 4509 if (chan == to_ppc440spe_adma_chan(ref->chan)) { 4510 list_del(&ref->node); 4511 kfree(ref); 4512 } 4513 } 4514 err_ref_alloc: 4515 if (adev->id != PPC440SPE_XOR_ID) { 4516 dma_unmap_page(&ofdev->dev, chan->pdest, 4517 PAGE_SIZE, DMA_BIDIRECTIONAL); 4518 dma_unmap_page(&ofdev->dev, chan->qdest, 4519 PAGE_SIZE, DMA_BIDIRECTIONAL); 4520 __free_page(chan->pdest_page); 4521 __free_page(chan->qdest_page); 4522 } 4523 err_page_alloc: 4524 kfree(chan); 4525 err_chan_alloc: 4526 if (adev->id == PPC440SPE_XOR_ID) 4527 iounmap(adev->xor_reg); 4528 else 4529 iounmap(adev->dma_reg); 4530 err_regs_alloc: 4531 dma_free_coherent(adev->dev, adev->pool_size, 4532 adev->dma_desc_pool_virt, 4533 adev->dma_desc_pool); 4534 err_dma_alloc: 4535 kfree(adev); 4536 err_adev_alloc: 4537 release_mem_region(res.start, resource_size(&res)); 4538 out: 4539 if (id < PPC440SPE_ADMA_ENGINES_NUM) 4540 ppc440spe_adma_devices[id] = initcode; 4541 4542 return ret; 4543 } 4544 4545 /** 4546 * ppc440spe_adma_remove - remove the asynch device 4547 */ 4548 static int ppc440spe_adma_remove(struct platform_device *ofdev) 4549 { 4550 struct ppc440spe_adma_device *adev = platform_get_drvdata(ofdev); 4551 struct device_node *np = ofdev->dev.of_node; 4552 struct resource res; 4553 struct dma_chan *chan, *_chan; 4554 struct ppc_dma_chan_ref *ref, *_ref; 4555 struct ppc440spe_adma_chan *ppc440spe_chan; 4556 4557 if (adev->id < PPC440SPE_ADMA_ENGINES_NUM) 4558 ppc440spe_adma_devices[adev->id] = -1; 4559 4560 dma_async_device_unregister(&adev->common); 4561 4562 list_for_each_entry_safe(chan, _chan, &adev->common.channels, 4563 device_node) { 4564 ppc440spe_chan = to_ppc440spe_adma_chan(chan); 4565 ppc440spe_adma_release_irqs(adev, ppc440spe_chan); 4566 tasklet_kill(&ppc440spe_chan->irq_tasklet); 4567 if (adev->id != PPC440SPE_XOR_ID) { 4568 dma_unmap_page(&ofdev->dev, ppc440spe_chan->pdest, 4569 PAGE_SIZE, DMA_BIDIRECTIONAL); 4570 dma_unmap_page(&ofdev->dev, ppc440spe_chan->qdest, 4571 PAGE_SIZE, DMA_BIDIRECTIONAL); 4572 __free_page(ppc440spe_chan->pdest_page); 4573 __free_page(ppc440spe_chan->qdest_page); 4574 } 4575 list_for_each_entry_safe(ref, _ref, &ppc440spe_adma_chan_list, 4576 node) { 4577 if (ppc440spe_chan == 4578 to_ppc440spe_adma_chan(ref->chan)) { 4579 list_del(&ref->node); 4580 kfree(ref); 4581 } 4582 } 4583 list_del(&chan->device_node); 4584 kfree(ppc440spe_chan); 4585 } 4586 4587 dma_free_coherent(adev->dev, adev->pool_size, 4588 adev->dma_desc_pool_virt, adev->dma_desc_pool); 4589 if (adev->id == PPC440SPE_XOR_ID) 4590 iounmap(adev->xor_reg); 4591 else 4592 iounmap(adev->dma_reg); 4593 of_address_to_resource(np, 0, &res); 4594 release_mem_region(res.start, resource_size(&res)); 4595 kfree(adev); 4596 return 0; 4597 } 4598 4599 /* 4600 * /sys driver interface to enable h/w RAID-6 capabilities 4601 * Files created in e.g. /sys/devices/plb.0/400100100.dma0/driver/ 4602 * directory are "devices", "enable" and "poly". 4603 * "devices" shows available engines. 4604 * "enable" is used to enable RAID-6 capabilities or to check 4605 * whether these has been activated. 4606 * "poly" allows setting/checking used polynomial (for PPC440SPe only). 4607 */ 4608 4609 static ssize_t show_ppc440spe_devices(struct device_driver *dev, char *buf) 4610 { 4611 ssize_t size = 0; 4612 int i; 4613 4614 for (i = 0; i < PPC440SPE_ADMA_ENGINES_NUM; i++) { 4615 if (ppc440spe_adma_devices[i] == -1) 4616 continue; 4617 size += snprintf(buf + size, PAGE_SIZE - size, 4618 "PPC440SP(E)-ADMA.%d: %s\n", i, 4619 ppc_adma_errors[ppc440spe_adma_devices[i]]); 4620 } 4621 return size; 4622 } 4623 4624 static ssize_t show_ppc440spe_r6enable(struct device_driver *dev, char *buf) 4625 { 4626 return snprintf(buf, PAGE_SIZE, 4627 "PPC440SP(e) RAID-6 capabilities are %sABLED.\n", 4628 ppc440spe_r6_enabled ? "EN" : "DIS"); 4629 } 4630 4631 static ssize_t store_ppc440spe_r6enable(struct device_driver *dev, 4632 const char *buf, size_t count) 4633 { 4634 unsigned long val; 4635 4636 if (!count || count > 11) 4637 return -EINVAL; 4638 4639 if (!ppc440spe_r6_tchan) 4640 return -EFAULT; 4641 4642 /* Write a key */ 4643 sscanf(buf, "%lx", &val); 4644 dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_XORBA, val); 4645 isync(); 4646 4647 /* Verify whether it really works now */ 4648 if (ppc440spe_test_raid6(ppc440spe_r6_tchan) == 0) { 4649 pr_info("PPC440SP(e) RAID-6 has been activated " 4650 "successfully\n"); 4651 ppc440spe_r6_enabled = 1; 4652 } else { 4653 pr_info("PPC440SP(e) RAID-6 hasn't been activated!" 4654 " Error key ?\n"); 4655 ppc440spe_r6_enabled = 0; 4656 } 4657 return count; 4658 } 4659 4660 static ssize_t show_ppc440spe_r6poly(struct device_driver *dev, char *buf) 4661 { 4662 ssize_t size = 0; 4663 u32 reg; 4664 4665 #ifdef CONFIG_440SP 4666 /* 440SP has fixed polynomial */ 4667 reg = 0x4d; 4668 #else 4669 reg = dcr_read(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL); 4670 reg >>= MQ0_CFBHL_POLY; 4671 reg &= 0xFF; 4672 #endif 4673 4674 size = snprintf(buf, PAGE_SIZE, "PPC440SP(e) RAID-6 driver " 4675 "uses 0x1%02x polynomial.\n", reg); 4676 return size; 4677 } 4678 4679 static ssize_t store_ppc440spe_r6poly(struct device_driver *dev, 4680 const char *buf, size_t count) 4681 { 4682 unsigned long reg, val; 4683 4684 #ifdef CONFIG_440SP 4685 /* 440SP uses default 0x14D polynomial only */ 4686 return -EINVAL; 4687 #endif 4688 4689 if (!count || count > 6) 4690 return -EINVAL; 4691 4692 /* e.g., 0x14D or 0x11D */ 4693 sscanf(buf, "%lx", &val); 4694 4695 if (val & ~0x1FF) 4696 return -EINVAL; 4697 4698 val &= 0xFF; 4699 reg = dcr_read(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL); 4700 reg &= ~(0xFF << MQ0_CFBHL_POLY); 4701 reg |= val << MQ0_CFBHL_POLY; 4702 dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL, reg); 4703 4704 return count; 4705 } 4706 4707 static DRIVER_ATTR(devices, S_IRUGO, show_ppc440spe_devices, NULL); 4708 static DRIVER_ATTR(enable, S_IRUGO | S_IWUSR, show_ppc440spe_r6enable, 4709 store_ppc440spe_r6enable); 4710 static DRIVER_ATTR(poly, S_IRUGO | S_IWUSR, show_ppc440spe_r6poly, 4711 store_ppc440spe_r6poly); 4712 4713 /* 4714 * Common initialisation for RAID engines; allocate memory for 4715 * DMAx FIFOs, perform configuration common for all DMA engines. 4716 * Further DMA engine specific configuration is done at probe time. 4717 */ 4718 static int ppc440spe_configure_raid_devices(void) 4719 { 4720 struct device_node *np; 4721 struct resource i2o_res; 4722 struct i2o_regs __iomem *i2o_reg; 4723 dcr_host_t i2o_dcr_host; 4724 unsigned int dcr_base, dcr_len; 4725 int i, ret; 4726 4727 np = of_find_compatible_node(NULL, NULL, "ibm,i2o-440spe"); 4728 if (!np) { 4729 pr_err("%s: can't find I2O device tree node\n", 4730 __func__); 4731 return -ENODEV; 4732 } 4733 4734 if (of_address_to_resource(np, 0, &i2o_res)) { 4735 of_node_put(np); 4736 return -EINVAL; 4737 } 4738 4739 i2o_reg = of_iomap(np, 0); 4740 if (!i2o_reg) { 4741 pr_err("%s: failed to map I2O registers\n", __func__); 4742 of_node_put(np); 4743 return -EINVAL; 4744 } 4745 4746 /* Get I2O DCRs base */ 4747 dcr_base = dcr_resource_start(np, 0); 4748 dcr_len = dcr_resource_len(np, 0); 4749 if (!dcr_base && !dcr_len) { 4750 pr_err("%s: can't get DCR registers base/len!\n", 4751 np->full_name); 4752 of_node_put(np); 4753 iounmap(i2o_reg); 4754 return -ENODEV; 4755 } 4756 4757 i2o_dcr_host = dcr_map(np, dcr_base, dcr_len); 4758 if (!DCR_MAP_OK(i2o_dcr_host)) { 4759 pr_err("%s: failed to map DCRs!\n", np->full_name); 4760 of_node_put(np); 4761 iounmap(i2o_reg); 4762 return -ENODEV; 4763 } 4764 of_node_put(np); 4765 4766 /* Provide memory regions for DMA's FIFOs: I2O, DMA0 and DMA1 share 4767 * the base address of FIFO memory space. 4768 * Actually we need twice more physical memory than programmed in the 4769 * <fsiz> register (because there are two FIFOs for each DMA: CP and CS) 4770 */ 4771 ppc440spe_dma_fifo_buf = kmalloc((DMA0_FIFO_SIZE + DMA1_FIFO_SIZE) << 1, 4772 GFP_KERNEL); 4773 if (!ppc440spe_dma_fifo_buf) { 4774 pr_err("%s: DMA FIFO buffer allocation failed.\n", __func__); 4775 iounmap(i2o_reg); 4776 dcr_unmap(i2o_dcr_host, dcr_len); 4777 return -ENOMEM; 4778 } 4779 4780 /* 4781 * Configure h/w 4782 */ 4783 /* Reset I2O/DMA */ 4784 mtdcri(SDR0, DCRN_SDR0_SRST, DCRN_SDR0_SRST_I2ODMA); 4785 mtdcri(SDR0, DCRN_SDR0_SRST, 0); 4786 4787 /* Setup the base address of mmaped registers */ 4788 dcr_write(i2o_dcr_host, DCRN_I2O0_IBAH, (u32)(i2o_res.start >> 32)); 4789 dcr_write(i2o_dcr_host, DCRN_I2O0_IBAL, (u32)(i2o_res.start) | 4790 I2O_REG_ENABLE); 4791 dcr_unmap(i2o_dcr_host, dcr_len); 4792 4793 /* Setup FIFO memory space base address */ 4794 iowrite32(0, &i2o_reg->ifbah); 4795 iowrite32(((u32)__pa(ppc440spe_dma_fifo_buf)), &i2o_reg->ifbal); 4796 4797 /* set zero FIFO size for I2O, so the whole 4798 * ppc440spe_dma_fifo_buf is used by DMAs. 4799 * DMAx_FIFOs will be configured while probe. 4800 */ 4801 iowrite32(0, &i2o_reg->ifsiz); 4802 iounmap(i2o_reg); 4803 4804 /* To prepare WXOR/RXOR functionality we need access to 4805 * Memory Queue Module DCRs (finally it will be enabled 4806 * via /sys interface of the ppc440spe ADMA driver). 4807 */ 4808 np = of_find_compatible_node(NULL, NULL, "ibm,mq-440spe"); 4809 if (!np) { 4810 pr_err("%s: can't find MQ device tree node\n", 4811 __func__); 4812 ret = -ENODEV; 4813 goto out_free; 4814 } 4815 4816 /* Get MQ DCRs base */ 4817 dcr_base = dcr_resource_start(np, 0); 4818 dcr_len = dcr_resource_len(np, 0); 4819 if (!dcr_base && !dcr_len) { 4820 pr_err("%s: can't get DCR registers base/len!\n", 4821 np->full_name); 4822 ret = -ENODEV; 4823 goto out_mq; 4824 } 4825 4826 ppc440spe_mq_dcr_host = dcr_map(np, dcr_base, dcr_len); 4827 if (!DCR_MAP_OK(ppc440spe_mq_dcr_host)) { 4828 pr_err("%s: failed to map DCRs!\n", np->full_name); 4829 ret = -ENODEV; 4830 goto out_mq; 4831 } 4832 of_node_put(np); 4833 ppc440spe_mq_dcr_len = dcr_len; 4834 4835 /* Set HB alias */ 4836 dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_BAUH, DMA_CUED_XOR_HB); 4837 4838 /* Set: 4839 * - LL transaction passing limit to 1; 4840 * - Memory controller cycle limit to 1; 4841 * - Galois Polynomial to 0x14d (default) 4842 */ 4843 dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL, 4844 (1 << MQ0_CFBHL_TPLM) | (1 << MQ0_CFBHL_HBCL) | 4845 (PPC440SPE_DEFAULT_POLY << MQ0_CFBHL_POLY)); 4846 4847 atomic_set(&ppc440spe_adma_err_irq_ref, 0); 4848 for (i = 0; i < PPC440SPE_ADMA_ENGINES_NUM; i++) 4849 ppc440spe_adma_devices[i] = -1; 4850 4851 return 0; 4852 4853 out_mq: 4854 of_node_put(np); 4855 out_free: 4856 kfree(ppc440spe_dma_fifo_buf); 4857 return ret; 4858 } 4859 4860 static const struct of_device_id ppc440spe_adma_of_match[] = { 4861 { .compatible = "ibm,dma-440spe", }, 4862 { .compatible = "amcc,xor-accelerator", }, 4863 {}, 4864 }; 4865 MODULE_DEVICE_TABLE(of, ppc440spe_adma_of_match); 4866 4867 static struct platform_driver ppc440spe_adma_driver = { 4868 .probe = ppc440spe_adma_probe, 4869 .remove = ppc440spe_adma_remove, 4870 .driver = { 4871 .name = "PPC440SP(E)-ADMA", 4872 .owner = THIS_MODULE, 4873 .of_match_table = ppc440spe_adma_of_match, 4874 }, 4875 }; 4876 4877 static __init int ppc440spe_adma_init(void) 4878 { 4879 int ret; 4880 4881 ret = ppc440spe_configure_raid_devices(); 4882 if (ret) 4883 return ret; 4884 4885 ret = platform_driver_register(&ppc440spe_adma_driver); 4886 if (ret) { 4887 pr_err("%s: failed to register platform driver\n", 4888 __func__); 4889 goto out_reg; 4890 } 4891 4892 /* Initialization status */ 4893 ret = driver_create_file(&ppc440spe_adma_driver.driver, 4894 &driver_attr_devices); 4895 if (ret) 4896 goto out_dev; 4897 4898 /* RAID-6 h/w enable entry */ 4899 ret = driver_create_file(&ppc440spe_adma_driver.driver, 4900 &driver_attr_enable); 4901 if (ret) 4902 goto out_en; 4903 4904 /* GF polynomial to use */ 4905 ret = driver_create_file(&ppc440spe_adma_driver.driver, 4906 &driver_attr_poly); 4907 if (!ret) 4908 return ret; 4909 4910 driver_remove_file(&ppc440spe_adma_driver.driver, 4911 &driver_attr_enable); 4912 out_en: 4913 driver_remove_file(&ppc440spe_adma_driver.driver, 4914 &driver_attr_devices); 4915 out_dev: 4916 /* User will not be able to enable h/w RAID-6 */ 4917 pr_err("%s: failed to create RAID-6 driver interface\n", 4918 __func__); 4919 platform_driver_unregister(&ppc440spe_adma_driver); 4920 out_reg: 4921 dcr_unmap(ppc440spe_mq_dcr_host, ppc440spe_mq_dcr_len); 4922 kfree(ppc440spe_dma_fifo_buf); 4923 return ret; 4924 } 4925 4926 static void __exit ppc440spe_adma_exit(void) 4927 { 4928 driver_remove_file(&ppc440spe_adma_driver.driver, 4929 &driver_attr_poly); 4930 driver_remove_file(&ppc440spe_adma_driver.driver, 4931 &driver_attr_enable); 4932 driver_remove_file(&ppc440spe_adma_driver.driver, 4933 &driver_attr_devices); 4934 platform_driver_unregister(&ppc440spe_adma_driver); 4935 dcr_unmap(ppc440spe_mq_dcr_host, ppc440spe_mq_dcr_len); 4936 kfree(ppc440spe_dma_fifo_buf); 4937 } 4938 4939 arch_initcall(ppc440spe_adma_init); 4940 module_exit(ppc440spe_adma_exit); 4941 4942 MODULE_AUTHOR("Yuri Tikhonov <yur@emcraft.com>"); 4943 MODULE_DESCRIPTION("PPC440SPE ADMA Engine Driver"); 4944 MODULE_LICENSE("GPL"); 4945