1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * AMD Cryptographic Coprocessor (CCP) driver 4 * 5 * Copyright (C) 2016,2017 Advanced Micro Devices, Inc. 6 * 7 * Author: Gary R Hook <gary.hook@amd.com> 8 */ 9 10 #include <linux/module.h> 11 #include <linux/kernel.h> 12 #include <linux/pci.h> 13 #include <linux/kthread.h> 14 #include <linux/debugfs.h> 15 #include <linux/dma-mapping.h> 16 #include <linux/interrupt.h> 17 #include <linux/compiler.h> 18 #include <linux/ccp.h> 19 20 #include "ccp-dev.h" 21 22 /* Allocate the requested number of contiguous LSB slots 23 * from the LSB bitmap. Look in the private range for this 24 * queue first; failing that, check the public area. 25 * If no space is available, wait around. 26 * Return: first slot number 27 */ 28 static u32 ccp_lsb_alloc(struct ccp_cmd_queue *cmd_q, unsigned int count) 29 { 30 struct ccp_device *ccp; 31 int start; 32 33 /* First look at the map for the queue */ 34 if (cmd_q->lsb >= 0) { 35 start = (u32)bitmap_find_next_zero_area(cmd_q->lsbmap, 36 LSB_SIZE, 37 0, count, 0); 38 if (start < LSB_SIZE) { 39 bitmap_set(cmd_q->lsbmap, start, count); 40 return start + cmd_q->lsb * LSB_SIZE; 41 } 42 } 43 44 /* No joy; try to get an entry from the shared blocks */ 45 ccp = cmd_q->ccp; 46 for (;;) { 47 mutex_lock(&ccp->sb_mutex); 48 49 start = (u32)bitmap_find_next_zero_area(ccp->lsbmap, 50 MAX_LSB_CNT * LSB_SIZE, 51 0, 52 count, 0); 53 if (start <= MAX_LSB_CNT * LSB_SIZE) { 54 bitmap_set(ccp->lsbmap, start, count); 55 56 mutex_unlock(&ccp->sb_mutex); 57 return start; 58 } 59 60 ccp->sb_avail = 0; 61 62 mutex_unlock(&ccp->sb_mutex); 63 64 /* Wait for KSB entries to become available */ 65 if (wait_event_interruptible(ccp->sb_queue, ccp->sb_avail)) 66 return 0; 67 } 68 } 69 70 /* Free a number of LSB slots from the bitmap, starting at 71 * the indicated starting slot number. 72 */ 73 static void ccp_lsb_free(struct ccp_cmd_queue *cmd_q, unsigned int start, 74 unsigned int count) 75 { 76 if (!start) 77 return; 78 79 if (cmd_q->lsb == start) { 80 /* An entry from the private LSB */ 81 bitmap_clear(cmd_q->lsbmap, start, count); 82 } else { 83 /* From the shared LSBs */ 84 struct ccp_device *ccp = cmd_q->ccp; 85 86 mutex_lock(&ccp->sb_mutex); 87 bitmap_clear(ccp->lsbmap, start, count); 88 ccp->sb_avail = 1; 89 mutex_unlock(&ccp->sb_mutex); 90 wake_up_interruptible_all(&ccp->sb_queue); 91 } 92 } 93 94 /* CCP version 5: Union to define the function field (cmd_reg1/dword0) */ 95 union ccp_function { 96 struct { 97 u16 size:7; 98 u16 encrypt:1; 99 u16 mode:5; 100 u16 type:2; 101 } aes; 102 struct { 103 u16 size:7; 104 u16 encrypt:1; 105 u16 rsvd:5; 106 u16 type:2; 107 } aes_xts; 108 struct { 109 u16 size:7; 110 u16 encrypt:1; 111 u16 mode:5; 112 u16 type:2; 113 } des3; 114 struct { 115 u16 rsvd1:10; 116 u16 type:4; 117 u16 rsvd2:1; 118 } sha; 119 struct { 120 u16 mode:3; 121 u16 size:12; 122 } rsa; 123 struct { 124 u16 byteswap:2; 125 u16 bitwise:3; 126 u16 reflect:2; 127 u16 rsvd:8; 128 } pt; 129 struct { 130 u16 rsvd:13; 131 } zlib; 132 struct { 133 u16 size:10; 134 u16 type:2; 135 u16 mode:3; 136 } ecc; 137 u16 raw; 138 }; 139 140 #define CCP_AES_SIZE(p) ((p)->aes.size) 141 #define CCP_AES_ENCRYPT(p) ((p)->aes.encrypt) 142 #define CCP_AES_MODE(p) ((p)->aes.mode) 143 #define CCP_AES_TYPE(p) ((p)->aes.type) 144 #define CCP_XTS_SIZE(p) ((p)->aes_xts.size) 145 #define CCP_XTS_TYPE(p) ((p)->aes_xts.type) 146 #define CCP_XTS_ENCRYPT(p) ((p)->aes_xts.encrypt) 147 #define CCP_DES3_SIZE(p) ((p)->des3.size) 148 #define CCP_DES3_ENCRYPT(p) ((p)->des3.encrypt) 149 #define CCP_DES3_MODE(p) ((p)->des3.mode) 150 #define CCP_DES3_TYPE(p) ((p)->des3.type) 151 #define CCP_SHA_TYPE(p) ((p)->sha.type) 152 #define CCP_RSA_SIZE(p) ((p)->rsa.size) 153 #define CCP_PT_BYTESWAP(p) ((p)->pt.byteswap) 154 #define CCP_PT_BITWISE(p) ((p)->pt.bitwise) 155 #define CCP_ECC_MODE(p) ((p)->ecc.mode) 156 #define CCP_ECC_AFFINE(p) ((p)->ecc.one) 157 158 /* Word 0 */ 159 #define CCP5_CMD_DW0(p) ((p)->dw0) 160 #define CCP5_CMD_SOC(p) (CCP5_CMD_DW0(p).soc) 161 #define CCP5_CMD_IOC(p) (CCP5_CMD_DW0(p).ioc) 162 #define CCP5_CMD_INIT(p) (CCP5_CMD_DW0(p).init) 163 #define CCP5_CMD_EOM(p) (CCP5_CMD_DW0(p).eom) 164 #define CCP5_CMD_FUNCTION(p) (CCP5_CMD_DW0(p).function) 165 #define CCP5_CMD_ENGINE(p) (CCP5_CMD_DW0(p).engine) 166 #define CCP5_CMD_PROT(p) (CCP5_CMD_DW0(p).prot) 167 168 /* Word 1 */ 169 #define CCP5_CMD_DW1(p) ((p)->length) 170 #define CCP5_CMD_LEN(p) (CCP5_CMD_DW1(p)) 171 172 /* Word 2 */ 173 #define CCP5_CMD_DW2(p) ((p)->src_lo) 174 #define CCP5_CMD_SRC_LO(p) (CCP5_CMD_DW2(p)) 175 176 /* Word 3 */ 177 #define CCP5_CMD_DW3(p) ((p)->dw3) 178 #define CCP5_CMD_SRC_MEM(p) ((p)->dw3.src_mem) 179 #define CCP5_CMD_SRC_HI(p) ((p)->dw3.src_hi) 180 #define CCP5_CMD_LSB_ID(p) ((p)->dw3.lsb_cxt_id) 181 #define CCP5_CMD_FIX_SRC(p) ((p)->dw3.fixed) 182 183 /* Words 4/5 */ 184 #define CCP5_CMD_DW4(p) ((p)->dw4) 185 #define CCP5_CMD_DST_LO(p) (CCP5_CMD_DW4(p).dst_lo) 186 #define CCP5_CMD_DW5(p) ((p)->dw5.fields.dst_hi) 187 #define CCP5_CMD_DST_HI(p) (CCP5_CMD_DW5(p)) 188 #define CCP5_CMD_DST_MEM(p) ((p)->dw5.fields.dst_mem) 189 #define CCP5_CMD_FIX_DST(p) ((p)->dw5.fields.fixed) 190 #define CCP5_CMD_SHA_LO(p) ((p)->dw4.sha_len_lo) 191 #define CCP5_CMD_SHA_HI(p) ((p)->dw5.sha_len_hi) 192 193 /* Word 6/7 */ 194 #define CCP5_CMD_DW6(p) ((p)->key_lo) 195 #define CCP5_CMD_KEY_LO(p) (CCP5_CMD_DW6(p)) 196 #define CCP5_CMD_DW7(p) ((p)->dw7) 197 #define CCP5_CMD_KEY_HI(p) ((p)->dw7.key_hi) 198 #define CCP5_CMD_KEY_MEM(p) ((p)->dw7.key_mem) 199 200 static inline u32 low_address(unsigned long addr) 201 { 202 return (u64)addr & 0x0ffffffff; 203 } 204 205 static inline u32 high_address(unsigned long addr) 206 { 207 return ((u64)addr >> 32) & 0x00000ffff; 208 } 209 210 static unsigned int ccp5_get_free_slots(struct ccp_cmd_queue *cmd_q) 211 { 212 unsigned int head_idx, n; 213 u32 head_lo, queue_start; 214 215 queue_start = low_address(cmd_q->qdma_tail); 216 head_lo = ioread32(cmd_q->reg_head_lo); 217 head_idx = (head_lo - queue_start) / sizeof(struct ccp5_desc); 218 219 n = head_idx + COMMANDS_PER_QUEUE - cmd_q->qidx - 1; 220 221 return n % COMMANDS_PER_QUEUE; /* Always one unused spot */ 222 } 223 224 static int ccp5_do_cmd(struct ccp5_desc *desc, 225 struct ccp_cmd_queue *cmd_q) 226 { 227 u32 *mP; 228 __le32 *dP; 229 u32 tail; 230 int i; 231 int ret = 0; 232 233 cmd_q->total_ops++; 234 235 if (CCP5_CMD_SOC(desc)) { 236 CCP5_CMD_IOC(desc) = 1; 237 CCP5_CMD_SOC(desc) = 0; 238 } 239 mutex_lock(&cmd_q->q_mutex); 240 241 mP = (u32 *) &cmd_q->qbase[cmd_q->qidx]; 242 dP = (__le32 *) desc; 243 for (i = 0; i < 8; i++) 244 mP[i] = cpu_to_le32(dP[i]); /* handle endianness */ 245 246 cmd_q->qidx = (cmd_q->qidx + 1) % COMMANDS_PER_QUEUE; 247 248 /* The data used by this command must be flushed to memory */ 249 wmb(); 250 251 /* Write the new tail address back to the queue register */ 252 tail = low_address(cmd_q->qdma_tail + cmd_q->qidx * Q_DESC_SIZE); 253 iowrite32(tail, cmd_q->reg_tail_lo); 254 255 /* Turn the queue back on using our cached control register */ 256 iowrite32(cmd_q->qcontrol | CMD5_Q_RUN, cmd_q->reg_control); 257 mutex_unlock(&cmd_q->q_mutex); 258 259 if (CCP5_CMD_IOC(desc)) { 260 /* Wait for the job to complete */ 261 ret = wait_event_interruptible(cmd_q->int_queue, 262 cmd_q->int_rcvd); 263 if (ret || cmd_q->cmd_error) { 264 /* Log the error and flush the queue by 265 * moving the head pointer 266 */ 267 if (cmd_q->cmd_error) 268 ccp_log_error(cmd_q->ccp, 269 cmd_q->cmd_error); 270 iowrite32(tail, cmd_q->reg_head_lo); 271 if (!ret) 272 ret = -EIO; 273 } 274 cmd_q->int_rcvd = 0; 275 } 276 277 return ret; 278 } 279 280 static int ccp5_perform_aes(struct ccp_op *op) 281 { 282 struct ccp5_desc desc; 283 union ccp_function function; 284 u32 key_addr = op->sb_key * LSB_ITEM_SIZE; 285 286 op->cmd_q->total_aes_ops++; 287 288 /* Zero out all the fields of the command desc */ 289 memset(&desc, 0, Q_DESC_SIZE); 290 291 CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_AES; 292 293 CCP5_CMD_SOC(&desc) = op->soc; 294 CCP5_CMD_IOC(&desc) = 1; 295 CCP5_CMD_INIT(&desc) = op->init; 296 CCP5_CMD_EOM(&desc) = op->eom; 297 CCP5_CMD_PROT(&desc) = 0; 298 299 function.raw = 0; 300 CCP_AES_ENCRYPT(&function) = op->u.aes.action; 301 CCP_AES_MODE(&function) = op->u.aes.mode; 302 CCP_AES_TYPE(&function) = op->u.aes.type; 303 CCP_AES_SIZE(&function) = op->u.aes.size; 304 305 CCP5_CMD_FUNCTION(&desc) = function.raw; 306 307 CCP5_CMD_LEN(&desc) = op->src.u.dma.length; 308 309 CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma); 310 CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma); 311 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM; 312 313 CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma); 314 CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma); 315 CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM; 316 317 CCP5_CMD_KEY_LO(&desc) = lower_32_bits(key_addr); 318 CCP5_CMD_KEY_HI(&desc) = 0; 319 CCP5_CMD_KEY_MEM(&desc) = CCP_MEMTYPE_SB; 320 CCP5_CMD_LSB_ID(&desc) = op->sb_ctx; 321 322 return ccp5_do_cmd(&desc, op->cmd_q); 323 } 324 325 static int ccp5_perform_xts_aes(struct ccp_op *op) 326 { 327 struct ccp5_desc desc; 328 union ccp_function function; 329 u32 key_addr = op->sb_key * LSB_ITEM_SIZE; 330 331 op->cmd_q->total_xts_aes_ops++; 332 333 /* Zero out all the fields of the command desc */ 334 memset(&desc, 0, Q_DESC_SIZE); 335 336 CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_XTS_AES_128; 337 338 CCP5_CMD_SOC(&desc) = op->soc; 339 CCP5_CMD_IOC(&desc) = 1; 340 CCP5_CMD_INIT(&desc) = op->init; 341 CCP5_CMD_EOM(&desc) = op->eom; 342 CCP5_CMD_PROT(&desc) = 0; 343 344 function.raw = 0; 345 CCP_XTS_TYPE(&function) = op->u.xts.type; 346 CCP_XTS_ENCRYPT(&function) = op->u.xts.action; 347 CCP_XTS_SIZE(&function) = op->u.xts.unit_size; 348 CCP5_CMD_FUNCTION(&desc) = function.raw; 349 350 CCP5_CMD_LEN(&desc) = op->src.u.dma.length; 351 352 CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma); 353 CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma); 354 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM; 355 356 CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma); 357 CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma); 358 CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM; 359 360 CCP5_CMD_KEY_LO(&desc) = lower_32_bits(key_addr); 361 CCP5_CMD_KEY_HI(&desc) = 0; 362 CCP5_CMD_KEY_MEM(&desc) = CCP_MEMTYPE_SB; 363 CCP5_CMD_LSB_ID(&desc) = op->sb_ctx; 364 365 return ccp5_do_cmd(&desc, op->cmd_q); 366 } 367 368 static int ccp5_perform_sha(struct ccp_op *op) 369 { 370 struct ccp5_desc desc; 371 union ccp_function function; 372 373 op->cmd_q->total_sha_ops++; 374 375 /* Zero out all the fields of the command desc */ 376 memset(&desc, 0, Q_DESC_SIZE); 377 378 CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_SHA; 379 380 CCP5_CMD_SOC(&desc) = op->soc; 381 CCP5_CMD_IOC(&desc) = 1; 382 CCP5_CMD_INIT(&desc) = 1; 383 CCP5_CMD_EOM(&desc) = op->eom; 384 CCP5_CMD_PROT(&desc) = 0; 385 386 function.raw = 0; 387 CCP_SHA_TYPE(&function) = op->u.sha.type; 388 CCP5_CMD_FUNCTION(&desc) = function.raw; 389 390 CCP5_CMD_LEN(&desc) = op->src.u.dma.length; 391 392 CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma); 393 CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma); 394 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM; 395 396 CCP5_CMD_LSB_ID(&desc) = op->sb_ctx; 397 398 if (op->eom) { 399 CCP5_CMD_SHA_LO(&desc) = lower_32_bits(op->u.sha.msg_bits); 400 CCP5_CMD_SHA_HI(&desc) = upper_32_bits(op->u.sha.msg_bits); 401 } else { 402 CCP5_CMD_SHA_LO(&desc) = 0; 403 CCP5_CMD_SHA_HI(&desc) = 0; 404 } 405 406 return ccp5_do_cmd(&desc, op->cmd_q); 407 } 408 409 static int ccp5_perform_des3(struct ccp_op *op) 410 { 411 struct ccp5_desc desc; 412 union ccp_function function; 413 u32 key_addr = op->sb_key * LSB_ITEM_SIZE; 414 415 op->cmd_q->total_3des_ops++; 416 417 /* Zero out all the fields of the command desc */ 418 memset(&desc, 0, sizeof(struct ccp5_desc)); 419 420 CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_DES3; 421 422 CCP5_CMD_SOC(&desc) = op->soc; 423 CCP5_CMD_IOC(&desc) = 1; 424 CCP5_CMD_INIT(&desc) = op->init; 425 CCP5_CMD_EOM(&desc) = op->eom; 426 CCP5_CMD_PROT(&desc) = 0; 427 428 function.raw = 0; 429 CCP_DES3_ENCRYPT(&function) = op->u.des3.action; 430 CCP_DES3_MODE(&function) = op->u.des3.mode; 431 CCP_DES3_TYPE(&function) = op->u.des3.type; 432 CCP5_CMD_FUNCTION(&desc) = function.raw; 433 434 CCP5_CMD_LEN(&desc) = op->src.u.dma.length; 435 436 CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma); 437 CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma); 438 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM; 439 440 CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma); 441 CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma); 442 CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM; 443 444 CCP5_CMD_KEY_LO(&desc) = lower_32_bits(key_addr); 445 CCP5_CMD_KEY_HI(&desc) = 0; 446 CCP5_CMD_KEY_MEM(&desc) = CCP_MEMTYPE_SB; 447 CCP5_CMD_LSB_ID(&desc) = op->sb_ctx; 448 449 return ccp5_do_cmd(&desc, op->cmd_q); 450 } 451 452 static int ccp5_perform_rsa(struct ccp_op *op) 453 { 454 struct ccp5_desc desc; 455 union ccp_function function; 456 457 op->cmd_q->total_rsa_ops++; 458 459 /* Zero out all the fields of the command desc */ 460 memset(&desc, 0, Q_DESC_SIZE); 461 462 CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_RSA; 463 464 CCP5_CMD_SOC(&desc) = op->soc; 465 CCP5_CMD_IOC(&desc) = 1; 466 CCP5_CMD_INIT(&desc) = 0; 467 CCP5_CMD_EOM(&desc) = 1; 468 CCP5_CMD_PROT(&desc) = 0; 469 470 function.raw = 0; 471 CCP_RSA_SIZE(&function) = (op->u.rsa.mod_size + 7) >> 3; 472 CCP5_CMD_FUNCTION(&desc) = function.raw; 473 474 CCP5_CMD_LEN(&desc) = op->u.rsa.input_len; 475 476 /* Source is from external memory */ 477 CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma); 478 CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma); 479 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM; 480 481 /* Destination is in external memory */ 482 CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma); 483 CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma); 484 CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM; 485 486 /* Key (Exponent) is in external memory */ 487 CCP5_CMD_KEY_LO(&desc) = ccp_addr_lo(&op->exp.u.dma); 488 CCP5_CMD_KEY_HI(&desc) = ccp_addr_hi(&op->exp.u.dma); 489 CCP5_CMD_KEY_MEM(&desc) = CCP_MEMTYPE_SYSTEM; 490 491 return ccp5_do_cmd(&desc, op->cmd_q); 492 } 493 494 static int ccp5_perform_passthru(struct ccp_op *op) 495 { 496 struct ccp5_desc desc; 497 union ccp_function function; 498 struct ccp_dma_info *saddr = &op->src.u.dma; 499 struct ccp_dma_info *daddr = &op->dst.u.dma; 500 501 502 op->cmd_q->total_pt_ops++; 503 504 memset(&desc, 0, Q_DESC_SIZE); 505 506 CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_PASSTHRU; 507 508 CCP5_CMD_SOC(&desc) = 0; 509 CCP5_CMD_IOC(&desc) = 1; 510 CCP5_CMD_INIT(&desc) = 0; 511 CCP5_CMD_EOM(&desc) = op->eom; 512 CCP5_CMD_PROT(&desc) = 0; 513 514 function.raw = 0; 515 CCP_PT_BYTESWAP(&function) = op->u.passthru.byte_swap; 516 CCP_PT_BITWISE(&function) = op->u.passthru.bit_mod; 517 CCP5_CMD_FUNCTION(&desc) = function.raw; 518 519 /* Length of source data is always 256 bytes */ 520 if (op->src.type == CCP_MEMTYPE_SYSTEM) 521 CCP5_CMD_LEN(&desc) = saddr->length; 522 else 523 CCP5_CMD_LEN(&desc) = daddr->length; 524 525 if (op->src.type == CCP_MEMTYPE_SYSTEM) { 526 CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma); 527 CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma); 528 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM; 529 530 if (op->u.passthru.bit_mod != CCP_PASSTHRU_BITWISE_NOOP) 531 CCP5_CMD_LSB_ID(&desc) = op->sb_key; 532 } else { 533 u32 key_addr = op->src.u.sb * CCP_SB_BYTES; 534 535 CCP5_CMD_SRC_LO(&desc) = lower_32_bits(key_addr); 536 CCP5_CMD_SRC_HI(&desc) = 0; 537 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SB; 538 } 539 540 if (op->dst.type == CCP_MEMTYPE_SYSTEM) { 541 CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma); 542 CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma); 543 CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM; 544 } else { 545 u32 key_addr = op->dst.u.sb * CCP_SB_BYTES; 546 547 CCP5_CMD_DST_LO(&desc) = lower_32_bits(key_addr); 548 CCP5_CMD_DST_HI(&desc) = 0; 549 CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SB; 550 } 551 552 return ccp5_do_cmd(&desc, op->cmd_q); 553 } 554 555 static int ccp5_perform_ecc(struct ccp_op *op) 556 { 557 struct ccp5_desc desc; 558 union ccp_function function; 559 560 op->cmd_q->total_ecc_ops++; 561 562 /* Zero out all the fields of the command desc */ 563 memset(&desc, 0, Q_DESC_SIZE); 564 565 CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_ECC; 566 567 CCP5_CMD_SOC(&desc) = 0; 568 CCP5_CMD_IOC(&desc) = 1; 569 CCP5_CMD_INIT(&desc) = 0; 570 CCP5_CMD_EOM(&desc) = 1; 571 CCP5_CMD_PROT(&desc) = 0; 572 573 function.raw = 0; 574 function.ecc.mode = op->u.ecc.function; 575 CCP5_CMD_FUNCTION(&desc) = function.raw; 576 577 CCP5_CMD_LEN(&desc) = op->src.u.dma.length; 578 579 CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma); 580 CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma); 581 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM; 582 583 CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma); 584 CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma); 585 CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM; 586 587 return ccp5_do_cmd(&desc, op->cmd_q); 588 } 589 590 static int ccp_find_lsb_regions(struct ccp_cmd_queue *cmd_q, u64 status) 591 { 592 int q_mask = 1 << cmd_q->id; 593 int queues = 0; 594 int j; 595 596 /* Build a bit mask to know which LSBs this queue has access to. 597 * Don't bother with segment 0 as it has special privileges. 598 */ 599 for (j = 1; j < MAX_LSB_CNT; j++) { 600 if (status & q_mask) 601 bitmap_set(cmd_q->lsbmask, j, 1); 602 status >>= LSB_REGION_WIDTH; 603 } 604 queues = bitmap_weight(cmd_q->lsbmask, MAX_LSB_CNT); 605 dev_dbg(cmd_q->ccp->dev, "Queue %d can access %d LSB regions\n", 606 cmd_q->id, queues); 607 608 return queues ? 0 : -EINVAL; 609 } 610 611 static int ccp_find_and_assign_lsb_to_q(struct ccp_device *ccp, 612 int lsb_cnt, int n_lsbs, 613 unsigned long *lsb_pub) 614 { 615 DECLARE_BITMAP(qlsb, MAX_LSB_CNT); 616 int bitno; 617 int qlsb_wgt; 618 int i; 619 620 /* For each queue: 621 * If the count of potential LSBs available to a queue matches the 622 * ordinal given to us in lsb_cnt: 623 * Copy the mask of possible LSBs for this queue into "qlsb"; 624 * For each bit in qlsb, see if the corresponding bit in the 625 * aggregation mask is set; if so, we have a match. 626 * If we have a match, clear the bit in the aggregation to 627 * mark it as no longer available. 628 * If there is no match, clear the bit in qlsb and keep looking. 629 */ 630 for (i = 0; i < ccp->cmd_q_count; i++) { 631 struct ccp_cmd_queue *cmd_q = &ccp->cmd_q[i]; 632 633 qlsb_wgt = bitmap_weight(cmd_q->lsbmask, MAX_LSB_CNT); 634 635 if (qlsb_wgt == lsb_cnt) { 636 bitmap_copy(qlsb, cmd_q->lsbmask, MAX_LSB_CNT); 637 638 bitno = find_first_bit(qlsb, MAX_LSB_CNT); 639 while (bitno < MAX_LSB_CNT) { 640 if (test_bit(bitno, lsb_pub)) { 641 /* We found an available LSB 642 * that this queue can access 643 */ 644 cmd_q->lsb = bitno; 645 bitmap_clear(lsb_pub, bitno, 1); 646 dev_dbg(ccp->dev, 647 "Queue %d gets LSB %d\n", 648 i, bitno); 649 break; 650 } 651 bitmap_clear(qlsb, bitno, 1); 652 bitno = find_first_bit(qlsb, MAX_LSB_CNT); 653 } 654 if (bitno >= MAX_LSB_CNT) 655 return -EINVAL; 656 n_lsbs--; 657 } 658 } 659 return n_lsbs; 660 } 661 662 /* For each queue, from the most- to least-constrained: 663 * find an LSB that can be assigned to the queue. If there are N queues that 664 * can only use M LSBs, where N > M, fail; otherwise, every queue will get a 665 * dedicated LSB. Remaining LSB regions become a shared resource. 666 * If we have fewer LSBs than queues, all LSB regions become shared resources. 667 */ 668 static int ccp_assign_lsbs(struct ccp_device *ccp) 669 { 670 DECLARE_BITMAP(lsb_pub, MAX_LSB_CNT); 671 DECLARE_BITMAP(qlsb, MAX_LSB_CNT); 672 int n_lsbs = 0; 673 int bitno; 674 int i, lsb_cnt; 675 int rc = 0; 676 677 bitmap_zero(lsb_pub, MAX_LSB_CNT); 678 679 /* Create an aggregate bitmap to get a total count of available LSBs */ 680 for (i = 0; i < ccp->cmd_q_count; i++) 681 bitmap_or(lsb_pub, 682 lsb_pub, ccp->cmd_q[i].lsbmask, 683 MAX_LSB_CNT); 684 685 n_lsbs = bitmap_weight(lsb_pub, MAX_LSB_CNT); 686 687 if (n_lsbs >= ccp->cmd_q_count) { 688 /* We have enough LSBS to give every queue a private LSB. 689 * Brute force search to start with the queues that are more 690 * constrained in LSB choice. When an LSB is privately 691 * assigned, it is removed from the public mask. 692 * This is an ugly N squared algorithm with some optimization. 693 */ 694 for (lsb_cnt = 1; 695 n_lsbs && (lsb_cnt <= MAX_LSB_CNT); 696 lsb_cnt++) { 697 rc = ccp_find_and_assign_lsb_to_q(ccp, lsb_cnt, n_lsbs, 698 lsb_pub); 699 if (rc < 0) 700 return -EINVAL; 701 n_lsbs = rc; 702 } 703 } 704 705 rc = 0; 706 /* What's left of the LSBs, according to the public mask, now become 707 * shared. Any zero bits in the lsb_pub mask represent an LSB region 708 * that can't be used as a shared resource, so mark the LSB slots for 709 * them as "in use". 710 */ 711 bitmap_copy(qlsb, lsb_pub, MAX_LSB_CNT); 712 713 bitno = find_first_zero_bit(qlsb, MAX_LSB_CNT); 714 while (bitno < MAX_LSB_CNT) { 715 bitmap_set(ccp->lsbmap, bitno * LSB_SIZE, LSB_SIZE); 716 bitmap_set(qlsb, bitno, 1); 717 bitno = find_first_zero_bit(qlsb, MAX_LSB_CNT); 718 } 719 720 return rc; 721 } 722 723 static void ccp5_disable_queue_interrupts(struct ccp_device *ccp) 724 { 725 unsigned int i; 726 727 for (i = 0; i < ccp->cmd_q_count; i++) 728 iowrite32(0x0, ccp->cmd_q[i].reg_int_enable); 729 } 730 731 static void ccp5_enable_queue_interrupts(struct ccp_device *ccp) 732 { 733 unsigned int i; 734 735 for (i = 0; i < ccp->cmd_q_count; i++) 736 iowrite32(SUPPORTED_INTERRUPTS, ccp->cmd_q[i].reg_int_enable); 737 } 738 739 static void ccp5_irq_bh(unsigned long data) 740 { 741 struct ccp_device *ccp = (struct ccp_device *)data; 742 u32 status; 743 unsigned int i; 744 745 for (i = 0; i < ccp->cmd_q_count; i++) { 746 struct ccp_cmd_queue *cmd_q = &ccp->cmd_q[i]; 747 748 status = ioread32(cmd_q->reg_interrupt_status); 749 750 if (status) { 751 cmd_q->int_status = status; 752 cmd_q->q_status = ioread32(cmd_q->reg_status); 753 cmd_q->q_int_status = ioread32(cmd_q->reg_int_status); 754 755 /* On error, only save the first error value */ 756 if ((status & INT_ERROR) && !cmd_q->cmd_error) 757 cmd_q->cmd_error = CMD_Q_ERROR(cmd_q->q_status); 758 759 cmd_q->int_rcvd = 1; 760 761 /* Acknowledge the interrupt and wake the kthread */ 762 iowrite32(status, cmd_q->reg_interrupt_status); 763 wake_up_interruptible(&cmd_q->int_queue); 764 } 765 } 766 ccp5_enable_queue_interrupts(ccp); 767 } 768 769 static irqreturn_t ccp5_irq_handler(int irq, void *data) 770 { 771 struct ccp_device *ccp = (struct ccp_device *)data; 772 773 ccp5_disable_queue_interrupts(ccp); 774 ccp->total_interrupts++; 775 if (ccp->use_tasklet) 776 tasklet_schedule(&ccp->irq_tasklet); 777 else 778 ccp5_irq_bh((unsigned long)ccp); 779 return IRQ_HANDLED; 780 } 781 782 static int ccp5_init(struct ccp_device *ccp) 783 { 784 struct device *dev = ccp->dev; 785 struct ccp_cmd_queue *cmd_q; 786 struct dma_pool *dma_pool; 787 char dma_pool_name[MAX_DMAPOOL_NAME_LEN]; 788 unsigned int qmr, i; 789 u64 status; 790 u32 status_lo, status_hi; 791 int ret; 792 793 /* Find available queues */ 794 qmr = ioread32(ccp->io_regs + Q_MASK_REG); 795 for (i = 0; i < MAX_HW_QUEUES; i++) { 796 797 if (!(qmr & (1 << i))) 798 continue; 799 800 /* Allocate a dma pool for this queue */ 801 snprintf(dma_pool_name, sizeof(dma_pool_name), "%s_q%d", 802 ccp->name, i); 803 dma_pool = dma_pool_create(dma_pool_name, dev, 804 CCP_DMAPOOL_MAX_SIZE, 805 CCP_DMAPOOL_ALIGN, 0); 806 if (!dma_pool) { 807 dev_err(dev, "unable to allocate dma pool\n"); 808 ret = -ENOMEM; 809 } 810 811 cmd_q = &ccp->cmd_q[ccp->cmd_q_count]; 812 ccp->cmd_q_count++; 813 814 cmd_q->ccp = ccp; 815 cmd_q->id = i; 816 cmd_q->dma_pool = dma_pool; 817 mutex_init(&cmd_q->q_mutex); 818 819 /* Page alignment satisfies our needs for N <= 128 */ 820 BUILD_BUG_ON(COMMANDS_PER_QUEUE > 128); 821 cmd_q->qsize = Q_SIZE(Q_DESC_SIZE); 822 cmd_q->qbase = dma_alloc_coherent(dev, cmd_q->qsize, 823 &cmd_q->qbase_dma, 824 GFP_KERNEL); 825 if (!cmd_q->qbase) { 826 dev_err(dev, "unable to allocate command queue\n"); 827 ret = -ENOMEM; 828 goto e_pool; 829 } 830 831 cmd_q->qidx = 0; 832 /* Preset some register values and masks that are queue 833 * number dependent 834 */ 835 cmd_q->reg_control = ccp->io_regs + 836 CMD5_Q_STATUS_INCR * (i + 1); 837 cmd_q->reg_tail_lo = cmd_q->reg_control + CMD5_Q_TAIL_LO_BASE; 838 cmd_q->reg_head_lo = cmd_q->reg_control + CMD5_Q_HEAD_LO_BASE; 839 cmd_q->reg_int_enable = cmd_q->reg_control + 840 CMD5_Q_INT_ENABLE_BASE; 841 cmd_q->reg_interrupt_status = cmd_q->reg_control + 842 CMD5_Q_INTERRUPT_STATUS_BASE; 843 cmd_q->reg_status = cmd_q->reg_control + CMD5_Q_STATUS_BASE; 844 cmd_q->reg_int_status = cmd_q->reg_control + 845 CMD5_Q_INT_STATUS_BASE; 846 cmd_q->reg_dma_status = cmd_q->reg_control + 847 CMD5_Q_DMA_STATUS_BASE; 848 cmd_q->reg_dma_read_status = cmd_q->reg_control + 849 CMD5_Q_DMA_READ_STATUS_BASE; 850 cmd_q->reg_dma_write_status = cmd_q->reg_control + 851 CMD5_Q_DMA_WRITE_STATUS_BASE; 852 853 init_waitqueue_head(&cmd_q->int_queue); 854 855 dev_dbg(dev, "queue #%u available\n", i); 856 } 857 858 if (ccp->cmd_q_count == 0) { 859 dev_notice(dev, "no command queues available\n"); 860 ret = -EIO; 861 goto e_pool; 862 } 863 864 /* Turn off the queues and disable interrupts until ready */ 865 ccp5_disable_queue_interrupts(ccp); 866 for (i = 0; i < ccp->cmd_q_count; i++) { 867 cmd_q = &ccp->cmd_q[i]; 868 869 cmd_q->qcontrol = 0; /* Start with nothing */ 870 iowrite32(cmd_q->qcontrol, cmd_q->reg_control); 871 872 ioread32(cmd_q->reg_int_status); 873 ioread32(cmd_q->reg_status); 874 875 /* Clear the interrupt status */ 876 iowrite32(SUPPORTED_INTERRUPTS, cmd_q->reg_interrupt_status); 877 } 878 879 dev_dbg(dev, "Requesting an IRQ...\n"); 880 /* Request an irq */ 881 ret = sp_request_ccp_irq(ccp->sp, ccp5_irq_handler, ccp->name, ccp); 882 if (ret) { 883 dev_err(dev, "unable to allocate an IRQ\n"); 884 goto e_pool; 885 } 886 /* Initialize the ISR tasklet */ 887 if (ccp->use_tasklet) 888 tasklet_init(&ccp->irq_tasklet, ccp5_irq_bh, 889 (unsigned long)ccp); 890 891 dev_dbg(dev, "Loading LSB map...\n"); 892 /* Copy the private LSB mask to the public registers */ 893 status_lo = ioread32(ccp->io_regs + LSB_PRIVATE_MASK_LO_OFFSET); 894 status_hi = ioread32(ccp->io_regs + LSB_PRIVATE_MASK_HI_OFFSET); 895 iowrite32(status_lo, ccp->io_regs + LSB_PUBLIC_MASK_LO_OFFSET); 896 iowrite32(status_hi, ccp->io_regs + LSB_PUBLIC_MASK_HI_OFFSET); 897 status = ((u64)status_hi<<30) | (u64)status_lo; 898 899 dev_dbg(dev, "Configuring virtual queues...\n"); 900 /* Configure size of each virtual queue accessible to host */ 901 for (i = 0; i < ccp->cmd_q_count; i++) { 902 u32 dma_addr_lo; 903 u32 dma_addr_hi; 904 905 cmd_q = &ccp->cmd_q[i]; 906 907 cmd_q->qcontrol &= ~(CMD5_Q_SIZE << CMD5_Q_SHIFT); 908 cmd_q->qcontrol |= QUEUE_SIZE_VAL << CMD5_Q_SHIFT; 909 910 cmd_q->qdma_tail = cmd_q->qbase_dma; 911 dma_addr_lo = low_address(cmd_q->qdma_tail); 912 iowrite32((u32)dma_addr_lo, cmd_q->reg_tail_lo); 913 iowrite32((u32)dma_addr_lo, cmd_q->reg_head_lo); 914 915 dma_addr_hi = high_address(cmd_q->qdma_tail); 916 cmd_q->qcontrol |= (dma_addr_hi << 16); 917 iowrite32(cmd_q->qcontrol, cmd_q->reg_control); 918 919 /* Find the LSB regions accessible to the queue */ 920 ccp_find_lsb_regions(cmd_q, status); 921 cmd_q->lsb = -1; /* Unassigned value */ 922 } 923 924 dev_dbg(dev, "Assigning LSBs...\n"); 925 ret = ccp_assign_lsbs(ccp); 926 if (ret) { 927 dev_err(dev, "Unable to assign LSBs (%d)\n", ret); 928 goto e_irq; 929 } 930 931 /* Optimization: pre-allocate LSB slots for each queue */ 932 for (i = 0; i < ccp->cmd_q_count; i++) { 933 ccp->cmd_q[i].sb_key = ccp_lsb_alloc(&ccp->cmd_q[i], 2); 934 ccp->cmd_q[i].sb_ctx = ccp_lsb_alloc(&ccp->cmd_q[i], 2); 935 } 936 937 dev_dbg(dev, "Starting threads...\n"); 938 /* Create a kthread for each queue */ 939 for (i = 0; i < ccp->cmd_q_count; i++) { 940 struct task_struct *kthread; 941 942 cmd_q = &ccp->cmd_q[i]; 943 944 kthread = kthread_create(ccp_cmd_queue_thread, cmd_q, 945 "%s-q%u", ccp->name, cmd_q->id); 946 if (IS_ERR(kthread)) { 947 dev_err(dev, "error creating queue thread (%ld)\n", 948 PTR_ERR(kthread)); 949 ret = PTR_ERR(kthread); 950 goto e_kthread; 951 } 952 953 cmd_q->kthread = kthread; 954 wake_up_process(kthread); 955 } 956 957 dev_dbg(dev, "Enabling interrupts...\n"); 958 ccp5_enable_queue_interrupts(ccp); 959 960 dev_dbg(dev, "Registering device...\n"); 961 /* Put this on the unit list to make it available */ 962 ccp_add_device(ccp); 963 964 ret = ccp_register_rng(ccp); 965 if (ret) 966 goto e_kthread; 967 968 /* Register the DMA engine support */ 969 ret = ccp_dmaengine_register(ccp); 970 if (ret) 971 goto e_hwrng; 972 973 /* Set up debugfs entries */ 974 ccp5_debugfs_setup(ccp); 975 976 return 0; 977 978 e_hwrng: 979 ccp_unregister_rng(ccp); 980 981 e_kthread: 982 for (i = 0; i < ccp->cmd_q_count; i++) 983 if (ccp->cmd_q[i].kthread) 984 kthread_stop(ccp->cmd_q[i].kthread); 985 986 e_irq: 987 sp_free_ccp_irq(ccp->sp, ccp); 988 989 e_pool: 990 for (i = 0; i < ccp->cmd_q_count; i++) 991 dma_pool_destroy(ccp->cmd_q[i].dma_pool); 992 993 return ret; 994 } 995 996 static void ccp5_destroy(struct ccp_device *ccp) 997 { 998 struct device *dev = ccp->dev; 999 struct ccp_cmd_queue *cmd_q; 1000 struct ccp_cmd *cmd; 1001 unsigned int i; 1002 1003 /* Unregister the DMA engine */ 1004 ccp_dmaengine_unregister(ccp); 1005 1006 /* Unregister the RNG */ 1007 ccp_unregister_rng(ccp); 1008 1009 /* Remove this device from the list of available units first */ 1010 ccp_del_device(ccp); 1011 1012 /* We're in the process of tearing down the entire driver; 1013 * when all the devices are gone clean up debugfs 1014 */ 1015 if (ccp_present()) 1016 ccp5_debugfs_destroy(); 1017 1018 /* Disable and clear interrupts */ 1019 ccp5_disable_queue_interrupts(ccp); 1020 for (i = 0; i < ccp->cmd_q_count; i++) { 1021 cmd_q = &ccp->cmd_q[i]; 1022 1023 /* Turn off the run bit */ 1024 iowrite32(cmd_q->qcontrol & ~CMD5_Q_RUN, cmd_q->reg_control); 1025 1026 /* Clear the interrupt status */ 1027 iowrite32(SUPPORTED_INTERRUPTS, cmd_q->reg_interrupt_status); 1028 ioread32(cmd_q->reg_int_status); 1029 ioread32(cmd_q->reg_status); 1030 } 1031 1032 /* Stop the queue kthreads */ 1033 for (i = 0; i < ccp->cmd_q_count; i++) 1034 if (ccp->cmd_q[i].kthread) 1035 kthread_stop(ccp->cmd_q[i].kthread); 1036 1037 sp_free_ccp_irq(ccp->sp, ccp); 1038 1039 for (i = 0; i < ccp->cmd_q_count; i++) { 1040 cmd_q = &ccp->cmd_q[i]; 1041 dma_free_coherent(dev, cmd_q->qsize, cmd_q->qbase, 1042 cmd_q->qbase_dma); 1043 } 1044 1045 /* Flush the cmd and backlog queue */ 1046 while (!list_empty(&ccp->cmd)) { 1047 /* Invoke the callback directly with an error code */ 1048 cmd = list_first_entry(&ccp->cmd, struct ccp_cmd, entry); 1049 list_del(&cmd->entry); 1050 cmd->callback(cmd->data, -ENODEV); 1051 } 1052 while (!list_empty(&ccp->backlog)) { 1053 /* Invoke the callback directly with an error code */ 1054 cmd = list_first_entry(&ccp->backlog, struct ccp_cmd, entry); 1055 list_del(&cmd->entry); 1056 cmd->callback(cmd->data, -ENODEV); 1057 } 1058 } 1059 1060 static void ccp5_config(struct ccp_device *ccp) 1061 { 1062 /* Public side */ 1063 iowrite32(0x0, ccp->io_regs + CMD5_REQID_CONFIG_OFFSET); 1064 } 1065 1066 static void ccp5other_config(struct ccp_device *ccp) 1067 { 1068 int i; 1069 u32 rnd; 1070 1071 /* We own all of the queues on the NTB CCP */ 1072 1073 iowrite32(0x00012D57, ccp->io_regs + CMD5_TRNG_CTL_OFFSET); 1074 iowrite32(0x00000003, ccp->io_regs + CMD5_CONFIG_0_OFFSET); 1075 for (i = 0; i < 12; i++) { 1076 rnd = ioread32(ccp->io_regs + TRNG_OUT_REG); 1077 iowrite32(rnd, ccp->io_regs + CMD5_AES_MASK_OFFSET); 1078 } 1079 1080 iowrite32(0x0000001F, ccp->io_regs + CMD5_QUEUE_MASK_OFFSET); 1081 iowrite32(0x00005B6D, ccp->io_regs + CMD5_QUEUE_PRIO_OFFSET); 1082 iowrite32(0x00000000, ccp->io_regs + CMD5_CMD_TIMEOUT_OFFSET); 1083 1084 iowrite32(0x3FFFFFFF, ccp->io_regs + LSB_PRIVATE_MASK_LO_OFFSET); 1085 iowrite32(0x000003FF, ccp->io_regs + LSB_PRIVATE_MASK_HI_OFFSET); 1086 1087 iowrite32(0x00108823, ccp->io_regs + CMD5_CLK_GATE_CTL_OFFSET); 1088 1089 ccp5_config(ccp); 1090 } 1091 1092 /* Version 5 adds some function, but is essentially the same as v5 */ 1093 static const struct ccp_actions ccp5_actions = { 1094 .aes = ccp5_perform_aes, 1095 .xts_aes = ccp5_perform_xts_aes, 1096 .sha = ccp5_perform_sha, 1097 .des3 = ccp5_perform_des3, 1098 .rsa = ccp5_perform_rsa, 1099 .passthru = ccp5_perform_passthru, 1100 .ecc = ccp5_perform_ecc, 1101 .sballoc = ccp_lsb_alloc, 1102 .sbfree = ccp_lsb_free, 1103 .init = ccp5_init, 1104 .destroy = ccp5_destroy, 1105 .get_free_slots = ccp5_get_free_slots, 1106 }; 1107 1108 const struct ccp_vdata ccpv5a = { 1109 .version = CCP_VERSION(5, 0), 1110 .setup = ccp5_config, 1111 .perform = &ccp5_actions, 1112 .offset = 0x0, 1113 .rsamax = CCP5_RSA_MAX_WIDTH, 1114 }; 1115 1116 const struct ccp_vdata ccpv5b = { 1117 .version = CCP_VERSION(5, 0), 1118 .dma_chan_attr = DMA_PRIVATE, 1119 .setup = ccp5other_config, 1120 .perform = &ccp5_actions, 1121 .offset = 0x0, 1122 .rsamax = CCP5_RSA_MAX_WIDTH, 1123 }; 1124