1 /* 2 * AMD Cryptographic Coprocessor (CCP) driver 3 * 4 * Copyright (C) 2016 Advanced Micro Devices, Inc. 5 * 6 * Author: Gary R Hook <gary.hook@amd.com> 7 * 8 * This program is free software; you can redistribute it and/or modify 9 * it under the terms of the GNU General Public License version 2 as 10 * published by the Free Software Foundation. 11 */ 12 13 #include <linux/module.h> 14 #include <linux/kernel.h> 15 #include <linux/pci.h> 16 #include <linux/kthread.h> 17 #include <linux/dma-mapping.h> 18 #include <linux/interrupt.h> 19 #include <linux/compiler.h> 20 #include <linux/ccp.h> 21 22 #include "ccp-dev.h" 23 24 /* Allocate the requested number of contiguous LSB slots 25 * from the LSB bitmap. Look in the private range for this 26 * queue first; failing that, check the public area. 27 * If no space is available, wait around. 28 * Return: first slot number 29 */ 30 static u32 ccp_lsb_alloc(struct ccp_cmd_queue *cmd_q, unsigned int count) 31 { 32 struct ccp_device *ccp; 33 int start; 34 35 /* First look at the map for the queue */ 36 if (cmd_q->lsb >= 0) { 37 start = (u32)bitmap_find_next_zero_area(cmd_q->lsbmap, 38 LSB_SIZE, 39 0, count, 0); 40 if (start < LSB_SIZE) { 41 bitmap_set(cmd_q->lsbmap, start, count); 42 return start + cmd_q->lsb * LSB_SIZE; 43 } 44 } 45 46 /* No joy; try to get an entry from the shared blocks */ 47 ccp = cmd_q->ccp; 48 for (;;) { 49 mutex_lock(&ccp->sb_mutex); 50 51 start = (u32)bitmap_find_next_zero_area(ccp->lsbmap, 52 MAX_LSB_CNT * LSB_SIZE, 53 0, 54 count, 0); 55 if (start <= MAX_LSB_CNT * LSB_SIZE) { 56 bitmap_set(ccp->lsbmap, start, count); 57 58 mutex_unlock(&ccp->sb_mutex); 59 return start; 60 } 61 62 ccp->sb_avail = 0; 63 64 mutex_unlock(&ccp->sb_mutex); 65 66 /* Wait for KSB entries to become available */ 67 if (wait_event_interruptible(ccp->sb_queue, ccp->sb_avail)) 68 return 0; 69 } 70 } 71 72 /* Free a number of LSB slots from the bitmap, starting at 73 * the indicated starting slot number. 74 */ 75 static void ccp_lsb_free(struct ccp_cmd_queue *cmd_q, unsigned int start, 76 unsigned int count) 77 { 78 if (!start) 79 return; 80 81 if (cmd_q->lsb == start) { 82 /* An entry from the private LSB */ 83 bitmap_clear(cmd_q->lsbmap, start, count); 84 } else { 85 /* From the shared LSBs */ 86 struct ccp_device *ccp = cmd_q->ccp; 87 88 mutex_lock(&ccp->sb_mutex); 89 bitmap_clear(ccp->lsbmap, start, count); 90 ccp->sb_avail = 1; 91 mutex_unlock(&ccp->sb_mutex); 92 wake_up_interruptible_all(&ccp->sb_queue); 93 } 94 } 95 96 /* CCP version 5: Union to define the function field (cmd_reg1/dword0) */ 97 union ccp_function { 98 struct { 99 u16 size:7; 100 u16 encrypt:1; 101 u16 mode:5; 102 u16 type:2; 103 } aes; 104 struct { 105 u16 size:7; 106 u16 encrypt:1; 107 u16 rsvd:5; 108 u16 type:2; 109 } aes_xts; 110 struct { 111 u16 size:7; 112 u16 encrypt:1; 113 u16 mode:5; 114 u16 type:2; 115 } des3; 116 struct { 117 u16 rsvd1:10; 118 u16 type:4; 119 u16 rsvd2:1; 120 } sha; 121 struct { 122 u16 mode:3; 123 u16 size:12; 124 } rsa; 125 struct { 126 u16 byteswap:2; 127 u16 bitwise:3; 128 u16 reflect:2; 129 u16 rsvd:8; 130 } pt; 131 struct { 132 u16 rsvd:13; 133 } zlib; 134 struct { 135 u16 size:10; 136 u16 type:2; 137 u16 mode:3; 138 } ecc; 139 u16 raw; 140 }; 141 142 #define CCP_AES_SIZE(p) ((p)->aes.size) 143 #define CCP_AES_ENCRYPT(p) ((p)->aes.encrypt) 144 #define CCP_AES_MODE(p) ((p)->aes.mode) 145 #define CCP_AES_TYPE(p) ((p)->aes.type) 146 #define CCP_XTS_SIZE(p) ((p)->aes_xts.size) 147 #define CCP_XTS_ENCRYPT(p) ((p)->aes_xts.encrypt) 148 #define CCP_DES3_SIZE(p) ((p)->des3.size) 149 #define CCP_DES3_ENCRYPT(p) ((p)->des3.encrypt) 150 #define CCP_DES3_MODE(p) ((p)->des3.mode) 151 #define CCP_DES3_TYPE(p) ((p)->des3.type) 152 #define CCP_SHA_TYPE(p) ((p)->sha.type) 153 #define CCP_RSA_SIZE(p) ((p)->rsa.size) 154 #define CCP_PT_BYTESWAP(p) ((p)->pt.byteswap) 155 #define CCP_PT_BITWISE(p) ((p)->pt.bitwise) 156 #define CCP_ECC_MODE(p) ((p)->ecc.mode) 157 #define CCP_ECC_AFFINE(p) ((p)->ecc.one) 158 159 /* Word 0 */ 160 #define CCP5_CMD_DW0(p) ((p)->dw0) 161 #define CCP5_CMD_SOC(p) (CCP5_CMD_DW0(p).soc) 162 #define CCP5_CMD_IOC(p) (CCP5_CMD_DW0(p).ioc) 163 #define CCP5_CMD_INIT(p) (CCP5_CMD_DW0(p).init) 164 #define CCP5_CMD_EOM(p) (CCP5_CMD_DW0(p).eom) 165 #define CCP5_CMD_FUNCTION(p) (CCP5_CMD_DW0(p).function) 166 #define CCP5_CMD_ENGINE(p) (CCP5_CMD_DW0(p).engine) 167 #define CCP5_CMD_PROT(p) (CCP5_CMD_DW0(p).prot) 168 169 /* Word 1 */ 170 #define CCP5_CMD_DW1(p) ((p)->length) 171 #define CCP5_CMD_LEN(p) (CCP5_CMD_DW1(p)) 172 173 /* Word 2 */ 174 #define CCP5_CMD_DW2(p) ((p)->src_lo) 175 #define CCP5_CMD_SRC_LO(p) (CCP5_CMD_DW2(p)) 176 177 /* Word 3 */ 178 #define CCP5_CMD_DW3(p) ((p)->dw3) 179 #define CCP5_CMD_SRC_MEM(p) ((p)->dw3.src_mem) 180 #define CCP5_CMD_SRC_HI(p) ((p)->dw3.src_hi) 181 #define CCP5_CMD_LSB_ID(p) ((p)->dw3.lsb_cxt_id) 182 #define CCP5_CMD_FIX_SRC(p) ((p)->dw3.fixed) 183 184 /* Words 4/5 */ 185 #define CCP5_CMD_DW4(p) ((p)->dw4) 186 #define CCP5_CMD_DST_LO(p) (CCP5_CMD_DW4(p).dst_lo) 187 #define CCP5_CMD_DW5(p) ((p)->dw5.fields.dst_hi) 188 #define CCP5_CMD_DST_HI(p) (CCP5_CMD_DW5(p)) 189 #define CCP5_CMD_DST_MEM(p) ((p)->dw5.fields.dst_mem) 190 #define CCP5_CMD_FIX_DST(p) ((p)->dw5.fields.fixed) 191 #define CCP5_CMD_SHA_LO(p) ((p)->dw4.sha_len_lo) 192 #define CCP5_CMD_SHA_HI(p) ((p)->dw5.sha_len_hi) 193 194 /* Word 6/7 */ 195 #define CCP5_CMD_DW6(p) ((p)->key_lo) 196 #define CCP5_CMD_KEY_LO(p) (CCP5_CMD_DW6(p)) 197 #define CCP5_CMD_DW7(p) ((p)->dw7) 198 #define CCP5_CMD_KEY_HI(p) ((p)->dw7.key_hi) 199 #define CCP5_CMD_KEY_MEM(p) ((p)->dw7.key_mem) 200 201 static inline u32 low_address(unsigned long addr) 202 { 203 return (u64)addr & 0x0ffffffff; 204 } 205 206 static inline u32 high_address(unsigned long addr) 207 { 208 return ((u64)addr >> 32) & 0x00000ffff; 209 } 210 211 static unsigned int ccp5_get_free_slots(struct ccp_cmd_queue *cmd_q) 212 { 213 unsigned int head_idx, n; 214 u32 head_lo, queue_start; 215 216 queue_start = low_address(cmd_q->qdma_tail); 217 head_lo = ioread32(cmd_q->reg_head_lo); 218 head_idx = (head_lo - queue_start) / sizeof(struct ccp5_desc); 219 220 n = head_idx + COMMANDS_PER_QUEUE - cmd_q->qidx - 1; 221 222 return n % COMMANDS_PER_QUEUE; /* Always one unused spot */ 223 } 224 225 static int ccp5_do_cmd(struct ccp5_desc *desc, 226 struct ccp_cmd_queue *cmd_q) 227 { 228 u32 *mP; 229 __le32 *dP; 230 u32 tail; 231 int i; 232 int ret = 0; 233 234 if (CCP5_CMD_SOC(desc)) { 235 CCP5_CMD_IOC(desc) = 1; 236 CCP5_CMD_SOC(desc) = 0; 237 } 238 mutex_lock(&cmd_q->q_mutex); 239 240 mP = (u32 *) &cmd_q->qbase[cmd_q->qidx]; 241 dP = (__le32 *) desc; 242 for (i = 0; i < 8; i++) 243 mP[i] = cpu_to_le32(dP[i]); /* handle endianness */ 244 245 cmd_q->qidx = (cmd_q->qidx + 1) % COMMANDS_PER_QUEUE; 246 247 /* The data used by this command must be flushed to memory */ 248 wmb(); 249 250 /* Write the new tail address back to the queue register */ 251 tail = low_address(cmd_q->qdma_tail + cmd_q->qidx * Q_DESC_SIZE); 252 iowrite32(tail, cmd_q->reg_tail_lo); 253 254 /* Turn the queue back on using our cached control register */ 255 iowrite32(cmd_q->qcontrol | CMD5_Q_RUN, cmd_q->reg_control); 256 mutex_unlock(&cmd_q->q_mutex); 257 258 if (CCP5_CMD_IOC(desc)) { 259 /* Wait for the job to complete */ 260 ret = wait_event_interruptible(cmd_q->int_queue, 261 cmd_q->int_rcvd); 262 if (ret || cmd_q->cmd_error) { 263 /* Log the error and flush the queue by 264 * moving the head pointer 265 */ 266 if (cmd_q->cmd_error) 267 ccp_log_error(cmd_q->ccp, 268 cmd_q->cmd_error); 269 iowrite32(tail, cmd_q->reg_head_lo); 270 if (!ret) 271 ret = -EIO; 272 } 273 cmd_q->int_rcvd = 0; 274 } 275 276 return ret; 277 } 278 279 static int ccp5_perform_aes(struct ccp_op *op) 280 { 281 struct ccp5_desc desc; 282 union ccp_function function; 283 u32 key_addr = op->sb_key * LSB_ITEM_SIZE; 284 285 /* Zero out all the fields of the command desc */ 286 memset(&desc, 0, Q_DESC_SIZE); 287 288 CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_AES; 289 290 CCP5_CMD_SOC(&desc) = op->soc; 291 CCP5_CMD_IOC(&desc) = 1; 292 CCP5_CMD_INIT(&desc) = op->init; 293 CCP5_CMD_EOM(&desc) = op->eom; 294 CCP5_CMD_PROT(&desc) = 0; 295 296 function.raw = 0; 297 CCP_AES_ENCRYPT(&function) = op->u.aes.action; 298 CCP_AES_MODE(&function) = op->u.aes.mode; 299 CCP_AES_TYPE(&function) = op->u.aes.type; 300 CCP_AES_SIZE(&function) = op->u.aes.size; 301 302 CCP5_CMD_FUNCTION(&desc) = function.raw; 303 304 CCP5_CMD_LEN(&desc) = op->src.u.dma.length; 305 306 CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma); 307 CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma); 308 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM; 309 310 CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma); 311 CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma); 312 CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM; 313 314 CCP5_CMD_KEY_LO(&desc) = lower_32_bits(key_addr); 315 CCP5_CMD_KEY_HI(&desc) = 0; 316 CCP5_CMD_KEY_MEM(&desc) = CCP_MEMTYPE_SB; 317 CCP5_CMD_LSB_ID(&desc) = op->sb_ctx; 318 319 return ccp5_do_cmd(&desc, op->cmd_q); 320 } 321 322 static int ccp5_perform_xts_aes(struct ccp_op *op) 323 { 324 struct ccp5_desc desc; 325 union ccp_function function; 326 u32 key_addr = op->sb_key * LSB_ITEM_SIZE; 327 328 /* Zero out all the fields of the command desc */ 329 memset(&desc, 0, Q_DESC_SIZE); 330 331 CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_XTS_AES_128; 332 333 CCP5_CMD_SOC(&desc) = op->soc; 334 CCP5_CMD_IOC(&desc) = 1; 335 CCP5_CMD_INIT(&desc) = op->init; 336 CCP5_CMD_EOM(&desc) = op->eom; 337 CCP5_CMD_PROT(&desc) = 0; 338 339 function.raw = 0; 340 CCP_XTS_ENCRYPT(&function) = op->u.xts.action; 341 CCP_XTS_SIZE(&function) = op->u.xts.unit_size; 342 CCP5_CMD_FUNCTION(&desc) = function.raw; 343 344 CCP5_CMD_LEN(&desc) = op->src.u.dma.length; 345 346 CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma); 347 CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma); 348 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM; 349 350 CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma); 351 CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma); 352 CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM; 353 354 CCP5_CMD_KEY_LO(&desc) = lower_32_bits(key_addr); 355 CCP5_CMD_KEY_HI(&desc) = 0; 356 CCP5_CMD_KEY_MEM(&desc) = CCP_MEMTYPE_SB; 357 CCP5_CMD_LSB_ID(&desc) = op->sb_ctx; 358 359 return ccp5_do_cmd(&desc, op->cmd_q); 360 } 361 362 static int ccp5_perform_sha(struct ccp_op *op) 363 { 364 struct ccp5_desc desc; 365 union ccp_function function; 366 367 /* Zero out all the fields of the command desc */ 368 memset(&desc, 0, Q_DESC_SIZE); 369 370 CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_SHA; 371 372 CCP5_CMD_SOC(&desc) = op->soc; 373 CCP5_CMD_IOC(&desc) = 1; 374 CCP5_CMD_INIT(&desc) = 1; 375 CCP5_CMD_EOM(&desc) = op->eom; 376 CCP5_CMD_PROT(&desc) = 0; 377 378 function.raw = 0; 379 CCP_SHA_TYPE(&function) = op->u.sha.type; 380 CCP5_CMD_FUNCTION(&desc) = function.raw; 381 382 CCP5_CMD_LEN(&desc) = op->src.u.dma.length; 383 384 CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma); 385 CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma); 386 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM; 387 388 CCP5_CMD_LSB_ID(&desc) = op->sb_ctx; 389 390 if (op->eom) { 391 CCP5_CMD_SHA_LO(&desc) = lower_32_bits(op->u.sha.msg_bits); 392 CCP5_CMD_SHA_HI(&desc) = upper_32_bits(op->u.sha.msg_bits); 393 } else { 394 CCP5_CMD_SHA_LO(&desc) = 0; 395 CCP5_CMD_SHA_HI(&desc) = 0; 396 } 397 398 return ccp5_do_cmd(&desc, op->cmd_q); 399 } 400 401 static int ccp5_perform_des3(struct ccp_op *op) 402 { 403 struct ccp5_desc desc; 404 union ccp_function function; 405 u32 key_addr = op->sb_key * LSB_ITEM_SIZE; 406 407 /* Zero out all the fields of the command desc */ 408 memset(&desc, 0, sizeof(struct ccp5_desc)); 409 410 CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_DES3; 411 412 CCP5_CMD_SOC(&desc) = op->soc; 413 CCP5_CMD_IOC(&desc) = 1; 414 CCP5_CMD_INIT(&desc) = op->init; 415 CCP5_CMD_EOM(&desc) = op->eom; 416 CCP5_CMD_PROT(&desc) = 0; 417 418 function.raw = 0; 419 CCP_DES3_ENCRYPT(&function) = op->u.des3.action; 420 CCP_DES3_MODE(&function) = op->u.des3.mode; 421 CCP_DES3_TYPE(&function) = op->u.des3.type; 422 CCP5_CMD_FUNCTION(&desc) = function.raw; 423 424 CCP5_CMD_LEN(&desc) = op->src.u.dma.length; 425 426 CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma); 427 CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma); 428 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM; 429 430 CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma); 431 CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma); 432 CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM; 433 434 CCP5_CMD_KEY_LO(&desc) = lower_32_bits(key_addr); 435 CCP5_CMD_KEY_HI(&desc) = 0; 436 CCP5_CMD_KEY_MEM(&desc) = CCP_MEMTYPE_SB; 437 CCP5_CMD_LSB_ID(&desc) = op->sb_ctx; 438 439 return ccp5_do_cmd(&desc, op->cmd_q); 440 } 441 442 static int ccp5_perform_rsa(struct ccp_op *op) 443 { 444 struct ccp5_desc desc; 445 union ccp_function function; 446 447 /* Zero out all the fields of the command desc */ 448 memset(&desc, 0, Q_DESC_SIZE); 449 450 CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_RSA; 451 452 CCP5_CMD_SOC(&desc) = op->soc; 453 CCP5_CMD_IOC(&desc) = 1; 454 CCP5_CMD_INIT(&desc) = 0; 455 CCP5_CMD_EOM(&desc) = 1; 456 CCP5_CMD_PROT(&desc) = 0; 457 458 function.raw = 0; 459 CCP_RSA_SIZE(&function) = op->u.rsa.mod_size >> 3; 460 CCP5_CMD_FUNCTION(&desc) = function.raw; 461 462 CCP5_CMD_LEN(&desc) = op->u.rsa.input_len; 463 464 /* Source is from external memory */ 465 CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma); 466 CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma); 467 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM; 468 469 /* Destination is in external memory */ 470 CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma); 471 CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma); 472 CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM; 473 474 /* Exponent is in LSB memory */ 475 CCP5_CMD_KEY_LO(&desc) = op->sb_key * LSB_ITEM_SIZE; 476 CCP5_CMD_KEY_HI(&desc) = 0; 477 CCP5_CMD_KEY_MEM(&desc) = CCP_MEMTYPE_SB; 478 479 return ccp5_do_cmd(&desc, op->cmd_q); 480 } 481 482 static int ccp5_perform_passthru(struct ccp_op *op) 483 { 484 struct ccp5_desc desc; 485 union ccp_function function; 486 struct ccp_dma_info *saddr = &op->src.u.dma; 487 struct ccp_dma_info *daddr = &op->dst.u.dma; 488 489 490 memset(&desc, 0, Q_DESC_SIZE); 491 492 CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_PASSTHRU; 493 494 CCP5_CMD_SOC(&desc) = 0; 495 CCP5_CMD_IOC(&desc) = 1; 496 CCP5_CMD_INIT(&desc) = 0; 497 CCP5_CMD_EOM(&desc) = op->eom; 498 CCP5_CMD_PROT(&desc) = 0; 499 500 function.raw = 0; 501 CCP_PT_BYTESWAP(&function) = op->u.passthru.byte_swap; 502 CCP_PT_BITWISE(&function) = op->u.passthru.bit_mod; 503 CCP5_CMD_FUNCTION(&desc) = function.raw; 504 505 /* Length of source data is always 256 bytes */ 506 if (op->src.type == CCP_MEMTYPE_SYSTEM) 507 CCP5_CMD_LEN(&desc) = saddr->length; 508 else 509 CCP5_CMD_LEN(&desc) = daddr->length; 510 511 if (op->src.type == CCP_MEMTYPE_SYSTEM) { 512 CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma); 513 CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma); 514 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM; 515 516 if (op->u.passthru.bit_mod != CCP_PASSTHRU_BITWISE_NOOP) 517 CCP5_CMD_LSB_ID(&desc) = op->sb_key; 518 } else { 519 u32 key_addr = op->src.u.sb * CCP_SB_BYTES; 520 521 CCP5_CMD_SRC_LO(&desc) = lower_32_bits(key_addr); 522 CCP5_CMD_SRC_HI(&desc) = 0; 523 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SB; 524 } 525 526 if (op->dst.type == CCP_MEMTYPE_SYSTEM) { 527 CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma); 528 CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma); 529 CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM; 530 } else { 531 u32 key_addr = op->dst.u.sb * CCP_SB_BYTES; 532 533 CCP5_CMD_DST_LO(&desc) = lower_32_bits(key_addr); 534 CCP5_CMD_DST_HI(&desc) = 0; 535 CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SB; 536 } 537 538 return ccp5_do_cmd(&desc, op->cmd_q); 539 } 540 541 static int ccp5_perform_ecc(struct ccp_op *op) 542 { 543 struct ccp5_desc desc; 544 union ccp_function function; 545 546 /* Zero out all the fields of the command desc */ 547 memset(&desc, 0, Q_DESC_SIZE); 548 549 CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_ECC; 550 551 CCP5_CMD_SOC(&desc) = 0; 552 CCP5_CMD_IOC(&desc) = 1; 553 CCP5_CMD_INIT(&desc) = 0; 554 CCP5_CMD_EOM(&desc) = 1; 555 CCP5_CMD_PROT(&desc) = 0; 556 557 function.raw = 0; 558 function.ecc.mode = op->u.ecc.function; 559 CCP5_CMD_FUNCTION(&desc) = function.raw; 560 561 CCP5_CMD_LEN(&desc) = op->src.u.dma.length; 562 563 CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma); 564 CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma); 565 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM; 566 567 CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma); 568 CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma); 569 CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM; 570 571 return ccp5_do_cmd(&desc, op->cmd_q); 572 } 573 574 static int ccp_find_lsb_regions(struct ccp_cmd_queue *cmd_q, u64 status) 575 { 576 int q_mask = 1 << cmd_q->id; 577 int queues = 0; 578 int j; 579 580 /* Build a bit mask to know which LSBs this queue has access to. 581 * Don't bother with segment 0 as it has special privileges. 582 */ 583 for (j = 1; j < MAX_LSB_CNT; j++) { 584 if (status & q_mask) 585 bitmap_set(cmd_q->lsbmask, j, 1); 586 status >>= LSB_REGION_WIDTH; 587 } 588 queues = bitmap_weight(cmd_q->lsbmask, MAX_LSB_CNT); 589 dev_dbg(cmd_q->ccp->dev, "Queue %d can access %d LSB regions\n", 590 cmd_q->id, queues); 591 592 return queues ? 0 : -EINVAL; 593 } 594 595 596 static int ccp_find_and_assign_lsb_to_q(struct ccp_device *ccp, 597 int lsb_cnt, int n_lsbs, 598 unsigned long *lsb_pub) 599 { 600 DECLARE_BITMAP(qlsb, MAX_LSB_CNT); 601 int bitno; 602 int qlsb_wgt; 603 int i; 604 605 /* For each queue: 606 * If the count of potential LSBs available to a queue matches the 607 * ordinal given to us in lsb_cnt: 608 * Copy the mask of possible LSBs for this queue into "qlsb"; 609 * For each bit in qlsb, see if the corresponding bit in the 610 * aggregation mask is set; if so, we have a match. 611 * If we have a match, clear the bit in the aggregation to 612 * mark it as no longer available. 613 * If there is no match, clear the bit in qlsb and keep looking. 614 */ 615 for (i = 0; i < ccp->cmd_q_count; i++) { 616 struct ccp_cmd_queue *cmd_q = &ccp->cmd_q[i]; 617 618 qlsb_wgt = bitmap_weight(cmd_q->lsbmask, MAX_LSB_CNT); 619 620 if (qlsb_wgt == lsb_cnt) { 621 bitmap_copy(qlsb, cmd_q->lsbmask, MAX_LSB_CNT); 622 623 bitno = find_first_bit(qlsb, MAX_LSB_CNT); 624 while (bitno < MAX_LSB_CNT) { 625 if (test_bit(bitno, lsb_pub)) { 626 /* We found an available LSB 627 * that this queue can access 628 */ 629 cmd_q->lsb = bitno; 630 bitmap_clear(lsb_pub, bitno, 1); 631 dev_dbg(ccp->dev, 632 "Queue %d gets LSB %d\n", 633 i, bitno); 634 break; 635 } 636 bitmap_clear(qlsb, bitno, 1); 637 bitno = find_first_bit(qlsb, MAX_LSB_CNT); 638 } 639 if (bitno >= MAX_LSB_CNT) 640 return -EINVAL; 641 n_lsbs--; 642 } 643 } 644 return n_lsbs; 645 } 646 647 /* For each queue, from the most- to least-constrained: 648 * find an LSB that can be assigned to the queue. If there are N queues that 649 * can only use M LSBs, where N > M, fail; otherwise, every queue will get a 650 * dedicated LSB. Remaining LSB regions become a shared resource. 651 * If we have fewer LSBs than queues, all LSB regions become shared resources. 652 */ 653 static int ccp_assign_lsbs(struct ccp_device *ccp) 654 { 655 DECLARE_BITMAP(lsb_pub, MAX_LSB_CNT); 656 DECLARE_BITMAP(qlsb, MAX_LSB_CNT); 657 int n_lsbs = 0; 658 int bitno; 659 int i, lsb_cnt; 660 int rc = 0; 661 662 bitmap_zero(lsb_pub, MAX_LSB_CNT); 663 664 /* Create an aggregate bitmap to get a total count of available LSBs */ 665 for (i = 0; i < ccp->cmd_q_count; i++) 666 bitmap_or(lsb_pub, 667 lsb_pub, ccp->cmd_q[i].lsbmask, 668 MAX_LSB_CNT); 669 670 n_lsbs = bitmap_weight(lsb_pub, MAX_LSB_CNT); 671 672 if (n_lsbs >= ccp->cmd_q_count) { 673 /* We have enough LSBS to give every queue a private LSB. 674 * Brute force search to start with the queues that are more 675 * constrained in LSB choice. When an LSB is privately 676 * assigned, it is removed from the public mask. 677 * This is an ugly N squared algorithm with some optimization. 678 */ 679 for (lsb_cnt = 1; 680 n_lsbs && (lsb_cnt <= MAX_LSB_CNT); 681 lsb_cnt++) { 682 rc = ccp_find_and_assign_lsb_to_q(ccp, lsb_cnt, n_lsbs, 683 lsb_pub); 684 if (rc < 0) 685 return -EINVAL; 686 n_lsbs = rc; 687 } 688 } 689 690 rc = 0; 691 /* What's left of the LSBs, according to the public mask, now become 692 * shared. Any zero bits in the lsb_pub mask represent an LSB region 693 * that can't be used as a shared resource, so mark the LSB slots for 694 * them as "in use". 695 */ 696 bitmap_copy(qlsb, lsb_pub, MAX_LSB_CNT); 697 698 bitno = find_first_zero_bit(qlsb, MAX_LSB_CNT); 699 while (bitno < MAX_LSB_CNT) { 700 bitmap_set(ccp->lsbmap, bitno * LSB_SIZE, LSB_SIZE); 701 bitmap_set(qlsb, bitno, 1); 702 bitno = find_first_zero_bit(qlsb, MAX_LSB_CNT); 703 } 704 705 return rc; 706 } 707 708 static void ccp5_disable_queue_interrupts(struct ccp_device *ccp) 709 { 710 unsigned int i; 711 712 for (i = 0; i < ccp->cmd_q_count; i++) 713 iowrite32(0x0, ccp->cmd_q[i].reg_int_enable); 714 } 715 716 static void ccp5_enable_queue_interrupts(struct ccp_device *ccp) 717 { 718 unsigned int i; 719 720 for (i = 0; i < ccp->cmd_q_count; i++) 721 iowrite32(SUPPORTED_INTERRUPTS, ccp->cmd_q[i].reg_int_enable); 722 } 723 724 static void ccp5_irq_bh(unsigned long data) 725 { 726 struct ccp_device *ccp = (struct ccp_device *)data; 727 u32 status; 728 unsigned int i; 729 730 for (i = 0; i < ccp->cmd_q_count; i++) { 731 struct ccp_cmd_queue *cmd_q = &ccp->cmd_q[i]; 732 733 status = ioread32(cmd_q->reg_interrupt_status); 734 735 if (status) { 736 cmd_q->int_status = status; 737 cmd_q->q_status = ioread32(cmd_q->reg_status); 738 cmd_q->q_int_status = ioread32(cmd_q->reg_int_status); 739 740 /* On error, only save the first error value */ 741 if ((status & INT_ERROR) && !cmd_q->cmd_error) 742 cmd_q->cmd_error = CMD_Q_ERROR(cmd_q->q_status); 743 744 cmd_q->int_rcvd = 1; 745 746 /* Acknowledge the interrupt and wake the kthread */ 747 iowrite32(status, cmd_q->reg_interrupt_status); 748 wake_up_interruptible(&cmd_q->int_queue); 749 } 750 } 751 ccp5_enable_queue_interrupts(ccp); 752 } 753 754 static irqreturn_t ccp5_irq_handler(int irq, void *data) 755 { 756 struct device *dev = data; 757 struct ccp_device *ccp = dev_get_drvdata(dev); 758 759 ccp5_disable_queue_interrupts(ccp); 760 if (ccp->use_tasklet) 761 tasklet_schedule(&ccp->irq_tasklet); 762 else 763 ccp5_irq_bh((unsigned long)ccp); 764 return IRQ_HANDLED; 765 } 766 767 static int ccp5_init(struct ccp_device *ccp) 768 { 769 struct device *dev = ccp->dev; 770 struct ccp_cmd_queue *cmd_q; 771 struct dma_pool *dma_pool; 772 char dma_pool_name[MAX_DMAPOOL_NAME_LEN]; 773 unsigned int qmr, qim, i; 774 u64 status; 775 u32 status_lo, status_hi; 776 int ret; 777 778 /* Find available queues */ 779 qim = 0; 780 qmr = ioread32(ccp->io_regs + Q_MASK_REG); 781 for (i = 0; i < MAX_HW_QUEUES; i++) { 782 783 if (!(qmr & (1 << i))) 784 continue; 785 786 /* Allocate a dma pool for this queue */ 787 snprintf(dma_pool_name, sizeof(dma_pool_name), "%s_q%d", 788 ccp->name, i); 789 dma_pool = dma_pool_create(dma_pool_name, dev, 790 CCP_DMAPOOL_MAX_SIZE, 791 CCP_DMAPOOL_ALIGN, 0); 792 if (!dma_pool) { 793 dev_err(dev, "unable to allocate dma pool\n"); 794 ret = -ENOMEM; 795 } 796 797 cmd_q = &ccp->cmd_q[ccp->cmd_q_count]; 798 ccp->cmd_q_count++; 799 800 cmd_q->ccp = ccp; 801 cmd_q->id = i; 802 cmd_q->dma_pool = dma_pool; 803 mutex_init(&cmd_q->q_mutex); 804 805 /* Page alignment satisfies our needs for N <= 128 */ 806 BUILD_BUG_ON(COMMANDS_PER_QUEUE > 128); 807 cmd_q->qsize = Q_SIZE(Q_DESC_SIZE); 808 cmd_q->qbase = dma_zalloc_coherent(dev, cmd_q->qsize, 809 &cmd_q->qbase_dma, 810 GFP_KERNEL); 811 if (!cmd_q->qbase) { 812 dev_err(dev, "unable to allocate command queue\n"); 813 ret = -ENOMEM; 814 goto e_pool; 815 } 816 817 cmd_q->qidx = 0; 818 /* Preset some register values and masks that are queue 819 * number dependent 820 */ 821 cmd_q->reg_control = ccp->io_regs + 822 CMD5_Q_STATUS_INCR * (i + 1); 823 cmd_q->reg_tail_lo = cmd_q->reg_control + CMD5_Q_TAIL_LO_BASE; 824 cmd_q->reg_head_lo = cmd_q->reg_control + CMD5_Q_HEAD_LO_BASE; 825 cmd_q->reg_int_enable = cmd_q->reg_control + 826 CMD5_Q_INT_ENABLE_BASE; 827 cmd_q->reg_interrupt_status = cmd_q->reg_control + 828 CMD5_Q_INTERRUPT_STATUS_BASE; 829 cmd_q->reg_status = cmd_q->reg_control + CMD5_Q_STATUS_BASE; 830 cmd_q->reg_int_status = cmd_q->reg_control + 831 CMD5_Q_INT_STATUS_BASE; 832 cmd_q->reg_dma_status = cmd_q->reg_control + 833 CMD5_Q_DMA_STATUS_BASE; 834 cmd_q->reg_dma_read_status = cmd_q->reg_control + 835 CMD5_Q_DMA_READ_STATUS_BASE; 836 cmd_q->reg_dma_write_status = cmd_q->reg_control + 837 CMD5_Q_DMA_WRITE_STATUS_BASE; 838 839 init_waitqueue_head(&cmd_q->int_queue); 840 841 dev_dbg(dev, "queue #%u available\n", i); 842 } 843 844 if (ccp->cmd_q_count == 0) { 845 dev_notice(dev, "no command queues available\n"); 846 ret = -EIO; 847 goto e_pool; 848 } 849 850 /* Turn off the queues and disable interrupts until ready */ 851 ccp5_disable_queue_interrupts(ccp); 852 for (i = 0; i < ccp->cmd_q_count; i++) { 853 cmd_q = &ccp->cmd_q[i]; 854 855 cmd_q->qcontrol = 0; /* Start with nothing */ 856 iowrite32(cmd_q->qcontrol, cmd_q->reg_control); 857 858 ioread32(cmd_q->reg_int_status); 859 ioread32(cmd_q->reg_status); 860 861 /* Clear the interrupt status */ 862 iowrite32(SUPPORTED_INTERRUPTS, cmd_q->reg_interrupt_status); 863 } 864 865 dev_dbg(dev, "Requesting an IRQ...\n"); 866 /* Request an irq */ 867 ret = ccp->get_irq(ccp); 868 if (ret) { 869 dev_err(dev, "unable to allocate an IRQ\n"); 870 goto e_pool; 871 } 872 /* Initialize the ISR tasklet */ 873 if (ccp->use_tasklet) 874 tasklet_init(&ccp->irq_tasklet, ccp5_irq_bh, 875 (unsigned long)ccp); 876 877 dev_dbg(dev, "Loading LSB map...\n"); 878 /* Copy the private LSB mask to the public registers */ 879 status_lo = ioread32(ccp->io_regs + LSB_PRIVATE_MASK_LO_OFFSET); 880 status_hi = ioread32(ccp->io_regs + LSB_PRIVATE_MASK_HI_OFFSET); 881 iowrite32(status_lo, ccp->io_regs + LSB_PUBLIC_MASK_LO_OFFSET); 882 iowrite32(status_hi, ccp->io_regs + LSB_PUBLIC_MASK_HI_OFFSET); 883 status = ((u64)status_hi<<30) | (u64)status_lo; 884 885 dev_dbg(dev, "Configuring virtual queues...\n"); 886 /* Configure size of each virtual queue accessible to host */ 887 for (i = 0; i < ccp->cmd_q_count; i++) { 888 u32 dma_addr_lo; 889 u32 dma_addr_hi; 890 891 cmd_q = &ccp->cmd_q[i]; 892 893 cmd_q->qcontrol &= ~(CMD5_Q_SIZE << CMD5_Q_SHIFT); 894 cmd_q->qcontrol |= QUEUE_SIZE_VAL << CMD5_Q_SHIFT; 895 896 cmd_q->qdma_tail = cmd_q->qbase_dma; 897 dma_addr_lo = low_address(cmd_q->qdma_tail); 898 iowrite32((u32)dma_addr_lo, cmd_q->reg_tail_lo); 899 iowrite32((u32)dma_addr_lo, cmd_q->reg_head_lo); 900 901 dma_addr_hi = high_address(cmd_q->qdma_tail); 902 cmd_q->qcontrol |= (dma_addr_hi << 16); 903 iowrite32(cmd_q->qcontrol, cmd_q->reg_control); 904 905 /* Find the LSB regions accessible to the queue */ 906 ccp_find_lsb_regions(cmd_q, status); 907 cmd_q->lsb = -1; /* Unassigned value */ 908 } 909 910 dev_dbg(dev, "Assigning LSBs...\n"); 911 ret = ccp_assign_lsbs(ccp); 912 if (ret) { 913 dev_err(dev, "Unable to assign LSBs (%d)\n", ret); 914 goto e_irq; 915 } 916 917 /* Optimization: pre-allocate LSB slots for each queue */ 918 for (i = 0; i < ccp->cmd_q_count; i++) { 919 ccp->cmd_q[i].sb_key = ccp_lsb_alloc(&ccp->cmd_q[i], 2); 920 ccp->cmd_q[i].sb_ctx = ccp_lsb_alloc(&ccp->cmd_q[i], 2); 921 } 922 923 dev_dbg(dev, "Starting threads...\n"); 924 /* Create a kthread for each queue */ 925 for (i = 0; i < ccp->cmd_q_count; i++) { 926 struct task_struct *kthread; 927 928 cmd_q = &ccp->cmd_q[i]; 929 930 kthread = kthread_create(ccp_cmd_queue_thread, cmd_q, 931 "%s-q%u", ccp->name, cmd_q->id); 932 if (IS_ERR(kthread)) { 933 dev_err(dev, "error creating queue thread (%ld)\n", 934 PTR_ERR(kthread)); 935 ret = PTR_ERR(kthread); 936 goto e_kthread; 937 } 938 939 cmd_q->kthread = kthread; 940 wake_up_process(kthread); 941 } 942 943 dev_dbg(dev, "Enabling interrupts...\n"); 944 ccp5_enable_queue_interrupts(ccp); 945 946 dev_dbg(dev, "Registering device...\n"); 947 /* Put this on the unit list to make it available */ 948 ccp_add_device(ccp); 949 950 ret = ccp_register_rng(ccp); 951 if (ret) 952 goto e_kthread; 953 954 /* Register the DMA engine support */ 955 ret = ccp_dmaengine_register(ccp); 956 if (ret) 957 goto e_hwrng; 958 959 return 0; 960 961 e_hwrng: 962 ccp_unregister_rng(ccp); 963 964 e_kthread: 965 for (i = 0; i < ccp->cmd_q_count; i++) 966 if (ccp->cmd_q[i].kthread) 967 kthread_stop(ccp->cmd_q[i].kthread); 968 969 e_irq: 970 ccp->free_irq(ccp); 971 972 e_pool: 973 for (i = 0; i < ccp->cmd_q_count; i++) 974 dma_pool_destroy(ccp->cmd_q[i].dma_pool); 975 976 return ret; 977 } 978 979 static void ccp5_destroy(struct ccp_device *ccp) 980 { 981 struct device *dev = ccp->dev; 982 struct ccp_cmd_queue *cmd_q; 983 struct ccp_cmd *cmd; 984 unsigned int i; 985 986 /* Unregister the DMA engine */ 987 ccp_dmaengine_unregister(ccp); 988 989 /* Unregister the RNG */ 990 ccp_unregister_rng(ccp); 991 992 /* Remove this device from the list of available units first */ 993 ccp_del_device(ccp); 994 995 /* Disable and clear interrupts */ 996 ccp5_disable_queue_interrupts(ccp); 997 for (i = 0; i < ccp->cmd_q_count; i++) { 998 cmd_q = &ccp->cmd_q[i]; 999 1000 /* Turn off the run bit */ 1001 iowrite32(cmd_q->qcontrol & ~CMD5_Q_RUN, cmd_q->reg_control); 1002 1003 /* Clear the interrupt status */ 1004 iowrite32(SUPPORTED_INTERRUPTS, cmd_q->reg_interrupt_status); 1005 ioread32(cmd_q->reg_int_status); 1006 ioread32(cmd_q->reg_status); 1007 } 1008 1009 /* Stop the queue kthreads */ 1010 for (i = 0; i < ccp->cmd_q_count; i++) 1011 if (ccp->cmd_q[i].kthread) 1012 kthread_stop(ccp->cmd_q[i].kthread); 1013 1014 ccp->free_irq(ccp); 1015 1016 for (i = 0; i < ccp->cmd_q_count; i++) { 1017 cmd_q = &ccp->cmd_q[i]; 1018 dma_free_coherent(dev, cmd_q->qsize, cmd_q->qbase, 1019 cmd_q->qbase_dma); 1020 } 1021 1022 /* Flush the cmd and backlog queue */ 1023 while (!list_empty(&ccp->cmd)) { 1024 /* Invoke the callback directly with an error code */ 1025 cmd = list_first_entry(&ccp->cmd, struct ccp_cmd, entry); 1026 list_del(&cmd->entry); 1027 cmd->callback(cmd->data, -ENODEV); 1028 } 1029 while (!list_empty(&ccp->backlog)) { 1030 /* Invoke the callback directly with an error code */ 1031 cmd = list_first_entry(&ccp->backlog, struct ccp_cmd, entry); 1032 list_del(&cmd->entry); 1033 cmd->callback(cmd->data, -ENODEV); 1034 } 1035 } 1036 1037 static void ccp5_config(struct ccp_device *ccp) 1038 { 1039 /* Public side */ 1040 iowrite32(0x0, ccp->io_regs + CMD5_REQID_CONFIG_OFFSET); 1041 } 1042 1043 static void ccp5other_config(struct ccp_device *ccp) 1044 { 1045 int i; 1046 u32 rnd; 1047 1048 /* We own all of the queues on the NTB CCP */ 1049 1050 iowrite32(0x00012D57, ccp->io_regs + CMD5_TRNG_CTL_OFFSET); 1051 iowrite32(0x00000003, ccp->io_regs + CMD5_CONFIG_0_OFFSET); 1052 for (i = 0; i < 12; i++) { 1053 rnd = ioread32(ccp->io_regs + TRNG_OUT_REG); 1054 iowrite32(rnd, ccp->io_regs + CMD5_AES_MASK_OFFSET); 1055 } 1056 1057 iowrite32(0x0000001F, ccp->io_regs + CMD5_QUEUE_MASK_OFFSET); 1058 iowrite32(0x00005B6D, ccp->io_regs + CMD5_QUEUE_PRIO_OFFSET); 1059 iowrite32(0x00000000, ccp->io_regs + CMD5_CMD_TIMEOUT_OFFSET); 1060 1061 iowrite32(0x3FFFFFFF, ccp->io_regs + LSB_PRIVATE_MASK_LO_OFFSET); 1062 iowrite32(0x000003FF, ccp->io_regs + LSB_PRIVATE_MASK_HI_OFFSET); 1063 1064 iowrite32(0x00108823, ccp->io_regs + CMD5_CLK_GATE_CTL_OFFSET); 1065 1066 ccp5_config(ccp); 1067 } 1068 1069 /* Version 5 adds some function, but is essentially the same as v5 */ 1070 static const struct ccp_actions ccp5_actions = { 1071 .aes = ccp5_perform_aes, 1072 .xts_aes = ccp5_perform_xts_aes, 1073 .sha = ccp5_perform_sha, 1074 .des3 = ccp5_perform_des3, 1075 .rsa = ccp5_perform_rsa, 1076 .passthru = ccp5_perform_passthru, 1077 .ecc = ccp5_perform_ecc, 1078 .sballoc = ccp_lsb_alloc, 1079 .sbfree = ccp_lsb_free, 1080 .init = ccp5_init, 1081 .destroy = ccp5_destroy, 1082 .get_free_slots = ccp5_get_free_slots, 1083 .irqhandler = ccp5_irq_handler, 1084 }; 1085 1086 const struct ccp_vdata ccpv5a = { 1087 .version = CCP_VERSION(5, 0), 1088 .setup = ccp5_config, 1089 .perform = &ccp5_actions, 1090 .bar = 2, 1091 .offset = 0x0, 1092 }; 1093 1094 const struct ccp_vdata ccpv5b = { 1095 .version = CCP_VERSION(5, 0), 1096 .dma_chan_attr = DMA_PRIVATE, 1097 .setup = ccp5other_config, 1098 .perform = &ccp5_actions, 1099 .bar = 2, 1100 .offset = 0x0, 1101 }; 1102