1 /* 2 * Driver for ARTPEC-6 crypto block using the kernel asynchronous crypto api. 3 * 4 * Copyright (C) 2014-2017 Axis Communications AB 5 */ 6 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 7 8 #include <linux/bitfield.h> 9 #include <linux/crypto.h> 10 #include <linux/debugfs.h> 11 #include <linux/delay.h> 12 #include <linux/dma-mapping.h> 13 #include <linux/fault-inject.h> 14 #include <linux/init.h> 15 #include <linux/interrupt.h> 16 #include <linux/kernel.h> 17 #include <linux/list.h> 18 #include <linux/module.h> 19 #include <linux/of.h> 20 #include <linux/platform_device.h> 21 #include <linux/scatterlist.h> 22 #include <linux/slab.h> 23 24 #include <crypto/aes.h> 25 #include <crypto/internal/aead.h> 26 #include <crypto/internal/hash.h> 27 #include <crypto/internal/skcipher.h> 28 #include <crypto/scatterwalk.h> 29 #include <crypto/sha.h> 30 #include <crypto/xts.h> 31 32 /* Max length of a line in all cache levels for Artpec SoCs. */ 33 #define ARTPEC_CACHE_LINE_MAX 32 34 35 #define PDMA_OUT_CFG 0x0000 36 #define PDMA_OUT_BUF_CFG 0x0004 37 #define PDMA_OUT_CMD 0x0008 38 #define PDMA_OUT_DESCRQ_PUSH 0x0010 39 #define PDMA_OUT_DESCRQ_STAT 0x0014 40 41 #define A6_PDMA_IN_CFG 0x0028 42 #define A6_PDMA_IN_BUF_CFG 0x002c 43 #define A6_PDMA_IN_CMD 0x0030 44 #define A6_PDMA_IN_STATQ_PUSH 0x0038 45 #define A6_PDMA_IN_DESCRQ_PUSH 0x0044 46 #define A6_PDMA_IN_DESCRQ_STAT 0x0048 47 #define A6_PDMA_INTR_MASK 0x0068 48 #define A6_PDMA_ACK_INTR 0x006c 49 #define A6_PDMA_MASKED_INTR 0x0074 50 51 #define A7_PDMA_IN_CFG 0x002c 52 #define A7_PDMA_IN_BUF_CFG 0x0030 53 #define A7_PDMA_IN_CMD 0x0034 54 #define A7_PDMA_IN_STATQ_PUSH 0x003c 55 #define A7_PDMA_IN_DESCRQ_PUSH 0x0048 56 #define A7_PDMA_IN_DESCRQ_STAT 0x004C 57 #define A7_PDMA_INTR_MASK 0x006c 58 #define A7_PDMA_ACK_INTR 0x0070 59 #define A7_PDMA_MASKED_INTR 0x0078 60 61 #define PDMA_OUT_CFG_EN BIT(0) 62 63 #define PDMA_OUT_BUF_CFG_DATA_BUF_SIZE GENMASK(4, 0) 64 #define PDMA_OUT_BUF_CFG_DESCR_BUF_SIZE GENMASK(9, 5) 65 66 #define PDMA_OUT_CMD_START BIT(0) 67 #define A6_PDMA_OUT_CMD_STOP BIT(3) 68 #define A7_PDMA_OUT_CMD_STOP BIT(2) 69 70 #define PDMA_OUT_DESCRQ_PUSH_LEN GENMASK(5, 0) 71 #define PDMA_OUT_DESCRQ_PUSH_ADDR GENMASK(31, 6) 72 73 #define PDMA_OUT_DESCRQ_STAT_LEVEL GENMASK(3, 0) 74 #define PDMA_OUT_DESCRQ_STAT_SIZE GENMASK(7, 4) 75 76 #define PDMA_IN_CFG_EN BIT(0) 77 78 #define PDMA_IN_BUF_CFG_DATA_BUF_SIZE GENMASK(4, 0) 79 #define PDMA_IN_BUF_CFG_DESCR_BUF_SIZE GENMASK(9, 5) 80 #define PDMA_IN_BUF_CFG_STAT_BUF_SIZE GENMASK(14, 10) 81 82 #define PDMA_IN_CMD_START BIT(0) 83 #define A6_PDMA_IN_CMD_FLUSH_STAT BIT(2) 84 #define A6_PDMA_IN_CMD_STOP BIT(3) 85 #define A7_PDMA_IN_CMD_FLUSH_STAT BIT(1) 86 #define A7_PDMA_IN_CMD_STOP BIT(2) 87 88 #define PDMA_IN_STATQ_PUSH_LEN GENMASK(5, 0) 89 #define PDMA_IN_STATQ_PUSH_ADDR GENMASK(31, 6) 90 91 #define PDMA_IN_DESCRQ_PUSH_LEN GENMASK(5, 0) 92 #define PDMA_IN_DESCRQ_PUSH_ADDR GENMASK(31, 6) 93 94 #define PDMA_IN_DESCRQ_STAT_LEVEL GENMASK(3, 0) 95 #define PDMA_IN_DESCRQ_STAT_SIZE GENMASK(7, 4) 96 97 #define A6_PDMA_INTR_MASK_IN_DATA BIT(2) 98 #define A6_PDMA_INTR_MASK_IN_EOP BIT(3) 99 #define A6_PDMA_INTR_MASK_IN_EOP_FLUSH BIT(4) 100 101 #define A7_PDMA_INTR_MASK_IN_DATA BIT(3) 102 #define A7_PDMA_INTR_MASK_IN_EOP BIT(4) 103 #define A7_PDMA_INTR_MASK_IN_EOP_FLUSH BIT(5) 104 105 #define A6_CRY_MD_OPER GENMASK(19, 16) 106 107 #define A6_CRY_MD_HASH_SEL_CTX GENMASK(21, 20) 108 #define A6_CRY_MD_HASH_HMAC_FIN BIT(23) 109 110 #define A6_CRY_MD_CIPHER_LEN GENMASK(21, 20) 111 #define A6_CRY_MD_CIPHER_DECR BIT(22) 112 #define A6_CRY_MD_CIPHER_TWEAK BIT(23) 113 #define A6_CRY_MD_CIPHER_DSEQ BIT(24) 114 115 #define A7_CRY_MD_OPER GENMASK(11, 8) 116 117 #define A7_CRY_MD_HASH_SEL_CTX GENMASK(13, 12) 118 #define A7_CRY_MD_HASH_HMAC_FIN BIT(15) 119 120 #define A7_CRY_MD_CIPHER_LEN GENMASK(13, 12) 121 #define A7_CRY_MD_CIPHER_DECR BIT(14) 122 #define A7_CRY_MD_CIPHER_TWEAK BIT(15) 123 #define A7_CRY_MD_CIPHER_DSEQ BIT(16) 124 125 /* DMA metadata constants */ 126 #define regk_crypto_aes_cbc 0x00000002 127 #define regk_crypto_aes_ctr 0x00000003 128 #define regk_crypto_aes_ecb 0x00000001 129 #define regk_crypto_aes_gcm 0x00000004 130 #define regk_crypto_aes_xts 0x00000005 131 #define regk_crypto_cache 0x00000002 132 #define a6_regk_crypto_dlkey 0x0000000a 133 #define a7_regk_crypto_dlkey 0x0000000e 134 #define regk_crypto_ext 0x00000001 135 #define regk_crypto_hmac_sha1 0x00000007 136 #define regk_crypto_hmac_sha256 0x00000009 137 #define regk_crypto_hmac_sha384 0x0000000b 138 #define regk_crypto_hmac_sha512 0x0000000d 139 #define regk_crypto_init 0x00000000 140 #define regk_crypto_key_128 0x00000000 141 #define regk_crypto_key_192 0x00000001 142 #define regk_crypto_key_256 0x00000002 143 #define regk_crypto_null 0x00000000 144 #define regk_crypto_sha1 0x00000006 145 #define regk_crypto_sha256 0x00000008 146 #define regk_crypto_sha384 0x0000000a 147 #define regk_crypto_sha512 0x0000000c 148 149 /* DMA descriptor structures */ 150 struct pdma_descr_ctrl { 151 unsigned char short_descr : 1; 152 unsigned char pad1 : 1; 153 unsigned char eop : 1; 154 unsigned char intr : 1; 155 unsigned char short_len : 3; 156 unsigned char pad2 : 1; 157 } __packed; 158 159 struct pdma_data_descr { 160 unsigned int len : 24; 161 unsigned int buf : 32; 162 } __packed; 163 164 struct pdma_short_descr { 165 unsigned char data[7]; 166 } __packed; 167 168 struct pdma_descr { 169 struct pdma_descr_ctrl ctrl; 170 union { 171 struct pdma_data_descr data; 172 struct pdma_short_descr shrt; 173 }; 174 }; 175 176 struct pdma_stat_descr { 177 unsigned char pad1 : 1; 178 unsigned char pad2 : 1; 179 unsigned char eop : 1; 180 unsigned char pad3 : 5; 181 unsigned int len : 24; 182 }; 183 184 /* Each descriptor array can hold max 64 entries */ 185 #define PDMA_DESCR_COUNT 64 186 187 #define MODULE_NAME "Artpec-6 CA" 188 189 /* Hash modes (including HMAC variants) */ 190 #define ARTPEC6_CRYPTO_HASH_SHA1 1 191 #define ARTPEC6_CRYPTO_HASH_SHA256 2 192 #define ARTPEC6_CRYPTO_HASH_SHA384 3 193 #define ARTPEC6_CRYPTO_HASH_SHA512 4 194 195 /* Crypto modes */ 196 #define ARTPEC6_CRYPTO_CIPHER_AES_ECB 1 197 #define ARTPEC6_CRYPTO_CIPHER_AES_CBC 2 198 #define ARTPEC6_CRYPTO_CIPHER_AES_CTR 3 199 #define ARTPEC6_CRYPTO_CIPHER_AES_XTS 5 200 201 /* The PDMA is a DMA-engine tightly coupled with a ciphering engine. 202 * It operates on a descriptor array with up to 64 descriptor entries. 203 * The arrays must be 64 byte aligned in memory. 204 * 205 * The ciphering unit has no registers and is completely controlled by 206 * a 4-byte metadata that is inserted at the beginning of each dma packet. 207 * 208 * A dma packet is a sequence of descriptors terminated by setting the .eop 209 * field in the final descriptor of the packet. 210 * 211 * Multiple packets are used for providing context data, key data and 212 * the plain/ciphertext. 213 * 214 * PDMA Descriptors (Array) 215 * +------+------+------+~~+-------+------+---- 216 * | 0 | 1 | 2 |~~| 11 EOP| 12 | .... 217 * +--+---+--+---+----+-+~~+-------+----+-+---- 218 * | | | | | 219 * | | | | | 220 * __|__ +-------++-------++-------+ +----+ 221 * | MD | |Payload||Payload||Payload| | MD | 222 * +-----+ +-------++-------++-------+ +----+ 223 */ 224 225 struct artpec6_crypto_bounce_buffer { 226 struct list_head list; 227 size_t length; 228 struct scatterlist *sg; 229 size_t offset; 230 /* buf is aligned to ARTPEC_CACHE_LINE_MAX and 231 * holds up to ARTPEC_CACHE_LINE_MAX bytes data. 232 */ 233 void *buf; 234 }; 235 236 struct artpec6_crypto_dma_map { 237 dma_addr_t dma_addr; 238 size_t size; 239 enum dma_data_direction dir; 240 }; 241 242 struct artpec6_crypto_dma_descriptors { 243 struct pdma_descr out[PDMA_DESCR_COUNT] __aligned(64); 244 struct pdma_descr in[PDMA_DESCR_COUNT] __aligned(64); 245 u32 stat[PDMA_DESCR_COUNT] __aligned(64); 246 struct list_head bounce_buffers; 247 /* Enough maps for all out/in buffers, and all three descr. arrays */ 248 struct artpec6_crypto_dma_map maps[PDMA_DESCR_COUNT * 2 + 2]; 249 dma_addr_t out_dma_addr; 250 dma_addr_t in_dma_addr; 251 dma_addr_t stat_dma_addr; 252 size_t out_cnt; 253 size_t in_cnt; 254 size_t map_count; 255 }; 256 257 enum artpec6_crypto_variant { 258 ARTPEC6_CRYPTO, 259 ARTPEC7_CRYPTO, 260 }; 261 262 struct artpec6_crypto { 263 void __iomem *base; 264 spinlock_t queue_lock; 265 struct list_head queue; /* waiting for pdma fifo space */ 266 struct list_head pending; /* submitted to pdma fifo */ 267 struct tasklet_struct task; 268 struct kmem_cache *dma_cache; 269 int pending_count; 270 struct timer_list timer; 271 enum artpec6_crypto_variant variant; 272 void *pad_buffer; /* cache-aligned block padding buffer */ 273 void *zero_buffer; 274 }; 275 276 enum artpec6_crypto_hash_flags { 277 HASH_FLAG_INIT_CTX = 2, 278 HASH_FLAG_UPDATE = 4, 279 HASH_FLAG_FINALIZE = 8, 280 HASH_FLAG_HMAC = 16, 281 HASH_FLAG_UPDATE_KEY = 32, 282 }; 283 284 struct artpec6_crypto_req_common { 285 struct list_head list; 286 struct artpec6_crypto_dma_descriptors *dma; 287 struct crypto_async_request *req; 288 void (*complete)(struct crypto_async_request *req); 289 gfp_t gfp_flags; 290 }; 291 292 struct artpec6_hash_request_context { 293 char partial_buffer[SHA512_BLOCK_SIZE]; 294 char partial_buffer_out[SHA512_BLOCK_SIZE]; 295 char key_buffer[SHA512_BLOCK_SIZE]; 296 char pad_buffer[SHA512_BLOCK_SIZE + 32]; 297 unsigned char digeststate[SHA512_DIGEST_SIZE]; 298 size_t partial_bytes; 299 u64 digcnt; 300 u32 key_md; 301 u32 hash_md; 302 enum artpec6_crypto_hash_flags hash_flags; 303 struct artpec6_crypto_req_common common; 304 }; 305 306 struct artpec6_hash_export_state { 307 char partial_buffer[SHA512_BLOCK_SIZE]; 308 unsigned char digeststate[SHA512_DIGEST_SIZE]; 309 size_t partial_bytes; 310 u64 digcnt; 311 int oper; 312 unsigned int hash_flags; 313 }; 314 315 struct artpec6_hashalg_context { 316 char hmac_key[SHA512_BLOCK_SIZE]; 317 size_t hmac_key_length; 318 struct crypto_shash *child_hash; 319 }; 320 321 struct artpec6_crypto_request_context { 322 u32 cipher_md; 323 bool decrypt; 324 struct artpec6_crypto_req_common common; 325 }; 326 327 struct artpec6_cryptotfm_context { 328 unsigned char aes_key[2*AES_MAX_KEY_SIZE]; 329 size_t key_length; 330 u32 key_md; 331 int crypto_type; 332 struct crypto_skcipher *fallback; 333 }; 334 335 struct artpec6_crypto_aead_hw_ctx { 336 __be64 aad_length_bits; 337 __be64 text_length_bits; 338 __u8 J0[AES_BLOCK_SIZE]; 339 }; 340 341 struct artpec6_crypto_aead_req_ctx { 342 struct artpec6_crypto_aead_hw_ctx hw_ctx; 343 u32 cipher_md; 344 bool decrypt; 345 struct artpec6_crypto_req_common common; 346 __u8 decryption_tag[AES_BLOCK_SIZE] ____cacheline_aligned; 347 }; 348 349 /* The crypto framework makes it hard to avoid this global. */ 350 static struct device *artpec6_crypto_dev; 351 352 #ifdef CONFIG_FAULT_INJECTION 353 static DECLARE_FAULT_ATTR(artpec6_crypto_fail_status_read); 354 static DECLARE_FAULT_ATTR(artpec6_crypto_fail_dma_array_full); 355 #endif 356 357 enum { 358 ARTPEC6_CRYPTO_PREPARE_HASH_NO_START, 359 ARTPEC6_CRYPTO_PREPARE_HASH_START, 360 }; 361 362 static int artpec6_crypto_prepare_aead(struct aead_request *areq); 363 static int artpec6_crypto_prepare_crypto(struct skcipher_request *areq); 364 static int artpec6_crypto_prepare_hash(struct ahash_request *areq); 365 366 static void 367 artpec6_crypto_complete_crypto(struct crypto_async_request *req); 368 static void 369 artpec6_crypto_complete_cbc_encrypt(struct crypto_async_request *req); 370 static void 371 artpec6_crypto_complete_cbc_decrypt(struct crypto_async_request *req); 372 static void 373 artpec6_crypto_complete_aead(struct crypto_async_request *req); 374 static void 375 artpec6_crypto_complete_hash(struct crypto_async_request *req); 376 377 static int 378 artpec6_crypto_common_destroy(struct artpec6_crypto_req_common *common); 379 380 static void 381 artpec6_crypto_start_dma(struct artpec6_crypto_req_common *common); 382 383 struct artpec6_crypto_walk { 384 struct scatterlist *sg; 385 size_t offset; 386 }; 387 388 static void artpec6_crypto_walk_init(struct artpec6_crypto_walk *awalk, 389 struct scatterlist *sg) 390 { 391 awalk->sg = sg; 392 awalk->offset = 0; 393 } 394 395 static size_t artpec6_crypto_walk_advance(struct artpec6_crypto_walk *awalk, 396 size_t nbytes) 397 { 398 while (nbytes && awalk->sg) { 399 size_t piece; 400 401 WARN_ON(awalk->offset > awalk->sg->length); 402 403 piece = min(nbytes, (size_t)awalk->sg->length - awalk->offset); 404 nbytes -= piece; 405 awalk->offset += piece; 406 if (awalk->offset == awalk->sg->length) { 407 awalk->sg = sg_next(awalk->sg); 408 awalk->offset = 0; 409 } 410 411 } 412 413 return nbytes; 414 } 415 416 static size_t 417 artpec6_crypto_walk_chunklen(const struct artpec6_crypto_walk *awalk) 418 { 419 WARN_ON(awalk->sg->length == awalk->offset); 420 421 return awalk->sg->length - awalk->offset; 422 } 423 424 static dma_addr_t 425 artpec6_crypto_walk_chunk_phys(const struct artpec6_crypto_walk *awalk) 426 { 427 return sg_phys(awalk->sg) + awalk->offset; 428 } 429 430 static void 431 artpec6_crypto_copy_bounce_buffers(struct artpec6_crypto_req_common *common) 432 { 433 struct artpec6_crypto_dma_descriptors *dma = common->dma; 434 struct artpec6_crypto_bounce_buffer *b; 435 struct artpec6_crypto_bounce_buffer *next; 436 437 list_for_each_entry_safe(b, next, &dma->bounce_buffers, list) { 438 pr_debug("bounce entry %p: %zu bytes @ %zu from %p\n", 439 b, b->length, b->offset, b->buf); 440 sg_pcopy_from_buffer(b->sg, 441 1, 442 b->buf, 443 b->length, 444 b->offset); 445 446 list_del(&b->list); 447 kfree(b); 448 } 449 } 450 451 static inline bool artpec6_crypto_busy(void) 452 { 453 struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev); 454 int fifo_count = ac->pending_count; 455 456 return fifo_count > 6; 457 } 458 459 static int artpec6_crypto_submit(struct artpec6_crypto_req_common *req) 460 { 461 struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev); 462 int ret = -EBUSY; 463 464 spin_lock_bh(&ac->queue_lock); 465 466 if (!artpec6_crypto_busy()) { 467 list_add_tail(&req->list, &ac->pending); 468 artpec6_crypto_start_dma(req); 469 ret = -EINPROGRESS; 470 } else if (req->req->flags & CRYPTO_TFM_REQ_MAY_BACKLOG) { 471 list_add_tail(&req->list, &ac->queue); 472 } else { 473 artpec6_crypto_common_destroy(req); 474 } 475 476 spin_unlock_bh(&ac->queue_lock); 477 478 return ret; 479 } 480 481 static void artpec6_crypto_start_dma(struct artpec6_crypto_req_common *common) 482 { 483 struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev); 484 enum artpec6_crypto_variant variant = ac->variant; 485 void __iomem *base = ac->base; 486 struct artpec6_crypto_dma_descriptors *dma = common->dma; 487 u32 ind, statd, outd; 488 489 /* Make descriptor content visible to the DMA before starting it. */ 490 wmb(); 491 492 ind = FIELD_PREP(PDMA_IN_DESCRQ_PUSH_LEN, dma->in_cnt - 1) | 493 FIELD_PREP(PDMA_IN_DESCRQ_PUSH_ADDR, dma->in_dma_addr >> 6); 494 495 statd = FIELD_PREP(PDMA_IN_STATQ_PUSH_LEN, dma->in_cnt - 1) | 496 FIELD_PREP(PDMA_IN_STATQ_PUSH_ADDR, dma->stat_dma_addr >> 6); 497 498 outd = FIELD_PREP(PDMA_OUT_DESCRQ_PUSH_LEN, dma->out_cnt - 1) | 499 FIELD_PREP(PDMA_OUT_DESCRQ_PUSH_ADDR, dma->out_dma_addr >> 6); 500 501 if (variant == ARTPEC6_CRYPTO) { 502 writel_relaxed(ind, base + A6_PDMA_IN_DESCRQ_PUSH); 503 writel_relaxed(statd, base + A6_PDMA_IN_STATQ_PUSH); 504 writel_relaxed(PDMA_IN_CMD_START, base + A6_PDMA_IN_CMD); 505 } else { 506 writel_relaxed(ind, base + A7_PDMA_IN_DESCRQ_PUSH); 507 writel_relaxed(statd, base + A7_PDMA_IN_STATQ_PUSH); 508 writel_relaxed(PDMA_IN_CMD_START, base + A7_PDMA_IN_CMD); 509 } 510 511 writel_relaxed(outd, base + PDMA_OUT_DESCRQ_PUSH); 512 writel_relaxed(PDMA_OUT_CMD_START, base + PDMA_OUT_CMD); 513 514 ac->pending_count++; 515 } 516 517 static void 518 artpec6_crypto_init_dma_operation(struct artpec6_crypto_req_common *common) 519 { 520 struct artpec6_crypto_dma_descriptors *dma = common->dma; 521 522 dma->out_cnt = 0; 523 dma->in_cnt = 0; 524 dma->map_count = 0; 525 INIT_LIST_HEAD(&dma->bounce_buffers); 526 } 527 528 static bool fault_inject_dma_descr(void) 529 { 530 #ifdef CONFIG_FAULT_INJECTION 531 return should_fail(&artpec6_crypto_fail_dma_array_full, 1); 532 #else 533 return false; 534 #endif 535 } 536 537 /** artpec6_crypto_setup_out_descr_phys - Setup an out channel with a 538 * physical address 539 * 540 * @addr: The physical address of the data buffer 541 * @len: The length of the data buffer 542 * @eop: True if this is the last buffer in the packet 543 * 544 * @return 0 on success or -ENOSPC if there are no more descriptors available 545 */ 546 static int 547 artpec6_crypto_setup_out_descr_phys(struct artpec6_crypto_req_common *common, 548 dma_addr_t addr, size_t len, bool eop) 549 { 550 struct artpec6_crypto_dma_descriptors *dma = common->dma; 551 struct pdma_descr *d; 552 553 if (dma->out_cnt >= PDMA_DESCR_COUNT || 554 fault_inject_dma_descr()) { 555 pr_err("No free OUT DMA descriptors available!\n"); 556 return -ENOSPC; 557 } 558 559 d = &dma->out[dma->out_cnt++]; 560 memset(d, 0, sizeof(*d)); 561 562 d->ctrl.short_descr = 0; 563 d->ctrl.eop = eop; 564 d->data.len = len; 565 d->data.buf = addr; 566 return 0; 567 } 568 569 /** artpec6_crypto_setup_out_descr_short - Setup a short out descriptor 570 * 571 * @dst: The virtual address of the data 572 * @len: The length of the data, must be between 1 to 7 bytes 573 * @eop: True if this is the last buffer in the packet 574 * 575 * @return 0 on success 576 * -ENOSPC if no more descriptors are available 577 * -EINVAL if the data length exceeds 7 bytes 578 */ 579 static int 580 artpec6_crypto_setup_out_descr_short(struct artpec6_crypto_req_common *common, 581 void *dst, unsigned int len, bool eop) 582 { 583 struct artpec6_crypto_dma_descriptors *dma = common->dma; 584 struct pdma_descr *d; 585 586 if (dma->out_cnt >= PDMA_DESCR_COUNT || 587 fault_inject_dma_descr()) { 588 pr_err("No free OUT DMA descriptors available!\n"); 589 return -ENOSPC; 590 } else if (len > 7 || len < 1) { 591 return -EINVAL; 592 } 593 d = &dma->out[dma->out_cnt++]; 594 memset(d, 0, sizeof(*d)); 595 596 d->ctrl.short_descr = 1; 597 d->ctrl.short_len = len; 598 d->ctrl.eop = eop; 599 memcpy(d->shrt.data, dst, len); 600 return 0; 601 } 602 603 static int artpec6_crypto_dma_map_page(struct artpec6_crypto_req_common *common, 604 struct page *page, size_t offset, 605 size_t size, 606 enum dma_data_direction dir, 607 dma_addr_t *dma_addr_out) 608 { 609 struct artpec6_crypto_dma_descriptors *dma = common->dma; 610 struct device *dev = artpec6_crypto_dev; 611 struct artpec6_crypto_dma_map *map; 612 dma_addr_t dma_addr; 613 614 *dma_addr_out = 0; 615 616 if (dma->map_count >= ARRAY_SIZE(dma->maps)) 617 return -ENOMEM; 618 619 dma_addr = dma_map_page(dev, page, offset, size, dir); 620 if (dma_mapping_error(dev, dma_addr)) 621 return -ENOMEM; 622 623 map = &dma->maps[dma->map_count++]; 624 map->size = size; 625 map->dma_addr = dma_addr; 626 map->dir = dir; 627 628 *dma_addr_out = dma_addr; 629 630 return 0; 631 } 632 633 static int 634 artpec6_crypto_dma_map_single(struct artpec6_crypto_req_common *common, 635 void *ptr, size_t size, 636 enum dma_data_direction dir, 637 dma_addr_t *dma_addr_out) 638 { 639 struct page *page = virt_to_page(ptr); 640 size_t offset = (uintptr_t)ptr & ~PAGE_MASK; 641 642 return artpec6_crypto_dma_map_page(common, page, offset, size, dir, 643 dma_addr_out); 644 } 645 646 static int 647 artpec6_crypto_dma_map_descs(struct artpec6_crypto_req_common *common) 648 { 649 struct artpec6_crypto_dma_descriptors *dma = common->dma; 650 int ret; 651 652 ret = artpec6_crypto_dma_map_single(common, dma->in, 653 sizeof(dma->in[0]) * dma->in_cnt, 654 DMA_TO_DEVICE, &dma->in_dma_addr); 655 if (ret) 656 return ret; 657 658 ret = artpec6_crypto_dma_map_single(common, dma->out, 659 sizeof(dma->out[0]) * dma->out_cnt, 660 DMA_TO_DEVICE, &dma->out_dma_addr); 661 if (ret) 662 return ret; 663 664 /* We only read one stat descriptor */ 665 dma->stat[dma->in_cnt - 1] = 0; 666 667 /* 668 * DMA_BIDIRECTIONAL since we need our zeroing of the stat descriptor 669 * to be written. 670 */ 671 return artpec6_crypto_dma_map_single(common, 672 dma->stat + dma->in_cnt - 1, 673 sizeof(dma->stat[0]), 674 DMA_BIDIRECTIONAL, 675 &dma->stat_dma_addr); 676 } 677 678 static void 679 artpec6_crypto_dma_unmap_all(struct artpec6_crypto_req_common *common) 680 { 681 struct artpec6_crypto_dma_descriptors *dma = common->dma; 682 struct device *dev = artpec6_crypto_dev; 683 int i; 684 685 for (i = 0; i < dma->map_count; i++) { 686 struct artpec6_crypto_dma_map *map = &dma->maps[i]; 687 688 dma_unmap_page(dev, map->dma_addr, map->size, map->dir); 689 } 690 691 dma->map_count = 0; 692 } 693 694 /** artpec6_crypto_setup_out_descr - Setup an out descriptor 695 * 696 * @dst: The virtual address of the data 697 * @len: The length of the data 698 * @eop: True if this is the last buffer in the packet 699 * @use_short: If this is true and the data length is 7 bytes or less then 700 * a short descriptor will be used 701 * 702 * @return 0 on success 703 * Any errors from artpec6_crypto_setup_out_descr_short() or 704 * setup_out_descr_phys() 705 */ 706 static int 707 artpec6_crypto_setup_out_descr(struct artpec6_crypto_req_common *common, 708 void *dst, unsigned int len, bool eop, 709 bool use_short) 710 { 711 if (use_short && len < 7) { 712 return artpec6_crypto_setup_out_descr_short(common, dst, len, 713 eop); 714 } else { 715 int ret; 716 dma_addr_t dma_addr; 717 718 ret = artpec6_crypto_dma_map_single(common, dst, len, 719 DMA_TO_DEVICE, 720 &dma_addr); 721 if (ret) 722 return ret; 723 724 return artpec6_crypto_setup_out_descr_phys(common, dma_addr, 725 len, eop); 726 } 727 } 728 729 /** artpec6_crypto_setup_in_descr_phys - Setup an in channel with a 730 * physical address 731 * 732 * @addr: The physical address of the data buffer 733 * @len: The length of the data buffer 734 * @intr: True if an interrupt should be fired after HW processing of this 735 * descriptor 736 * 737 */ 738 static int 739 artpec6_crypto_setup_in_descr_phys(struct artpec6_crypto_req_common *common, 740 dma_addr_t addr, unsigned int len, bool intr) 741 { 742 struct artpec6_crypto_dma_descriptors *dma = common->dma; 743 struct pdma_descr *d; 744 745 if (dma->in_cnt >= PDMA_DESCR_COUNT || 746 fault_inject_dma_descr()) { 747 pr_err("No free IN DMA descriptors available!\n"); 748 return -ENOSPC; 749 } 750 d = &dma->in[dma->in_cnt++]; 751 memset(d, 0, sizeof(*d)); 752 753 d->ctrl.intr = intr; 754 d->data.len = len; 755 d->data.buf = addr; 756 return 0; 757 } 758 759 /** artpec6_crypto_setup_in_descr - Setup an in channel descriptor 760 * 761 * @buffer: The virtual address to of the data buffer 762 * @len: The length of the data buffer 763 * @last: If this is the last data buffer in the request (i.e. an interrupt 764 * is needed 765 * 766 * Short descriptors are not used for the in channel 767 */ 768 static int 769 artpec6_crypto_setup_in_descr(struct artpec6_crypto_req_common *common, 770 void *buffer, unsigned int len, bool last) 771 { 772 dma_addr_t dma_addr; 773 int ret; 774 775 ret = artpec6_crypto_dma_map_single(common, buffer, len, 776 DMA_FROM_DEVICE, &dma_addr); 777 if (ret) 778 return ret; 779 780 return artpec6_crypto_setup_in_descr_phys(common, dma_addr, len, last); 781 } 782 783 static struct artpec6_crypto_bounce_buffer * 784 artpec6_crypto_alloc_bounce(gfp_t flags) 785 { 786 void *base; 787 size_t alloc_size = sizeof(struct artpec6_crypto_bounce_buffer) + 788 2 * ARTPEC_CACHE_LINE_MAX; 789 struct artpec6_crypto_bounce_buffer *bbuf = kzalloc(alloc_size, flags); 790 791 if (!bbuf) 792 return NULL; 793 794 base = bbuf + 1; 795 bbuf->buf = PTR_ALIGN(base, ARTPEC_CACHE_LINE_MAX); 796 return bbuf; 797 } 798 799 static int setup_bounce_buffer_in(struct artpec6_crypto_req_common *common, 800 struct artpec6_crypto_walk *walk, size_t size) 801 { 802 struct artpec6_crypto_bounce_buffer *bbuf; 803 int ret; 804 805 bbuf = artpec6_crypto_alloc_bounce(common->gfp_flags); 806 if (!bbuf) 807 return -ENOMEM; 808 809 bbuf->length = size; 810 bbuf->sg = walk->sg; 811 bbuf->offset = walk->offset; 812 813 ret = artpec6_crypto_setup_in_descr(common, bbuf->buf, size, false); 814 if (ret) { 815 kfree(bbuf); 816 return ret; 817 } 818 819 pr_debug("BOUNCE %zu offset %zu\n", size, walk->offset); 820 list_add_tail(&bbuf->list, &common->dma->bounce_buffers); 821 return 0; 822 } 823 824 static int 825 artpec6_crypto_setup_sg_descrs_in(struct artpec6_crypto_req_common *common, 826 struct artpec6_crypto_walk *walk, 827 size_t count) 828 { 829 size_t chunk; 830 int ret; 831 dma_addr_t addr; 832 833 while (walk->sg && count) { 834 chunk = min(count, artpec6_crypto_walk_chunklen(walk)); 835 addr = artpec6_crypto_walk_chunk_phys(walk); 836 837 /* When destination buffers are not aligned to the cache line 838 * size we need bounce buffers. The DMA-API requires that the 839 * entire line is owned by the DMA buffer and this holds also 840 * for the case when coherent DMA is used. 841 */ 842 if (!IS_ALIGNED(addr, ARTPEC_CACHE_LINE_MAX)) { 843 chunk = min_t(dma_addr_t, chunk, 844 ALIGN(addr, ARTPEC_CACHE_LINE_MAX) - 845 addr); 846 847 pr_debug("CHUNK-b %pad:%zu\n", &addr, chunk); 848 ret = setup_bounce_buffer_in(common, walk, chunk); 849 } else if (chunk < ARTPEC_CACHE_LINE_MAX) { 850 pr_debug("CHUNK-b %pad:%zu\n", &addr, chunk); 851 ret = setup_bounce_buffer_in(common, walk, chunk); 852 } else { 853 dma_addr_t dma_addr; 854 855 chunk = chunk & ~(ARTPEC_CACHE_LINE_MAX-1); 856 857 pr_debug("CHUNK %pad:%zu\n", &addr, chunk); 858 859 ret = artpec6_crypto_dma_map_page(common, 860 sg_page(walk->sg), 861 walk->sg->offset + 862 walk->offset, 863 chunk, 864 DMA_FROM_DEVICE, 865 &dma_addr); 866 if (ret) 867 return ret; 868 869 ret = artpec6_crypto_setup_in_descr_phys(common, 870 dma_addr, 871 chunk, false); 872 } 873 874 if (ret) 875 return ret; 876 877 count = count - chunk; 878 artpec6_crypto_walk_advance(walk, chunk); 879 } 880 881 if (count) 882 pr_err("EOL unexpected %zu bytes left\n", count); 883 884 return count ? -EINVAL : 0; 885 } 886 887 static int 888 artpec6_crypto_setup_sg_descrs_out(struct artpec6_crypto_req_common *common, 889 struct artpec6_crypto_walk *walk, 890 size_t count) 891 { 892 size_t chunk; 893 int ret; 894 dma_addr_t addr; 895 896 while (walk->sg && count) { 897 chunk = min(count, artpec6_crypto_walk_chunklen(walk)); 898 addr = artpec6_crypto_walk_chunk_phys(walk); 899 900 pr_debug("OUT-CHUNK %pad:%zu\n", &addr, chunk); 901 902 if (addr & 3) { 903 char buf[3]; 904 905 chunk = min_t(size_t, chunk, (4-(addr&3))); 906 907 sg_pcopy_to_buffer(walk->sg, 1, buf, chunk, 908 walk->offset); 909 910 ret = artpec6_crypto_setup_out_descr_short(common, buf, 911 chunk, 912 false); 913 } else { 914 dma_addr_t dma_addr; 915 916 ret = artpec6_crypto_dma_map_page(common, 917 sg_page(walk->sg), 918 walk->sg->offset + 919 walk->offset, 920 chunk, 921 DMA_TO_DEVICE, 922 &dma_addr); 923 if (ret) 924 return ret; 925 926 ret = artpec6_crypto_setup_out_descr_phys(common, 927 dma_addr, 928 chunk, false); 929 } 930 931 if (ret) 932 return ret; 933 934 count = count - chunk; 935 artpec6_crypto_walk_advance(walk, chunk); 936 } 937 938 if (count) 939 pr_err("EOL unexpected %zu bytes left\n", count); 940 941 return count ? -EINVAL : 0; 942 } 943 944 945 /** artpec6_crypto_terminate_out_descrs - Set the EOP on the last out descriptor 946 * 947 * If the out descriptor list is non-empty, then the eop flag on the 948 * last used out descriptor will be set. 949 * 950 * @return 0 on success 951 * -EINVAL if the out descriptor is empty or has overflown 952 */ 953 static int 954 artpec6_crypto_terminate_out_descrs(struct artpec6_crypto_req_common *common) 955 { 956 struct artpec6_crypto_dma_descriptors *dma = common->dma; 957 struct pdma_descr *d; 958 959 if (!dma->out_cnt || dma->out_cnt > PDMA_DESCR_COUNT) { 960 pr_err("%s: OUT descriptor list is %s\n", 961 MODULE_NAME, dma->out_cnt ? "empty" : "full"); 962 return -EINVAL; 963 964 } 965 966 d = &dma->out[dma->out_cnt-1]; 967 d->ctrl.eop = 1; 968 969 return 0; 970 } 971 972 /** artpec6_crypto_terminate_in_descrs - Set the interrupt flag on the last 973 * in descriptor 974 * 975 * See artpec6_crypto_terminate_out_descrs() for return values 976 */ 977 static int 978 artpec6_crypto_terminate_in_descrs(struct artpec6_crypto_req_common *common) 979 { 980 struct artpec6_crypto_dma_descriptors *dma = common->dma; 981 struct pdma_descr *d; 982 983 if (!dma->in_cnt || dma->in_cnt > PDMA_DESCR_COUNT) { 984 pr_err("%s: IN descriptor list is %s\n", 985 MODULE_NAME, dma->in_cnt ? "empty" : "full"); 986 return -EINVAL; 987 } 988 989 d = &dma->in[dma->in_cnt-1]; 990 d->ctrl.intr = 1; 991 return 0; 992 } 993 994 /** create_hash_pad - Create a Secure Hash conformant pad 995 * 996 * @dst: The destination buffer to write the pad. Must be at least 64 bytes 997 * @dgstlen: The total length of the hash digest in bytes 998 * @bitcount: The total length of the digest in bits 999 * 1000 * @return The total number of padding bytes written to @dst 1001 */ 1002 static size_t 1003 create_hash_pad(int oper, unsigned char *dst, u64 dgstlen, u64 bitcount) 1004 { 1005 unsigned int mod, target, diff, pad_bytes, size_bytes; 1006 __be64 bits = __cpu_to_be64(bitcount); 1007 1008 switch (oper) { 1009 case regk_crypto_sha1: 1010 case regk_crypto_sha256: 1011 case regk_crypto_hmac_sha1: 1012 case regk_crypto_hmac_sha256: 1013 target = 448 / 8; 1014 mod = 512 / 8; 1015 size_bytes = 8; 1016 break; 1017 default: 1018 target = 896 / 8; 1019 mod = 1024 / 8; 1020 size_bytes = 16; 1021 break; 1022 } 1023 1024 target -= 1; 1025 diff = dgstlen & (mod - 1); 1026 pad_bytes = diff > target ? target + mod - diff : target - diff; 1027 1028 memset(dst + 1, 0, pad_bytes); 1029 dst[0] = 0x80; 1030 1031 if (size_bytes == 16) { 1032 memset(dst + 1 + pad_bytes, 0, 8); 1033 memcpy(dst + 1 + pad_bytes + 8, &bits, 8); 1034 } else { 1035 memcpy(dst + 1 + pad_bytes, &bits, 8); 1036 } 1037 1038 return pad_bytes + size_bytes + 1; 1039 } 1040 1041 static int artpec6_crypto_common_init(struct artpec6_crypto_req_common *common, 1042 struct crypto_async_request *parent, 1043 void (*complete)(struct crypto_async_request *req), 1044 struct scatterlist *dstsg, unsigned int nbytes) 1045 { 1046 gfp_t flags; 1047 struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev); 1048 1049 flags = (parent->flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 1050 GFP_KERNEL : GFP_ATOMIC; 1051 1052 common->gfp_flags = flags; 1053 common->dma = kmem_cache_alloc(ac->dma_cache, flags); 1054 if (!common->dma) 1055 return -ENOMEM; 1056 1057 common->req = parent; 1058 common->complete = complete; 1059 return 0; 1060 } 1061 1062 static void 1063 artpec6_crypto_bounce_destroy(struct artpec6_crypto_dma_descriptors *dma) 1064 { 1065 struct artpec6_crypto_bounce_buffer *b; 1066 struct artpec6_crypto_bounce_buffer *next; 1067 1068 list_for_each_entry_safe(b, next, &dma->bounce_buffers, list) { 1069 kfree(b); 1070 } 1071 } 1072 1073 static int 1074 artpec6_crypto_common_destroy(struct artpec6_crypto_req_common *common) 1075 { 1076 struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev); 1077 1078 artpec6_crypto_dma_unmap_all(common); 1079 artpec6_crypto_bounce_destroy(common->dma); 1080 kmem_cache_free(ac->dma_cache, common->dma); 1081 common->dma = NULL; 1082 return 0; 1083 } 1084 1085 /* 1086 * Ciphering functions. 1087 */ 1088 static int artpec6_crypto_encrypt(struct skcipher_request *req) 1089 { 1090 struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req); 1091 struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(cipher); 1092 struct artpec6_crypto_request_context *req_ctx = NULL; 1093 void (*complete)(struct crypto_async_request *req); 1094 int ret; 1095 1096 req_ctx = skcipher_request_ctx(req); 1097 1098 switch (ctx->crypto_type) { 1099 case ARTPEC6_CRYPTO_CIPHER_AES_CBC: 1100 case ARTPEC6_CRYPTO_CIPHER_AES_ECB: 1101 case ARTPEC6_CRYPTO_CIPHER_AES_XTS: 1102 req_ctx->decrypt = 0; 1103 break; 1104 default: 1105 break; 1106 } 1107 1108 switch (ctx->crypto_type) { 1109 case ARTPEC6_CRYPTO_CIPHER_AES_CBC: 1110 complete = artpec6_crypto_complete_cbc_encrypt; 1111 break; 1112 default: 1113 complete = artpec6_crypto_complete_crypto; 1114 break; 1115 } 1116 1117 ret = artpec6_crypto_common_init(&req_ctx->common, 1118 &req->base, 1119 complete, 1120 req->dst, req->cryptlen); 1121 if (ret) 1122 return ret; 1123 1124 ret = artpec6_crypto_prepare_crypto(req); 1125 if (ret) { 1126 artpec6_crypto_common_destroy(&req_ctx->common); 1127 return ret; 1128 } 1129 1130 return artpec6_crypto_submit(&req_ctx->common); 1131 } 1132 1133 static int artpec6_crypto_decrypt(struct skcipher_request *req) 1134 { 1135 int ret; 1136 struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req); 1137 struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(cipher); 1138 struct artpec6_crypto_request_context *req_ctx = NULL; 1139 void (*complete)(struct crypto_async_request *req); 1140 1141 req_ctx = skcipher_request_ctx(req); 1142 1143 switch (ctx->crypto_type) { 1144 case ARTPEC6_CRYPTO_CIPHER_AES_CBC: 1145 case ARTPEC6_CRYPTO_CIPHER_AES_ECB: 1146 case ARTPEC6_CRYPTO_CIPHER_AES_XTS: 1147 req_ctx->decrypt = 1; 1148 break; 1149 default: 1150 break; 1151 } 1152 1153 1154 switch (ctx->crypto_type) { 1155 case ARTPEC6_CRYPTO_CIPHER_AES_CBC: 1156 complete = artpec6_crypto_complete_cbc_decrypt; 1157 break; 1158 default: 1159 complete = artpec6_crypto_complete_crypto; 1160 break; 1161 } 1162 1163 ret = artpec6_crypto_common_init(&req_ctx->common, &req->base, 1164 complete, 1165 req->dst, req->cryptlen); 1166 if (ret) 1167 return ret; 1168 1169 ret = artpec6_crypto_prepare_crypto(req); 1170 if (ret) { 1171 artpec6_crypto_common_destroy(&req_ctx->common); 1172 return ret; 1173 } 1174 1175 return artpec6_crypto_submit(&req_ctx->common); 1176 } 1177 1178 static int 1179 artpec6_crypto_ctr_crypt(struct skcipher_request *req, bool encrypt) 1180 { 1181 struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req); 1182 struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(cipher); 1183 size_t iv_len = crypto_skcipher_ivsize(cipher); 1184 unsigned int counter = be32_to_cpup((__be32 *) 1185 (req->iv + iv_len - 4)); 1186 unsigned int nblks = ALIGN(req->cryptlen, AES_BLOCK_SIZE) / 1187 AES_BLOCK_SIZE; 1188 1189 /* 1190 * The hardware uses only the last 32-bits as the counter while the 1191 * kernel tests (aes_ctr_enc_tv_template[4] for example) expect that 1192 * the whole IV is a counter. So fallback if the counter is going to 1193 * overlow. 1194 */ 1195 if (counter + nblks < counter) { 1196 int ret; 1197 1198 pr_debug("counter %x will overflow (nblks %u), falling back\n", 1199 counter, counter + nblks); 1200 1201 ret = crypto_skcipher_setkey(ctx->fallback, ctx->aes_key, 1202 ctx->key_length); 1203 if (ret) 1204 return ret; 1205 1206 { 1207 SKCIPHER_REQUEST_ON_STACK(subreq, ctx->fallback); 1208 1209 skcipher_request_set_tfm(subreq, ctx->fallback); 1210 skcipher_request_set_callback(subreq, req->base.flags, 1211 NULL, NULL); 1212 skcipher_request_set_crypt(subreq, req->src, req->dst, 1213 req->cryptlen, req->iv); 1214 ret = encrypt ? crypto_skcipher_encrypt(subreq) 1215 : crypto_skcipher_decrypt(subreq); 1216 skcipher_request_zero(subreq); 1217 } 1218 return ret; 1219 } 1220 1221 return encrypt ? artpec6_crypto_encrypt(req) 1222 : artpec6_crypto_decrypt(req); 1223 } 1224 1225 static int artpec6_crypto_ctr_encrypt(struct skcipher_request *req) 1226 { 1227 return artpec6_crypto_ctr_crypt(req, true); 1228 } 1229 1230 static int artpec6_crypto_ctr_decrypt(struct skcipher_request *req) 1231 { 1232 return artpec6_crypto_ctr_crypt(req, false); 1233 } 1234 1235 /* 1236 * AEAD functions 1237 */ 1238 static int artpec6_crypto_aead_init(struct crypto_aead *tfm) 1239 { 1240 struct artpec6_cryptotfm_context *tfm_ctx = crypto_aead_ctx(tfm); 1241 1242 memset(tfm_ctx, 0, sizeof(*tfm_ctx)); 1243 1244 crypto_aead_set_reqsize(tfm, 1245 sizeof(struct artpec6_crypto_aead_req_ctx)); 1246 1247 return 0; 1248 } 1249 1250 static int artpec6_crypto_aead_set_key(struct crypto_aead *tfm, const u8 *key, 1251 unsigned int len) 1252 { 1253 struct artpec6_cryptotfm_context *ctx = crypto_tfm_ctx(&tfm->base); 1254 1255 if (len != 16 && len != 24 && len != 32) { 1256 crypto_aead_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); 1257 return -1; 1258 } 1259 1260 ctx->key_length = len; 1261 1262 memcpy(ctx->aes_key, key, len); 1263 return 0; 1264 } 1265 1266 static int artpec6_crypto_aead_encrypt(struct aead_request *req) 1267 { 1268 int ret; 1269 struct artpec6_crypto_aead_req_ctx *req_ctx = aead_request_ctx(req); 1270 1271 req_ctx->decrypt = false; 1272 ret = artpec6_crypto_common_init(&req_ctx->common, &req->base, 1273 artpec6_crypto_complete_aead, 1274 NULL, 0); 1275 if (ret) 1276 return ret; 1277 1278 ret = artpec6_crypto_prepare_aead(req); 1279 if (ret) { 1280 artpec6_crypto_common_destroy(&req_ctx->common); 1281 return ret; 1282 } 1283 1284 return artpec6_crypto_submit(&req_ctx->common); 1285 } 1286 1287 static int artpec6_crypto_aead_decrypt(struct aead_request *req) 1288 { 1289 int ret; 1290 struct artpec6_crypto_aead_req_ctx *req_ctx = aead_request_ctx(req); 1291 1292 req_ctx->decrypt = true; 1293 if (req->cryptlen < AES_BLOCK_SIZE) 1294 return -EINVAL; 1295 1296 ret = artpec6_crypto_common_init(&req_ctx->common, 1297 &req->base, 1298 artpec6_crypto_complete_aead, 1299 NULL, 0); 1300 if (ret) 1301 return ret; 1302 1303 ret = artpec6_crypto_prepare_aead(req); 1304 if (ret) { 1305 artpec6_crypto_common_destroy(&req_ctx->common); 1306 return ret; 1307 } 1308 1309 return artpec6_crypto_submit(&req_ctx->common); 1310 } 1311 1312 static int artpec6_crypto_prepare_hash(struct ahash_request *areq) 1313 { 1314 struct artpec6_hashalg_context *ctx = crypto_tfm_ctx(areq->base.tfm); 1315 struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(areq); 1316 size_t digestsize = crypto_ahash_digestsize(crypto_ahash_reqtfm(areq)); 1317 size_t contextsize = digestsize == SHA384_DIGEST_SIZE ? 1318 SHA512_DIGEST_SIZE : digestsize; 1319 size_t blocksize = crypto_tfm_alg_blocksize( 1320 crypto_ahash_tfm(crypto_ahash_reqtfm(areq))); 1321 struct artpec6_crypto_req_common *common = &req_ctx->common; 1322 struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev); 1323 enum artpec6_crypto_variant variant = ac->variant; 1324 u32 sel_ctx; 1325 bool ext_ctx = false; 1326 bool run_hw = false; 1327 int error = 0; 1328 1329 artpec6_crypto_init_dma_operation(common); 1330 1331 /* Upload HMAC key, must be first the first packet */ 1332 if (req_ctx->hash_flags & HASH_FLAG_HMAC) { 1333 if (variant == ARTPEC6_CRYPTO) { 1334 req_ctx->key_md = FIELD_PREP(A6_CRY_MD_OPER, 1335 a6_regk_crypto_dlkey); 1336 } else { 1337 req_ctx->key_md = FIELD_PREP(A7_CRY_MD_OPER, 1338 a7_regk_crypto_dlkey); 1339 } 1340 1341 /* Copy and pad up the key */ 1342 memcpy(req_ctx->key_buffer, ctx->hmac_key, 1343 ctx->hmac_key_length); 1344 memset(req_ctx->key_buffer + ctx->hmac_key_length, 0, 1345 blocksize - ctx->hmac_key_length); 1346 1347 error = artpec6_crypto_setup_out_descr(common, 1348 (void *)&req_ctx->key_md, 1349 sizeof(req_ctx->key_md), false, false); 1350 if (error) 1351 return error; 1352 1353 error = artpec6_crypto_setup_out_descr(common, 1354 req_ctx->key_buffer, blocksize, 1355 true, false); 1356 if (error) 1357 return error; 1358 } 1359 1360 if (!(req_ctx->hash_flags & HASH_FLAG_INIT_CTX)) { 1361 /* Restore context */ 1362 sel_ctx = regk_crypto_ext; 1363 ext_ctx = true; 1364 } else { 1365 sel_ctx = regk_crypto_init; 1366 } 1367 1368 if (variant == ARTPEC6_CRYPTO) { 1369 req_ctx->hash_md &= ~A6_CRY_MD_HASH_SEL_CTX; 1370 req_ctx->hash_md |= FIELD_PREP(A6_CRY_MD_HASH_SEL_CTX, sel_ctx); 1371 1372 /* If this is the final round, set the final flag */ 1373 if (req_ctx->hash_flags & HASH_FLAG_FINALIZE) 1374 req_ctx->hash_md |= A6_CRY_MD_HASH_HMAC_FIN; 1375 } else { 1376 req_ctx->hash_md &= ~A7_CRY_MD_HASH_SEL_CTX; 1377 req_ctx->hash_md |= FIELD_PREP(A7_CRY_MD_HASH_SEL_CTX, sel_ctx); 1378 1379 /* If this is the final round, set the final flag */ 1380 if (req_ctx->hash_flags & HASH_FLAG_FINALIZE) 1381 req_ctx->hash_md |= A7_CRY_MD_HASH_HMAC_FIN; 1382 } 1383 1384 /* Setup up metadata descriptors */ 1385 error = artpec6_crypto_setup_out_descr(common, 1386 (void *)&req_ctx->hash_md, 1387 sizeof(req_ctx->hash_md), false, false); 1388 if (error) 1389 return error; 1390 1391 error = artpec6_crypto_setup_in_descr(common, ac->pad_buffer, 4, false); 1392 if (error) 1393 return error; 1394 1395 if (ext_ctx) { 1396 error = artpec6_crypto_setup_out_descr(common, 1397 req_ctx->digeststate, 1398 contextsize, false, false); 1399 1400 if (error) 1401 return error; 1402 } 1403 1404 if (req_ctx->hash_flags & HASH_FLAG_UPDATE) { 1405 size_t done_bytes = 0; 1406 size_t total_bytes = areq->nbytes + req_ctx->partial_bytes; 1407 size_t ready_bytes = round_down(total_bytes, blocksize); 1408 struct artpec6_crypto_walk walk; 1409 1410 run_hw = ready_bytes > 0; 1411 if (req_ctx->partial_bytes && ready_bytes) { 1412 /* We have a partial buffer and will at least some bytes 1413 * to the HW. Empty this partial buffer before tackling 1414 * the SG lists 1415 */ 1416 memcpy(req_ctx->partial_buffer_out, 1417 req_ctx->partial_buffer, 1418 req_ctx->partial_bytes); 1419 1420 error = artpec6_crypto_setup_out_descr(common, 1421 req_ctx->partial_buffer_out, 1422 req_ctx->partial_bytes, 1423 false, true); 1424 if (error) 1425 return error; 1426 1427 /* Reset partial buffer */ 1428 done_bytes += req_ctx->partial_bytes; 1429 req_ctx->partial_bytes = 0; 1430 } 1431 1432 artpec6_crypto_walk_init(&walk, areq->src); 1433 1434 error = artpec6_crypto_setup_sg_descrs_out(common, &walk, 1435 ready_bytes - 1436 done_bytes); 1437 if (error) 1438 return error; 1439 1440 if (walk.sg) { 1441 size_t sg_skip = ready_bytes - done_bytes; 1442 size_t sg_rem = areq->nbytes - sg_skip; 1443 1444 sg_pcopy_to_buffer(areq->src, sg_nents(areq->src), 1445 req_ctx->partial_buffer + 1446 req_ctx->partial_bytes, 1447 sg_rem, sg_skip); 1448 1449 req_ctx->partial_bytes += sg_rem; 1450 } 1451 1452 req_ctx->digcnt += ready_bytes; 1453 req_ctx->hash_flags &= ~(HASH_FLAG_UPDATE); 1454 } 1455 1456 /* Finalize */ 1457 if (req_ctx->hash_flags & HASH_FLAG_FINALIZE) { 1458 bool needtrim = contextsize != digestsize; 1459 size_t hash_pad_len; 1460 u64 digest_bits; 1461 u32 oper; 1462 1463 if (variant == ARTPEC6_CRYPTO) 1464 oper = FIELD_GET(A6_CRY_MD_OPER, req_ctx->hash_md); 1465 else 1466 oper = FIELD_GET(A7_CRY_MD_OPER, req_ctx->hash_md); 1467 1468 /* Write out the partial buffer if present */ 1469 if (req_ctx->partial_bytes) { 1470 memcpy(req_ctx->partial_buffer_out, 1471 req_ctx->partial_buffer, 1472 req_ctx->partial_bytes); 1473 error = artpec6_crypto_setup_out_descr(common, 1474 req_ctx->partial_buffer_out, 1475 req_ctx->partial_bytes, 1476 false, true); 1477 if (error) 1478 return error; 1479 1480 req_ctx->digcnt += req_ctx->partial_bytes; 1481 req_ctx->partial_bytes = 0; 1482 } 1483 1484 if (req_ctx->hash_flags & HASH_FLAG_HMAC) 1485 digest_bits = 8 * (req_ctx->digcnt + blocksize); 1486 else 1487 digest_bits = 8 * req_ctx->digcnt; 1488 1489 /* Add the hash pad */ 1490 hash_pad_len = create_hash_pad(oper, req_ctx->pad_buffer, 1491 req_ctx->digcnt, digest_bits); 1492 error = artpec6_crypto_setup_out_descr(common, 1493 req_ctx->pad_buffer, 1494 hash_pad_len, false, 1495 true); 1496 req_ctx->digcnt = 0; 1497 1498 if (error) 1499 return error; 1500 1501 /* Descriptor for the final result */ 1502 error = artpec6_crypto_setup_in_descr(common, areq->result, 1503 digestsize, 1504 !needtrim); 1505 if (error) 1506 return error; 1507 1508 if (needtrim) { 1509 /* Discard the extra context bytes for SHA-384 */ 1510 error = artpec6_crypto_setup_in_descr(common, 1511 req_ctx->partial_buffer, 1512 digestsize - contextsize, true); 1513 if (error) 1514 return error; 1515 } 1516 1517 } else { /* This is not the final operation for this request */ 1518 if (!run_hw) 1519 return ARTPEC6_CRYPTO_PREPARE_HASH_NO_START; 1520 1521 /* Save the result to the context */ 1522 error = artpec6_crypto_setup_in_descr(common, 1523 req_ctx->digeststate, 1524 contextsize, false); 1525 if (error) 1526 return error; 1527 /* fall through */ 1528 } 1529 1530 req_ctx->hash_flags &= ~(HASH_FLAG_INIT_CTX | HASH_FLAG_UPDATE | 1531 HASH_FLAG_FINALIZE); 1532 1533 error = artpec6_crypto_terminate_in_descrs(common); 1534 if (error) 1535 return error; 1536 1537 error = artpec6_crypto_terminate_out_descrs(common); 1538 if (error) 1539 return error; 1540 1541 error = artpec6_crypto_dma_map_descs(common); 1542 if (error) 1543 return error; 1544 1545 return ARTPEC6_CRYPTO_PREPARE_HASH_START; 1546 } 1547 1548 1549 static int artpec6_crypto_aes_ecb_init(struct crypto_skcipher *tfm) 1550 { 1551 struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(tfm); 1552 1553 tfm->reqsize = sizeof(struct artpec6_crypto_request_context); 1554 ctx->crypto_type = ARTPEC6_CRYPTO_CIPHER_AES_ECB; 1555 1556 return 0; 1557 } 1558 1559 static int artpec6_crypto_aes_ctr_init(struct crypto_skcipher *tfm) 1560 { 1561 struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(tfm); 1562 1563 ctx->fallback = crypto_alloc_skcipher(crypto_tfm_alg_name(&tfm->base), 1564 0, 1565 CRYPTO_ALG_ASYNC | 1566 CRYPTO_ALG_NEED_FALLBACK); 1567 if (IS_ERR(ctx->fallback)) 1568 return PTR_ERR(ctx->fallback); 1569 1570 tfm->reqsize = sizeof(struct artpec6_crypto_request_context); 1571 ctx->crypto_type = ARTPEC6_CRYPTO_CIPHER_AES_CTR; 1572 1573 return 0; 1574 } 1575 1576 static int artpec6_crypto_aes_cbc_init(struct crypto_skcipher *tfm) 1577 { 1578 struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(tfm); 1579 1580 tfm->reqsize = sizeof(struct artpec6_crypto_request_context); 1581 ctx->crypto_type = ARTPEC6_CRYPTO_CIPHER_AES_CBC; 1582 1583 return 0; 1584 } 1585 1586 static int artpec6_crypto_aes_xts_init(struct crypto_skcipher *tfm) 1587 { 1588 struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(tfm); 1589 1590 tfm->reqsize = sizeof(struct artpec6_crypto_request_context); 1591 ctx->crypto_type = ARTPEC6_CRYPTO_CIPHER_AES_XTS; 1592 1593 return 0; 1594 } 1595 1596 static void artpec6_crypto_aes_exit(struct crypto_skcipher *tfm) 1597 { 1598 struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(tfm); 1599 1600 memset(ctx, 0, sizeof(*ctx)); 1601 } 1602 1603 static void artpec6_crypto_aes_ctr_exit(struct crypto_skcipher *tfm) 1604 { 1605 struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(tfm); 1606 1607 crypto_free_skcipher(ctx->fallback); 1608 artpec6_crypto_aes_exit(tfm); 1609 } 1610 1611 static int 1612 artpec6_crypto_cipher_set_key(struct crypto_skcipher *cipher, const u8 *key, 1613 unsigned int keylen) 1614 { 1615 struct artpec6_cryptotfm_context *ctx = 1616 crypto_skcipher_ctx(cipher); 1617 1618 switch (keylen) { 1619 case 16: 1620 case 24: 1621 case 32: 1622 break; 1623 default: 1624 crypto_skcipher_set_flags(cipher, 1625 CRYPTO_TFM_RES_BAD_KEY_LEN); 1626 return -EINVAL; 1627 } 1628 1629 memcpy(ctx->aes_key, key, keylen); 1630 ctx->key_length = keylen; 1631 return 0; 1632 } 1633 1634 static int 1635 artpec6_crypto_xts_set_key(struct crypto_skcipher *cipher, const u8 *key, 1636 unsigned int keylen) 1637 { 1638 struct artpec6_cryptotfm_context *ctx = 1639 crypto_skcipher_ctx(cipher); 1640 int ret; 1641 1642 ret = xts_check_key(&cipher->base, key, keylen); 1643 if (ret) 1644 return ret; 1645 1646 switch (keylen) { 1647 case 32: 1648 case 48: 1649 case 64: 1650 break; 1651 default: 1652 crypto_skcipher_set_flags(cipher, 1653 CRYPTO_TFM_RES_BAD_KEY_LEN); 1654 return -EINVAL; 1655 } 1656 1657 memcpy(ctx->aes_key, key, keylen); 1658 ctx->key_length = keylen; 1659 return 0; 1660 } 1661 1662 /** artpec6_crypto_process_crypto - Prepare an async block cipher crypto request 1663 * 1664 * @req: The asynch request to process 1665 * 1666 * @return 0 if the dma job was successfully prepared 1667 * <0 on error 1668 * 1669 * This function sets up the PDMA descriptors for a block cipher request. 1670 * 1671 * The required padding is added for AES-CTR using a statically defined 1672 * buffer. 1673 * 1674 * The PDMA descriptor list will be as follows: 1675 * 1676 * OUT: [KEY_MD][KEY][EOP]<CIPHER_MD>[IV]<data_0>...[data_n][AES-CTR_pad]<eop> 1677 * IN: <CIPHER_MD><data_0>...[data_n]<intr> 1678 * 1679 */ 1680 static int artpec6_crypto_prepare_crypto(struct skcipher_request *areq) 1681 { 1682 int ret; 1683 struct artpec6_crypto_walk walk; 1684 struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(areq); 1685 struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(cipher); 1686 struct artpec6_crypto_request_context *req_ctx = NULL; 1687 size_t iv_len = crypto_skcipher_ivsize(cipher); 1688 struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev); 1689 enum artpec6_crypto_variant variant = ac->variant; 1690 struct artpec6_crypto_req_common *common; 1691 bool cipher_decr = false; 1692 size_t cipher_klen; 1693 u32 cipher_len = 0; /* Same as regk_crypto_key_128 for NULL crypto */ 1694 u32 oper; 1695 1696 req_ctx = skcipher_request_ctx(areq); 1697 common = &req_ctx->common; 1698 1699 artpec6_crypto_init_dma_operation(common); 1700 1701 if (variant == ARTPEC6_CRYPTO) 1702 ctx->key_md = FIELD_PREP(A6_CRY_MD_OPER, a6_regk_crypto_dlkey); 1703 else 1704 ctx->key_md = FIELD_PREP(A7_CRY_MD_OPER, a7_regk_crypto_dlkey); 1705 1706 ret = artpec6_crypto_setup_out_descr(common, (void *)&ctx->key_md, 1707 sizeof(ctx->key_md), false, false); 1708 if (ret) 1709 return ret; 1710 1711 ret = artpec6_crypto_setup_out_descr(common, ctx->aes_key, 1712 ctx->key_length, true, false); 1713 if (ret) 1714 return ret; 1715 1716 req_ctx->cipher_md = 0; 1717 1718 if (ctx->crypto_type == ARTPEC6_CRYPTO_CIPHER_AES_XTS) 1719 cipher_klen = ctx->key_length/2; 1720 else 1721 cipher_klen = ctx->key_length; 1722 1723 /* Metadata */ 1724 switch (cipher_klen) { 1725 case 16: 1726 cipher_len = regk_crypto_key_128; 1727 break; 1728 case 24: 1729 cipher_len = regk_crypto_key_192; 1730 break; 1731 case 32: 1732 cipher_len = regk_crypto_key_256; 1733 break; 1734 default: 1735 pr_err("%s: Invalid key length %d!\n", 1736 MODULE_NAME, ctx->key_length); 1737 return -EINVAL; 1738 } 1739 1740 switch (ctx->crypto_type) { 1741 case ARTPEC6_CRYPTO_CIPHER_AES_ECB: 1742 oper = regk_crypto_aes_ecb; 1743 cipher_decr = req_ctx->decrypt; 1744 break; 1745 1746 case ARTPEC6_CRYPTO_CIPHER_AES_CBC: 1747 oper = regk_crypto_aes_cbc; 1748 cipher_decr = req_ctx->decrypt; 1749 break; 1750 1751 case ARTPEC6_CRYPTO_CIPHER_AES_CTR: 1752 oper = regk_crypto_aes_ctr; 1753 cipher_decr = false; 1754 break; 1755 1756 case ARTPEC6_CRYPTO_CIPHER_AES_XTS: 1757 oper = regk_crypto_aes_xts; 1758 cipher_decr = req_ctx->decrypt; 1759 1760 if (variant == ARTPEC6_CRYPTO) 1761 req_ctx->cipher_md |= A6_CRY_MD_CIPHER_DSEQ; 1762 else 1763 req_ctx->cipher_md |= A7_CRY_MD_CIPHER_DSEQ; 1764 break; 1765 1766 default: 1767 pr_err("%s: Invalid cipher mode %d!\n", 1768 MODULE_NAME, ctx->crypto_type); 1769 return -EINVAL; 1770 } 1771 1772 if (variant == ARTPEC6_CRYPTO) { 1773 req_ctx->cipher_md |= FIELD_PREP(A6_CRY_MD_OPER, oper); 1774 req_ctx->cipher_md |= FIELD_PREP(A6_CRY_MD_CIPHER_LEN, 1775 cipher_len); 1776 if (cipher_decr) 1777 req_ctx->cipher_md |= A6_CRY_MD_CIPHER_DECR; 1778 } else { 1779 req_ctx->cipher_md |= FIELD_PREP(A7_CRY_MD_OPER, oper); 1780 req_ctx->cipher_md |= FIELD_PREP(A7_CRY_MD_CIPHER_LEN, 1781 cipher_len); 1782 if (cipher_decr) 1783 req_ctx->cipher_md |= A7_CRY_MD_CIPHER_DECR; 1784 } 1785 1786 ret = artpec6_crypto_setup_out_descr(common, 1787 &req_ctx->cipher_md, 1788 sizeof(req_ctx->cipher_md), 1789 false, false); 1790 if (ret) 1791 return ret; 1792 1793 ret = artpec6_crypto_setup_in_descr(common, ac->pad_buffer, 4, false); 1794 if (ret) 1795 return ret; 1796 1797 if (iv_len) { 1798 ret = artpec6_crypto_setup_out_descr(common, areq->iv, iv_len, 1799 false, false); 1800 if (ret) 1801 return ret; 1802 } 1803 /* Data out */ 1804 artpec6_crypto_walk_init(&walk, areq->src); 1805 ret = artpec6_crypto_setup_sg_descrs_out(common, &walk, areq->cryptlen); 1806 if (ret) 1807 return ret; 1808 1809 /* Data in */ 1810 artpec6_crypto_walk_init(&walk, areq->dst); 1811 ret = artpec6_crypto_setup_sg_descrs_in(common, &walk, areq->cryptlen); 1812 if (ret) 1813 return ret; 1814 1815 /* CTR-mode padding required by the HW. */ 1816 if (ctx->crypto_type == ARTPEC6_CRYPTO_CIPHER_AES_CTR || 1817 ctx->crypto_type == ARTPEC6_CRYPTO_CIPHER_AES_XTS) { 1818 size_t pad = ALIGN(areq->cryptlen, AES_BLOCK_SIZE) - 1819 areq->cryptlen; 1820 1821 if (pad) { 1822 ret = artpec6_crypto_setup_out_descr(common, 1823 ac->pad_buffer, 1824 pad, false, false); 1825 if (ret) 1826 return ret; 1827 1828 ret = artpec6_crypto_setup_in_descr(common, 1829 ac->pad_buffer, pad, 1830 false); 1831 if (ret) 1832 return ret; 1833 } 1834 } 1835 1836 ret = artpec6_crypto_terminate_out_descrs(common); 1837 if (ret) 1838 return ret; 1839 1840 ret = artpec6_crypto_terminate_in_descrs(common); 1841 if (ret) 1842 return ret; 1843 1844 return artpec6_crypto_dma_map_descs(common); 1845 } 1846 1847 static int artpec6_crypto_prepare_aead(struct aead_request *areq) 1848 { 1849 size_t count; 1850 int ret; 1851 size_t input_length; 1852 struct artpec6_cryptotfm_context *ctx = crypto_tfm_ctx(areq->base.tfm); 1853 struct artpec6_crypto_aead_req_ctx *req_ctx = aead_request_ctx(areq); 1854 struct crypto_aead *cipher = crypto_aead_reqtfm(areq); 1855 struct artpec6_crypto_req_common *common = &req_ctx->common; 1856 struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev); 1857 enum artpec6_crypto_variant variant = ac->variant; 1858 u32 md_cipher_len; 1859 1860 artpec6_crypto_init_dma_operation(common); 1861 1862 /* Key */ 1863 if (variant == ARTPEC6_CRYPTO) { 1864 ctx->key_md = FIELD_PREP(A6_CRY_MD_OPER, 1865 a6_regk_crypto_dlkey); 1866 } else { 1867 ctx->key_md = FIELD_PREP(A7_CRY_MD_OPER, 1868 a7_regk_crypto_dlkey); 1869 } 1870 ret = artpec6_crypto_setup_out_descr(common, (void *)&ctx->key_md, 1871 sizeof(ctx->key_md), false, false); 1872 if (ret) 1873 return ret; 1874 1875 ret = artpec6_crypto_setup_out_descr(common, ctx->aes_key, 1876 ctx->key_length, true, false); 1877 if (ret) 1878 return ret; 1879 1880 req_ctx->cipher_md = 0; 1881 1882 switch (ctx->key_length) { 1883 case 16: 1884 md_cipher_len = regk_crypto_key_128; 1885 break; 1886 case 24: 1887 md_cipher_len = regk_crypto_key_192; 1888 break; 1889 case 32: 1890 md_cipher_len = regk_crypto_key_256; 1891 break; 1892 default: 1893 return -EINVAL; 1894 } 1895 1896 if (variant == ARTPEC6_CRYPTO) { 1897 req_ctx->cipher_md |= FIELD_PREP(A6_CRY_MD_OPER, 1898 regk_crypto_aes_gcm); 1899 req_ctx->cipher_md |= FIELD_PREP(A6_CRY_MD_CIPHER_LEN, 1900 md_cipher_len); 1901 if (req_ctx->decrypt) 1902 req_ctx->cipher_md |= A6_CRY_MD_CIPHER_DECR; 1903 } else { 1904 req_ctx->cipher_md |= FIELD_PREP(A7_CRY_MD_OPER, 1905 regk_crypto_aes_gcm); 1906 req_ctx->cipher_md |= FIELD_PREP(A7_CRY_MD_CIPHER_LEN, 1907 md_cipher_len); 1908 if (req_ctx->decrypt) 1909 req_ctx->cipher_md |= A7_CRY_MD_CIPHER_DECR; 1910 } 1911 1912 ret = artpec6_crypto_setup_out_descr(common, 1913 (void *) &req_ctx->cipher_md, 1914 sizeof(req_ctx->cipher_md), false, 1915 false); 1916 if (ret) 1917 return ret; 1918 1919 ret = artpec6_crypto_setup_in_descr(common, ac->pad_buffer, 4, false); 1920 if (ret) 1921 return ret; 1922 1923 /* For the decryption, cryptlen includes the tag. */ 1924 input_length = areq->cryptlen; 1925 if (req_ctx->decrypt) 1926 input_length -= AES_BLOCK_SIZE; 1927 1928 /* Prepare the context buffer */ 1929 req_ctx->hw_ctx.aad_length_bits = 1930 __cpu_to_be64(8*areq->assoclen); 1931 1932 req_ctx->hw_ctx.text_length_bits = 1933 __cpu_to_be64(8*input_length); 1934 1935 memcpy(req_ctx->hw_ctx.J0, areq->iv, crypto_aead_ivsize(cipher)); 1936 // The HW omits the initial increment of the counter field. 1937 crypto_inc(req_ctx->hw_ctx.J0+12, 4); 1938 1939 ret = artpec6_crypto_setup_out_descr(common, &req_ctx->hw_ctx, 1940 sizeof(struct artpec6_crypto_aead_hw_ctx), false, false); 1941 if (ret) 1942 return ret; 1943 1944 { 1945 struct artpec6_crypto_walk walk; 1946 1947 artpec6_crypto_walk_init(&walk, areq->src); 1948 1949 /* Associated data */ 1950 count = areq->assoclen; 1951 ret = artpec6_crypto_setup_sg_descrs_out(common, &walk, count); 1952 if (ret) 1953 return ret; 1954 1955 if (!IS_ALIGNED(areq->assoclen, 16)) { 1956 size_t assoc_pad = 16 - (areq->assoclen % 16); 1957 /* The HW mandates zero padding here */ 1958 ret = artpec6_crypto_setup_out_descr(common, 1959 ac->zero_buffer, 1960 assoc_pad, false, 1961 false); 1962 if (ret) 1963 return ret; 1964 } 1965 1966 /* Data to crypto */ 1967 count = input_length; 1968 ret = artpec6_crypto_setup_sg_descrs_out(common, &walk, count); 1969 if (ret) 1970 return ret; 1971 1972 if (!IS_ALIGNED(input_length, 16)) { 1973 size_t crypto_pad = 16 - (input_length % 16); 1974 /* The HW mandates zero padding here */ 1975 ret = artpec6_crypto_setup_out_descr(common, 1976 ac->zero_buffer, 1977 crypto_pad, 1978 false, 1979 false); 1980 if (ret) 1981 return ret; 1982 } 1983 } 1984 1985 /* Data from crypto */ 1986 { 1987 struct artpec6_crypto_walk walk; 1988 size_t output_len = areq->cryptlen; 1989 1990 if (req_ctx->decrypt) 1991 output_len -= AES_BLOCK_SIZE; 1992 1993 artpec6_crypto_walk_init(&walk, areq->dst); 1994 1995 /* skip associated data in the output */ 1996 count = artpec6_crypto_walk_advance(&walk, areq->assoclen); 1997 if (count) 1998 return -EINVAL; 1999 2000 count = output_len; 2001 ret = artpec6_crypto_setup_sg_descrs_in(common, &walk, count); 2002 if (ret) 2003 return ret; 2004 2005 /* Put padding between the cryptotext and the auth tag */ 2006 if (!IS_ALIGNED(output_len, 16)) { 2007 size_t crypto_pad = 16 - (output_len % 16); 2008 2009 ret = artpec6_crypto_setup_in_descr(common, 2010 ac->pad_buffer, 2011 crypto_pad, false); 2012 if (ret) 2013 return ret; 2014 } 2015 2016 /* The authentication tag shall follow immediately after 2017 * the output ciphertext. For decryption it is put in a context 2018 * buffer for later compare against the input tag. 2019 */ 2020 count = AES_BLOCK_SIZE; 2021 2022 if (req_ctx->decrypt) { 2023 ret = artpec6_crypto_setup_in_descr(common, 2024 req_ctx->decryption_tag, count, false); 2025 if (ret) 2026 return ret; 2027 2028 } else { 2029 ret = artpec6_crypto_setup_sg_descrs_in(common, &walk, 2030 count); 2031 if (ret) 2032 return ret; 2033 } 2034 2035 } 2036 2037 ret = artpec6_crypto_terminate_in_descrs(common); 2038 if (ret) 2039 return ret; 2040 2041 ret = artpec6_crypto_terminate_out_descrs(common); 2042 if (ret) 2043 return ret; 2044 2045 return artpec6_crypto_dma_map_descs(common); 2046 } 2047 2048 static void artpec6_crypto_process_queue(struct artpec6_crypto *ac) 2049 { 2050 struct artpec6_crypto_req_common *req; 2051 2052 while (!list_empty(&ac->queue) && !artpec6_crypto_busy()) { 2053 req = list_first_entry(&ac->queue, 2054 struct artpec6_crypto_req_common, 2055 list); 2056 list_move_tail(&req->list, &ac->pending); 2057 artpec6_crypto_start_dma(req); 2058 2059 req->req->complete(req->req, -EINPROGRESS); 2060 } 2061 2062 /* 2063 * In some cases, the hardware can raise an in_eop_flush interrupt 2064 * before actually updating the status, so we have an timer which will 2065 * recheck the status on timeout. Since the cases are expected to be 2066 * very rare, we use a relatively large timeout value. There should be 2067 * no noticeable negative effect if we timeout spuriously. 2068 */ 2069 if (ac->pending_count) 2070 mod_timer(&ac->timer, jiffies + msecs_to_jiffies(100)); 2071 else 2072 del_timer(&ac->timer); 2073 } 2074 2075 static void artpec6_crypto_timeout(struct timer_list *t) 2076 { 2077 struct artpec6_crypto *ac = from_timer(ac, t, timer); 2078 2079 dev_info_ratelimited(artpec6_crypto_dev, "timeout\n"); 2080 2081 tasklet_schedule(&ac->task); 2082 } 2083 2084 static void artpec6_crypto_task(unsigned long data) 2085 { 2086 struct artpec6_crypto *ac = (struct artpec6_crypto *)data; 2087 struct artpec6_crypto_req_common *req; 2088 struct artpec6_crypto_req_common *n; 2089 2090 if (list_empty(&ac->pending)) { 2091 pr_debug("Spurious IRQ\n"); 2092 return; 2093 } 2094 2095 spin_lock_bh(&ac->queue_lock); 2096 2097 list_for_each_entry_safe(req, n, &ac->pending, list) { 2098 struct artpec6_crypto_dma_descriptors *dma = req->dma; 2099 u32 stat; 2100 2101 dma_sync_single_for_cpu(artpec6_crypto_dev, dma->stat_dma_addr, 2102 sizeof(dma->stat[0]), 2103 DMA_BIDIRECTIONAL); 2104 2105 stat = req->dma->stat[req->dma->in_cnt-1]; 2106 2107 /* A non-zero final status descriptor indicates 2108 * this job has finished. 2109 */ 2110 pr_debug("Request %p status is %X\n", req, stat); 2111 if (!stat) 2112 break; 2113 2114 /* Allow testing of timeout handling with fault injection */ 2115 #ifdef CONFIG_FAULT_INJECTION 2116 if (should_fail(&artpec6_crypto_fail_status_read, 1)) 2117 continue; 2118 #endif 2119 2120 pr_debug("Completing request %p\n", req); 2121 2122 list_del(&req->list); 2123 2124 artpec6_crypto_dma_unmap_all(req); 2125 artpec6_crypto_copy_bounce_buffers(req); 2126 2127 ac->pending_count--; 2128 artpec6_crypto_common_destroy(req); 2129 req->complete(req->req); 2130 } 2131 2132 artpec6_crypto_process_queue(ac); 2133 2134 spin_unlock_bh(&ac->queue_lock); 2135 } 2136 2137 static void artpec6_crypto_complete_crypto(struct crypto_async_request *req) 2138 { 2139 req->complete(req, 0); 2140 } 2141 2142 static void 2143 artpec6_crypto_complete_cbc_decrypt(struct crypto_async_request *req) 2144 { 2145 struct skcipher_request *cipher_req = container_of(req, 2146 struct skcipher_request, base); 2147 2148 scatterwalk_map_and_copy(cipher_req->iv, cipher_req->src, 2149 cipher_req->cryptlen - AES_BLOCK_SIZE, 2150 AES_BLOCK_SIZE, 0); 2151 req->complete(req, 0); 2152 } 2153 2154 static void 2155 artpec6_crypto_complete_cbc_encrypt(struct crypto_async_request *req) 2156 { 2157 struct skcipher_request *cipher_req = container_of(req, 2158 struct skcipher_request, base); 2159 2160 scatterwalk_map_and_copy(cipher_req->iv, cipher_req->dst, 2161 cipher_req->cryptlen - AES_BLOCK_SIZE, 2162 AES_BLOCK_SIZE, 0); 2163 req->complete(req, 0); 2164 } 2165 2166 static void artpec6_crypto_complete_aead(struct crypto_async_request *req) 2167 { 2168 int result = 0; 2169 2170 /* Verify GCM hashtag. */ 2171 struct aead_request *areq = container_of(req, 2172 struct aead_request, base); 2173 struct artpec6_crypto_aead_req_ctx *req_ctx = aead_request_ctx(areq); 2174 2175 if (req_ctx->decrypt) { 2176 u8 input_tag[AES_BLOCK_SIZE]; 2177 2178 sg_pcopy_to_buffer(areq->src, 2179 sg_nents(areq->src), 2180 input_tag, 2181 AES_BLOCK_SIZE, 2182 areq->assoclen + areq->cryptlen - 2183 AES_BLOCK_SIZE); 2184 2185 if (memcmp(req_ctx->decryption_tag, 2186 input_tag, 2187 AES_BLOCK_SIZE)) { 2188 pr_debug("***EBADMSG:\n"); 2189 print_hex_dump_debug("ref:", DUMP_PREFIX_ADDRESS, 32, 1, 2190 input_tag, AES_BLOCK_SIZE, true); 2191 print_hex_dump_debug("out:", DUMP_PREFIX_ADDRESS, 32, 1, 2192 req_ctx->decryption_tag, 2193 AES_BLOCK_SIZE, true); 2194 2195 result = -EBADMSG; 2196 } 2197 } 2198 2199 req->complete(req, result); 2200 } 2201 2202 static void artpec6_crypto_complete_hash(struct crypto_async_request *req) 2203 { 2204 req->complete(req, 0); 2205 } 2206 2207 2208 /*------------------- Hash functions -----------------------------------------*/ 2209 static int 2210 artpec6_crypto_hash_set_key(struct crypto_ahash *tfm, 2211 const u8 *key, unsigned int keylen) 2212 { 2213 struct artpec6_hashalg_context *tfm_ctx = crypto_tfm_ctx(&tfm->base); 2214 size_t blocksize; 2215 int ret; 2216 2217 if (!keylen) { 2218 pr_err("Invalid length (%d) of HMAC key\n", 2219 keylen); 2220 return -EINVAL; 2221 } 2222 2223 memset(tfm_ctx->hmac_key, 0, sizeof(tfm_ctx->hmac_key)); 2224 2225 blocksize = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm)); 2226 2227 if (keylen > blocksize) { 2228 SHASH_DESC_ON_STACK(hdesc, tfm_ctx->child_hash); 2229 2230 hdesc->tfm = tfm_ctx->child_hash; 2231 hdesc->flags = crypto_ahash_get_flags(tfm) & 2232 CRYPTO_TFM_REQ_MAY_SLEEP; 2233 2234 tfm_ctx->hmac_key_length = blocksize; 2235 ret = crypto_shash_digest(hdesc, key, keylen, 2236 tfm_ctx->hmac_key); 2237 if (ret) 2238 return ret; 2239 2240 } else { 2241 memcpy(tfm_ctx->hmac_key, key, keylen); 2242 tfm_ctx->hmac_key_length = keylen; 2243 } 2244 2245 return 0; 2246 } 2247 2248 static int 2249 artpec6_crypto_init_hash(struct ahash_request *req, u8 type, int hmac) 2250 { 2251 struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev); 2252 enum artpec6_crypto_variant variant = ac->variant; 2253 struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req); 2254 u32 oper; 2255 2256 memset(req_ctx, 0, sizeof(*req_ctx)); 2257 2258 req_ctx->hash_flags = HASH_FLAG_INIT_CTX; 2259 if (hmac) 2260 req_ctx->hash_flags |= (HASH_FLAG_HMAC | HASH_FLAG_UPDATE_KEY); 2261 2262 switch (type) { 2263 case ARTPEC6_CRYPTO_HASH_SHA1: 2264 oper = hmac ? regk_crypto_hmac_sha1 : regk_crypto_sha1; 2265 break; 2266 case ARTPEC6_CRYPTO_HASH_SHA256: 2267 oper = hmac ? regk_crypto_hmac_sha256 : regk_crypto_sha256; 2268 break; 2269 case ARTPEC6_CRYPTO_HASH_SHA384: 2270 oper = hmac ? regk_crypto_hmac_sha384 : regk_crypto_sha384; 2271 break; 2272 case ARTPEC6_CRYPTO_HASH_SHA512: 2273 oper = hmac ? regk_crypto_hmac_sha512 : regk_crypto_sha512; 2274 break; 2275 2276 default: 2277 pr_err("%s: Unsupported hash type 0x%x\n", MODULE_NAME, type); 2278 return -EINVAL; 2279 } 2280 2281 if (variant == ARTPEC6_CRYPTO) 2282 req_ctx->hash_md = FIELD_PREP(A6_CRY_MD_OPER, oper); 2283 else 2284 req_ctx->hash_md = FIELD_PREP(A7_CRY_MD_OPER, oper); 2285 2286 return 0; 2287 } 2288 2289 static int artpec6_crypto_prepare_submit_hash(struct ahash_request *req) 2290 { 2291 struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req); 2292 int ret; 2293 2294 if (!req_ctx->common.dma) { 2295 ret = artpec6_crypto_common_init(&req_ctx->common, 2296 &req->base, 2297 artpec6_crypto_complete_hash, 2298 NULL, 0); 2299 2300 if (ret) 2301 return ret; 2302 } 2303 2304 ret = artpec6_crypto_prepare_hash(req); 2305 switch (ret) { 2306 case ARTPEC6_CRYPTO_PREPARE_HASH_START: 2307 ret = artpec6_crypto_submit(&req_ctx->common); 2308 break; 2309 2310 case ARTPEC6_CRYPTO_PREPARE_HASH_NO_START: 2311 ret = 0; 2312 /* Fallthrough */ 2313 2314 default: 2315 artpec6_crypto_common_destroy(&req_ctx->common); 2316 break; 2317 } 2318 2319 return ret; 2320 } 2321 2322 static int artpec6_crypto_hash_final(struct ahash_request *req) 2323 { 2324 struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req); 2325 2326 req_ctx->hash_flags |= HASH_FLAG_FINALIZE; 2327 2328 return artpec6_crypto_prepare_submit_hash(req); 2329 } 2330 2331 static int artpec6_crypto_hash_update(struct ahash_request *req) 2332 { 2333 struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req); 2334 2335 req_ctx->hash_flags |= HASH_FLAG_UPDATE; 2336 2337 return artpec6_crypto_prepare_submit_hash(req); 2338 } 2339 2340 static int artpec6_crypto_sha1_init(struct ahash_request *req) 2341 { 2342 return artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA1, 0); 2343 } 2344 2345 static int artpec6_crypto_sha1_digest(struct ahash_request *req) 2346 { 2347 struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req); 2348 2349 artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA1, 0); 2350 2351 req_ctx->hash_flags |= HASH_FLAG_UPDATE | HASH_FLAG_FINALIZE; 2352 2353 return artpec6_crypto_prepare_submit_hash(req); 2354 } 2355 2356 static int artpec6_crypto_sha256_init(struct ahash_request *req) 2357 { 2358 return artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA256, 0); 2359 } 2360 2361 static int artpec6_crypto_sha256_digest(struct ahash_request *req) 2362 { 2363 struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req); 2364 2365 artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA256, 0); 2366 req_ctx->hash_flags |= HASH_FLAG_UPDATE | HASH_FLAG_FINALIZE; 2367 2368 return artpec6_crypto_prepare_submit_hash(req); 2369 } 2370 2371 static int __maybe_unused artpec6_crypto_sha384_init(struct ahash_request *req) 2372 { 2373 return artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA384, 0); 2374 } 2375 2376 static int __maybe_unused 2377 artpec6_crypto_sha384_digest(struct ahash_request *req) 2378 { 2379 struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req); 2380 2381 artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA384, 0); 2382 req_ctx->hash_flags |= HASH_FLAG_UPDATE | HASH_FLAG_FINALIZE; 2383 2384 return artpec6_crypto_prepare_submit_hash(req); 2385 } 2386 2387 static int artpec6_crypto_sha512_init(struct ahash_request *req) 2388 { 2389 return artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA512, 0); 2390 } 2391 2392 static int artpec6_crypto_sha512_digest(struct ahash_request *req) 2393 { 2394 struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req); 2395 2396 artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA512, 0); 2397 req_ctx->hash_flags |= HASH_FLAG_UPDATE | HASH_FLAG_FINALIZE; 2398 2399 return artpec6_crypto_prepare_submit_hash(req); 2400 } 2401 2402 static int artpec6_crypto_hmac_sha256_init(struct ahash_request *req) 2403 { 2404 return artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA256, 1); 2405 } 2406 2407 static int __maybe_unused 2408 artpec6_crypto_hmac_sha384_init(struct ahash_request *req) 2409 { 2410 return artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA384, 1); 2411 } 2412 2413 static int artpec6_crypto_hmac_sha512_init(struct ahash_request *req) 2414 { 2415 return artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA512, 1); 2416 } 2417 2418 static int artpec6_crypto_hmac_sha256_digest(struct ahash_request *req) 2419 { 2420 struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req); 2421 2422 artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA256, 1); 2423 req_ctx->hash_flags |= HASH_FLAG_UPDATE | HASH_FLAG_FINALIZE; 2424 2425 return artpec6_crypto_prepare_submit_hash(req); 2426 } 2427 2428 static int __maybe_unused 2429 artpec6_crypto_hmac_sha384_digest(struct ahash_request *req) 2430 { 2431 struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req); 2432 2433 artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA384, 1); 2434 req_ctx->hash_flags |= HASH_FLAG_UPDATE | HASH_FLAG_FINALIZE; 2435 2436 return artpec6_crypto_prepare_submit_hash(req); 2437 } 2438 2439 static int artpec6_crypto_hmac_sha512_digest(struct ahash_request *req) 2440 { 2441 struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req); 2442 2443 artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA512, 1); 2444 req_ctx->hash_flags |= HASH_FLAG_UPDATE | HASH_FLAG_FINALIZE; 2445 2446 return artpec6_crypto_prepare_submit_hash(req); 2447 } 2448 2449 static int artpec6_crypto_ahash_init_common(struct crypto_tfm *tfm, 2450 const char *base_hash_name) 2451 { 2452 struct artpec6_hashalg_context *tfm_ctx = crypto_tfm_ctx(tfm); 2453 2454 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), 2455 sizeof(struct artpec6_hash_request_context)); 2456 memset(tfm_ctx, 0, sizeof(*tfm_ctx)); 2457 2458 if (base_hash_name) { 2459 struct crypto_shash *child; 2460 2461 child = crypto_alloc_shash(base_hash_name, 0, 2462 CRYPTO_ALG_NEED_FALLBACK); 2463 2464 if (IS_ERR(child)) 2465 return PTR_ERR(child); 2466 2467 tfm_ctx->child_hash = child; 2468 } 2469 2470 return 0; 2471 } 2472 2473 static int artpec6_crypto_ahash_init(struct crypto_tfm *tfm) 2474 { 2475 return artpec6_crypto_ahash_init_common(tfm, NULL); 2476 } 2477 2478 static int artpec6_crypto_ahash_init_hmac_sha256(struct crypto_tfm *tfm) 2479 { 2480 return artpec6_crypto_ahash_init_common(tfm, "sha256"); 2481 } 2482 2483 static int __maybe_unused 2484 artpec6_crypto_ahash_init_hmac_sha384(struct crypto_tfm *tfm) 2485 { 2486 return artpec6_crypto_ahash_init_common(tfm, "sha384"); 2487 } 2488 2489 static int artpec6_crypto_ahash_init_hmac_sha512(struct crypto_tfm *tfm) 2490 { 2491 return artpec6_crypto_ahash_init_common(tfm, "sha512"); 2492 } 2493 2494 static void artpec6_crypto_ahash_exit(struct crypto_tfm *tfm) 2495 { 2496 struct artpec6_hashalg_context *tfm_ctx = crypto_tfm_ctx(tfm); 2497 2498 if (tfm_ctx->child_hash) 2499 crypto_free_shash(tfm_ctx->child_hash); 2500 2501 memset(tfm_ctx->hmac_key, 0, sizeof(tfm_ctx->hmac_key)); 2502 tfm_ctx->hmac_key_length = 0; 2503 } 2504 2505 static int artpec6_crypto_hash_export(struct ahash_request *req, void *out) 2506 { 2507 const struct artpec6_hash_request_context *ctx = ahash_request_ctx(req); 2508 struct artpec6_hash_export_state *state = out; 2509 struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev); 2510 enum artpec6_crypto_variant variant = ac->variant; 2511 2512 BUILD_BUG_ON(sizeof(state->partial_buffer) != 2513 sizeof(ctx->partial_buffer)); 2514 BUILD_BUG_ON(sizeof(state->digeststate) != sizeof(ctx->digeststate)); 2515 2516 state->digcnt = ctx->digcnt; 2517 state->partial_bytes = ctx->partial_bytes; 2518 state->hash_flags = ctx->hash_flags; 2519 2520 if (variant == ARTPEC6_CRYPTO) 2521 state->oper = FIELD_GET(A6_CRY_MD_OPER, ctx->hash_md); 2522 else 2523 state->oper = FIELD_GET(A7_CRY_MD_OPER, ctx->hash_md); 2524 2525 memcpy(state->partial_buffer, ctx->partial_buffer, 2526 sizeof(state->partial_buffer)); 2527 memcpy(state->digeststate, ctx->digeststate, 2528 sizeof(state->digeststate)); 2529 2530 return 0; 2531 } 2532 2533 static int artpec6_crypto_hash_import(struct ahash_request *req, const void *in) 2534 { 2535 struct artpec6_hash_request_context *ctx = ahash_request_ctx(req); 2536 const struct artpec6_hash_export_state *state = in; 2537 struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev); 2538 enum artpec6_crypto_variant variant = ac->variant; 2539 2540 memset(ctx, 0, sizeof(*ctx)); 2541 2542 ctx->digcnt = state->digcnt; 2543 ctx->partial_bytes = state->partial_bytes; 2544 ctx->hash_flags = state->hash_flags; 2545 2546 if (variant == ARTPEC6_CRYPTO) 2547 ctx->hash_md = FIELD_PREP(A6_CRY_MD_OPER, state->oper); 2548 else 2549 ctx->hash_md = FIELD_PREP(A7_CRY_MD_OPER, state->oper); 2550 2551 memcpy(ctx->partial_buffer, state->partial_buffer, 2552 sizeof(state->partial_buffer)); 2553 memcpy(ctx->digeststate, state->digeststate, 2554 sizeof(state->digeststate)); 2555 2556 return 0; 2557 } 2558 2559 static int init_crypto_hw(struct artpec6_crypto *ac) 2560 { 2561 enum artpec6_crypto_variant variant = ac->variant; 2562 void __iomem *base = ac->base; 2563 u32 out_descr_buf_size; 2564 u32 out_data_buf_size; 2565 u32 in_data_buf_size; 2566 u32 in_descr_buf_size; 2567 u32 in_stat_buf_size; 2568 u32 in, out; 2569 2570 /* 2571 * The PDMA unit contains 1984 bytes of internal memory for the OUT 2572 * channels and 1024 bytes for the IN channel. This is an elastic 2573 * memory used to internally store the descriptors and data. The values 2574 * ares specified in 64 byte incremements. Trustzone buffers are not 2575 * used at this stage. 2576 */ 2577 out_data_buf_size = 16; /* 1024 bytes for data */ 2578 out_descr_buf_size = 15; /* 960 bytes for descriptors */ 2579 in_data_buf_size = 8; /* 512 bytes for data */ 2580 in_descr_buf_size = 4; /* 256 bytes for descriptors */ 2581 in_stat_buf_size = 4; /* 256 bytes for stat descrs */ 2582 2583 BUILD_BUG_ON_MSG((out_data_buf_size 2584 + out_descr_buf_size) * 64 > 1984, 2585 "Invalid OUT configuration"); 2586 2587 BUILD_BUG_ON_MSG((in_data_buf_size 2588 + in_descr_buf_size 2589 + in_stat_buf_size) * 64 > 1024, 2590 "Invalid IN configuration"); 2591 2592 in = FIELD_PREP(PDMA_IN_BUF_CFG_DATA_BUF_SIZE, in_data_buf_size) | 2593 FIELD_PREP(PDMA_IN_BUF_CFG_DESCR_BUF_SIZE, in_descr_buf_size) | 2594 FIELD_PREP(PDMA_IN_BUF_CFG_STAT_BUF_SIZE, in_stat_buf_size); 2595 2596 out = FIELD_PREP(PDMA_OUT_BUF_CFG_DATA_BUF_SIZE, out_data_buf_size) | 2597 FIELD_PREP(PDMA_OUT_BUF_CFG_DESCR_BUF_SIZE, out_descr_buf_size); 2598 2599 writel_relaxed(out, base + PDMA_OUT_BUF_CFG); 2600 writel_relaxed(PDMA_OUT_CFG_EN, base + PDMA_OUT_CFG); 2601 2602 if (variant == ARTPEC6_CRYPTO) { 2603 writel_relaxed(in, base + A6_PDMA_IN_BUF_CFG); 2604 writel_relaxed(PDMA_IN_CFG_EN, base + A6_PDMA_IN_CFG); 2605 writel_relaxed(A6_PDMA_INTR_MASK_IN_DATA | 2606 A6_PDMA_INTR_MASK_IN_EOP_FLUSH, 2607 base + A6_PDMA_INTR_MASK); 2608 } else { 2609 writel_relaxed(in, base + A7_PDMA_IN_BUF_CFG); 2610 writel_relaxed(PDMA_IN_CFG_EN, base + A7_PDMA_IN_CFG); 2611 writel_relaxed(A7_PDMA_INTR_MASK_IN_DATA | 2612 A7_PDMA_INTR_MASK_IN_EOP_FLUSH, 2613 base + A7_PDMA_INTR_MASK); 2614 } 2615 2616 return 0; 2617 } 2618 2619 static void artpec6_crypto_disable_hw(struct artpec6_crypto *ac) 2620 { 2621 enum artpec6_crypto_variant variant = ac->variant; 2622 void __iomem *base = ac->base; 2623 2624 if (variant == ARTPEC6_CRYPTO) { 2625 writel_relaxed(A6_PDMA_IN_CMD_STOP, base + A6_PDMA_IN_CMD); 2626 writel_relaxed(0, base + A6_PDMA_IN_CFG); 2627 writel_relaxed(A6_PDMA_OUT_CMD_STOP, base + PDMA_OUT_CMD); 2628 } else { 2629 writel_relaxed(A7_PDMA_IN_CMD_STOP, base + A7_PDMA_IN_CMD); 2630 writel_relaxed(0, base + A7_PDMA_IN_CFG); 2631 writel_relaxed(A7_PDMA_OUT_CMD_STOP, base + PDMA_OUT_CMD); 2632 } 2633 2634 writel_relaxed(0, base + PDMA_OUT_CFG); 2635 2636 } 2637 2638 static irqreturn_t artpec6_crypto_irq(int irq, void *dev_id) 2639 { 2640 struct artpec6_crypto *ac = dev_id; 2641 enum artpec6_crypto_variant variant = ac->variant; 2642 void __iomem *base = ac->base; 2643 u32 mask_in_data, mask_in_eop_flush; 2644 u32 in_cmd_flush_stat, in_cmd_reg; 2645 u32 ack_intr_reg; 2646 u32 ack = 0; 2647 u32 intr; 2648 2649 if (variant == ARTPEC6_CRYPTO) { 2650 intr = readl_relaxed(base + A6_PDMA_MASKED_INTR); 2651 mask_in_data = A6_PDMA_INTR_MASK_IN_DATA; 2652 mask_in_eop_flush = A6_PDMA_INTR_MASK_IN_EOP_FLUSH; 2653 in_cmd_flush_stat = A6_PDMA_IN_CMD_FLUSH_STAT; 2654 in_cmd_reg = A6_PDMA_IN_CMD; 2655 ack_intr_reg = A6_PDMA_ACK_INTR; 2656 } else { 2657 intr = readl_relaxed(base + A7_PDMA_MASKED_INTR); 2658 mask_in_data = A7_PDMA_INTR_MASK_IN_DATA; 2659 mask_in_eop_flush = A7_PDMA_INTR_MASK_IN_EOP_FLUSH; 2660 in_cmd_flush_stat = A7_PDMA_IN_CMD_FLUSH_STAT; 2661 in_cmd_reg = A7_PDMA_IN_CMD; 2662 ack_intr_reg = A7_PDMA_ACK_INTR; 2663 } 2664 2665 /* We get two interrupt notifications from each job. 2666 * The in_data means all data was sent to memory and then 2667 * we request a status flush command to write the per-job 2668 * status to its status vector. This ensures that the 2669 * tasklet can detect exactly how many submitted jobs 2670 * that have finished. 2671 */ 2672 if (intr & mask_in_data) 2673 ack |= mask_in_data; 2674 2675 if (intr & mask_in_eop_flush) 2676 ack |= mask_in_eop_flush; 2677 else 2678 writel_relaxed(in_cmd_flush_stat, base + in_cmd_reg); 2679 2680 writel_relaxed(ack, base + ack_intr_reg); 2681 2682 if (intr & mask_in_eop_flush) 2683 tasklet_schedule(&ac->task); 2684 2685 return IRQ_HANDLED; 2686 } 2687 2688 /*------------------- Algorithm definitions ----------------------------------*/ 2689 2690 /* Hashes */ 2691 static struct ahash_alg hash_algos[] = { 2692 /* SHA-1 */ 2693 { 2694 .init = artpec6_crypto_sha1_init, 2695 .update = artpec6_crypto_hash_update, 2696 .final = artpec6_crypto_hash_final, 2697 .digest = artpec6_crypto_sha1_digest, 2698 .import = artpec6_crypto_hash_import, 2699 .export = artpec6_crypto_hash_export, 2700 .halg.digestsize = SHA1_DIGEST_SIZE, 2701 .halg.statesize = sizeof(struct artpec6_hash_export_state), 2702 .halg.base = { 2703 .cra_name = "sha1", 2704 .cra_driver_name = "artpec-sha1", 2705 .cra_priority = 300, 2706 .cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC, 2707 .cra_blocksize = SHA1_BLOCK_SIZE, 2708 .cra_ctxsize = sizeof(struct artpec6_hashalg_context), 2709 .cra_alignmask = 3, 2710 .cra_module = THIS_MODULE, 2711 .cra_init = artpec6_crypto_ahash_init, 2712 .cra_exit = artpec6_crypto_ahash_exit, 2713 } 2714 }, 2715 /* SHA-256 */ 2716 { 2717 .init = artpec6_crypto_sha256_init, 2718 .update = artpec6_crypto_hash_update, 2719 .final = artpec6_crypto_hash_final, 2720 .digest = artpec6_crypto_sha256_digest, 2721 .import = artpec6_crypto_hash_import, 2722 .export = artpec6_crypto_hash_export, 2723 .halg.digestsize = SHA256_DIGEST_SIZE, 2724 .halg.statesize = sizeof(struct artpec6_hash_export_state), 2725 .halg.base = { 2726 .cra_name = "sha256", 2727 .cra_driver_name = "artpec-sha256", 2728 .cra_priority = 300, 2729 .cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC, 2730 .cra_blocksize = SHA256_BLOCK_SIZE, 2731 .cra_ctxsize = sizeof(struct artpec6_hashalg_context), 2732 .cra_alignmask = 3, 2733 .cra_module = THIS_MODULE, 2734 .cra_init = artpec6_crypto_ahash_init, 2735 .cra_exit = artpec6_crypto_ahash_exit, 2736 } 2737 }, 2738 /* HMAC SHA-256 */ 2739 { 2740 .init = artpec6_crypto_hmac_sha256_init, 2741 .update = artpec6_crypto_hash_update, 2742 .final = artpec6_crypto_hash_final, 2743 .digest = artpec6_crypto_hmac_sha256_digest, 2744 .import = artpec6_crypto_hash_import, 2745 .export = artpec6_crypto_hash_export, 2746 .setkey = artpec6_crypto_hash_set_key, 2747 .halg.digestsize = SHA256_DIGEST_SIZE, 2748 .halg.statesize = sizeof(struct artpec6_hash_export_state), 2749 .halg.base = { 2750 .cra_name = "hmac(sha256)", 2751 .cra_driver_name = "artpec-hmac-sha256", 2752 .cra_priority = 300, 2753 .cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC, 2754 .cra_blocksize = SHA256_BLOCK_SIZE, 2755 .cra_ctxsize = sizeof(struct artpec6_hashalg_context), 2756 .cra_alignmask = 3, 2757 .cra_module = THIS_MODULE, 2758 .cra_init = artpec6_crypto_ahash_init_hmac_sha256, 2759 .cra_exit = artpec6_crypto_ahash_exit, 2760 } 2761 }, 2762 }; 2763 2764 static struct ahash_alg artpec7_hash_algos[] = { 2765 /* SHA-384 */ 2766 { 2767 .init = artpec6_crypto_sha384_init, 2768 .update = artpec6_crypto_hash_update, 2769 .final = artpec6_crypto_hash_final, 2770 .digest = artpec6_crypto_sha384_digest, 2771 .import = artpec6_crypto_hash_import, 2772 .export = artpec6_crypto_hash_export, 2773 .halg.digestsize = SHA384_DIGEST_SIZE, 2774 .halg.statesize = sizeof(struct artpec6_hash_export_state), 2775 .halg.base = { 2776 .cra_name = "sha384", 2777 .cra_driver_name = "artpec-sha384", 2778 .cra_priority = 300, 2779 .cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC, 2780 .cra_blocksize = SHA384_BLOCK_SIZE, 2781 .cra_ctxsize = sizeof(struct artpec6_hashalg_context), 2782 .cra_alignmask = 3, 2783 .cra_module = THIS_MODULE, 2784 .cra_init = artpec6_crypto_ahash_init, 2785 .cra_exit = artpec6_crypto_ahash_exit, 2786 } 2787 }, 2788 /* HMAC SHA-384 */ 2789 { 2790 .init = artpec6_crypto_hmac_sha384_init, 2791 .update = artpec6_crypto_hash_update, 2792 .final = artpec6_crypto_hash_final, 2793 .digest = artpec6_crypto_hmac_sha384_digest, 2794 .import = artpec6_crypto_hash_import, 2795 .export = artpec6_crypto_hash_export, 2796 .setkey = artpec6_crypto_hash_set_key, 2797 .halg.digestsize = SHA384_DIGEST_SIZE, 2798 .halg.statesize = sizeof(struct artpec6_hash_export_state), 2799 .halg.base = { 2800 .cra_name = "hmac(sha384)", 2801 .cra_driver_name = "artpec-hmac-sha384", 2802 .cra_priority = 300, 2803 .cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC, 2804 .cra_blocksize = SHA384_BLOCK_SIZE, 2805 .cra_ctxsize = sizeof(struct artpec6_hashalg_context), 2806 .cra_alignmask = 3, 2807 .cra_module = THIS_MODULE, 2808 .cra_init = artpec6_crypto_ahash_init_hmac_sha384, 2809 .cra_exit = artpec6_crypto_ahash_exit, 2810 } 2811 }, 2812 /* SHA-512 */ 2813 { 2814 .init = artpec6_crypto_sha512_init, 2815 .update = artpec6_crypto_hash_update, 2816 .final = artpec6_crypto_hash_final, 2817 .digest = artpec6_crypto_sha512_digest, 2818 .import = artpec6_crypto_hash_import, 2819 .export = artpec6_crypto_hash_export, 2820 .halg.digestsize = SHA512_DIGEST_SIZE, 2821 .halg.statesize = sizeof(struct artpec6_hash_export_state), 2822 .halg.base = { 2823 .cra_name = "sha512", 2824 .cra_driver_name = "artpec-sha512", 2825 .cra_priority = 300, 2826 .cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC, 2827 .cra_blocksize = SHA512_BLOCK_SIZE, 2828 .cra_ctxsize = sizeof(struct artpec6_hashalg_context), 2829 .cra_alignmask = 3, 2830 .cra_module = THIS_MODULE, 2831 .cra_init = artpec6_crypto_ahash_init, 2832 .cra_exit = artpec6_crypto_ahash_exit, 2833 } 2834 }, 2835 /* HMAC SHA-512 */ 2836 { 2837 .init = artpec6_crypto_hmac_sha512_init, 2838 .update = artpec6_crypto_hash_update, 2839 .final = artpec6_crypto_hash_final, 2840 .digest = artpec6_crypto_hmac_sha512_digest, 2841 .import = artpec6_crypto_hash_import, 2842 .export = artpec6_crypto_hash_export, 2843 .setkey = artpec6_crypto_hash_set_key, 2844 .halg.digestsize = SHA512_DIGEST_SIZE, 2845 .halg.statesize = sizeof(struct artpec6_hash_export_state), 2846 .halg.base = { 2847 .cra_name = "hmac(sha512)", 2848 .cra_driver_name = "artpec-hmac-sha512", 2849 .cra_priority = 300, 2850 .cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC, 2851 .cra_blocksize = SHA512_BLOCK_SIZE, 2852 .cra_ctxsize = sizeof(struct artpec6_hashalg_context), 2853 .cra_alignmask = 3, 2854 .cra_module = THIS_MODULE, 2855 .cra_init = artpec6_crypto_ahash_init_hmac_sha512, 2856 .cra_exit = artpec6_crypto_ahash_exit, 2857 } 2858 }, 2859 }; 2860 2861 /* Crypto */ 2862 static struct skcipher_alg crypto_algos[] = { 2863 /* AES - ECB */ 2864 { 2865 .base = { 2866 .cra_name = "ecb(aes)", 2867 .cra_driver_name = "artpec6-ecb-aes", 2868 .cra_priority = 300, 2869 .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | 2870 CRYPTO_ALG_ASYNC, 2871 .cra_blocksize = AES_BLOCK_SIZE, 2872 .cra_ctxsize = sizeof(struct artpec6_cryptotfm_context), 2873 .cra_alignmask = 3, 2874 .cra_module = THIS_MODULE, 2875 }, 2876 .min_keysize = AES_MIN_KEY_SIZE, 2877 .max_keysize = AES_MAX_KEY_SIZE, 2878 .setkey = artpec6_crypto_cipher_set_key, 2879 .encrypt = artpec6_crypto_encrypt, 2880 .decrypt = artpec6_crypto_decrypt, 2881 .init = artpec6_crypto_aes_ecb_init, 2882 .exit = artpec6_crypto_aes_exit, 2883 }, 2884 /* AES - CTR */ 2885 { 2886 .base = { 2887 .cra_name = "ctr(aes)", 2888 .cra_driver_name = "artpec6-ctr-aes", 2889 .cra_priority = 300, 2890 .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | 2891 CRYPTO_ALG_ASYNC | 2892 CRYPTO_ALG_NEED_FALLBACK, 2893 .cra_blocksize = 1, 2894 .cra_ctxsize = sizeof(struct artpec6_cryptotfm_context), 2895 .cra_alignmask = 3, 2896 .cra_module = THIS_MODULE, 2897 }, 2898 .min_keysize = AES_MIN_KEY_SIZE, 2899 .max_keysize = AES_MAX_KEY_SIZE, 2900 .ivsize = AES_BLOCK_SIZE, 2901 .setkey = artpec6_crypto_cipher_set_key, 2902 .encrypt = artpec6_crypto_ctr_encrypt, 2903 .decrypt = artpec6_crypto_ctr_decrypt, 2904 .init = artpec6_crypto_aes_ctr_init, 2905 .exit = artpec6_crypto_aes_ctr_exit, 2906 }, 2907 /* AES - CBC */ 2908 { 2909 .base = { 2910 .cra_name = "cbc(aes)", 2911 .cra_driver_name = "artpec6-cbc-aes", 2912 .cra_priority = 300, 2913 .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | 2914 CRYPTO_ALG_ASYNC, 2915 .cra_blocksize = AES_BLOCK_SIZE, 2916 .cra_ctxsize = sizeof(struct artpec6_cryptotfm_context), 2917 .cra_alignmask = 3, 2918 .cra_module = THIS_MODULE, 2919 }, 2920 .min_keysize = AES_MIN_KEY_SIZE, 2921 .max_keysize = AES_MAX_KEY_SIZE, 2922 .ivsize = AES_BLOCK_SIZE, 2923 .setkey = artpec6_crypto_cipher_set_key, 2924 .encrypt = artpec6_crypto_encrypt, 2925 .decrypt = artpec6_crypto_decrypt, 2926 .init = artpec6_crypto_aes_cbc_init, 2927 .exit = artpec6_crypto_aes_exit 2928 }, 2929 /* AES - XTS */ 2930 { 2931 .base = { 2932 .cra_name = "xts(aes)", 2933 .cra_driver_name = "artpec6-xts-aes", 2934 .cra_priority = 300, 2935 .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | 2936 CRYPTO_ALG_ASYNC, 2937 .cra_blocksize = 1, 2938 .cra_ctxsize = sizeof(struct artpec6_cryptotfm_context), 2939 .cra_alignmask = 3, 2940 .cra_module = THIS_MODULE, 2941 }, 2942 .min_keysize = 2*AES_MIN_KEY_SIZE, 2943 .max_keysize = 2*AES_MAX_KEY_SIZE, 2944 .ivsize = 16, 2945 .setkey = artpec6_crypto_xts_set_key, 2946 .encrypt = artpec6_crypto_encrypt, 2947 .decrypt = artpec6_crypto_decrypt, 2948 .init = artpec6_crypto_aes_xts_init, 2949 .exit = artpec6_crypto_aes_exit, 2950 }, 2951 }; 2952 2953 static struct aead_alg aead_algos[] = { 2954 { 2955 .init = artpec6_crypto_aead_init, 2956 .setkey = artpec6_crypto_aead_set_key, 2957 .encrypt = artpec6_crypto_aead_encrypt, 2958 .decrypt = artpec6_crypto_aead_decrypt, 2959 .ivsize = AES_BLOCK_SIZE, 2960 .maxauthsize = AES_BLOCK_SIZE, 2961 2962 .base = { 2963 .cra_name = "gcm(aes)", 2964 .cra_driver_name = "artpec-gcm-aes", 2965 .cra_priority = 300, 2966 .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC | 2967 CRYPTO_ALG_KERN_DRIVER_ONLY, 2968 .cra_blocksize = 1, 2969 .cra_ctxsize = sizeof(struct artpec6_cryptotfm_context), 2970 .cra_alignmask = 3, 2971 .cra_module = THIS_MODULE, 2972 }, 2973 } 2974 }; 2975 2976 #ifdef CONFIG_DEBUG_FS 2977 2978 struct dbgfs_u32 { 2979 char *name; 2980 mode_t mode; 2981 u32 *flag; 2982 char *desc; 2983 }; 2984 2985 static struct dentry *dbgfs_root; 2986 2987 static void artpec6_crypto_init_debugfs(void) 2988 { 2989 dbgfs_root = debugfs_create_dir("artpec6_crypto", NULL); 2990 2991 if (!dbgfs_root || IS_ERR(dbgfs_root)) { 2992 dbgfs_root = NULL; 2993 pr_err("%s: Could not initialise debugfs!\n", MODULE_NAME); 2994 return; 2995 } 2996 2997 #ifdef CONFIG_FAULT_INJECTION 2998 fault_create_debugfs_attr("fail_status_read", dbgfs_root, 2999 &artpec6_crypto_fail_status_read); 3000 3001 fault_create_debugfs_attr("fail_dma_array_full", dbgfs_root, 3002 &artpec6_crypto_fail_dma_array_full); 3003 #endif 3004 } 3005 3006 static void artpec6_crypto_free_debugfs(void) 3007 { 3008 if (!dbgfs_root) 3009 return; 3010 3011 debugfs_remove_recursive(dbgfs_root); 3012 dbgfs_root = NULL; 3013 } 3014 #endif 3015 3016 static const struct of_device_id artpec6_crypto_of_match[] = { 3017 { .compatible = "axis,artpec6-crypto", .data = (void *)ARTPEC6_CRYPTO }, 3018 { .compatible = "axis,artpec7-crypto", .data = (void *)ARTPEC7_CRYPTO }, 3019 {} 3020 }; 3021 MODULE_DEVICE_TABLE(of, artpec6_crypto_of_match); 3022 3023 static int artpec6_crypto_probe(struct platform_device *pdev) 3024 { 3025 const struct of_device_id *match; 3026 enum artpec6_crypto_variant variant; 3027 struct artpec6_crypto *ac; 3028 struct device *dev = &pdev->dev; 3029 void __iomem *base; 3030 struct resource *res; 3031 int irq; 3032 int err; 3033 3034 if (artpec6_crypto_dev) 3035 return -ENODEV; 3036 3037 match = of_match_node(artpec6_crypto_of_match, dev->of_node); 3038 if (!match) 3039 return -EINVAL; 3040 3041 variant = (enum artpec6_crypto_variant)match->data; 3042 3043 res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 3044 if (!res) 3045 return -ENODEV; 3046 3047 base = devm_ioremap_resource(&pdev->dev, res); 3048 if (IS_ERR(base)) 3049 return PTR_ERR(base); 3050 3051 irq = platform_get_irq(pdev, 0); 3052 if (irq < 0) 3053 return -ENODEV; 3054 3055 ac = devm_kzalloc(&pdev->dev, sizeof(struct artpec6_crypto), 3056 GFP_KERNEL); 3057 if (!ac) 3058 return -ENOMEM; 3059 3060 platform_set_drvdata(pdev, ac); 3061 ac->variant = variant; 3062 3063 spin_lock_init(&ac->queue_lock); 3064 INIT_LIST_HEAD(&ac->queue); 3065 INIT_LIST_HEAD(&ac->pending); 3066 timer_setup(&ac->timer, artpec6_crypto_timeout, 0); 3067 3068 ac->base = base; 3069 3070 ac->dma_cache = kmem_cache_create("artpec6_crypto_dma", 3071 sizeof(struct artpec6_crypto_dma_descriptors), 3072 64, 3073 0, 3074 NULL); 3075 if (!ac->dma_cache) 3076 return -ENOMEM; 3077 3078 #ifdef CONFIG_DEBUG_FS 3079 artpec6_crypto_init_debugfs(); 3080 #endif 3081 3082 tasklet_init(&ac->task, artpec6_crypto_task, 3083 (unsigned long)ac); 3084 3085 ac->pad_buffer = devm_kzalloc(&pdev->dev, 2 * ARTPEC_CACHE_LINE_MAX, 3086 GFP_KERNEL); 3087 if (!ac->pad_buffer) 3088 return -ENOMEM; 3089 ac->pad_buffer = PTR_ALIGN(ac->pad_buffer, ARTPEC_CACHE_LINE_MAX); 3090 3091 ac->zero_buffer = devm_kzalloc(&pdev->dev, 2 * ARTPEC_CACHE_LINE_MAX, 3092 GFP_KERNEL); 3093 if (!ac->zero_buffer) 3094 return -ENOMEM; 3095 ac->zero_buffer = PTR_ALIGN(ac->zero_buffer, ARTPEC_CACHE_LINE_MAX); 3096 3097 err = init_crypto_hw(ac); 3098 if (err) 3099 goto free_cache; 3100 3101 err = devm_request_irq(&pdev->dev, irq, artpec6_crypto_irq, 0, 3102 "artpec6-crypto", ac); 3103 if (err) 3104 goto disable_hw; 3105 3106 artpec6_crypto_dev = &pdev->dev; 3107 3108 err = crypto_register_ahashes(hash_algos, ARRAY_SIZE(hash_algos)); 3109 if (err) { 3110 dev_err(dev, "Failed to register ahashes\n"); 3111 goto disable_hw; 3112 } 3113 3114 if (variant != ARTPEC6_CRYPTO) { 3115 err = crypto_register_ahashes(artpec7_hash_algos, 3116 ARRAY_SIZE(artpec7_hash_algos)); 3117 if (err) { 3118 dev_err(dev, "Failed to register ahashes\n"); 3119 goto unregister_ahashes; 3120 } 3121 } 3122 3123 err = crypto_register_skciphers(crypto_algos, ARRAY_SIZE(crypto_algos)); 3124 if (err) { 3125 dev_err(dev, "Failed to register ciphers\n"); 3126 goto unregister_a7_ahashes; 3127 } 3128 3129 err = crypto_register_aeads(aead_algos, ARRAY_SIZE(aead_algos)); 3130 if (err) { 3131 dev_err(dev, "Failed to register aeads\n"); 3132 goto unregister_algs; 3133 } 3134 3135 return 0; 3136 3137 unregister_algs: 3138 crypto_unregister_skciphers(crypto_algos, ARRAY_SIZE(crypto_algos)); 3139 unregister_a7_ahashes: 3140 if (variant != ARTPEC6_CRYPTO) 3141 crypto_unregister_ahashes(artpec7_hash_algos, 3142 ARRAY_SIZE(artpec7_hash_algos)); 3143 unregister_ahashes: 3144 crypto_unregister_ahashes(hash_algos, ARRAY_SIZE(hash_algos)); 3145 disable_hw: 3146 artpec6_crypto_disable_hw(ac); 3147 free_cache: 3148 kmem_cache_destroy(ac->dma_cache); 3149 return err; 3150 } 3151 3152 static int artpec6_crypto_remove(struct platform_device *pdev) 3153 { 3154 struct artpec6_crypto *ac = platform_get_drvdata(pdev); 3155 int irq = platform_get_irq(pdev, 0); 3156 3157 crypto_unregister_ahashes(hash_algos, ARRAY_SIZE(hash_algos)); 3158 if (ac->variant != ARTPEC6_CRYPTO) 3159 crypto_unregister_ahashes(artpec7_hash_algos, 3160 ARRAY_SIZE(artpec7_hash_algos)); 3161 crypto_unregister_skciphers(crypto_algos, ARRAY_SIZE(crypto_algos)); 3162 crypto_unregister_aeads(aead_algos, ARRAY_SIZE(aead_algos)); 3163 3164 tasklet_disable(&ac->task); 3165 devm_free_irq(&pdev->dev, irq, ac); 3166 tasklet_kill(&ac->task); 3167 del_timer_sync(&ac->timer); 3168 3169 artpec6_crypto_disable_hw(ac); 3170 3171 kmem_cache_destroy(ac->dma_cache); 3172 #ifdef CONFIG_DEBUG_FS 3173 artpec6_crypto_free_debugfs(); 3174 #endif 3175 return 0; 3176 } 3177 3178 static struct platform_driver artpec6_crypto_driver = { 3179 .probe = artpec6_crypto_probe, 3180 .remove = artpec6_crypto_remove, 3181 .driver = { 3182 .name = "artpec6-crypto", 3183 .owner = THIS_MODULE, 3184 .of_match_table = artpec6_crypto_of_match, 3185 }, 3186 }; 3187 3188 module_platform_driver(artpec6_crypto_driver); 3189 3190 MODULE_AUTHOR("Axis Communications AB"); 3191 MODULE_DESCRIPTION("ARTPEC-6 Crypto driver"); 3192 MODULE_LICENSE("GPL"); 3193