1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Driver for ARTPEC-6 crypto block using the kernel asynchronous crypto api. 4 * 5 * Copyright (C) 2014-2017 Axis Communications AB 6 */ 7 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 8 9 #include <linux/bitfield.h> 10 #include <linux/crypto.h> 11 #include <linux/debugfs.h> 12 #include <linux/delay.h> 13 #include <linux/dma-mapping.h> 14 #include <linux/fault-inject.h> 15 #include <linux/init.h> 16 #include <linux/interrupt.h> 17 #include <linux/kernel.h> 18 #include <linux/list.h> 19 #include <linux/module.h> 20 #include <linux/of.h> 21 #include <linux/platform_device.h> 22 #include <linux/scatterlist.h> 23 #include <linux/slab.h> 24 25 #include <crypto/aes.h> 26 #include <crypto/gcm.h> 27 #include <crypto/internal/aead.h> 28 #include <crypto/internal/hash.h> 29 #include <crypto/internal/skcipher.h> 30 #include <crypto/scatterwalk.h> 31 #include <crypto/sha.h> 32 #include <crypto/xts.h> 33 34 /* Max length of a line in all cache levels for Artpec SoCs. */ 35 #define ARTPEC_CACHE_LINE_MAX 32 36 37 #define PDMA_OUT_CFG 0x0000 38 #define PDMA_OUT_BUF_CFG 0x0004 39 #define PDMA_OUT_CMD 0x0008 40 #define PDMA_OUT_DESCRQ_PUSH 0x0010 41 #define PDMA_OUT_DESCRQ_STAT 0x0014 42 43 #define A6_PDMA_IN_CFG 0x0028 44 #define A6_PDMA_IN_BUF_CFG 0x002c 45 #define A6_PDMA_IN_CMD 0x0030 46 #define A6_PDMA_IN_STATQ_PUSH 0x0038 47 #define A6_PDMA_IN_DESCRQ_PUSH 0x0044 48 #define A6_PDMA_IN_DESCRQ_STAT 0x0048 49 #define A6_PDMA_INTR_MASK 0x0068 50 #define A6_PDMA_ACK_INTR 0x006c 51 #define A6_PDMA_MASKED_INTR 0x0074 52 53 #define A7_PDMA_IN_CFG 0x002c 54 #define A7_PDMA_IN_BUF_CFG 0x0030 55 #define A7_PDMA_IN_CMD 0x0034 56 #define A7_PDMA_IN_STATQ_PUSH 0x003c 57 #define A7_PDMA_IN_DESCRQ_PUSH 0x0048 58 #define A7_PDMA_IN_DESCRQ_STAT 0x004C 59 #define A7_PDMA_INTR_MASK 0x006c 60 #define A7_PDMA_ACK_INTR 0x0070 61 #define A7_PDMA_MASKED_INTR 0x0078 62 63 #define PDMA_OUT_CFG_EN BIT(0) 64 65 #define PDMA_OUT_BUF_CFG_DATA_BUF_SIZE GENMASK(4, 0) 66 #define PDMA_OUT_BUF_CFG_DESCR_BUF_SIZE GENMASK(9, 5) 67 68 #define PDMA_OUT_CMD_START BIT(0) 69 #define A6_PDMA_OUT_CMD_STOP BIT(3) 70 #define A7_PDMA_OUT_CMD_STOP BIT(2) 71 72 #define PDMA_OUT_DESCRQ_PUSH_LEN GENMASK(5, 0) 73 #define PDMA_OUT_DESCRQ_PUSH_ADDR GENMASK(31, 6) 74 75 #define PDMA_OUT_DESCRQ_STAT_LEVEL GENMASK(3, 0) 76 #define PDMA_OUT_DESCRQ_STAT_SIZE GENMASK(7, 4) 77 78 #define PDMA_IN_CFG_EN BIT(0) 79 80 #define PDMA_IN_BUF_CFG_DATA_BUF_SIZE GENMASK(4, 0) 81 #define PDMA_IN_BUF_CFG_DESCR_BUF_SIZE GENMASK(9, 5) 82 #define PDMA_IN_BUF_CFG_STAT_BUF_SIZE GENMASK(14, 10) 83 84 #define PDMA_IN_CMD_START BIT(0) 85 #define A6_PDMA_IN_CMD_FLUSH_STAT BIT(2) 86 #define A6_PDMA_IN_CMD_STOP BIT(3) 87 #define A7_PDMA_IN_CMD_FLUSH_STAT BIT(1) 88 #define A7_PDMA_IN_CMD_STOP BIT(2) 89 90 #define PDMA_IN_STATQ_PUSH_LEN GENMASK(5, 0) 91 #define PDMA_IN_STATQ_PUSH_ADDR GENMASK(31, 6) 92 93 #define PDMA_IN_DESCRQ_PUSH_LEN GENMASK(5, 0) 94 #define PDMA_IN_DESCRQ_PUSH_ADDR GENMASK(31, 6) 95 96 #define PDMA_IN_DESCRQ_STAT_LEVEL GENMASK(3, 0) 97 #define PDMA_IN_DESCRQ_STAT_SIZE GENMASK(7, 4) 98 99 #define A6_PDMA_INTR_MASK_IN_DATA BIT(2) 100 #define A6_PDMA_INTR_MASK_IN_EOP BIT(3) 101 #define A6_PDMA_INTR_MASK_IN_EOP_FLUSH BIT(4) 102 103 #define A7_PDMA_INTR_MASK_IN_DATA BIT(3) 104 #define A7_PDMA_INTR_MASK_IN_EOP BIT(4) 105 #define A7_PDMA_INTR_MASK_IN_EOP_FLUSH BIT(5) 106 107 #define A6_CRY_MD_OPER GENMASK(19, 16) 108 109 #define A6_CRY_MD_HASH_SEL_CTX GENMASK(21, 20) 110 #define A6_CRY_MD_HASH_HMAC_FIN BIT(23) 111 112 #define A6_CRY_MD_CIPHER_LEN GENMASK(21, 20) 113 #define A6_CRY_MD_CIPHER_DECR BIT(22) 114 #define A6_CRY_MD_CIPHER_TWEAK BIT(23) 115 #define A6_CRY_MD_CIPHER_DSEQ BIT(24) 116 117 #define A7_CRY_MD_OPER GENMASK(11, 8) 118 119 #define A7_CRY_MD_HASH_SEL_CTX GENMASK(13, 12) 120 #define A7_CRY_MD_HASH_HMAC_FIN BIT(15) 121 122 #define A7_CRY_MD_CIPHER_LEN GENMASK(13, 12) 123 #define A7_CRY_MD_CIPHER_DECR BIT(14) 124 #define A7_CRY_MD_CIPHER_TWEAK BIT(15) 125 #define A7_CRY_MD_CIPHER_DSEQ BIT(16) 126 127 /* DMA metadata constants */ 128 #define regk_crypto_aes_cbc 0x00000002 129 #define regk_crypto_aes_ctr 0x00000003 130 #define regk_crypto_aes_ecb 0x00000001 131 #define regk_crypto_aes_gcm 0x00000004 132 #define regk_crypto_aes_xts 0x00000005 133 #define regk_crypto_cache 0x00000002 134 #define a6_regk_crypto_dlkey 0x0000000a 135 #define a7_regk_crypto_dlkey 0x0000000e 136 #define regk_crypto_ext 0x00000001 137 #define regk_crypto_hmac_sha1 0x00000007 138 #define regk_crypto_hmac_sha256 0x00000009 139 #define regk_crypto_init 0x00000000 140 #define regk_crypto_key_128 0x00000000 141 #define regk_crypto_key_192 0x00000001 142 #define regk_crypto_key_256 0x00000002 143 #define regk_crypto_null 0x00000000 144 #define regk_crypto_sha1 0x00000006 145 #define regk_crypto_sha256 0x00000008 146 147 /* DMA descriptor structures */ 148 struct pdma_descr_ctrl { 149 unsigned char short_descr : 1; 150 unsigned char pad1 : 1; 151 unsigned char eop : 1; 152 unsigned char intr : 1; 153 unsigned char short_len : 3; 154 unsigned char pad2 : 1; 155 } __packed; 156 157 struct pdma_data_descr { 158 unsigned int len : 24; 159 unsigned int buf : 32; 160 } __packed; 161 162 struct pdma_short_descr { 163 unsigned char data[7]; 164 } __packed; 165 166 struct pdma_descr { 167 struct pdma_descr_ctrl ctrl; 168 union { 169 struct pdma_data_descr data; 170 struct pdma_short_descr shrt; 171 }; 172 }; 173 174 struct pdma_stat_descr { 175 unsigned char pad1 : 1; 176 unsigned char pad2 : 1; 177 unsigned char eop : 1; 178 unsigned char pad3 : 5; 179 unsigned int len : 24; 180 }; 181 182 /* Each descriptor array can hold max 64 entries */ 183 #define PDMA_DESCR_COUNT 64 184 185 #define MODULE_NAME "Artpec-6 CA" 186 187 /* Hash modes (including HMAC variants) */ 188 #define ARTPEC6_CRYPTO_HASH_SHA1 1 189 #define ARTPEC6_CRYPTO_HASH_SHA256 2 190 191 /* Crypto modes */ 192 #define ARTPEC6_CRYPTO_CIPHER_AES_ECB 1 193 #define ARTPEC6_CRYPTO_CIPHER_AES_CBC 2 194 #define ARTPEC6_CRYPTO_CIPHER_AES_CTR 3 195 #define ARTPEC6_CRYPTO_CIPHER_AES_XTS 5 196 197 /* The PDMA is a DMA-engine tightly coupled with a ciphering engine. 198 * It operates on a descriptor array with up to 64 descriptor entries. 199 * The arrays must be 64 byte aligned in memory. 200 * 201 * The ciphering unit has no registers and is completely controlled by 202 * a 4-byte metadata that is inserted at the beginning of each dma packet. 203 * 204 * A dma packet is a sequence of descriptors terminated by setting the .eop 205 * field in the final descriptor of the packet. 206 * 207 * Multiple packets are used for providing context data, key data and 208 * the plain/ciphertext. 209 * 210 * PDMA Descriptors (Array) 211 * +------+------+------+~~+-------+------+---- 212 * | 0 | 1 | 2 |~~| 11 EOP| 12 | .... 213 * +--+---+--+---+----+-+~~+-------+----+-+---- 214 * | | | | | 215 * | | | | | 216 * __|__ +-------++-------++-------+ +----+ 217 * | MD | |Payload||Payload||Payload| | MD | 218 * +-----+ +-------++-------++-------+ +----+ 219 */ 220 221 struct artpec6_crypto_bounce_buffer { 222 struct list_head list; 223 size_t length; 224 struct scatterlist *sg; 225 size_t offset; 226 /* buf is aligned to ARTPEC_CACHE_LINE_MAX and 227 * holds up to ARTPEC_CACHE_LINE_MAX bytes data. 228 */ 229 void *buf; 230 }; 231 232 struct artpec6_crypto_dma_map { 233 dma_addr_t dma_addr; 234 size_t size; 235 enum dma_data_direction dir; 236 }; 237 238 struct artpec6_crypto_dma_descriptors { 239 struct pdma_descr out[PDMA_DESCR_COUNT] __aligned(64); 240 struct pdma_descr in[PDMA_DESCR_COUNT] __aligned(64); 241 u32 stat[PDMA_DESCR_COUNT] __aligned(64); 242 struct list_head bounce_buffers; 243 /* Enough maps for all out/in buffers, and all three descr. arrays */ 244 struct artpec6_crypto_dma_map maps[PDMA_DESCR_COUNT * 2 + 2]; 245 dma_addr_t out_dma_addr; 246 dma_addr_t in_dma_addr; 247 dma_addr_t stat_dma_addr; 248 size_t out_cnt; 249 size_t in_cnt; 250 size_t map_count; 251 }; 252 253 enum artpec6_crypto_variant { 254 ARTPEC6_CRYPTO, 255 ARTPEC7_CRYPTO, 256 }; 257 258 struct artpec6_crypto { 259 void __iomem *base; 260 spinlock_t queue_lock; 261 struct list_head queue; /* waiting for pdma fifo space */ 262 struct list_head pending; /* submitted to pdma fifo */ 263 struct tasklet_struct task; 264 struct kmem_cache *dma_cache; 265 int pending_count; 266 struct timer_list timer; 267 enum artpec6_crypto_variant variant; 268 void *pad_buffer; /* cache-aligned block padding buffer */ 269 void *zero_buffer; 270 }; 271 272 enum artpec6_crypto_hash_flags { 273 HASH_FLAG_INIT_CTX = 2, 274 HASH_FLAG_UPDATE = 4, 275 HASH_FLAG_FINALIZE = 8, 276 HASH_FLAG_HMAC = 16, 277 HASH_FLAG_UPDATE_KEY = 32, 278 }; 279 280 struct artpec6_crypto_req_common { 281 struct list_head list; 282 struct list_head complete_in_progress; 283 struct artpec6_crypto_dma_descriptors *dma; 284 struct crypto_async_request *req; 285 void (*complete)(struct crypto_async_request *req); 286 gfp_t gfp_flags; 287 }; 288 289 struct artpec6_hash_request_context { 290 char partial_buffer[SHA256_BLOCK_SIZE]; 291 char partial_buffer_out[SHA256_BLOCK_SIZE]; 292 char key_buffer[SHA256_BLOCK_SIZE]; 293 char pad_buffer[SHA256_BLOCK_SIZE + 32]; 294 unsigned char digeststate[SHA256_DIGEST_SIZE]; 295 size_t partial_bytes; 296 u64 digcnt; 297 u32 key_md; 298 u32 hash_md; 299 enum artpec6_crypto_hash_flags hash_flags; 300 struct artpec6_crypto_req_common common; 301 }; 302 303 struct artpec6_hash_export_state { 304 char partial_buffer[SHA256_BLOCK_SIZE]; 305 unsigned char digeststate[SHA256_DIGEST_SIZE]; 306 size_t partial_bytes; 307 u64 digcnt; 308 int oper; 309 unsigned int hash_flags; 310 }; 311 312 struct artpec6_hashalg_context { 313 char hmac_key[SHA256_BLOCK_SIZE]; 314 size_t hmac_key_length; 315 struct crypto_shash *child_hash; 316 }; 317 318 struct artpec6_crypto_request_context { 319 u32 cipher_md; 320 bool decrypt; 321 struct artpec6_crypto_req_common common; 322 }; 323 324 struct artpec6_cryptotfm_context { 325 unsigned char aes_key[2*AES_MAX_KEY_SIZE]; 326 size_t key_length; 327 u32 key_md; 328 int crypto_type; 329 struct crypto_sync_skcipher *fallback; 330 }; 331 332 struct artpec6_crypto_aead_hw_ctx { 333 __be64 aad_length_bits; 334 __be64 text_length_bits; 335 __u8 J0[AES_BLOCK_SIZE]; 336 }; 337 338 struct artpec6_crypto_aead_req_ctx { 339 struct artpec6_crypto_aead_hw_ctx hw_ctx; 340 u32 cipher_md; 341 bool decrypt; 342 struct artpec6_crypto_req_common common; 343 __u8 decryption_tag[AES_BLOCK_SIZE] ____cacheline_aligned; 344 }; 345 346 /* The crypto framework makes it hard to avoid this global. */ 347 static struct device *artpec6_crypto_dev; 348 349 #ifdef CONFIG_FAULT_INJECTION 350 static DECLARE_FAULT_ATTR(artpec6_crypto_fail_status_read); 351 static DECLARE_FAULT_ATTR(artpec6_crypto_fail_dma_array_full); 352 #endif 353 354 enum { 355 ARTPEC6_CRYPTO_PREPARE_HASH_NO_START, 356 ARTPEC6_CRYPTO_PREPARE_HASH_START, 357 }; 358 359 static int artpec6_crypto_prepare_aead(struct aead_request *areq); 360 static int artpec6_crypto_prepare_crypto(struct skcipher_request *areq); 361 static int artpec6_crypto_prepare_hash(struct ahash_request *areq); 362 363 static void 364 artpec6_crypto_complete_crypto(struct crypto_async_request *req); 365 static void 366 artpec6_crypto_complete_cbc_encrypt(struct crypto_async_request *req); 367 static void 368 artpec6_crypto_complete_cbc_decrypt(struct crypto_async_request *req); 369 static void 370 artpec6_crypto_complete_aead(struct crypto_async_request *req); 371 static void 372 artpec6_crypto_complete_hash(struct crypto_async_request *req); 373 374 static int 375 artpec6_crypto_common_destroy(struct artpec6_crypto_req_common *common); 376 377 static void 378 artpec6_crypto_start_dma(struct artpec6_crypto_req_common *common); 379 380 struct artpec6_crypto_walk { 381 struct scatterlist *sg; 382 size_t offset; 383 }; 384 385 static void artpec6_crypto_walk_init(struct artpec6_crypto_walk *awalk, 386 struct scatterlist *sg) 387 { 388 awalk->sg = sg; 389 awalk->offset = 0; 390 } 391 392 static size_t artpec6_crypto_walk_advance(struct artpec6_crypto_walk *awalk, 393 size_t nbytes) 394 { 395 while (nbytes && awalk->sg) { 396 size_t piece; 397 398 WARN_ON(awalk->offset > awalk->sg->length); 399 400 piece = min(nbytes, (size_t)awalk->sg->length - awalk->offset); 401 nbytes -= piece; 402 awalk->offset += piece; 403 if (awalk->offset == awalk->sg->length) { 404 awalk->sg = sg_next(awalk->sg); 405 awalk->offset = 0; 406 } 407 408 } 409 410 return nbytes; 411 } 412 413 static size_t 414 artpec6_crypto_walk_chunklen(const struct artpec6_crypto_walk *awalk) 415 { 416 WARN_ON(awalk->sg->length == awalk->offset); 417 418 return awalk->sg->length - awalk->offset; 419 } 420 421 static dma_addr_t 422 artpec6_crypto_walk_chunk_phys(const struct artpec6_crypto_walk *awalk) 423 { 424 return sg_phys(awalk->sg) + awalk->offset; 425 } 426 427 static void 428 artpec6_crypto_copy_bounce_buffers(struct artpec6_crypto_req_common *common) 429 { 430 struct artpec6_crypto_dma_descriptors *dma = common->dma; 431 struct artpec6_crypto_bounce_buffer *b; 432 struct artpec6_crypto_bounce_buffer *next; 433 434 list_for_each_entry_safe(b, next, &dma->bounce_buffers, list) { 435 pr_debug("bounce entry %p: %zu bytes @ %zu from %p\n", 436 b, b->length, b->offset, b->buf); 437 sg_pcopy_from_buffer(b->sg, 438 1, 439 b->buf, 440 b->length, 441 b->offset); 442 443 list_del(&b->list); 444 kfree(b); 445 } 446 } 447 448 static inline bool artpec6_crypto_busy(void) 449 { 450 struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev); 451 int fifo_count = ac->pending_count; 452 453 return fifo_count > 6; 454 } 455 456 static int artpec6_crypto_submit(struct artpec6_crypto_req_common *req) 457 { 458 struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev); 459 int ret = -EBUSY; 460 461 spin_lock_bh(&ac->queue_lock); 462 463 if (!artpec6_crypto_busy()) { 464 list_add_tail(&req->list, &ac->pending); 465 artpec6_crypto_start_dma(req); 466 ret = -EINPROGRESS; 467 } else if (req->req->flags & CRYPTO_TFM_REQ_MAY_BACKLOG) { 468 list_add_tail(&req->list, &ac->queue); 469 } else { 470 artpec6_crypto_common_destroy(req); 471 } 472 473 spin_unlock_bh(&ac->queue_lock); 474 475 return ret; 476 } 477 478 static void artpec6_crypto_start_dma(struct artpec6_crypto_req_common *common) 479 { 480 struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev); 481 enum artpec6_crypto_variant variant = ac->variant; 482 void __iomem *base = ac->base; 483 struct artpec6_crypto_dma_descriptors *dma = common->dma; 484 u32 ind, statd, outd; 485 486 /* Make descriptor content visible to the DMA before starting it. */ 487 wmb(); 488 489 ind = FIELD_PREP(PDMA_IN_DESCRQ_PUSH_LEN, dma->in_cnt - 1) | 490 FIELD_PREP(PDMA_IN_DESCRQ_PUSH_ADDR, dma->in_dma_addr >> 6); 491 492 statd = FIELD_PREP(PDMA_IN_STATQ_PUSH_LEN, dma->in_cnt - 1) | 493 FIELD_PREP(PDMA_IN_STATQ_PUSH_ADDR, dma->stat_dma_addr >> 6); 494 495 outd = FIELD_PREP(PDMA_OUT_DESCRQ_PUSH_LEN, dma->out_cnt - 1) | 496 FIELD_PREP(PDMA_OUT_DESCRQ_PUSH_ADDR, dma->out_dma_addr >> 6); 497 498 if (variant == ARTPEC6_CRYPTO) { 499 writel_relaxed(ind, base + A6_PDMA_IN_DESCRQ_PUSH); 500 writel_relaxed(statd, base + A6_PDMA_IN_STATQ_PUSH); 501 writel_relaxed(PDMA_IN_CMD_START, base + A6_PDMA_IN_CMD); 502 } else { 503 writel_relaxed(ind, base + A7_PDMA_IN_DESCRQ_PUSH); 504 writel_relaxed(statd, base + A7_PDMA_IN_STATQ_PUSH); 505 writel_relaxed(PDMA_IN_CMD_START, base + A7_PDMA_IN_CMD); 506 } 507 508 writel_relaxed(outd, base + PDMA_OUT_DESCRQ_PUSH); 509 writel_relaxed(PDMA_OUT_CMD_START, base + PDMA_OUT_CMD); 510 511 ac->pending_count++; 512 } 513 514 static void 515 artpec6_crypto_init_dma_operation(struct artpec6_crypto_req_common *common) 516 { 517 struct artpec6_crypto_dma_descriptors *dma = common->dma; 518 519 dma->out_cnt = 0; 520 dma->in_cnt = 0; 521 dma->map_count = 0; 522 INIT_LIST_HEAD(&dma->bounce_buffers); 523 } 524 525 static bool fault_inject_dma_descr(void) 526 { 527 #ifdef CONFIG_FAULT_INJECTION 528 return should_fail(&artpec6_crypto_fail_dma_array_full, 1); 529 #else 530 return false; 531 #endif 532 } 533 534 /** artpec6_crypto_setup_out_descr_phys - Setup an out channel with a 535 * physical address 536 * 537 * @addr: The physical address of the data buffer 538 * @len: The length of the data buffer 539 * @eop: True if this is the last buffer in the packet 540 * 541 * @return 0 on success or -ENOSPC if there are no more descriptors available 542 */ 543 static int 544 artpec6_crypto_setup_out_descr_phys(struct artpec6_crypto_req_common *common, 545 dma_addr_t addr, size_t len, bool eop) 546 { 547 struct artpec6_crypto_dma_descriptors *dma = common->dma; 548 struct pdma_descr *d; 549 550 if (dma->out_cnt >= PDMA_DESCR_COUNT || 551 fault_inject_dma_descr()) { 552 pr_err("No free OUT DMA descriptors available!\n"); 553 return -ENOSPC; 554 } 555 556 d = &dma->out[dma->out_cnt++]; 557 memset(d, 0, sizeof(*d)); 558 559 d->ctrl.short_descr = 0; 560 d->ctrl.eop = eop; 561 d->data.len = len; 562 d->data.buf = addr; 563 return 0; 564 } 565 566 /** artpec6_crypto_setup_out_descr_short - Setup a short out descriptor 567 * 568 * @dst: The virtual address of the data 569 * @len: The length of the data, must be between 1 to 7 bytes 570 * @eop: True if this is the last buffer in the packet 571 * 572 * @return 0 on success 573 * -ENOSPC if no more descriptors are available 574 * -EINVAL if the data length exceeds 7 bytes 575 */ 576 static int 577 artpec6_crypto_setup_out_descr_short(struct artpec6_crypto_req_common *common, 578 void *dst, unsigned int len, bool eop) 579 { 580 struct artpec6_crypto_dma_descriptors *dma = common->dma; 581 struct pdma_descr *d; 582 583 if (dma->out_cnt >= PDMA_DESCR_COUNT || 584 fault_inject_dma_descr()) { 585 pr_err("No free OUT DMA descriptors available!\n"); 586 return -ENOSPC; 587 } else if (len > 7 || len < 1) { 588 return -EINVAL; 589 } 590 d = &dma->out[dma->out_cnt++]; 591 memset(d, 0, sizeof(*d)); 592 593 d->ctrl.short_descr = 1; 594 d->ctrl.short_len = len; 595 d->ctrl.eop = eop; 596 memcpy(d->shrt.data, dst, len); 597 return 0; 598 } 599 600 static int artpec6_crypto_dma_map_page(struct artpec6_crypto_req_common *common, 601 struct page *page, size_t offset, 602 size_t size, 603 enum dma_data_direction dir, 604 dma_addr_t *dma_addr_out) 605 { 606 struct artpec6_crypto_dma_descriptors *dma = common->dma; 607 struct device *dev = artpec6_crypto_dev; 608 struct artpec6_crypto_dma_map *map; 609 dma_addr_t dma_addr; 610 611 *dma_addr_out = 0; 612 613 if (dma->map_count >= ARRAY_SIZE(dma->maps)) 614 return -ENOMEM; 615 616 dma_addr = dma_map_page(dev, page, offset, size, dir); 617 if (dma_mapping_error(dev, dma_addr)) 618 return -ENOMEM; 619 620 map = &dma->maps[dma->map_count++]; 621 map->size = size; 622 map->dma_addr = dma_addr; 623 map->dir = dir; 624 625 *dma_addr_out = dma_addr; 626 627 return 0; 628 } 629 630 static int 631 artpec6_crypto_dma_map_single(struct artpec6_crypto_req_common *common, 632 void *ptr, size_t size, 633 enum dma_data_direction dir, 634 dma_addr_t *dma_addr_out) 635 { 636 struct page *page = virt_to_page(ptr); 637 size_t offset = (uintptr_t)ptr & ~PAGE_MASK; 638 639 return artpec6_crypto_dma_map_page(common, page, offset, size, dir, 640 dma_addr_out); 641 } 642 643 static int 644 artpec6_crypto_dma_map_descs(struct artpec6_crypto_req_common *common) 645 { 646 struct artpec6_crypto_dma_descriptors *dma = common->dma; 647 int ret; 648 649 ret = artpec6_crypto_dma_map_single(common, dma->in, 650 sizeof(dma->in[0]) * dma->in_cnt, 651 DMA_TO_DEVICE, &dma->in_dma_addr); 652 if (ret) 653 return ret; 654 655 ret = artpec6_crypto_dma_map_single(common, dma->out, 656 sizeof(dma->out[0]) * dma->out_cnt, 657 DMA_TO_DEVICE, &dma->out_dma_addr); 658 if (ret) 659 return ret; 660 661 /* We only read one stat descriptor */ 662 dma->stat[dma->in_cnt - 1] = 0; 663 664 /* 665 * DMA_BIDIRECTIONAL since we need our zeroing of the stat descriptor 666 * to be written. 667 */ 668 return artpec6_crypto_dma_map_single(common, 669 dma->stat, 670 sizeof(dma->stat[0]) * dma->in_cnt, 671 DMA_BIDIRECTIONAL, 672 &dma->stat_dma_addr); 673 } 674 675 static void 676 artpec6_crypto_dma_unmap_all(struct artpec6_crypto_req_common *common) 677 { 678 struct artpec6_crypto_dma_descriptors *dma = common->dma; 679 struct device *dev = artpec6_crypto_dev; 680 int i; 681 682 for (i = 0; i < dma->map_count; i++) { 683 struct artpec6_crypto_dma_map *map = &dma->maps[i]; 684 685 dma_unmap_page(dev, map->dma_addr, map->size, map->dir); 686 } 687 688 dma->map_count = 0; 689 } 690 691 /** artpec6_crypto_setup_out_descr - Setup an out descriptor 692 * 693 * @dst: The virtual address of the data 694 * @len: The length of the data 695 * @eop: True if this is the last buffer in the packet 696 * @use_short: If this is true and the data length is 7 bytes or less then 697 * a short descriptor will be used 698 * 699 * @return 0 on success 700 * Any errors from artpec6_crypto_setup_out_descr_short() or 701 * setup_out_descr_phys() 702 */ 703 static int 704 artpec6_crypto_setup_out_descr(struct artpec6_crypto_req_common *common, 705 void *dst, unsigned int len, bool eop, 706 bool use_short) 707 { 708 if (use_short && len < 7) { 709 return artpec6_crypto_setup_out_descr_short(common, dst, len, 710 eop); 711 } else { 712 int ret; 713 dma_addr_t dma_addr; 714 715 ret = artpec6_crypto_dma_map_single(common, dst, len, 716 DMA_TO_DEVICE, 717 &dma_addr); 718 if (ret) 719 return ret; 720 721 return artpec6_crypto_setup_out_descr_phys(common, dma_addr, 722 len, eop); 723 } 724 } 725 726 /** artpec6_crypto_setup_in_descr_phys - Setup an in channel with a 727 * physical address 728 * 729 * @addr: The physical address of the data buffer 730 * @len: The length of the data buffer 731 * @intr: True if an interrupt should be fired after HW processing of this 732 * descriptor 733 * 734 */ 735 static int 736 artpec6_crypto_setup_in_descr_phys(struct artpec6_crypto_req_common *common, 737 dma_addr_t addr, unsigned int len, bool intr) 738 { 739 struct artpec6_crypto_dma_descriptors *dma = common->dma; 740 struct pdma_descr *d; 741 742 if (dma->in_cnt >= PDMA_DESCR_COUNT || 743 fault_inject_dma_descr()) { 744 pr_err("No free IN DMA descriptors available!\n"); 745 return -ENOSPC; 746 } 747 d = &dma->in[dma->in_cnt++]; 748 memset(d, 0, sizeof(*d)); 749 750 d->ctrl.intr = intr; 751 d->data.len = len; 752 d->data.buf = addr; 753 return 0; 754 } 755 756 /** artpec6_crypto_setup_in_descr - Setup an in channel descriptor 757 * 758 * @buffer: The virtual address to of the data buffer 759 * @len: The length of the data buffer 760 * @last: If this is the last data buffer in the request (i.e. an interrupt 761 * is needed 762 * 763 * Short descriptors are not used for the in channel 764 */ 765 static int 766 artpec6_crypto_setup_in_descr(struct artpec6_crypto_req_common *common, 767 void *buffer, unsigned int len, bool last) 768 { 769 dma_addr_t dma_addr; 770 int ret; 771 772 ret = artpec6_crypto_dma_map_single(common, buffer, len, 773 DMA_FROM_DEVICE, &dma_addr); 774 if (ret) 775 return ret; 776 777 return artpec6_crypto_setup_in_descr_phys(common, dma_addr, len, last); 778 } 779 780 static struct artpec6_crypto_bounce_buffer * 781 artpec6_crypto_alloc_bounce(gfp_t flags) 782 { 783 void *base; 784 size_t alloc_size = sizeof(struct artpec6_crypto_bounce_buffer) + 785 2 * ARTPEC_CACHE_LINE_MAX; 786 struct artpec6_crypto_bounce_buffer *bbuf = kzalloc(alloc_size, flags); 787 788 if (!bbuf) 789 return NULL; 790 791 base = bbuf + 1; 792 bbuf->buf = PTR_ALIGN(base, ARTPEC_CACHE_LINE_MAX); 793 return bbuf; 794 } 795 796 static int setup_bounce_buffer_in(struct artpec6_crypto_req_common *common, 797 struct artpec6_crypto_walk *walk, size_t size) 798 { 799 struct artpec6_crypto_bounce_buffer *bbuf; 800 int ret; 801 802 bbuf = artpec6_crypto_alloc_bounce(common->gfp_flags); 803 if (!bbuf) 804 return -ENOMEM; 805 806 bbuf->length = size; 807 bbuf->sg = walk->sg; 808 bbuf->offset = walk->offset; 809 810 ret = artpec6_crypto_setup_in_descr(common, bbuf->buf, size, false); 811 if (ret) { 812 kfree(bbuf); 813 return ret; 814 } 815 816 pr_debug("BOUNCE %zu offset %zu\n", size, walk->offset); 817 list_add_tail(&bbuf->list, &common->dma->bounce_buffers); 818 return 0; 819 } 820 821 static int 822 artpec6_crypto_setup_sg_descrs_in(struct artpec6_crypto_req_common *common, 823 struct artpec6_crypto_walk *walk, 824 size_t count) 825 { 826 size_t chunk; 827 int ret; 828 dma_addr_t addr; 829 830 while (walk->sg && count) { 831 chunk = min(count, artpec6_crypto_walk_chunklen(walk)); 832 addr = artpec6_crypto_walk_chunk_phys(walk); 833 834 /* When destination buffers are not aligned to the cache line 835 * size we need bounce buffers. The DMA-API requires that the 836 * entire line is owned by the DMA buffer and this holds also 837 * for the case when coherent DMA is used. 838 */ 839 if (!IS_ALIGNED(addr, ARTPEC_CACHE_LINE_MAX)) { 840 chunk = min_t(dma_addr_t, chunk, 841 ALIGN(addr, ARTPEC_CACHE_LINE_MAX) - 842 addr); 843 844 pr_debug("CHUNK-b %pad:%zu\n", &addr, chunk); 845 ret = setup_bounce_buffer_in(common, walk, chunk); 846 } else if (chunk < ARTPEC_CACHE_LINE_MAX) { 847 pr_debug("CHUNK-b %pad:%zu\n", &addr, chunk); 848 ret = setup_bounce_buffer_in(common, walk, chunk); 849 } else { 850 dma_addr_t dma_addr; 851 852 chunk = chunk & ~(ARTPEC_CACHE_LINE_MAX-1); 853 854 pr_debug("CHUNK %pad:%zu\n", &addr, chunk); 855 856 ret = artpec6_crypto_dma_map_page(common, 857 sg_page(walk->sg), 858 walk->sg->offset + 859 walk->offset, 860 chunk, 861 DMA_FROM_DEVICE, 862 &dma_addr); 863 if (ret) 864 return ret; 865 866 ret = artpec6_crypto_setup_in_descr_phys(common, 867 dma_addr, 868 chunk, false); 869 } 870 871 if (ret) 872 return ret; 873 874 count = count - chunk; 875 artpec6_crypto_walk_advance(walk, chunk); 876 } 877 878 if (count) 879 pr_err("EOL unexpected %zu bytes left\n", count); 880 881 return count ? -EINVAL : 0; 882 } 883 884 static int 885 artpec6_crypto_setup_sg_descrs_out(struct artpec6_crypto_req_common *common, 886 struct artpec6_crypto_walk *walk, 887 size_t count) 888 { 889 size_t chunk; 890 int ret; 891 dma_addr_t addr; 892 893 while (walk->sg && count) { 894 chunk = min(count, artpec6_crypto_walk_chunklen(walk)); 895 addr = artpec6_crypto_walk_chunk_phys(walk); 896 897 pr_debug("OUT-CHUNK %pad:%zu\n", &addr, chunk); 898 899 if (addr & 3) { 900 char buf[3]; 901 902 chunk = min_t(size_t, chunk, (4-(addr&3))); 903 904 sg_pcopy_to_buffer(walk->sg, 1, buf, chunk, 905 walk->offset); 906 907 ret = artpec6_crypto_setup_out_descr_short(common, buf, 908 chunk, 909 false); 910 } else { 911 dma_addr_t dma_addr; 912 913 ret = artpec6_crypto_dma_map_page(common, 914 sg_page(walk->sg), 915 walk->sg->offset + 916 walk->offset, 917 chunk, 918 DMA_TO_DEVICE, 919 &dma_addr); 920 if (ret) 921 return ret; 922 923 ret = artpec6_crypto_setup_out_descr_phys(common, 924 dma_addr, 925 chunk, false); 926 } 927 928 if (ret) 929 return ret; 930 931 count = count - chunk; 932 artpec6_crypto_walk_advance(walk, chunk); 933 } 934 935 if (count) 936 pr_err("EOL unexpected %zu bytes left\n", count); 937 938 return count ? -EINVAL : 0; 939 } 940 941 942 /** artpec6_crypto_terminate_out_descrs - Set the EOP on the last out descriptor 943 * 944 * If the out descriptor list is non-empty, then the eop flag on the 945 * last used out descriptor will be set. 946 * 947 * @return 0 on success 948 * -EINVAL if the out descriptor is empty or has overflown 949 */ 950 static int 951 artpec6_crypto_terminate_out_descrs(struct artpec6_crypto_req_common *common) 952 { 953 struct artpec6_crypto_dma_descriptors *dma = common->dma; 954 struct pdma_descr *d; 955 956 if (!dma->out_cnt || dma->out_cnt > PDMA_DESCR_COUNT) { 957 pr_err("%s: OUT descriptor list is %s\n", 958 MODULE_NAME, dma->out_cnt ? "empty" : "full"); 959 return -EINVAL; 960 961 } 962 963 d = &dma->out[dma->out_cnt-1]; 964 d->ctrl.eop = 1; 965 966 return 0; 967 } 968 969 /** artpec6_crypto_terminate_in_descrs - Set the interrupt flag on the last 970 * in descriptor 971 * 972 * See artpec6_crypto_terminate_out_descrs() for return values 973 */ 974 static int 975 artpec6_crypto_terminate_in_descrs(struct artpec6_crypto_req_common *common) 976 { 977 struct artpec6_crypto_dma_descriptors *dma = common->dma; 978 struct pdma_descr *d; 979 980 if (!dma->in_cnt || dma->in_cnt > PDMA_DESCR_COUNT) { 981 pr_err("%s: IN descriptor list is %s\n", 982 MODULE_NAME, dma->in_cnt ? "empty" : "full"); 983 return -EINVAL; 984 } 985 986 d = &dma->in[dma->in_cnt-1]; 987 d->ctrl.intr = 1; 988 return 0; 989 } 990 991 /** create_hash_pad - Create a Secure Hash conformant pad 992 * 993 * @dst: The destination buffer to write the pad. Must be at least 64 bytes 994 * @dgstlen: The total length of the hash digest in bytes 995 * @bitcount: The total length of the digest in bits 996 * 997 * @return The total number of padding bytes written to @dst 998 */ 999 static size_t 1000 create_hash_pad(int oper, unsigned char *dst, u64 dgstlen, u64 bitcount) 1001 { 1002 unsigned int mod, target, diff, pad_bytes, size_bytes; 1003 __be64 bits = __cpu_to_be64(bitcount); 1004 1005 switch (oper) { 1006 case regk_crypto_sha1: 1007 case regk_crypto_sha256: 1008 case regk_crypto_hmac_sha1: 1009 case regk_crypto_hmac_sha256: 1010 target = 448 / 8; 1011 mod = 512 / 8; 1012 size_bytes = 8; 1013 break; 1014 default: 1015 target = 896 / 8; 1016 mod = 1024 / 8; 1017 size_bytes = 16; 1018 break; 1019 } 1020 1021 target -= 1; 1022 diff = dgstlen & (mod - 1); 1023 pad_bytes = diff > target ? target + mod - diff : target - diff; 1024 1025 memset(dst + 1, 0, pad_bytes); 1026 dst[0] = 0x80; 1027 1028 if (size_bytes == 16) { 1029 memset(dst + 1 + pad_bytes, 0, 8); 1030 memcpy(dst + 1 + pad_bytes + 8, &bits, 8); 1031 } else { 1032 memcpy(dst + 1 + pad_bytes, &bits, 8); 1033 } 1034 1035 return pad_bytes + size_bytes + 1; 1036 } 1037 1038 static int artpec6_crypto_common_init(struct artpec6_crypto_req_common *common, 1039 struct crypto_async_request *parent, 1040 void (*complete)(struct crypto_async_request *req), 1041 struct scatterlist *dstsg, unsigned int nbytes) 1042 { 1043 gfp_t flags; 1044 struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev); 1045 1046 flags = (parent->flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 1047 GFP_KERNEL : GFP_ATOMIC; 1048 1049 common->gfp_flags = flags; 1050 common->dma = kmem_cache_alloc(ac->dma_cache, flags); 1051 if (!common->dma) 1052 return -ENOMEM; 1053 1054 common->req = parent; 1055 common->complete = complete; 1056 return 0; 1057 } 1058 1059 static void 1060 artpec6_crypto_bounce_destroy(struct artpec6_crypto_dma_descriptors *dma) 1061 { 1062 struct artpec6_crypto_bounce_buffer *b; 1063 struct artpec6_crypto_bounce_buffer *next; 1064 1065 list_for_each_entry_safe(b, next, &dma->bounce_buffers, list) { 1066 kfree(b); 1067 } 1068 } 1069 1070 static int 1071 artpec6_crypto_common_destroy(struct artpec6_crypto_req_common *common) 1072 { 1073 struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev); 1074 1075 artpec6_crypto_dma_unmap_all(common); 1076 artpec6_crypto_bounce_destroy(common->dma); 1077 kmem_cache_free(ac->dma_cache, common->dma); 1078 common->dma = NULL; 1079 return 0; 1080 } 1081 1082 /* 1083 * Ciphering functions. 1084 */ 1085 static int artpec6_crypto_encrypt(struct skcipher_request *req) 1086 { 1087 struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req); 1088 struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(cipher); 1089 struct artpec6_crypto_request_context *req_ctx = NULL; 1090 void (*complete)(struct crypto_async_request *req); 1091 int ret; 1092 1093 req_ctx = skcipher_request_ctx(req); 1094 1095 switch (ctx->crypto_type) { 1096 case ARTPEC6_CRYPTO_CIPHER_AES_CBC: 1097 case ARTPEC6_CRYPTO_CIPHER_AES_ECB: 1098 case ARTPEC6_CRYPTO_CIPHER_AES_XTS: 1099 req_ctx->decrypt = 0; 1100 break; 1101 default: 1102 break; 1103 } 1104 1105 switch (ctx->crypto_type) { 1106 case ARTPEC6_CRYPTO_CIPHER_AES_CBC: 1107 complete = artpec6_crypto_complete_cbc_encrypt; 1108 break; 1109 default: 1110 complete = artpec6_crypto_complete_crypto; 1111 break; 1112 } 1113 1114 ret = artpec6_crypto_common_init(&req_ctx->common, 1115 &req->base, 1116 complete, 1117 req->dst, req->cryptlen); 1118 if (ret) 1119 return ret; 1120 1121 ret = artpec6_crypto_prepare_crypto(req); 1122 if (ret) { 1123 artpec6_crypto_common_destroy(&req_ctx->common); 1124 return ret; 1125 } 1126 1127 return artpec6_crypto_submit(&req_ctx->common); 1128 } 1129 1130 static int artpec6_crypto_decrypt(struct skcipher_request *req) 1131 { 1132 int ret; 1133 struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req); 1134 struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(cipher); 1135 struct artpec6_crypto_request_context *req_ctx = NULL; 1136 void (*complete)(struct crypto_async_request *req); 1137 1138 req_ctx = skcipher_request_ctx(req); 1139 1140 switch (ctx->crypto_type) { 1141 case ARTPEC6_CRYPTO_CIPHER_AES_CBC: 1142 case ARTPEC6_CRYPTO_CIPHER_AES_ECB: 1143 case ARTPEC6_CRYPTO_CIPHER_AES_XTS: 1144 req_ctx->decrypt = 1; 1145 break; 1146 default: 1147 break; 1148 } 1149 1150 1151 switch (ctx->crypto_type) { 1152 case ARTPEC6_CRYPTO_CIPHER_AES_CBC: 1153 complete = artpec6_crypto_complete_cbc_decrypt; 1154 break; 1155 default: 1156 complete = artpec6_crypto_complete_crypto; 1157 break; 1158 } 1159 1160 ret = artpec6_crypto_common_init(&req_ctx->common, &req->base, 1161 complete, 1162 req->dst, req->cryptlen); 1163 if (ret) 1164 return ret; 1165 1166 ret = artpec6_crypto_prepare_crypto(req); 1167 if (ret) { 1168 artpec6_crypto_common_destroy(&req_ctx->common); 1169 return ret; 1170 } 1171 1172 return artpec6_crypto_submit(&req_ctx->common); 1173 } 1174 1175 static int 1176 artpec6_crypto_ctr_crypt(struct skcipher_request *req, bool encrypt) 1177 { 1178 struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req); 1179 struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(cipher); 1180 size_t iv_len = crypto_skcipher_ivsize(cipher); 1181 unsigned int counter = be32_to_cpup((__be32 *) 1182 (req->iv + iv_len - 4)); 1183 unsigned int nblks = ALIGN(req->cryptlen, AES_BLOCK_SIZE) / 1184 AES_BLOCK_SIZE; 1185 1186 /* 1187 * The hardware uses only the last 32-bits as the counter while the 1188 * kernel tests (aes_ctr_enc_tv_template[4] for example) expect that 1189 * the whole IV is a counter. So fallback if the counter is going to 1190 * overlow. 1191 */ 1192 if (counter + nblks < counter) { 1193 int ret; 1194 1195 pr_debug("counter %x will overflow (nblks %u), falling back\n", 1196 counter, counter + nblks); 1197 1198 ret = crypto_sync_skcipher_setkey(ctx->fallback, ctx->aes_key, 1199 ctx->key_length); 1200 if (ret) 1201 return ret; 1202 1203 { 1204 SYNC_SKCIPHER_REQUEST_ON_STACK(subreq, ctx->fallback); 1205 1206 skcipher_request_set_sync_tfm(subreq, ctx->fallback); 1207 skcipher_request_set_callback(subreq, req->base.flags, 1208 NULL, NULL); 1209 skcipher_request_set_crypt(subreq, req->src, req->dst, 1210 req->cryptlen, req->iv); 1211 ret = encrypt ? crypto_skcipher_encrypt(subreq) 1212 : crypto_skcipher_decrypt(subreq); 1213 skcipher_request_zero(subreq); 1214 } 1215 return ret; 1216 } 1217 1218 return encrypt ? artpec6_crypto_encrypt(req) 1219 : artpec6_crypto_decrypt(req); 1220 } 1221 1222 static int artpec6_crypto_ctr_encrypt(struct skcipher_request *req) 1223 { 1224 return artpec6_crypto_ctr_crypt(req, true); 1225 } 1226 1227 static int artpec6_crypto_ctr_decrypt(struct skcipher_request *req) 1228 { 1229 return artpec6_crypto_ctr_crypt(req, false); 1230 } 1231 1232 /* 1233 * AEAD functions 1234 */ 1235 static int artpec6_crypto_aead_init(struct crypto_aead *tfm) 1236 { 1237 struct artpec6_cryptotfm_context *tfm_ctx = crypto_aead_ctx(tfm); 1238 1239 memset(tfm_ctx, 0, sizeof(*tfm_ctx)); 1240 1241 crypto_aead_set_reqsize(tfm, 1242 sizeof(struct artpec6_crypto_aead_req_ctx)); 1243 1244 return 0; 1245 } 1246 1247 static int artpec6_crypto_aead_set_key(struct crypto_aead *tfm, const u8 *key, 1248 unsigned int len) 1249 { 1250 struct artpec6_cryptotfm_context *ctx = crypto_tfm_ctx(&tfm->base); 1251 1252 if (len != 16 && len != 24 && len != 32) { 1253 crypto_aead_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); 1254 return -1; 1255 } 1256 1257 ctx->key_length = len; 1258 1259 memcpy(ctx->aes_key, key, len); 1260 return 0; 1261 } 1262 1263 static int artpec6_crypto_aead_encrypt(struct aead_request *req) 1264 { 1265 int ret; 1266 struct artpec6_crypto_aead_req_ctx *req_ctx = aead_request_ctx(req); 1267 1268 req_ctx->decrypt = false; 1269 ret = artpec6_crypto_common_init(&req_ctx->common, &req->base, 1270 artpec6_crypto_complete_aead, 1271 NULL, 0); 1272 if (ret) 1273 return ret; 1274 1275 ret = artpec6_crypto_prepare_aead(req); 1276 if (ret) { 1277 artpec6_crypto_common_destroy(&req_ctx->common); 1278 return ret; 1279 } 1280 1281 return artpec6_crypto_submit(&req_ctx->common); 1282 } 1283 1284 static int artpec6_crypto_aead_decrypt(struct aead_request *req) 1285 { 1286 int ret; 1287 struct artpec6_crypto_aead_req_ctx *req_ctx = aead_request_ctx(req); 1288 1289 req_ctx->decrypt = true; 1290 if (req->cryptlen < AES_BLOCK_SIZE) 1291 return -EINVAL; 1292 1293 ret = artpec6_crypto_common_init(&req_ctx->common, 1294 &req->base, 1295 artpec6_crypto_complete_aead, 1296 NULL, 0); 1297 if (ret) 1298 return ret; 1299 1300 ret = artpec6_crypto_prepare_aead(req); 1301 if (ret) { 1302 artpec6_crypto_common_destroy(&req_ctx->common); 1303 return ret; 1304 } 1305 1306 return artpec6_crypto_submit(&req_ctx->common); 1307 } 1308 1309 static int artpec6_crypto_prepare_hash(struct ahash_request *areq) 1310 { 1311 struct artpec6_hashalg_context *ctx = crypto_tfm_ctx(areq->base.tfm); 1312 struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(areq); 1313 size_t digestsize = crypto_ahash_digestsize(crypto_ahash_reqtfm(areq)); 1314 size_t contextsize = digestsize; 1315 size_t blocksize = crypto_tfm_alg_blocksize( 1316 crypto_ahash_tfm(crypto_ahash_reqtfm(areq))); 1317 struct artpec6_crypto_req_common *common = &req_ctx->common; 1318 struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev); 1319 enum artpec6_crypto_variant variant = ac->variant; 1320 u32 sel_ctx; 1321 bool ext_ctx = false; 1322 bool run_hw = false; 1323 int error = 0; 1324 1325 artpec6_crypto_init_dma_operation(common); 1326 1327 /* Upload HMAC key, must be first the first packet */ 1328 if (req_ctx->hash_flags & HASH_FLAG_HMAC) { 1329 if (variant == ARTPEC6_CRYPTO) { 1330 req_ctx->key_md = FIELD_PREP(A6_CRY_MD_OPER, 1331 a6_regk_crypto_dlkey); 1332 } else { 1333 req_ctx->key_md = FIELD_PREP(A7_CRY_MD_OPER, 1334 a7_regk_crypto_dlkey); 1335 } 1336 1337 /* Copy and pad up the key */ 1338 memcpy(req_ctx->key_buffer, ctx->hmac_key, 1339 ctx->hmac_key_length); 1340 memset(req_ctx->key_buffer + ctx->hmac_key_length, 0, 1341 blocksize - ctx->hmac_key_length); 1342 1343 error = artpec6_crypto_setup_out_descr(common, 1344 (void *)&req_ctx->key_md, 1345 sizeof(req_ctx->key_md), false, false); 1346 if (error) 1347 return error; 1348 1349 error = artpec6_crypto_setup_out_descr(common, 1350 req_ctx->key_buffer, blocksize, 1351 true, false); 1352 if (error) 1353 return error; 1354 } 1355 1356 if (!(req_ctx->hash_flags & HASH_FLAG_INIT_CTX)) { 1357 /* Restore context */ 1358 sel_ctx = regk_crypto_ext; 1359 ext_ctx = true; 1360 } else { 1361 sel_ctx = regk_crypto_init; 1362 } 1363 1364 if (variant == ARTPEC6_CRYPTO) { 1365 req_ctx->hash_md &= ~A6_CRY_MD_HASH_SEL_CTX; 1366 req_ctx->hash_md |= FIELD_PREP(A6_CRY_MD_HASH_SEL_CTX, sel_ctx); 1367 1368 /* If this is the final round, set the final flag */ 1369 if (req_ctx->hash_flags & HASH_FLAG_FINALIZE) 1370 req_ctx->hash_md |= A6_CRY_MD_HASH_HMAC_FIN; 1371 } else { 1372 req_ctx->hash_md &= ~A7_CRY_MD_HASH_SEL_CTX; 1373 req_ctx->hash_md |= FIELD_PREP(A7_CRY_MD_HASH_SEL_CTX, sel_ctx); 1374 1375 /* If this is the final round, set the final flag */ 1376 if (req_ctx->hash_flags & HASH_FLAG_FINALIZE) 1377 req_ctx->hash_md |= A7_CRY_MD_HASH_HMAC_FIN; 1378 } 1379 1380 /* Setup up metadata descriptors */ 1381 error = artpec6_crypto_setup_out_descr(common, 1382 (void *)&req_ctx->hash_md, 1383 sizeof(req_ctx->hash_md), false, false); 1384 if (error) 1385 return error; 1386 1387 error = artpec6_crypto_setup_in_descr(common, ac->pad_buffer, 4, false); 1388 if (error) 1389 return error; 1390 1391 if (ext_ctx) { 1392 error = artpec6_crypto_setup_out_descr(common, 1393 req_ctx->digeststate, 1394 contextsize, false, false); 1395 1396 if (error) 1397 return error; 1398 } 1399 1400 if (req_ctx->hash_flags & HASH_FLAG_UPDATE) { 1401 size_t done_bytes = 0; 1402 size_t total_bytes = areq->nbytes + req_ctx->partial_bytes; 1403 size_t ready_bytes = round_down(total_bytes, blocksize); 1404 struct artpec6_crypto_walk walk; 1405 1406 run_hw = ready_bytes > 0; 1407 if (req_ctx->partial_bytes && ready_bytes) { 1408 /* We have a partial buffer and will at least some bytes 1409 * to the HW. Empty this partial buffer before tackling 1410 * the SG lists 1411 */ 1412 memcpy(req_ctx->partial_buffer_out, 1413 req_ctx->partial_buffer, 1414 req_ctx->partial_bytes); 1415 1416 error = artpec6_crypto_setup_out_descr(common, 1417 req_ctx->partial_buffer_out, 1418 req_ctx->partial_bytes, 1419 false, true); 1420 if (error) 1421 return error; 1422 1423 /* Reset partial buffer */ 1424 done_bytes += req_ctx->partial_bytes; 1425 req_ctx->partial_bytes = 0; 1426 } 1427 1428 artpec6_crypto_walk_init(&walk, areq->src); 1429 1430 error = artpec6_crypto_setup_sg_descrs_out(common, &walk, 1431 ready_bytes - 1432 done_bytes); 1433 if (error) 1434 return error; 1435 1436 if (walk.sg) { 1437 size_t sg_skip = ready_bytes - done_bytes; 1438 size_t sg_rem = areq->nbytes - sg_skip; 1439 1440 sg_pcopy_to_buffer(areq->src, sg_nents(areq->src), 1441 req_ctx->partial_buffer + 1442 req_ctx->partial_bytes, 1443 sg_rem, sg_skip); 1444 1445 req_ctx->partial_bytes += sg_rem; 1446 } 1447 1448 req_ctx->digcnt += ready_bytes; 1449 req_ctx->hash_flags &= ~(HASH_FLAG_UPDATE); 1450 } 1451 1452 /* Finalize */ 1453 if (req_ctx->hash_flags & HASH_FLAG_FINALIZE) { 1454 size_t hash_pad_len; 1455 u64 digest_bits; 1456 u32 oper; 1457 1458 if (variant == ARTPEC6_CRYPTO) 1459 oper = FIELD_GET(A6_CRY_MD_OPER, req_ctx->hash_md); 1460 else 1461 oper = FIELD_GET(A7_CRY_MD_OPER, req_ctx->hash_md); 1462 1463 /* Write out the partial buffer if present */ 1464 if (req_ctx->partial_bytes) { 1465 memcpy(req_ctx->partial_buffer_out, 1466 req_ctx->partial_buffer, 1467 req_ctx->partial_bytes); 1468 error = artpec6_crypto_setup_out_descr(common, 1469 req_ctx->partial_buffer_out, 1470 req_ctx->partial_bytes, 1471 false, true); 1472 if (error) 1473 return error; 1474 1475 req_ctx->digcnt += req_ctx->partial_bytes; 1476 req_ctx->partial_bytes = 0; 1477 } 1478 1479 if (req_ctx->hash_flags & HASH_FLAG_HMAC) 1480 digest_bits = 8 * (req_ctx->digcnt + blocksize); 1481 else 1482 digest_bits = 8 * req_ctx->digcnt; 1483 1484 /* Add the hash pad */ 1485 hash_pad_len = create_hash_pad(oper, req_ctx->pad_buffer, 1486 req_ctx->digcnt, digest_bits); 1487 error = artpec6_crypto_setup_out_descr(common, 1488 req_ctx->pad_buffer, 1489 hash_pad_len, false, 1490 true); 1491 req_ctx->digcnt = 0; 1492 1493 if (error) 1494 return error; 1495 1496 /* Descriptor for the final result */ 1497 error = artpec6_crypto_setup_in_descr(common, areq->result, 1498 digestsize, 1499 true); 1500 if (error) 1501 return error; 1502 1503 } else { /* This is not the final operation for this request */ 1504 if (!run_hw) 1505 return ARTPEC6_CRYPTO_PREPARE_HASH_NO_START; 1506 1507 /* Save the result to the context */ 1508 error = artpec6_crypto_setup_in_descr(common, 1509 req_ctx->digeststate, 1510 contextsize, false); 1511 if (error) 1512 return error; 1513 /* fall through */ 1514 } 1515 1516 req_ctx->hash_flags &= ~(HASH_FLAG_INIT_CTX | HASH_FLAG_UPDATE | 1517 HASH_FLAG_FINALIZE); 1518 1519 error = artpec6_crypto_terminate_in_descrs(common); 1520 if (error) 1521 return error; 1522 1523 error = artpec6_crypto_terminate_out_descrs(common); 1524 if (error) 1525 return error; 1526 1527 error = artpec6_crypto_dma_map_descs(common); 1528 if (error) 1529 return error; 1530 1531 return ARTPEC6_CRYPTO_PREPARE_HASH_START; 1532 } 1533 1534 1535 static int artpec6_crypto_aes_ecb_init(struct crypto_skcipher *tfm) 1536 { 1537 struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(tfm); 1538 1539 tfm->reqsize = sizeof(struct artpec6_crypto_request_context); 1540 ctx->crypto_type = ARTPEC6_CRYPTO_CIPHER_AES_ECB; 1541 1542 return 0; 1543 } 1544 1545 static int artpec6_crypto_aes_ctr_init(struct crypto_skcipher *tfm) 1546 { 1547 struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(tfm); 1548 1549 ctx->fallback = 1550 crypto_alloc_sync_skcipher(crypto_tfm_alg_name(&tfm->base), 1551 0, CRYPTO_ALG_NEED_FALLBACK); 1552 if (IS_ERR(ctx->fallback)) 1553 return PTR_ERR(ctx->fallback); 1554 1555 tfm->reqsize = sizeof(struct artpec6_crypto_request_context); 1556 ctx->crypto_type = ARTPEC6_CRYPTO_CIPHER_AES_CTR; 1557 1558 return 0; 1559 } 1560 1561 static int artpec6_crypto_aes_cbc_init(struct crypto_skcipher *tfm) 1562 { 1563 struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(tfm); 1564 1565 tfm->reqsize = sizeof(struct artpec6_crypto_request_context); 1566 ctx->crypto_type = ARTPEC6_CRYPTO_CIPHER_AES_CBC; 1567 1568 return 0; 1569 } 1570 1571 static int artpec6_crypto_aes_xts_init(struct crypto_skcipher *tfm) 1572 { 1573 struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(tfm); 1574 1575 tfm->reqsize = sizeof(struct artpec6_crypto_request_context); 1576 ctx->crypto_type = ARTPEC6_CRYPTO_CIPHER_AES_XTS; 1577 1578 return 0; 1579 } 1580 1581 static void artpec6_crypto_aes_exit(struct crypto_skcipher *tfm) 1582 { 1583 struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(tfm); 1584 1585 memset(ctx, 0, sizeof(*ctx)); 1586 } 1587 1588 static void artpec6_crypto_aes_ctr_exit(struct crypto_skcipher *tfm) 1589 { 1590 struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(tfm); 1591 1592 crypto_free_sync_skcipher(ctx->fallback); 1593 artpec6_crypto_aes_exit(tfm); 1594 } 1595 1596 static int 1597 artpec6_crypto_cipher_set_key(struct crypto_skcipher *cipher, const u8 *key, 1598 unsigned int keylen) 1599 { 1600 struct artpec6_cryptotfm_context *ctx = 1601 crypto_skcipher_ctx(cipher); 1602 1603 switch (keylen) { 1604 case 16: 1605 case 24: 1606 case 32: 1607 break; 1608 default: 1609 crypto_skcipher_set_flags(cipher, 1610 CRYPTO_TFM_RES_BAD_KEY_LEN); 1611 return -EINVAL; 1612 } 1613 1614 memcpy(ctx->aes_key, key, keylen); 1615 ctx->key_length = keylen; 1616 return 0; 1617 } 1618 1619 static int 1620 artpec6_crypto_xts_set_key(struct crypto_skcipher *cipher, const u8 *key, 1621 unsigned int keylen) 1622 { 1623 struct artpec6_cryptotfm_context *ctx = 1624 crypto_skcipher_ctx(cipher); 1625 int ret; 1626 1627 ret = xts_check_key(&cipher->base, key, keylen); 1628 if (ret) 1629 return ret; 1630 1631 switch (keylen) { 1632 case 32: 1633 case 48: 1634 case 64: 1635 break; 1636 default: 1637 crypto_skcipher_set_flags(cipher, 1638 CRYPTO_TFM_RES_BAD_KEY_LEN); 1639 return -EINVAL; 1640 } 1641 1642 memcpy(ctx->aes_key, key, keylen); 1643 ctx->key_length = keylen; 1644 return 0; 1645 } 1646 1647 /** artpec6_crypto_process_crypto - Prepare an async block cipher crypto request 1648 * 1649 * @req: The asynch request to process 1650 * 1651 * @return 0 if the dma job was successfully prepared 1652 * <0 on error 1653 * 1654 * This function sets up the PDMA descriptors for a block cipher request. 1655 * 1656 * The required padding is added for AES-CTR using a statically defined 1657 * buffer. 1658 * 1659 * The PDMA descriptor list will be as follows: 1660 * 1661 * OUT: [KEY_MD][KEY][EOP]<CIPHER_MD>[IV]<data_0>...[data_n][AES-CTR_pad]<eop> 1662 * IN: <CIPHER_MD><data_0>...[data_n]<intr> 1663 * 1664 */ 1665 static int artpec6_crypto_prepare_crypto(struct skcipher_request *areq) 1666 { 1667 int ret; 1668 struct artpec6_crypto_walk walk; 1669 struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(areq); 1670 struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(cipher); 1671 struct artpec6_crypto_request_context *req_ctx = NULL; 1672 size_t iv_len = crypto_skcipher_ivsize(cipher); 1673 struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev); 1674 enum artpec6_crypto_variant variant = ac->variant; 1675 struct artpec6_crypto_req_common *common; 1676 bool cipher_decr = false; 1677 size_t cipher_klen; 1678 u32 cipher_len = 0; /* Same as regk_crypto_key_128 for NULL crypto */ 1679 u32 oper; 1680 1681 req_ctx = skcipher_request_ctx(areq); 1682 common = &req_ctx->common; 1683 1684 artpec6_crypto_init_dma_operation(common); 1685 1686 if (variant == ARTPEC6_CRYPTO) 1687 ctx->key_md = FIELD_PREP(A6_CRY_MD_OPER, a6_regk_crypto_dlkey); 1688 else 1689 ctx->key_md = FIELD_PREP(A7_CRY_MD_OPER, a7_regk_crypto_dlkey); 1690 1691 ret = artpec6_crypto_setup_out_descr(common, (void *)&ctx->key_md, 1692 sizeof(ctx->key_md), false, false); 1693 if (ret) 1694 return ret; 1695 1696 ret = artpec6_crypto_setup_out_descr(common, ctx->aes_key, 1697 ctx->key_length, true, false); 1698 if (ret) 1699 return ret; 1700 1701 req_ctx->cipher_md = 0; 1702 1703 if (ctx->crypto_type == ARTPEC6_CRYPTO_CIPHER_AES_XTS) 1704 cipher_klen = ctx->key_length/2; 1705 else 1706 cipher_klen = ctx->key_length; 1707 1708 /* Metadata */ 1709 switch (cipher_klen) { 1710 case 16: 1711 cipher_len = regk_crypto_key_128; 1712 break; 1713 case 24: 1714 cipher_len = regk_crypto_key_192; 1715 break; 1716 case 32: 1717 cipher_len = regk_crypto_key_256; 1718 break; 1719 default: 1720 pr_err("%s: Invalid key length %d!\n", 1721 MODULE_NAME, ctx->key_length); 1722 return -EINVAL; 1723 } 1724 1725 switch (ctx->crypto_type) { 1726 case ARTPEC6_CRYPTO_CIPHER_AES_ECB: 1727 oper = regk_crypto_aes_ecb; 1728 cipher_decr = req_ctx->decrypt; 1729 break; 1730 1731 case ARTPEC6_CRYPTO_CIPHER_AES_CBC: 1732 oper = regk_crypto_aes_cbc; 1733 cipher_decr = req_ctx->decrypt; 1734 break; 1735 1736 case ARTPEC6_CRYPTO_CIPHER_AES_CTR: 1737 oper = regk_crypto_aes_ctr; 1738 cipher_decr = false; 1739 break; 1740 1741 case ARTPEC6_CRYPTO_CIPHER_AES_XTS: 1742 oper = regk_crypto_aes_xts; 1743 cipher_decr = req_ctx->decrypt; 1744 1745 if (variant == ARTPEC6_CRYPTO) 1746 req_ctx->cipher_md |= A6_CRY_MD_CIPHER_DSEQ; 1747 else 1748 req_ctx->cipher_md |= A7_CRY_MD_CIPHER_DSEQ; 1749 break; 1750 1751 default: 1752 pr_err("%s: Invalid cipher mode %d!\n", 1753 MODULE_NAME, ctx->crypto_type); 1754 return -EINVAL; 1755 } 1756 1757 if (variant == ARTPEC6_CRYPTO) { 1758 req_ctx->cipher_md |= FIELD_PREP(A6_CRY_MD_OPER, oper); 1759 req_ctx->cipher_md |= FIELD_PREP(A6_CRY_MD_CIPHER_LEN, 1760 cipher_len); 1761 if (cipher_decr) 1762 req_ctx->cipher_md |= A6_CRY_MD_CIPHER_DECR; 1763 } else { 1764 req_ctx->cipher_md |= FIELD_PREP(A7_CRY_MD_OPER, oper); 1765 req_ctx->cipher_md |= FIELD_PREP(A7_CRY_MD_CIPHER_LEN, 1766 cipher_len); 1767 if (cipher_decr) 1768 req_ctx->cipher_md |= A7_CRY_MD_CIPHER_DECR; 1769 } 1770 1771 ret = artpec6_crypto_setup_out_descr(common, 1772 &req_ctx->cipher_md, 1773 sizeof(req_ctx->cipher_md), 1774 false, false); 1775 if (ret) 1776 return ret; 1777 1778 ret = artpec6_crypto_setup_in_descr(common, ac->pad_buffer, 4, false); 1779 if (ret) 1780 return ret; 1781 1782 if (iv_len) { 1783 ret = artpec6_crypto_setup_out_descr(common, areq->iv, iv_len, 1784 false, false); 1785 if (ret) 1786 return ret; 1787 } 1788 /* Data out */ 1789 artpec6_crypto_walk_init(&walk, areq->src); 1790 ret = artpec6_crypto_setup_sg_descrs_out(common, &walk, areq->cryptlen); 1791 if (ret) 1792 return ret; 1793 1794 /* Data in */ 1795 artpec6_crypto_walk_init(&walk, areq->dst); 1796 ret = artpec6_crypto_setup_sg_descrs_in(common, &walk, areq->cryptlen); 1797 if (ret) 1798 return ret; 1799 1800 /* CTR-mode padding required by the HW. */ 1801 if (ctx->crypto_type == ARTPEC6_CRYPTO_CIPHER_AES_CTR || 1802 ctx->crypto_type == ARTPEC6_CRYPTO_CIPHER_AES_XTS) { 1803 size_t pad = ALIGN(areq->cryptlen, AES_BLOCK_SIZE) - 1804 areq->cryptlen; 1805 1806 if (pad) { 1807 ret = artpec6_crypto_setup_out_descr(common, 1808 ac->pad_buffer, 1809 pad, false, false); 1810 if (ret) 1811 return ret; 1812 1813 ret = artpec6_crypto_setup_in_descr(common, 1814 ac->pad_buffer, pad, 1815 false); 1816 if (ret) 1817 return ret; 1818 } 1819 } 1820 1821 ret = artpec6_crypto_terminate_out_descrs(common); 1822 if (ret) 1823 return ret; 1824 1825 ret = artpec6_crypto_terminate_in_descrs(common); 1826 if (ret) 1827 return ret; 1828 1829 return artpec6_crypto_dma_map_descs(common); 1830 } 1831 1832 static int artpec6_crypto_prepare_aead(struct aead_request *areq) 1833 { 1834 size_t count; 1835 int ret; 1836 size_t input_length; 1837 struct artpec6_cryptotfm_context *ctx = crypto_tfm_ctx(areq->base.tfm); 1838 struct artpec6_crypto_aead_req_ctx *req_ctx = aead_request_ctx(areq); 1839 struct crypto_aead *cipher = crypto_aead_reqtfm(areq); 1840 struct artpec6_crypto_req_common *common = &req_ctx->common; 1841 struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev); 1842 enum artpec6_crypto_variant variant = ac->variant; 1843 u32 md_cipher_len; 1844 1845 artpec6_crypto_init_dma_operation(common); 1846 1847 /* Key */ 1848 if (variant == ARTPEC6_CRYPTO) { 1849 ctx->key_md = FIELD_PREP(A6_CRY_MD_OPER, 1850 a6_regk_crypto_dlkey); 1851 } else { 1852 ctx->key_md = FIELD_PREP(A7_CRY_MD_OPER, 1853 a7_regk_crypto_dlkey); 1854 } 1855 ret = artpec6_crypto_setup_out_descr(common, (void *)&ctx->key_md, 1856 sizeof(ctx->key_md), false, false); 1857 if (ret) 1858 return ret; 1859 1860 ret = artpec6_crypto_setup_out_descr(common, ctx->aes_key, 1861 ctx->key_length, true, false); 1862 if (ret) 1863 return ret; 1864 1865 req_ctx->cipher_md = 0; 1866 1867 switch (ctx->key_length) { 1868 case 16: 1869 md_cipher_len = regk_crypto_key_128; 1870 break; 1871 case 24: 1872 md_cipher_len = regk_crypto_key_192; 1873 break; 1874 case 32: 1875 md_cipher_len = regk_crypto_key_256; 1876 break; 1877 default: 1878 return -EINVAL; 1879 } 1880 1881 if (variant == ARTPEC6_CRYPTO) { 1882 req_ctx->cipher_md |= FIELD_PREP(A6_CRY_MD_OPER, 1883 regk_crypto_aes_gcm); 1884 req_ctx->cipher_md |= FIELD_PREP(A6_CRY_MD_CIPHER_LEN, 1885 md_cipher_len); 1886 if (req_ctx->decrypt) 1887 req_ctx->cipher_md |= A6_CRY_MD_CIPHER_DECR; 1888 } else { 1889 req_ctx->cipher_md |= FIELD_PREP(A7_CRY_MD_OPER, 1890 regk_crypto_aes_gcm); 1891 req_ctx->cipher_md |= FIELD_PREP(A7_CRY_MD_CIPHER_LEN, 1892 md_cipher_len); 1893 if (req_ctx->decrypt) 1894 req_ctx->cipher_md |= A7_CRY_MD_CIPHER_DECR; 1895 } 1896 1897 ret = artpec6_crypto_setup_out_descr(common, 1898 (void *) &req_ctx->cipher_md, 1899 sizeof(req_ctx->cipher_md), false, 1900 false); 1901 if (ret) 1902 return ret; 1903 1904 ret = artpec6_crypto_setup_in_descr(common, ac->pad_buffer, 4, false); 1905 if (ret) 1906 return ret; 1907 1908 /* For the decryption, cryptlen includes the tag. */ 1909 input_length = areq->cryptlen; 1910 if (req_ctx->decrypt) 1911 input_length -= crypto_aead_authsize(cipher); 1912 1913 /* Prepare the context buffer */ 1914 req_ctx->hw_ctx.aad_length_bits = 1915 __cpu_to_be64(8*areq->assoclen); 1916 1917 req_ctx->hw_ctx.text_length_bits = 1918 __cpu_to_be64(8*input_length); 1919 1920 memcpy(req_ctx->hw_ctx.J0, areq->iv, crypto_aead_ivsize(cipher)); 1921 // The HW omits the initial increment of the counter field. 1922 memcpy(req_ctx->hw_ctx.J0 + GCM_AES_IV_SIZE, "\x00\x00\x00\x01", 4); 1923 1924 ret = artpec6_crypto_setup_out_descr(common, &req_ctx->hw_ctx, 1925 sizeof(struct artpec6_crypto_aead_hw_ctx), false, false); 1926 if (ret) 1927 return ret; 1928 1929 { 1930 struct artpec6_crypto_walk walk; 1931 1932 artpec6_crypto_walk_init(&walk, areq->src); 1933 1934 /* Associated data */ 1935 count = areq->assoclen; 1936 ret = artpec6_crypto_setup_sg_descrs_out(common, &walk, count); 1937 if (ret) 1938 return ret; 1939 1940 if (!IS_ALIGNED(areq->assoclen, 16)) { 1941 size_t assoc_pad = 16 - (areq->assoclen % 16); 1942 /* The HW mandates zero padding here */ 1943 ret = artpec6_crypto_setup_out_descr(common, 1944 ac->zero_buffer, 1945 assoc_pad, false, 1946 false); 1947 if (ret) 1948 return ret; 1949 } 1950 1951 /* Data to crypto */ 1952 count = input_length; 1953 ret = artpec6_crypto_setup_sg_descrs_out(common, &walk, count); 1954 if (ret) 1955 return ret; 1956 1957 if (!IS_ALIGNED(input_length, 16)) { 1958 size_t crypto_pad = 16 - (input_length % 16); 1959 /* The HW mandates zero padding here */ 1960 ret = artpec6_crypto_setup_out_descr(common, 1961 ac->zero_buffer, 1962 crypto_pad, 1963 false, 1964 false); 1965 if (ret) 1966 return ret; 1967 } 1968 } 1969 1970 /* Data from crypto */ 1971 { 1972 struct artpec6_crypto_walk walk; 1973 size_t output_len = areq->cryptlen; 1974 1975 if (req_ctx->decrypt) 1976 output_len -= crypto_aead_authsize(cipher); 1977 1978 artpec6_crypto_walk_init(&walk, areq->dst); 1979 1980 /* skip associated data in the output */ 1981 count = artpec6_crypto_walk_advance(&walk, areq->assoclen); 1982 if (count) 1983 return -EINVAL; 1984 1985 count = output_len; 1986 ret = artpec6_crypto_setup_sg_descrs_in(common, &walk, count); 1987 if (ret) 1988 return ret; 1989 1990 /* Put padding between the cryptotext and the auth tag */ 1991 if (!IS_ALIGNED(output_len, 16)) { 1992 size_t crypto_pad = 16 - (output_len % 16); 1993 1994 ret = artpec6_crypto_setup_in_descr(common, 1995 ac->pad_buffer, 1996 crypto_pad, false); 1997 if (ret) 1998 return ret; 1999 } 2000 2001 /* The authentication tag shall follow immediately after 2002 * the output ciphertext. For decryption it is put in a context 2003 * buffer for later compare against the input tag. 2004 */ 2005 2006 if (req_ctx->decrypt) { 2007 ret = artpec6_crypto_setup_in_descr(common, 2008 req_ctx->decryption_tag, AES_BLOCK_SIZE, false); 2009 if (ret) 2010 return ret; 2011 2012 } else { 2013 /* For encryption the requested tag size may be smaller 2014 * than the hardware's generated tag. 2015 */ 2016 size_t authsize = crypto_aead_authsize(cipher); 2017 2018 ret = artpec6_crypto_setup_sg_descrs_in(common, &walk, 2019 authsize); 2020 if (ret) 2021 return ret; 2022 2023 if (authsize < AES_BLOCK_SIZE) { 2024 count = AES_BLOCK_SIZE - authsize; 2025 ret = artpec6_crypto_setup_in_descr(common, 2026 ac->pad_buffer, 2027 count, false); 2028 if (ret) 2029 return ret; 2030 } 2031 } 2032 2033 } 2034 2035 ret = artpec6_crypto_terminate_in_descrs(common); 2036 if (ret) 2037 return ret; 2038 2039 ret = artpec6_crypto_terminate_out_descrs(common); 2040 if (ret) 2041 return ret; 2042 2043 return artpec6_crypto_dma_map_descs(common); 2044 } 2045 2046 static void artpec6_crypto_process_queue(struct artpec6_crypto *ac, 2047 struct list_head *completions) 2048 { 2049 struct artpec6_crypto_req_common *req; 2050 2051 while (!list_empty(&ac->queue) && !artpec6_crypto_busy()) { 2052 req = list_first_entry(&ac->queue, 2053 struct artpec6_crypto_req_common, 2054 list); 2055 list_move_tail(&req->list, &ac->pending); 2056 artpec6_crypto_start_dma(req); 2057 2058 list_add_tail(&req->complete_in_progress, completions); 2059 } 2060 2061 /* 2062 * In some cases, the hardware can raise an in_eop_flush interrupt 2063 * before actually updating the status, so we have an timer which will 2064 * recheck the status on timeout. Since the cases are expected to be 2065 * very rare, we use a relatively large timeout value. There should be 2066 * no noticeable negative effect if we timeout spuriously. 2067 */ 2068 if (ac->pending_count) 2069 mod_timer(&ac->timer, jiffies + msecs_to_jiffies(100)); 2070 else 2071 del_timer(&ac->timer); 2072 } 2073 2074 static void artpec6_crypto_timeout(struct timer_list *t) 2075 { 2076 struct artpec6_crypto *ac = from_timer(ac, t, timer); 2077 2078 dev_info_ratelimited(artpec6_crypto_dev, "timeout\n"); 2079 2080 tasklet_schedule(&ac->task); 2081 } 2082 2083 static void artpec6_crypto_task(unsigned long data) 2084 { 2085 struct artpec6_crypto *ac = (struct artpec6_crypto *)data; 2086 struct artpec6_crypto_req_common *req; 2087 struct artpec6_crypto_req_common *n; 2088 struct list_head complete_done; 2089 struct list_head complete_in_progress; 2090 2091 INIT_LIST_HEAD(&complete_done); 2092 INIT_LIST_HEAD(&complete_in_progress); 2093 2094 if (list_empty(&ac->pending)) { 2095 pr_debug("Spurious IRQ\n"); 2096 return; 2097 } 2098 2099 spin_lock_bh(&ac->queue_lock); 2100 2101 list_for_each_entry_safe(req, n, &ac->pending, list) { 2102 struct artpec6_crypto_dma_descriptors *dma = req->dma; 2103 u32 stat; 2104 dma_addr_t stataddr; 2105 2106 stataddr = dma->stat_dma_addr + 4 * (req->dma->in_cnt - 1); 2107 dma_sync_single_for_cpu(artpec6_crypto_dev, 2108 stataddr, 2109 4, 2110 DMA_BIDIRECTIONAL); 2111 2112 stat = req->dma->stat[req->dma->in_cnt-1]; 2113 2114 /* A non-zero final status descriptor indicates 2115 * this job has finished. 2116 */ 2117 pr_debug("Request %p status is %X\n", req, stat); 2118 if (!stat) 2119 break; 2120 2121 /* Allow testing of timeout handling with fault injection */ 2122 #ifdef CONFIG_FAULT_INJECTION 2123 if (should_fail(&artpec6_crypto_fail_status_read, 1)) 2124 continue; 2125 #endif 2126 2127 pr_debug("Completing request %p\n", req); 2128 2129 list_move_tail(&req->list, &complete_done); 2130 2131 ac->pending_count--; 2132 } 2133 2134 artpec6_crypto_process_queue(ac, &complete_in_progress); 2135 2136 spin_unlock_bh(&ac->queue_lock); 2137 2138 /* Perform the completion callbacks without holding the queue lock 2139 * to allow new request submissions from the callbacks. 2140 */ 2141 list_for_each_entry_safe(req, n, &complete_done, list) { 2142 artpec6_crypto_dma_unmap_all(req); 2143 artpec6_crypto_copy_bounce_buffers(req); 2144 artpec6_crypto_common_destroy(req); 2145 2146 req->complete(req->req); 2147 } 2148 2149 list_for_each_entry_safe(req, n, &complete_in_progress, 2150 complete_in_progress) { 2151 req->req->complete(req->req, -EINPROGRESS); 2152 } 2153 } 2154 2155 static void artpec6_crypto_complete_crypto(struct crypto_async_request *req) 2156 { 2157 req->complete(req, 0); 2158 } 2159 2160 static void 2161 artpec6_crypto_complete_cbc_decrypt(struct crypto_async_request *req) 2162 { 2163 struct skcipher_request *cipher_req = container_of(req, 2164 struct skcipher_request, base); 2165 2166 scatterwalk_map_and_copy(cipher_req->iv, cipher_req->src, 2167 cipher_req->cryptlen - AES_BLOCK_SIZE, 2168 AES_BLOCK_SIZE, 0); 2169 req->complete(req, 0); 2170 } 2171 2172 static void 2173 artpec6_crypto_complete_cbc_encrypt(struct crypto_async_request *req) 2174 { 2175 struct skcipher_request *cipher_req = container_of(req, 2176 struct skcipher_request, base); 2177 2178 scatterwalk_map_and_copy(cipher_req->iv, cipher_req->dst, 2179 cipher_req->cryptlen - AES_BLOCK_SIZE, 2180 AES_BLOCK_SIZE, 0); 2181 req->complete(req, 0); 2182 } 2183 2184 static void artpec6_crypto_complete_aead(struct crypto_async_request *req) 2185 { 2186 int result = 0; 2187 2188 /* Verify GCM hashtag. */ 2189 struct aead_request *areq = container_of(req, 2190 struct aead_request, base); 2191 struct crypto_aead *aead = crypto_aead_reqtfm(areq); 2192 struct artpec6_crypto_aead_req_ctx *req_ctx = aead_request_ctx(areq); 2193 2194 if (req_ctx->decrypt) { 2195 u8 input_tag[AES_BLOCK_SIZE]; 2196 unsigned int authsize = crypto_aead_authsize(aead); 2197 2198 sg_pcopy_to_buffer(areq->src, 2199 sg_nents(areq->src), 2200 input_tag, 2201 authsize, 2202 areq->assoclen + areq->cryptlen - 2203 authsize); 2204 2205 if (crypto_memneq(req_ctx->decryption_tag, 2206 input_tag, 2207 authsize)) { 2208 pr_debug("***EBADMSG:\n"); 2209 print_hex_dump_debug("ref:", DUMP_PREFIX_ADDRESS, 32, 1, 2210 input_tag, authsize, true); 2211 print_hex_dump_debug("out:", DUMP_PREFIX_ADDRESS, 32, 1, 2212 req_ctx->decryption_tag, 2213 authsize, true); 2214 2215 result = -EBADMSG; 2216 } 2217 } 2218 2219 req->complete(req, result); 2220 } 2221 2222 static void artpec6_crypto_complete_hash(struct crypto_async_request *req) 2223 { 2224 req->complete(req, 0); 2225 } 2226 2227 2228 /*------------------- Hash functions -----------------------------------------*/ 2229 static int 2230 artpec6_crypto_hash_set_key(struct crypto_ahash *tfm, 2231 const u8 *key, unsigned int keylen) 2232 { 2233 struct artpec6_hashalg_context *tfm_ctx = crypto_tfm_ctx(&tfm->base); 2234 size_t blocksize; 2235 int ret; 2236 2237 if (!keylen) { 2238 pr_err("Invalid length (%d) of HMAC key\n", 2239 keylen); 2240 return -EINVAL; 2241 } 2242 2243 memset(tfm_ctx->hmac_key, 0, sizeof(tfm_ctx->hmac_key)); 2244 2245 blocksize = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm)); 2246 2247 if (keylen > blocksize) { 2248 SHASH_DESC_ON_STACK(hdesc, tfm_ctx->child_hash); 2249 2250 hdesc->tfm = tfm_ctx->child_hash; 2251 2252 tfm_ctx->hmac_key_length = blocksize; 2253 ret = crypto_shash_digest(hdesc, key, keylen, 2254 tfm_ctx->hmac_key); 2255 if (ret) 2256 return ret; 2257 2258 } else { 2259 memcpy(tfm_ctx->hmac_key, key, keylen); 2260 tfm_ctx->hmac_key_length = keylen; 2261 } 2262 2263 return 0; 2264 } 2265 2266 static int 2267 artpec6_crypto_init_hash(struct ahash_request *req, u8 type, int hmac) 2268 { 2269 struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev); 2270 enum artpec6_crypto_variant variant = ac->variant; 2271 struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req); 2272 u32 oper; 2273 2274 memset(req_ctx, 0, sizeof(*req_ctx)); 2275 2276 req_ctx->hash_flags = HASH_FLAG_INIT_CTX; 2277 if (hmac) 2278 req_ctx->hash_flags |= (HASH_FLAG_HMAC | HASH_FLAG_UPDATE_KEY); 2279 2280 switch (type) { 2281 case ARTPEC6_CRYPTO_HASH_SHA1: 2282 oper = hmac ? regk_crypto_hmac_sha1 : regk_crypto_sha1; 2283 break; 2284 case ARTPEC6_CRYPTO_HASH_SHA256: 2285 oper = hmac ? regk_crypto_hmac_sha256 : regk_crypto_sha256; 2286 break; 2287 default: 2288 pr_err("%s: Unsupported hash type 0x%x\n", MODULE_NAME, type); 2289 return -EINVAL; 2290 } 2291 2292 if (variant == ARTPEC6_CRYPTO) 2293 req_ctx->hash_md = FIELD_PREP(A6_CRY_MD_OPER, oper); 2294 else 2295 req_ctx->hash_md = FIELD_PREP(A7_CRY_MD_OPER, oper); 2296 2297 return 0; 2298 } 2299 2300 static int artpec6_crypto_prepare_submit_hash(struct ahash_request *req) 2301 { 2302 struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req); 2303 int ret; 2304 2305 if (!req_ctx->common.dma) { 2306 ret = artpec6_crypto_common_init(&req_ctx->common, 2307 &req->base, 2308 artpec6_crypto_complete_hash, 2309 NULL, 0); 2310 2311 if (ret) 2312 return ret; 2313 } 2314 2315 ret = artpec6_crypto_prepare_hash(req); 2316 switch (ret) { 2317 case ARTPEC6_CRYPTO_PREPARE_HASH_START: 2318 ret = artpec6_crypto_submit(&req_ctx->common); 2319 break; 2320 2321 case ARTPEC6_CRYPTO_PREPARE_HASH_NO_START: 2322 ret = 0; 2323 /* Fallthrough */ 2324 2325 default: 2326 artpec6_crypto_common_destroy(&req_ctx->common); 2327 break; 2328 } 2329 2330 return ret; 2331 } 2332 2333 static int artpec6_crypto_hash_final(struct ahash_request *req) 2334 { 2335 struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req); 2336 2337 req_ctx->hash_flags |= HASH_FLAG_FINALIZE; 2338 2339 return artpec6_crypto_prepare_submit_hash(req); 2340 } 2341 2342 static int artpec6_crypto_hash_update(struct ahash_request *req) 2343 { 2344 struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req); 2345 2346 req_ctx->hash_flags |= HASH_FLAG_UPDATE; 2347 2348 return artpec6_crypto_prepare_submit_hash(req); 2349 } 2350 2351 static int artpec6_crypto_sha1_init(struct ahash_request *req) 2352 { 2353 return artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA1, 0); 2354 } 2355 2356 static int artpec6_crypto_sha1_digest(struct ahash_request *req) 2357 { 2358 struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req); 2359 2360 artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA1, 0); 2361 2362 req_ctx->hash_flags |= HASH_FLAG_UPDATE | HASH_FLAG_FINALIZE; 2363 2364 return artpec6_crypto_prepare_submit_hash(req); 2365 } 2366 2367 static int artpec6_crypto_sha256_init(struct ahash_request *req) 2368 { 2369 return artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA256, 0); 2370 } 2371 2372 static int artpec6_crypto_sha256_digest(struct ahash_request *req) 2373 { 2374 struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req); 2375 2376 artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA256, 0); 2377 req_ctx->hash_flags |= HASH_FLAG_UPDATE | HASH_FLAG_FINALIZE; 2378 2379 return artpec6_crypto_prepare_submit_hash(req); 2380 } 2381 2382 static int artpec6_crypto_hmac_sha256_init(struct ahash_request *req) 2383 { 2384 return artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA256, 1); 2385 } 2386 2387 static int artpec6_crypto_hmac_sha256_digest(struct ahash_request *req) 2388 { 2389 struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req); 2390 2391 artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA256, 1); 2392 req_ctx->hash_flags |= HASH_FLAG_UPDATE | HASH_FLAG_FINALIZE; 2393 2394 return artpec6_crypto_prepare_submit_hash(req); 2395 } 2396 2397 static int artpec6_crypto_ahash_init_common(struct crypto_tfm *tfm, 2398 const char *base_hash_name) 2399 { 2400 struct artpec6_hashalg_context *tfm_ctx = crypto_tfm_ctx(tfm); 2401 2402 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), 2403 sizeof(struct artpec6_hash_request_context)); 2404 memset(tfm_ctx, 0, sizeof(*tfm_ctx)); 2405 2406 if (base_hash_name) { 2407 struct crypto_shash *child; 2408 2409 child = crypto_alloc_shash(base_hash_name, 0, 2410 CRYPTO_ALG_NEED_FALLBACK); 2411 2412 if (IS_ERR(child)) 2413 return PTR_ERR(child); 2414 2415 tfm_ctx->child_hash = child; 2416 } 2417 2418 return 0; 2419 } 2420 2421 static int artpec6_crypto_ahash_init(struct crypto_tfm *tfm) 2422 { 2423 return artpec6_crypto_ahash_init_common(tfm, NULL); 2424 } 2425 2426 static int artpec6_crypto_ahash_init_hmac_sha256(struct crypto_tfm *tfm) 2427 { 2428 return artpec6_crypto_ahash_init_common(tfm, "sha256"); 2429 } 2430 2431 static void artpec6_crypto_ahash_exit(struct crypto_tfm *tfm) 2432 { 2433 struct artpec6_hashalg_context *tfm_ctx = crypto_tfm_ctx(tfm); 2434 2435 if (tfm_ctx->child_hash) 2436 crypto_free_shash(tfm_ctx->child_hash); 2437 2438 memset(tfm_ctx->hmac_key, 0, sizeof(tfm_ctx->hmac_key)); 2439 tfm_ctx->hmac_key_length = 0; 2440 } 2441 2442 static int artpec6_crypto_hash_export(struct ahash_request *req, void *out) 2443 { 2444 const struct artpec6_hash_request_context *ctx = ahash_request_ctx(req); 2445 struct artpec6_hash_export_state *state = out; 2446 struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev); 2447 enum artpec6_crypto_variant variant = ac->variant; 2448 2449 BUILD_BUG_ON(sizeof(state->partial_buffer) != 2450 sizeof(ctx->partial_buffer)); 2451 BUILD_BUG_ON(sizeof(state->digeststate) != sizeof(ctx->digeststate)); 2452 2453 state->digcnt = ctx->digcnt; 2454 state->partial_bytes = ctx->partial_bytes; 2455 state->hash_flags = ctx->hash_flags; 2456 2457 if (variant == ARTPEC6_CRYPTO) 2458 state->oper = FIELD_GET(A6_CRY_MD_OPER, ctx->hash_md); 2459 else 2460 state->oper = FIELD_GET(A7_CRY_MD_OPER, ctx->hash_md); 2461 2462 memcpy(state->partial_buffer, ctx->partial_buffer, 2463 sizeof(state->partial_buffer)); 2464 memcpy(state->digeststate, ctx->digeststate, 2465 sizeof(state->digeststate)); 2466 2467 return 0; 2468 } 2469 2470 static int artpec6_crypto_hash_import(struct ahash_request *req, const void *in) 2471 { 2472 struct artpec6_hash_request_context *ctx = ahash_request_ctx(req); 2473 const struct artpec6_hash_export_state *state = in; 2474 struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev); 2475 enum artpec6_crypto_variant variant = ac->variant; 2476 2477 memset(ctx, 0, sizeof(*ctx)); 2478 2479 ctx->digcnt = state->digcnt; 2480 ctx->partial_bytes = state->partial_bytes; 2481 ctx->hash_flags = state->hash_flags; 2482 2483 if (variant == ARTPEC6_CRYPTO) 2484 ctx->hash_md = FIELD_PREP(A6_CRY_MD_OPER, state->oper); 2485 else 2486 ctx->hash_md = FIELD_PREP(A7_CRY_MD_OPER, state->oper); 2487 2488 memcpy(ctx->partial_buffer, state->partial_buffer, 2489 sizeof(state->partial_buffer)); 2490 memcpy(ctx->digeststate, state->digeststate, 2491 sizeof(state->digeststate)); 2492 2493 return 0; 2494 } 2495 2496 static int init_crypto_hw(struct artpec6_crypto *ac) 2497 { 2498 enum artpec6_crypto_variant variant = ac->variant; 2499 void __iomem *base = ac->base; 2500 u32 out_descr_buf_size; 2501 u32 out_data_buf_size; 2502 u32 in_data_buf_size; 2503 u32 in_descr_buf_size; 2504 u32 in_stat_buf_size; 2505 u32 in, out; 2506 2507 /* 2508 * The PDMA unit contains 1984 bytes of internal memory for the OUT 2509 * channels and 1024 bytes for the IN channel. This is an elastic 2510 * memory used to internally store the descriptors and data. The values 2511 * ares specified in 64 byte incremements. Trustzone buffers are not 2512 * used at this stage. 2513 */ 2514 out_data_buf_size = 16; /* 1024 bytes for data */ 2515 out_descr_buf_size = 15; /* 960 bytes for descriptors */ 2516 in_data_buf_size = 8; /* 512 bytes for data */ 2517 in_descr_buf_size = 4; /* 256 bytes for descriptors */ 2518 in_stat_buf_size = 4; /* 256 bytes for stat descrs */ 2519 2520 BUILD_BUG_ON_MSG((out_data_buf_size 2521 + out_descr_buf_size) * 64 > 1984, 2522 "Invalid OUT configuration"); 2523 2524 BUILD_BUG_ON_MSG((in_data_buf_size 2525 + in_descr_buf_size 2526 + in_stat_buf_size) * 64 > 1024, 2527 "Invalid IN configuration"); 2528 2529 in = FIELD_PREP(PDMA_IN_BUF_CFG_DATA_BUF_SIZE, in_data_buf_size) | 2530 FIELD_PREP(PDMA_IN_BUF_CFG_DESCR_BUF_SIZE, in_descr_buf_size) | 2531 FIELD_PREP(PDMA_IN_BUF_CFG_STAT_BUF_SIZE, in_stat_buf_size); 2532 2533 out = FIELD_PREP(PDMA_OUT_BUF_CFG_DATA_BUF_SIZE, out_data_buf_size) | 2534 FIELD_PREP(PDMA_OUT_BUF_CFG_DESCR_BUF_SIZE, out_descr_buf_size); 2535 2536 writel_relaxed(out, base + PDMA_OUT_BUF_CFG); 2537 writel_relaxed(PDMA_OUT_CFG_EN, base + PDMA_OUT_CFG); 2538 2539 if (variant == ARTPEC6_CRYPTO) { 2540 writel_relaxed(in, base + A6_PDMA_IN_BUF_CFG); 2541 writel_relaxed(PDMA_IN_CFG_EN, base + A6_PDMA_IN_CFG); 2542 writel_relaxed(A6_PDMA_INTR_MASK_IN_DATA | 2543 A6_PDMA_INTR_MASK_IN_EOP_FLUSH, 2544 base + A6_PDMA_INTR_MASK); 2545 } else { 2546 writel_relaxed(in, base + A7_PDMA_IN_BUF_CFG); 2547 writel_relaxed(PDMA_IN_CFG_EN, base + A7_PDMA_IN_CFG); 2548 writel_relaxed(A7_PDMA_INTR_MASK_IN_DATA | 2549 A7_PDMA_INTR_MASK_IN_EOP_FLUSH, 2550 base + A7_PDMA_INTR_MASK); 2551 } 2552 2553 return 0; 2554 } 2555 2556 static void artpec6_crypto_disable_hw(struct artpec6_crypto *ac) 2557 { 2558 enum artpec6_crypto_variant variant = ac->variant; 2559 void __iomem *base = ac->base; 2560 2561 if (variant == ARTPEC6_CRYPTO) { 2562 writel_relaxed(A6_PDMA_IN_CMD_STOP, base + A6_PDMA_IN_CMD); 2563 writel_relaxed(0, base + A6_PDMA_IN_CFG); 2564 writel_relaxed(A6_PDMA_OUT_CMD_STOP, base + PDMA_OUT_CMD); 2565 } else { 2566 writel_relaxed(A7_PDMA_IN_CMD_STOP, base + A7_PDMA_IN_CMD); 2567 writel_relaxed(0, base + A7_PDMA_IN_CFG); 2568 writel_relaxed(A7_PDMA_OUT_CMD_STOP, base + PDMA_OUT_CMD); 2569 } 2570 2571 writel_relaxed(0, base + PDMA_OUT_CFG); 2572 2573 } 2574 2575 static irqreturn_t artpec6_crypto_irq(int irq, void *dev_id) 2576 { 2577 struct artpec6_crypto *ac = dev_id; 2578 enum artpec6_crypto_variant variant = ac->variant; 2579 void __iomem *base = ac->base; 2580 u32 mask_in_data, mask_in_eop_flush; 2581 u32 in_cmd_flush_stat, in_cmd_reg; 2582 u32 ack_intr_reg; 2583 u32 ack = 0; 2584 u32 intr; 2585 2586 if (variant == ARTPEC6_CRYPTO) { 2587 intr = readl_relaxed(base + A6_PDMA_MASKED_INTR); 2588 mask_in_data = A6_PDMA_INTR_MASK_IN_DATA; 2589 mask_in_eop_flush = A6_PDMA_INTR_MASK_IN_EOP_FLUSH; 2590 in_cmd_flush_stat = A6_PDMA_IN_CMD_FLUSH_STAT; 2591 in_cmd_reg = A6_PDMA_IN_CMD; 2592 ack_intr_reg = A6_PDMA_ACK_INTR; 2593 } else { 2594 intr = readl_relaxed(base + A7_PDMA_MASKED_INTR); 2595 mask_in_data = A7_PDMA_INTR_MASK_IN_DATA; 2596 mask_in_eop_flush = A7_PDMA_INTR_MASK_IN_EOP_FLUSH; 2597 in_cmd_flush_stat = A7_PDMA_IN_CMD_FLUSH_STAT; 2598 in_cmd_reg = A7_PDMA_IN_CMD; 2599 ack_intr_reg = A7_PDMA_ACK_INTR; 2600 } 2601 2602 /* We get two interrupt notifications from each job. 2603 * The in_data means all data was sent to memory and then 2604 * we request a status flush command to write the per-job 2605 * status to its status vector. This ensures that the 2606 * tasklet can detect exactly how many submitted jobs 2607 * that have finished. 2608 */ 2609 if (intr & mask_in_data) 2610 ack |= mask_in_data; 2611 2612 if (intr & mask_in_eop_flush) 2613 ack |= mask_in_eop_flush; 2614 else 2615 writel_relaxed(in_cmd_flush_stat, base + in_cmd_reg); 2616 2617 writel_relaxed(ack, base + ack_intr_reg); 2618 2619 if (intr & mask_in_eop_flush) 2620 tasklet_schedule(&ac->task); 2621 2622 return IRQ_HANDLED; 2623 } 2624 2625 /*------------------- Algorithm definitions ----------------------------------*/ 2626 2627 /* Hashes */ 2628 static struct ahash_alg hash_algos[] = { 2629 /* SHA-1 */ 2630 { 2631 .init = artpec6_crypto_sha1_init, 2632 .update = artpec6_crypto_hash_update, 2633 .final = artpec6_crypto_hash_final, 2634 .digest = artpec6_crypto_sha1_digest, 2635 .import = artpec6_crypto_hash_import, 2636 .export = artpec6_crypto_hash_export, 2637 .halg.digestsize = SHA1_DIGEST_SIZE, 2638 .halg.statesize = sizeof(struct artpec6_hash_export_state), 2639 .halg.base = { 2640 .cra_name = "sha1", 2641 .cra_driver_name = "artpec-sha1", 2642 .cra_priority = 300, 2643 .cra_flags = CRYPTO_ALG_ASYNC, 2644 .cra_blocksize = SHA1_BLOCK_SIZE, 2645 .cra_ctxsize = sizeof(struct artpec6_hashalg_context), 2646 .cra_alignmask = 3, 2647 .cra_module = THIS_MODULE, 2648 .cra_init = artpec6_crypto_ahash_init, 2649 .cra_exit = artpec6_crypto_ahash_exit, 2650 } 2651 }, 2652 /* SHA-256 */ 2653 { 2654 .init = artpec6_crypto_sha256_init, 2655 .update = artpec6_crypto_hash_update, 2656 .final = artpec6_crypto_hash_final, 2657 .digest = artpec6_crypto_sha256_digest, 2658 .import = artpec6_crypto_hash_import, 2659 .export = artpec6_crypto_hash_export, 2660 .halg.digestsize = SHA256_DIGEST_SIZE, 2661 .halg.statesize = sizeof(struct artpec6_hash_export_state), 2662 .halg.base = { 2663 .cra_name = "sha256", 2664 .cra_driver_name = "artpec-sha256", 2665 .cra_priority = 300, 2666 .cra_flags = CRYPTO_ALG_ASYNC, 2667 .cra_blocksize = SHA256_BLOCK_SIZE, 2668 .cra_ctxsize = sizeof(struct artpec6_hashalg_context), 2669 .cra_alignmask = 3, 2670 .cra_module = THIS_MODULE, 2671 .cra_init = artpec6_crypto_ahash_init, 2672 .cra_exit = artpec6_crypto_ahash_exit, 2673 } 2674 }, 2675 /* HMAC SHA-256 */ 2676 { 2677 .init = artpec6_crypto_hmac_sha256_init, 2678 .update = artpec6_crypto_hash_update, 2679 .final = artpec6_crypto_hash_final, 2680 .digest = artpec6_crypto_hmac_sha256_digest, 2681 .import = artpec6_crypto_hash_import, 2682 .export = artpec6_crypto_hash_export, 2683 .setkey = artpec6_crypto_hash_set_key, 2684 .halg.digestsize = SHA256_DIGEST_SIZE, 2685 .halg.statesize = sizeof(struct artpec6_hash_export_state), 2686 .halg.base = { 2687 .cra_name = "hmac(sha256)", 2688 .cra_driver_name = "artpec-hmac-sha256", 2689 .cra_priority = 300, 2690 .cra_flags = CRYPTO_ALG_ASYNC, 2691 .cra_blocksize = SHA256_BLOCK_SIZE, 2692 .cra_ctxsize = sizeof(struct artpec6_hashalg_context), 2693 .cra_alignmask = 3, 2694 .cra_module = THIS_MODULE, 2695 .cra_init = artpec6_crypto_ahash_init_hmac_sha256, 2696 .cra_exit = artpec6_crypto_ahash_exit, 2697 } 2698 }, 2699 }; 2700 2701 /* Crypto */ 2702 static struct skcipher_alg crypto_algos[] = { 2703 /* AES - ECB */ 2704 { 2705 .base = { 2706 .cra_name = "ecb(aes)", 2707 .cra_driver_name = "artpec6-ecb-aes", 2708 .cra_priority = 300, 2709 .cra_flags = CRYPTO_ALG_ASYNC, 2710 .cra_blocksize = AES_BLOCK_SIZE, 2711 .cra_ctxsize = sizeof(struct artpec6_cryptotfm_context), 2712 .cra_alignmask = 3, 2713 .cra_module = THIS_MODULE, 2714 }, 2715 .min_keysize = AES_MIN_KEY_SIZE, 2716 .max_keysize = AES_MAX_KEY_SIZE, 2717 .setkey = artpec6_crypto_cipher_set_key, 2718 .encrypt = artpec6_crypto_encrypt, 2719 .decrypt = artpec6_crypto_decrypt, 2720 .init = artpec6_crypto_aes_ecb_init, 2721 .exit = artpec6_crypto_aes_exit, 2722 }, 2723 /* AES - CTR */ 2724 { 2725 .base = { 2726 .cra_name = "ctr(aes)", 2727 .cra_driver_name = "artpec6-ctr-aes", 2728 .cra_priority = 300, 2729 .cra_flags = CRYPTO_ALG_ASYNC | 2730 CRYPTO_ALG_NEED_FALLBACK, 2731 .cra_blocksize = 1, 2732 .cra_ctxsize = sizeof(struct artpec6_cryptotfm_context), 2733 .cra_alignmask = 3, 2734 .cra_module = THIS_MODULE, 2735 }, 2736 .min_keysize = AES_MIN_KEY_SIZE, 2737 .max_keysize = AES_MAX_KEY_SIZE, 2738 .ivsize = AES_BLOCK_SIZE, 2739 .setkey = artpec6_crypto_cipher_set_key, 2740 .encrypt = artpec6_crypto_ctr_encrypt, 2741 .decrypt = artpec6_crypto_ctr_decrypt, 2742 .init = artpec6_crypto_aes_ctr_init, 2743 .exit = artpec6_crypto_aes_ctr_exit, 2744 }, 2745 /* AES - CBC */ 2746 { 2747 .base = { 2748 .cra_name = "cbc(aes)", 2749 .cra_driver_name = "artpec6-cbc-aes", 2750 .cra_priority = 300, 2751 .cra_flags = CRYPTO_ALG_ASYNC, 2752 .cra_blocksize = AES_BLOCK_SIZE, 2753 .cra_ctxsize = sizeof(struct artpec6_cryptotfm_context), 2754 .cra_alignmask = 3, 2755 .cra_module = THIS_MODULE, 2756 }, 2757 .min_keysize = AES_MIN_KEY_SIZE, 2758 .max_keysize = AES_MAX_KEY_SIZE, 2759 .ivsize = AES_BLOCK_SIZE, 2760 .setkey = artpec6_crypto_cipher_set_key, 2761 .encrypt = artpec6_crypto_encrypt, 2762 .decrypt = artpec6_crypto_decrypt, 2763 .init = artpec6_crypto_aes_cbc_init, 2764 .exit = artpec6_crypto_aes_exit 2765 }, 2766 /* AES - XTS */ 2767 { 2768 .base = { 2769 .cra_name = "xts(aes)", 2770 .cra_driver_name = "artpec6-xts-aes", 2771 .cra_priority = 300, 2772 .cra_flags = CRYPTO_ALG_ASYNC, 2773 .cra_blocksize = 1, 2774 .cra_ctxsize = sizeof(struct artpec6_cryptotfm_context), 2775 .cra_alignmask = 3, 2776 .cra_module = THIS_MODULE, 2777 }, 2778 .min_keysize = 2*AES_MIN_KEY_SIZE, 2779 .max_keysize = 2*AES_MAX_KEY_SIZE, 2780 .ivsize = 16, 2781 .setkey = artpec6_crypto_xts_set_key, 2782 .encrypt = artpec6_crypto_encrypt, 2783 .decrypt = artpec6_crypto_decrypt, 2784 .init = artpec6_crypto_aes_xts_init, 2785 .exit = artpec6_crypto_aes_exit, 2786 }, 2787 }; 2788 2789 static struct aead_alg aead_algos[] = { 2790 { 2791 .init = artpec6_crypto_aead_init, 2792 .setkey = artpec6_crypto_aead_set_key, 2793 .encrypt = artpec6_crypto_aead_encrypt, 2794 .decrypt = artpec6_crypto_aead_decrypt, 2795 .ivsize = GCM_AES_IV_SIZE, 2796 .maxauthsize = AES_BLOCK_SIZE, 2797 2798 .base = { 2799 .cra_name = "gcm(aes)", 2800 .cra_driver_name = "artpec-gcm-aes", 2801 .cra_priority = 300, 2802 .cra_flags = CRYPTO_ALG_ASYNC | 2803 CRYPTO_ALG_KERN_DRIVER_ONLY, 2804 .cra_blocksize = 1, 2805 .cra_ctxsize = sizeof(struct artpec6_cryptotfm_context), 2806 .cra_alignmask = 3, 2807 .cra_module = THIS_MODULE, 2808 }, 2809 } 2810 }; 2811 2812 #ifdef CONFIG_DEBUG_FS 2813 2814 struct dbgfs_u32 { 2815 char *name; 2816 mode_t mode; 2817 u32 *flag; 2818 char *desc; 2819 }; 2820 2821 static struct dentry *dbgfs_root; 2822 2823 static void artpec6_crypto_init_debugfs(void) 2824 { 2825 dbgfs_root = debugfs_create_dir("artpec6_crypto", NULL); 2826 2827 #ifdef CONFIG_FAULT_INJECTION 2828 fault_create_debugfs_attr("fail_status_read", dbgfs_root, 2829 &artpec6_crypto_fail_status_read); 2830 2831 fault_create_debugfs_attr("fail_dma_array_full", dbgfs_root, 2832 &artpec6_crypto_fail_dma_array_full); 2833 #endif 2834 } 2835 2836 static void artpec6_crypto_free_debugfs(void) 2837 { 2838 debugfs_remove_recursive(dbgfs_root); 2839 dbgfs_root = NULL; 2840 } 2841 #endif 2842 2843 static const struct of_device_id artpec6_crypto_of_match[] = { 2844 { .compatible = "axis,artpec6-crypto", .data = (void *)ARTPEC6_CRYPTO }, 2845 { .compatible = "axis,artpec7-crypto", .data = (void *)ARTPEC7_CRYPTO }, 2846 {} 2847 }; 2848 MODULE_DEVICE_TABLE(of, artpec6_crypto_of_match); 2849 2850 static int artpec6_crypto_probe(struct platform_device *pdev) 2851 { 2852 const struct of_device_id *match; 2853 enum artpec6_crypto_variant variant; 2854 struct artpec6_crypto *ac; 2855 struct device *dev = &pdev->dev; 2856 void __iomem *base; 2857 struct resource *res; 2858 int irq; 2859 int err; 2860 2861 if (artpec6_crypto_dev) 2862 return -ENODEV; 2863 2864 match = of_match_node(artpec6_crypto_of_match, dev->of_node); 2865 if (!match) 2866 return -EINVAL; 2867 2868 variant = (enum artpec6_crypto_variant)match->data; 2869 2870 res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 2871 base = devm_ioremap_resource(&pdev->dev, res); 2872 if (IS_ERR(base)) 2873 return PTR_ERR(base); 2874 2875 irq = platform_get_irq(pdev, 0); 2876 if (irq < 0) 2877 return -ENODEV; 2878 2879 ac = devm_kzalloc(&pdev->dev, sizeof(struct artpec6_crypto), 2880 GFP_KERNEL); 2881 if (!ac) 2882 return -ENOMEM; 2883 2884 platform_set_drvdata(pdev, ac); 2885 ac->variant = variant; 2886 2887 spin_lock_init(&ac->queue_lock); 2888 INIT_LIST_HEAD(&ac->queue); 2889 INIT_LIST_HEAD(&ac->pending); 2890 timer_setup(&ac->timer, artpec6_crypto_timeout, 0); 2891 2892 ac->base = base; 2893 2894 ac->dma_cache = kmem_cache_create("artpec6_crypto_dma", 2895 sizeof(struct artpec6_crypto_dma_descriptors), 2896 64, 2897 0, 2898 NULL); 2899 if (!ac->dma_cache) 2900 return -ENOMEM; 2901 2902 #ifdef CONFIG_DEBUG_FS 2903 artpec6_crypto_init_debugfs(); 2904 #endif 2905 2906 tasklet_init(&ac->task, artpec6_crypto_task, 2907 (unsigned long)ac); 2908 2909 ac->pad_buffer = devm_kzalloc(&pdev->dev, 2 * ARTPEC_CACHE_LINE_MAX, 2910 GFP_KERNEL); 2911 if (!ac->pad_buffer) 2912 return -ENOMEM; 2913 ac->pad_buffer = PTR_ALIGN(ac->pad_buffer, ARTPEC_CACHE_LINE_MAX); 2914 2915 ac->zero_buffer = devm_kzalloc(&pdev->dev, 2 * ARTPEC_CACHE_LINE_MAX, 2916 GFP_KERNEL); 2917 if (!ac->zero_buffer) 2918 return -ENOMEM; 2919 ac->zero_buffer = PTR_ALIGN(ac->zero_buffer, ARTPEC_CACHE_LINE_MAX); 2920 2921 err = init_crypto_hw(ac); 2922 if (err) 2923 goto free_cache; 2924 2925 err = devm_request_irq(&pdev->dev, irq, artpec6_crypto_irq, 0, 2926 "artpec6-crypto", ac); 2927 if (err) 2928 goto disable_hw; 2929 2930 artpec6_crypto_dev = &pdev->dev; 2931 2932 err = crypto_register_ahashes(hash_algos, ARRAY_SIZE(hash_algos)); 2933 if (err) { 2934 dev_err(dev, "Failed to register ahashes\n"); 2935 goto disable_hw; 2936 } 2937 2938 err = crypto_register_skciphers(crypto_algos, ARRAY_SIZE(crypto_algos)); 2939 if (err) { 2940 dev_err(dev, "Failed to register ciphers\n"); 2941 goto unregister_ahashes; 2942 } 2943 2944 err = crypto_register_aeads(aead_algos, ARRAY_SIZE(aead_algos)); 2945 if (err) { 2946 dev_err(dev, "Failed to register aeads\n"); 2947 goto unregister_algs; 2948 } 2949 2950 return 0; 2951 2952 unregister_algs: 2953 crypto_unregister_skciphers(crypto_algos, ARRAY_SIZE(crypto_algos)); 2954 unregister_ahashes: 2955 crypto_unregister_ahashes(hash_algos, ARRAY_SIZE(hash_algos)); 2956 disable_hw: 2957 artpec6_crypto_disable_hw(ac); 2958 free_cache: 2959 kmem_cache_destroy(ac->dma_cache); 2960 return err; 2961 } 2962 2963 static int artpec6_crypto_remove(struct platform_device *pdev) 2964 { 2965 struct artpec6_crypto *ac = platform_get_drvdata(pdev); 2966 int irq = platform_get_irq(pdev, 0); 2967 2968 crypto_unregister_ahashes(hash_algos, ARRAY_SIZE(hash_algos)); 2969 crypto_unregister_skciphers(crypto_algos, ARRAY_SIZE(crypto_algos)); 2970 crypto_unregister_aeads(aead_algos, ARRAY_SIZE(aead_algos)); 2971 2972 tasklet_disable(&ac->task); 2973 devm_free_irq(&pdev->dev, irq, ac); 2974 tasklet_kill(&ac->task); 2975 del_timer_sync(&ac->timer); 2976 2977 artpec6_crypto_disable_hw(ac); 2978 2979 kmem_cache_destroy(ac->dma_cache); 2980 #ifdef CONFIG_DEBUG_FS 2981 artpec6_crypto_free_debugfs(); 2982 #endif 2983 return 0; 2984 } 2985 2986 static struct platform_driver artpec6_crypto_driver = { 2987 .probe = artpec6_crypto_probe, 2988 .remove = artpec6_crypto_remove, 2989 .driver = { 2990 .name = "artpec6-crypto", 2991 .of_match_table = artpec6_crypto_of_match, 2992 }, 2993 }; 2994 2995 module_platform_driver(artpec6_crypto_driver); 2996 2997 MODULE_AUTHOR("Axis Communications AB"); 2998 MODULE_DESCRIPTION("ARTPEC-6 Crypto driver"); 2999 MODULE_LICENSE("GPL"); 3000