1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Intel Keem Bay OCS AES Crypto Driver. 4 * 5 * Copyright (C) 2018-2020 Intel Corporation 6 */ 7 8 #include <linux/dma-mapping.h> 9 #include <linux/interrupt.h> 10 #include <linux/platform_device.h> 11 #include <linux/slab.h> 12 #include <linux/swab.h> 13 14 #include <asm/byteorder.h> 15 #include <asm/errno.h> 16 17 #include <crypto/aes.h> 18 #include <crypto/gcm.h> 19 20 #include "ocs-aes.h" 21 22 #define AES_COMMAND_OFFSET 0x0000 23 #define AES_KEY_0_OFFSET 0x0004 24 #define AES_KEY_1_OFFSET 0x0008 25 #define AES_KEY_2_OFFSET 0x000C 26 #define AES_KEY_3_OFFSET 0x0010 27 #define AES_KEY_4_OFFSET 0x0014 28 #define AES_KEY_5_OFFSET 0x0018 29 #define AES_KEY_6_OFFSET 0x001C 30 #define AES_KEY_7_OFFSET 0x0020 31 #define AES_IV_0_OFFSET 0x0024 32 #define AES_IV_1_OFFSET 0x0028 33 #define AES_IV_2_OFFSET 0x002C 34 #define AES_IV_3_OFFSET 0x0030 35 #define AES_ACTIVE_OFFSET 0x0034 36 #define AES_STATUS_OFFSET 0x0038 37 #define AES_KEY_SIZE_OFFSET 0x0044 38 #define AES_IER_OFFSET 0x0048 39 #define AES_ISR_OFFSET 0x005C 40 #define AES_MULTIPURPOSE1_0_OFFSET 0x0200 41 #define AES_MULTIPURPOSE1_1_OFFSET 0x0204 42 #define AES_MULTIPURPOSE1_2_OFFSET 0x0208 43 #define AES_MULTIPURPOSE1_3_OFFSET 0x020C 44 #define AES_MULTIPURPOSE2_0_OFFSET 0x0220 45 #define AES_MULTIPURPOSE2_1_OFFSET 0x0224 46 #define AES_MULTIPURPOSE2_2_OFFSET 0x0228 47 #define AES_MULTIPURPOSE2_3_OFFSET 0x022C 48 #define AES_BYTE_ORDER_CFG_OFFSET 0x02C0 49 #define AES_TLEN_OFFSET 0x0300 50 #define AES_T_MAC_0_OFFSET 0x0304 51 #define AES_T_MAC_1_OFFSET 0x0308 52 #define AES_T_MAC_2_OFFSET 0x030C 53 #define AES_T_MAC_3_OFFSET 0x0310 54 #define AES_PLEN_OFFSET 0x0314 55 #define AES_A_DMA_SRC_ADDR_OFFSET 0x0400 56 #define AES_A_DMA_DST_ADDR_OFFSET 0x0404 57 #define AES_A_DMA_SRC_SIZE_OFFSET 0x0408 58 #define AES_A_DMA_DST_SIZE_OFFSET 0x040C 59 #define AES_A_DMA_DMA_MODE_OFFSET 0x0410 60 #define AES_A_DMA_NEXT_SRC_DESCR_OFFSET 0x0418 61 #define AES_A_DMA_NEXT_DST_DESCR_OFFSET 0x041C 62 #define AES_A_DMA_WHILE_ACTIVE_MODE_OFFSET 0x0420 63 #define AES_A_DMA_LOG_OFFSET 0x0424 64 #define AES_A_DMA_STATUS_OFFSET 0x0428 65 #define AES_A_DMA_PERF_CNTR_OFFSET 0x042C 66 #define AES_A_DMA_MSI_ISR_OFFSET 0x0480 67 #define AES_A_DMA_MSI_IER_OFFSET 0x0484 68 #define AES_A_DMA_MSI_MASK_OFFSET 0x0488 69 #define AES_A_DMA_INBUFFER_WRITE_FIFO_OFFSET 0x0600 70 #define AES_A_DMA_OUTBUFFER_READ_FIFO_OFFSET 0x0700 71 72 /* 73 * AES_A_DMA_DMA_MODE register. 74 * Default: 0x00000000. 75 * bit[31] ACTIVE 76 * This bit activates the DMA. When the DMA finishes, it resets 77 * this bit to zero. 78 * bit[30:26] Unused by this driver. 79 * bit[25] SRC_LINK_LIST_EN 80 * Source link list enable bit. When the linked list is terminated 81 * this bit is reset by the DMA. 82 * bit[24] DST_LINK_LIST_EN 83 * Destination link list enable bit. When the linked list is 84 * terminated this bit is reset by the DMA. 85 * bit[23:0] Unused by this driver. 86 */ 87 #define AES_A_DMA_DMA_MODE_ACTIVE BIT(31) 88 #define AES_A_DMA_DMA_MODE_SRC_LINK_LIST_EN BIT(25) 89 #define AES_A_DMA_DMA_MODE_DST_LINK_LIST_EN BIT(24) 90 91 /* 92 * AES_ACTIVE register 93 * default 0x00000000 94 * bit[31:10] Reserved 95 * bit[9] LAST_ADATA 96 * bit[8] LAST_GCX 97 * bit[7:2] Reserved 98 * bit[1] TERMINATION 99 * bit[0] TRIGGER 100 */ 101 #define AES_ACTIVE_LAST_ADATA BIT(9) 102 #define AES_ACTIVE_LAST_CCM_GCM BIT(8) 103 #define AES_ACTIVE_TERMINATION BIT(1) 104 #define AES_ACTIVE_TRIGGER BIT(0) 105 106 #define AES_DISABLE_INT 0x00000000 107 #define AES_DMA_CPD_ERR_INT BIT(8) 108 #define AES_DMA_OUTBUF_RD_ERR_INT BIT(7) 109 #define AES_DMA_OUTBUF_WR_ERR_INT BIT(6) 110 #define AES_DMA_INBUF_RD_ERR_INT BIT(5) 111 #define AES_DMA_INBUF_WR_ERR_INT BIT(4) 112 #define AES_DMA_BAD_COMP_INT BIT(3) 113 #define AES_DMA_SAI_INT BIT(2) 114 #define AES_DMA_SRC_DONE_INT BIT(0) 115 #define AES_COMPLETE_INT BIT(1) 116 117 #define AES_DMA_MSI_MASK_CLEAR BIT(0) 118 119 #define AES_128_BIT_KEY 0x00000000 120 #define AES_256_BIT_KEY BIT(0) 121 122 #define AES_DEACTIVATE_PERF_CNTR 0x00000000 123 #define AES_ACTIVATE_PERF_CNTR BIT(0) 124 125 #define AES_MAX_TAG_SIZE_U32 4 126 127 #define OCS_LL_DMA_FLAG_TERMINATE BIT(31) 128 129 /* 130 * There is an inconsistency in the documentation. This is documented as a 131 * 11-bit value, but it is actually 10-bits. 132 */ 133 #define AES_DMA_STATUS_INPUT_BUFFER_OCCUPANCY_MASK 0x3FF 134 135 /* 136 * During CCM decrypt, the OCS block needs to finish processing the ciphertext 137 * before the tag is written. For 128-bit mode this required delay is 28 OCS 138 * clock cycles. For 256-bit mode it is 36 OCS clock cycles. 139 */ 140 #define CCM_DECRYPT_DELAY_TAG_CLK_COUNT 36UL 141 142 /* 143 * During CCM decrypt there must be a delay of at least 42 OCS clock cycles 144 * between setting the TRIGGER bit in AES_ACTIVE and setting the LAST_CCM_GCM 145 * bit in the same register (as stated in the OCS databook) 146 */ 147 #define CCM_DECRYPT_DELAY_LAST_GCX_CLK_COUNT 42UL 148 149 /* See RFC3610 section 2.2 */ 150 #define L_PRIME_MIN (1) 151 #define L_PRIME_MAX (7) 152 /* 153 * CCM IV format from RFC 3610 section 2.3 154 * 155 * Octet Number Contents 156 * ------------ --------- 157 * 0 Flags 158 * 1 ... 15-L Nonce N 159 * 16-L ... 15 Counter i 160 * 161 * Flags = L' = L - 1 162 */ 163 #define L_PRIME_IDX 0 164 #define COUNTER_START(lprime) (16 - ((lprime) + 1)) 165 #define COUNTER_LEN(lprime) ((lprime) + 1) 166 167 enum aes_counter_mode { 168 AES_CTR_M_NO_INC = 0, 169 AES_CTR_M_32_INC = 1, 170 AES_CTR_M_64_INC = 2, 171 AES_CTR_M_128_INC = 3, 172 }; 173 174 /** 175 * struct ocs_dma_linked_list - OCS DMA linked list entry. 176 * @src_addr: Source address of the data. 177 * @src_len: Length of data to be fetched. 178 * @next: Next dma_list to fetch. 179 * @ll_flags: Flags (Freeze @ terminate) for the DMA engine. 180 */ 181 struct ocs_dma_linked_list { 182 u32 src_addr; 183 u32 src_len; 184 u32 next; 185 u32 ll_flags; 186 } __packed; 187 188 /* 189 * Set endianness of inputs and outputs 190 * AES_BYTE_ORDER_CFG 191 * default 0x00000000 192 * bit [10] - KEY_HI_LO_SWAP 193 * bit [9] - KEY_HI_SWAP_DWORDS_IN_OCTWORD 194 * bit [8] - KEY_HI_SWAP_BYTES_IN_DWORD 195 * bit [7] - KEY_LO_SWAP_DWORDS_IN_OCTWORD 196 * bit [6] - KEY_LO_SWAP_BYTES_IN_DWORD 197 * bit [5] - IV_SWAP_DWORDS_IN_OCTWORD 198 * bit [4] - IV_SWAP_BYTES_IN_DWORD 199 * bit [3] - DOUT_SWAP_DWORDS_IN_OCTWORD 200 * bit [2] - DOUT_SWAP_BYTES_IN_DWORD 201 * bit [1] - DOUT_SWAP_DWORDS_IN_OCTWORD 202 * bit [0] - DOUT_SWAP_BYTES_IN_DWORD 203 */ 204 static inline void aes_a_set_endianness(const struct ocs_aes_dev *aes_dev) 205 { 206 iowrite32(0x7FF, aes_dev->base_reg + AES_BYTE_ORDER_CFG_OFFSET); 207 } 208 209 /* Trigger AES process start. */ 210 static inline void aes_a_op_trigger(const struct ocs_aes_dev *aes_dev) 211 { 212 iowrite32(AES_ACTIVE_TRIGGER, aes_dev->base_reg + AES_ACTIVE_OFFSET); 213 } 214 215 /* Indicate last bulk of data. */ 216 static inline void aes_a_op_termination(const struct ocs_aes_dev *aes_dev) 217 { 218 iowrite32(AES_ACTIVE_TERMINATION, 219 aes_dev->base_reg + AES_ACTIVE_OFFSET); 220 } 221 222 /* 223 * Set LAST_CCM_GCM in AES_ACTIVE register and clear all other bits. 224 * 225 * Called when DMA is programmed to fetch the last batch of data. 226 * - For AES-CCM it is called for the last batch of Payload data and Ciphertext 227 * data. 228 * - For AES-GCM, it is called for the last batch of Plaintext data and 229 * Ciphertext data. 230 */ 231 static inline void aes_a_set_last_gcx(const struct ocs_aes_dev *aes_dev) 232 { 233 iowrite32(AES_ACTIVE_LAST_CCM_GCM, 234 aes_dev->base_reg + AES_ACTIVE_OFFSET); 235 } 236 237 /* Wait for LAST_CCM_GCM bit to be unset. */ 238 static inline void aes_a_wait_last_gcx(const struct ocs_aes_dev *aes_dev) 239 { 240 u32 aes_active_reg; 241 242 do { 243 aes_active_reg = ioread32(aes_dev->base_reg + 244 AES_ACTIVE_OFFSET); 245 } while (aes_active_reg & AES_ACTIVE_LAST_CCM_GCM); 246 } 247 248 /* Wait for 10 bits of input occupancy. */ 249 static void aes_a_dma_wait_input_buffer_occupancy(const struct ocs_aes_dev *aes_dev) 250 { 251 u32 reg; 252 253 do { 254 reg = ioread32(aes_dev->base_reg + AES_A_DMA_STATUS_OFFSET); 255 } while (reg & AES_DMA_STATUS_INPUT_BUFFER_OCCUPANCY_MASK); 256 } 257 258 /* 259 * Set LAST_CCM_GCM and LAST_ADATA bits in AES_ACTIVE register (and clear all 260 * other bits). 261 * 262 * Called when DMA is programmed to fetch the last batch of Associated Data 263 * (CCM case) or Additional Authenticated Data (GCM case). 264 */ 265 static inline void aes_a_set_last_gcx_and_adata(const struct ocs_aes_dev *aes_dev) 266 { 267 iowrite32(AES_ACTIVE_LAST_ADATA | AES_ACTIVE_LAST_CCM_GCM, 268 aes_dev->base_reg + AES_ACTIVE_OFFSET); 269 } 270 271 /* Set DMA src and dst transfer size to 0 */ 272 static inline void aes_a_dma_set_xfer_size_zero(const struct ocs_aes_dev *aes_dev) 273 { 274 iowrite32(0, aes_dev->base_reg + AES_A_DMA_SRC_SIZE_OFFSET); 275 iowrite32(0, aes_dev->base_reg + AES_A_DMA_DST_SIZE_OFFSET); 276 } 277 278 /* Activate DMA for zero-byte transfer case. */ 279 static inline void aes_a_dma_active(const struct ocs_aes_dev *aes_dev) 280 { 281 iowrite32(AES_A_DMA_DMA_MODE_ACTIVE, 282 aes_dev->base_reg + AES_A_DMA_DMA_MODE_OFFSET); 283 } 284 285 /* Activate DMA and enable src linked list */ 286 static inline void aes_a_dma_active_src_ll_en(const struct ocs_aes_dev *aes_dev) 287 { 288 iowrite32(AES_A_DMA_DMA_MODE_ACTIVE | 289 AES_A_DMA_DMA_MODE_SRC_LINK_LIST_EN, 290 aes_dev->base_reg + AES_A_DMA_DMA_MODE_OFFSET); 291 } 292 293 /* Activate DMA and enable dst linked list */ 294 static inline void aes_a_dma_active_dst_ll_en(const struct ocs_aes_dev *aes_dev) 295 { 296 iowrite32(AES_A_DMA_DMA_MODE_ACTIVE | 297 AES_A_DMA_DMA_MODE_DST_LINK_LIST_EN, 298 aes_dev->base_reg + AES_A_DMA_DMA_MODE_OFFSET); 299 } 300 301 /* Activate DMA and enable src and dst linked lists */ 302 static inline void aes_a_dma_active_src_dst_ll_en(const struct ocs_aes_dev *aes_dev) 303 { 304 iowrite32(AES_A_DMA_DMA_MODE_ACTIVE | 305 AES_A_DMA_DMA_MODE_SRC_LINK_LIST_EN | 306 AES_A_DMA_DMA_MODE_DST_LINK_LIST_EN, 307 aes_dev->base_reg + AES_A_DMA_DMA_MODE_OFFSET); 308 } 309 310 /* Reset PERF_CNTR to 0 and activate it */ 311 static inline void aes_a_dma_reset_and_activate_perf_cntr(const struct ocs_aes_dev *aes_dev) 312 { 313 iowrite32(0x00000000, aes_dev->base_reg + AES_A_DMA_PERF_CNTR_OFFSET); 314 iowrite32(AES_ACTIVATE_PERF_CNTR, 315 aes_dev->base_reg + AES_A_DMA_WHILE_ACTIVE_MODE_OFFSET); 316 } 317 318 /* Wait until PERF_CNTR is > delay, then deactivate it */ 319 static inline void aes_a_dma_wait_and_deactivate_perf_cntr(const struct ocs_aes_dev *aes_dev, 320 int delay) 321 { 322 while (ioread32(aes_dev->base_reg + AES_A_DMA_PERF_CNTR_OFFSET) < delay) 323 ; 324 iowrite32(AES_DEACTIVATE_PERF_CNTR, 325 aes_dev->base_reg + AES_A_DMA_WHILE_ACTIVE_MODE_OFFSET); 326 } 327 328 /* Disable AES and DMA IRQ. */ 329 static void aes_irq_disable(struct ocs_aes_dev *aes_dev) 330 { 331 u32 isr_val = 0; 332 333 /* Disable interrupts */ 334 iowrite32(AES_DISABLE_INT, 335 aes_dev->base_reg + AES_A_DMA_MSI_IER_OFFSET); 336 iowrite32(AES_DISABLE_INT, aes_dev->base_reg + AES_IER_OFFSET); 337 338 /* Clear any pending interrupt */ 339 isr_val = ioread32(aes_dev->base_reg + AES_A_DMA_MSI_ISR_OFFSET); 340 if (isr_val) 341 iowrite32(isr_val, 342 aes_dev->base_reg + AES_A_DMA_MSI_ISR_OFFSET); 343 344 isr_val = ioread32(aes_dev->base_reg + AES_A_DMA_MSI_MASK_OFFSET); 345 if (isr_val) 346 iowrite32(isr_val, 347 aes_dev->base_reg + AES_A_DMA_MSI_MASK_OFFSET); 348 349 isr_val = ioread32(aes_dev->base_reg + AES_ISR_OFFSET); 350 if (isr_val) 351 iowrite32(isr_val, aes_dev->base_reg + AES_ISR_OFFSET); 352 } 353 354 /* Enable AES or DMA IRQ. IRQ is disabled once fired. */ 355 static void aes_irq_enable(struct ocs_aes_dev *aes_dev, u8 irq) 356 { 357 if (irq == AES_COMPLETE_INT) { 358 /* Ensure DMA error interrupts are enabled */ 359 iowrite32(AES_DMA_CPD_ERR_INT | 360 AES_DMA_OUTBUF_RD_ERR_INT | 361 AES_DMA_OUTBUF_WR_ERR_INT | 362 AES_DMA_INBUF_RD_ERR_INT | 363 AES_DMA_INBUF_WR_ERR_INT | 364 AES_DMA_BAD_COMP_INT | 365 AES_DMA_SAI_INT, 366 aes_dev->base_reg + AES_A_DMA_MSI_IER_OFFSET); 367 /* 368 * AES_IER 369 * default 0x00000000 370 * bits [31:3] - reserved 371 * bit [2] - EN_SKS_ERR 372 * bit [1] - EN_AES_COMPLETE 373 * bit [0] - reserved 374 */ 375 iowrite32(AES_COMPLETE_INT, aes_dev->base_reg + AES_IER_OFFSET); 376 return; 377 } 378 if (irq == AES_DMA_SRC_DONE_INT) { 379 /* Ensure AES interrupts are disabled */ 380 iowrite32(AES_DISABLE_INT, aes_dev->base_reg + AES_IER_OFFSET); 381 /* 382 * DMA_MSI_IER 383 * default 0x00000000 384 * bits [31:9] - reserved 385 * bit [8] - CPD_ERR_INT_EN 386 * bit [7] - OUTBUF_RD_ERR_INT_EN 387 * bit [6] - OUTBUF_WR_ERR_INT_EN 388 * bit [5] - INBUF_RD_ERR_INT_EN 389 * bit [4] - INBUF_WR_ERR_INT_EN 390 * bit [3] - BAD_COMP_INT_EN 391 * bit [2] - SAI_INT_EN 392 * bit [1] - DST_DONE_INT_EN 393 * bit [0] - SRC_DONE_INT_EN 394 */ 395 iowrite32(AES_DMA_CPD_ERR_INT | 396 AES_DMA_OUTBUF_RD_ERR_INT | 397 AES_DMA_OUTBUF_WR_ERR_INT | 398 AES_DMA_INBUF_RD_ERR_INT | 399 AES_DMA_INBUF_WR_ERR_INT | 400 AES_DMA_BAD_COMP_INT | 401 AES_DMA_SAI_INT | 402 AES_DMA_SRC_DONE_INT, 403 aes_dev->base_reg + AES_A_DMA_MSI_IER_OFFSET); 404 } 405 } 406 407 /* Enable and wait for IRQ (either from OCS AES engine or DMA) */ 408 static int ocs_aes_irq_enable_and_wait(struct ocs_aes_dev *aes_dev, u8 irq) 409 { 410 int rc; 411 412 reinit_completion(&aes_dev->irq_completion); 413 aes_irq_enable(aes_dev, irq); 414 rc = wait_for_completion_interruptible(&aes_dev->irq_completion); 415 if (rc) 416 return rc; 417 418 return aes_dev->dma_err_mask ? -EIO : 0; 419 } 420 421 /* Configure DMA to OCS, linked list mode */ 422 static inline void dma_to_ocs_aes_ll(struct ocs_aes_dev *aes_dev, 423 dma_addr_t dma_list) 424 { 425 iowrite32(0, aes_dev->base_reg + AES_A_DMA_SRC_SIZE_OFFSET); 426 iowrite32(dma_list, 427 aes_dev->base_reg + AES_A_DMA_NEXT_SRC_DESCR_OFFSET); 428 } 429 430 /* Configure DMA from OCS, linked list mode */ 431 static inline void dma_from_ocs_aes_ll(struct ocs_aes_dev *aes_dev, 432 dma_addr_t dma_list) 433 { 434 iowrite32(0, aes_dev->base_reg + AES_A_DMA_DST_SIZE_OFFSET); 435 iowrite32(dma_list, 436 aes_dev->base_reg + AES_A_DMA_NEXT_DST_DESCR_OFFSET); 437 } 438 439 irqreturn_t ocs_aes_irq_handler(int irq, void *dev_id) 440 { 441 struct ocs_aes_dev *aes_dev = dev_id; 442 u32 aes_dma_isr; 443 444 /* Read DMA ISR status. */ 445 aes_dma_isr = ioread32(aes_dev->base_reg + AES_A_DMA_MSI_ISR_OFFSET); 446 447 /* Disable and clear interrupts. */ 448 aes_irq_disable(aes_dev); 449 450 /* Save DMA error status. */ 451 aes_dev->dma_err_mask = aes_dma_isr & 452 (AES_DMA_CPD_ERR_INT | 453 AES_DMA_OUTBUF_RD_ERR_INT | 454 AES_DMA_OUTBUF_WR_ERR_INT | 455 AES_DMA_INBUF_RD_ERR_INT | 456 AES_DMA_INBUF_WR_ERR_INT | 457 AES_DMA_BAD_COMP_INT | 458 AES_DMA_SAI_INT); 459 460 /* Signal IRQ completion. */ 461 complete(&aes_dev->irq_completion); 462 463 return IRQ_HANDLED; 464 } 465 466 /** 467 * ocs_aes_set_key() - Write key into OCS AES hardware. 468 * @aes_dev: The OCS AES device to write the key to. 469 * @key_size: The size of the key (in bytes). 470 * @key: The key to write. 471 * @cipher: The cipher the key is for. 472 * 473 * For AES @key_size must be either 16 or 32. For SM4 @key_size must be 16. 474 * 475 * Return: 0 on success, negative error code otherwise. 476 */ 477 int ocs_aes_set_key(struct ocs_aes_dev *aes_dev, u32 key_size, const u8 *key, 478 enum ocs_cipher cipher) 479 { 480 const u32 *key_u32; 481 u32 val; 482 int i; 483 484 /* OCS AES supports 128-bit and 256-bit keys only. */ 485 if (cipher == OCS_AES && !(key_size == 32 || key_size == 16)) { 486 dev_err(aes_dev->dev, 487 "%d-bit keys not supported by AES cipher\n", 488 key_size * 8); 489 return -EINVAL; 490 } 491 /* OCS SM4 supports 128-bit keys only. */ 492 if (cipher == OCS_SM4 && key_size != 16) { 493 dev_err(aes_dev->dev, 494 "%d-bit keys not supported for SM4 cipher\n", 495 key_size * 8); 496 return -EINVAL; 497 } 498 499 if (!key) 500 return -EINVAL; 501 502 key_u32 = (const u32 *)key; 503 504 /* Write key to AES_KEY[0-7] registers */ 505 for (i = 0; i < (key_size / sizeof(u32)); i++) { 506 iowrite32(key_u32[i], 507 aes_dev->base_reg + AES_KEY_0_OFFSET + 508 (i * sizeof(u32))); 509 } 510 /* 511 * Write key size 512 * bits [31:1] - reserved 513 * bit [0] - AES_KEY_SIZE 514 * 0 - 128 bit key 515 * 1 - 256 bit key 516 */ 517 val = (key_size == 16) ? AES_128_BIT_KEY : AES_256_BIT_KEY; 518 iowrite32(val, aes_dev->base_reg + AES_KEY_SIZE_OFFSET); 519 520 return 0; 521 } 522 523 /* Write AES_COMMAND */ 524 static inline void set_ocs_aes_command(struct ocs_aes_dev *aes_dev, 525 enum ocs_cipher cipher, 526 enum ocs_mode mode, 527 enum ocs_instruction instruction) 528 { 529 u32 val; 530 531 /* AES_COMMAND 532 * default 0x000000CC 533 * bit [14] - CIPHER_SELECT 534 * 0 - AES 535 * 1 - SM4 536 * bits [11:8] - OCS_AES_MODE 537 * 0000 - ECB 538 * 0001 - CBC 539 * 0010 - CTR 540 * 0110 - CCM 541 * 0111 - GCM 542 * 1001 - CTS 543 * bits [7:6] - AES_INSTRUCTION 544 * 00 - ENCRYPT 545 * 01 - DECRYPT 546 * 10 - EXPAND 547 * 11 - BYPASS 548 * bits [3:2] - CTR_M_BITS 549 * 00 - No increment 550 * 01 - Least significant 32 bits are incremented 551 * 10 - Least significant 64 bits are incremented 552 * 11 - Full 128 bits are incremented 553 */ 554 val = (cipher << 14) | (mode << 8) | (instruction << 6) | 555 (AES_CTR_M_128_INC << 2); 556 iowrite32(val, aes_dev->base_reg + AES_COMMAND_OFFSET); 557 } 558 559 static void ocs_aes_init(struct ocs_aes_dev *aes_dev, 560 enum ocs_mode mode, 561 enum ocs_cipher cipher, 562 enum ocs_instruction instruction) 563 { 564 /* Ensure interrupts are disabled and pending interrupts cleared. */ 565 aes_irq_disable(aes_dev); 566 567 /* Set endianness recommended by data-sheet. */ 568 aes_a_set_endianness(aes_dev); 569 570 /* Set AES_COMMAND register. */ 571 set_ocs_aes_command(aes_dev, cipher, mode, instruction); 572 } 573 574 /* 575 * Write the byte length of the last AES/SM4 block of Payload data (without 576 * zero padding and without the length of the MAC) in register AES_PLEN. 577 */ 578 static inline void ocs_aes_write_last_data_blk_len(struct ocs_aes_dev *aes_dev, 579 u32 size) 580 { 581 u32 val; 582 583 if (size == 0) { 584 val = 0; 585 goto exit; 586 } 587 588 val = size % AES_BLOCK_SIZE; 589 if (val == 0) 590 val = AES_BLOCK_SIZE; 591 592 exit: 593 iowrite32(val, aes_dev->base_reg + AES_PLEN_OFFSET); 594 } 595 596 /* 597 * Validate inputs according to mode. 598 * If OK return 0; else return -EINVAL. 599 */ 600 static int ocs_aes_validate_inputs(dma_addr_t src_dma_list, u32 src_size, 601 const u8 *iv, u32 iv_size, 602 dma_addr_t aad_dma_list, u32 aad_size, 603 const u8 *tag, u32 tag_size, 604 enum ocs_cipher cipher, enum ocs_mode mode, 605 enum ocs_instruction instruction, 606 dma_addr_t dst_dma_list) 607 { 608 /* Ensure cipher, mode and instruction are valid. */ 609 if (!(cipher == OCS_AES || cipher == OCS_SM4)) 610 return -EINVAL; 611 612 if (mode != OCS_MODE_ECB && mode != OCS_MODE_CBC && 613 mode != OCS_MODE_CTR && mode != OCS_MODE_CCM && 614 mode != OCS_MODE_GCM && mode != OCS_MODE_CTS) 615 return -EINVAL; 616 617 if (instruction != OCS_ENCRYPT && instruction != OCS_DECRYPT && 618 instruction != OCS_EXPAND && instruction != OCS_BYPASS) 619 return -EINVAL; 620 621 /* 622 * When instruction is OCS_BYPASS, OCS simply copies data from source 623 * to destination using DMA. 624 * 625 * AES mode is irrelevant, but both source and destination DMA 626 * linked-list must be defined. 627 */ 628 if (instruction == OCS_BYPASS) { 629 if (src_dma_list == DMA_MAPPING_ERROR || 630 dst_dma_list == DMA_MAPPING_ERROR) 631 return -EINVAL; 632 633 return 0; 634 } 635 636 /* 637 * For performance reasons switch based on mode to limit unnecessary 638 * conditionals for each mode 639 */ 640 switch (mode) { 641 case OCS_MODE_ECB: 642 /* Ensure input length is multiple of block size */ 643 if (src_size % AES_BLOCK_SIZE != 0) 644 return -EINVAL; 645 646 /* Ensure source and destination linked lists are created */ 647 if (src_dma_list == DMA_MAPPING_ERROR || 648 dst_dma_list == DMA_MAPPING_ERROR) 649 return -EINVAL; 650 651 return 0; 652 653 case OCS_MODE_CBC: 654 /* Ensure input length is multiple of block size */ 655 if (src_size % AES_BLOCK_SIZE != 0) 656 return -EINVAL; 657 658 /* Ensure source and destination linked lists are created */ 659 if (src_dma_list == DMA_MAPPING_ERROR || 660 dst_dma_list == DMA_MAPPING_ERROR) 661 return -EINVAL; 662 663 /* Ensure IV is present and block size in length */ 664 if (!iv || iv_size != AES_BLOCK_SIZE) 665 return -EINVAL; 666 667 return 0; 668 669 case OCS_MODE_CTR: 670 /* Ensure input length of 1 byte or greater */ 671 if (src_size == 0) 672 return -EINVAL; 673 674 /* Ensure source and destination linked lists are created */ 675 if (src_dma_list == DMA_MAPPING_ERROR || 676 dst_dma_list == DMA_MAPPING_ERROR) 677 return -EINVAL; 678 679 /* Ensure IV is present and block size in length */ 680 if (!iv || iv_size != AES_BLOCK_SIZE) 681 return -EINVAL; 682 683 return 0; 684 685 case OCS_MODE_CTS: 686 /* Ensure input length >= block size */ 687 if (src_size < AES_BLOCK_SIZE) 688 return -EINVAL; 689 690 /* Ensure source and destination linked lists are created */ 691 if (src_dma_list == DMA_MAPPING_ERROR || 692 dst_dma_list == DMA_MAPPING_ERROR) 693 return -EINVAL; 694 695 /* Ensure IV is present and block size in length */ 696 if (!iv || iv_size != AES_BLOCK_SIZE) 697 return -EINVAL; 698 699 return 0; 700 701 case OCS_MODE_GCM: 702 /* Ensure IV is present and GCM_AES_IV_SIZE in length */ 703 if (!iv || iv_size != GCM_AES_IV_SIZE) 704 return -EINVAL; 705 706 /* 707 * If input data present ensure source and destination linked 708 * lists are created 709 */ 710 if (src_size && (src_dma_list == DMA_MAPPING_ERROR || 711 dst_dma_list == DMA_MAPPING_ERROR)) 712 return -EINVAL; 713 714 /* If aad present ensure aad linked list is created */ 715 if (aad_size && aad_dma_list == DMA_MAPPING_ERROR) 716 return -EINVAL; 717 718 /* Ensure tag destination is set */ 719 if (!tag) 720 return -EINVAL; 721 722 /* Just ensure that tag_size doesn't cause overflows. */ 723 if (tag_size > (AES_MAX_TAG_SIZE_U32 * sizeof(u32))) 724 return -EINVAL; 725 726 return 0; 727 728 case OCS_MODE_CCM: 729 /* Ensure IV is present and block size in length */ 730 if (!iv || iv_size != AES_BLOCK_SIZE) 731 return -EINVAL; 732 733 /* 2 <= L <= 8, so 1 <= L' <= 7 */ 734 if (iv[L_PRIME_IDX] < L_PRIME_MIN || 735 iv[L_PRIME_IDX] > L_PRIME_MAX) 736 return -EINVAL; 737 738 /* If aad present ensure aad linked list is created */ 739 if (aad_size && aad_dma_list == DMA_MAPPING_ERROR) 740 return -EINVAL; 741 742 /* Just ensure that tag_size doesn't cause overflows. */ 743 if (tag_size > (AES_MAX_TAG_SIZE_U32 * sizeof(u32))) 744 return -EINVAL; 745 746 if (instruction == OCS_DECRYPT) { 747 /* 748 * If input data present ensure source and destination 749 * linked lists are created 750 */ 751 if (src_size && (src_dma_list == DMA_MAPPING_ERROR || 752 dst_dma_list == DMA_MAPPING_ERROR)) 753 return -EINVAL; 754 755 /* Ensure input tag is present */ 756 if (!tag) 757 return -EINVAL; 758 759 return 0; 760 } 761 762 /* Instruction == OCS_ENCRYPT */ 763 764 /* 765 * Destination linked list always required (for tag even if no 766 * input data) 767 */ 768 if (dst_dma_list == DMA_MAPPING_ERROR) 769 return -EINVAL; 770 771 /* If input data present ensure src linked list is created */ 772 if (src_size && src_dma_list == DMA_MAPPING_ERROR) 773 return -EINVAL; 774 775 return 0; 776 777 default: 778 return -EINVAL; 779 } 780 } 781 782 /** 783 * ocs_aes_op() - Perform AES/SM4 operation. 784 * @aes_dev: The OCS AES device to use. 785 * @mode: The mode to use (ECB, CBC, CTR, or CTS). 786 * @cipher: The cipher to use (AES or SM4). 787 * @instruction: The instruction to perform (encrypt or decrypt). 788 * @dst_dma_list: The OCS DMA list mapping output memory. 789 * @src_dma_list: The OCS DMA list mapping input payload data. 790 * @src_size: The amount of data mapped by @src_dma_list. 791 * @iv: The IV vector. 792 * @iv_size: The size (in bytes) of @iv. 793 * 794 * Return: 0 on success, negative error code otherwise. 795 */ 796 int ocs_aes_op(struct ocs_aes_dev *aes_dev, 797 enum ocs_mode mode, 798 enum ocs_cipher cipher, 799 enum ocs_instruction instruction, 800 dma_addr_t dst_dma_list, 801 dma_addr_t src_dma_list, 802 u32 src_size, 803 u8 *iv, 804 u32 iv_size) 805 { 806 u32 *iv32; 807 int rc; 808 809 rc = ocs_aes_validate_inputs(src_dma_list, src_size, iv, iv_size, 0, 0, 810 NULL, 0, cipher, mode, instruction, 811 dst_dma_list); 812 if (rc) 813 return rc; 814 /* 815 * ocs_aes_validate_inputs() is a generic check, now ensure mode is not 816 * GCM or CCM. 817 */ 818 if (mode == OCS_MODE_GCM || mode == OCS_MODE_CCM) 819 return -EINVAL; 820 821 /* Cast IV to u32 array. */ 822 iv32 = (u32 *)iv; 823 824 ocs_aes_init(aes_dev, mode, cipher, instruction); 825 826 if (mode == OCS_MODE_CTS) { 827 /* Write the byte length of the last data block to engine. */ 828 ocs_aes_write_last_data_blk_len(aes_dev, src_size); 829 } 830 831 /* ECB is the only mode that doesn't use IV. */ 832 if (mode != OCS_MODE_ECB) { 833 iowrite32(iv32[0], aes_dev->base_reg + AES_IV_0_OFFSET); 834 iowrite32(iv32[1], aes_dev->base_reg + AES_IV_1_OFFSET); 835 iowrite32(iv32[2], aes_dev->base_reg + AES_IV_2_OFFSET); 836 iowrite32(iv32[3], aes_dev->base_reg + AES_IV_3_OFFSET); 837 } 838 839 /* Set AES_ACTIVE.TRIGGER to start the operation. */ 840 aes_a_op_trigger(aes_dev); 841 842 /* Configure and activate input / output DMA. */ 843 dma_to_ocs_aes_ll(aes_dev, src_dma_list); 844 dma_from_ocs_aes_ll(aes_dev, dst_dma_list); 845 aes_a_dma_active_src_dst_ll_en(aes_dev); 846 847 if (mode == OCS_MODE_CTS) { 848 /* 849 * For CTS mode, instruct engine to activate ciphertext 850 * stealing if last block of data is incomplete. 851 */ 852 aes_a_set_last_gcx(aes_dev); 853 } else { 854 /* For all other modes, just write the 'termination' bit. */ 855 aes_a_op_termination(aes_dev); 856 } 857 858 /* Wait for engine to complete processing. */ 859 rc = ocs_aes_irq_enable_and_wait(aes_dev, AES_COMPLETE_INT); 860 if (rc) 861 return rc; 862 863 if (mode == OCS_MODE_CTR) { 864 /* Read back IV for streaming mode */ 865 iv32[0] = ioread32(aes_dev->base_reg + AES_IV_0_OFFSET); 866 iv32[1] = ioread32(aes_dev->base_reg + AES_IV_1_OFFSET); 867 iv32[2] = ioread32(aes_dev->base_reg + AES_IV_2_OFFSET); 868 iv32[3] = ioread32(aes_dev->base_reg + AES_IV_3_OFFSET); 869 } 870 871 return 0; 872 } 873 874 /* Compute and write J0 to engine registers. */ 875 static void ocs_aes_gcm_write_j0(const struct ocs_aes_dev *aes_dev, 876 const u8 *iv) 877 { 878 const u32 *j0 = (u32 *)iv; 879 880 /* 881 * IV must be 12 bytes; Other sizes not supported as Linux crypto API 882 * does only expects/allows 12 byte IV for GCM 883 */ 884 iowrite32(0x00000001, aes_dev->base_reg + AES_IV_0_OFFSET); 885 iowrite32(__swab32(j0[2]), aes_dev->base_reg + AES_IV_1_OFFSET); 886 iowrite32(__swab32(j0[1]), aes_dev->base_reg + AES_IV_2_OFFSET); 887 iowrite32(__swab32(j0[0]), aes_dev->base_reg + AES_IV_3_OFFSET); 888 } 889 890 /* Read GCM tag from engine registers. */ 891 static inline void ocs_aes_gcm_read_tag(struct ocs_aes_dev *aes_dev, 892 u8 *tag, u32 tag_size) 893 { 894 u32 tag_u32[AES_MAX_TAG_SIZE_U32]; 895 896 /* 897 * The Authentication Tag T is stored in Little Endian order in the 898 * registers with the most significant bytes stored from AES_T_MAC[3] 899 * downward. 900 */ 901 tag_u32[0] = __swab32(ioread32(aes_dev->base_reg + AES_T_MAC_3_OFFSET)); 902 tag_u32[1] = __swab32(ioread32(aes_dev->base_reg + AES_T_MAC_2_OFFSET)); 903 tag_u32[2] = __swab32(ioread32(aes_dev->base_reg + AES_T_MAC_1_OFFSET)); 904 tag_u32[3] = __swab32(ioread32(aes_dev->base_reg + AES_T_MAC_0_OFFSET)); 905 906 memcpy(tag, tag_u32, tag_size); 907 } 908 909 /** 910 * ocs_aes_gcm_op() - Perform GCM operation. 911 * @aes_dev: The OCS AES device to use. 912 * @cipher: The Cipher to use (AES or SM4). 913 * @instruction: The instruction to perform (encrypt or decrypt). 914 * @dst_dma_list: The OCS DMA list mapping output memory. 915 * @src_dma_list: The OCS DMA list mapping input payload data. 916 * @src_size: The amount of data mapped by @src_dma_list. 917 * @iv: The input IV vector. 918 * @aad_dma_list: The OCS DMA list mapping input AAD data. 919 * @aad_size: The amount of data mapped by @aad_dma_list. 920 * @out_tag: Where to store computed tag. 921 * @tag_size: The size (in bytes) of @out_tag. 922 * 923 * Return: 0 on success, negative error code otherwise. 924 */ 925 int ocs_aes_gcm_op(struct ocs_aes_dev *aes_dev, 926 enum ocs_cipher cipher, 927 enum ocs_instruction instruction, 928 dma_addr_t dst_dma_list, 929 dma_addr_t src_dma_list, 930 u32 src_size, 931 const u8 *iv, 932 dma_addr_t aad_dma_list, 933 u32 aad_size, 934 u8 *out_tag, 935 u32 tag_size) 936 { 937 u64 bit_len; 938 u32 val; 939 int rc; 940 941 rc = ocs_aes_validate_inputs(src_dma_list, src_size, iv, 942 GCM_AES_IV_SIZE, aad_dma_list, 943 aad_size, out_tag, tag_size, cipher, 944 OCS_MODE_GCM, instruction, 945 dst_dma_list); 946 if (rc) 947 return rc; 948 949 ocs_aes_init(aes_dev, OCS_MODE_GCM, cipher, instruction); 950 951 /* Compute and write J0 to OCS HW. */ 952 ocs_aes_gcm_write_j0(aes_dev, iv); 953 954 /* Write out_tag byte length */ 955 iowrite32(tag_size, aes_dev->base_reg + AES_TLEN_OFFSET); 956 957 /* Write the byte length of the last plaintext / ciphertext block. */ 958 ocs_aes_write_last_data_blk_len(aes_dev, src_size); 959 960 /* Write ciphertext bit length */ 961 bit_len = (u64)src_size * 8; 962 val = bit_len & 0xFFFFFFFF; 963 iowrite32(val, aes_dev->base_reg + AES_MULTIPURPOSE2_0_OFFSET); 964 val = bit_len >> 32; 965 iowrite32(val, aes_dev->base_reg + AES_MULTIPURPOSE2_1_OFFSET); 966 967 /* Write aad bit length */ 968 bit_len = (u64)aad_size * 8; 969 val = bit_len & 0xFFFFFFFF; 970 iowrite32(val, aes_dev->base_reg + AES_MULTIPURPOSE2_2_OFFSET); 971 val = bit_len >> 32; 972 iowrite32(val, aes_dev->base_reg + AES_MULTIPURPOSE2_3_OFFSET); 973 974 /* Set AES_ACTIVE.TRIGGER to start the operation. */ 975 aes_a_op_trigger(aes_dev); 976 977 /* Process AAD. */ 978 if (aad_size) { 979 /* If aad present, configure DMA to feed it to the engine. */ 980 dma_to_ocs_aes_ll(aes_dev, aad_dma_list); 981 aes_a_dma_active_src_ll_en(aes_dev); 982 983 /* Instructs engine to pad last block of aad, if needed. */ 984 aes_a_set_last_gcx_and_adata(aes_dev); 985 986 /* Wait for DMA transfer to complete. */ 987 rc = ocs_aes_irq_enable_and_wait(aes_dev, AES_DMA_SRC_DONE_INT); 988 if (rc) 989 return rc; 990 } else { 991 aes_a_set_last_gcx_and_adata(aes_dev); 992 } 993 994 /* Wait until adata (if present) has been processed. */ 995 aes_a_wait_last_gcx(aes_dev); 996 aes_a_dma_wait_input_buffer_occupancy(aes_dev); 997 998 /* Now process payload. */ 999 if (src_size) { 1000 /* Configure and activate DMA for both input and output data. */ 1001 dma_to_ocs_aes_ll(aes_dev, src_dma_list); 1002 dma_from_ocs_aes_ll(aes_dev, dst_dma_list); 1003 aes_a_dma_active_src_dst_ll_en(aes_dev); 1004 } else { 1005 aes_a_dma_set_xfer_size_zero(aes_dev); 1006 aes_a_dma_active(aes_dev); 1007 } 1008 1009 /* Instruct AES/SMA4 engine payload processing is over. */ 1010 aes_a_set_last_gcx(aes_dev); 1011 1012 /* Wait for OCS AES engine to complete processing. */ 1013 rc = ocs_aes_irq_enable_and_wait(aes_dev, AES_COMPLETE_INT); 1014 if (rc) 1015 return rc; 1016 1017 ocs_aes_gcm_read_tag(aes_dev, out_tag, tag_size); 1018 1019 return 0; 1020 } 1021 1022 /* Write encrypted tag to AES/SM4 engine. */ 1023 static void ocs_aes_ccm_write_encrypted_tag(struct ocs_aes_dev *aes_dev, 1024 const u8 *in_tag, u32 tag_size) 1025 { 1026 int i; 1027 1028 /* Ensure DMA input buffer is empty */ 1029 aes_a_dma_wait_input_buffer_occupancy(aes_dev); 1030 1031 /* 1032 * During CCM decrypt, the OCS block needs to finish processing the 1033 * ciphertext before the tag is written. So delay needed after DMA has 1034 * completed writing the ciphertext 1035 */ 1036 aes_a_dma_reset_and_activate_perf_cntr(aes_dev); 1037 aes_a_dma_wait_and_deactivate_perf_cntr(aes_dev, 1038 CCM_DECRYPT_DELAY_TAG_CLK_COUNT); 1039 1040 /* Write encrypted tag to AES/SM4 engine. */ 1041 for (i = 0; i < tag_size; i++) { 1042 iowrite8(in_tag[i], aes_dev->base_reg + 1043 AES_A_DMA_INBUFFER_WRITE_FIFO_OFFSET); 1044 } 1045 } 1046 1047 /* 1048 * Write B0 CCM block to OCS AES HW. 1049 * 1050 * Note: B0 format is documented in NIST Special Publication 800-38C 1051 * https://nvlpubs.nist.gov/nistpubs/Legacy/SP/nistspecialpublication800-38c.pdf 1052 * (see Section A.2.1) 1053 */ 1054 static int ocs_aes_ccm_write_b0(const struct ocs_aes_dev *aes_dev, 1055 const u8 *iv, u32 adata_size, u32 tag_size, 1056 u32 cryptlen) 1057 { 1058 u8 b0[16]; /* CCM B0 block is 16 bytes long. */ 1059 int i, q; 1060 1061 /* Initialize B0 to 0. */ 1062 memset(b0, 0, sizeof(b0)); 1063 1064 /* 1065 * B0[0] is the 'Flags Octet' and has the following structure: 1066 * bit 7: Reserved 1067 * bit 6: Adata flag 1068 * bit 5-3: t value encoded as (t-2)/2 1069 * bit 2-0: q value encoded as q - 1 1070 */ 1071 /* If there is AAD data, set the Adata flag. */ 1072 if (adata_size) 1073 b0[0] |= BIT(6); 1074 /* 1075 * t denotes the octet length of T. 1076 * t can only be an element of { 4, 6, 8, 10, 12, 14, 16} and is 1077 * encoded as (t - 2) / 2 1078 */ 1079 b0[0] |= (((tag_size - 2) / 2) & 0x7) << 3; 1080 /* 1081 * q is the octet length of Q. 1082 * q can only be an element of {2, 3, 4, 5, 6, 7, 8} and is encoded as 1083 * q - 1 == iv[0] & 0x7; 1084 */ 1085 b0[0] |= iv[0] & 0x7; 1086 /* 1087 * Copy the Nonce N from IV to B0; N is located in iv[1]..iv[15 - q] 1088 * and must be copied to b0[1]..b0[15-q]. 1089 * q == (iv[0] & 0x7) + 1 1090 */ 1091 q = (iv[0] & 0x7) + 1; 1092 for (i = 1; i <= 15 - q; i++) 1093 b0[i] = iv[i]; 1094 /* 1095 * The rest of B0 must contain Q, i.e., the message length. 1096 * Q is encoded in q octets, in big-endian order, so to write it, we 1097 * start from the end of B0 and we move backward. 1098 */ 1099 i = sizeof(b0) - 1; 1100 while (q) { 1101 b0[i] = cryptlen & 0xff; 1102 cryptlen >>= 8; 1103 i--; 1104 q--; 1105 } 1106 /* 1107 * If cryptlen is not zero at this point, it means that its original 1108 * value was too big. 1109 */ 1110 if (cryptlen) 1111 return -EOVERFLOW; 1112 /* Now write B0 to OCS AES input buffer. */ 1113 for (i = 0; i < sizeof(b0); i++) 1114 iowrite8(b0[i], aes_dev->base_reg + 1115 AES_A_DMA_INBUFFER_WRITE_FIFO_OFFSET); 1116 return 0; 1117 } 1118 1119 /* 1120 * Write adata length to OCS AES HW. 1121 * 1122 * Note: adata len encoding is documented in NIST Special Publication 800-38C 1123 * https://nvlpubs.nist.gov/nistpubs/Legacy/SP/nistspecialpublication800-38c.pdf 1124 * (see Section A.2.2) 1125 */ 1126 static void ocs_aes_ccm_write_adata_len(const struct ocs_aes_dev *aes_dev, 1127 u64 adata_len) 1128 { 1129 u8 enc_a[10]; /* Maximum encoded size: 10 octets. */ 1130 int i, len; 1131 1132 /* 1133 * adata_len ('a') is encoded as follows: 1134 * If 0 < a < 2^16 - 2^8 ==> 'a' encoded as [a]16, i.e., two octets 1135 * (big endian). 1136 * If 2^16 - 2^8 ≤ a < 2^32 ==> 'a' encoded as 0xff || 0xfe || [a]32, 1137 * i.e., six octets (big endian). 1138 * If 2^32 ≤ a < 2^64 ==> 'a' encoded as 0xff || 0xff || [a]64, 1139 * i.e., ten octets (big endian). 1140 */ 1141 if (adata_len < 65280) { 1142 len = 2; 1143 *(__be16 *)enc_a = cpu_to_be16(adata_len); 1144 } else if (adata_len <= 0xFFFFFFFF) { 1145 len = 6; 1146 *(__be16 *)enc_a = cpu_to_be16(0xfffe); 1147 *(__be32 *)&enc_a[2] = cpu_to_be32(adata_len); 1148 } else { /* adata_len >= 2^32 */ 1149 len = 10; 1150 *(__be16 *)enc_a = cpu_to_be16(0xffff); 1151 *(__be64 *)&enc_a[2] = cpu_to_be64(adata_len); 1152 } 1153 for (i = 0; i < len; i++) 1154 iowrite8(enc_a[i], 1155 aes_dev->base_reg + 1156 AES_A_DMA_INBUFFER_WRITE_FIFO_OFFSET); 1157 } 1158 1159 static int ocs_aes_ccm_do_adata(struct ocs_aes_dev *aes_dev, 1160 dma_addr_t adata_dma_list, u32 adata_size) 1161 { 1162 int rc; 1163 1164 if (!adata_size) { 1165 /* Since no aad the LAST_GCX bit can be set now */ 1166 aes_a_set_last_gcx_and_adata(aes_dev); 1167 goto exit; 1168 } 1169 1170 /* Adata case. */ 1171 1172 /* 1173 * Form the encoding of the Associated data length and write it 1174 * to the AES/SM4 input buffer. 1175 */ 1176 ocs_aes_ccm_write_adata_len(aes_dev, adata_size); 1177 1178 /* Configure the AES/SM4 DMA to fetch the Associated Data */ 1179 dma_to_ocs_aes_ll(aes_dev, adata_dma_list); 1180 1181 /* Activate DMA to fetch Associated data. */ 1182 aes_a_dma_active_src_ll_en(aes_dev); 1183 1184 /* Set LAST_GCX and LAST_ADATA in AES ACTIVE register. */ 1185 aes_a_set_last_gcx_and_adata(aes_dev); 1186 1187 /* Wait for DMA transfer to complete. */ 1188 rc = ocs_aes_irq_enable_and_wait(aes_dev, AES_DMA_SRC_DONE_INT); 1189 if (rc) 1190 return rc; 1191 1192 exit: 1193 /* Wait until adata (if present) has been processed. */ 1194 aes_a_wait_last_gcx(aes_dev); 1195 aes_a_dma_wait_input_buffer_occupancy(aes_dev); 1196 1197 return 0; 1198 } 1199 1200 static int ocs_aes_ccm_encrypt_do_payload(struct ocs_aes_dev *aes_dev, 1201 dma_addr_t dst_dma_list, 1202 dma_addr_t src_dma_list, 1203 u32 src_size) 1204 { 1205 if (src_size) { 1206 /* 1207 * Configure and activate DMA for both input and output 1208 * data. 1209 */ 1210 dma_to_ocs_aes_ll(aes_dev, src_dma_list); 1211 dma_from_ocs_aes_ll(aes_dev, dst_dma_list); 1212 aes_a_dma_active_src_dst_ll_en(aes_dev); 1213 } else { 1214 /* Configure and activate DMA for output data only. */ 1215 dma_from_ocs_aes_ll(aes_dev, dst_dma_list); 1216 aes_a_dma_active_dst_ll_en(aes_dev); 1217 } 1218 1219 /* 1220 * Set the LAST GCX bit in AES_ACTIVE Register to instruct 1221 * AES/SM4 engine to pad the last block of data. 1222 */ 1223 aes_a_set_last_gcx(aes_dev); 1224 1225 /* We are done, wait for IRQ and return. */ 1226 return ocs_aes_irq_enable_and_wait(aes_dev, AES_COMPLETE_INT); 1227 } 1228 1229 static int ocs_aes_ccm_decrypt_do_payload(struct ocs_aes_dev *aes_dev, 1230 dma_addr_t dst_dma_list, 1231 dma_addr_t src_dma_list, 1232 u32 src_size) 1233 { 1234 if (!src_size) { 1235 /* Let engine process 0-length input. */ 1236 aes_a_dma_set_xfer_size_zero(aes_dev); 1237 aes_a_dma_active(aes_dev); 1238 aes_a_set_last_gcx(aes_dev); 1239 1240 return 0; 1241 } 1242 1243 /* 1244 * Configure and activate DMA for both input and output 1245 * data. 1246 */ 1247 dma_to_ocs_aes_ll(aes_dev, src_dma_list); 1248 dma_from_ocs_aes_ll(aes_dev, dst_dma_list); 1249 aes_a_dma_active_src_dst_ll_en(aes_dev); 1250 /* 1251 * Set the LAST GCX bit in AES_ACTIVE Register; this allows the 1252 * AES/SM4 engine to differentiate between encrypted data and 1253 * encrypted MAC. 1254 */ 1255 aes_a_set_last_gcx(aes_dev); 1256 /* 1257 * Enable DMA DONE interrupt; once DMA transfer is over, 1258 * interrupt handler will process the MAC/tag. 1259 */ 1260 return ocs_aes_irq_enable_and_wait(aes_dev, AES_DMA_SRC_DONE_INT); 1261 } 1262 1263 /* 1264 * Compare Tag to Yr. 1265 * 1266 * Only used at the end of CCM decrypt. If tag == yr, message authentication 1267 * has succeeded. 1268 */ 1269 static inline int ccm_compare_tag_to_yr(struct ocs_aes_dev *aes_dev, 1270 u8 tag_size_bytes) 1271 { 1272 u32 tag[AES_MAX_TAG_SIZE_U32]; 1273 u32 yr[AES_MAX_TAG_SIZE_U32]; 1274 u8 i; 1275 1276 /* Read Tag and Yr from AES registers. */ 1277 for (i = 0; i < AES_MAX_TAG_SIZE_U32; i++) { 1278 tag[i] = ioread32(aes_dev->base_reg + 1279 AES_T_MAC_0_OFFSET + (i * sizeof(u32))); 1280 yr[i] = ioread32(aes_dev->base_reg + 1281 AES_MULTIPURPOSE2_0_OFFSET + 1282 (i * sizeof(u32))); 1283 } 1284 1285 return memcmp(tag, yr, tag_size_bytes) ? -EBADMSG : 0; 1286 } 1287 1288 /** 1289 * ocs_aes_ccm_op() - Perform CCM operation. 1290 * @aes_dev: The OCS AES device to use. 1291 * @cipher: The Cipher to use (AES or SM4). 1292 * @instruction: The instruction to perform (encrypt or decrypt). 1293 * @dst_dma_list: The OCS DMA list mapping output memory. 1294 * @src_dma_list: The OCS DMA list mapping input payload data. 1295 * @src_size: The amount of data mapped by @src_dma_list. 1296 * @iv: The input IV vector. 1297 * @adata_dma_list: The OCS DMA list mapping input A-data. 1298 * @adata_size: The amount of data mapped by @adata_dma_list. 1299 * @in_tag: Input tag. 1300 * @tag_size: The size (in bytes) of @in_tag. 1301 * 1302 * Note: for encrypt the tag is appended to the ciphertext (in the memory 1303 * mapped by @dst_dma_list). 1304 * 1305 * Return: 0 on success, negative error code otherwise. 1306 */ 1307 int ocs_aes_ccm_op(struct ocs_aes_dev *aes_dev, 1308 enum ocs_cipher cipher, 1309 enum ocs_instruction instruction, 1310 dma_addr_t dst_dma_list, 1311 dma_addr_t src_dma_list, 1312 u32 src_size, 1313 u8 *iv, 1314 dma_addr_t adata_dma_list, 1315 u32 adata_size, 1316 u8 *in_tag, 1317 u32 tag_size) 1318 { 1319 u32 *iv_32; 1320 u8 lprime; 1321 int rc; 1322 1323 rc = ocs_aes_validate_inputs(src_dma_list, src_size, iv, 1324 AES_BLOCK_SIZE, adata_dma_list, adata_size, 1325 in_tag, tag_size, cipher, OCS_MODE_CCM, 1326 instruction, dst_dma_list); 1327 if (rc) 1328 return rc; 1329 1330 ocs_aes_init(aes_dev, OCS_MODE_CCM, cipher, instruction); 1331 1332 /* 1333 * Note: rfc 3610 and NIST 800-38C require counter of zero to encrypt 1334 * auth tag so ensure this is the case 1335 */ 1336 lprime = iv[L_PRIME_IDX]; 1337 memset(&iv[COUNTER_START(lprime)], 0, COUNTER_LEN(lprime)); 1338 1339 /* 1340 * Nonce is already converted to ctr0 before being passed into this 1341 * function as iv. 1342 */ 1343 iv_32 = (u32 *)iv; 1344 iowrite32(__swab32(iv_32[0]), 1345 aes_dev->base_reg + AES_MULTIPURPOSE1_3_OFFSET); 1346 iowrite32(__swab32(iv_32[1]), 1347 aes_dev->base_reg + AES_MULTIPURPOSE1_2_OFFSET); 1348 iowrite32(__swab32(iv_32[2]), 1349 aes_dev->base_reg + AES_MULTIPURPOSE1_1_OFFSET); 1350 iowrite32(__swab32(iv_32[3]), 1351 aes_dev->base_reg + AES_MULTIPURPOSE1_0_OFFSET); 1352 1353 /* Write MAC/tag length in register AES_TLEN */ 1354 iowrite32(tag_size, aes_dev->base_reg + AES_TLEN_OFFSET); 1355 /* 1356 * Write the byte length of the last AES/SM4 block of Payload data 1357 * (without zero padding and without the length of the MAC) in register 1358 * AES_PLEN. 1359 */ 1360 ocs_aes_write_last_data_blk_len(aes_dev, src_size); 1361 1362 /* Set AES_ACTIVE.TRIGGER to start the operation. */ 1363 aes_a_op_trigger(aes_dev); 1364 1365 aes_a_dma_reset_and_activate_perf_cntr(aes_dev); 1366 1367 /* Form block B0 and write it to the AES/SM4 input buffer. */ 1368 rc = ocs_aes_ccm_write_b0(aes_dev, iv, adata_size, tag_size, src_size); 1369 if (rc) 1370 return rc; 1371 /* 1372 * Ensure there has been at least CCM_DECRYPT_DELAY_LAST_GCX_CLK_COUNT 1373 * clock cycles since TRIGGER bit was set 1374 */ 1375 aes_a_dma_wait_and_deactivate_perf_cntr(aes_dev, 1376 CCM_DECRYPT_DELAY_LAST_GCX_CLK_COUNT); 1377 1378 /* Process Adata. */ 1379 ocs_aes_ccm_do_adata(aes_dev, adata_dma_list, adata_size); 1380 1381 /* For Encrypt case we just process the payload and return. */ 1382 if (instruction == OCS_ENCRYPT) { 1383 return ocs_aes_ccm_encrypt_do_payload(aes_dev, dst_dma_list, 1384 src_dma_list, src_size); 1385 } 1386 /* For Decypt we need to process the payload and then the tag. */ 1387 rc = ocs_aes_ccm_decrypt_do_payload(aes_dev, dst_dma_list, 1388 src_dma_list, src_size); 1389 if (rc) 1390 return rc; 1391 1392 /* Process MAC/tag directly: feed tag to engine and wait for IRQ. */ 1393 ocs_aes_ccm_write_encrypted_tag(aes_dev, in_tag, tag_size); 1394 rc = ocs_aes_irq_enable_and_wait(aes_dev, AES_COMPLETE_INT); 1395 if (rc) 1396 return rc; 1397 1398 return ccm_compare_tag_to_yr(aes_dev, tag_size); 1399 } 1400 1401 /** 1402 * ocs_create_linked_list_from_sg() - Create OCS DMA linked list from SG list. 1403 * @aes_dev: The OCS AES device the list will be created for. 1404 * @sg: The SG list OCS DMA linked list will be created from. When 1405 * passed to this function, @sg must have been already mapped 1406 * with dma_map_sg(). 1407 * @sg_dma_count: The number of DMA-mapped entries in @sg. This must be the 1408 * value returned by dma_map_sg() when @sg was mapped. 1409 * @dll_desc: The OCS DMA dma_list to use to store information about the 1410 * created linked list. 1411 * @data_size: The size of the data (from the SG list) to be mapped into the 1412 * OCS DMA linked list. 1413 * @data_offset: The offset (within the SG list) of the data to be mapped. 1414 * 1415 * Return: 0 on success, negative error code otherwise. 1416 */ 1417 int ocs_create_linked_list_from_sg(const struct ocs_aes_dev *aes_dev, 1418 struct scatterlist *sg, 1419 int sg_dma_count, 1420 struct ocs_dll_desc *dll_desc, 1421 size_t data_size, size_t data_offset) 1422 { 1423 struct ocs_dma_linked_list *ll = NULL; 1424 struct scatterlist *sg_tmp; 1425 unsigned int tmp; 1426 int dma_nents; 1427 int i; 1428 1429 if (!dll_desc || !sg || !aes_dev) 1430 return -EINVAL; 1431 1432 /* Default values for when no ddl_desc is created. */ 1433 dll_desc->vaddr = NULL; 1434 dll_desc->dma_addr = DMA_MAPPING_ERROR; 1435 dll_desc->size = 0; 1436 1437 if (data_size == 0) 1438 return 0; 1439 1440 /* Loop over sg_list until we reach entry at specified offset. */ 1441 while (data_offset >= sg_dma_len(sg)) { 1442 data_offset -= sg_dma_len(sg); 1443 sg_dma_count--; 1444 sg = sg_next(sg); 1445 /* If we reach the end of the list, offset was invalid. */ 1446 if (!sg || sg_dma_count == 0) 1447 return -EINVAL; 1448 } 1449 1450 /* Compute number of DMA-mapped SG entries to add into OCS DMA list. */ 1451 dma_nents = 0; 1452 tmp = 0; 1453 sg_tmp = sg; 1454 while (tmp < data_offset + data_size) { 1455 /* If we reach the end of the list, data_size was invalid. */ 1456 if (!sg_tmp) 1457 return -EINVAL; 1458 tmp += sg_dma_len(sg_tmp); 1459 dma_nents++; 1460 sg_tmp = sg_next(sg_tmp); 1461 } 1462 if (dma_nents > sg_dma_count) 1463 return -EINVAL; 1464 1465 /* Allocate the DMA list, one entry for each SG entry. */ 1466 dll_desc->size = sizeof(struct ocs_dma_linked_list) * dma_nents; 1467 dll_desc->vaddr = dma_alloc_coherent(aes_dev->dev, dll_desc->size, 1468 &dll_desc->dma_addr, GFP_KERNEL); 1469 if (!dll_desc->vaddr) 1470 return -ENOMEM; 1471 1472 /* Populate DMA linked list entries. */ 1473 ll = dll_desc->vaddr; 1474 for (i = 0; i < dma_nents; i++, sg = sg_next(sg)) { 1475 ll[i].src_addr = sg_dma_address(sg) + data_offset; 1476 ll[i].src_len = (sg_dma_len(sg) - data_offset) < data_size ? 1477 (sg_dma_len(sg) - data_offset) : data_size; 1478 data_offset = 0; 1479 data_size -= ll[i].src_len; 1480 /* Current element points to the DMA address of the next one. */ 1481 ll[i].next = dll_desc->dma_addr + (sizeof(*ll) * (i + 1)); 1482 ll[i].ll_flags = 0; 1483 } 1484 /* Terminate last element. */ 1485 ll[i - 1].next = 0; 1486 ll[i - 1].ll_flags = OCS_LL_DMA_FLAG_TERMINATE; 1487 1488 return 0; 1489 } 1490