1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * K3 SA2UL crypto accelerator driver 4 * 5 * Copyright (C) 2018-2020 Texas Instruments Incorporated - http://www.ti.com 6 * 7 * Authors: Keerthy 8 * Vitaly Andrianov 9 * Tero Kristo 10 */ 11 #include <linux/clk.h> 12 #include <linux/dmaengine.h> 13 #include <linux/dmapool.h> 14 #include <linux/module.h> 15 #include <linux/of_device.h> 16 #include <linux/platform_device.h> 17 #include <linux/pm_runtime.h> 18 19 #include <crypto/aes.h> 20 #include <crypto/authenc.h> 21 #include <crypto/des.h> 22 #include <crypto/internal/aead.h> 23 #include <crypto/internal/hash.h> 24 #include <crypto/internal/skcipher.h> 25 #include <crypto/scatterwalk.h> 26 #include <crypto/sha.h> 27 28 #include "sa2ul.h" 29 30 /* Byte offset for key in encryption security context */ 31 #define SC_ENC_KEY_OFFSET (1 + 27 + 4) 32 /* Byte offset for Aux-1 in encryption security context */ 33 #define SC_ENC_AUX1_OFFSET (1 + 27 + 4 + 32) 34 35 #define SA_CMDL_UPD_ENC 0x0001 36 #define SA_CMDL_UPD_AUTH 0x0002 37 #define SA_CMDL_UPD_ENC_IV 0x0004 38 #define SA_CMDL_UPD_AUTH_IV 0x0008 39 #define SA_CMDL_UPD_AUX_KEY 0x0010 40 41 #define SA_AUTH_SUBKEY_LEN 16 42 #define SA_CMDL_PAYLOAD_LENGTH_MASK 0xFFFF 43 #define SA_CMDL_SOP_BYPASS_LEN_MASK 0xFF000000 44 45 #define MODE_CONTROL_BYTES 27 46 #define SA_HASH_PROCESSING 0 47 #define SA_CRYPTO_PROCESSING 0 48 #define SA_UPLOAD_HASH_TO_TLR BIT(6) 49 50 #define SA_SW0_FLAGS_MASK 0xF0000 51 #define SA_SW0_CMDL_INFO_MASK 0x1F00000 52 #define SA_SW0_CMDL_PRESENT BIT(4) 53 #define SA_SW0_ENG_ID_MASK 0x3E000000 54 #define SA_SW0_DEST_INFO_PRESENT BIT(30) 55 #define SA_SW2_EGRESS_LENGTH 0xFF000000 56 #define SA_BASIC_HASH 0x10 57 58 #define SHA256_DIGEST_WORDS 8 59 /* Make 32-bit word from 4 bytes */ 60 #define SA_MK_U32(b0, b1, b2, b3) (((b0) << 24) | ((b1) << 16) | \ 61 ((b2) << 8) | (b3)) 62 63 /* size of SCCTL structure in bytes */ 64 #define SA_SCCTL_SZ 16 65 66 /* Max Authentication tag size */ 67 #define SA_MAX_AUTH_TAG_SZ 64 68 69 #define PRIV_ID 0x1 70 #define PRIV 0x1 71 72 static struct device *sa_k3_dev; 73 74 /** 75 * struct sa_cmdl_cfg - Command label configuration descriptor 76 * @aalg: authentication algorithm ID 77 * @enc_eng_id: Encryption Engine ID supported by the SA hardware 78 * @auth_eng_id: Authentication Engine ID 79 * @iv_size: Initialization Vector size 80 * @akey: Authentication key 81 * @akey_len: Authentication key length 82 * @enc: True, if this is an encode request 83 */ 84 struct sa_cmdl_cfg { 85 int aalg; 86 u8 enc_eng_id; 87 u8 auth_eng_id; 88 u8 iv_size; 89 const u8 *akey; 90 u16 akey_len; 91 bool enc; 92 }; 93 94 /** 95 * struct algo_data - Crypto algorithm specific data 96 * @enc_eng: Encryption engine info structure 97 * @auth_eng: Authentication engine info structure 98 * @auth_ctrl: Authentication control word 99 * @hash_size: Size of digest 100 * @iv_idx: iv index in psdata 101 * @iv_out_size: iv out size 102 * @ealg_id: Encryption Algorithm ID 103 * @aalg_id: Authentication algorithm ID 104 * @mci_enc: Mode Control Instruction for Encryption algorithm 105 * @mci_dec: Mode Control Instruction for Decryption 106 * @inv_key: Whether the encryption algorithm demands key inversion 107 * @ctx: Pointer to the algorithm context 108 * @keyed_mac: Whether the authentication algorithm has key 109 * @prep_iopad: Function pointer to generate intermediate ipad/opad 110 */ 111 struct algo_data { 112 struct sa_eng_info enc_eng; 113 struct sa_eng_info auth_eng; 114 u8 auth_ctrl; 115 u8 hash_size; 116 u8 iv_idx; 117 u8 iv_out_size; 118 u8 ealg_id; 119 u8 aalg_id; 120 u8 *mci_enc; 121 u8 *mci_dec; 122 bool inv_key; 123 struct sa_tfm_ctx *ctx; 124 bool keyed_mac; 125 void (*prep_iopad)(struct algo_data *algo, const u8 *key, 126 u16 key_sz, __be32 *ipad, __be32 *opad); 127 }; 128 129 /** 130 * struct sa_alg_tmpl: A generic template encompassing crypto/aead algorithms 131 * @type: Type of the crypto algorithm. 132 * @alg: Union of crypto algorithm definitions. 133 * @registered: Flag indicating if the crypto algorithm is already registered 134 */ 135 struct sa_alg_tmpl { 136 u32 type; /* CRYPTO_ALG_TYPE from <linux/crypto.h> */ 137 union { 138 struct skcipher_alg skcipher; 139 struct ahash_alg ahash; 140 struct aead_alg aead; 141 } alg; 142 bool registered; 143 }; 144 145 /** 146 * struct sa_rx_data: RX Packet miscellaneous data place holder 147 * @req: crypto request data pointer 148 * @ddev: pointer to the DMA device 149 * @tx_in: dma_async_tx_descriptor pointer for rx channel 150 * @split_src_sg: Set if the src sg is split and needs to be freed up 151 * @split_dst_sg: Set if the dst sg is split and needs to be freed up 152 * @enc: Flag indicating either encryption or decryption 153 * @enc_iv_size: Initialisation vector size 154 * @iv_idx: Initialisation vector index 155 * @rx_sg: Static scatterlist entry for overriding RX data 156 * @tx_sg: Static scatterlist entry for overriding TX data 157 * @src: Source data pointer 158 * @dst: Destination data pointer 159 */ 160 struct sa_rx_data { 161 void *req; 162 struct device *ddev; 163 struct dma_async_tx_descriptor *tx_in; 164 struct scatterlist *split_src_sg; 165 struct scatterlist *split_dst_sg; 166 u8 enc; 167 u8 enc_iv_size; 168 u8 iv_idx; 169 struct scatterlist rx_sg; 170 struct scatterlist tx_sg; 171 struct scatterlist *src; 172 struct scatterlist *dst; 173 }; 174 175 /** 176 * struct sa_req: SA request definition 177 * @dev: device for the request 178 * @size: total data to the xmitted via DMA 179 * @enc_offset: offset of cipher data 180 * @enc_size: data to be passed to cipher engine 181 * @enc_iv: cipher IV 182 * @auth_offset: offset of the authentication data 183 * @auth_size: size of the authentication data 184 * @auth_iv: authentication IV 185 * @type: algorithm type for the request 186 * @cmdl: command label pointer 187 * @base: pointer to the base request 188 * @ctx: pointer to the algorithm context data 189 * @enc: true if this is an encode request 190 * @src: source data 191 * @dst: destination data 192 * @callback: DMA callback for the request 193 * @mdata_size: metadata size passed to DMA 194 */ 195 struct sa_req { 196 struct device *dev; 197 u16 size; 198 u8 enc_offset; 199 u16 enc_size; 200 u8 *enc_iv; 201 u8 auth_offset; 202 u16 auth_size; 203 u8 *auth_iv; 204 u32 type; 205 u32 *cmdl; 206 struct crypto_async_request *base; 207 struct sa_tfm_ctx *ctx; 208 bool enc; 209 struct scatterlist *src; 210 struct scatterlist *dst; 211 dma_async_tx_callback callback; 212 u16 mdata_size; 213 }; 214 215 /* 216 * Mode Control Instructions for various Key lengths 128, 192, 256 217 * For CBC (Cipher Block Chaining) mode for encryption 218 */ 219 static u8 mci_cbc_enc_array[3][MODE_CONTROL_BYTES] = { 220 { 0x61, 0x00, 0x00, 0x18, 0x88, 0x0a, 0xaa, 0x4b, 0x7e, 0x00, 221 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 222 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, 223 { 0x61, 0x00, 0x00, 0x18, 0x88, 0x4a, 0xaa, 0x4b, 0x7e, 0x00, 224 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 225 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, 226 { 0x61, 0x00, 0x00, 0x18, 0x88, 0x8a, 0xaa, 0x4b, 0x7e, 0x00, 227 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 228 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, 229 }; 230 231 /* 232 * Mode Control Instructions for various Key lengths 128, 192, 256 233 * For CBC (Cipher Block Chaining) mode for decryption 234 */ 235 static u8 mci_cbc_dec_array[3][MODE_CONTROL_BYTES] = { 236 { 0x71, 0x00, 0x00, 0x80, 0x8a, 0xca, 0x98, 0xf4, 0x40, 0xc0, 237 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 238 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, 239 { 0x71, 0x00, 0x00, 0x84, 0x8a, 0xca, 0x98, 0xf4, 0x40, 0xc0, 240 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 241 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, 242 { 0x71, 0x00, 0x00, 0x88, 0x8a, 0xca, 0x98, 0xf4, 0x40, 0xc0, 243 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 244 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, 245 }; 246 247 /* 248 * Mode Control Instructions for various Key lengths 128, 192, 256 249 * For CBC (Cipher Block Chaining) mode for encryption 250 */ 251 static u8 mci_cbc_enc_no_iv_array[3][MODE_CONTROL_BYTES] = { 252 { 0x21, 0x00, 0x00, 0x18, 0x88, 0x0a, 0xaa, 0x4b, 0x7e, 0x00, 253 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 254 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, 255 { 0x21, 0x00, 0x00, 0x18, 0x88, 0x4a, 0xaa, 0x4b, 0x7e, 0x00, 256 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 257 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, 258 { 0x21, 0x00, 0x00, 0x18, 0x88, 0x8a, 0xaa, 0x4b, 0x7e, 0x00, 259 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 260 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, 261 }; 262 263 /* 264 * Mode Control Instructions for various Key lengths 128, 192, 256 265 * For CBC (Cipher Block Chaining) mode for decryption 266 */ 267 static u8 mci_cbc_dec_no_iv_array[3][MODE_CONTROL_BYTES] = { 268 { 0x31, 0x00, 0x00, 0x80, 0x8a, 0xca, 0x98, 0xf4, 0x40, 0xc0, 269 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 270 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, 271 { 0x31, 0x00, 0x00, 0x84, 0x8a, 0xca, 0x98, 0xf4, 0x40, 0xc0, 272 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 273 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, 274 { 0x31, 0x00, 0x00, 0x88, 0x8a, 0xca, 0x98, 0xf4, 0x40, 0xc0, 275 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 276 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, 277 }; 278 279 /* 280 * Mode Control Instructions for various Key lengths 128, 192, 256 281 * For ECB (Electronic Code Book) mode for encryption 282 */ 283 static u8 mci_ecb_enc_array[3][27] = { 284 { 0x21, 0x00, 0x00, 0x80, 0x8a, 0x04, 0xb7, 0x90, 0x00, 0x00, 285 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 286 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, 287 { 0x21, 0x00, 0x00, 0x84, 0x8a, 0x04, 0xb7, 0x90, 0x00, 0x00, 288 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 289 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, 290 { 0x21, 0x00, 0x00, 0x88, 0x8a, 0x04, 0xb7, 0x90, 0x00, 0x00, 291 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 292 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, 293 }; 294 295 /* 296 * Mode Control Instructions for various Key lengths 128, 192, 256 297 * For ECB (Electronic Code Book) mode for decryption 298 */ 299 static u8 mci_ecb_dec_array[3][27] = { 300 { 0x31, 0x00, 0x00, 0x80, 0x8a, 0x04, 0xb7, 0x90, 0x00, 0x00, 301 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 302 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, 303 { 0x31, 0x00, 0x00, 0x84, 0x8a, 0x04, 0xb7, 0x90, 0x00, 0x00, 304 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 305 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, 306 { 0x31, 0x00, 0x00, 0x88, 0x8a, 0x04, 0xb7, 0x90, 0x00, 0x00, 307 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 308 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, 309 }; 310 311 /* 312 * Mode Control Instructions for DES algorithm 313 * For CBC (Cipher Block Chaining) mode and ECB mode 314 * encryption and for decryption respectively 315 */ 316 static u8 mci_cbc_3des_enc_array[MODE_CONTROL_BYTES] = { 317 0x60, 0x00, 0x00, 0x18, 0x88, 0x52, 0xaa, 0x4b, 0x7e, 0x00, 0x00, 0x00, 318 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 319 0x00, 0x00, 0x00, 320 }; 321 322 static u8 mci_cbc_3des_dec_array[MODE_CONTROL_BYTES] = { 323 0x70, 0x00, 0x00, 0x85, 0x0a, 0xca, 0x98, 0xf4, 0x40, 0xc0, 0x00, 0x00, 324 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 325 0x00, 0x00, 0x00, 326 }; 327 328 static u8 mci_ecb_3des_enc_array[MODE_CONTROL_BYTES] = { 329 0x20, 0x00, 0x00, 0x85, 0x0a, 0x04, 0xb7, 0x90, 0x00, 0x00, 0x00, 0x00, 330 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 331 0x00, 0x00, 0x00, 332 }; 333 334 static u8 mci_ecb_3des_dec_array[MODE_CONTROL_BYTES] = { 335 0x30, 0x00, 0x00, 0x85, 0x0a, 0x04, 0xb7, 0x90, 0x00, 0x00, 0x00, 0x00, 336 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 337 0x00, 0x00, 0x00, 338 }; 339 340 /* 341 * Perform 16 byte or 128 bit swizzling 342 * The SA2UL Expects the security context to 343 * be in little Endian and the bus width is 128 bits or 16 bytes 344 * Hence swap 16 bytes at a time from higher to lower address 345 */ 346 static void sa_swiz_128(u8 *in, u16 len) 347 { 348 u8 data[16]; 349 int i, j; 350 351 for (i = 0; i < len; i += 16) { 352 memcpy(data, &in[i], 16); 353 for (j = 0; j < 16; j++) 354 in[i + j] = data[15 - j]; 355 } 356 } 357 358 /* Prepare the ipad and opad from key as per SHA algorithm step 1*/ 359 static void prepare_kiopad(u8 *k_ipad, u8 *k_opad, const u8 *key, u16 key_sz) 360 { 361 int i; 362 363 for (i = 0; i < key_sz; i++) { 364 k_ipad[i] = key[i] ^ 0x36; 365 k_opad[i] = key[i] ^ 0x5c; 366 } 367 368 /* Instead of XOR with 0 */ 369 for (; i < SHA1_BLOCK_SIZE; i++) { 370 k_ipad[i] = 0x36; 371 k_opad[i] = 0x5c; 372 } 373 } 374 375 static void sa_export_shash(struct shash_desc *hash, int block_size, 376 int digest_size, __be32 *out) 377 { 378 union { 379 struct sha1_state sha1; 380 struct sha256_state sha256; 381 struct sha512_state sha512; 382 } sha; 383 void *state; 384 u32 *result; 385 int i; 386 387 switch (digest_size) { 388 case SHA1_DIGEST_SIZE: 389 state = &sha.sha1; 390 result = sha.sha1.state; 391 break; 392 case SHA256_DIGEST_SIZE: 393 state = &sha.sha256; 394 result = sha.sha256.state; 395 break; 396 default: 397 dev_err(sa_k3_dev, "%s: bad digest_size=%d\n", __func__, 398 digest_size); 399 return; 400 } 401 402 crypto_shash_export(hash, state); 403 404 for (i = 0; i < digest_size >> 2; i++) 405 out[i] = cpu_to_be32(result[i]); 406 } 407 408 static void sa_prepare_iopads(struct algo_data *data, const u8 *key, 409 u16 key_sz, __be32 *ipad, __be32 *opad) 410 { 411 SHASH_DESC_ON_STACK(shash, data->ctx->shash); 412 int block_size = crypto_shash_blocksize(data->ctx->shash); 413 int digest_size = crypto_shash_digestsize(data->ctx->shash); 414 u8 k_ipad[SHA1_BLOCK_SIZE]; 415 u8 k_opad[SHA1_BLOCK_SIZE]; 416 417 shash->tfm = data->ctx->shash; 418 419 prepare_kiopad(k_ipad, k_opad, key, key_sz); 420 421 memzero_explicit(ipad, block_size); 422 memzero_explicit(opad, block_size); 423 424 crypto_shash_init(shash); 425 crypto_shash_update(shash, k_ipad, block_size); 426 sa_export_shash(shash, block_size, digest_size, ipad); 427 428 crypto_shash_init(shash); 429 crypto_shash_update(shash, k_opad, block_size); 430 431 sa_export_shash(shash, block_size, digest_size, opad); 432 } 433 434 /* Derive the inverse key used in AES-CBC decryption operation */ 435 static inline int sa_aes_inv_key(u8 *inv_key, const u8 *key, u16 key_sz) 436 { 437 struct crypto_aes_ctx ctx; 438 int key_pos; 439 440 if (aes_expandkey(&ctx, key, key_sz)) { 441 dev_err(sa_k3_dev, "%s: bad key len(%d)\n", __func__, key_sz); 442 return -EINVAL; 443 } 444 445 /* work around to get the right inverse for AES_KEYSIZE_192 size keys */ 446 if (key_sz == AES_KEYSIZE_192) { 447 ctx.key_enc[52] = ctx.key_enc[51] ^ ctx.key_enc[46]; 448 ctx.key_enc[53] = ctx.key_enc[52] ^ ctx.key_enc[47]; 449 } 450 451 /* Based crypto_aes_expand_key logic */ 452 switch (key_sz) { 453 case AES_KEYSIZE_128: 454 case AES_KEYSIZE_192: 455 key_pos = key_sz + 24; 456 break; 457 458 case AES_KEYSIZE_256: 459 key_pos = key_sz + 24 - 4; 460 break; 461 462 default: 463 dev_err(sa_k3_dev, "%s: bad key len(%d)\n", __func__, key_sz); 464 return -EINVAL; 465 } 466 467 memcpy(inv_key, &ctx.key_enc[key_pos], key_sz); 468 return 0; 469 } 470 471 /* Set Security context for the encryption engine */ 472 static int sa_set_sc_enc(struct algo_data *ad, const u8 *key, u16 key_sz, 473 u8 enc, u8 *sc_buf) 474 { 475 const u8 *mci = NULL; 476 477 /* Set Encryption mode selector to crypto processing */ 478 sc_buf[0] = SA_CRYPTO_PROCESSING; 479 480 if (enc) 481 mci = ad->mci_enc; 482 else 483 mci = ad->mci_dec; 484 /* Set the mode control instructions in security context */ 485 if (mci) 486 memcpy(&sc_buf[1], mci, MODE_CONTROL_BYTES); 487 488 /* For AES-CBC decryption get the inverse key */ 489 if (ad->inv_key && !enc) { 490 if (sa_aes_inv_key(&sc_buf[SC_ENC_KEY_OFFSET], key, key_sz)) 491 return -EINVAL; 492 /* For all other cases: key is used */ 493 } else { 494 memcpy(&sc_buf[SC_ENC_KEY_OFFSET], key, key_sz); 495 } 496 497 return 0; 498 } 499 500 /* Set Security context for the authentication engine */ 501 static void sa_set_sc_auth(struct algo_data *ad, const u8 *key, u16 key_sz, 502 u8 *sc_buf) 503 { 504 __be32 ipad[64], opad[64]; 505 506 /* Set Authentication mode selector to hash processing */ 507 sc_buf[0] = SA_HASH_PROCESSING; 508 /* Auth SW ctrl word: bit[6]=1 (upload computed hash to TLR section) */ 509 sc_buf[1] = SA_UPLOAD_HASH_TO_TLR; 510 sc_buf[1] |= ad->auth_ctrl; 511 512 /* Copy the keys or ipad/opad */ 513 if (ad->keyed_mac) { 514 ad->prep_iopad(ad, key, key_sz, ipad, opad); 515 516 /* Copy ipad to AuthKey */ 517 memcpy(&sc_buf[32], ipad, ad->hash_size); 518 /* Copy opad to Aux-1 */ 519 memcpy(&sc_buf[64], opad, ad->hash_size); 520 } else { 521 /* basic hash */ 522 sc_buf[1] |= SA_BASIC_HASH; 523 } 524 } 525 526 static inline void sa_copy_iv(__be32 *out, const u8 *iv, bool size16) 527 { 528 int j; 529 530 for (j = 0; j < ((size16) ? 4 : 2); j++) { 531 *out = cpu_to_be32(*((u32 *)iv)); 532 iv += 4; 533 out++; 534 } 535 } 536 537 /* Format general command label */ 538 static int sa_format_cmdl_gen(struct sa_cmdl_cfg *cfg, u8 *cmdl, 539 struct sa_cmdl_upd_info *upd_info) 540 { 541 u8 enc_offset = 0, auth_offset = 0, total = 0; 542 u8 enc_next_eng = SA_ENG_ID_OUTPORT2; 543 u8 auth_next_eng = SA_ENG_ID_OUTPORT2; 544 u32 *word_ptr = (u32 *)cmdl; 545 int i; 546 547 /* Clear the command label */ 548 memzero_explicit(cmdl, (SA_MAX_CMDL_WORDS * sizeof(u32))); 549 550 /* Iniialize the command update structure */ 551 memzero_explicit(upd_info, sizeof(*upd_info)); 552 553 if (cfg->enc_eng_id && cfg->auth_eng_id) { 554 if (cfg->enc) { 555 auth_offset = SA_CMDL_HEADER_SIZE_BYTES; 556 enc_next_eng = cfg->auth_eng_id; 557 558 if (cfg->iv_size) 559 auth_offset += cfg->iv_size; 560 } else { 561 enc_offset = SA_CMDL_HEADER_SIZE_BYTES; 562 auth_next_eng = cfg->enc_eng_id; 563 } 564 } 565 566 if (cfg->enc_eng_id) { 567 upd_info->flags |= SA_CMDL_UPD_ENC; 568 upd_info->enc_size.index = enc_offset >> 2; 569 upd_info->enc_offset.index = upd_info->enc_size.index + 1; 570 /* Encryption command label */ 571 cmdl[enc_offset + SA_CMDL_OFFSET_NESC] = enc_next_eng; 572 573 /* Encryption modes requiring IV */ 574 if (cfg->iv_size) { 575 upd_info->flags |= SA_CMDL_UPD_ENC_IV; 576 upd_info->enc_iv.index = 577 (enc_offset + SA_CMDL_HEADER_SIZE_BYTES) >> 2; 578 upd_info->enc_iv.size = cfg->iv_size; 579 580 cmdl[enc_offset + SA_CMDL_OFFSET_LABEL_LEN] = 581 SA_CMDL_HEADER_SIZE_BYTES + cfg->iv_size; 582 583 cmdl[enc_offset + SA_CMDL_OFFSET_OPTION_CTRL1] = 584 (SA_CTX_ENC_AUX2_OFFSET | (cfg->iv_size >> 3)); 585 total += SA_CMDL_HEADER_SIZE_BYTES + cfg->iv_size; 586 } else { 587 cmdl[enc_offset + SA_CMDL_OFFSET_LABEL_LEN] = 588 SA_CMDL_HEADER_SIZE_BYTES; 589 total += SA_CMDL_HEADER_SIZE_BYTES; 590 } 591 } 592 593 if (cfg->auth_eng_id) { 594 upd_info->flags |= SA_CMDL_UPD_AUTH; 595 upd_info->auth_size.index = auth_offset >> 2; 596 upd_info->auth_offset.index = upd_info->auth_size.index + 1; 597 cmdl[auth_offset + SA_CMDL_OFFSET_NESC] = auth_next_eng; 598 cmdl[auth_offset + SA_CMDL_OFFSET_LABEL_LEN] = 599 SA_CMDL_HEADER_SIZE_BYTES; 600 total += SA_CMDL_HEADER_SIZE_BYTES; 601 } 602 603 total = roundup(total, 8); 604 605 for (i = 0; i < total / 4; i++) 606 word_ptr[i] = swab32(word_ptr[i]); 607 608 return total; 609 } 610 611 /* Update Command label */ 612 static inline void sa_update_cmdl(struct sa_req *req, u32 *cmdl, 613 struct sa_cmdl_upd_info *upd_info) 614 { 615 int i = 0, j; 616 617 if (likely(upd_info->flags & SA_CMDL_UPD_ENC)) { 618 cmdl[upd_info->enc_size.index] &= ~SA_CMDL_PAYLOAD_LENGTH_MASK; 619 cmdl[upd_info->enc_size.index] |= req->enc_size; 620 cmdl[upd_info->enc_offset.index] &= 621 ~SA_CMDL_SOP_BYPASS_LEN_MASK; 622 cmdl[upd_info->enc_offset.index] |= 623 ((u32)req->enc_offset << 624 __ffs(SA_CMDL_SOP_BYPASS_LEN_MASK)); 625 626 if (likely(upd_info->flags & SA_CMDL_UPD_ENC_IV)) { 627 __be32 *data = (__be32 *)&cmdl[upd_info->enc_iv.index]; 628 u32 *enc_iv = (u32 *)req->enc_iv; 629 630 for (j = 0; i < upd_info->enc_iv.size; i += 4, j++) { 631 data[j] = cpu_to_be32(*enc_iv); 632 enc_iv++; 633 } 634 } 635 } 636 637 if (likely(upd_info->flags & SA_CMDL_UPD_AUTH)) { 638 cmdl[upd_info->auth_size.index] &= ~SA_CMDL_PAYLOAD_LENGTH_MASK; 639 cmdl[upd_info->auth_size.index] |= req->auth_size; 640 cmdl[upd_info->auth_offset.index] &= 641 ~SA_CMDL_SOP_BYPASS_LEN_MASK; 642 cmdl[upd_info->auth_offset.index] |= 643 ((u32)req->auth_offset << 644 __ffs(SA_CMDL_SOP_BYPASS_LEN_MASK)); 645 if (upd_info->flags & SA_CMDL_UPD_AUTH_IV) { 646 sa_copy_iv((void *)&cmdl[upd_info->auth_iv.index], 647 req->auth_iv, 648 (upd_info->auth_iv.size > 8)); 649 } 650 if (upd_info->flags & SA_CMDL_UPD_AUX_KEY) { 651 int offset = (req->auth_size & 0xF) ? 4 : 0; 652 653 memcpy(&cmdl[upd_info->aux_key_info.index], 654 &upd_info->aux_key[offset], 16); 655 } 656 } 657 } 658 659 /* Format SWINFO words to be sent to SA */ 660 static 661 void sa_set_swinfo(u8 eng_id, u16 sc_id, dma_addr_t sc_phys, 662 u8 cmdl_present, u8 cmdl_offset, u8 flags, 663 u8 hash_size, u32 *swinfo) 664 { 665 swinfo[0] = sc_id; 666 swinfo[0] |= (flags << __ffs(SA_SW0_FLAGS_MASK)); 667 if (likely(cmdl_present)) 668 swinfo[0] |= ((cmdl_offset | SA_SW0_CMDL_PRESENT) << 669 __ffs(SA_SW0_CMDL_INFO_MASK)); 670 swinfo[0] |= (eng_id << __ffs(SA_SW0_ENG_ID_MASK)); 671 672 swinfo[0] |= SA_SW0_DEST_INFO_PRESENT; 673 swinfo[1] = (u32)(sc_phys & 0xFFFFFFFFULL); 674 swinfo[2] = (u32)((sc_phys & 0xFFFFFFFF00000000ULL) >> 32); 675 swinfo[2] |= (hash_size << __ffs(SA_SW2_EGRESS_LENGTH)); 676 } 677 678 /* Dump the security context */ 679 static void sa_dump_sc(u8 *buf, dma_addr_t dma_addr) 680 { 681 #ifdef DEBUG 682 dev_info(sa_k3_dev, "Security context dump:: 0x%pad\n", &dma_addr); 683 print_hex_dump(KERN_CONT, "", DUMP_PREFIX_OFFSET, 684 16, 1, buf, SA_CTX_MAX_SZ, false); 685 #endif 686 } 687 688 static 689 int sa_init_sc(struct sa_ctx_info *ctx, const u8 *enc_key, 690 u16 enc_key_sz, const u8 *auth_key, u16 auth_key_sz, 691 struct algo_data *ad, u8 enc, u32 *swinfo) 692 { 693 int enc_sc_offset = 0; 694 int auth_sc_offset = 0; 695 u8 *sc_buf = ctx->sc; 696 u16 sc_id = ctx->sc_id; 697 u8 first_engine = 0; 698 699 memzero_explicit(sc_buf, SA_CTX_MAX_SZ); 700 701 if (ad->auth_eng.eng_id) { 702 if (enc) 703 first_engine = ad->enc_eng.eng_id; 704 else 705 first_engine = ad->auth_eng.eng_id; 706 707 enc_sc_offset = SA_CTX_PHP_PE_CTX_SZ; 708 auth_sc_offset = enc_sc_offset + ad->enc_eng.sc_size; 709 sc_buf[1] = SA_SCCTL_FE_AUTH_ENC; 710 if (!ad->hash_size) 711 return -EINVAL; 712 ad->hash_size = roundup(ad->hash_size, 8); 713 714 } else if (ad->enc_eng.eng_id && !ad->auth_eng.eng_id) { 715 enc_sc_offset = SA_CTX_PHP_PE_CTX_SZ; 716 first_engine = ad->enc_eng.eng_id; 717 sc_buf[1] = SA_SCCTL_FE_ENC; 718 ad->hash_size = ad->iv_out_size; 719 } 720 721 /* SCCTL Owner info: 0=host, 1=CP_ACE */ 722 sc_buf[SA_CTX_SCCTL_OWNER_OFFSET] = 0; 723 memcpy(&sc_buf[2], &sc_id, 2); 724 sc_buf[4] = 0x0; 725 sc_buf[5] = PRIV_ID; 726 sc_buf[6] = PRIV; 727 sc_buf[7] = 0x0; 728 729 /* Prepare context for encryption engine */ 730 if (ad->enc_eng.sc_size) { 731 if (sa_set_sc_enc(ad, enc_key, enc_key_sz, enc, 732 &sc_buf[enc_sc_offset])) 733 return -EINVAL; 734 } 735 736 /* Prepare context for authentication engine */ 737 if (ad->auth_eng.sc_size) 738 sa_set_sc_auth(ad, auth_key, auth_key_sz, 739 &sc_buf[auth_sc_offset]); 740 741 /* Set the ownership of context to CP_ACE */ 742 sc_buf[SA_CTX_SCCTL_OWNER_OFFSET] = 0x80; 743 744 /* swizzle the security context */ 745 sa_swiz_128(sc_buf, SA_CTX_MAX_SZ); 746 747 sa_set_swinfo(first_engine, ctx->sc_id, ctx->sc_phys, 1, 0, 748 SA_SW_INFO_FLAG_EVICT, ad->hash_size, swinfo); 749 750 sa_dump_sc(sc_buf, ctx->sc_phys); 751 752 return 0; 753 } 754 755 /* Free the per direction context memory */ 756 static void sa_free_ctx_info(struct sa_ctx_info *ctx, 757 struct sa_crypto_data *data) 758 { 759 unsigned long bn; 760 761 bn = ctx->sc_id - data->sc_id_start; 762 spin_lock(&data->scid_lock); 763 __clear_bit(bn, data->ctx_bm); 764 data->sc_id--; 765 spin_unlock(&data->scid_lock); 766 767 if (ctx->sc) { 768 dma_pool_free(data->sc_pool, ctx->sc, ctx->sc_phys); 769 ctx->sc = NULL; 770 } 771 } 772 773 static int sa_init_ctx_info(struct sa_ctx_info *ctx, 774 struct sa_crypto_data *data) 775 { 776 unsigned long bn; 777 int err; 778 779 spin_lock(&data->scid_lock); 780 bn = find_first_zero_bit(data->ctx_bm, SA_MAX_NUM_CTX); 781 __set_bit(bn, data->ctx_bm); 782 data->sc_id++; 783 spin_unlock(&data->scid_lock); 784 785 ctx->sc_id = (u16)(data->sc_id_start + bn); 786 787 ctx->sc = dma_pool_alloc(data->sc_pool, GFP_KERNEL, &ctx->sc_phys); 788 if (!ctx->sc) { 789 dev_err(&data->pdev->dev, "Failed to allocate SC memory\n"); 790 err = -ENOMEM; 791 goto scid_rollback; 792 } 793 794 return 0; 795 796 scid_rollback: 797 spin_lock(&data->scid_lock); 798 __clear_bit(bn, data->ctx_bm); 799 data->sc_id--; 800 spin_unlock(&data->scid_lock); 801 802 return err; 803 } 804 805 static void sa_cipher_cra_exit(struct crypto_skcipher *tfm) 806 { 807 struct sa_tfm_ctx *ctx = crypto_skcipher_ctx(tfm); 808 struct sa_crypto_data *data = dev_get_drvdata(sa_k3_dev); 809 810 dev_dbg(sa_k3_dev, "%s(0x%p) sc-ids(0x%x(0x%pad), 0x%x(0x%pad))\n", 811 __func__, tfm, ctx->enc.sc_id, &ctx->enc.sc_phys, 812 ctx->dec.sc_id, &ctx->dec.sc_phys); 813 814 sa_free_ctx_info(&ctx->enc, data); 815 sa_free_ctx_info(&ctx->dec, data); 816 817 crypto_free_sync_skcipher(ctx->fallback.skcipher); 818 } 819 820 static int sa_cipher_cra_init(struct crypto_skcipher *tfm) 821 { 822 struct sa_tfm_ctx *ctx = crypto_skcipher_ctx(tfm); 823 struct sa_crypto_data *data = dev_get_drvdata(sa_k3_dev); 824 const char *name = crypto_tfm_alg_name(&tfm->base); 825 int ret; 826 827 memzero_explicit(ctx, sizeof(*ctx)); 828 ctx->dev_data = data; 829 830 ret = sa_init_ctx_info(&ctx->enc, data); 831 if (ret) 832 return ret; 833 ret = sa_init_ctx_info(&ctx->dec, data); 834 if (ret) { 835 sa_free_ctx_info(&ctx->enc, data); 836 return ret; 837 } 838 839 ctx->fallback.skcipher = 840 crypto_alloc_sync_skcipher(name, 0, CRYPTO_ALG_NEED_FALLBACK); 841 842 if (IS_ERR(ctx->fallback.skcipher)) { 843 dev_err(sa_k3_dev, "Error allocating fallback algo %s\n", name); 844 return PTR_ERR(ctx->fallback.skcipher); 845 } 846 847 dev_dbg(sa_k3_dev, "%s(0x%p) sc-ids(0x%x(0x%pad), 0x%x(0x%pad))\n", 848 __func__, tfm, ctx->enc.sc_id, &ctx->enc.sc_phys, 849 ctx->dec.sc_id, &ctx->dec.sc_phys); 850 return 0; 851 } 852 853 static int sa_cipher_setkey(struct crypto_skcipher *tfm, const u8 *key, 854 unsigned int keylen, struct algo_data *ad) 855 { 856 struct sa_tfm_ctx *ctx = crypto_skcipher_ctx(tfm); 857 int cmdl_len; 858 struct sa_cmdl_cfg cfg; 859 int ret; 860 861 if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_192 && 862 keylen != AES_KEYSIZE_256) 863 return -EINVAL; 864 865 ad->enc_eng.eng_id = SA_ENG_ID_EM1; 866 ad->enc_eng.sc_size = SA_CTX_ENC_TYPE1_SZ; 867 868 memzero_explicit(&cfg, sizeof(cfg)); 869 cfg.enc_eng_id = ad->enc_eng.eng_id; 870 cfg.iv_size = crypto_skcipher_ivsize(tfm); 871 872 crypto_sync_skcipher_clear_flags(ctx->fallback.skcipher, 873 CRYPTO_TFM_REQ_MASK); 874 crypto_sync_skcipher_set_flags(ctx->fallback.skcipher, 875 tfm->base.crt_flags & 876 CRYPTO_TFM_REQ_MASK); 877 ret = crypto_sync_skcipher_setkey(ctx->fallback.skcipher, key, keylen); 878 if (ret) 879 return ret; 880 881 /* Setup Encryption Security Context & Command label template */ 882 if (sa_init_sc(&ctx->enc, key, keylen, NULL, 0, ad, 1, 883 &ctx->enc.epib[1])) 884 goto badkey; 885 886 cmdl_len = sa_format_cmdl_gen(&cfg, 887 (u8 *)ctx->enc.cmdl, 888 &ctx->enc.cmdl_upd_info); 889 if (cmdl_len <= 0 || (cmdl_len > SA_MAX_CMDL_WORDS * sizeof(u32))) 890 goto badkey; 891 892 ctx->enc.cmdl_size = cmdl_len; 893 894 /* Setup Decryption Security Context & Command label template */ 895 if (sa_init_sc(&ctx->dec, key, keylen, NULL, 0, ad, 0, 896 &ctx->dec.epib[1])) 897 goto badkey; 898 899 cfg.enc_eng_id = ad->enc_eng.eng_id; 900 cmdl_len = sa_format_cmdl_gen(&cfg, (u8 *)ctx->dec.cmdl, 901 &ctx->dec.cmdl_upd_info); 902 903 if (cmdl_len <= 0 || (cmdl_len > SA_MAX_CMDL_WORDS * sizeof(u32))) 904 goto badkey; 905 906 ctx->dec.cmdl_size = cmdl_len; 907 ctx->iv_idx = ad->iv_idx; 908 909 return 0; 910 911 badkey: 912 dev_err(sa_k3_dev, "%s: badkey\n", __func__); 913 return -EINVAL; 914 } 915 916 static int sa_aes_cbc_setkey(struct crypto_skcipher *tfm, const u8 *key, 917 unsigned int keylen) 918 { 919 struct algo_data ad = { 0 }; 920 /* Convert the key size (16/24/32) to the key size index (0/1/2) */ 921 int key_idx = (keylen >> 3) - 2; 922 923 if (key_idx >= 3) 924 return -EINVAL; 925 926 ad.mci_enc = mci_cbc_enc_array[key_idx]; 927 ad.mci_dec = mci_cbc_dec_array[key_idx]; 928 ad.inv_key = true; 929 ad.ealg_id = SA_EALG_ID_AES_CBC; 930 ad.iv_idx = 4; 931 ad.iv_out_size = 16; 932 933 return sa_cipher_setkey(tfm, key, keylen, &ad); 934 } 935 936 static int sa_aes_ecb_setkey(struct crypto_skcipher *tfm, const u8 *key, 937 unsigned int keylen) 938 { 939 struct algo_data ad = { 0 }; 940 /* Convert the key size (16/24/32) to the key size index (0/1/2) */ 941 int key_idx = (keylen >> 3) - 2; 942 943 if (key_idx >= 3) 944 return -EINVAL; 945 946 ad.mci_enc = mci_ecb_enc_array[key_idx]; 947 ad.mci_dec = mci_ecb_dec_array[key_idx]; 948 ad.inv_key = true; 949 ad.ealg_id = SA_EALG_ID_AES_ECB; 950 951 return sa_cipher_setkey(tfm, key, keylen, &ad); 952 } 953 954 static int sa_3des_cbc_setkey(struct crypto_skcipher *tfm, const u8 *key, 955 unsigned int keylen) 956 { 957 struct algo_data ad = { 0 }; 958 959 ad.mci_enc = mci_cbc_3des_enc_array; 960 ad.mci_dec = mci_cbc_3des_dec_array; 961 ad.ealg_id = SA_EALG_ID_3DES_CBC; 962 ad.iv_idx = 6; 963 ad.iv_out_size = 8; 964 965 return sa_cipher_setkey(tfm, key, keylen, &ad); 966 } 967 968 static int sa_3des_ecb_setkey(struct crypto_skcipher *tfm, const u8 *key, 969 unsigned int keylen) 970 { 971 struct algo_data ad = { 0 }; 972 973 ad.mci_enc = mci_ecb_3des_enc_array; 974 ad.mci_dec = mci_ecb_3des_dec_array; 975 976 return sa_cipher_setkey(tfm, key, keylen, &ad); 977 } 978 979 static void sa_aes_dma_in_callback(void *data) 980 { 981 struct sa_rx_data *rxd = (struct sa_rx_data *)data; 982 struct skcipher_request *req; 983 int sglen; 984 u32 *result; 985 __be32 *mdptr; 986 size_t ml, pl; 987 int i; 988 enum dma_data_direction dir_src; 989 bool diff_dst; 990 991 req = container_of(rxd->req, struct skcipher_request, base); 992 sglen = sg_nents_for_len(req->src, req->cryptlen); 993 994 diff_dst = (req->src != req->dst) ? true : false; 995 dir_src = diff_dst ? DMA_TO_DEVICE : DMA_BIDIRECTIONAL; 996 997 if (req->iv) { 998 mdptr = (__be32 *)dmaengine_desc_get_metadata_ptr(rxd->tx_in, &pl, 999 &ml); 1000 result = (u32 *)req->iv; 1001 1002 for (i = 0; i < (rxd->enc_iv_size / 4); i++) 1003 result[i] = be32_to_cpu(mdptr[i + rxd->iv_idx]); 1004 } 1005 1006 dma_unmap_sg(rxd->ddev, req->src, sglen, dir_src); 1007 kfree(rxd->split_src_sg); 1008 1009 if (diff_dst) { 1010 sglen = sg_nents_for_len(req->dst, req->cryptlen); 1011 1012 dma_unmap_sg(rxd->ddev, req->dst, sglen, 1013 DMA_FROM_DEVICE); 1014 kfree(rxd->split_dst_sg); 1015 } 1016 1017 kfree(rxd); 1018 1019 skcipher_request_complete(req, 0); 1020 } 1021 1022 static void 1023 sa_prepare_tx_desc(u32 *mdptr, u32 pslen, u32 *psdata, u32 epiblen, u32 *epib) 1024 { 1025 u32 *out, *in; 1026 int i; 1027 1028 for (out = mdptr, in = epib, i = 0; i < epiblen / sizeof(u32); i++) 1029 *out++ = *in++; 1030 1031 mdptr[4] = (0xFFFF << 16); 1032 for (out = &mdptr[5], in = psdata, i = 0; 1033 i < pslen / sizeof(u32); i++) 1034 *out++ = *in++; 1035 } 1036 1037 static int sa_run(struct sa_req *req) 1038 { 1039 struct sa_rx_data *rxd; 1040 gfp_t gfp_flags; 1041 u32 cmdl[SA_MAX_CMDL_WORDS]; 1042 struct sa_crypto_data *pdata = dev_get_drvdata(sa_k3_dev); 1043 struct device *ddev; 1044 struct dma_chan *dma_rx; 1045 int sg_nents, src_nents, dst_nents; 1046 int mapped_src_nents, mapped_dst_nents; 1047 struct scatterlist *src, *dst; 1048 size_t pl, ml, split_size; 1049 struct sa_ctx_info *sa_ctx = req->enc ? &req->ctx->enc : &req->ctx->dec; 1050 int ret; 1051 struct dma_async_tx_descriptor *tx_out; 1052 u32 *mdptr; 1053 bool diff_dst; 1054 enum dma_data_direction dir_src; 1055 1056 gfp_flags = req->base->flags & CRYPTO_TFM_REQ_MAY_SLEEP ? 1057 GFP_KERNEL : GFP_ATOMIC; 1058 1059 rxd = kzalloc(sizeof(*rxd), gfp_flags); 1060 if (!rxd) 1061 return -ENOMEM; 1062 1063 if (req->src != req->dst) { 1064 diff_dst = true; 1065 dir_src = DMA_TO_DEVICE; 1066 } else { 1067 diff_dst = false; 1068 dir_src = DMA_BIDIRECTIONAL; 1069 } 1070 1071 /* 1072 * SA2UL has an interesting feature where the receive DMA channel 1073 * is selected based on the data passed to the engine. Within the 1074 * transition range, there is also a space where it is impossible 1075 * to determine where the data will end up, and this should be 1076 * avoided. This will be handled by the SW fallback mechanism by 1077 * the individual algorithm implementations. 1078 */ 1079 if (req->size >= 256) 1080 dma_rx = pdata->dma_rx2; 1081 else 1082 dma_rx = pdata->dma_rx1; 1083 1084 ddev = dma_rx->device->dev; 1085 1086 memcpy(cmdl, sa_ctx->cmdl, sa_ctx->cmdl_size); 1087 1088 sa_update_cmdl(req, cmdl, &sa_ctx->cmdl_upd_info); 1089 1090 if (req->type != CRYPTO_ALG_TYPE_AHASH) { 1091 if (req->enc) 1092 req->type |= 1093 (SA_REQ_SUBTYPE_ENC << SA_REQ_SUBTYPE_SHIFT); 1094 else 1095 req->type |= 1096 (SA_REQ_SUBTYPE_DEC << SA_REQ_SUBTYPE_SHIFT); 1097 } 1098 1099 cmdl[sa_ctx->cmdl_size / sizeof(u32)] = req->type; 1100 1101 /* 1102 * Map the packets, first we check if the data fits into a single 1103 * sg entry and use that if possible. If it does not fit, we check 1104 * if we need to do sg_split to align the scatterlist data on the 1105 * actual data size being processed by the crypto engine. 1106 */ 1107 src = req->src; 1108 sg_nents = sg_nents_for_len(src, req->size); 1109 1110 split_size = req->size; 1111 1112 if (sg_nents == 1 && split_size <= req->src->length) { 1113 src = &rxd->rx_sg; 1114 sg_init_table(src, 1); 1115 sg_set_page(src, sg_page(req->src), split_size, 1116 req->src->offset); 1117 src_nents = 1; 1118 dma_map_sg(ddev, src, sg_nents, dir_src); 1119 } else { 1120 mapped_src_nents = dma_map_sg(ddev, req->src, sg_nents, 1121 dir_src); 1122 ret = sg_split(req->src, mapped_src_nents, 0, 1, &split_size, 1123 &src, &src_nents, gfp_flags); 1124 if (ret) { 1125 src_nents = sg_nents; 1126 src = req->src; 1127 } else { 1128 rxd->split_src_sg = src; 1129 } 1130 } 1131 1132 if (!diff_dst) { 1133 dst_nents = src_nents; 1134 dst = src; 1135 } else { 1136 dst_nents = sg_nents_for_len(req->dst, req->size); 1137 1138 if (dst_nents == 1 && split_size <= req->dst->length) { 1139 dst = &rxd->tx_sg; 1140 sg_init_table(dst, 1); 1141 sg_set_page(dst, sg_page(req->dst), split_size, 1142 req->dst->offset); 1143 dst_nents = 1; 1144 dma_map_sg(ddev, dst, dst_nents, DMA_FROM_DEVICE); 1145 } else { 1146 mapped_dst_nents = dma_map_sg(ddev, req->dst, dst_nents, 1147 DMA_FROM_DEVICE); 1148 ret = sg_split(req->dst, mapped_dst_nents, 0, 1, 1149 &split_size, &dst, &dst_nents, 1150 gfp_flags); 1151 if (ret) { 1152 dst_nents = dst_nents; 1153 dst = req->dst; 1154 } else { 1155 rxd->split_dst_sg = dst; 1156 } 1157 } 1158 } 1159 1160 if (unlikely(src_nents != sg_nents)) { 1161 dev_warn_ratelimited(sa_k3_dev, "failed to map tx pkt\n"); 1162 ret = -EIO; 1163 goto err_cleanup; 1164 } 1165 1166 rxd->tx_in = dmaengine_prep_slave_sg(dma_rx, dst, dst_nents, 1167 DMA_DEV_TO_MEM, 1168 DMA_PREP_INTERRUPT | DMA_CTRL_ACK); 1169 if (!rxd->tx_in) { 1170 dev_err(pdata->dev, "IN prep_slave_sg() failed\n"); 1171 ret = -EINVAL; 1172 goto err_cleanup; 1173 } 1174 1175 rxd->req = (void *)req->base; 1176 rxd->enc = req->enc; 1177 rxd->ddev = ddev; 1178 rxd->src = src; 1179 rxd->dst = dst; 1180 rxd->iv_idx = req->ctx->iv_idx; 1181 rxd->enc_iv_size = sa_ctx->cmdl_upd_info.enc_iv.size; 1182 rxd->tx_in->callback = req->callback; 1183 rxd->tx_in->callback_param = rxd; 1184 1185 tx_out = dmaengine_prep_slave_sg(pdata->dma_tx, src, 1186 src_nents, DMA_MEM_TO_DEV, 1187 DMA_PREP_INTERRUPT | DMA_CTRL_ACK); 1188 1189 if (!tx_out) { 1190 dev_err(pdata->dev, "OUT prep_slave_sg() failed\n"); 1191 ret = -EINVAL; 1192 goto err_cleanup; 1193 } 1194 1195 /* 1196 * Prepare metadata for DMA engine. This essentially describes the 1197 * crypto algorithm to be used, data sizes, different keys etc. 1198 */ 1199 mdptr = (u32 *)dmaengine_desc_get_metadata_ptr(tx_out, &pl, &ml); 1200 1201 sa_prepare_tx_desc(mdptr, (sa_ctx->cmdl_size + (SA_PSDATA_CTX_WORDS * 1202 sizeof(u32))), cmdl, sizeof(sa_ctx->epib), 1203 sa_ctx->epib); 1204 1205 ml = sa_ctx->cmdl_size + (SA_PSDATA_CTX_WORDS * sizeof(u32)); 1206 dmaengine_desc_set_metadata_len(tx_out, req->mdata_size); 1207 1208 dmaengine_submit(tx_out); 1209 dmaengine_submit(rxd->tx_in); 1210 1211 dma_async_issue_pending(dma_rx); 1212 dma_async_issue_pending(pdata->dma_tx); 1213 1214 return -EINPROGRESS; 1215 1216 err_cleanup: 1217 dma_unmap_sg(ddev, req->src, sg_nents, DMA_TO_DEVICE); 1218 kfree(rxd->split_src_sg); 1219 1220 if (req->src != req->dst) { 1221 dst_nents = sg_nents_for_len(req->dst, req->size); 1222 dma_unmap_sg(ddev, req->dst, dst_nents, DMA_FROM_DEVICE); 1223 kfree(rxd->split_dst_sg); 1224 } 1225 1226 kfree(rxd); 1227 1228 return ret; 1229 } 1230 1231 static int sa_cipher_run(struct skcipher_request *req, u8 *iv, int enc) 1232 { 1233 struct sa_tfm_ctx *ctx = 1234 crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); 1235 struct crypto_alg *alg = req->base.tfm->__crt_alg; 1236 struct sa_req sa_req = { 0 }; 1237 int ret; 1238 1239 if (!req->cryptlen) 1240 return 0; 1241 1242 if (req->cryptlen % alg->cra_blocksize) 1243 return -EINVAL; 1244 1245 /* Use SW fallback if the data size is not supported */ 1246 if (req->cryptlen > SA_MAX_DATA_SZ || 1247 (req->cryptlen >= SA_UNSAFE_DATA_SZ_MIN && 1248 req->cryptlen <= SA_UNSAFE_DATA_SZ_MAX)) { 1249 SYNC_SKCIPHER_REQUEST_ON_STACK(subreq, ctx->fallback.skcipher); 1250 1251 skcipher_request_set_sync_tfm(subreq, ctx->fallback.skcipher); 1252 skcipher_request_set_callback(subreq, req->base.flags, 1253 NULL, NULL); 1254 skcipher_request_set_crypt(subreq, req->src, req->dst, 1255 req->cryptlen, req->iv); 1256 if (enc) 1257 ret = crypto_skcipher_encrypt(subreq); 1258 else 1259 ret = crypto_skcipher_decrypt(subreq); 1260 1261 skcipher_request_zero(subreq); 1262 return ret; 1263 } 1264 1265 sa_req.size = req->cryptlen; 1266 sa_req.enc_size = req->cryptlen; 1267 sa_req.src = req->src; 1268 sa_req.dst = req->dst; 1269 sa_req.enc_iv = iv; 1270 sa_req.type = CRYPTO_ALG_TYPE_SKCIPHER; 1271 sa_req.enc = enc; 1272 sa_req.callback = sa_aes_dma_in_callback; 1273 sa_req.mdata_size = 44; 1274 sa_req.base = &req->base; 1275 sa_req.ctx = ctx; 1276 1277 return sa_run(&sa_req); 1278 } 1279 1280 static int sa_encrypt(struct skcipher_request *req) 1281 { 1282 return sa_cipher_run(req, req->iv, 1); 1283 } 1284 1285 static int sa_decrypt(struct skcipher_request *req) 1286 { 1287 return sa_cipher_run(req, req->iv, 0); 1288 } 1289 1290 static void sa_sha_dma_in_callback(void *data) 1291 { 1292 struct sa_rx_data *rxd = (struct sa_rx_data *)data; 1293 struct ahash_request *req; 1294 struct crypto_ahash *tfm; 1295 unsigned int authsize; 1296 int i, sg_nents; 1297 size_t ml, pl; 1298 u32 *result; 1299 __be32 *mdptr; 1300 1301 req = container_of(rxd->req, struct ahash_request, base); 1302 tfm = crypto_ahash_reqtfm(req); 1303 authsize = crypto_ahash_digestsize(tfm); 1304 1305 mdptr = (__be32 *)dmaengine_desc_get_metadata_ptr(rxd->tx_in, &pl, &ml); 1306 result = (u32 *)req->result; 1307 1308 for (i = 0; i < (authsize / 4); i++) 1309 result[i] = be32_to_cpu(mdptr[i + 4]); 1310 1311 sg_nents = sg_nents_for_len(req->src, req->nbytes); 1312 dma_unmap_sg(rxd->ddev, req->src, sg_nents, DMA_FROM_DEVICE); 1313 1314 kfree(rxd->split_src_sg); 1315 1316 kfree(rxd); 1317 1318 ahash_request_complete(req, 0); 1319 } 1320 1321 static int zero_message_process(struct ahash_request *req) 1322 { 1323 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 1324 int sa_digest_size = crypto_ahash_digestsize(tfm); 1325 1326 switch (sa_digest_size) { 1327 case SHA1_DIGEST_SIZE: 1328 memcpy(req->result, sha1_zero_message_hash, sa_digest_size); 1329 break; 1330 case SHA256_DIGEST_SIZE: 1331 memcpy(req->result, sha256_zero_message_hash, sa_digest_size); 1332 break; 1333 case SHA512_DIGEST_SIZE: 1334 memcpy(req->result, sha512_zero_message_hash, sa_digest_size); 1335 break; 1336 default: 1337 return -EINVAL; 1338 } 1339 1340 return 0; 1341 } 1342 1343 static int sa_sha_run(struct ahash_request *req) 1344 { 1345 struct sa_tfm_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(req)); 1346 struct sa_sha_req_ctx *rctx = ahash_request_ctx(req); 1347 struct sa_req sa_req = { 0 }; 1348 size_t auth_len; 1349 1350 auth_len = req->nbytes; 1351 1352 if (!auth_len) 1353 return zero_message_process(req); 1354 1355 if (auth_len > SA_MAX_DATA_SZ || 1356 (auth_len >= SA_UNSAFE_DATA_SZ_MIN && 1357 auth_len <= SA_UNSAFE_DATA_SZ_MAX)) { 1358 struct ahash_request *subreq = &rctx->fallback_req; 1359 int ret = 0; 1360 1361 ahash_request_set_tfm(subreq, ctx->fallback.ahash); 1362 subreq->base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; 1363 1364 crypto_ahash_init(subreq); 1365 1366 subreq->nbytes = auth_len; 1367 subreq->src = req->src; 1368 subreq->result = req->result; 1369 1370 ret |= crypto_ahash_update(subreq); 1371 1372 subreq->nbytes = 0; 1373 1374 ret |= crypto_ahash_final(subreq); 1375 1376 return ret; 1377 } 1378 1379 sa_req.size = auth_len; 1380 sa_req.auth_size = auth_len; 1381 sa_req.src = req->src; 1382 sa_req.dst = req->src; 1383 sa_req.enc = true; 1384 sa_req.type = CRYPTO_ALG_TYPE_AHASH; 1385 sa_req.callback = sa_sha_dma_in_callback; 1386 sa_req.mdata_size = 28; 1387 sa_req.ctx = ctx; 1388 sa_req.base = &req->base; 1389 1390 return sa_run(&sa_req); 1391 } 1392 1393 static int sa_sha_setup(struct sa_tfm_ctx *ctx, struct algo_data *ad) 1394 { 1395 int bs = crypto_shash_blocksize(ctx->shash); 1396 int cmdl_len; 1397 struct sa_cmdl_cfg cfg; 1398 1399 ad->enc_eng.sc_size = SA_CTX_ENC_TYPE1_SZ; 1400 ad->auth_eng.eng_id = SA_ENG_ID_AM1; 1401 ad->auth_eng.sc_size = SA_CTX_AUTH_TYPE2_SZ; 1402 1403 memset(ctx->authkey, 0, bs); 1404 memset(&cfg, 0, sizeof(cfg)); 1405 cfg.aalg = ad->aalg_id; 1406 cfg.enc_eng_id = ad->enc_eng.eng_id; 1407 cfg.auth_eng_id = ad->auth_eng.eng_id; 1408 cfg.iv_size = 0; 1409 cfg.akey = NULL; 1410 cfg.akey_len = 0; 1411 1412 /* Setup Encryption Security Context & Command label template */ 1413 if (sa_init_sc(&ctx->enc, NULL, 0, NULL, 0, ad, 0, 1414 &ctx->enc.epib[1])) 1415 goto badkey; 1416 1417 cmdl_len = sa_format_cmdl_gen(&cfg, 1418 (u8 *)ctx->enc.cmdl, 1419 &ctx->enc.cmdl_upd_info); 1420 if (cmdl_len <= 0 || (cmdl_len > SA_MAX_CMDL_WORDS * sizeof(u32))) 1421 goto badkey; 1422 1423 ctx->enc.cmdl_size = cmdl_len; 1424 1425 return 0; 1426 1427 badkey: 1428 dev_err(sa_k3_dev, "%s: badkey\n", __func__); 1429 return -EINVAL; 1430 } 1431 1432 static int sa_sha_cra_init_alg(struct crypto_tfm *tfm, const char *alg_base) 1433 { 1434 struct sa_tfm_ctx *ctx = crypto_tfm_ctx(tfm); 1435 struct sa_crypto_data *data = dev_get_drvdata(sa_k3_dev); 1436 int ret; 1437 1438 memset(ctx, 0, sizeof(*ctx)); 1439 ctx->dev_data = data; 1440 ret = sa_init_ctx_info(&ctx->enc, data); 1441 if (ret) 1442 return ret; 1443 1444 if (alg_base) { 1445 ctx->shash = crypto_alloc_shash(alg_base, 0, 1446 CRYPTO_ALG_NEED_FALLBACK); 1447 if (IS_ERR(ctx->shash)) { 1448 dev_err(sa_k3_dev, "base driver %s couldn't be loaded\n", 1449 alg_base); 1450 return PTR_ERR(ctx->shash); 1451 } 1452 /* for fallback */ 1453 ctx->fallback.ahash = 1454 crypto_alloc_ahash(alg_base, 0, 1455 CRYPTO_ALG_NEED_FALLBACK); 1456 if (IS_ERR(ctx->fallback.ahash)) { 1457 dev_err(ctx->dev_data->dev, 1458 "Could not load fallback driver\n"); 1459 return PTR_ERR(ctx->fallback.ahash); 1460 } 1461 } 1462 1463 dev_dbg(sa_k3_dev, "%s(0x%p) sc-ids(0x%x(0x%pad), 0x%x(0x%pad))\n", 1464 __func__, tfm, ctx->enc.sc_id, &ctx->enc.sc_phys, 1465 ctx->dec.sc_id, &ctx->dec.sc_phys); 1466 1467 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), 1468 sizeof(struct sa_sha_req_ctx) + 1469 crypto_ahash_reqsize(ctx->fallback.ahash)); 1470 1471 return 0; 1472 } 1473 1474 static int sa_sha_digest(struct ahash_request *req) 1475 { 1476 return sa_sha_run(req); 1477 } 1478 1479 static int sa_sha_init(struct ahash_request *req) 1480 { 1481 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 1482 struct sa_sha_req_ctx *rctx = ahash_request_ctx(req); 1483 struct sa_tfm_ctx *ctx = crypto_ahash_ctx(tfm); 1484 1485 dev_dbg(sa_k3_dev, "init: digest size: %d, rctx=%llx\n", 1486 crypto_ahash_digestsize(tfm), (u64)rctx); 1487 1488 ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback.ahash); 1489 rctx->fallback_req.base.flags = 1490 req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; 1491 1492 return crypto_ahash_init(&rctx->fallback_req); 1493 } 1494 1495 static int sa_sha_update(struct ahash_request *req) 1496 { 1497 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 1498 struct sa_sha_req_ctx *rctx = ahash_request_ctx(req); 1499 struct sa_tfm_ctx *ctx = crypto_ahash_ctx(tfm); 1500 1501 ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback.ahash); 1502 rctx->fallback_req.base.flags = 1503 req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; 1504 rctx->fallback_req.nbytes = req->nbytes; 1505 rctx->fallback_req.src = req->src; 1506 1507 return crypto_ahash_update(&rctx->fallback_req); 1508 } 1509 1510 static int sa_sha_final(struct ahash_request *req) 1511 { 1512 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 1513 struct sa_sha_req_ctx *rctx = ahash_request_ctx(req); 1514 struct sa_tfm_ctx *ctx = crypto_ahash_ctx(tfm); 1515 1516 ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback.ahash); 1517 rctx->fallback_req.base.flags = 1518 req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; 1519 rctx->fallback_req.result = req->result; 1520 1521 return crypto_ahash_final(&rctx->fallback_req); 1522 } 1523 1524 static int sa_sha_finup(struct ahash_request *req) 1525 { 1526 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 1527 struct sa_sha_req_ctx *rctx = ahash_request_ctx(req); 1528 struct sa_tfm_ctx *ctx = crypto_ahash_ctx(tfm); 1529 1530 ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback.ahash); 1531 rctx->fallback_req.base.flags = 1532 req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; 1533 1534 rctx->fallback_req.nbytes = req->nbytes; 1535 rctx->fallback_req.src = req->src; 1536 rctx->fallback_req.result = req->result; 1537 1538 return crypto_ahash_finup(&rctx->fallback_req); 1539 } 1540 1541 static int sa_sha_import(struct ahash_request *req, const void *in) 1542 { 1543 struct sa_sha_req_ctx *rctx = ahash_request_ctx(req); 1544 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 1545 struct sa_tfm_ctx *ctx = crypto_ahash_ctx(tfm); 1546 1547 ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback.ahash); 1548 rctx->fallback_req.base.flags = req->base.flags & 1549 CRYPTO_TFM_REQ_MAY_SLEEP; 1550 1551 return crypto_ahash_import(&rctx->fallback_req, in); 1552 } 1553 1554 static int sa_sha_export(struct ahash_request *req, void *out) 1555 { 1556 struct sa_sha_req_ctx *rctx = ahash_request_ctx(req); 1557 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 1558 struct sa_tfm_ctx *ctx = crypto_ahash_ctx(tfm); 1559 struct ahash_request *subreq = &rctx->fallback_req; 1560 1561 ahash_request_set_tfm(subreq, ctx->fallback.ahash); 1562 subreq->base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; 1563 1564 return crypto_ahash_export(subreq, out); 1565 } 1566 1567 static int sa_sha1_cra_init(struct crypto_tfm *tfm) 1568 { 1569 struct algo_data ad = { 0 }; 1570 struct sa_tfm_ctx *ctx = crypto_tfm_ctx(tfm); 1571 1572 sa_sha_cra_init_alg(tfm, "sha1"); 1573 1574 ad.aalg_id = SA_AALG_ID_SHA1; 1575 ad.hash_size = SHA1_DIGEST_SIZE; 1576 ad.auth_ctrl = SA_AUTH_SW_CTRL_SHA1; 1577 1578 sa_sha_setup(ctx, &ad); 1579 1580 return 0; 1581 } 1582 1583 static int sa_sha256_cra_init(struct crypto_tfm *tfm) 1584 { 1585 struct algo_data ad = { 0 }; 1586 struct sa_tfm_ctx *ctx = crypto_tfm_ctx(tfm); 1587 1588 sa_sha_cra_init_alg(tfm, "sha256"); 1589 1590 ad.aalg_id = SA_AALG_ID_SHA2_256; 1591 ad.hash_size = SHA256_DIGEST_SIZE; 1592 ad.auth_ctrl = SA_AUTH_SW_CTRL_SHA256; 1593 1594 sa_sha_setup(ctx, &ad); 1595 1596 return 0; 1597 } 1598 1599 static int sa_sha512_cra_init(struct crypto_tfm *tfm) 1600 { 1601 struct algo_data ad = { 0 }; 1602 struct sa_tfm_ctx *ctx = crypto_tfm_ctx(tfm); 1603 1604 sa_sha_cra_init_alg(tfm, "sha512"); 1605 1606 ad.aalg_id = SA_AALG_ID_SHA2_512; 1607 ad.hash_size = SHA512_DIGEST_SIZE; 1608 ad.auth_ctrl = SA_AUTH_SW_CTRL_SHA512; 1609 1610 sa_sha_setup(ctx, &ad); 1611 1612 return 0; 1613 } 1614 1615 static void sa_sha_cra_exit(struct crypto_tfm *tfm) 1616 { 1617 struct sa_tfm_ctx *ctx = crypto_tfm_ctx(tfm); 1618 struct sa_crypto_data *data = dev_get_drvdata(sa_k3_dev); 1619 1620 dev_dbg(sa_k3_dev, "%s(0x%p) sc-ids(0x%x(0x%pad), 0x%x(0x%pad))\n", 1621 __func__, tfm, ctx->enc.sc_id, &ctx->enc.sc_phys, 1622 ctx->dec.sc_id, &ctx->dec.sc_phys); 1623 1624 if (crypto_tfm_alg_type(tfm) == CRYPTO_ALG_TYPE_AHASH) 1625 sa_free_ctx_info(&ctx->enc, data); 1626 1627 crypto_free_shash(ctx->shash); 1628 crypto_free_ahash(ctx->fallback.ahash); 1629 } 1630 1631 static void sa_aead_dma_in_callback(void *data) 1632 { 1633 struct sa_rx_data *rxd = (struct sa_rx_data *)data; 1634 struct aead_request *req; 1635 struct crypto_aead *tfm; 1636 unsigned int start; 1637 unsigned int authsize; 1638 u8 auth_tag[SA_MAX_AUTH_TAG_SZ]; 1639 size_t pl, ml; 1640 int i, sglen; 1641 int err = 0; 1642 u16 auth_len; 1643 u32 *mdptr; 1644 bool diff_dst; 1645 enum dma_data_direction dir_src; 1646 1647 req = container_of(rxd->req, struct aead_request, base); 1648 tfm = crypto_aead_reqtfm(req); 1649 start = req->assoclen + req->cryptlen; 1650 authsize = crypto_aead_authsize(tfm); 1651 1652 diff_dst = (req->src != req->dst) ? true : false; 1653 dir_src = diff_dst ? DMA_TO_DEVICE : DMA_BIDIRECTIONAL; 1654 1655 mdptr = (u32 *)dmaengine_desc_get_metadata_ptr(rxd->tx_in, &pl, &ml); 1656 for (i = 0; i < (authsize / 4); i++) 1657 mdptr[i + 4] = swab32(mdptr[i + 4]); 1658 1659 auth_len = req->assoclen + req->cryptlen; 1660 if (!rxd->enc) 1661 auth_len -= authsize; 1662 1663 sglen = sg_nents_for_len(rxd->src, auth_len); 1664 dma_unmap_sg(rxd->ddev, rxd->src, sglen, dir_src); 1665 kfree(rxd->split_src_sg); 1666 1667 if (diff_dst) { 1668 sglen = sg_nents_for_len(rxd->dst, auth_len); 1669 dma_unmap_sg(rxd->ddev, rxd->dst, sglen, DMA_FROM_DEVICE); 1670 kfree(rxd->split_dst_sg); 1671 } 1672 1673 if (rxd->enc) { 1674 scatterwalk_map_and_copy(&mdptr[4], req->dst, start, authsize, 1675 1); 1676 } else { 1677 start -= authsize; 1678 scatterwalk_map_and_copy(auth_tag, req->src, start, authsize, 1679 0); 1680 1681 err = memcmp(&mdptr[4], auth_tag, authsize) ? -EBADMSG : 0; 1682 } 1683 1684 kfree(rxd); 1685 1686 aead_request_complete(req, err); 1687 } 1688 1689 static int sa_cra_init_aead(struct crypto_aead *tfm, const char *hash, 1690 const char *fallback) 1691 { 1692 struct sa_tfm_ctx *ctx = crypto_aead_ctx(tfm); 1693 struct sa_crypto_data *data = dev_get_drvdata(sa_k3_dev); 1694 int ret; 1695 1696 memzero_explicit(ctx, sizeof(*ctx)); 1697 1698 ctx->shash = crypto_alloc_shash(hash, 0, CRYPTO_ALG_NEED_FALLBACK); 1699 if (IS_ERR(ctx->shash)) { 1700 dev_err(sa_k3_dev, "base driver %s couldn't be loaded\n", hash); 1701 return PTR_ERR(ctx->shash); 1702 } 1703 1704 ctx->fallback.aead = crypto_alloc_aead(fallback, 0, 1705 CRYPTO_ALG_NEED_FALLBACK); 1706 1707 if (IS_ERR(ctx->fallback.aead)) { 1708 dev_err(sa_k3_dev, "fallback driver %s couldn't be loaded\n", 1709 fallback); 1710 return PTR_ERR(ctx->fallback.aead); 1711 } 1712 1713 crypto_aead_set_reqsize(tfm, sizeof(struct aead_request) + 1714 crypto_aead_reqsize(ctx->fallback.aead)); 1715 1716 ret = sa_init_ctx_info(&ctx->enc, data); 1717 if (ret) 1718 return ret; 1719 1720 ret = sa_init_ctx_info(&ctx->dec, data); 1721 if (ret) { 1722 sa_free_ctx_info(&ctx->enc, data); 1723 return ret; 1724 } 1725 1726 dev_dbg(sa_k3_dev, "%s(0x%p) sc-ids(0x%x(0x%pad), 0x%x(0x%pad))\n", 1727 __func__, tfm, ctx->enc.sc_id, &ctx->enc.sc_phys, 1728 ctx->dec.sc_id, &ctx->dec.sc_phys); 1729 1730 return ret; 1731 } 1732 1733 static int sa_cra_init_aead_sha1(struct crypto_aead *tfm) 1734 { 1735 return sa_cra_init_aead(tfm, "sha1", 1736 "authenc(hmac(sha1-ce),cbc(aes-ce))"); 1737 } 1738 1739 static int sa_cra_init_aead_sha256(struct crypto_aead *tfm) 1740 { 1741 return sa_cra_init_aead(tfm, "sha256", 1742 "authenc(hmac(sha256-ce),cbc(aes-ce))"); 1743 } 1744 1745 static void sa_exit_tfm_aead(struct crypto_aead *tfm) 1746 { 1747 struct sa_tfm_ctx *ctx = crypto_aead_ctx(tfm); 1748 struct sa_crypto_data *data = dev_get_drvdata(sa_k3_dev); 1749 1750 crypto_free_shash(ctx->shash); 1751 crypto_free_aead(ctx->fallback.aead); 1752 1753 sa_free_ctx_info(&ctx->enc, data); 1754 sa_free_ctx_info(&ctx->dec, data); 1755 } 1756 1757 /* AEAD algorithm configuration interface function */ 1758 static int sa_aead_setkey(struct crypto_aead *authenc, 1759 const u8 *key, unsigned int keylen, 1760 struct algo_data *ad) 1761 { 1762 struct sa_tfm_ctx *ctx = crypto_aead_ctx(authenc); 1763 struct crypto_authenc_keys keys; 1764 int cmdl_len; 1765 struct sa_cmdl_cfg cfg; 1766 int key_idx; 1767 1768 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) 1769 return -EINVAL; 1770 1771 /* Convert the key size (16/24/32) to the key size index (0/1/2) */ 1772 key_idx = (keys.enckeylen >> 3) - 2; 1773 if (key_idx >= 3) 1774 return -EINVAL; 1775 1776 ad->ctx = ctx; 1777 ad->enc_eng.eng_id = SA_ENG_ID_EM1; 1778 ad->enc_eng.sc_size = SA_CTX_ENC_TYPE1_SZ; 1779 ad->auth_eng.eng_id = SA_ENG_ID_AM1; 1780 ad->auth_eng.sc_size = SA_CTX_AUTH_TYPE2_SZ; 1781 ad->mci_enc = mci_cbc_enc_no_iv_array[key_idx]; 1782 ad->mci_dec = mci_cbc_dec_no_iv_array[key_idx]; 1783 ad->inv_key = true; 1784 ad->keyed_mac = true; 1785 ad->ealg_id = SA_EALG_ID_AES_CBC; 1786 ad->prep_iopad = sa_prepare_iopads; 1787 1788 memset(&cfg, 0, sizeof(cfg)); 1789 cfg.enc = true; 1790 cfg.aalg = ad->aalg_id; 1791 cfg.enc_eng_id = ad->enc_eng.eng_id; 1792 cfg.auth_eng_id = ad->auth_eng.eng_id; 1793 cfg.iv_size = crypto_aead_ivsize(authenc); 1794 cfg.akey = keys.authkey; 1795 cfg.akey_len = keys.authkeylen; 1796 1797 /* Setup Encryption Security Context & Command label template */ 1798 if (sa_init_sc(&ctx->enc, keys.enckey, keys.enckeylen, 1799 keys.authkey, keys.authkeylen, 1800 ad, 1, &ctx->enc.epib[1])) 1801 return -EINVAL; 1802 1803 cmdl_len = sa_format_cmdl_gen(&cfg, 1804 (u8 *)ctx->enc.cmdl, 1805 &ctx->enc.cmdl_upd_info); 1806 if (cmdl_len <= 0 || (cmdl_len > SA_MAX_CMDL_WORDS * sizeof(u32))) 1807 return -EINVAL; 1808 1809 ctx->enc.cmdl_size = cmdl_len; 1810 1811 /* Setup Decryption Security Context & Command label template */ 1812 if (sa_init_sc(&ctx->dec, keys.enckey, keys.enckeylen, 1813 keys.authkey, keys.authkeylen, 1814 ad, 0, &ctx->dec.epib[1])) 1815 return -EINVAL; 1816 1817 cfg.enc = false; 1818 cmdl_len = sa_format_cmdl_gen(&cfg, (u8 *)ctx->dec.cmdl, 1819 &ctx->dec.cmdl_upd_info); 1820 1821 if (cmdl_len <= 0 || (cmdl_len > SA_MAX_CMDL_WORDS * sizeof(u32))) 1822 return -EINVAL; 1823 1824 ctx->dec.cmdl_size = cmdl_len; 1825 1826 crypto_aead_clear_flags(ctx->fallback.aead, CRYPTO_TFM_REQ_MASK); 1827 crypto_aead_set_flags(ctx->fallback.aead, 1828 crypto_aead_get_flags(authenc) & 1829 CRYPTO_TFM_REQ_MASK); 1830 crypto_aead_setkey(ctx->fallback.aead, key, keylen); 1831 1832 return 0; 1833 } 1834 1835 static int sa_aead_setauthsize(struct crypto_aead *tfm, unsigned int authsize) 1836 { 1837 struct sa_tfm_ctx *ctx = crypto_tfm_ctx(crypto_aead_tfm(tfm)); 1838 1839 return crypto_aead_setauthsize(ctx->fallback.aead, authsize); 1840 } 1841 1842 static int sa_aead_cbc_sha1_setkey(struct crypto_aead *authenc, 1843 const u8 *key, unsigned int keylen) 1844 { 1845 struct algo_data ad = { 0 }; 1846 1847 ad.ealg_id = SA_EALG_ID_AES_CBC; 1848 ad.aalg_id = SA_AALG_ID_HMAC_SHA1; 1849 ad.hash_size = SHA1_DIGEST_SIZE; 1850 ad.auth_ctrl = SA_AUTH_SW_CTRL_SHA1; 1851 1852 return sa_aead_setkey(authenc, key, keylen, &ad); 1853 } 1854 1855 static int sa_aead_cbc_sha256_setkey(struct crypto_aead *authenc, 1856 const u8 *key, unsigned int keylen) 1857 { 1858 struct algo_data ad = { 0 }; 1859 1860 ad.ealg_id = SA_EALG_ID_AES_CBC; 1861 ad.aalg_id = SA_AALG_ID_HMAC_SHA2_256; 1862 ad.hash_size = SHA256_DIGEST_SIZE; 1863 ad.auth_ctrl = SA_AUTH_SW_CTRL_SHA256; 1864 1865 return sa_aead_setkey(authenc, key, keylen, &ad); 1866 } 1867 1868 static int sa_aead_run(struct aead_request *req, u8 *iv, int enc) 1869 { 1870 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1871 struct sa_tfm_ctx *ctx = crypto_aead_ctx(tfm); 1872 struct sa_req sa_req = { 0 }; 1873 size_t auth_size, enc_size; 1874 1875 enc_size = req->cryptlen; 1876 auth_size = req->assoclen + req->cryptlen; 1877 1878 if (!enc) { 1879 enc_size -= crypto_aead_authsize(tfm); 1880 auth_size -= crypto_aead_authsize(tfm); 1881 } 1882 1883 if (auth_size > SA_MAX_DATA_SZ || 1884 (auth_size >= SA_UNSAFE_DATA_SZ_MIN && 1885 auth_size <= SA_UNSAFE_DATA_SZ_MAX)) { 1886 struct aead_request *subreq = aead_request_ctx(req); 1887 int ret; 1888 1889 aead_request_set_tfm(subreq, ctx->fallback.aead); 1890 aead_request_set_callback(subreq, req->base.flags, 1891 req->base.complete, req->base.data); 1892 aead_request_set_crypt(subreq, req->src, req->dst, 1893 req->cryptlen, req->iv); 1894 aead_request_set_ad(subreq, req->assoclen); 1895 1896 ret = enc ? crypto_aead_encrypt(subreq) : 1897 crypto_aead_decrypt(subreq); 1898 return ret; 1899 } 1900 1901 sa_req.enc_offset = req->assoclen; 1902 sa_req.enc_size = enc_size; 1903 sa_req.auth_size = auth_size; 1904 sa_req.size = auth_size; 1905 sa_req.enc_iv = iv; 1906 sa_req.type = CRYPTO_ALG_TYPE_AEAD; 1907 sa_req.enc = enc; 1908 sa_req.callback = sa_aead_dma_in_callback; 1909 sa_req.mdata_size = 52; 1910 sa_req.base = &req->base; 1911 sa_req.ctx = ctx; 1912 sa_req.src = req->src; 1913 sa_req.dst = req->dst; 1914 1915 return sa_run(&sa_req); 1916 } 1917 1918 /* AEAD algorithm encrypt interface function */ 1919 static int sa_aead_encrypt(struct aead_request *req) 1920 { 1921 return sa_aead_run(req, req->iv, 1); 1922 } 1923 1924 /* AEAD algorithm decrypt interface function */ 1925 static int sa_aead_decrypt(struct aead_request *req) 1926 { 1927 return sa_aead_run(req, req->iv, 0); 1928 } 1929 1930 static struct sa_alg_tmpl sa_algs[] = { 1931 { 1932 .type = CRYPTO_ALG_TYPE_SKCIPHER, 1933 .alg.skcipher = { 1934 .base.cra_name = "cbc(aes)", 1935 .base.cra_driver_name = "cbc-aes-sa2ul", 1936 .base.cra_priority = 30000, 1937 .base.cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | 1938 CRYPTO_ALG_KERN_DRIVER_ONLY | 1939 CRYPTO_ALG_ASYNC | 1940 CRYPTO_ALG_NEED_FALLBACK, 1941 .base.cra_blocksize = AES_BLOCK_SIZE, 1942 .base.cra_ctxsize = sizeof(struct sa_tfm_ctx), 1943 .base.cra_module = THIS_MODULE, 1944 .init = sa_cipher_cra_init, 1945 .exit = sa_cipher_cra_exit, 1946 .min_keysize = AES_MIN_KEY_SIZE, 1947 .max_keysize = AES_MAX_KEY_SIZE, 1948 .ivsize = AES_BLOCK_SIZE, 1949 .setkey = sa_aes_cbc_setkey, 1950 .encrypt = sa_encrypt, 1951 .decrypt = sa_decrypt, 1952 } 1953 }, 1954 { 1955 .type = CRYPTO_ALG_TYPE_SKCIPHER, 1956 .alg.skcipher = { 1957 .base.cra_name = "ecb(aes)", 1958 .base.cra_driver_name = "ecb-aes-sa2ul", 1959 .base.cra_priority = 30000, 1960 .base.cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | 1961 CRYPTO_ALG_KERN_DRIVER_ONLY | 1962 CRYPTO_ALG_ASYNC | 1963 CRYPTO_ALG_NEED_FALLBACK, 1964 .base.cra_blocksize = AES_BLOCK_SIZE, 1965 .base.cra_ctxsize = sizeof(struct sa_tfm_ctx), 1966 .base.cra_module = THIS_MODULE, 1967 .init = sa_cipher_cra_init, 1968 .exit = sa_cipher_cra_exit, 1969 .min_keysize = AES_MIN_KEY_SIZE, 1970 .max_keysize = AES_MAX_KEY_SIZE, 1971 .setkey = sa_aes_ecb_setkey, 1972 .encrypt = sa_encrypt, 1973 .decrypt = sa_decrypt, 1974 } 1975 }, 1976 { 1977 .type = CRYPTO_ALG_TYPE_SKCIPHER, 1978 .alg.skcipher = { 1979 .base.cra_name = "cbc(des3_ede)", 1980 .base.cra_driver_name = "cbc-des3-sa2ul", 1981 .base.cra_priority = 30000, 1982 .base.cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | 1983 CRYPTO_ALG_KERN_DRIVER_ONLY | 1984 CRYPTO_ALG_ASYNC | 1985 CRYPTO_ALG_NEED_FALLBACK, 1986 .base.cra_blocksize = DES_BLOCK_SIZE, 1987 .base.cra_ctxsize = sizeof(struct sa_tfm_ctx), 1988 .base.cra_module = THIS_MODULE, 1989 .init = sa_cipher_cra_init, 1990 .exit = sa_cipher_cra_exit, 1991 .min_keysize = 3 * DES_KEY_SIZE, 1992 .max_keysize = 3 * DES_KEY_SIZE, 1993 .ivsize = DES_BLOCK_SIZE, 1994 .setkey = sa_3des_cbc_setkey, 1995 .encrypt = sa_encrypt, 1996 .decrypt = sa_decrypt, 1997 } 1998 }, 1999 { 2000 .type = CRYPTO_ALG_TYPE_SKCIPHER, 2001 .alg.skcipher = { 2002 .base.cra_name = "ecb(des3_ede)", 2003 .base.cra_driver_name = "ecb-des3-sa2ul", 2004 .base.cra_priority = 30000, 2005 .base.cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | 2006 CRYPTO_ALG_KERN_DRIVER_ONLY | 2007 CRYPTO_ALG_ASYNC | 2008 CRYPTO_ALG_NEED_FALLBACK, 2009 .base.cra_blocksize = DES_BLOCK_SIZE, 2010 .base.cra_ctxsize = sizeof(struct sa_tfm_ctx), 2011 .base.cra_module = THIS_MODULE, 2012 .init = sa_cipher_cra_init, 2013 .exit = sa_cipher_cra_exit, 2014 .min_keysize = 3 * DES_KEY_SIZE, 2015 .max_keysize = 3 * DES_KEY_SIZE, 2016 .setkey = sa_3des_ecb_setkey, 2017 .encrypt = sa_encrypt, 2018 .decrypt = sa_decrypt, 2019 } 2020 }, 2021 { 2022 .type = CRYPTO_ALG_TYPE_AHASH, 2023 .alg.ahash = { 2024 .halg.base = { 2025 .cra_name = "sha1", 2026 .cra_driver_name = "sha1-sa2ul", 2027 .cra_priority = 400, 2028 .cra_flags = CRYPTO_ALG_TYPE_AHASH | 2029 CRYPTO_ALG_ASYNC | 2030 CRYPTO_ALG_KERN_DRIVER_ONLY | 2031 CRYPTO_ALG_NEED_FALLBACK, 2032 .cra_blocksize = SHA1_BLOCK_SIZE, 2033 .cra_ctxsize = sizeof(struct sa_tfm_ctx), 2034 .cra_module = THIS_MODULE, 2035 .cra_init = sa_sha1_cra_init, 2036 .cra_exit = sa_sha_cra_exit, 2037 }, 2038 .halg.digestsize = SHA1_DIGEST_SIZE, 2039 .halg.statesize = sizeof(struct sa_sha_req_ctx) + 2040 sizeof(struct sha1_state), 2041 .init = sa_sha_init, 2042 .update = sa_sha_update, 2043 .final = sa_sha_final, 2044 .finup = sa_sha_finup, 2045 .digest = sa_sha_digest, 2046 .export = sa_sha_export, 2047 .import = sa_sha_import, 2048 }, 2049 }, 2050 { 2051 .type = CRYPTO_ALG_TYPE_AHASH, 2052 .alg.ahash = { 2053 .halg.base = { 2054 .cra_name = "sha256", 2055 .cra_driver_name = "sha256-sa2ul", 2056 .cra_priority = 400, 2057 .cra_flags = CRYPTO_ALG_TYPE_AHASH | 2058 CRYPTO_ALG_ASYNC | 2059 CRYPTO_ALG_KERN_DRIVER_ONLY | 2060 CRYPTO_ALG_NEED_FALLBACK, 2061 .cra_blocksize = SHA256_BLOCK_SIZE, 2062 .cra_ctxsize = sizeof(struct sa_tfm_ctx), 2063 .cra_module = THIS_MODULE, 2064 .cra_init = sa_sha256_cra_init, 2065 .cra_exit = sa_sha_cra_exit, 2066 }, 2067 .halg.digestsize = SHA256_DIGEST_SIZE, 2068 .halg.statesize = sizeof(struct sa_sha_req_ctx) + 2069 sizeof(struct sha256_state), 2070 .init = sa_sha_init, 2071 .update = sa_sha_update, 2072 .final = sa_sha_final, 2073 .finup = sa_sha_finup, 2074 .digest = sa_sha_digest, 2075 .export = sa_sha_export, 2076 .import = sa_sha_import, 2077 }, 2078 }, 2079 { 2080 .type = CRYPTO_ALG_TYPE_AHASH, 2081 .alg.ahash = { 2082 .halg.base = { 2083 .cra_name = "sha512", 2084 .cra_driver_name = "sha512-sa2ul", 2085 .cra_priority = 400, 2086 .cra_flags = CRYPTO_ALG_TYPE_AHASH | 2087 CRYPTO_ALG_ASYNC | 2088 CRYPTO_ALG_KERN_DRIVER_ONLY | 2089 CRYPTO_ALG_NEED_FALLBACK, 2090 .cra_blocksize = SHA512_BLOCK_SIZE, 2091 .cra_ctxsize = sizeof(struct sa_tfm_ctx), 2092 .cra_module = THIS_MODULE, 2093 .cra_init = sa_sha512_cra_init, 2094 .cra_exit = sa_sha_cra_exit, 2095 }, 2096 .halg.digestsize = SHA512_DIGEST_SIZE, 2097 .halg.statesize = sizeof(struct sa_sha_req_ctx) + 2098 sizeof(struct sha512_state), 2099 .init = sa_sha_init, 2100 .update = sa_sha_update, 2101 .final = sa_sha_final, 2102 .finup = sa_sha_finup, 2103 .digest = sa_sha_digest, 2104 .export = sa_sha_export, 2105 .import = sa_sha_import, 2106 }, 2107 }, 2108 { 2109 .type = CRYPTO_ALG_TYPE_AEAD, 2110 .alg.aead = { 2111 .base = { 2112 .cra_name = "authenc(hmac(sha1),cbc(aes))", 2113 .cra_driver_name = 2114 "authenc(hmac(sha1),cbc(aes))-sa2ul", 2115 .cra_blocksize = AES_BLOCK_SIZE, 2116 .cra_flags = CRYPTO_ALG_TYPE_AEAD | 2117 CRYPTO_ALG_KERN_DRIVER_ONLY | 2118 CRYPTO_ALG_ASYNC | 2119 CRYPTO_ALG_NEED_FALLBACK, 2120 .cra_ctxsize = sizeof(struct sa_tfm_ctx), 2121 .cra_module = THIS_MODULE, 2122 .cra_priority = 3000, 2123 }, 2124 .ivsize = AES_BLOCK_SIZE, 2125 .maxauthsize = SHA1_DIGEST_SIZE, 2126 2127 .init = sa_cra_init_aead_sha1, 2128 .exit = sa_exit_tfm_aead, 2129 .setkey = sa_aead_cbc_sha1_setkey, 2130 .setauthsize = sa_aead_setauthsize, 2131 .encrypt = sa_aead_encrypt, 2132 .decrypt = sa_aead_decrypt, 2133 }, 2134 }, 2135 { 2136 .type = CRYPTO_ALG_TYPE_AEAD, 2137 .alg.aead = { 2138 .base = { 2139 .cra_name = "authenc(hmac(sha256),cbc(aes))", 2140 .cra_driver_name = 2141 "authenc(hmac(sha256),cbc(aes))-sa2ul", 2142 .cra_blocksize = AES_BLOCK_SIZE, 2143 .cra_flags = CRYPTO_ALG_TYPE_AEAD | 2144 CRYPTO_ALG_KERN_DRIVER_ONLY | 2145 CRYPTO_ALG_ASYNC | 2146 CRYPTO_ALG_NEED_FALLBACK, 2147 .cra_ctxsize = sizeof(struct sa_tfm_ctx), 2148 .cra_module = THIS_MODULE, 2149 .cra_alignmask = 0, 2150 .cra_priority = 3000, 2151 }, 2152 .ivsize = AES_BLOCK_SIZE, 2153 .maxauthsize = SHA256_DIGEST_SIZE, 2154 2155 .init = sa_cra_init_aead_sha256, 2156 .exit = sa_exit_tfm_aead, 2157 .setkey = sa_aead_cbc_sha256_setkey, 2158 .setauthsize = sa_aead_setauthsize, 2159 .encrypt = sa_aead_encrypt, 2160 .decrypt = sa_aead_decrypt, 2161 }, 2162 }, 2163 }; 2164 2165 /* Register the algorithms in crypto framework */ 2166 static void sa_register_algos(const struct device *dev) 2167 { 2168 char *alg_name; 2169 u32 type; 2170 int i, err; 2171 2172 for (i = 0; i < ARRAY_SIZE(sa_algs); i++) { 2173 type = sa_algs[i].type; 2174 if (type == CRYPTO_ALG_TYPE_SKCIPHER) { 2175 alg_name = sa_algs[i].alg.skcipher.base.cra_name; 2176 err = crypto_register_skcipher(&sa_algs[i].alg.skcipher); 2177 } else if (type == CRYPTO_ALG_TYPE_AHASH) { 2178 alg_name = sa_algs[i].alg.ahash.halg.base.cra_name; 2179 err = crypto_register_ahash(&sa_algs[i].alg.ahash); 2180 } else if (type == CRYPTO_ALG_TYPE_AEAD) { 2181 alg_name = sa_algs[i].alg.aead.base.cra_name; 2182 err = crypto_register_aead(&sa_algs[i].alg.aead); 2183 } else { 2184 dev_err(dev, 2185 "un-supported crypto algorithm (%d)", 2186 sa_algs[i].type); 2187 continue; 2188 } 2189 2190 if (err) 2191 dev_err(dev, "Failed to register '%s'\n", alg_name); 2192 else 2193 sa_algs[i].registered = true; 2194 } 2195 } 2196 2197 /* Unregister the algorithms in crypto framework */ 2198 static void sa_unregister_algos(const struct device *dev) 2199 { 2200 u32 type; 2201 int i; 2202 2203 for (i = 0; i < ARRAY_SIZE(sa_algs); i++) { 2204 type = sa_algs[i].type; 2205 if (!sa_algs[i].registered) 2206 continue; 2207 if (type == CRYPTO_ALG_TYPE_SKCIPHER) 2208 crypto_unregister_skcipher(&sa_algs[i].alg.skcipher); 2209 else if (type == CRYPTO_ALG_TYPE_AHASH) 2210 crypto_unregister_ahash(&sa_algs[i].alg.ahash); 2211 else if (type == CRYPTO_ALG_TYPE_AEAD) 2212 crypto_unregister_aead(&sa_algs[i].alg.aead); 2213 2214 sa_algs[i].registered = false; 2215 } 2216 } 2217 2218 static int sa_init_mem(struct sa_crypto_data *dev_data) 2219 { 2220 struct device *dev = &dev_data->pdev->dev; 2221 /* Setup dma pool for security context buffers */ 2222 dev_data->sc_pool = dma_pool_create("keystone-sc", dev, 2223 SA_CTX_MAX_SZ, 64, 0); 2224 if (!dev_data->sc_pool) { 2225 dev_err(dev, "Failed to create dma pool"); 2226 return -ENOMEM; 2227 } 2228 2229 return 0; 2230 } 2231 2232 static int sa_dma_init(struct sa_crypto_data *dd) 2233 { 2234 int ret; 2235 struct dma_slave_config cfg; 2236 2237 dd->dma_rx1 = NULL; 2238 dd->dma_tx = NULL; 2239 dd->dma_rx2 = NULL; 2240 2241 ret = dma_coerce_mask_and_coherent(dd->dev, DMA_BIT_MASK(48)); 2242 if (ret) 2243 return ret; 2244 2245 dd->dma_rx1 = dma_request_chan(dd->dev, "rx1"); 2246 if (IS_ERR(dd->dma_rx1)) { 2247 if (PTR_ERR(dd->dma_rx1) != -EPROBE_DEFER) 2248 dev_err(dd->dev, "Unable to request rx1 DMA channel\n"); 2249 return PTR_ERR(dd->dma_rx1); 2250 } 2251 2252 dd->dma_rx2 = dma_request_chan(dd->dev, "rx2"); 2253 if (IS_ERR(dd->dma_rx2)) { 2254 dma_release_channel(dd->dma_rx1); 2255 if (PTR_ERR(dd->dma_rx2) != -EPROBE_DEFER) 2256 dev_err(dd->dev, "Unable to request rx2 DMA channel\n"); 2257 return PTR_ERR(dd->dma_rx2); 2258 } 2259 2260 dd->dma_tx = dma_request_chan(dd->dev, "tx"); 2261 if (IS_ERR(dd->dma_tx)) { 2262 if (PTR_ERR(dd->dma_tx) != -EPROBE_DEFER) 2263 dev_err(dd->dev, "Unable to request tx DMA channel\n"); 2264 ret = PTR_ERR(dd->dma_tx); 2265 goto err_dma_tx; 2266 } 2267 2268 memzero_explicit(&cfg, sizeof(cfg)); 2269 2270 cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; 2271 cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; 2272 cfg.src_maxburst = 4; 2273 cfg.dst_maxburst = 4; 2274 2275 ret = dmaengine_slave_config(dd->dma_rx1, &cfg); 2276 if (ret) { 2277 dev_err(dd->dev, "can't configure IN dmaengine slave: %d\n", 2278 ret); 2279 return ret; 2280 } 2281 2282 ret = dmaengine_slave_config(dd->dma_rx2, &cfg); 2283 if (ret) { 2284 dev_err(dd->dev, "can't configure IN dmaengine slave: %d\n", 2285 ret); 2286 return ret; 2287 } 2288 2289 ret = dmaengine_slave_config(dd->dma_tx, &cfg); 2290 if (ret) { 2291 dev_err(dd->dev, "can't configure OUT dmaengine slave: %d\n", 2292 ret); 2293 return ret; 2294 } 2295 2296 return 0; 2297 2298 err_dma_tx: 2299 dma_release_channel(dd->dma_rx1); 2300 dma_release_channel(dd->dma_rx2); 2301 2302 return ret; 2303 } 2304 2305 static int sa_link_child(struct device *dev, void *data) 2306 { 2307 struct device *parent = data; 2308 2309 device_link_add(dev, parent, DL_FLAG_AUTOPROBE_CONSUMER); 2310 2311 return 0; 2312 } 2313 2314 static int sa_ul_probe(struct platform_device *pdev) 2315 { 2316 struct device *dev = &pdev->dev; 2317 struct device_node *node = dev->of_node; 2318 struct resource *res; 2319 static void __iomem *saul_base; 2320 struct sa_crypto_data *dev_data; 2321 u32 val; 2322 int ret; 2323 2324 dev_data = devm_kzalloc(dev, sizeof(*dev_data), GFP_KERNEL); 2325 if (!dev_data) 2326 return -ENOMEM; 2327 2328 sa_k3_dev = dev; 2329 dev_data->dev = dev; 2330 dev_data->pdev = pdev; 2331 platform_set_drvdata(pdev, dev_data); 2332 dev_set_drvdata(sa_k3_dev, dev_data); 2333 2334 pm_runtime_enable(dev); 2335 ret = pm_runtime_get_sync(dev); 2336 if (ret) { 2337 dev_err(&pdev->dev, "%s: failed to get sync: %d\n", __func__, 2338 ret); 2339 return ret; 2340 } 2341 2342 sa_init_mem(dev_data); 2343 ret = sa_dma_init(dev_data); 2344 if (ret) 2345 goto disable_pm_runtime; 2346 2347 spin_lock_init(&dev_data->scid_lock); 2348 res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 2349 saul_base = devm_ioremap_resource(dev, res); 2350 2351 dev_data->base = saul_base; 2352 val = SA_EEC_ENCSS_EN | SA_EEC_AUTHSS_EN | SA_EEC_CTXCACH_EN | 2353 SA_EEC_CPPI_PORT_IN_EN | SA_EEC_CPPI_PORT_OUT_EN | 2354 SA_EEC_TRNG_EN; 2355 2356 writel_relaxed(val, saul_base + SA_ENGINE_ENABLE_CONTROL); 2357 2358 sa_register_algos(dev); 2359 2360 ret = of_platform_populate(node, NULL, NULL, &pdev->dev); 2361 if (ret) 2362 goto release_dma; 2363 2364 device_for_each_child(&pdev->dev, &pdev->dev, sa_link_child); 2365 2366 return 0; 2367 2368 release_dma: 2369 sa_unregister_algos(&pdev->dev); 2370 2371 dma_release_channel(dev_data->dma_rx2); 2372 dma_release_channel(dev_data->dma_rx1); 2373 dma_release_channel(dev_data->dma_tx); 2374 2375 dma_pool_destroy(dev_data->sc_pool); 2376 2377 disable_pm_runtime: 2378 pm_runtime_put_sync(&pdev->dev); 2379 pm_runtime_disable(&pdev->dev); 2380 2381 return ret; 2382 } 2383 2384 static int sa_ul_remove(struct platform_device *pdev) 2385 { 2386 struct sa_crypto_data *dev_data = platform_get_drvdata(pdev); 2387 2388 sa_unregister_algos(&pdev->dev); 2389 2390 dma_release_channel(dev_data->dma_rx2); 2391 dma_release_channel(dev_data->dma_rx1); 2392 dma_release_channel(dev_data->dma_tx); 2393 2394 dma_pool_destroy(dev_data->sc_pool); 2395 2396 platform_set_drvdata(pdev, NULL); 2397 2398 pm_runtime_put_sync(&pdev->dev); 2399 pm_runtime_disable(&pdev->dev); 2400 2401 return 0; 2402 } 2403 2404 static const struct of_device_id of_match[] = { 2405 {.compatible = "ti,j721e-sa2ul",}, 2406 {.compatible = "ti,am654-sa2ul",}, 2407 {}, 2408 }; 2409 MODULE_DEVICE_TABLE(of, of_match); 2410 2411 static struct platform_driver sa_ul_driver = { 2412 .probe = sa_ul_probe, 2413 .remove = sa_ul_remove, 2414 .driver = { 2415 .name = "saul-crypto", 2416 .of_match_table = of_match, 2417 }, 2418 }; 2419 module_platform_driver(sa_ul_driver); 2420 MODULE_LICENSE("GPL v2"); 2421