1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * K3 SA2UL crypto accelerator driver 4 * 5 * Copyright (C) 2018-2020 Texas Instruments Incorporated - http://www.ti.com 6 * 7 * Authors: Keerthy 8 * Vitaly Andrianov 9 * Tero Kristo 10 */ 11 #include <linux/clk.h> 12 #include <linux/dmaengine.h> 13 #include <linux/dmapool.h> 14 #include <linux/module.h> 15 #include <linux/of_device.h> 16 #include <linux/platform_device.h> 17 #include <linux/pm_runtime.h> 18 19 #include <crypto/aes.h> 20 #include <crypto/authenc.h> 21 #include <crypto/des.h> 22 #include <crypto/internal/aead.h> 23 #include <crypto/internal/hash.h> 24 #include <crypto/internal/skcipher.h> 25 #include <crypto/scatterwalk.h> 26 #include <crypto/sha.h> 27 28 #include "sa2ul.h" 29 30 /* Byte offset for key in encryption security context */ 31 #define SC_ENC_KEY_OFFSET (1 + 27 + 4) 32 /* Byte offset for Aux-1 in encryption security context */ 33 #define SC_ENC_AUX1_OFFSET (1 + 27 + 4 + 32) 34 35 #define SA_CMDL_UPD_ENC 0x0001 36 #define SA_CMDL_UPD_AUTH 0x0002 37 #define SA_CMDL_UPD_ENC_IV 0x0004 38 #define SA_CMDL_UPD_AUTH_IV 0x0008 39 #define SA_CMDL_UPD_AUX_KEY 0x0010 40 41 #define SA_AUTH_SUBKEY_LEN 16 42 #define SA_CMDL_PAYLOAD_LENGTH_MASK 0xFFFF 43 #define SA_CMDL_SOP_BYPASS_LEN_MASK 0xFF000000 44 45 #define MODE_CONTROL_BYTES 27 46 #define SA_HASH_PROCESSING 0 47 #define SA_CRYPTO_PROCESSING 0 48 #define SA_UPLOAD_HASH_TO_TLR BIT(6) 49 50 #define SA_SW0_FLAGS_MASK 0xF0000 51 #define SA_SW0_CMDL_INFO_MASK 0x1F00000 52 #define SA_SW0_CMDL_PRESENT BIT(4) 53 #define SA_SW0_ENG_ID_MASK 0x3E000000 54 #define SA_SW0_DEST_INFO_PRESENT BIT(30) 55 #define SA_SW2_EGRESS_LENGTH 0xFF000000 56 #define SA_BASIC_HASH 0x10 57 58 #define SHA256_DIGEST_WORDS 8 59 /* Make 32-bit word from 4 bytes */ 60 #define SA_MK_U32(b0, b1, b2, b3) (((b0) << 24) | ((b1) << 16) | \ 61 ((b2) << 8) | (b3)) 62 63 /* size of SCCTL structure in bytes */ 64 #define SA_SCCTL_SZ 16 65 66 /* Max Authentication tag size */ 67 #define SA_MAX_AUTH_TAG_SZ 64 68 69 #define PRIV_ID 0x1 70 #define PRIV 0x1 71 72 static struct device *sa_k3_dev; 73 74 /** 75 * struct sa_cmdl_cfg - Command label configuration descriptor 76 * @aalg: authentication algorithm ID 77 * @enc_eng_id: Encryption Engine ID supported by the SA hardware 78 * @auth_eng_id: Authentication Engine ID 79 * @iv_size: Initialization Vector size 80 * @akey: Authentication key 81 * @akey_len: Authentication key length 82 * @enc: True, if this is an encode request 83 */ 84 struct sa_cmdl_cfg { 85 int aalg; 86 u8 enc_eng_id; 87 u8 auth_eng_id; 88 u8 iv_size; 89 const u8 *akey; 90 u16 akey_len; 91 bool enc; 92 }; 93 94 /** 95 * struct algo_data - Crypto algorithm specific data 96 * @enc_eng: Encryption engine info structure 97 * @auth_eng: Authentication engine info structure 98 * @auth_ctrl: Authentication control word 99 * @hash_size: Size of digest 100 * @iv_idx: iv index in psdata 101 * @iv_out_size: iv out size 102 * @ealg_id: Encryption Algorithm ID 103 * @aalg_id: Authentication algorithm ID 104 * @mci_enc: Mode Control Instruction for Encryption algorithm 105 * @mci_dec: Mode Control Instruction for Decryption 106 * @inv_key: Whether the encryption algorithm demands key inversion 107 * @ctx: Pointer to the algorithm context 108 * @keyed_mac: Whether the authentication algorithm has key 109 * @prep_iopad: Function pointer to generate intermediate ipad/opad 110 */ 111 struct algo_data { 112 struct sa_eng_info enc_eng; 113 struct sa_eng_info auth_eng; 114 u8 auth_ctrl; 115 u8 hash_size; 116 u8 iv_idx; 117 u8 iv_out_size; 118 u8 ealg_id; 119 u8 aalg_id; 120 u8 *mci_enc; 121 u8 *mci_dec; 122 bool inv_key; 123 struct sa_tfm_ctx *ctx; 124 bool keyed_mac; 125 void (*prep_iopad)(struct algo_data *algo, const u8 *key, 126 u16 key_sz, __be32 *ipad, __be32 *opad); 127 }; 128 129 /** 130 * struct sa_alg_tmpl: A generic template encompassing crypto/aead algorithms 131 * @type: Type of the crypto algorithm. 132 * @alg: Union of crypto algorithm definitions. 133 * @registered: Flag indicating if the crypto algorithm is already registered 134 */ 135 struct sa_alg_tmpl { 136 u32 type; /* CRYPTO_ALG_TYPE from <linux/crypto.h> */ 137 union { 138 struct skcipher_alg skcipher; 139 struct ahash_alg ahash; 140 struct aead_alg aead; 141 } alg; 142 bool registered; 143 }; 144 145 /** 146 * struct sa_mapped_sg: scatterlist information for tx and rx 147 * @mapped: Set to true if the @sgt is mapped 148 * @dir: mapping direction used for @sgt 149 * @split_sg: Set if the sg is split and needs to be freed up 150 * @static_sg: Static scatterlist entry for overriding data 151 * @sgt: scatterlist table for DMA API use 152 */ 153 struct sa_mapped_sg { 154 bool mapped; 155 enum dma_data_direction dir; 156 struct scatterlist static_sg; 157 struct scatterlist *split_sg; 158 struct sg_table sgt; 159 }; 160 /** 161 * struct sa_rx_data: RX Packet miscellaneous data place holder 162 * @req: crypto request data pointer 163 * @ddev: pointer to the DMA device 164 * @tx_in: dma_async_tx_descriptor pointer for rx channel 165 * @mapped_sg: Information on tx (0) and rx (1) scatterlist DMA mapping 166 * @enc: Flag indicating either encryption or decryption 167 * @enc_iv_size: Initialisation vector size 168 * @iv_idx: Initialisation vector index 169 */ 170 struct sa_rx_data { 171 void *req; 172 struct device *ddev; 173 struct dma_async_tx_descriptor *tx_in; 174 struct sa_mapped_sg mapped_sg[2]; 175 u8 enc; 176 u8 enc_iv_size; 177 u8 iv_idx; 178 }; 179 180 /** 181 * struct sa_req: SA request definition 182 * @dev: device for the request 183 * @size: total data to the xmitted via DMA 184 * @enc_offset: offset of cipher data 185 * @enc_size: data to be passed to cipher engine 186 * @enc_iv: cipher IV 187 * @auth_offset: offset of the authentication data 188 * @auth_size: size of the authentication data 189 * @auth_iv: authentication IV 190 * @type: algorithm type for the request 191 * @cmdl: command label pointer 192 * @base: pointer to the base request 193 * @ctx: pointer to the algorithm context data 194 * @enc: true if this is an encode request 195 * @src: source data 196 * @dst: destination data 197 * @callback: DMA callback for the request 198 * @mdata_size: metadata size passed to DMA 199 */ 200 struct sa_req { 201 struct device *dev; 202 u16 size; 203 u8 enc_offset; 204 u16 enc_size; 205 u8 *enc_iv; 206 u8 auth_offset; 207 u16 auth_size; 208 u8 *auth_iv; 209 u32 type; 210 u32 *cmdl; 211 struct crypto_async_request *base; 212 struct sa_tfm_ctx *ctx; 213 bool enc; 214 struct scatterlist *src; 215 struct scatterlist *dst; 216 dma_async_tx_callback callback; 217 u16 mdata_size; 218 }; 219 220 /* 221 * Mode Control Instructions for various Key lengths 128, 192, 256 222 * For CBC (Cipher Block Chaining) mode for encryption 223 */ 224 static u8 mci_cbc_enc_array[3][MODE_CONTROL_BYTES] = { 225 { 0x61, 0x00, 0x00, 0x18, 0x88, 0x0a, 0xaa, 0x4b, 0x7e, 0x00, 226 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 227 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, 228 { 0x61, 0x00, 0x00, 0x18, 0x88, 0x4a, 0xaa, 0x4b, 0x7e, 0x00, 229 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 230 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, 231 { 0x61, 0x00, 0x00, 0x18, 0x88, 0x8a, 0xaa, 0x4b, 0x7e, 0x00, 232 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 233 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, 234 }; 235 236 /* 237 * Mode Control Instructions for various Key lengths 128, 192, 256 238 * For CBC (Cipher Block Chaining) mode for decryption 239 */ 240 static u8 mci_cbc_dec_array[3][MODE_CONTROL_BYTES] = { 241 { 0x71, 0x00, 0x00, 0x80, 0x8a, 0xca, 0x98, 0xf4, 0x40, 0xc0, 242 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 243 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, 244 { 0x71, 0x00, 0x00, 0x84, 0x8a, 0xca, 0x98, 0xf4, 0x40, 0xc0, 245 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 246 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, 247 { 0x71, 0x00, 0x00, 0x88, 0x8a, 0xca, 0x98, 0xf4, 0x40, 0xc0, 248 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 249 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, 250 }; 251 252 /* 253 * Mode Control Instructions for various Key lengths 128, 192, 256 254 * For CBC (Cipher Block Chaining) mode for encryption 255 */ 256 static u8 mci_cbc_enc_no_iv_array[3][MODE_CONTROL_BYTES] = { 257 { 0x21, 0x00, 0x00, 0x18, 0x88, 0x0a, 0xaa, 0x4b, 0x7e, 0x00, 258 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 259 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, 260 { 0x21, 0x00, 0x00, 0x18, 0x88, 0x4a, 0xaa, 0x4b, 0x7e, 0x00, 261 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 262 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, 263 { 0x21, 0x00, 0x00, 0x18, 0x88, 0x8a, 0xaa, 0x4b, 0x7e, 0x00, 264 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 265 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, 266 }; 267 268 /* 269 * Mode Control Instructions for various Key lengths 128, 192, 256 270 * For CBC (Cipher Block Chaining) mode for decryption 271 */ 272 static u8 mci_cbc_dec_no_iv_array[3][MODE_CONTROL_BYTES] = { 273 { 0x31, 0x00, 0x00, 0x80, 0x8a, 0xca, 0x98, 0xf4, 0x40, 0xc0, 274 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 275 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, 276 { 0x31, 0x00, 0x00, 0x84, 0x8a, 0xca, 0x98, 0xf4, 0x40, 0xc0, 277 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 278 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, 279 { 0x31, 0x00, 0x00, 0x88, 0x8a, 0xca, 0x98, 0xf4, 0x40, 0xc0, 280 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 281 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, 282 }; 283 284 /* 285 * Mode Control Instructions for various Key lengths 128, 192, 256 286 * For ECB (Electronic Code Book) mode for encryption 287 */ 288 static u8 mci_ecb_enc_array[3][27] = { 289 { 0x21, 0x00, 0x00, 0x80, 0x8a, 0x04, 0xb7, 0x90, 0x00, 0x00, 290 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 291 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, 292 { 0x21, 0x00, 0x00, 0x84, 0x8a, 0x04, 0xb7, 0x90, 0x00, 0x00, 293 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 294 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, 295 { 0x21, 0x00, 0x00, 0x88, 0x8a, 0x04, 0xb7, 0x90, 0x00, 0x00, 296 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 297 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, 298 }; 299 300 /* 301 * Mode Control Instructions for various Key lengths 128, 192, 256 302 * For ECB (Electronic Code Book) mode for decryption 303 */ 304 static u8 mci_ecb_dec_array[3][27] = { 305 { 0x31, 0x00, 0x00, 0x80, 0x8a, 0x04, 0xb7, 0x90, 0x00, 0x00, 306 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 307 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, 308 { 0x31, 0x00, 0x00, 0x84, 0x8a, 0x04, 0xb7, 0x90, 0x00, 0x00, 309 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 310 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, 311 { 0x31, 0x00, 0x00, 0x88, 0x8a, 0x04, 0xb7, 0x90, 0x00, 0x00, 312 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 313 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, 314 }; 315 316 /* 317 * Mode Control Instructions for DES algorithm 318 * For CBC (Cipher Block Chaining) mode and ECB mode 319 * encryption and for decryption respectively 320 */ 321 static u8 mci_cbc_3des_enc_array[MODE_CONTROL_BYTES] = { 322 0x60, 0x00, 0x00, 0x18, 0x88, 0x52, 0xaa, 0x4b, 0x7e, 0x00, 0x00, 0x00, 323 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 324 0x00, 0x00, 0x00, 325 }; 326 327 static u8 mci_cbc_3des_dec_array[MODE_CONTROL_BYTES] = { 328 0x70, 0x00, 0x00, 0x85, 0x0a, 0xca, 0x98, 0xf4, 0x40, 0xc0, 0x00, 0x00, 329 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 330 0x00, 0x00, 0x00, 331 }; 332 333 static u8 mci_ecb_3des_enc_array[MODE_CONTROL_BYTES] = { 334 0x20, 0x00, 0x00, 0x85, 0x0a, 0x04, 0xb7, 0x90, 0x00, 0x00, 0x00, 0x00, 335 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 336 0x00, 0x00, 0x00, 337 }; 338 339 static u8 mci_ecb_3des_dec_array[MODE_CONTROL_BYTES] = { 340 0x30, 0x00, 0x00, 0x85, 0x0a, 0x04, 0xb7, 0x90, 0x00, 0x00, 0x00, 0x00, 341 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 342 0x00, 0x00, 0x00, 343 }; 344 345 /* 346 * Perform 16 byte or 128 bit swizzling 347 * The SA2UL Expects the security context to 348 * be in little Endian and the bus width is 128 bits or 16 bytes 349 * Hence swap 16 bytes at a time from higher to lower address 350 */ 351 static void sa_swiz_128(u8 *in, u16 len) 352 { 353 u8 data[16]; 354 int i, j; 355 356 for (i = 0; i < len; i += 16) { 357 memcpy(data, &in[i], 16); 358 for (j = 0; j < 16; j++) 359 in[i + j] = data[15 - j]; 360 } 361 } 362 363 /* Prepare the ipad and opad from key as per SHA algorithm step 1*/ 364 static void prepare_kiopad(u8 *k_ipad, u8 *k_opad, const u8 *key, u16 key_sz) 365 { 366 int i; 367 368 for (i = 0; i < key_sz; i++) { 369 k_ipad[i] = key[i] ^ 0x36; 370 k_opad[i] = key[i] ^ 0x5c; 371 } 372 373 /* Instead of XOR with 0 */ 374 for (; i < SHA1_BLOCK_SIZE; i++) { 375 k_ipad[i] = 0x36; 376 k_opad[i] = 0x5c; 377 } 378 } 379 380 static void sa_export_shash(struct shash_desc *hash, int block_size, 381 int digest_size, __be32 *out) 382 { 383 union { 384 struct sha1_state sha1; 385 struct sha256_state sha256; 386 struct sha512_state sha512; 387 } sha; 388 void *state; 389 u32 *result; 390 int i; 391 392 switch (digest_size) { 393 case SHA1_DIGEST_SIZE: 394 state = &sha.sha1; 395 result = sha.sha1.state; 396 break; 397 case SHA256_DIGEST_SIZE: 398 state = &sha.sha256; 399 result = sha.sha256.state; 400 break; 401 default: 402 dev_err(sa_k3_dev, "%s: bad digest_size=%d\n", __func__, 403 digest_size); 404 return; 405 } 406 407 crypto_shash_export(hash, state); 408 409 for (i = 0; i < digest_size >> 2; i++) 410 out[i] = cpu_to_be32(result[i]); 411 } 412 413 static void sa_prepare_iopads(struct algo_data *data, const u8 *key, 414 u16 key_sz, __be32 *ipad, __be32 *opad) 415 { 416 SHASH_DESC_ON_STACK(shash, data->ctx->shash); 417 int block_size = crypto_shash_blocksize(data->ctx->shash); 418 int digest_size = crypto_shash_digestsize(data->ctx->shash); 419 u8 k_ipad[SHA1_BLOCK_SIZE]; 420 u8 k_opad[SHA1_BLOCK_SIZE]; 421 422 shash->tfm = data->ctx->shash; 423 424 prepare_kiopad(k_ipad, k_opad, key, key_sz); 425 426 memzero_explicit(ipad, block_size); 427 memzero_explicit(opad, block_size); 428 429 crypto_shash_init(shash); 430 crypto_shash_update(shash, k_ipad, block_size); 431 sa_export_shash(shash, block_size, digest_size, ipad); 432 433 crypto_shash_init(shash); 434 crypto_shash_update(shash, k_opad, block_size); 435 436 sa_export_shash(shash, block_size, digest_size, opad); 437 } 438 439 /* Derive the inverse key used in AES-CBC decryption operation */ 440 static inline int sa_aes_inv_key(u8 *inv_key, const u8 *key, u16 key_sz) 441 { 442 struct crypto_aes_ctx ctx; 443 int key_pos; 444 445 if (aes_expandkey(&ctx, key, key_sz)) { 446 dev_err(sa_k3_dev, "%s: bad key len(%d)\n", __func__, key_sz); 447 return -EINVAL; 448 } 449 450 /* work around to get the right inverse for AES_KEYSIZE_192 size keys */ 451 if (key_sz == AES_KEYSIZE_192) { 452 ctx.key_enc[52] = ctx.key_enc[51] ^ ctx.key_enc[46]; 453 ctx.key_enc[53] = ctx.key_enc[52] ^ ctx.key_enc[47]; 454 } 455 456 /* Based crypto_aes_expand_key logic */ 457 switch (key_sz) { 458 case AES_KEYSIZE_128: 459 case AES_KEYSIZE_192: 460 key_pos = key_sz + 24; 461 break; 462 463 case AES_KEYSIZE_256: 464 key_pos = key_sz + 24 - 4; 465 break; 466 467 default: 468 dev_err(sa_k3_dev, "%s: bad key len(%d)\n", __func__, key_sz); 469 return -EINVAL; 470 } 471 472 memcpy(inv_key, &ctx.key_enc[key_pos], key_sz); 473 return 0; 474 } 475 476 /* Set Security context for the encryption engine */ 477 static int sa_set_sc_enc(struct algo_data *ad, const u8 *key, u16 key_sz, 478 u8 enc, u8 *sc_buf) 479 { 480 const u8 *mci = NULL; 481 482 /* Set Encryption mode selector to crypto processing */ 483 sc_buf[0] = SA_CRYPTO_PROCESSING; 484 485 if (enc) 486 mci = ad->mci_enc; 487 else 488 mci = ad->mci_dec; 489 /* Set the mode control instructions in security context */ 490 if (mci) 491 memcpy(&sc_buf[1], mci, MODE_CONTROL_BYTES); 492 493 /* For AES-CBC decryption get the inverse key */ 494 if (ad->inv_key && !enc) { 495 if (sa_aes_inv_key(&sc_buf[SC_ENC_KEY_OFFSET], key, key_sz)) 496 return -EINVAL; 497 /* For all other cases: key is used */ 498 } else { 499 memcpy(&sc_buf[SC_ENC_KEY_OFFSET], key, key_sz); 500 } 501 502 return 0; 503 } 504 505 /* Set Security context for the authentication engine */ 506 static void sa_set_sc_auth(struct algo_data *ad, const u8 *key, u16 key_sz, 507 u8 *sc_buf) 508 { 509 __be32 ipad[64], opad[64]; 510 511 /* Set Authentication mode selector to hash processing */ 512 sc_buf[0] = SA_HASH_PROCESSING; 513 /* Auth SW ctrl word: bit[6]=1 (upload computed hash to TLR section) */ 514 sc_buf[1] = SA_UPLOAD_HASH_TO_TLR; 515 sc_buf[1] |= ad->auth_ctrl; 516 517 /* Copy the keys or ipad/opad */ 518 if (ad->keyed_mac) { 519 ad->prep_iopad(ad, key, key_sz, ipad, opad); 520 521 /* Copy ipad to AuthKey */ 522 memcpy(&sc_buf[32], ipad, ad->hash_size); 523 /* Copy opad to Aux-1 */ 524 memcpy(&sc_buf[64], opad, ad->hash_size); 525 } else { 526 /* basic hash */ 527 sc_buf[1] |= SA_BASIC_HASH; 528 } 529 } 530 531 static inline void sa_copy_iv(__be32 *out, const u8 *iv, bool size16) 532 { 533 int j; 534 535 for (j = 0; j < ((size16) ? 4 : 2); j++) { 536 *out = cpu_to_be32(*((u32 *)iv)); 537 iv += 4; 538 out++; 539 } 540 } 541 542 /* Format general command label */ 543 static int sa_format_cmdl_gen(struct sa_cmdl_cfg *cfg, u8 *cmdl, 544 struct sa_cmdl_upd_info *upd_info) 545 { 546 u8 enc_offset = 0, auth_offset = 0, total = 0; 547 u8 enc_next_eng = SA_ENG_ID_OUTPORT2; 548 u8 auth_next_eng = SA_ENG_ID_OUTPORT2; 549 u32 *word_ptr = (u32 *)cmdl; 550 int i; 551 552 /* Clear the command label */ 553 memzero_explicit(cmdl, (SA_MAX_CMDL_WORDS * sizeof(u32))); 554 555 /* Iniialize the command update structure */ 556 memzero_explicit(upd_info, sizeof(*upd_info)); 557 558 if (cfg->enc_eng_id && cfg->auth_eng_id) { 559 if (cfg->enc) { 560 auth_offset = SA_CMDL_HEADER_SIZE_BYTES; 561 enc_next_eng = cfg->auth_eng_id; 562 563 if (cfg->iv_size) 564 auth_offset += cfg->iv_size; 565 } else { 566 enc_offset = SA_CMDL_HEADER_SIZE_BYTES; 567 auth_next_eng = cfg->enc_eng_id; 568 } 569 } 570 571 if (cfg->enc_eng_id) { 572 upd_info->flags |= SA_CMDL_UPD_ENC; 573 upd_info->enc_size.index = enc_offset >> 2; 574 upd_info->enc_offset.index = upd_info->enc_size.index + 1; 575 /* Encryption command label */ 576 cmdl[enc_offset + SA_CMDL_OFFSET_NESC] = enc_next_eng; 577 578 /* Encryption modes requiring IV */ 579 if (cfg->iv_size) { 580 upd_info->flags |= SA_CMDL_UPD_ENC_IV; 581 upd_info->enc_iv.index = 582 (enc_offset + SA_CMDL_HEADER_SIZE_BYTES) >> 2; 583 upd_info->enc_iv.size = cfg->iv_size; 584 585 cmdl[enc_offset + SA_CMDL_OFFSET_LABEL_LEN] = 586 SA_CMDL_HEADER_SIZE_BYTES + cfg->iv_size; 587 588 cmdl[enc_offset + SA_CMDL_OFFSET_OPTION_CTRL1] = 589 (SA_CTX_ENC_AUX2_OFFSET | (cfg->iv_size >> 3)); 590 total += SA_CMDL_HEADER_SIZE_BYTES + cfg->iv_size; 591 } else { 592 cmdl[enc_offset + SA_CMDL_OFFSET_LABEL_LEN] = 593 SA_CMDL_HEADER_SIZE_BYTES; 594 total += SA_CMDL_HEADER_SIZE_BYTES; 595 } 596 } 597 598 if (cfg->auth_eng_id) { 599 upd_info->flags |= SA_CMDL_UPD_AUTH; 600 upd_info->auth_size.index = auth_offset >> 2; 601 upd_info->auth_offset.index = upd_info->auth_size.index + 1; 602 cmdl[auth_offset + SA_CMDL_OFFSET_NESC] = auth_next_eng; 603 cmdl[auth_offset + SA_CMDL_OFFSET_LABEL_LEN] = 604 SA_CMDL_HEADER_SIZE_BYTES; 605 total += SA_CMDL_HEADER_SIZE_BYTES; 606 } 607 608 total = roundup(total, 8); 609 610 for (i = 0; i < total / 4; i++) 611 word_ptr[i] = swab32(word_ptr[i]); 612 613 return total; 614 } 615 616 /* Update Command label */ 617 static inline void sa_update_cmdl(struct sa_req *req, u32 *cmdl, 618 struct sa_cmdl_upd_info *upd_info) 619 { 620 int i = 0, j; 621 622 if (likely(upd_info->flags & SA_CMDL_UPD_ENC)) { 623 cmdl[upd_info->enc_size.index] &= ~SA_CMDL_PAYLOAD_LENGTH_MASK; 624 cmdl[upd_info->enc_size.index] |= req->enc_size; 625 cmdl[upd_info->enc_offset.index] &= 626 ~SA_CMDL_SOP_BYPASS_LEN_MASK; 627 cmdl[upd_info->enc_offset.index] |= 628 ((u32)req->enc_offset << 629 __ffs(SA_CMDL_SOP_BYPASS_LEN_MASK)); 630 631 if (likely(upd_info->flags & SA_CMDL_UPD_ENC_IV)) { 632 __be32 *data = (__be32 *)&cmdl[upd_info->enc_iv.index]; 633 u32 *enc_iv = (u32 *)req->enc_iv; 634 635 for (j = 0; i < upd_info->enc_iv.size; i += 4, j++) { 636 data[j] = cpu_to_be32(*enc_iv); 637 enc_iv++; 638 } 639 } 640 } 641 642 if (likely(upd_info->flags & SA_CMDL_UPD_AUTH)) { 643 cmdl[upd_info->auth_size.index] &= ~SA_CMDL_PAYLOAD_LENGTH_MASK; 644 cmdl[upd_info->auth_size.index] |= req->auth_size; 645 cmdl[upd_info->auth_offset.index] &= 646 ~SA_CMDL_SOP_BYPASS_LEN_MASK; 647 cmdl[upd_info->auth_offset.index] |= 648 ((u32)req->auth_offset << 649 __ffs(SA_CMDL_SOP_BYPASS_LEN_MASK)); 650 if (upd_info->flags & SA_CMDL_UPD_AUTH_IV) { 651 sa_copy_iv((void *)&cmdl[upd_info->auth_iv.index], 652 req->auth_iv, 653 (upd_info->auth_iv.size > 8)); 654 } 655 if (upd_info->flags & SA_CMDL_UPD_AUX_KEY) { 656 int offset = (req->auth_size & 0xF) ? 4 : 0; 657 658 memcpy(&cmdl[upd_info->aux_key_info.index], 659 &upd_info->aux_key[offset], 16); 660 } 661 } 662 } 663 664 /* Format SWINFO words to be sent to SA */ 665 static 666 void sa_set_swinfo(u8 eng_id, u16 sc_id, dma_addr_t sc_phys, 667 u8 cmdl_present, u8 cmdl_offset, u8 flags, 668 u8 hash_size, u32 *swinfo) 669 { 670 swinfo[0] = sc_id; 671 swinfo[0] |= (flags << __ffs(SA_SW0_FLAGS_MASK)); 672 if (likely(cmdl_present)) 673 swinfo[0] |= ((cmdl_offset | SA_SW0_CMDL_PRESENT) << 674 __ffs(SA_SW0_CMDL_INFO_MASK)); 675 swinfo[0] |= (eng_id << __ffs(SA_SW0_ENG_ID_MASK)); 676 677 swinfo[0] |= SA_SW0_DEST_INFO_PRESENT; 678 swinfo[1] = (u32)(sc_phys & 0xFFFFFFFFULL); 679 swinfo[2] = (u32)((sc_phys & 0xFFFFFFFF00000000ULL) >> 32); 680 swinfo[2] |= (hash_size << __ffs(SA_SW2_EGRESS_LENGTH)); 681 } 682 683 /* Dump the security context */ 684 static void sa_dump_sc(u8 *buf, dma_addr_t dma_addr) 685 { 686 #ifdef DEBUG 687 dev_info(sa_k3_dev, "Security context dump:: 0x%pad\n", &dma_addr); 688 print_hex_dump(KERN_CONT, "", DUMP_PREFIX_OFFSET, 689 16, 1, buf, SA_CTX_MAX_SZ, false); 690 #endif 691 } 692 693 static 694 int sa_init_sc(struct sa_ctx_info *ctx, const u8 *enc_key, 695 u16 enc_key_sz, const u8 *auth_key, u16 auth_key_sz, 696 struct algo_data *ad, u8 enc, u32 *swinfo) 697 { 698 int enc_sc_offset = 0; 699 int auth_sc_offset = 0; 700 u8 *sc_buf = ctx->sc; 701 u16 sc_id = ctx->sc_id; 702 u8 first_engine = 0; 703 704 memzero_explicit(sc_buf, SA_CTX_MAX_SZ); 705 706 if (ad->auth_eng.eng_id) { 707 if (enc) 708 first_engine = ad->enc_eng.eng_id; 709 else 710 first_engine = ad->auth_eng.eng_id; 711 712 enc_sc_offset = SA_CTX_PHP_PE_CTX_SZ; 713 auth_sc_offset = enc_sc_offset + ad->enc_eng.sc_size; 714 sc_buf[1] = SA_SCCTL_FE_AUTH_ENC; 715 if (!ad->hash_size) 716 return -EINVAL; 717 ad->hash_size = roundup(ad->hash_size, 8); 718 719 } else if (ad->enc_eng.eng_id && !ad->auth_eng.eng_id) { 720 enc_sc_offset = SA_CTX_PHP_PE_CTX_SZ; 721 first_engine = ad->enc_eng.eng_id; 722 sc_buf[1] = SA_SCCTL_FE_ENC; 723 ad->hash_size = ad->iv_out_size; 724 } 725 726 /* SCCTL Owner info: 0=host, 1=CP_ACE */ 727 sc_buf[SA_CTX_SCCTL_OWNER_OFFSET] = 0; 728 memcpy(&sc_buf[2], &sc_id, 2); 729 sc_buf[4] = 0x0; 730 sc_buf[5] = PRIV_ID; 731 sc_buf[6] = PRIV; 732 sc_buf[7] = 0x0; 733 734 /* Prepare context for encryption engine */ 735 if (ad->enc_eng.sc_size) { 736 if (sa_set_sc_enc(ad, enc_key, enc_key_sz, enc, 737 &sc_buf[enc_sc_offset])) 738 return -EINVAL; 739 } 740 741 /* Prepare context for authentication engine */ 742 if (ad->auth_eng.sc_size) 743 sa_set_sc_auth(ad, auth_key, auth_key_sz, 744 &sc_buf[auth_sc_offset]); 745 746 /* Set the ownership of context to CP_ACE */ 747 sc_buf[SA_CTX_SCCTL_OWNER_OFFSET] = 0x80; 748 749 /* swizzle the security context */ 750 sa_swiz_128(sc_buf, SA_CTX_MAX_SZ); 751 752 sa_set_swinfo(first_engine, ctx->sc_id, ctx->sc_phys, 1, 0, 753 SA_SW_INFO_FLAG_EVICT, ad->hash_size, swinfo); 754 755 sa_dump_sc(sc_buf, ctx->sc_phys); 756 757 return 0; 758 } 759 760 /* Free the per direction context memory */ 761 static void sa_free_ctx_info(struct sa_ctx_info *ctx, 762 struct sa_crypto_data *data) 763 { 764 unsigned long bn; 765 766 bn = ctx->sc_id - data->sc_id_start; 767 spin_lock(&data->scid_lock); 768 __clear_bit(bn, data->ctx_bm); 769 data->sc_id--; 770 spin_unlock(&data->scid_lock); 771 772 if (ctx->sc) { 773 dma_pool_free(data->sc_pool, ctx->sc, ctx->sc_phys); 774 ctx->sc = NULL; 775 } 776 } 777 778 static int sa_init_ctx_info(struct sa_ctx_info *ctx, 779 struct sa_crypto_data *data) 780 { 781 unsigned long bn; 782 int err; 783 784 spin_lock(&data->scid_lock); 785 bn = find_first_zero_bit(data->ctx_bm, SA_MAX_NUM_CTX); 786 __set_bit(bn, data->ctx_bm); 787 data->sc_id++; 788 spin_unlock(&data->scid_lock); 789 790 ctx->sc_id = (u16)(data->sc_id_start + bn); 791 792 ctx->sc = dma_pool_alloc(data->sc_pool, GFP_KERNEL, &ctx->sc_phys); 793 if (!ctx->sc) { 794 dev_err(&data->pdev->dev, "Failed to allocate SC memory\n"); 795 err = -ENOMEM; 796 goto scid_rollback; 797 } 798 799 return 0; 800 801 scid_rollback: 802 spin_lock(&data->scid_lock); 803 __clear_bit(bn, data->ctx_bm); 804 data->sc_id--; 805 spin_unlock(&data->scid_lock); 806 807 return err; 808 } 809 810 static void sa_cipher_cra_exit(struct crypto_skcipher *tfm) 811 { 812 struct sa_tfm_ctx *ctx = crypto_skcipher_ctx(tfm); 813 struct sa_crypto_data *data = dev_get_drvdata(sa_k3_dev); 814 815 dev_dbg(sa_k3_dev, "%s(0x%p) sc-ids(0x%x(0x%pad), 0x%x(0x%pad))\n", 816 __func__, tfm, ctx->enc.sc_id, &ctx->enc.sc_phys, 817 ctx->dec.sc_id, &ctx->dec.sc_phys); 818 819 sa_free_ctx_info(&ctx->enc, data); 820 sa_free_ctx_info(&ctx->dec, data); 821 822 crypto_free_sync_skcipher(ctx->fallback.skcipher); 823 } 824 825 static int sa_cipher_cra_init(struct crypto_skcipher *tfm) 826 { 827 struct sa_tfm_ctx *ctx = crypto_skcipher_ctx(tfm); 828 struct sa_crypto_data *data = dev_get_drvdata(sa_k3_dev); 829 const char *name = crypto_tfm_alg_name(&tfm->base); 830 int ret; 831 832 memzero_explicit(ctx, sizeof(*ctx)); 833 ctx->dev_data = data; 834 835 ret = sa_init_ctx_info(&ctx->enc, data); 836 if (ret) 837 return ret; 838 ret = sa_init_ctx_info(&ctx->dec, data); 839 if (ret) { 840 sa_free_ctx_info(&ctx->enc, data); 841 return ret; 842 } 843 844 ctx->fallback.skcipher = 845 crypto_alloc_sync_skcipher(name, 0, CRYPTO_ALG_NEED_FALLBACK); 846 847 if (IS_ERR(ctx->fallback.skcipher)) { 848 dev_err(sa_k3_dev, "Error allocating fallback algo %s\n", name); 849 return PTR_ERR(ctx->fallback.skcipher); 850 } 851 852 dev_dbg(sa_k3_dev, "%s(0x%p) sc-ids(0x%x(0x%pad), 0x%x(0x%pad))\n", 853 __func__, tfm, ctx->enc.sc_id, &ctx->enc.sc_phys, 854 ctx->dec.sc_id, &ctx->dec.sc_phys); 855 return 0; 856 } 857 858 static int sa_cipher_setkey(struct crypto_skcipher *tfm, const u8 *key, 859 unsigned int keylen, struct algo_data *ad) 860 { 861 struct sa_tfm_ctx *ctx = crypto_skcipher_ctx(tfm); 862 int cmdl_len; 863 struct sa_cmdl_cfg cfg; 864 int ret; 865 866 if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_192 && 867 keylen != AES_KEYSIZE_256) 868 return -EINVAL; 869 870 ad->enc_eng.eng_id = SA_ENG_ID_EM1; 871 ad->enc_eng.sc_size = SA_CTX_ENC_TYPE1_SZ; 872 873 memzero_explicit(&cfg, sizeof(cfg)); 874 cfg.enc_eng_id = ad->enc_eng.eng_id; 875 cfg.iv_size = crypto_skcipher_ivsize(tfm); 876 877 crypto_sync_skcipher_clear_flags(ctx->fallback.skcipher, 878 CRYPTO_TFM_REQ_MASK); 879 crypto_sync_skcipher_set_flags(ctx->fallback.skcipher, 880 tfm->base.crt_flags & 881 CRYPTO_TFM_REQ_MASK); 882 ret = crypto_sync_skcipher_setkey(ctx->fallback.skcipher, key, keylen); 883 if (ret) 884 return ret; 885 886 /* Setup Encryption Security Context & Command label template */ 887 if (sa_init_sc(&ctx->enc, key, keylen, NULL, 0, ad, 1, 888 &ctx->enc.epib[1])) 889 goto badkey; 890 891 cmdl_len = sa_format_cmdl_gen(&cfg, 892 (u8 *)ctx->enc.cmdl, 893 &ctx->enc.cmdl_upd_info); 894 if (cmdl_len <= 0 || (cmdl_len > SA_MAX_CMDL_WORDS * sizeof(u32))) 895 goto badkey; 896 897 ctx->enc.cmdl_size = cmdl_len; 898 899 /* Setup Decryption Security Context & Command label template */ 900 if (sa_init_sc(&ctx->dec, key, keylen, NULL, 0, ad, 0, 901 &ctx->dec.epib[1])) 902 goto badkey; 903 904 cfg.enc_eng_id = ad->enc_eng.eng_id; 905 cmdl_len = sa_format_cmdl_gen(&cfg, (u8 *)ctx->dec.cmdl, 906 &ctx->dec.cmdl_upd_info); 907 908 if (cmdl_len <= 0 || (cmdl_len > SA_MAX_CMDL_WORDS * sizeof(u32))) 909 goto badkey; 910 911 ctx->dec.cmdl_size = cmdl_len; 912 ctx->iv_idx = ad->iv_idx; 913 914 return 0; 915 916 badkey: 917 dev_err(sa_k3_dev, "%s: badkey\n", __func__); 918 return -EINVAL; 919 } 920 921 static int sa_aes_cbc_setkey(struct crypto_skcipher *tfm, const u8 *key, 922 unsigned int keylen) 923 { 924 struct algo_data ad = { 0 }; 925 /* Convert the key size (16/24/32) to the key size index (0/1/2) */ 926 int key_idx = (keylen >> 3) - 2; 927 928 if (key_idx >= 3) 929 return -EINVAL; 930 931 ad.mci_enc = mci_cbc_enc_array[key_idx]; 932 ad.mci_dec = mci_cbc_dec_array[key_idx]; 933 ad.inv_key = true; 934 ad.ealg_id = SA_EALG_ID_AES_CBC; 935 ad.iv_idx = 4; 936 ad.iv_out_size = 16; 937 938 return sa_cipher_setkey(tfm, key, keylen, &ad); 939 } 940 941 static int sa_aes_ecb_setkey(struct crypto_skcipher *tfm, const u8 *key, 942 unsigned int keylen) 943 { 944 struct algo_data ad = { 0 }; 945 /* Convert the key size (16/24/32) to the key size index (0/1/2) */ 946 int key_idx = (keylen >> 3) - 2; 947 948 if (key_idx >= 3) 949 return -EINVAL; 950 951 ad.mci_enc = mci_ecb_enc_array[key_idx]; 952 ad.mci_dec = mci_ecb_dec_array[key_idx]; 953 ad.inv_key = true; 954 ad.ealg_id = SA_EALG_ID_AES_ECB; 955 956 return sa_cipher_setkey(tfm, key, keylen, &ad); 957 } 958 959 static int sa_3des_cbc_setkey(struct crypto_skcipher *tfm, const u8 *key, 960 unsigned int keylen) 961 { 962 struct algo_data ad = { 0 }; 963 964 ad.mci_enc = mci_cbc_3des_enc_array; 965 ad.mci_dec = mci_cbc_3des_dec_array; 966 ad.ealg_id = SA_EALG_ID_3DES_CBC; 967 ad.iv_idx = 6; 968 ad.iv_out_size = 8; 969 970 return sa_cipher_setkey(tfm, key, keylen, &ad); 971 } 972 973 static int sa_3des_ecb_setkey(struct crypto_skcipher *tfm, const u8 *key, 974 unsigned int keylen) 975 { 976 struct algo_data ad = { 0 }; 977 978 ad.mci_enc = mci_ecb_3des_enc_array; 979 ad.mci_dec = mci_ecb_3des_dec_array; 980 981 return sa_cipher_setkey(tfm, key, keylen, &ad); 982 } 983 984 static void sa_sync_from_device(struct sa_rx_data *rxd) 985 { 986 struct sg_table *sgt; 987 988 if (rxd->mapped_sg[0].dir == DMA_BIDIRECTIONAL) 989 sgt = &rxd->mapped_sg[0].sgt; 990 else 991 sgt = &rxd->mapped_sg[1].sgt; 992 993 dma_sync_sgtable_for_cpu(rxd->ddev, sgt, DMA_FROM_DEVICE); 994 } 995 996 static void sa_free_sa_rx_data(struct sa_rx_data *rxd) 997 { 998 int i; 999 1000 for (i = 0; i < ARRAY_SIZE(rxd->mapped_sg); i++) { 1001 struct sa_mapped_sg *mapped_sg = &rxd->mapped_sg[i]; 1002 1003 if (mapped_sg->mapped) { 1004 dma_unmap_sgtable(rxd->ddev, &mapped_sg->sgt, 1005 mapped_sg->dir, 0); 1006 kfree(mapped_sg->split_sg); 1007 } 1008 } 1009 1010 kfree(rxd); 1011 } 1012 1013 static void sa_aes_dma_in_callback(void *data) 1014 { 1015 struct sa_rx_data *rxd = (struct sa_rx_data *)data; 1016 struct skcipher_request *req; 1017 u32 *result; 1018 __be32 *mdptr; 1019 size_t ml, pl; 1020 int i; 1021 1022 sa_sync_from_device(rxd); 1023 req = container_of(rxd->req, struct skcipher_request, base); 1024 1025 if (req->iv) { 1026 mdptr = (__be32 *)dmaengine_desc_get_metadata_ptr(rxd->tx_in, &pl, 1027 &ml); 1028 result = (u32 *)req->iv; 1029 1030 for (i = 0; i < (rxd->enc_iv_size / 4); i++) 1031 result[i] = be32_to_cpu(mdptr[i + rxd->iv_idx]); 1032 } 1033 1034 sa_free_sa_rx_data(rxd); 1035 1036 skcipher_request_complete(req, 0); 1037 } 1038 1039 static void 1040 sa_prepare_tx_desc(u32 *mdptr, u32 pslen, u32 *psdata, u32 epiblen, u32 *epib) 1041 { 1042 u32 *out, *in; 1043 int i; 1044 1045 for (out = mdptr, in = epib, i = 0; i < epiblen / sizeof(u32); i++) 1046 *out++ = *in++; 1047 1048 mdptr[4] = (0xFFFF << 16); 1049 for (out = &mdptr[5], in = psdata, i = 0; 1050 i < pslen / sizeof(u32); i++) 1051 *out++ = *in++; 1052 } 1053 1054 static int sa_run(struct sa_req *req) 1055 { 1056 struct sa_rx_data *rxd; 1057 gfp_t gfp_flags; 1058 u32 cmdl[SA_MAX_CMDL_WORDS]; 1059 struct sa_crypto_data *pdata = dev_get_drvdata(sa_k3_dev); 1060 struct device *ddev; 1061 struct dma_chan *dma_rx; 1062 int sg_nents, src_nents, dst_nents; 1063 struct scatterlist *src, *dst; 1064 size_t pl, ml, split_size; 1065 struct sa_ctx_info *sa_ctx = req->enc ? &req->ctx->enc : &req->ctx->dec; 1066 int ret; 1067 struct dma_async_tx_descriptor *tx_out; 1068 u32 *mdptr; 1069 bool diff_dst; 1070 enum dma_data_direction dir_src; 1071 struct sa_mapped_sg *mapped_sg; 1072 1073 gfp_flags = req->base->flags & CRYPTO_TFM_REQ_MAY_SLEEP ? 1074 GFP_KERNEL : GFP_ATOMIC; 1075 1076 rxd = kzalloc(sizeof(*rxd), gfp_flags); 1077 if (!rxd) 1078 return -ENOMEM; 1079 1080 if (req->src != req->dst) { 1081 diff_dst = true; 1082 dir_src = DMA_TO_DEVICE; 1083 } else { 1084 diff_dst = false; 1085 dir_src = DMA_BIDIRECTIONAL; 1086 } 1087 1088 /* 1089 * SA2UL has an interesting feature where the receive DMA channel 1090 * is selected based on the data passed to the engine. Within the 1091 * transition range, there is also a space where it is impossible 1092 * to determine where the data will end up, and this should be 1093 * avoided. This will be handled by the SW fallback mechanism by 1094 * the individual algorithm implementations. 1095 */ 1096 if (req->size >= 256) 1097 dma_rx = pdata->dma_rx2; 1098 else 1099 dma_rx = pdata->dma_rx1; 1100 1101 ddev = dma_rx->device->dev; 1102 rxd->ddev = ddev; 1103 1104 memcpy(cmdl, sa_ctx->cmdl, sa_ctx->cmdl_size); 1105 1106 sa_update_cmdl(req, cmdl, &sa_ctx->cmdl_upd_info); 1107 1108 if (req->type != CRYPTO_ALG_TYPE_AHASH) { 1109 if (req->enc) 1110 req->type |= 1111 (SA_REQ_SUBTYPE_ENC << SA_REQ_SUBTYPE_SHIFT); 1112 else 1113 req->type |= 1114 (SA_REQ_SUBTYPE_DEC << SA_REQ_SUBTYPE_SHIFT); 1115 } 1116 1117 cmdl[sa_ctx->cmdl_size / sizeof(u32)] = req->type; 1118 1119 /* 1120 * Map the packets, first we check if the data fits into a single 1121 * sg entry and use that if possible. If it does not fit, we check 1122 * if we need to do sg_split to align the scatterlist data on the 1123 * actual data size being processed by the crypto engine. 1124 */ 1125 src = req->src; 1126 sg_nents = sg_nents_for_len(src, req->size); 1127 1128 split_size = req->size; 1129 1130 mapped_sg = &rxd->mapped_sg[0]; 1131 if (sg_nents == 1 && split_size <= req->src->length) { 1132 src = &mapped_sg->static_sg; 1133 src_nents = 1; 1134 sg_init_table(src, 1); 1135 sg_set_page(src, sg_page(req->src), split_size, 1136 req->src->offset); 1137 1138 mapped_sg->sgt.sgl = src; 1139 mapped_sg->sgt.orig_nents = src_nents; 1140 ret = dma_map_sgtable(ddev, &mapped_sg->sgt, dir_src, 0); 1141 if (ret) 1142 return ret; 1143 1144 mapped_sg->dir = dir_src; 1145 mapped_sg->mapped = true; 1146 } else { 1147 mapped_sg->sgt.sgl = req->src; 1148 mapped_sg->sgt.orig_nents = sg_nents; 1149 ret = dma_map_sgtable(ddev, &mapped_sg->sgt, dir_src, 0); 1150 if (ret) 1151 return ret; 1152 1153 mapped_sg->dir = dir_src; 1154 mapped_sg->mapped = true; 1155 1156 ret = sg_split(mapped_sg->sgt.sgl, mapped_sg->sgt.nents, 0, 1, 1157 &split_size, &src, &src_nents, gfp_flags); 1158 if (ret) { 1159 src_nents = mapped_sg->sgt.nents; 1160 src = mapped_sg->sgt.sgl; 1161 } else { 1162 mapped_sg->split_sg = src; 1163 } 1164 } 1165 1166 dma_sync_sgtable_for_device(ddev, &mapped_sg->sgt, DMA_TO_DEVICE); 1167 1168 if (!diff_dst) { 1169 dst_nents = src_nents; 1170 dst = src; 1171 } else { 1172 dst_nents = sg_nents_for_len(req->dst, req->size); 1173 mapped_sg = &rxd->mapped_sg[1]; 1174 1175 if (dst_nents == 1 && split_size <= req->dst->length) { 1176 dst = &mapped_sg->static_sg; 1177 dst_nents = 1; 1178 sg_init_table(dst, 1); 1179 sg_set_page(dst, sg_page(req->dst), split_size, 1180 req->dst->offset); 1181 1182 mapped_sg->sgt.sgl = dst; 1183 mapped_sg->sgt.orig_nents = dst_nents; 1184 ret = dma_map_sgtable(ddev, &mapped_sg->sgt, 1185 DMA_FROM_DEVICE, 0); 1186 if (ret) 1187 goto err_cleanup; 1188 1189 mapped_sg->dir = DMA_FROM_DEVICE; 1190 mapped_sg->mapped = true; 1191 } else { 1192 mapped_sg->sgt.sgl = req->dst; 1193 mapped_sg->sgt.orig_nents = dst_nents; 1194 ret = dma_map_sgtable(ddev, &mapped_sg->sgt, 1195 DMA_FROM_DEVICE, 0); 1196 if (ret) 1197 goto err_cleanup; 1198 1199 mapped_sg->dir = DMA_FROM_DEVICE; 1200 mapped_sg->mapped = true; 1201 1202 ret = sg_split(mapped_sg->sgt.sgl, mapped_sg->sgt.nents, 1203 0, 1, &split_size, &dst, &dst_nents, 1204 gfp_flags); 1205 if (ret) { 1206 dst_nents = mapped_sg->sgt.nents; 1207 dst = mapped_sg->sgt.sgl; 1208 } else { 1209 mapped_sg->split_sg = dst; 1210 } 1211 } 1212 } 1213 1214 rxd->tx_in = dmaengine_prep_slave_sg(dma_rx, dst, dst_nents, 1215 DMA_DEV_TO_MEM, 1216 DMA_PREP_INTERRUPT | DMA_CTRL_ACK); 1217 if (!rxd->tx_in) { 1218 dev_err(pdata->dev, "IN prep_slave_sg() failed\n"); 1219 ret = -EINVAL; 1220 goto err_cleanup; 1221 } 1222 1223 rxd->req = (void *)req->base; 1224 rxd->enc = req->enc; 1225 rxd->iv_idx = req->ctx->iv_idx; 1226 rxd->enc_iv_size = sa_ctx->cmdl_upd_info.enc_iv.size; 1227 rxd->tx_in->callback = req->callback; 1228 rxd->tx_in->callback_param = rxd; 1229 1230 tx_out = dmaengine_prep_slave_sg(pdata->dma_tx, src, 1231 src_nents, DMA_MEM_TO_DEV, 1232 DMA_PREP_INTERRUPT | DMA_CTRL_ACK); 1233 1234 if (!tx_out) { 1235 dev_err(pdata->dev, "OUT prep_slave_sg() failed\n"); 1236 ret = -EINVAL; 1237 goto err_cleanup; 1238 } 1239 1240 /* 1241 * Prepare metadata for DMA engine. This essentially describes the 1242 * crypto algorithm to be used, data sizes, different keys etc. 1243 */ 1244 mdptr = (u32 *)dmaengine_desc_get_metadata_ptr(tx_out, &pl, &ml); 1245 1246 sa_prepare_tx_desc(mdptr, (sa_ctx->cmdl_size + (SA_PSDATA_CTX_WORDS * 1247 sizeof(u32))), cmdl, sizeof(sa_ctx->epib), 1248 sa_ctx->epib); 1249 1250 ml = sa_ctx->cmdl_size + (SA_PSDATA_CTX_WORDS * sizeof(u32)); 1251 dmaengine_desc_set_metadata_len(tx_out, req->mdata_size); 1252 1253 dmaengine_submit(tx_out); 1254 dmaengine_submit(rxd->tx_in); 1255 1256 dma_async_issue_pending(dma_rx); 1257 dma_async_issue_pending(pdata->dma_tx); 1258 1259 return -EINPROGRESS; 1260 1261 err_cleanup: 1262 sa_free_sa_rx_data(rxd); 1263 1264 return ret; 1265 } 1266 1267 static int sa_cipher_run(struct skcipher_request *req, u8 *iv, int enc) 1268 { 1269 struct sa_tfm_ctx *ctx = 1270 crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); 1271 struct crypto_alg *alg = req->base.tfm->__crt_alg; 1272 struct sa_req sa_req = { 0 }; 1273 int ret; 1274 1275 if (!req->cryptlen) 1276 return 0; 1277 1278 if (req->cryptlen % alg->cra_blocksize) 1279 return -EINVAL; 1280 1281 /* Use SW fallback if the data size is not supported */ 1282 if (req->cryptlen > SA_MAX_DATA_SZ || 1283 (req->cryptlen >= SA_UNSAFE_DATA_SZ_MIN && 1284 req->cryptlen <= SA_UNSAFE_DATA_SZ_MAX)) { 1285 SYNC_SKCIPHER_REQUEST_ON_STACK(subreq, ctx->fallback.skcipher); 1286 1287 skcipher_request_set_sync_tfm(subreq, ctx->fallback.skcipher); 1288 skcipher_request_set_callback(subreq, req->base.flags, 1289 NULL, NULL); 1290 skcipher_request_set_crypt(subreq, req->src, req->dst, 1291 req->cryptlen, req->iv); 1292 if (enc) 1293 ret = crypto_skcipher_encrypt(subreq); 1294 else 1295 ret = crypto_skcipher_decrypt(subreq); 1296 1297 skcipher_request_zero(subreq); 1298 return ret; 1299 } 1300 1301 sa_req.size = req->cryptlen; 1302 sa_req.enc_size = req->cryptlen; 1303 sa_req.src = req->src; 1304 sa_req.dst = req->dst; 1305 sa_req.enc_iv = iv; 1306 sa_req.type = CRYPTO_ALG_TYPE_SKCIPHER; 1307 sa_req.enc = enc; 1308 sa_req.callback = sa_aes_dma_in_callback; 1309 sa_req.mdata_size = 44; 1310 sa_req.base = &req->base; 1311 sa_req.ctx = ctx; 1312 1313 return sa_run(&sa_req); 1314 } 1315 1316 static int sa_encrypt(struct skcipher_request *req) 1317 { 1318 return sa_cipher_run(req, req->iv, 1); 1319 } 1320 1321 static int sa_decrypt(struct skcipher_request *req) 1322 { 1323 return sa_cipher_run(req, req->iv, 0); 1324 } 1325 1326 static void sa_sha_dma_in_callback(void *data) 1327 { 1328 struct sa_rx_data *rxd = (struct sa_rx_data *)data; 1329 struct ahash_request *req; 1330 struct crypto_ahash *tfm; 1331 unsigned int authsize; 1332 int i; 1333 size_t ml, pl; 1334 u32 *result; 1335 __be32 *mdptr; 1336 1337 sa_sync_from_device(rxd); 1338 req = container_of(rxd->req, struct ahash_request, base); 1339 tfm = crypto_ahash_reqtfm(req); 1340 authsize = crypto_ahash_digestsize(tfm); 1341 1342 mdptr = (__be32 *)dmaengine_desc_get_metadata_ptr(rxd->tx_in, &pl, &ml); 1343 result = (u32 *)req->result; 1344 1345 for (i = 0; i < (authsize / 4); i++) 1346 result[i] = be32_to_cpu(mdptr[i + 4]); 1347 1348 sa_free_sa_rx_data(rxd); 1349 1350 ahash_request_complete(req, 0); 1351 } 1352 1353 static int zero_message_process(struct ahash_request *req) 1354 { 1355 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 1356 int sa_digest_size = crypto_ahash_digestsize(tfm); 1357 1358 switch (sa_digest_size) { 1359 case SHA1_DIGEST_SIZE: 1360 memcpy(req->result, sha1_zero_message_hash, sa_digest_size); 1361 break; 1362 case SHA256_DIGEST_SIZE: 1363 memcpy(req->result, sha256_zero_message_hash, sa_digest_size); 1364 break; 1365 case SHA512_DIGEST_SIZE: 1366 memcpy(req->result, sha512_zero_message_hash, sa_digest_size); 1367 break; 1368 default: 1369 return -EINVAL; 1370 } 1371 1372 return 0; 1373 } 1374 1375 static int sa_sha_run(struct ahash_request *req) 1376 { 1377 struct sa_tfm_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(req)); 1378 struct sa_sha_req_ctx *rctx = ahash_request_ctx(req); 1379 struct sa_req sa_req = { 0 }; 1380 size_t auth_len; 1381 1382 auth_len = req->nbytes; 1383 1384 if (!auth_len) 1385 return zero_message_process(req); 1386 1387 if (auth_len > SA_MAX_DATA_SZ || 1388 (auth_len >= SA_UNSAFE_DATA_SZ_MIN && 1389 auth_len <= SA_UNSAFE_DATA_SZ_MAX)) { 1390 struct ahash_request *subreq = &rctx->fallback_req; 1391 int ret = 0; 1392 1393 ahash_request_set_tfm(subreq, ctx->fallback.ahash); 1394 subreq->base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; 1395 1396 crypto_ahash_init(subreq); 1397 1398 subreq->nbytes = auth_len; 1399 subreq->src = req->src; 1400 subreq->result = req->result; 1401 1402 ret |= crypto_ahash_update(subreq); 1403 1404 subreq->nbytes = 0; 1405 1406 ret |= crypto_ahash_final(subreq); 1407 1408 return ret; 1409 } 1410 1411 sa_req.size = auth_len; 1412 sa_req.auth_size = auth_len; 1413 sa_req.src = req->src; 1414 sa_req.dst = req->src; 1415 sa_req.enc = true; 1416 sa_req.type = CRYPTO_ALG_TYPE_AHASH; 1417 sa_req.callback = sa_sha_dma_in_callback; 1418 sa_req.mdata_size = 28; 1419 sa_req.ctx = ctx; 1420 sa_req.base = &req->base; 1421 1422 return sa_run(&sa_req); 1423 } 1424 1425 static int sa_sha_setup(struct sa_tfm_ctx *ctx, struct algo_data *ad) 1426 { 1427 int bs = crypto_shash_blocksize(ctx->shash); 1428 int cmdl_len; 1429 struct sa_cmdl_cfg cfg; 1430 1431 ad->enc_eng.sc_size = SA_CTX_ENC_TYPE1_SZ; 1432 ad->auth_eng.eng_id = SA_ENG_ID_AM1; 1433 ad->auth_eng.sc_size = SA_CTX_AUTH_TYPE2_SZ; 1434 1435 memset(ctx->authkey, 0, bs); 1436 memset(&cfg, 0, sizeof(cfg)); 1437 cfg.aalg = ad->aalg_id; 1438 cfg.enc_eng_id = ad->enc_eng.eng_id; 1439 cfg.auth_eng_id = ad->auth_eng.eng_id; 1440 cfg.iv_size = 0; 1441 cfg.akey = NULL; 1442 cfg.akey_len = 0; 1443 1444 /* Setup Encryption Security Context & Command label template */ 1445 if (sa_init_sc(&ctx->enc, NULL, 0, NULL, 0, ad, 0, 1446 &ctx->enc.epib[1])) 1447 goto badkey; 1448 1449 cmdl_len = sa_format_cmdl_gen(&cfg, 1450 (u8 *)ctx->enc.cmdl, 1451 &ctx->enc.cmdl_upd_info); 1452 if (cmdl_len <= 0 || (cmdl_len > SA_MAX_CMDL_WORDS * sizeof(u32))) 1453 goto badkey; 1454 1455 ctx->enc.cmdl_size = cmdl_len; 1456 1457 return 0; 1458 1459 badkey: 1460 dev_err(sa_k3_dev, "%s: badkey\n", __func__); 1461 return -EINVAL; 1462 } 1463 1464 static int sa_sha_cra_init_alg(struct crypto_tfm *tfm, const char *alg_base) 1465 { 1466 struct sa_tfm_ctx *ctx = crypto_tfm_ctx(tfm); 1467 struct sa_crypto_data *data = dev_get_drvdata(sa_k3_dev); 1468 int ret; 1469 1470 memset(ctx, 0, sizeof(*ctx)); 1471 ctx->dev_data = data; 1472 ret = sa_init_ctx_info(&ctx->enc, data); 1473 if (ret) 1474 return ret; 1475 1476 if (alg_base) { 1477 ctx->shash = crypto_alloc_shash(alg_base, 0, 1478 CRYPTO_ALG_NEED_FALLBACK); 1479 if (IS_ERR(ctx->shash)) { 1480 dev_err(sa_k3_dev, "base driver %s couldn't be loaded\n", 1481 alg_base); 1482 return PTR_ERR(ctx->shash); 1483 } 1484 /* for fallback */ 1485 ctx->fallback.ahash = 1486 crypto_alloc_ahash(alg_base, 0, 1487 CRYPTO_ALG_NEED_FALLBACK); 1488 if (IS_ERR(ctx->fallback.ahash)) { 1489 dev_err(ctx->dev_data->dev, 1490 "Could not load fallback driver\n"); 1491 return PTR_ERR(ctx->fallback.ahash); 1492 } 1493 } 1494 1495 dev_dbg(sa_k3_dev, "%s(0x%p) sc-ids(0x%x(0x%pad), 0x%x(0x%pad))\n", 1496 __func__, tfm, ctx->enc.sc_id, &ctx->enc.sc_phys, 1497 ctx->dec.sc_id, &ctx->dec.sc_phys); 1498 1499 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), 1500 sizeof(struct sa_sha_req_ctx) + 1501 crypto_ahash_reqsize(ctx->fallback.ahash)); 1502 1503 return 0; 1504 } 1505 1506 static int sa_sha_digest(struct ahash_request *req) 1507 { 1508 return sa_sha_run(req); 1509 } 1510 1511 static int sa_sha_init(struct ahash_request *req) 1512 { 1513 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 1514 struct sa_sha_req_ctx *rctx = ahash_request_ctx(req); 1515 struct sa_tfm_ctx *ctx = crypto_ahash_ctx(tfm); 1516 1517 dev_dbg(sa_k3_dev, "init: digest size: %u, rctx=%p\n", 1518 crypto_ahash_digestsize(tfm), rctx); 1519 1520 ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback.ahash); 1521 rctx->fallback_req.base.flags = 1522 req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; 1523 1524 return crypto_ahash_init(&rctx->fallback_req); 1525 } 1526 1527 static int sa_sha_update(struct ahash_request *req) 1528 { 1529 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 1530 struct sa_sha_req_ctx *rctx = ahash_request_ctx(req); 1531 struct sa_tfm_ctx *ctx = crypto_ahash_ctx(tfm); 1532 1533 ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback.ahash); 1534 rctx->fallback_req.base.flags = 1535 req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; 1536 rctx->fallback_req.nbytes = req->nbytes; 1537 rctx->fallback_req.src = req->src; 1538 1539 return crypto_ahash_update(&rctx->fallback_req); 1540 } 1541 1542 static int sa_sha_final(struct ahash_request *req) 1543 { 1544 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 1545 struct sa_sha_req_ctx *rctx = ahash_request_ctx(req); 1546 struct sa_tfm_ctx *ctx = crypto_ahash_ctx(tfm); 1547 1548 ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback.ahash); 1549 rctx->fallback_req.base.flags = 1550 req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; 1551 rctx->fallback_req.result = req->result; 1552 1553 return crypto_ahash_final(&rctx->fallback_req); 1554 } 1555 1556 static int sa_sha_finup(struct ahash_request *req) 1557 { 1558 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 1559 struct sa_sha_req_ctx *rctx = ahash_request_ctx(req); 1560 struct sa_tfm_ctx *ctx = crypto_ahash_ctx(tfm); 1561 1562 ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback.ahash); 1563 rctx->fallback_req.base.flags = 1564 req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; 1565 1566 rctx->fallback_req.nbytes = req->nbytes; 1567 rctx->fallback_req.src = req->src; 1568 rctx->fallback_req.result = req->result; 1569 1570 return crypto_ahash_finup(&rctx->fallback_req); 1571 } 1572 1573 static int sa_sha_import(struct ahash_request *req, const void *in) 1574 { 1575 struct sa_sha_req_ctx *rctx = ahash_request_ctx(req); 1576 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 1577 struct sa_tfm_ctx *ctx = crypto_ahash_ctx(tfm); 1578 1579 ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback.ahash); 1580 rctx->fallback_req.base.flags = req->base.flags & 1581 CRYPTO_TFM_REQ_MAY_SLEEP; 1582 1583 return crypto_ahash_import(&rctx->fallback_req, in); 1584 } 1585 1586 static int sa_sha_export(struct ahash_request *req, void *out) 1587 { 1588 struct sa_sha_req_ctx *rctx = ahash_request_ctx(req); 1589 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 1590 struct sa_tfm_ctx *ctx = crypto_ahash_ctx(tfm); 1591 struct ahash_request *subreq = &rctx->fallback_req; 1592 1593 ahash_request_set_tfm(subreq, ctx->fallback.ahash); 1594 subreq->base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; 1595 1596 return crypto_ahash_export(subreq, out); 1597 } 1598 1599 static int sa_sha1_cra_init(struct crypto_tfm *tfm) 1600 { 1601 struct algo_data ad = { 0 }; 1602 struct sa_tfm_ctx *ctx = crypto_tfm_ctx(tfm); 1603 1604 sa_sha_cra_init_alg(tfm, "sha1"); 1605 1606 ad.aalg_id = SA_AALG_ID_SHA1; 1607 ad.hash_size = SHA1_DIGEST_SIZE; 1608 ad.auth_ctrl = SA_AUTH_SW_CTRL_SHA1; 1609 1610 sa_sha_setup(ctx, &ad); 1611 1612 return 0; 1613 } 1614 1615 static int sa_sha256_cra_init(struct crypto_tfm *tfm) 1616 { 1617 struct algo_data ad = { 0 }; 1618 struct sa_tfm_ctx *ctx = crypto_tfm_ctx(tfm); 1619 1620 sa_sha_cra_init_alg(tfm, "sha256"); 1621 1622 ad.aalg_id = SA_AALG_ID_SHA2_256; 1623 ad.hash_size = SHA256_DIGEST_SIZE; 1624 ad.auth_ctrl = SA_AUTH_SW_CTRL_SHA256; 1625 1626 sa_sha_setup(ctx, &ad); 1627 1628 return 0; 1629 } 1630 1631 static int sa_sha512_cra_init(struct crypto_tfm *tfm) 1632 { 1633 struct algo_data ad = { 0 }; 1634 struct sa_tfm_ctx *ctx = crypto_tfm_ctx(tfm); 1635 1636 sa_sha_cra_init_alg(tfm, "sha512"); 1637 1638 ad.aalg_id = SA_AALG_ID_SHA2_512; 1639 ad.hash_size = SHA512_DIGEST_SIZE; 1640 ad.auth_ctrl = SA_AUTH_SW_CTRL_SHA512; 1641 1642 sa_sha_setup(ctx, &ad); 1643 1644 return 0; 1645 } 1646 1647 static void sa_sha_cra_exit(struct crypto_tfm *tfm) 1648 { 1649 struct sa_tfm_ctx *ctx = crypto_tfm_ctx(tfm); 1650 struct sa_crypto_data *data = dev_get_drvdata(sa_k3_dev); 1651 1652 dev_dbg(sa_k3_dev, "%s(0x%p) sc-ids(0x%x(0x%pad), 0x%x(0x%pad))\n", 1653 __func__, tfm, ctx->enc.sc_id, &ctx->enc.sc_phys, 1654 ctx->dec.sc_id, &ctx->dec.sc_phys); 1655 1656 if (crypto_tfm_alg_type(tfm) == CRYPTO_ALG_TYPE_AHASH) 1657 sa_free_ctx_info(&ctx->enc, data); 1658 1659 crypto_free_shash(ctx->shash); 1660 crypto_free_ahash(ctx->fallback.ahash); 1661 } 1662 1663 static void sa_aead_dma_in_callback(void *data) 1664 { 1665 struct sa_rx_data *rxd = (struct sa_rx_data *)data; 1666 struct aead_request *req; 1667 struct crypto_aead *tfm; 1668 unsigned int start; 1669 unsigned int authsize; 1670 u8 auth_tag[SA_MAX_AUTH_TAG_SZ]; 1671 size_t pl, ml; 1672 int i; 1673 int err = 0; 1674 u16 auth_len; 1675 u32 *mdptr; 1676 1677 sa_sync_from_device(rxd); 1678 req = container_of(rxd->req, struct aead_request, base); 1679 tfm = crypto_aead_reqtfm(req); 1680 start = req->assoclen + req->cryptlen; 1681 authsize = crypto_aead_authsize(tfm); 1682 1683 mdptr = (u32 *)dmaengine_desc_get_metadata_ptr(rxd->tx_in, &pl, &ml); 1684 for (i = 0; i < (authsize / 4); i++) 1685 mdptr[i + 4] = swab32(mdptr[i + 4]); 1686 1687 auth_len = req->assoclen + req->cryptlen; 1688 1689 if (rxd->enc) { 1690 scatterwalk_map_and_copy(&mdptr[4], req->dst, start, authsize, 1691 1); 1692 } else { 1693 auth_len -= authsize; 1694 start -= authsize; 1695 scatterwalk_map_and_copy(auth_tag, req->src, start, authsize, 1696 0); 1697 1698 err = memcmp(&mdptr[4], auth_tag, authsize) ? -EBADMSG : 0; 1699 } 1700 1701 sa_free_sa_rx_data(rxd); 1702 1703 aead_request_complete(req, err); 1704 } 1705 1706 static int sa_cra_init_aead(struct crypto_aead *tfm, const char *hash, 1707 const char *fallback) 1708 { 1709 struct sa_tfm_ctx *ctx = crypto_aead_ctx(tfm); 1710 struct sa_crypto_data *data = dev_get_drvdata(sa_k3_dev); 1711 int ret; 1712 1713 memzero_explicit(ctx, sizeof(*ctx)); 1714 1715 ctx->shash = crypto_alloc_shash(hash, 0, CRYPTO_ALG_NEED_FALLBACK); 1716 if (IS_ERR(ctx->shash)) { 1717 dev_err(sa_k3_dev, "base driver %s couldn't be loaded\n", hash); 1718 return PTR_ERR(ctx->shash); 1719 } 1720 1721 ctx->fallback.aead = crypto_alloc_aead(fallback, 0, 1722 CRYPTO_ALG_NEED_FALLBACK); 1723 1724 if (IS_ERR(ctx->fallback.aead)) { 1725 dev_err(sa_k3_dev, "fallback driver %s couldn't be loaded\n", 1726 fallback); 1727 return PTR_ERR(ctx->fallback.aead); 1728 } 1729 1730 crypto_aead_set_reqsize(tfm, sizeof(struct aead_request) + 1731 crypto_aead_reqsize(ctx->fallback.aead)); 1732 1733 ret = sa_init_ctx_info(&ctx->enc, data); 1734 if (ret) 1735 return ret; 1736 1737 ret = sa_init_ctx_info(&ctx->dec, data); 1738 if (ret) { 1739 sa_free_ctx_info(&ctx->enc, data); 1740 return ret; 1741 } 1742 1743 dev_dbg(sa_k3_dev, "%s(0x%p) sc-ids(0x%x(0x%pad), 0x%x(0x%pad))\n", 1744 __func__, tfm, ctx->enc.sc_id, &ctx->enc.sc_phys, 1745 ctx->dec.sc_id, &ctx->dec.sc_phys); 1746 1747 return ret; 1748 } 1749 1750 static int sa_cra_init_aead_sha1(struct crypto_aead *tfm) 1751 { 1752 return sa_cra_init_aead(tfm, "sha1", 1753 "authenc(hmac(sha1-ce),cbc(aes-ce))"); 1754 } 1755 1756 static int sa_cra_init_aead_sha256(struct crypto_aead *tfm) 1757 { 1758 return sa_cra_init_aead(tfm, "sha256", 1759 "authenc(hmac(sha256-ce),cbc(aes-ce))"); 1760 } 1761 1762 static void sa_exit_tfm_aead(struct crypto_aead *tfm) 1763 { 1764 struct sa_tfm_ctx *ctx = crypto_aead_ctx(tfm); 1765 struct sa_crypto_data *data = dev_get_drvdata(sa_k3_dev); 1766 1767 crypto_free_shash(ctx->shash); 1768 crypto_free_aead(ctx->fallback.aead); 1769 1770 sa_free_ctx_info(&ctx->enc, data); 1771 sa_free_ctx_info(&ctx->dec, data); 1772 } 1773 1774 /* AEAD algorithm configuration interface function */ 1775 static int sa_aead_setkey(struct crypto_aead *authenc, 1776 const u8 *key, unsigned int keylen, 1777 struct algo_data *ad) 1778 { 1779 struct sa_tfm_ctx *ctx = crypto_aead_ctx(authenc); 1780 struct crypto_authenc_keys keys; 1781 int cmdl_len; 1782 struct sa_cmdl_cfg cfg; 1783 int key_idx; 1784 1785 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) 1786 return -EINVAL; 1787 1788 /* Convert the key size (16/24/32) to the key size index (0/1/2) */ 1789 key_idx = (keys.enckeylen >> 3) - 2; 1790 if (key_idx >= 3) 1791 return -EINVAL; 1792 1793 ad->ctx = ctx; 1794 ad->enc_eng.eng_id = SA_ENG_ID_EM1; 1795 ad->enc_eng.sc_size = SA_CTX_ENC_TYPE1_SZ; 1796 ad->auth_eng.eng_id = SA_ENG_ID_AM1; 1797 ad->auth_eng.sc_size = SA_CTX_AUTH_TYPE2_SZ; 1798 ad->mci_enc = mci_cbc_enc_no_iv_array[key_idx]; 1799 ad->mci_dec = mci_cbc_dec_no_iv_array[key_idx]; 1800 ad->inv_key = true; 1801 ad->keyed_mac = true; 1802 ad->ealg_id = SA_EALG_ID_AES_CBC; 1803 ad->prep_iopad = sa_prepare_iopads; 1804 1805 memset(&cfg, 0, sizeof(cfg)); 1806 cfg.enc = true; 1807 cfg.aalg = ad->aalg_id; 1808 cfg.enc_eng_id = ad->enc_eng.eng_id; 1809 cfg.auth_eng_id = ad->auth_eng.eng_id; 1810 cfg.iv_size = crypto_aead_ivsize(authenc); 1811 cfg.akey = keys.authkey; 1812 cfg.akey_len = keys.authkeylen; 1813 1814 /* Setup Encryption Security Context & Command label template */ 1815 if (sa_init_sc(&ctx->enc, keys.enckey, keys.enckeylen, 1816 keys.authkey, keys.authkeylen, 1817 ad, 1, &ctx->enc.epib[1])) 1818 return -EINVAL; 1819 1820 cmdl_len = sa_format_cmdl_gen(&cfg, 1821 (u8 *)ctx->enc.cmdl, 1822 &ctx->enc.cmdl_upd_info); 1823 if (cmdl_len <= 0 || (cmdl_len > SA_MAX_CMDL_WORDS * sizeof(u32))) 1824 return -EINVAL; 1825 1826 ctx->enc.cmdl_size = cmdl_len; 1827 1828 /* Setup Decryption Security Context & Command label template */ 1829 if (sa_init_sc(&ctx->dec, keys.enckey, keys.enckeylen, 1830 keys.authkey, keys.authkeylen, 1831 ad, 0, &ctx->dec.epib[1])) 1832 return -EINVAL; 1833 1834 cfg.enc = false; 1835 cmdl_len = sa_format_cmdl_gen(&cfg, (u8 *)ctx->dec.cmdl, 1836 &ctx->dec.cmdl_upd_info); 1837 1838 if (cmdl_len <= 0 || (cmdl_len > SA_MAX_CMDL_WORDS * sizeof(u32))) 1839 return -EINVAL; 1840 1841 ctx->dec.cmdl_size = cmdl_len; 1842 1843 crypto_aead_clear_flags(ctx->fallback.aead, CRYPTO_TFM_REQ_MASK); 1844 crypto_aead_set_flags(ctx->fallback.aead, 1845 crypto_aead_get_flags(authenc) & 1846 CRYPTO_TFM_REQ_MASK); 1847 crypto_aead_setkey(ctx->fallback.aead, key, keylen); 1848 1849 return 0; 1850 } 1851 1852 static int sa_aead_setauthsize(struct crypto_aead *tfm, unsigned int authsize) 1853 { 1854 struct sa_tfm_ctx *ctx = crypto_tfm_ctx(crypto_aead_tfm(tfm)); 1855 1856 return crypto_aead_setauthsize(ctx->fallback.aead, authsize); 1857 } 1858 1859 static int sa_aead_cbc_sha1_setkey(struct crypto_aead *authenc, 1860 const u8 *key, unsigned int keylen) 1861 { 1862 struct algo_data ad = { 0 }; 1863 1864 ad.ealg_id = SA_EALG_ID_AES_CBC; 1865 ad.aalg_id = SA_AALG_ID_HMAC_SHA1; 1866 ad.hash_size = SHA1_DIGEST_SIZE; 1867 ad.auth_ctrl = SA_AUTH_SW_CTRL_SHA1; 1868 1869 return sa_aead_setkey(authenc, key, keylen, &ad); 1870 } 1871 1872 static int sa_aead_cbc_sha256_setkey(struct crypto_aead *authenc, 1873 const u8 *key, unsigned int keylen) 1874 { 1875 struct algo_data ad = { 0 }; 1876 1877 ad.ealg_id = SA_EALG_ID_AES_CBC; 1878 ad.aalg_id = SA_AALG_ID_HMAC_SHA2_256; 1879 ad.hash_size = SHA256_DIGEST_SIZE; 1880 ad.auth_ctrl = SA_AUTH_SW_CTRL_SHA256; 1881 1882 return sa_aead_setkey(authenc, key, keylen, &ad); 1883 } 1884 1885 static int sa_aead_run(struct aead_request *req, u8 *iv, int enc) 1886 { 1887 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1888 struct sa_tfm_ctx *ctx = crypto_aead_ctx(tfm); 1889 struct sa_req sa_req = { 0 }; 1890 size_t auth_size, enc_size; 1891 1892 enc_size = req->cryptlen; 1893 auth_size = req->assoclen + req->cryptlen; 1894 1895 if (!enc) { 1896 enc_size -= crypto_aead_authsize(tfm); 1897 auth_size -= crypto_aead_authsize(tfm); 1898 } 1899 1900 if (auth_size > SA_MAX_DATA_SZ || 1901 (auth_size >= SA_UNSAFE_DATA_SZ_MIN && 1902 auth_size <= SA_UNSAFE_DATA_SZ_MAX)) { 1903 struct aead_request *subreq = aead_request_ctx(req); 1904 int ret; 1905 1906 aead_request_set_tfm(subreq, ctx->fallback.aead); 1907 aead_request_set_callback(subreq, req->base.flags, 1908 req->base.complete, req->base.data); 1909 aead_request_set_crypt(subreq, req->src, req->dst, 1910 req->cryptlen, req->iv); 1911 aead_request_set_ad(subreq, req->assoclen); 1912 1913 ret = enc ? crypto_aead_encrypt(subreq) : 1914 crypto_aead_decrypt(subreq); 1915 return ret; 1916 } 1917 1918 sa_req.enc_offset = req->assoclen; 1919 sa_req.enc_size = enc_size; 1920 sa_req.auth_size = auth_size; 1921 sa_req.size = auth_size; 1922 sa_req.enc_iv = iv; 1923 sa_req.type = CRYPTO_ALG_TYPE_AEAD; 1924 sa_req.enc = enc; 1925 sa_req.callback = sa_aead_dma_in_callback; 1926 sa_req.mdata_size = 52; 1927 sa_req.base = &req->base; 1928 sa_req.ctx = ctx; 1929 sa_req.src = req->src; 1930 sa_req.dst = req->dst; 1931 1932 return sa_run(&sa_req); 1933 } 1934 1935 /* AEAD algorithm encrypt interface function */ 1936 static int sa_aead_encrypt(struct aead_request *req) 1937 { 1938 return sa_aead_run(req, req->iv, 1); 1939 } 1940 1941 /* AEAD algorithm decrypt interface function */ 1942 static int sa_aead_decrypt(struct aead_request *req) 1943 { 1944 return sa_aead_run(req, req->iv, 0); 1945 } 1946 1947 static struct sa_alg_tmpl sa_algs[] = { 1948 { 1949 .type = CRYPTO_ALG_TYPE_SKCIPHER, 1950 .alg.skcipher = { 1951 .base.cra_name = "cbc(aes)", 1952 .base.cra_driver_name = "cbc-aes-sa2ul", 1953 .base.cra_priority = 30000, 1954 .base.cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | 1955 CRYPTO_ALG_KERN_DRIVER_ONLY | 1956 CRYPTO_ALG_ASYNC | 1957 CRYPTO_ALG_NEED_FALLBACK, 1958 .base.cra_blocksize = AES_BLOCK_SIZE, 1959 .base.cra_ctxsize = sizeof(struct sa_tfm_ctx), 1960 .base.cra_module = THIS_MODULE, 1961 .init = sa_cipher_cra_init, 1962 .exit = sa_cipher_cra_exit, 1963 .min_keysize = AES_MIN_KEY_SIZE, 1964 .max_keysize = AES_MAX_KEY_SIZE, 1965 .ivsize = AES_BLOCK_SIZE, 1966 .setkey = sa_aes_cbc_setkey, 1967 .encrypt = sa_encrypt, 1968 .decrypt = sa_decrypt, 1969 } 1970 }, 1971 { 1972 .type = CRYPTO_ALG_TYPE_SKCIPHER, 1973 .alg.skcipher = { 1974 .base.cra_name = "ecb(aes)", 1975 .base.cra_driver_name = "ecb-aes-sa2ul", 1976 .base.cra_priority = 30000, 1977 .base.cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | 1978 CRYPTO_ALG_KERN_DRIVER_ONLY | 1979 CRYPTO_ALG_ASYNC | 1980 CRYPTO_ALG_NEED_FALLBACK, 1981 .base.cra_blocksize = AES_BLOCK_SIZE, 1982 .base.cra_ctxsize = sizeof(struct sa_tfm_ctx), 1983 .base.cra_module = THIS_MODULE, 1984 .init = sa_cipher_cra_init, 1985 .exit = sa_cipher_cra_exit, 1986 .min_keysize = AES_MIN_KEY_SIZE, 1987 .max_keysize = AES_MAX_KEY_SIZE, 1988 .setkey = sa_aes_ecb_setkey, 1989 .encrypt = sa_encrypt, 1990 .decrypt = sa_decrypt, 1991 } 1992 }, 1993 { 1994 .type = CRYPTO_ALG_TYPE_SKCIPHER, 1995 .alg.skcipher = { 1996 .base.cra_name = "cbc(des3_ede)", 1997 .base.cra_driver_name = "cbc-des3-sa2ul", 1998 .base.cra_priority = 30000, 1999 .base.cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | 2000 CRYPTO_ALG_KERN_DRIVER_ONLY | 2001 CRYPTO_ALG_ASYNC | 2002 CRYPTO_ALG_NEED_FALLBACK, 2003 .base.cra_blocksize = DES_BLOCK_SIZE, 2004 .base.cra_ctxsize = sizeof(struct sa_tfm_ctx), 2005 .base.cra_module = THIS_MODULE, 2006 .init = sa_cipher_cra_init, 2007 .exit = sa_cipher_cra_exit, 2008 .min_keysize = 3 * DES_KEY_SIZE, 2009 .max_keysize = 3 * DES_KEY_SIZE, 2010 .ivsize = DES_BLOCK_SIZE, 2011 .setkey = sa_3des_cbc_setkey, 2012 .encrypt = sa_encrypt, 2013 .decrypt = sa_decrypt, 2014 } 2015 }, 2016 { 2017 .type = CRYPTO_ALG_TYPE_SKCIPHER, 2018 .alg.skcipher = { 2019 .base.cra_name = "ecb(des3_ede)", 2020 .base.cra_driver_name = "ecb-des3-sa2ul", 2021 .base.cra_priority = 30000, 2022 .base.cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | 2023 CRYPTO_ALG_KERN_DRIVER_ONLY | 2024 CRYPTO_ALG_ASYNC | 2025 CRYPTO_ALG_NEED_FALLBACK, 2026 .base.cra_blocksize = DES_BLOCK_SIZE, 2027 .base.cra_ctxsize = sizeof(struct sa_tfm_ctx), 2028 .base.cra_module = THIS_MODULE, 2029 .init = sa_cipher_cra_init, 2030 .exit = sa_cipher_cra_exit, 2031 .min_keysize = 3 * DES_KEY_SIZE, 2032 .max_keysize = 3 * DES_KEY_SIZE, 2033 .setkey = sa_3des_ecb_setkey, 2034 .encrypt = sa_encrypt, 2035 .decrypt = sa_decrypt, 2036 } 2037 }, 2038 { 2039 .type = CRYPTO_ALG_TYPE_AHASH, 2040 .alg.ahash = { 2041 .halg.base = { 2042 .cra_name = "sha1", 2043 .cra_driver_name = "sha1-sa2ul", 2044 .cra_priority = 400, 2045 .cra_flags = CRYPTO_ALG_TYPE_AHASH | 2046 CRYPTO_ALG_ASYNC | 2047 CRYPTO_ALG_KERN_DRIVER_ONLY | 2048 CRYPTO_ALG_NEED_FALLBACK, 2049 .cra_blocksize = SHA1_BLOCK_SIZE, 2050 .cra_ctxsize = sizeof(struct sa_tfm_ctx), 2051 .cra_module = THIS_MODULE, 2052 .cra_init = sa_sha1_cra_init, 2053 .cra_exit = sa_sha_cra_exit, 2054 }, 2055 .halg.digestsize = SHA1_DIGEST_SIZE, 2056 .halg.statesize = sizeof(struct sa_sha_req_ctx) + 2057 sizeof(struct sha1_state), 2058 .init = sa_sha_init, 2059 .update = sa_sha_update, 2060 .final = sa_sha_final, 2061 .finup = sa_sha_finup, 2062 .digest = sa_sha_digest, 2063 .export = sa_sha_export, 2064 .import = sa_sha_import, 2065 }, 2066 }, 2067 { 2068 .type = CRYPTO_ALG_TYPE_AHASH, 2069 .alg.ahash = { 2070 .halg.base = { 2071 .cra_name = "sha256", 2072 .cra_driver_name = "sha256-sa2ul", 2073 .cra_priority = 400, 2074 .cra_flags = CRYPTO_ALG_TYPE_AHASH | 2075 CRYPTO_ALG_ASYNC | 2076 CRYPTO_ALG_KERN_DRIVER_ONLY | 2077 CRYPTO_ALG_NEED_FALLBACK, 2078 .cra_blocksize = SHA256_BLOCK_SIZE, 2079 .cra_ctxsize = sizeof(struct sa_tfm_ctx), 2080 .cra_module = THIS_MODULE, 2081 .cra_init = sa_sha256_cra_init, 2082 .cra_exit = sa_sha_cra_exit, 2083 }, 2084 .halg.digestsize = SHA256_DIGEST_SIZE, 2085 .halg.statesize = sizeof(struct sa_sha_req_ctx) + 2086 sizeof(struct sha256_state), 2087 .init = sa_sha_init, 2088 .update = sa_sha_update, 2089 .final = sa_sha_final, 2090 .finup = sa_sha_finup, 2091 .digest = sa_sha_digest, 2092 .export = sa_sha_export, 2093 .import = sa_sha_import, 2094 }, 2095 }, 2096 { 2097 .type = CRYPTO_ALG_TYPE_AHASH, 2098 .alg.ahash = { 2099 .halg.base = { 2100 .cra_name = "sha512", 2101 .cra_driver_name = "sha512-sa2ul", 2102 .cra_priority = 400, 2103 .cra_flags = CRYPTO_ALG_TYPE_AHASH | 2104 CRYPTO_ALG_ASYNC | 2105 CRYPTO_ALG_KERN_DRIVER_ONLY | 2106 CRYPTO_ALG_NEED_FALLBACK, 2107 .cra_blocksize = SHA512_BLOCK_SIZE, 2108 .cra_ctxsize = sizeof(struct sa_tfm_ctx), 2109 .cra_module = THIS_MODULE, 2110 .cra_init = sa_sha512_cra_init, 2111 .cra_exit = sa_sha_cra_exit, 2112 }, 2113 .halg.digestsize = SHA512_DIGEST_SIZE, 2114 .halg.statesize = sizeof(struct sa_sha_req_ctx) + 2115 sizeof(struct sha512_state), 2116 .init = sa_sha_init, 2117 .update = sa_sha_update, 2118 .final = sa_sha_final, 2119 .finup = sa_sha_finup, 2120 .digest = sa_sha_digest, 2121 .export = sa_sha_export, 2122 .import = sa_sha_import, 2123 }, 2124 }, 2125 { 2126 .type = CRYPTO_ALG_TYPE_AEAD, 2127 .alg.aead = { 2128 .base = { 2129 .cra_name = "authenc(hmac(sha1),cbc(aes))", 2130 .cra_driver_name = 2131 "authenc(hmac(sha1),cbc(aes))-sa2ul", 2132 .cra_blocksize = AES_BLOCK_SIZE, 2133 .cra_flags = CRYPTO_ALG_TYPE_AEAD | 2134 CRYPTO_ALG_KERN_DRIVER_ONLY | 2135 CRYPTO_ALG_ASYNC | 2136 CRYPTO_ALG_NEED_FALLBACK, 2137 .cra_ctxsize = sizeof(struct sa_tfm_ctx), 2138 .cra_module = THIS_MODULE, 2139 .cra_priority = 3000, 2140 }, 2141 .ivsize = AES_BLOCK_SIZE, 2142 .maxauthsize = SHA1_DIGEST_SIZE, 2143 2144 .init = sa_cra_init_aead_sha1, 2145 .exit = sa_exit_tfm_aead, 2146 .setkey = sa_aead_cbc_sha1_setkey, 2147 .setauthsize = sa_aead_setauthsize, 2148 .encrypt = sa_aead_encrypt, 2149 .decrypt = sa_aead_decrypt, 2150 }, 2151 }, 2152 { 2153 .type = CRYPTO_ALG_TYPE_AEAD, 2154 .alg.aead = { 2155 .base = { 2156 .cra_name = "authenc(hmac(sha256),cbc(aes))", 2157 .cra_driver_name = 2158 "authenc(hmac(sha256),cbc(aes))-sa2ul", 2159 .cra_blocksize = AES_BLOCK_SIZE, 2160 .cra_flags = CRYPTO_ALG_TYPE_AEAD | 2161 CRYPTO_ALG_KERN_DRIVER_ONLY | 2162 CRYPTO_ALG_ASYNC | 2163 CRYPTO_ALG_NEED_FALLBACK, 2164 .cra_ctxsize = sizeof(struct sa_tfm_ctx), 2165 .cra_module = THIS_MODULE, 2166 .cra_alignmask = 0, 2167 .cra_priority = 3000, 2168 }, 2169 .ivsize = AES_BLOCK_SIZE, 2170 .maxauthsize = SHA256_DIGEST_SIZE, 2171 2172 .init = sa_cra_init_aead_sha256, 2173 .exit = sa_exit_tfm_aead, 2174 .setkey = sa_aead_cbc_sha256_setkey, 2175 .setauthsize = sa_aead_setauthsize, 2176 .encrypt = sa_aead_encrypt, 2177 .decrypt = sa_aead_decrypt, 2178 }, 2179 }, 2180 }; 2181 2182 /* Register the algorithms in crypto framework */ 2183 static void sa_register_algos(const struct device *dev) 2184 { 2185 char *alg_name; 2186 u32 type; 2187 int i, err; 2188 2189 for (i = 0; i < ARRAY_SIZE(sa_algs); i++) { 2190 type = sa_algs[i].type; 2191 if (type == CRYPTO_ALG_TYPE_SKCIPHER) { 2192 alg_name = sa_algs[i].alg.skcipher.base.cra_name; 2193 err = crypto_register_skcipher(&sa_algs[i].alg.skcipher); 2194 } else if (type == CRYPTO_ALG_TYPE_AHASH) { 2195 alg_name = sa_algs[i].alg.ahash.halg.base.cra_name; 2196 err = crypto_register_ahash(&sa_algs[i].alg.ahash); 2197 } else if (type == CRYPTO_ALG_TYPE_AEAD) { 2198 alg_name = sa_algs[i].alg.aead.base.cra_name; 2199 err = crypto_register_aead(&sa_algs[i].alg.aead); 2200 } else { 2201 dev_err(dev, 2202 "un-supported crypto algorithm (%d)", 2203 sa_algs[i].type); 2204 continue; 2205 } 2206 2207 if (err) 2208 dev_err(dev, "Failed to register '%s'\n", alg_name); 2209 else 2210 sa_algs[i].registered = true; 2211 } 2212 } 2213 2214 /* Unregister the algorithms in crypto framework */ 2215 static void sa_unregister_algos(const struct device *dev) 2216 { 2217 u32 type; 2218 int i; 2219 2220 for (i = 0; i < ARRAY_SIZE(sa_algs); i++) { 2221 type = sa_algs[i].type; 2222 if (!sa_algs[i].registered) 2223 continue; 2224 if (type == CRYPTO_ALG_TYPE_SKCIPHER) 2225 crypto_unregister_skcipher(&sa_algs[i].alg.skcipher); 2226 else if (type == CRYPTO_ALG_TYPE_AHASH) 2227 crypto_unregister_ahash(&sa_algs[i].alg.ahash); 2228 else if (type == CRYPTO_ALG_TYPE_AEAD) 2229 crypto_unregister_aead(&sa_algs[i].alg.aead); 2230 2231 sa_algs[i].registered = false; 2232 } 2233 } 2234 2235 static int sa_init_mem(struct sa_crypto_data *dev_data) 2236 { 2237 struct device *dev = &dev_data->pdev->dev; 2238 /* Setup dma pool for security context buffers */ 2239 dev_data->sc_pool = dma_pool_create("keystone-sc", dev, 2240 SA_CTX_MAX_SZ, 64, 0); 2241 if (!dev_data->sc_pool) { 2242 dev_err(dev, "Failed to create dma pool"); 2243 return -ENOMEM; 2244 } 2245 2246 return 0; 2247 } 2248 2249 static int sa_dma_init(struct sa_crypto_data *dd) 2250 { 2251 int ret; 2252 struct dma_slave_config cfg; 2253 2254 dd->dma_rx1 = NULL; 2255 dd->dma_tx = NULL; 2256 dd->dma_rx2 = NULL; 2257 2258 ret = dma_coerce_mask_and_coherent(dd->dev, DMA_BIT_MASK(48)); 2259 if (ret) 2260 return ret; 2261 2262 dd->dma_rx1 = dma_request_chan(dd->dev, "rx1"); 2263 if (IS_ERR(dd->dma_rx1)) 2264 return dev_err_probe(dd->dev, PTR_ERR(dd->dma_rx1), 2265 "Unable to request rx1 DMA channel\n"); 2266 2267 dd->dma_rx2 = dma_request_chan(dd->dev, "rx2"); 2268 if (IS_ERR(dd->dma_rx2)) { 2269 dma_release_channel(dd->dma_rx1); 2270 return dev_err_probe(dd->dev, PTR_ERR(dd->dma_rx2), 2271 "Unable to request rx2 DMA channel\n"); 2272 } 2273 2274 dd->dma_tx = dma_request_chan(dd->dev, "tx"); 2275 if (IS_ERR(dd->dma_tx)) { 2276 ret = dev_err_probe(dd->dev, PTR_ERR(dd->dma_tx), 2277 "Unable to request tx DMA channel\n"); 2278 goto err_dma_tx; 2279 } 2280 2281 memzero_explicit(&cfg, sizeof(cfg)); 2282 2283 cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; 2284 cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; 2285 cfg.src_maxburst = 4; 2286 cfg.dst_maxburst = 4; 2287 2288 ret = dmaengine_slave_config(dd->dma_rx1, &cfg); 2289 if (ret) { 2290 dev_err(dd->dev, "can't configure IN dmaengine slave: %d\n", 2291 ret); 2292 return ret; 2293 } 2294 2295 ret = dmaengine_slave_config(dd->dma_rx2, &cfg); 2296 if (ret) { 2297 dev_err(dd->dev, "can't configure IN dmaengine slave: %d\n", 2298 ret); 2299 return ret; 2300 } 2301 2302 ret = dmaengine_slave_config(dd->dma_tx, &cfg); 2303 if (ret) { 2304 dev_err(dd->dev, "can't configure OUT dmaengine slave: %d\n", 2305 ret); 2306 return ret; 2307 } 2308 2309 return 0; 2310 2311 err_dma_tx: 2312 dma_release_channel(dd->dma_rx1); 2313 dma_release_channel(dd->dma_rx2); 2314 2315 return ret; 2316 } 2317 2318 static int sa_link_child(struct device *dev, void *data) 2319 { 2320 struct device *parent = data; 2321 2322 device_link_add(dev, parent, DL_FLAG_AUTOPROBE_CONSUMER); 2323 2324 return 0; 2325 } 2326 2327 static int sa_ul_probe(struct platform_device *pdev) 2328 { 2329 struct device *dev = &pdev->dev; 2330 struct device_node *node = dev->of_node; 2331 struct resource *res; 2332 static void __iomem *saul_base; 2333 struct sa_crypto_data *dev_data; 2334 u32 val; 2335 int ret; 2336 2337 dev_data = devm_kzalloc(dev, sizeof(*dev_data), GFP_KERNEL); 2338 if (!dev_data) 2339 return -ENOMEM; 2340 2341 sa_k3_dev = dev; 2342 dev_data->dev = dev; 2343 dev_data->pdev = pdev; 2344 platform_set_drvdata(pdev, dev_data); 2345 dev_set_drvdata(sa_k3_dev, dev_data); 2346 2347 pm_runtime_enable(dev); 2348 ret = pm_runtime_get_sync(dev); 2349 if (ret < 0) { 2350 dev_err(&pdev->dev, "%s: failed to get sync: %d\n", __func__, 2351 ret); 2352 return ret; 2353 } 2354 2355 sa_init_mem(dev_data); 2356 ret = sa_dma_init(dev_data); 2357 if (ret) 2358 goto disable_pm_runtime; 2359 2360 spin_lock_init(&dev_data->scid_lock); 2361 res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 2362 saul_base = devm_ioremap_resource(dev, res); 2363 2364 dev_data->base = saul_base; 2365 val = SA_EEC_ENCSS_EN | SA_EEC_AUTHSS_EN | SA_EEC_CTXCACH_EN | 2366 SA_EEC_CPPI_PORT_IN_EN | SA_EEC_CPPI_PORT_OUT_EN | 2367 SA_EEC_TRNG_EN; 2368 2369 writel_relaxed(val, saul_base + SA_ENGINE_ENABLE_CONTROL); 2370 2371 sa_register_algos(dev); 2372 2373 ret = of_platform_populate(node, NULL, NULL, &pdev->dev); 2374 if (ret) 2375 goto release_dma; 2376 2377 device_for_each_child(&pdev->dev, &pdev->dev, sa_link_child); 2378 2379 return 0; 2380 2381 release_dma: 2382 sa_unregister_algos(&pdev->dev); 2383 2384 dma_release_channel(dev_data->dma_rx2); 2385 dma_release_channel(dev_data->dma_rx1); 2386 dma_release_channel(dev_data->dma_tx); 2387 2388 dma_pool_destroy(dev_data->sc_pool); 2389 2390 disable_pm_runtime: 2391 pm_runtime_put_sync(&pdev->dev); 2392 pm_runtime_disable(&pdev->dev); 2393 2394 return ret; 2395 } 2396 2397 static int sa_ul_remove(struct platform_device *pdev) 2398 { 2399 struct sa_crypto_data *dev_data = platform_get_drvdata(pdev); 2400 2401 sa_unregister_algos(&pdev->dev); 2402 2403 dma_release_channel(dev_data->dma_rx2); 2404 dma_release_channel(dev_data->dma_rx1); 2405 dma_release_channel(dev_data->dma_tx); 2406 2407 dma_pool_destroy(dev_data->sc_pool); 2408 2409 platform_set_drvdata(pdev, NULL); 2410 2411 pm_runtime_put_sync(&pdev->dev); 2412 pm_runtime_disable(&pdev->dev); 2413 2414 return 0; 2415 } 2416 2417 static const struct of_device_id of_match[] = { 2418 {.compatible = "ti,j721e-sa2ul",}, 2419 {.compatible = "ti,am654-sa2ul",}, 2420 {}, 2421 }; 2422 MODULE_DEVICE_TABLE(of, of_match); 2423 2424 static struct platform_driver sa_ul_driver = { 2425 .probe = sa_ul_probe, 2426 .remove = sa_ul_remove, 2427 .driver = { 2428 .name = "saul-crypto", 2429 .of_match_table = of_match, 2430 }, 2431 }; 2432 module_platform_driver(sa_ul_driver); 2433 MODULE_LICENSE("GPL v2"); 2434