1 // SPDX-License-Identifier: GPL-2.0 2 #include <linux/kernel.h> 3 #include <linux/printk.h> 4 #include <linux/crypto.h> 5 #include <linux/rtnetlink.h> 6 7 #include <crypto/aead.h> 8 #include <crypto/authenc.h> 9 #include <crypto/des.h> 10 #include <crypto/sha.h> 11 #include <crypto/internal/aead.h> 12 #include <crypto/scatterwalk.h> 13 #include <crypto/gcm.h> 14 15 #include "nitrox_dev.h" 16 #include "nitrox_common.h" 17 #include "nitrox_req.h" 18 19 #define GCM_AES_SALT_SIZE 4 20 21 /** 22 * struct nitrox_crypt_params - Params to set nitrox crypto request. 23 * @cryptlen: Encryption/Decryption data length 24 * @authlen: Assoc data length + Cryptlen 25 * @srclen: Input buffer length 26 * @dstlen: Output buffer length 27 * @iv: IV data 28 * @ivsize: IV data length 29 * @ctrl_arg: Identifies the request type (ENCRYPT/DECRYPT) 30 */ 31 struct nitrox_crypt_params { 32 unsigned int cryptlen; 33 unsigned int authlen; 34 unsigned int srclen; 35 unsigned int dstlen; 36 u8 *iv; 37 int ivsize; 38 u8 ctrl_arg; 39 }; 40 41 union gph_p3 { 42 struct { 43 #ifdef __BIG_ENDIAN_BITFIELD 44 u16 iv_offset : 8; 45 u16 auth_offset : 8; 46 #else 47 u16 auth_offset : 8; 48 u16 iv_offset : 8; 49 #endif 50 }; 51 u16 param; 52 }; 53 54 static int nitrox_aes_gcm_setkey(struct crypto_aead *aead, const u8 *key, 55 unsigned int keylen) 56 { 57 int aes_keylen; 58 struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead); 59 struct flexi_crypto_context *fctx; 60 union fc_ctx_flags flags; 61 62 aes_keylen = flexi_aes_keylen(keylen); 63 if (aes_keylen < 0) { 64 crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN); 65 return -EINVAL; 66 } 67 68 /* fill crypto context */ 69 fctx = nctx->u.fctx; 70 flags.f = be64_to_cpu(fctx->flags.f); 71 flags.w0.aes_keylen = aes_keylen; 72 fctx->flags.f = cpu_to_be64(flags.f); 73 74 /* copy enc key to context */ 75 memset(&fctx->crypto, 0, sizeof(fctx->crypto)); 76 memcpy(fctx->crypto.u.key, key, keylen); 77 78 return 0; 79 } 80 81 static int nitrox_aead_setauthsize(struct crypto_aead *aead, 82 unsigned int authsize) 83 { 84 struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead); 85 struct flexi_crypto_context *fctx = nctx->u.fctx; 86 union fc_ctx_flags flags; 87 88 flags.f = be64_to_cpu(fctx->flags.f); 89 flags.w0.mac_len = authsize; 90 fctx->flags.f = cpu_to_be64(flags.f); 91 92 aead->authsize = authsize; 93 94 return 0; 95 } 96 97 static int alloc_src_sglist(struct aead_request *areq, char *iv, int ivsize, 98 int buflen) 99 { 100 struct nitrox_kcrypt_request *nkreq = aead_request_ctx(areq); 101 int nents = sg_nents_for_len(areq->src, buflen) + 1; 102 int ret; 103 104 if (nents < 0) 105 return nents; 106 107 /* Allocate buffer to hold IV and input scatterlist array */ 108 ret = alloc_src_req_buf(nkreq, nents, ivsize); 109 if (ret) 110 return ret; 111 112 nitrox_creq_copy_iv(nkreq->src, iv, ivsize); 113 nitrox_creq_set_src_sg(nkreq, nents, ivsize, areq->src, buflen); 114 115 return 0; 116 } 117 118 static int alloc_dst_sglist(struct aead_request *areq, int ivsize, int buflen) 119 { 120 struct nitrox_kcrypt_request *nkreq = aead_request_ctx(areq); 121 int nents = sg_nents_for_len(areq->dst, buflen) + 3; 122 int ret; 123 124 if (nents < 0) 125 return nents; 126 127 /* Allocate buffer to hold ORH, COMPLETION and output scatterlist 128 * array 129 */ 130 ret = alloc_dst_req_buf(nkreq, nents); 131 if (ret) 132 return ret; 133 134 nitrox_creq_set_orh(nkreq); 135 nitrox_creq_set_comp(nkreq); 136 nitrox_creq_set_dst_sg(nkreq, nents, ivsize, areq->dst, buflen); 137 138 return 0; 139 } 140 141 static void free_src_sglist(struct aead_request *areq) 142 { 143 struct nitrox_kcrypt_request *nkreq = aead_request_ctx(areq); 144 145 kfree(nkreq->src); 146 } 147 148 static void free_dst_sglist(struct aead_request *areq) 149 { 150 struct nitrox_kcrypt_request *nkreq = aead_request_ctx(areq); 151 152 kfree(nkreq->dst); 153 } 154 155 static int nitrox_set_creq(struct aead_request *areq, 156 struct nitrox_crypt_params *params) 157 { 158 struct nitrox_kcrypt_request *nkreq = aead_request_ctx(areq); 159 struct se_crypto_request *creq = &nkreq->creq; 160 struct crypto_aead *aead = crypto_aead_reqtfm(areq); 161 union gph_p3 param3; 162 struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead); 163 int ret; 164 165 creq->flags = areq->base.flags; 166 creq->gfp = (areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 167 GFP_KERNEL : GFP_ATOMIC; 168 169 creq->ctrl.value = 0; 170 creq->opcode = FLEXI_CRYPTO_ENCRYPT_HMAC; 171 creq->ctrl.s.arg = params->ctrl_arg; 172 173 creq->gph.param0 = cpu_to_be16(params->cryptlen); 174 creq->gph.param1 = cpu_to_be16(params->authlen); 175 creq->gph.param2 = cpu_to_be16(params->ivsize + areq->assoclen); 176 param3.iv_offset = 0; 177 param3.auth_offset = params->ivsize; 178 creq->gph.param3 = cpu_to_be16(param3.param); 179 180 creq->ctx_handle = nctx->u.ctx_handle; 181 creq->ctrl.s.ctxl = sizeof(struct flexi_crypto_context); 182 183 ret = alloc_src_sglist(areq, params->iv, params->ivsize, 184 params->srclen); 185 if (ret) 186 return ret; 187 188 ret = alloc_dst_sglist(areq, params->ivsize, params->dstlen); 189 if (ret) { 190 free_src_sglist(areq); 191 return ret; 192 } 193 194 return 0; 195 } 196 197 static void nitrox_aead_callback(void *arg, int err) 198 { 199 struct aead_request *areq = arg; 200 201 free_src_sglist(areq); 202 free_dst_sglist(areq); 203 if (err) { 204 pr_err_ratelimited("request failed status 0x%0x\n", err); 205 err = -EINVAL; 206 } 207 208 areq->base.complete(&areq->base, err); 209 } 210 211 static int nitrox_aes_gcm_enc(struct aead_request *areq) 212 { 213 struct crypto_aead *aead = crypto_aead_reqtfm(areq); 214 struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead); 215 struct nitrox_kcrypt_request *nkreq = aead_request_ctx(areq); 216 struct se_crypto_request *creq = &nkreq->creq; 217 struct flexi_crypto_context *fctx = nctx->u.fctx; 218 struct nitrox_crypt_params params; 219 int ret; 220 221 memcpy(fctx->crypto.iv, areq->iv, GCM_AES_SALT_SIZE); 222 223 memset(¶ms, 0, sizeof(params)); 224 params.cryptlen = areq->cryptlen; 225 params.authlen = areq->assoclen + params.cryptlen; 226 params.srclen = params.authlen; 227 params.dstlen = params.srclen + aead->authsize; 228 params.iv = &areq->iv[GCM_AES_SALT_SIZE]; 229 params.ivsize = GCM_AES_IV_SIZE - GCM_AES_SALT_SIZE; 230 params.ctrl_arg = ENCRYPT; 231 ret = nitrox_set_creq(areq, ¶ms); 232 if (ret) 233 return ret; 234 235 /* send the crypto request */ 236 return nitrox_process_se_request(nctx->ndev, creq, nitrox_aead_callback, 237 areq); 238 } 239 240 static int nitrox_aes_gcm_dec(struct aead_request *areq) 241 { 242 struct crypto_aead *aead = crypto_aead_reqtfm(areq); 243 struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead); 244 struct nitrox_kcrypt_request *nkreq = aead_request_ctx(areq); 245 struct se_crypto_request *creq = &nkreq->creq; 246 struct flexi_crypto_context *fctx = nctx->u.fctx; 247 struct nitrox_crypt_params params; 248 int ret; 249 250 memcpy(fctx->crypto.iv, areq->iv, GCM_AES_SALT_SIZE); 251 252 memset(¶ms, 0, sizeof(params)); 253 params.cryptlen = areq->cryptlen - aead->authsize; 254 params.authlen = areq->assoclen + params.cryptlen; 255 params.srclen = areq->cryptlen + areq->assoclen; 256 params.dstlen = params.srclen - aead->authsize; 257 params.iv = &areq->iv[GCM_AES_SALT_SIZE]; 258 params.ivsize = GCM_AES_IV_SIZE - GCM_AES_SALT_SIZE; 259 params.ctrl_arg = DECRYPT; 260 ret = nitrox_set_creq(areq, ¶ms); 261 if (ret) 262 return ret; 263 264 /* send the crypto request */ 265 return nitrox_process_se_request(nctx->ndev, creq, nitrox_aead_callback, 266 areq); 267 } 268 269 static int nitrox_aead_init(struct crypto_aead *aead) 270 { 271 struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead); 272 struct crypto_ctx_hdr *chdr; 273 274 /* get the first device */ 275 nctx->ndev = nitrox_get_first_device(); 276 if (!nctx->ndev) 277 return -ENODEV; 278 279 /* allocate nitrox crypto context */ 280 chdr = crypto_alloc_context(nctx->ndev); 281 if (!chdr) { 282 nitrox_put_device(nctx->ndev); 283 return -ENOMEM; 284 } 285 nctx->chdr = chdr; 286 nctx->u.ctx_handle = (uintptr_t)((u8 *)chdr->vaddr + 287 sizeof(struct ctx_hdr)); 288 nctx->u.fctx->flags.f = 0; 289 290 return 0; 291 } 292 293 static int nitrox_aes_gcm_init(struct crypto_aead *aead) 294 { 295 int ret; 296 struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead); 297 union fc_ctx_flags *flags; 298 299 ret = nitrox_aead_init(aead); 300 if (ret) 301 return ret; 302 303 flags = &nctx->u.fctx->flags; 304 flags->w0.cipher_type = CIPHER_AES_GCM; 305 flags->w0.hash_type = AUTH_NULL; 306 flags->w0.iv_source = IV_FROM_DPTR; 307 /* ask microcode to calculate ipad/opad */ 308 flags->w0.auth_input_type = 1; 309 flags->f = be64_to_cpu(flags->f); 310 311 crypto_aead_set_reqsize(aead, sizeof(struct aead_request) + 312 sizeof(struct nitrox_kcrypt_request)); 313 314 return 0; 315 } 316 317 static void nitrox_aead_exit(struct crypto_aead *aead) 318 { 319 struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead); 320 321 /* free the nitrox crypto context */ 322 if (nctx->u.ctx_handle) { 323 struct flexi_crypto_context *fctx = nctx->u.fctx; 324 325 memzero_explicit(&fctx->crypto, sizeof(struct crypto_keys)); 326 memzero_explicit(&fctx->auth, sizeof(struct auth_keys)); 327 crypto_free_context((void *)nctx->chdr); 328 } 329 nitrox_put_device(nctx->ndev); 330 331 nctx->u.ctx_handle = 0; 332 nctx->ndev = NULL; 333 } 334 335 static struct aead_alg nitrox_aeads[] = { { 336 .base = { 337 .cra_name = "gcm(aes)", 338 .cra_driver_name = "n5_aes_gcm", 339 .cra_priority = PRIO, 340 .cra_flags = CRYPTO_ALG_ASYNC, 341 .cra_blocksize = AES_BLOCK_SIZE, 342 .cra_ctxsize = sizeof(struct nitrox_crypto_ctx), 343 .cra_alignmask = 0, 344 .cra_module = THIS_MODULE, 345 }, 346 .setkey = nitrox_aes_gcm_setkey, 347 .setauthsize = nitrox_aead_setauthsize, 348 .encrypt = nitrox_aes_gcm_enc, 349 .decrypt = nitrox_aes_gcm_dec, 350 .init = nitrox_aes_gcm_init, 351 .exit = nitrox_aead_exit, 352 .ivsize = GCM_AES_IV_SIZE, 353 .maxauthsize = AES_BLOCK_SIZE, 354 } }; 355 356 int nitrox_register_aeads(void) 357 { 358 return crypto_register_aeads(nitrox_aeads, ARRAY_SIZE(nitrox_aeads)); 359 } 360 361 void nitrox_unregister_aeads(void) 362 { 363 crypto_unregister_aeads(nitrox_aeads, ARRAY_SIZE(nitrox_aeads)); 364 } 365