1 // SPDX-License-Identifier: GPL-2.0
2 #include <linux/kernel.h>
3 #include <linux/printk.h>
4 #include <linux/crypto.h>
5 #include <linux/rtnetlink.h>
6 
7 #include <crypto/aead.h>
8 #include <crypto/authenc.h>
9 #include <crypto/des.h>
10 #include <crypto/sha.h>
11 #include <crypto/internal/aead.h>
12 #include <crypto/scatterwalk.h>
13 #include <crypto/gcm.h>
14 
15 #include "nitrox_dev.h"
16 #include "nitrox_common.h"
17 #include "nitrox_req.h"
18 
19 #define GCM_AES_SALT_SIZE	4
20 
21 /**
22  * struct nitrox_crypt_params - Params to set nitrox crypto request.
23  * @cryptlen: Encryption/Decryption data length
24  * @authlen: Assoc data length + Cryptlen
25  * @srclen: Input buffer length
26  * @dstlen: Output buffer length
27  * @iv: IV data
28  * @ivsize: IV data length
29  * @ctrl_arg: Identifies the request type (ENCRYPT/DECRYPT)
30  */
31 struct nitrox_crypt_params {
32 	unsigned int cryptlen;
33 	unsigned int authlen;
34 	unsigned int srclen;
35 	unsigned int dstlen;
36 	u8 *iv;
37 	int ivsize;
38 	u8 ctrl_arg;
39 };
40 
41 union gph_p3 {
42 	struct {
43 #ifdef __BIG_ENDIAN_BITFIELD
44 		u16 iv_offset : 8;
45 		u16 auth_offset	: 8;
46 #else
47 		u16 auth_offset	: 8;
48 		u16 iv_offset : 8;
49 #endif
50 	};
51 	u16 param;
52 };
53 
54 static int nitrox_aes_gcm_setkey(struct crypto_aead *aead, const u8 *key,
55 				 unsigned int keylen)
56 {
57 	int aes_keylen;
58 	struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
59 	struct flexi_crypto_context *fctx;
60 	union fc_ctx_flags flags;
61 
62 	aes_keylen = flexi_aes_keylen(keylen);
63 	if (aes_keylen < 0) {
64 		crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN);
65 		return -EINVAL;
66 	}
67 
68 	/* fill crypto context */
69 	fctx = nctx->u.fctx;
70 	flags.f = be64_to_cpu(fctx->flags.f);
71 	flags.w0.aes_keylen = aes_keylen;
72 	fctx->flags.f = cpu_to_be64(flags.f);
73 
74 	/* copy enc key to context */
75 	memset(&fctx->crypto, 0, sizeof(fctx->crypto));
76 	memcpy(fctx->crypto.u.key, key, keylen);
77 
78 	return 0;
79 }
80 
81 static int nitrox_aead_setauthsize(struct crypto_aead *aead,
82 				   unsigned int authsize)
83 {
84 	struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
85 	struct flexi_crypto_context *fctx = nctx->u.fctx;
86 	union fc_ctx_flags flags;
87 
88 	flags.f = be64_to_cpu(fctx->flags.f);
89 	flags.w0.mac_len = authsize;
90 	fctx->flags.f = cpu_to_be64(flags.f);
91 
92 	aead->authsize = authsize;
93 
94 	return 0;
95 }
96 
97 static int alloc_src_sglist(struct aead_request *areq, char *iv, int ivsize,
98 			    int buflen)
99 {
100 	struct nitrox_kcrypt_request *nkreq = aead_request_ctx(areq);
101 	int nents = sg_nents_for_len(areq->src, buflen) + 1;
102 	int ret;
103 
104 	if (nents < 0)
105 		return nents;
106 
107 	/* Allocate buffer to hold IV and input scatterlist array */
108 	ret = alloc_src_req_buf(nkreq, nents, ivsize);
109 	if (ret)
110 		return ret;
111 
112 	nitrox_creq_copy_iv(nkreq->src, iv, ivsize);
113 	nitrox_creq_set_src_sg(nkreq, nents, ivsize, areq->src, buflen);
114 
115 	return 0;
116 }
117 
118 static int alloc_dst_sglist(struct aead_request *areq, int ivsize, int buflen)
119 {
120 	struct nitrox_kcrypt_request *nkreq = aead_request_ctx(areq);
121 	int nents = sg_nents_for_len(areq->dst, buflen) + 3;
122 	int ret;
123 
124 	if (nents < 0)
125 		return nents;
126 
127 	/* Allocate buffer to hold ORH, COMPLETION and output scatterlist
128 	 * array
129 	 */
130 	ret = alloc_dst_req_buf(nkreq, nents);
131 	if (ret)
132 		return ret;
133 
134 	nitrox_creq_set_orh(nkreq);
135 	nitrox_creq_set_comp(nkreq);
136 	nitrox_creq_set_dst_sg(nkreq, nents, ivsize, areq->dst, buflen);
137 
138 	return 0;
139 }
140 
141 static void free_src_sglist(struct aead_request *areq)
142 {
143 	struct nitrox_kcrypt_request *nkreq = aead_request_ctx(areq);
144 
145 	kfree(nkreq->src);
146 }
147 
148 static void free_dst_sglist(struct aead_request *areq)
149 {
150 	struct nitrox_kcrypt_request *nkreq = aead_request_ctx(areq);
151 
152 	kfree(nkreq->dst);
153 }
154 
155 static int nitrox_set_creq(struct aead_request *areq,
156 			   struct nitrox_crypt_params *params)
157 {
158 	struct nitrox_kcrypt_request *nkreq = aead_request_ctx(areq);
159 	struct se_crypto_request *creq = &nkreq->creq;
160 	struct crypto_aead *aead = crypto_aead_reqtfm(areq);
161 	union gph_p3 param3;
162 	struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
163 	int ret;
164 
165 	creq->flags = areq->base.flags;
166 	creq->gfp = (areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
167 		GFP_KERNEL : GFP_ATOMIC;
168 
169 	creq->ctrl.value = 0;
170 	creq->opcode = FLEXI_CRYPTO_ENCRYPT_HMAC;
171 	creq->ctrl.s.arg = params->ctrl_arg;
172 
173 	creq->gph.param0 = cpu_to_be16(params->cryptlen);
174 	creq->gph.param1 = cpu_to_be16(params->authlen);
175 	creq->gph.param2 = cpu_to_be16(params->ivsize + areq->assoclen);
176 	param3.iv_offset = 0;
177 	param3.auth_offset = params->ivsize;
178 	creq->gph.param3 = cpu_to_be16(param3.param);
179 
180 	creq->ctx_handle = nctx->u.ctx_handle;
181 	creq->ctrl.s.ctxl = sizeof(struct flexi_crypto_context);
182 
183 	ret = alloc_src_sglist(areq, params->iv, params->ivsize,
184 			       params->srclen);
185 	if (ret)
186 		return ret;
187 
188 	ret = alloc_dst_sglist(areq, params->ivsize, params->dstlen);
189 	if (ret) {
190 		free_src_sglist(areq);
191 		return ret;
192 	}
193 
194 	return 0;
195 }
196 
197 static void nitrox_aead_callback(void *arg, int err)
198 {
199 	struct aead_request *areq = arg;
200 
201 	free_src_sglist(areq);
202 	free_dst_sglist(areq);
203 	if (err) {
204 		pr_err_ratelimited("request failed status 0x%0x\n", err);
205 		err = -EINVAL;
206 	}
207 
208 	areq->base.complete(&areq->base, err);
209 }
210 
211 static int nitrox_aes_gcm_enc(struct aead_request *areq)
212 {
213 	struct crypto_aead *aead = crypto_aead_reqtfm(areq);
214 	struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
215 	struct nitrox_kcrypt_request *nkreq = aead_request_ctx(areq);
216 	struct se_crypto_request *creq = &nkreq->creq;
217 	struct flexi_crypto_context *fctx = nctx->u.fctx;
218 	struct nitrox_crypt_params params;
219 	int ret;
220 
221 	memcpy(fctx->crypto.iv, areq->iv, GCM_AES_SALT_SIZE);
222 
223 	memset(&params, 0, sizeof(params));
224 	params.cryptlen = areq->cryptlen;
225 	params.authlen = areq->assoclen + params.cryptlen;
226 	params.srclen = params.authlen;
227 	params.dstlen = params.srclen + aead->authsize;
228 	params.iv = &areq->iv[GCM_AES_SALT_SIZE];
229 	params.ivsize = GCM_AES_IV_SIZE - GCM_AES_SALT_SIZE;
230 	params.ctrl_arg = ENCRYPT;
231 	ret = nitrox_set_creq(areq, &params);
232 	if (ret)
233 		return ret;
234 
235 	/* send the crypto request */
236 	return nitrox_process_se_request(nctx->ndev, creq, nitrox_aead_callback,
237 					 areq);
238 }
239 
240 static int nitrox_aes_gcm_dec(struct aead_request *areq)
241 {
242 	struct crypto_aead *aead = crypto_aead_reqtfm(areq);
243 	struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
244 	struct nitrox_kcrypt_request *nkreq = aead_request_ctx(areq);
245 	struct se_crypto_request *creq = &nkreq->creq;
246 	struct flexi_crypto_context *fctx = nctx->u.fctx;
247 	struct nitrox_crypt_params params;
248 	int ret;
249 
250 	memcpy(fctx->crypto.iv, areq->iv, GCM_AES_SALT_SIZE);
251 
252 	memset(&params, 0, sizeof(params));
253 	params.cryptlen = areq->cryptlen - aead->authsize;
254 	params.authlen = areq->assoclen + params.cryptlen;
255 	params.srclen = areq->cryptlen + areq->assoclen;
256 	params.dstlen = params.srclen - aead->authsize;
257 	params.iv = &areq->iv[GCM_AES_SALT_SIZE];
258 	params.ivsize = GCM_AES_IV_SIZE - GCM_AES_SALT_SIZE;
259 	params.ctrl_arg = DECRYPT;
260 	ret = nitrox_set_creq(areq, &params);
261 	if (ret)
262 		return ret;
263 
264 	/* send the crypto request */
265 	return nitrox_process_se_request(nctx->ndev, creq, nitrox_aead_callback,
266 					 areq);
267 }
268 
269 static int nitrox_aead_init(struct crypto_aead *aead)
270 {
271 	struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
272 	struct crypto_ctx_hdr *chdr;
273 
274 	/* get the first device */
275 	nctx->ndev = nitrox_get_first_device();
276 	if (!nctx->ndev)
277 		return -ENODEV;
278 
279 	/* allocate nitrox crypto context */
280 	chdr = crypto_alloc_context(nctx->ndev);
281 	if (!chdr) {
282 		nitrox_put_device(nctx->ndev);
283 		return -ENOMEM;
284 	}
285 	nctx->chdr = chdr;
286 	nctx->u.ctx_handle = (uintptr_t)((u8 *)chdr->vaddr +
287 					 sizeof(struct ctx_hdr));
288 	nctx->u.fctx->flags.f = 0;
289 
290 	return 0;
291 }
292 
293 static int nitrox_aes_gcm_init(struct crypto_aead *aead)
294 {
295 	int ret;
296 	struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
297 	union fc_ctx_flags *flags;
298 
299 	ret = nitrox_aead_init(aead);
300 	if (ret)
301 		return ret;
302 
303 	flags = &nctx->u.fctx->flags;
304 	flags->w0.cipher_type = CIPHER_AES_GCM;
305 	flags->w0.hash_type = AUTH_NULL;
306 	flags->w0.iv_source = IV_FROM_DPTR;
307 	/* ask microcode to calculate ipad/opad */
308 	flags->w0.auth_input_type = 1;
309 	flags->f = be64_to_cpu(flags->f);
310 
311 	crypto_aead_set_reqsize(aead, sizeof(struct aead_request) +
312 				sizeof(struct nitrox_kcrypt_request));
313 
314 	return 0;
315 }
316 
317 static void nitrox_aead_exit(struct crypto_aead *aead)
318 {
319 	struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
320 
321 	/* free the nitrox crypto context */
322 	if (nctx->u.ctx_handle) {
323 		struct flexi_crypto_context *fctx = nctx->u.fctx;
324 
325 		memzero_explicit(&fctx->crypto, sizeof(struct crypto_keys));
326 		memzero_explicit(&fctx->auth, sizeof(struct auth_keys));
327 		crypto_free_context((void *)nctx->chdr);
328 	}
329 	nitrox_put_device(nctx->ndev);
330 
331 	nctx->u.ctx_handle = 0;
332 	nctx->ndev = NULL;
333 }
334 
335 static struct aead_alg nitrox_aeads[] = { {
336 	.base = {
337 		.cra_name = "gcm(aes)",
338 		.cra_driver_name = "n5_aes_gcm",
339 		.cra_priority = PRIO,
340 		.cra_flags = CRYPTO_ALG_ASYNC,
341 		.cra_blocksize = AES_BLOCK_SIZE,
342 		.cra_ctxsize = sizeof(struct nitrox_crypto_ctx),
343 		.cra_alignmask = 0,
344 		.cra_module = THIS_MODULE,
345 	},
346 	.setkey = nitrox_aes_gcm_setkey,
347 	.setauthsize = nitrox_aead_setauthsize,
348 	.encrypt = nitrox_aes_gcm_enc,
349 	.decrypt = nitrox_aes_gcm_dec,
350 	.init = nitrox_aes_gcm_init,
351 	.exit = nitrox_aead_exit,
352 	.ivsize = GCM_AES_IV_SIZE,
353 	.maxauthsize = AES_BLOCK_SIZE,
354 } };
355 
356 int nitrox_register_aeads(void)
357 {
358 	return crypto_register_aeads(nitrox_aeads, ARRAY_SIZE(nitrox_aeads));
359 }
360 
361 void nitrox_unregister_aeads(void)
362 {
363 	crypto_unregister_aeads(nitrox_aeads, ARRAY_SIZE(nitrox_aeads));
364 }
365