1 /*
2  * Support for Intel AES-NI instructions. This file contains glue
3  * code, the real AES implementation is in intel-aes_asm.S.
4  *
5  * Copyright (C) 2008, Intel Corp.
6  *    Author: Huang Ying <ying.huang@intel.com>
7  *
8  * This program is free software; you can redistribute it and/or modify
9  * it under the terms of the GNU General Public License as published by
10  * the Free Software Foundation; either version 2 of the License, or
11  * (at your option) any later version.
12  */
13 
14 #include <linux/hardirq.h>
15 #include <linux/types.h>
16 #include <linux/crypto.h>
17 #include <linux/err.h>
18 #include <crypto/algapi.h>
19 #include <crypto/aes.h>
20 #include <crypto/cryptd.h>
21 #include <asm/i387.h>
22 #include <asm/aes.h>
23 
24 struct async_aes_ctx {
25 	struct cryptd_ablkcipher *cryptd_tfm;
26 };
27 
28 #define AESNI_ALIGN	16
29 #define AES_BLOCK_MASK	(~(AES_BLOCK_SIZE-1))
30 
31 asmlinkage int aesni_set_key(struct crypto_aes_ctx *ctx, const u8 *in_key,
32 			     unsigned int key_len);
33 asmlinkage void aesni_enc(struct crypto_aes_ctx *ctx, u8 *out,
34 			  const u8 *in);
35 asmlinkage void aesni_dec(struct crypto_aes_ctx *ctx, u8 *out,
36 			  const u8 *in);
37 asmlinkage void aesni_ecb_enc(struct crypto_aes_ctx *ctx, u8 *out,
38 			      const u8 *in, unsigned int len);
39 asmlinkage void aesni_ecb_dec(struct crypto_aes_ctx *ctx, u8 *out,
40 			      const u8 *in, unsigned int len);
41 asmlinkage void aesni_cbc_enc(struct crypto_aes_ctx *ctx, u8 *out,
42 			      const u8 *in, unsigned int len, u8 *iv);
43 asmlinkage void aesni_cbc_dec(struct crypto_aes_ctx *ctx, u8 *out,
44 			      const u8 *in, unsigned int len, u8 *iv);
45 
46 static inline int kernel_fpu_using(void)
47 {
48 	if (in_interrupt() && !(read_cr0() & X86_CR0_TS))
49 		return 1;
50 	return 0;
51 }
52 
53 static inline struct crypto_aes_ctx *aes_ctx(void *raw_ctx)
54 {
55 	unsigned long addr = (unsigned long)raw_ctx;
56 	unsigned long align = AESNI_ALIGN;
57 
58 	if (align <= crypto_tfm_ctx_alignment())
59 		align = 1;
60 	return (struct crypto_aes_ctx *)ALIGN(addr, align);
61 }
62 
63 static int aes_set_key_common(struct crypto_tfm *tfm, void *raw_ctx,
64 			      const u8 *in_key, unsigned int key_len)
65 {
66 	struct crypto_aes_ctx *ctx = aes_ctx(raw_ctx);
67 	u32 *flags = &tfm->crt_flags;
68 	int err;
69 
70 	if (key_len != AES_KEYSIZE_128 && key_len != AES_KEYSIZE_192 &&
71 	    key_len != AES_KEYSIZE_256) {
72 		*flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
73 		return -EINVAL;
74 	}
75 
76 	if (kernel_fpu_using())
77 		err = crypto_aes_expand_key(ctx, in_key, key_len);
78 	else {
79 		kernel_fpu_begin();
80 		err = aesni_set_key(ctx, in_key, key_len);
81 		kernel_fpu_end();
82 	}
83 
84 	return err;
85 }
86 
87 static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key,
88 		       unsigned int key_len)
89 {
90 	return aes_set_key_common(tfm, crypto_tfm_ctx(tfm), in_key, key_len);
91 }
92 
93 static void aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
94 {
95 	struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
96 
97 	if (kernel_fpu_using())
98 		crypto_aes_encrypt_x86(ctx, dst, src);
99 	else {
100 		kernel_fpu_begin();
101 		aesni_enc(ctx, dst, src);
102 		kernel_fpu_end();
103 	}
104 }
105 
106 static void aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
107 {
108 	struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
109 
110 	if (kernel_fpu_using())
111 		crypto_aes_decrypt_x86(ctx, dst, src);
112 	else {
113 		kernel_fpu_begin();
114 		aesni_dec(ctx, dst, src);
115 		kernel_fpu_end();
116 	}
117 }
118 
119 static struct crypto_alg aesni_alg = {
120 	.cra_name		= "aes",
121 	.cra_driver_name	= "aes-aesni",
122 	.cra_priority		= 300,
123 	.cra_flags		= CRYPTO_ALG_TYPE_CIPHER,
124 	.cra_blocksize		= AES_BLOCK_SIZE,
125 	.cra_ctxsize		= sizeof(struct crypto_aes_ctx)+AESNI_ALIGN-1,
126 	.cra_alignmask		= 0,
127 	.cra_module		= THIS_MODULE,
128 	.cra_list		= LIST_HEAD_INIT(aesni_alg.cra_list),
129 	.cra_u	= {
130 		.cipher	= {
131 			.cia_min_keysize	= AES_MIN_KEY_SIZE,
132 			.cia_max_keysize	= AES_MAX_KEY_SIZE,
133 			.cia_setkey		= aes_set_key,
134 			.cia_encrypt		= aes_encrypt,
135 			.cia_decrypt		= aes_decrypt
136 		}
137 	}
138 };
139 
140 static int ecb_encrypt(struct blkcipher_desc *desc,
141 		       struct scatterlist *dst, struct scatterlist *src,
142 		       unsigned int nbytes)
143 {
144 	struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm));
145 	struct blkcipher_walk walk;
146 	int err;
147 
148 	blkcipher_walk_init(&walk, dst, src, nbytes);
149 	err = blkcipher_walk_virt(desc, &walk);
150 
151 	kernel_fpu_begin();
152 	while ((nbytes = walk.nbytes)) {
153 		aesni_ecb_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
154 			      nbytes & AES_BLOCK_MASK);
155 		nbytes &= AES_BLOCK_SIZE - 1;
156 		err = blkcipher_walk_done(desc, &walk, nbytes);
157 	}
158 	kernel_fpu_end();
159 
160 	return err;
161 }
162 
163 static int ecb_decrypt(struct blkcipher_desc *desc,
164 		       struct scatterlist *dst, struct scatterlist *src,
165 		       unsigned int nbytes)
166 {
167 	struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm));
168 	struct blkcipher_walk walk;
169 	int err;
170 
171 	blkcipher_walk_init(&walk, dst, src, nbytes);
172 	err = blkcipher_walk_virt(desc, &walk);
173 
174 	kernel_fpu_begin();
175 	while ((nbytes = walk.nbytes)) {
176 		aesni_ecb_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr,
177 			      nbytes & AES_BLOCK_MASK);
178 		nbytes &= AES_BLOCK_SIZE - 1;
179 		err = blkcipher_walk_done(desc, &walk, nbytes);
180 	}
181 	kernel_fpu_end();
182 
183 	return err;
184 }
185 
186 static struct crypto_alg blk_ecb_alg = {
187 	.cra_name		= "__ecb-aes-aesni",
188 	.cra_driver_name	= "__driver-ecb-aes-aesni",
189 	.cra_priority		= 0,
190 	.cra_flags		= CRYPTO_ALG_TYPE_BLKCIPHER,
191 	.cra_blocksize		= AES_BLOCK_SIZE,
192 	.cra_ctxsize		= sizeof(struct crypto_aes_ctx)+AESNI_ALIGN-1,
193 	.cra_alignmask		= 0,
194 	.cra_type		= &crypto_blkcipher_type,
195 	.cra_module		= THIS_MODULE,
196 	.cra_list		= LIST_HEAD_INIT(blk_ecb_alg.cra_list),
197 	.cra_u = {
198 		.blkcipher = {
199 			.min_keysize	= AES_MIN_KEY_SIZE,
200 			.max_keysize	= AES_MAX_KEY_SIZE,
201 			.setkey		= aes_set_key,
202 			.encrypt	= ecb_encrypt,
203 			.decrypt	= ecb_decrypt,
204 		},
205 	},
206 };
207 
208 static int cbc_encrypt(struct blkcipher_desc *desc,
209 		       struct scatterlist *dst, struct scatterlist *src,
210 		       unsigned int nbytes)
211 {
212 	struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm));
213 	struct blkcipher_walk walk;
214 	int err;
215 
216 	blkcipher_walk_init(&walk, dst, src, nbytes);
217 	err = blkcipher_walk_virt(desc, &walk);
218 
219 	kernel_fpu_begin();
220 	while ((nbytes = walk.nbytes)) {
221 		aesni_cbc_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
222 			      nbytes & AES_BLOCK_MASK, walk.iv);
223 		nbytes &= AES_BLOCK_SIZE - 1;
224 		err = blkcipher_walk_done(desc, &walk, nbytes);
225 	}
226 	kernel_fpu_end();
227 
228 	return err;
229 }
230 
231 static int cbc_decrypt(struct blkcipher_desc *desc,
232 		       struct scatterlist *dst, struct scatterlist *src,
233 		       unsigned int nbytes)
234 {
235 	struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm));
236 	struct blkcipher_walk walk;
237 	int err;
238 
239 	blkcipher_walk_init(&walk, dst, src, nbytes);
240 	err = blkcipher_walk_virt(desc, &walk);
241 
242 	kernel_fpu_begin();
243 	while ((nbytes = walk.nbytes)) {
244 		aesni_cbc_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr,
245 			      nbytes & AES_BLOCK_MASK, walk.iv);
246 		nbytes &= AES_BLOCK_SIZE - 1;
247 		err = blkcipher_walk_done(desc, &walk, nbytes);
248 	}
249 	kernel_fpu_end();
250 
251 	return err;
252 }
253 
254 static struct crypto_alg blk_cbc_alg = {
255 	.cra_name		= "__cbc-aes-aesni",
256 	.cra_driver_name	= "__driver-cbc-aes-aesni",
257 	.cra_priority		= 0,
258 	.cra_flags		= CRYPTO_ALG_TYPE_BLKCIPHER,
259 	.cra_blocksize		= AES_BLOCK_SIZE,
260 	.cra_ctxsize		= sizeof(struct crypto_aes_ctx)+AESNI_ALIGN-1,
261 	.cra_alignmask		= 0,
262 	.cra_type		= &crypto_blkcipher_type,
263 	.cra_module		= THIS_MODULE,
264 	.cra_list		= LIST_HEAD_INIT(blk_cbc_alg.cra_list),
265 	.cra_u = {
266 		.blkcipher = {
267 			.min_keysize	= AES_MIN_KEY_SIZE,
268 			.max_keysize	= AES_MAX_KEY_SIZE,
269 			.setkey		= aes_set_key,
270 			.encrypt	= cbc_encrypt,
271 			.decrypt	= cbc_decrypt,
272 		},
273 	},
274 };
275 
276 static int ablk_set_key(struct crypto_ablkcipher *tfm, const u8 *key,
277 			unsigned int key_len)
278 {
279 	struct async_aes_ctx *ctx = crypto_ablkcipher_ctx(tfm);
280 
281 	return crypto_ablkcipher_setkey(&ctx->cryptd_tfm->base, key, key_len);
282 }
283 
284 static int ablk_encrypt(struct ablkcipher_request *req)
285 {
286 	struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
287 	struct async_aes_ctx *ctx = crypto_ablkcipher_ctx(tfm);
288 
289 	if (kernel_fpu_using()) {
290 		struct ablkcipher_request *cryptd_req =
291 			ablkcipher_request_ctx(req);
292 		memcpy(cryptd_req, req, sizeof(*req));
293 		ablkcipher_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base);
294 		return crypto_ablkcipher_encrypt(cryptd_req);
295 	} else {
296 		struct blkcipher_desc desc;
297 		desc.tfm = cryptd_ablkcipher_child(ctx->cryptd_tfm);
298 		desc.info = req->info;
299 		desc.flags = 0;
300 		return crypto_blkcipher_crt(desc.tfm)->encrypt(
301 			&desc, req->dst, req->src, req->nbytes);
302 	}
303 }
304 
305 static int ablk_decrypt(struct ablkcipher_request *req)
306 {
307 	struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
308 	struct async_aes_ctx *ctx = crypto_ablkcipher_ctx(tfm);
309 
310 	if (kernel_fpu_using()) {
311 		struct ablkcipher_request *cryptd_req =
312 			ablkcipher_request_ctx(req);
313 		memcpy(cryptd_req, req, sizeof(*req));
314 		ablkcipher_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base);
315 		return crypto_ablkcipher_decrypt(cryptd_req);
316 	} else {
317 		struct blkcipher_desc desc;
318 		desc.tfm = cryptd_ablkcipher_child(ctx->cryptd_tfm);
319 		desc.info = req->info;
320 		desc.flags = 0;
321 		return crypto_blkcipher_crt(desc.tfm)->decrypt(
322 			&desc, req->dst, req->src, req->nbytes);
323 	}
324 }
325 
326 static void ablk_exit(struct crypto_tfm *tfm)
327 {
328 	struct async_aes_ctx *ctx = crypto_tfm_ctx(tfm);
329 
330 	cryptd_free_ablkcipher(ctx->cryptd_tfm);
331 }
332 
333 static void ablk_init_common(struct crypto_tfm *tfm,
334 			     struct cryptd_ablkcipher *cryptd_tfm)
335 {
336 	struct async_aes_ctx *ctx = crypto_tfm_ctx(tfm);
337 
338 	ctx->cryptd_tfm = cryptd_tfm;
339 	tfm->crt_ablkcipher.reqsize = sizeof(struct ablkcipher_request) +
340 		crypto_ablkcipher_reqsize(&cryptd_tfm->base);
341 }
342 
343 static int ablk_ecb_init(struct crypto_tfm *tfm)
344 {
345 	struct cryptd_ablkcipher *cryptd_tfm;
346 
347 	cryptd_tfm = cryptd_alloc_ablkcipher("__driver-ecb-aes-aesni", 0, 0);
348 	if (IS_ERR(cryptd_tfm))
349 		return PTR_ERR(cryptd_tfm);
350 	ablk_init_common(tfm, cryptd_tfm);
351 	return 0;
352 }
353 
354 static struct crypto_alg ablk_ecb_alg = {
355 	.cra_name		= "ecb(aes)",
356 	.cra_driver_name	= "ecb-aes-aesni",
357 	.cra_priority		= 400,
358 	.cra_flags		= CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
359 	.cra_blocksize		= AES_BLOCK_SIZE,
360 	.cra_ctxsize		= sizeof(struct async_aes_ctx),
361 	.cra_alignmask		= 0,
362 	.cra_type		= &crypto_ablkcipher_type,
363 	.cra_module		= THIS_MODULE,
364 	.cra_list		= LIST_HEAD_INIT(ablk_ecb_alg.cra_list),
365 	.cra_init		= ablk_ecb_init,
366 	.cra_exit		= ablk_exit,
367 	.cra_u = {
368 		.ablkcipher = {
369 			.min_keysize	= AES_MIN_KEY_SIZE,
370 			.max_keysize	= AES_MAX_KEY_SIZE,
371 			.setkey		= ablk_set_key,
372 			.encrypt	= ablk_encrypt,
373 			.decrypt	= ablk_decrypt,
374 		},
375 	},
376 };
377 
378 static int ablk_cbc_init(struct crypto_tfm *tfm)
379 {
380 	struct cryptd_ablkcipher *cryptd_tfm;
381 
382 	cryptd_tfm = cryptd_alloc_ablkcipher("__driver-cbc-aes-aesni", 0, 0);
383 	if (IS_ERR(cryptd_tfm))
384 		return PTR_ERR(cryptd_tfm);
385 	ablk_init_common(tfm, cryptd_tfm);
386 	return 0;
387 }
388 
389 static struct crypto_alg ablk_cbc_alg = {
390 	.cra_name		= "cbc(aes)",
391 	.cra_driver_name	= "cbc-aes-aesni",
392 	.cra_priority		= 400,
393 	.cra_flags		= CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
394 	.cra_blocksize		= AES_BLOCK_SIZE,
395 	.cra_ctxsize		= sizeof(struct async_aes_ctx),
396 	.cra_alignmask		= 0,
397 	.cra_type		= &crypto_ablkcipher_type,
398 	.cra_module		= THIS_MODULE,
399 	.cra_list		= LIST_HEAD_INIT(ablk_cbc_alg.cra_list),
400 	.cra_init		= ablk_cbc_init,
401 	.cra_exit		= ablk_exit,
402 	.cra_u = {
403 		.ablkcipher = {
404 			.min_keysize	= AES_MIN_KEY_SIZE,
405 			.max_keysize	= AES_MAX_KEY_SIZE,
406 			.ivsize		= AES_BLOCK_SIZE,
407 			.setkey		= ablk_set_key,
408 			.encrypt	= ablk_encrypt,
409 			.decrypt	= ablk_decrypt,
410 		},
411 	},
412 };
413 
414 static int __init aesni_init(void)
415 {
416 	int err;
417 
418 	if (!cpu_has_aes) {
419 		printk(KERN_ERR "Intel AES-NI instructions are not detected.\n");
420 		return -ENODEV;
421 	}
422 	if ((err = crypto_register_alg(&aesni_alg)))
423 		goto aes_err;
424 	if ((err = crypto_register_alg(&blk_ecb_alg)))
425 		goto blk_ecb_err;
426 	if ((err = crypto_register_alg(&blk_cbc_alg)))
427 		goto blk_cbc_err;
428 	if ((err = crypto_register_alg(&ablk_ecb_alg)))
429 		goto ablk_ecb_err;
430 	if ((err = crypto_register_alg(&ablk_cbc_alg)))
431 		goto ablk_cbc_err;
432 
433 	return err;
434 
435 ablk_cbc_err:
436 	crypto_unregister_alg(&ablk_ecb_alg);
437 ablk_ecb_err:
438 	crypto_unregister_alg(&blk_cbc_alg);
439 blk_cbc_err:
440 	crypto_unregister_alg(&blk_ecb_alg);
441 blk_ecb_err:
442 	crypto_unregister_alg(&aesni_alg);
443 aes_err:
444 	return err;
445 }
446 
447 static void __exit aesni_exit(void)
448 {
449 	crypto_unregister_alg(&ablk_cbc_alg);
450 	crypto_unregister_alg(&ablk_ecb_alg);
451 	crypto_unregister_alg(&blk_cbc_alg);
452 	crypto_unregister_alg(&blk_ecb_alg);
453 	crypto_unregister_alg(&aesni_alg);
454 }
455 
456 module_init(aesni_init);
457 module_exit(aesni_exit);
458 
459 MODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm, Intel AES-NI instructions optimized");
460 MODULE_LICENSE("GPL");
461 MODULE_ALIAS("aes");
462