1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Accelerated GHASH implementation with ARMv8 PMULL instructions.
4  *
5  * Copyright (C) 2014 - 2018 Linaro Ltd. <ard.biesheuvel@linaro.org>
6  */
7 
8 #include <asm/neon.h>
9 #include <asm/simd.h>
10 #include <asm/unaligned.h>
11 #include <crypto/aes.h>
12 #include <crypto/algapi.h>
13 #include <crypto/b128ops.h>
14 #include <crypto/gf128mul.h>
15 #include <crypto/internal/aead.h>
16 #include <crypto/internal/hash.h>
17 #include <crypto/internal/simd.h>
18 #include <crypto/internal/skcipher.h>
19 #include <crypto/scatterwalk.h>
20 #include <linux/cpufeature.h>
21 #include <linux/crypto.h>
22 #include <linux/module.h>
23 
24 MODULE_DESCRIPTION("GHASH and AES-GCM using ARMv8 Crypto Extensions");
25 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
26 MODULE_LICENSE("GPL v2");
27 MODULE_ALIAS_CRYPTO("ghash");
28 
29 #define GHASH_BLOCK_SIZE	16
30 #define GHASH_DIGEST_SIZE	16
31 #define GCM_IV_SIZE		12
32 
33 struct ghash_key {
34 	u64			h[2];
35 	u64			h2[2];
36 	u64			h3[2];
37 	u64			h4[2];
38 
39 	be128			k;
40 };
41 
42 struct ghash_desc_ctx {
43 	u64 digest[GHASH_DIGEST_SIZE/sizeof(u64)];
44 	u8 buf[GHASH_BLOCK_SIZE];
45 	u32 count;
46 };
47 
48 struct gcm_aes_ctx {
49 	struct crypto_aes_ctx	aes_key;
50 	struct ghash_key	ghash_key;
51 };
52 
53 asmlinkage void pmull_ghash_update_p64(int blocks, u64 dg[], const char *src,
54 				       struct ghash_key const *k,
55 				       const char *head);
56 
57 asmlinkage void pmull_ghash_update_p8(int blocks, u64 dg[], const char *src,
58 				      struct ghash_key const *k,
59 				      const char *head);
60 
61 asmlinkage void pmull_gcm_encrypt(int bytes, u8 dst[], const u8 src[],
62 				  struct ghash_key const *k, u64 dg[],
63 				  u8 ctr[], u32 const rk[], int rounds,
64 				  u8 tag[]);
65 
66 asmlinkage void pmull_gcm_decrypt(int bytes, u8 dst[], const u8 src[],
67 				  struct ghash_key const *k, u64 dg[],
68 				  u8 ctr[], u32 const rk[], int rounds,
69 				  u8 tag[]);
70 
71 static int ghash_init(struct shash_desc *desc)
72 {
73 	struct ghash_desc_ctx *ctx = shash_desc_ctx(desc);
74 
75 	*ctx = (struct ghash_desc_ctx){};
76 	return 0;
77 }
78 
79 static void ghash_do_update(int blocks, u64 dg[], const char *src,
80 			    struct ghash_key *key, const char *head,
81 			    void (*simd_update)(int blocks, u64 dg[],
82 						const char *src,
83 						struct ghash_key const *k,
84 						const char *head))
85 {
86 	if (likely(crypto_simd_usable() && simd_update)) {
87 		kernel_neon_begin();
88 		simd_update(blocks, dg, src, key, head);
89 		kernel_neon_end();
90 	} else {
91 		be128 dst = { cpu_to_be64(dg[1]), cpu_to_be64(dg[0]) };
92 
93 		do {
94 			const u8 *in = src;
95 
96 			if (head) {
97 				in = head;
98 				blocks++;
99 				head = NULL;
100 			} else {
101 				src += GHASH_BLOCK_SIZE;
102 			}
103 
104 			crypto_xor((u8 *)&dst, in, GHASH_BLOCK_SIZE);
105 			gf128mul_lle(&dst, &key->k);
106 		} while (--blocks);
107 
108 		dg[0] = be64_to_cpu(dst.b);
109 		dg[1] = be64_to_cpu(dst.a);
110 	}
111 }
112 
113 /* avoid hogging the CPU for too long */
114 #define MAX_BLOCKS	(SZ_64K / GHASH_BLOCK_SIZE)
115 
116 static int __ghash_update(struct shash_desc *desc, const u8 *src,
117 			  unsigned int len,
118 			  void (*simd_update)(int blocks, u64 dg[],
119 					      const char *src,
120 					      struct ghash_key const *k,
121 					      const char *head))
122 {
123 	struct ghash_desc_ctx *ctx = shash_desc_ctx(desc);
124 	unsigned int partial = ctx->count % GHASH_BLOCK_SIZE;
125 
126 	ctx->count += len;
127 
128 	if ((partial + len) >= GHASH_BLOCK_SIZE) {
129 		struct ghash_key *key = crypto_shash_ctx(desc->tfm);
130 		int blocks;
131 
132 		if (partial) {
133 			int p = GHASH_BLOCK_SIZE - partial;
134 
135 			memcpy(ctx->buf + partial, src, p);
136 			src += p;
137 			len -= p;
138 		}
139 
140 		blocks = len / GHASH_BLOCK_SIZE;
141 		len %= GHASH_BLOCK_SIZE;
142 
143 		do {
144 			int chunk = min(blocks, MAX_BLOCKS);
145 
146 			ghash_do_update(chunk, ctx->digest, src, key,
147 					partial ? ctx->buf : NULL,
148 					simd_update);
149 
150 			blocks -= chunk;
151 			src += chunk * GHASH_BLOCK_SIZE;
152 			partial = 0;
153 		} while (unlikely(blocks > 0));
154 	}
155 	if (len)
156 		memcpy(ctx->buf + partial, src, len);
157 	return 0;
158 }
159 
160 static int ghash_update_p8(struct shash_desc *desc, const u8 *src,
161 			   unsigned int len)
162 {
163 	return __ghash_update(desc, src, len, pmull_ghash_update_p8);
164 }
165 
166 static int ghash_update_p64(struct shash_desc *desc, const u8 *src,
167 			    unsigned int len)
168 {
169 	return __ghash_update(desc, src, len, pmull_ghash_update_p64);
170 }
171 
172 static int ghash_final_p8(struct shash_desc *desc, u8 *dst)
173 {
174 	struct ghash_desc_ctx *ctx = shash_desc_ctx(desc);
175 	unsigned int partial = ctx->count % GHASH_BLOCK_SIZE;
176 
177 	if (partial) {
178 		struct ghash_key *key = crypto_shash_ctx(desc->tfm);
179 
180 		memset(ctx->buf + partial, 0, GHASH_BLOCK_SIZE - partial);
181 
182 		ghash_do_update(1, ctx->digest, ctx->buf, key, NULL,
183 				pmull_ghash_update_p8);
184 	}
185 	put_unaligned_be64(ctx->digest[1], dst);
186 	put_unaligned_be64(ctx->digest[0], dst + 8);
187 
188 	*ctx = (struct ghash_desc_ctx){};
189 	return 0;
190 }
191 
192 static int ghash_final_p64(struct shash_desc *desc, u8 *dst)
193 {
194 	struct ghash_desc_ctx *ctx = shash_desc_ctx(desc);
195 	unsigned int partial = ctx->count % GHASH_BLOCK_SIZE;
196 
197 	if (partial) {
198 		struct ghash_key *key = crypto_shash_ctx(desc->tfm);
199 
200 		memset(ctx->buf + partial, 0, GHASH_BLOCK_SIZE - partial);
201 
202 		ghash_do_update(1, ctx->digest, ctx->buf, key, NULL,
203 				pmull_ghash_update_p64);
204 	}
205 	put_unaligned_be64(ctx->digest[1], dst);
206 	put_unaligned_be64(ctx->digest[0], dst + 8);
207 
208 	*ctx = (struct ghash_desc_ctx){};
209 	return 0;
210 }
211 
212 static void ghash_reflect(u64 h[], const be128 *k)
213 {
214 	u64 carry = be64_to_cpu(k->a) & BIT(63) ? 1 : 0;
215 
216 	h[0] = (be64_to_cpu(k->b) << 1) | carry;
217 	h[1] = (be64_to_cpu(k->a) << 1) | (be64_to_cpu(k->b) >> 63);
218 
219 	if (carry)
220 		h[1] ^= 0xc200000000000000UL;
221 }
222 
223 static int __ghash_setkey(struct ghash_key *key,
224 			  const u8 *inkey, unsigned int keylen)
225 {
226 	be128 h;
227 
228 	/* needed for the fallback */
229 	memcpy(&key->k, inkey, GHASH_BLOCK_SIZE);
230 
231 	ghash_reflect(key->h, &key->k);
232 
233 	h = key->k;
234 	gf128mul_lle(&h, &key->k);
235 	ghash_reflect(key->h2, &h);
236 
237 	gf128mul_lle(&h, &key->k);
238 	ghash_reflect(key->h3, &h);
239 
240 	gf128mul_lle(&h, &key->k);
241 	ghash_reflect(key->h4, &h);
242 
243 	return 0;
244 }
245 
246 static int ghash_setkey(struct crypto_shash *tfm,
247 			const u8 *inkey, unsigned int keylen)
248 {
249 	struct ghash_key *key = crypto_shash_ctx(tfm);
250 
251 	if (keylen != GHASH_BLOCK_SIZE)
252 		return -EINVAL;
253 
254 	return __ghash_setkey(key, inkey, keylen);
255 }
256 
257 static struct shash_alg ghash_alg[] = {{
258 	.base.cra_name		= "ghash",
259 	.base.cra_driver_name	= "ghash-neon",
260 	.base.cra_priority	= 150,
261 	.base.cra_blocksize	= GHASH_BLOCK_SIZE,
262 	.base.cra_ctxsize	= sizeof(struct ghash_key),
263 	.base.cra_module	= THIS_MODULE,
264 
265 	.digestsize		= GHASH_DIGEST_SIZE,
266 	.init			= ghash_init,
267 	.update			= ghash_update_p8,
268 	.final			= ghash_final_p8,
269 	.setkey			= ghash_setkey,
270 	.descsize		= sizeof(struct ghash_desc_ctx),
271 }, {
272 	.base.cra_name		= "ghash",
273 	.base.cra_driver_name	= "ghash-ce",
274 	.base.cra_priority	= 200,
275 	.base.cra_blocksize	= GHASH_BLOCK_SIZE,
276 	.base.cra_ctxsize	= sizeof(struct ghash_key),
277 	.base.cra_module	= THIS_MODULE,
278 
279 	.digestsize		= GHASH_DIGEST_SIZE,
280 	.init			= ghash_init,
281 	.update			= ghash_update_p64,
282 	.final			= ghash_final_p64,
283 	.setkey			= ghash_setkey,
284 	.descsize		= sizeof(struct ghash_desc_ctx),
285 }};
286 
287 static int num_rounds(struct crypto_aes_ctx *ctx)
288 {
289 	/*
290 	 * # of rounds specified by AES:
291 	 * 128 bit key		10 rounds
292 	 * 192 bit key		12 rounds
293 	 * 256 bit key		14 rounds
294 	 * => n byte key	=> 6 + (n/4) rounds
295 	 */
296 	return 6 + ctx->key_length / 4;
297 }
298 
299 static int gcm_setkey(struct crypto_aead *tfm, const u8 *inkey,
300 		      unsigned int keylen)
301 {
302 	struct gcm_aes_ctx *ctx = crypto_aead_ctx(tfm);
303 	u8 key[GHASH_BLOCK_SIZE];
304 	int ret;
305 
306 	ret = aes_expandkey(&ctx->aes_key, inkey, keylen);
307 	if (ret)
308 		return -EINVAL;
309 
310 	aes_encrypt(&ctx->aes_key, key, (u8[AES_BLOCK_SIZE]){});
311 
312 	return __ghash_setkey(&ctx->ghash_key, key, sizeof(be128));
313 }
314 
315 static int gcm_setauthsize(struct crypto_aead *tfm, unsigned int authsize)
316 {
317 	switch (authsize) {
318 	case 4:
319 	case 8:
320 	case 12 ... 16:
321 		break;
322 	default:
323 		return -EINVAL;
324 	}
325 	return 0;
326 }
327 
328 static void gcm_update_mac(u64 dg[], const u8 *src, int count, u8 buf[],
329 			   int *buf_count, struct gcm_aes_ctx *ctx)
330 {
331 	if (*buf_count > 0) {
332 		int buf_added = min(count, GHASH_BLOCK_SIZE - *buf_count);
333 
334 		memcpy(&buf[*buf_count], src, buf_added);
335 
336 		*buf_count += buf_added;
337 		src += buf_added;
338 		count -= buf_added;
339 	}
340 
341 	if (count >= GHASH_BLOCK_SIZE || *buf_count == GHASH_BLOCK_SIZE) {
342 		int blocks = count / GHASH_BLOCK_SIZE;
343 
344 		ghash_do_update(blocks, dg, src, &ctx->ghash_key,
345 				*buf_count ? buf : NULL,
346 				pmull_ghash_update_p64);
347 
348 		src += blocks * GHASH_BLOCK_SIZE;
349 		count %= GHASH_BLOCK_SIZE;
350 		*buf_count = 0;
351 	}
352 
353 	if (count > 0) {
354 		memcpy(buf, src, count);
355 		*buf_count = count;
356 	}
357 }
358 
359 static void gcm_calculate_auth_mac(struct aead_request *req, u64 dg[])
360 {
361 	struct crypto_aead *aead = crypto_aead_reqtfm(req);
362 	struct gcm_aes_ctx *ctx = crypto_aead_ctx(aead);
363 	u8 buf[GHASH_BLOCK_SIZE];
364 	struct scatter_walk walk;
365 	u32 len = req->assoclen;
366 	int buf_count = 0;
367 
368 	scatterwalk_start(&walk, req->src);
369 
370 	do {
371 		u32 n = scatterwalk_clamp(&walk, len);
372 		u8 *p;
373 
374 		if (!n) {
375 			scatterwalk_start(&walk, sg_next(walk.sg));
376 			n = scatterwalk_clamp(&walk, len);
377 		}
378 		p = scatterwalk_map(&walk);
379 
380 		gcm_update_mac(dg, p, n, buf, &buf_count, ctx);
381 		len -= n;
382 
383 		scatterwalk_unmap(p);
384 		scatterwalk_advance(&walk, n);
385 		scatterwalk_done(&walk, 0, len);
386 	} while (len);
387 
388 	if (buf_count) {
389 		memset(&buf[buf_count], 0, GHASH_BLOCK_SIZE - buf_count);
390 		ghash_do_update(1, dg, buf, &ctx->ghash_key, NULL,
391 				pmull_ghash_update_p64);
392 	}
393 }
394 
395 static int gcm_encrypt(struct aead_request *req)
396 {
397 	struct crypto_aead *aead = crypto_aead_reqtfm(req);
398 	struct gcm_aes_ctx *ctx = crypto_aead_ctx(aead);
399 	int nrounds = num_rounds(&ctx->aes_key);
400 	struct skcipher_walk walk;
401 	u8 buf[AES_BLOCK_SIZE];
402 	u8 iv[AES_BLOCK_SIZE];
403 	u64 dg[2] = {};
404 	u128 lengths;
405 	u8 *tag;
406 	int err;
407 
408 	lengths.a = cpu_to_be64(req->assoclen * 8);
409 	lengths.b = cpu_to_be64(req->cryptlen * 8);
410 
411 	if (req->assoclen)
412 		gcm_calculate_auth_mac(req, dg);
413 
414 	memcpy(iv, req->iv, GCM_IV_SIZE);
415 	put_unaligned_be32(2, iv + GCM_IV_SIZE);
416 
417 	err = skcipher_walk_aead_encrypt(&walk, req, false);
418 
419 	if (likely(crypto_simd_usable())) {
420 		do {
421 			const u8 *src = walk.src.virt.addr;
422 			u8 *dst = walk.dst.virt.addr;
423 			int nbytes = walk.nbytes;
424 
425 			tag = (u8 *)&lengths;
426 
427 			if (unlikely(nbytes > 0 && nbytes < AES_BLOCK_SIZE)) {
428 				src = dst = memcpy(buf + sizeof(buf) - nbytes,
429 						   src, nbytes);
430 			} else if (nbytes < walk.total) {
431 				nbytes &= ~(AES_BLOCK_SIZE - 1);
432 				tag = NULL;
433 			}
434 
435 			kernel_neon_begin();
436 			pmull_gcm_encrypt(nbytes, dst, src, &ctx->ghash_key, dg,
437 					  iv, ctx->aes_key.key_enc, nrounds,
438 					  tag);
439 			kernel_neon_end();
440 
441 			if (unlikely(!nbytes))
442 				break;
443 
444 			if (unlikely(nbytes > 0 && nbytes < AES_BLOCK_SIZE))
445 				memcpy(walk.dst.virt.addr,
446 				       buf + sizeof(buf) - nbytes, nbytes);
447 
448 			err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
449 		} while (walk.nbytes);
450 	} else {
451 		while (walk.nbytes >= AES_BLOCK_SIZE) {
452 			int blocks = walk.nbytes / AES_BLOCK_SIZE;
453 			const u8 *src = walk.src.virt.addr;
454 			u8 *dst = walk.dst.virt.addr;
455 			int remaining = blocks;
456 
457 			do {
458 				aes_encrypt(&ctx->aes_key, buf, iv);
459 				crypto_xor_cpy(dst, src, buf, AES_BLOCK_SIZE);
460 				crypto_inc(iv, AES_BLOCK_SIZE);
461 
462 				dst += AES_BLOCK_SIZE;
463 				src += AES_BLOCK_SIZE;
464 			} while (--remaining > 0);
465 
466 			ghash_do_update(blocks, dg, walk.dst.virt.addr,
467 					&ctx->ghash_key, NULL, NULL);
468 
469 			err = skcipher_walk_done(&walk,
470 						 walk.nbytes % AES_BLOCK_SIZE);
471 		}
472 
473 		/* handle the tail */
474 		if (walk.nbytes) {
475 			aes_encrypt(&ctx->aes_key, buf, iv);
476 
477 			crypto_xor_cpy(walk.dst.virt.addr, walk.src.virt.addr,
478 				       buf, walk.nbytes);
479 
480 			memcpy(buf, walk.dst.virt.addr, walk.nbytes);
481 			memset(buf + walk.nbytes, 0, sizeof(buf) - walk.nbytes);
482 		}
483 
484 		tag = (u8 *)&lengths;
485 		ghash_do_update(1, dg, tag, &ctx->ghash_key,
486 				walk.nbytes ? buf : NULL, NULL);
487 
488 		if (walk.nbytes)
489 			err = skcipher_walk_done(&walk, 0);
490 
491 		put_unaligned_be64(dg[1], tag);
492 		put_unaligned_be64(dg[0], tag + 8);
493 		put_unaligned_be32(1, iv + GCM_IV_SIZE);
494 		aes_encrypt(&ctx->aes_key, iv, iv);
495 		crypto_xor(tag, iv, AES_BLOCK_SIZE);
496 	}
497 
498 	if (err)
499 		return err;
500 
501 	/* copy authtag to end of dst */
502 	scatterwalk_map_and_copy(tag, req->dst, req->assoclen + req->cryptlen,
503 				 crypto_aead_authsize(aead), 1);
504 
505 	return 0;
506 }
507 
508 static int gcm_decrypt(struct aead_request *req)
509 {
510 	struct crypto_aead *aead = crypto_aead_reqtfm(req);
511 	struct gcm_aes_ctx *ctx = crypto_aead_ctx(aead);
512 	unsigned int authsize = crypto_aead_authsize(aead);
513 	int nrounds = num_rounds(&ctx->aes_key);
514 	struct skcipher_walk walk;
515 	u8 buf[AES_BLOCK_SIZE];
516 	u8 iv[AES_BLOCK_SIZE];
517 	u64 dg[2] = {};
518 	u128 lengths;
519 	u8 *tag;
520 	int err;
521 
522 	lengths.a = cpu_to_be64(req->assoclen * 8);
523 	lengths.b = cpu_to_be64((req->cryptlen - authsize) * 8);
524 
525 	if (req->assoclen)
526 		gcm_calculate_auth_mac(req, dg);
527 
528 	memcpy(iv, req->iv, GCM_IV_SIZE);
529 	put_unaligned_be32(2, iv + GCM_IV_SIZE);
530 
531 	err = skcipher_walk_aead_decrypt(&walk, req, false);
532 
533 	if (likely(crypto_simd_usable())) {
534 		do {
535 			const u8 *src = walk.src.virt.addr;
536 			u8 *dst = walk.dst.virt.addr;
537 			int nbytes = walk.nbytes;
538 
539 			tag = (u8 *)&lengths;
540 
541 			if (unlikely(nbytes > 0 && nbytes < AES_BLOCK_SIZE)) {
542 				src = dst = memcpy(buf + sizeof(buf) - nbytes,
543 						   src, nbytes);
544 			} else if (nbytes < walk.total) {
545 				nbytes &= ~(AES_BLOCK_SIZE - 1);
546 				tag = NULL;
547 			}
548 
549 			kernel_neon_begin();
550 			pmull_gcm_decrypt(nbytes, dst, src, &ctx->ghash_key, dg,
551 					  iv, ctx->aes_key.key_enc, nrounds,
552 					  tag);
553 			kernel_neon_end();
554 
555 			if (unlikely(!nbytes))
556 				break;
557 
558 			if (unlikely(nbytes > 0 && nbytes < AES_BLOCK_SIZE))
559 				memcpy(walk.dst.virt.addr,
560 				       buf + sizeof(buf) - nbytes, nbytes);
561 
562 			err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
563 		} while (walk.nbytes);
564 	} else {
565 		while (walk.nbytes >= AES_BLOCK_SIZE) {
566 			int blocks = walk.nbytes / AES_BLOCK_SIZE;
567 			const u8 *src = walk.src.virt.addr;
568 			u8 *dst = walk.dst.virt.addr;
569 
570 			ghash_do_update(blocks, dg, walk.src.virt.addr,
571 					&ctx->ghash_key, NULL, NULL);
572 
573 			do {
574 				aes_encrypt(&ctx->aes_key, buf, iv);
575 				crypto_xor_cpy(dst, src, buf, AES_BLOCK_SIZE);
576 				crypto_inc(iv, AES_BLOCK_SIZE);
577 
578 				dst += AES_BLOCK_SIZE;
579 				src += AES_BLOCK_SIZE;
580 			} while (--blocks > 0);
581 
582 			err = skcipher_walk_done(&walk,
583 						 walk.nbytes % AES_BLOCK_SIZE);
584 		}
585 
586 		/* handle the tail */
587 		if (walk.nbytes) {
588 			memcpy(buf, walk.src.virt.addr, walk.nbytes);
589 			memset(buf + walk.nbytes, 0, sizeof(buf) - walk.nbytes);
590 		}
591 
592 		tag = (u8 *)&lengths;
593 		ghash_do_update(1, dg, tag, &ctx->ghash_key,
594 				walk.nbytes ? buf : NULL, NULL);
595 
596 		if (walk.nbytes) {
597 			aes_encrypt(&ctx->aes_key, buf, iv);
598 
599 			crypto_xor_cpy(walk.dst.virt.addr, walk.src.virt.addr,
600 				       buf, walk.nbytes);
601 
602 			err = skcipher_walk_done(&walk, 0);
603 		}
604 
605 		put_unaligned_be64(dg[1], tag);
606 		put_unaligned_be64(dg[0], tag + 8);
607 		put_unaligned_be32(1, iv + GCM_IV_SIZE);
608 		aes_encrypt(&ctx->aes_key, iv, iv);
609 		crypto_xor(tag, iv, AES_BLOCK_SIZE);
610 	}
611 
612 	if (err)
613 		return err;
614 
615 	/* compare calculated auth tag with the stored one */
616 	scatterwalk_map_and_copy(buf, req->src,
617 				 req->assoclen + req->cryptlen - authsize,
618 				 authsize, 0);
619 
620 	if (crypto_memneq(tag, buf, authsize))
621 		return -EBADMSG;
622 	return 0;
623 }
624 
625 static struct aead_alg gcm_aes_alg = {
626 	.ivsize			= GCM_IV_SIZE,
627 	.chunksize		= AES_BLOCK_SIZE,
628 	.maxauthsize		= AES_BLOCK_SIZE,
629 	.setkey			= gcm_setkey,
630 	.setauthsize		= gcm_setauthsize,
631 	.encrypt		= gcm_encrypt,
632 	.decrypt		= gcm_decrypt,
633 
634 	.base.cra_name		= "gcm(aes)",
635 	.base.cra_driver_name	= "gcm-aes-ce",
636 	.base.cra_priority	= 300,
637 	.base.cra_blocksize	= 1,
638 	.base.cra_ctxsize	= sizeof(struct gcm_aes_ctx),
639 	.base.cra_module	= THIS_MODULE,
640 };
641 
642 static int __init ghash_ce_mod_init(void)
643 {
644 	int ret;
645 
646 	if (!cpu_have_named_feature(ASIMD))
647 		return -ENODEV;
648 
649 	if (cpu_have_named_feature(PMULL))
650 		ret = crypto_register_shashes(ghash_alg,
651 					      ARRAY_SIZE(ghash_alg));
652 	else
653 		/* only register the first array element */
654 		ret = crypto_register_shash(ghash_alg);
655 
656 	if (ret)
657 		return ret;
658 
659 	if (cpu_have_named_feature(PMULL)) {
660 		ret = crypto_register_aead(&gcm_aes_alg);
661 		if (ret)
662 			crypto_unregister_shashes(ghash_alg,
663 						  ARRAY_SIZE(ghash_alg));
664 	}
665 	return ret;
666 }
667 
668 static void __exit ghash_ce_mod_exit(void)
669 {
670 	if (cpu_have_named_feature(PMULL))
671 		crypto_unregister_shashes(ghash_alg, ARRAY_SIZE(ghash_alg));
672 	else
673 		crypto_unregister_shash(ghash_alg);
674 	crypto_unregister_aead(&gcm_aes_alg);
675 }
676 
677 static const struct cpu_feature ghash_cpu_feature[] = {
678 	{ cpu_feature(PMULL) }, { }
679 };
680 MODULE_DEVICE_TABLE(cpu, ghash_cpu_feature);
681 
682 module_init(ghash_ce_mod_init);
683 module_exit(ghash_ce_mod_exit);
684