xref: /openbmc/linux/lib/crypto/poly1305-donna64.c (revision 224d80c5)
1 // SPDX-License-Identifier: GPL-2.0 OR MIT
2 /*
3  * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
4  *
5  * This is based in part on Andrew Moon's poly1305-donna, which is in the
6  * public domain.
7  */
8 
9 #include <linux/kernel.h>
10 #include <asm/unaligned.h>
11 #include <crypto/internal/poly1305.h>
12 
poly1305_core_setkey(struct poly1305_core_key * key,const u8 raw_key[POLY1305_BLOCK_SIZE])13 void poly1305_core_setkey(struct poly1305_core_key *key,
14 			  const u8 raw_key[POLY1305_BLOCK_SIZE])
15 {
16 	u64 t0, t1;
17 
18 	/* r &= 0xffffffc0ffffffc0ffffffc0fffffff */
19 	t0 = get_unaligned_le64(&raw_key[0]);
20 	t1 = get_unaligned_le64(&raw_key[8]);
21 
22 	key->key.r64[0] = t0 & 0xffc0fffffffULL;
23 	key->key.r64[1] = ((t0 >> 44) | (t1 << 20)) & 0xfffffc0ffffULL;
24 	key->key.r64[2] = ((t1 >> 24)) & 0x00ffffffc0fULL;
25 
26 	/* s = 20*r */
27 	key->precomputed_s.r64[0] = key->key.r64[1] * 20;
28 	key->precomputed_s.r64[1] = key->key.r64[2] * 20;
29 }
30 EXPORT_SYMBOL(poly1305_core_setkey);
31 
poly1305_core_blocks(struct poly1305_state * state,const struct poly1305_core_key * key,const void * src,unsigned int nblocks,u32 hibit)32 void poly1305_core_blocks(struct poly1305_state *state,
33 			  const struct poly1305_core_key *key, const void *src,
34 			  unsigned int nblocks, u32 hibit)
35 {
36 	const u8 *input = src;
37 	u64 hibit64;
38 	u64 r0, r1, r2;
39 	u64 s1, s2;
40 	u64 h0, h1, h2;
41 	u64 c;
42 	u128 d0, d1, d2, d;
43 
44 	if (!nblocks)
45 		return;
46 
47 	hibit64 = ((u64)hibit) << 40;
48 
49 	r0 = key->key.r64[0];
50 	r1 = key->key.r64[1];
51 	r2 = key->key.r64[2];
52 
53 	h0 = state->h64[0];
54 	h1 = state->h64[1];
55 	h2 = state->h64[2];
56 
57 	s1 = key->precomputed_s.r64[0];
58 	s2 = key->precomputed_s.r64[1];
59 
60 	do {
61 		u64 t0, t1;
62 
63 		/* h += m[i] */
64 		t0 = get_unaligned_le64(&input[0]);
65 		t1 = get_unaligned_le64(&input[8]);
66 
67 		h0 += t0 & 0xfffffffffffULL;
68 		h1 += ((t0 >> 44) | (t1 << 20)) & 0xfffffffffffULL;
69 		h2 += (((t1 >> 24)) & 0x3ffffffffffULL) | hibit64;
70 
71 		/* h *= r */
72 		d0 = (u128)h0 * r0;
73 		d = (u128)h1 * s2;
74 		d0 += d;
75 		d = (u128)h2 * s1;
76 		d0 += d;
77 		d1 = (u128)h0 * r1;
78 		d = (u128)h1 * r0;
79 		d1 += d;
80 		d = (u128)h2 * s2;
81 		d1 += d;
82 		d2 = (u128)h0 * r2;
83 		d = (u128)h1 * r1;
84 		d2 += d;
85 		d = (u128)h2 * r0;
86 		d2 += d;
87 
88 		/* (partial) h %= p */
89 		c = (u64)(d0 >> 44);
90 		h0 = (u64)d0 & 0xfffffffffffULL;
91 		d1 += c;
92 		c = (u64)(d1 >> 44);
93 		h1 = (u64)d1 & 0xfffffffffffULL;
94 		d2 += c;
95 		c = (u64)(d2 >> 42);
96 		h2 = (u64)d2 & 0x3ffffffffffULL;
97 		h0 += c * 5;
98 		c = h0 >> 44;
99 		h0 = h0 & 0xfffffffffffULL;
100 		h1 += c;
101 
102 		input += POLY1305_BLOCK_SIZE;
103 	} while (--nblocks);
104 
105 	state->h64[0] = h0;
106 	state->h64[1] = h1;
107 	state->h64[2] = h2;
108 }
109 EXPORT_SYMBOL(poly1305_core_blocks);
110 
poly1305_core_emit(const struct poly1305_state * state,const u32 nonce[4],void * dst)111 void poly1305_core_emit(const struct poly1305_state *state, const u32 nonce[4],
112 			void *dst)
113 {
114 	u8 *mac = dst;
115 	u64 h0, h1, h2, c;
116 	u64 g0, g1, g2;
117 	u64 t0, t1;
118 
119 	/* fully carry h */
120 	h0 = state->h64[0];
121 	h1 = state->h64[1];
122 	h2 = state->h64[2];
123 
124 	c = h1 >> 44;
125 	h1 &= 0xfffffffffffULL;
126 	h2 += c;
127 	c = h2 >> 42;
128 	h2 &= 0x3ffffffffffULL;
129 	h0 += c * 5;
130 	c = h0 >> 44;
131 	h0 &= 0xfffffffffffULL;
132 	h1 += c;
133 	c = h1 >> 44;
134 	h1 &= 0xfffffffffffULL;
135 	h2 += c;
136 	c = h2 >> 42;
137 	h2 &= 0x3ffffffffffULL;
138 	h0 += c * 5;
139 	c = h0 >> 44;
140 	h0 &= 0xfffffffffffULL;
141 	h1 += c;
142 
143 	/* compute h + -p */
144 	g0 = h0 + 5;
145 	c  = g0 >> 44;
146 	g0 &= 0xfffffffffffULL;
147 	g1 = h1 + c;
148 	c  = g1 >> 44;
149 	g1 &= 0xfffffffffffULL;
150 	g2 = h2 + c - (1ULL << 42);
151 
152 	/* select h if h < p, or h + -p if h >= p */
153 	c = (g2 >> ((sizeof(u64) * 8) - 1)) - 1;
154 	g0 &= c;
155 	g1 &= c;
156 	g2 &= c;
157 	c  = ~c;
158 	h0 = (h0 & c) | g0;
159 	h1 = (h1 & c) | g1;
160 	h2 = (h2 & c) | g2;
161 
162 	if (likely(nonce)) {
163 		/* h = (h + nonce) */
164 		t0 = ((u64)nonce[1] << 32) | nonce[0];
165 		t1 = ((u64)nonce[3] << 32) | nonce[2];
166 
167 		h0 += t0 & 0xfffffffffffULL;
168 		c = h0 >> 44;
169 		h0 &= 0xfffffffffffULL;
170 		h1 += (((t0 >> 44) | (t1 << 20)) & 0xfffffffffffULL) + c;
171 		c = h1 >> 44;
172 		h1 &= 0xfffffffffffULL;
173 		h2 += (((t1 >> 24)) & 0x3ffffffffffULL) + c;
174 		h2 &= 0x3ffffffffffULL;
175 	}
176 
177 	/* mac = h % (2^128) */
178 	h0 = h0 | (h1 << 44);
179 	h1 = (h1 >> 20) | (h2 << 24);
180 
181 	put_unaligned_le64(h0, &mac[0]);
182 	put_unaligned_le64(h1, &mac[8]);
183 }
184 EXPORT_SYMBOL(poly1305_core_emit);
185