1f1939f7cSShane Wang /*
2bb296481SEric Biggers * VMAC: Message Authentication Code using Universal Hashing
3bb296481SEric Biggers *
4bb296481SEric Biggers * Reference: https://tools.ietf.org/html/draft-krovetz-vmac-01
5bb296481SEric Biggers *
6f1939f7cSShane Wang * Copyright (c) 2009, Intel Corporation.
7bb296481SEric Biggers * Copyright (c) 2018, Google Inc.
8f1939f7cSShane Wang *
9f1939f7cSShane Wang * This program is free software; you can redistribute it and/or modify it
10f1939f7cSShane Wang * under the terms and conditions of the GNU General Public License,
11f1939f7cSShane Wang * version 2, as published by the Free Software Foundation.
12f1939f7cSShane Wang *
13f1939f7cSShane Wang * This program is distributed in the hope it will be useful, but WITHOUT
14f1939f7cSShane Wang * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
15f1939f7cSShane Wang * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
16f1939f7cSShane Wang * more details.
17f1939f7cSShane Wang *
18f1939f7cSShane Wang * You should have received a copy of the GNU General Public License along with
19f1939f7cSShane Wang * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
20f1939f7cSShane Wang * Place - Suite 330, Boston, MA 02111-1307 USA.
21f1939f7cSShane Wang */
22f1939f7cSShane Wang
23bb296481SEric Biggers /*
24bb296481SEric Biggers * Derived from:
25f1939f7cSShane Wang * VMAC and VHASH Implementation by Ted Krovetz (tdk@acm.org) and Wei Dai.
26f1939f7cSShane Wang * This implementation is herby placed in the public domain.
27f1939f7cSShane Wang * The authors offers no warranty. Use at your own risk.
28f1939f7cSShane Wang * Last modified: 17 APR 08, 1700 PDT
29bb296481SEric Biggers */
30f1939f7cSShane Wang
31bb296481SEric Biggers #include <asm/unaligned.h>
32f1939f7cSShane Wang #include <linux/init.h>
33f1939f7cSShane Wang #include <linux/types.h>
34f1939f7cSShane Wang #include <linux/crypto.h>
354bb33cc8SPaul Gortmaker #include <linux/module.h>
36f1939f7cSShane Wang #include <linux/scatterlist.h>
37f1939f7cSShane Wang #include <asm/byteorder.h>
38f1939f7cSShane Wang #include <crypto/scatterwalk.h>
39*0eb76ba2SArd Biesheuvel #include <crypto/internal/cipher.h>
40f1939f7cSShane Wang #include <crypto/internal/hash.h>
41f1939f7cSShane Wang
42f1939f7cSShane Wang /*
43bb296481SEric Biggers * User definable settings.
44bb296481SEric Biggers */
45bb296481SEric Biggers #define VMAC_TAG_LEN 64
46bb296481SEric Biggers #define VMAC_KEY_SIZE 128/* Must be 128, 192 or 256 */
47bb296481SEric Biggers #define VMAC_KEY_LEN (VMAC_KEY_SIZE/8)
48bb296481SEric Biggers #define VMAC_NHBYTES 128/* Must 2^i for any 3 < i < 13 Standard = 128*/
49ed331adaSEric Biggers #define VMAC_NONCEBYTES 16
50bb296481SEric Biggers
51bb296481SEric Biggers /* per-transform (per-key) context */
52bb296481SEric Biggers struct vmac_tfm_ctx {
53bb296481SEric Biggers struct crypto_cipher *cipher;
54bb296481SEric Biggers u64 nhkey[(VMAC_NHBYTES/8)+2*(VMAC_TAG_LEN/64-1)];
55bb296481SEric Biggers u64 polykey[2*VMAC_TAG_LEN/64];
56bb296481SEric Biggers u64 l3key[2*VMAC_TAG_LEN/64];
57bb296481SEric Biggers };
58bb296481SEric Biggers
59bb296481SEric Biggers /* per-request context */
60bb296481SEric Biggers struct vmac_desc_ctx {
61bb296481SEric Biggers union {
62bb296481SEric Biggers u8 partial[VMAC_NHBYTES]; /* partial block */
63bb296481SEric Biggers __le64 partial_words[VMAC_NHBYTES / 8];
64bb296481SEric Biggers };
65bb296481SEric Biggers unsigned int partial_size; /* size of the partial block */
66bb296481SEric Biggers bool first_block_processed;
67bb296481SEric Biggers u64 polytmp[2*VMAC_TAG_LEN/64]; /* running total of L2-hash */
68ed331adaSEric Biggers union {
69ed331adaSEric Biggers u8 bytes[VMAC_NONCEBYTES];
70ed331adaSEric Biggers __be64 pads[VMAC_NONCEBYTES / 8];
71ed331adaSEric Biggers } nonce;
72ed331adaSEric Biggers unsigned int nonce_size; /* nonce bytes filled so far */
73bb296481SEric Biggers };
74bb296481SEric Biggers
75bb296481SEric Biggers /*
76f1939f7cSShane Wang * Constants and masks
77f1939f7cSShane Wang */
78f1939f7cSShane Wang #define UINT64_C(x) x##ULL
7966ce0b0fSJussi Kivilinna static const u64 p64 = UINT64_C(0xfffffffffffffeff); /* 2^64 - 257 prime */
8066ce0b0fSJussi Kivilinna static const u64 m62 = UINT64_C(0x3fffffffffffffff); /* 62-bit mask */
8166ce0b0fSJussi Kivilinna static const u64 m63 = UINT64_C(0x7fffffffffffffff); /* 63-bit mask */
8266ce0b0fSJussi Kivilinna static const u64 m64 = UINT64_C(0xffffffffffffffff); /* 64-bit mask */
8366ce0b0fSJussi Kivilinna static const u64 mpoly = UINT64_C(0x1fffffff1fffffff); /* Poly key mask */
84f1939f7cSShane Wang
85304a204eSShane Wang #define pe64_to_cpup le64_to_cpup /* Prefer little endian */
86304a204eSShane Wang
87f1939f7cSShane Wang #ifdef __LITTLE_ENDIAN
88f1939f7cSShane Wang #define INDEX_HIGH 1
89f1939f7cSShane Wang #define INDEX_LOW 0
90f1939f7cSShane Wang #else
91f1939f7cSShane Wang #define INDEX_HIGH 0
92f1939f7cSShane Wang #define INDEX_LOW 1
93f1939f7cSShane Wang #endif
94f1939f7cSShane Wang
95f1939f7cSShane Wang /*
96f1939f7cSShane Wang * The following routines are used in this implementation. They are
97f1939f7cSShane Wang * written via macros to simulate zero-overhead call-by-reference.
98f1939f7cSShane Wang *
99f1939f7cSShane Wang * MUL64: 64x64->128-bit multiplication
100f1939f7cSShane Wang * PMUL64: assumes top bits cleared on inputs
101f1939f7cSShane Wang * ADD128: 128x128->128-bit addition
102f1939f7cSShane Wang */
103f1939f7cSShane Wang
104f1939f7cSShane Wang #define ADD128(rh, rl, ih, il) \
105f1939f7cSShane Wang do { \
106f1939f7cSShane Wang u64 _il = (il); \
107f1939f7cSShane Wang (rl) += (_il); \
108f1939f7cSShane Wang if ((rl) < (_il)) \
109f1939f7cSShane Wang (rh)++; \
110f1939f7cSShane Wang (rh) += (ih); \
111f1939f7cSShane Wang } while (0)
112f1939f7cSShane Wang
113f1939f7cSShane Wang #define MUL32(i1, i2) ((u64)(u32)(i1)*(u32)(i2))
114f1939f7cSShane Wang
115f1939f7cSShane Wang #define PMUL64(rh, rl, i1, i2) /* Assumes m doesn't overflow */ \
116f1939f7cSShane Wang do { \
117f1939f7cSShane Wang u64 _i1 = (i1), _i2 = (i2); \
118f1939f7cSShane Wang u64 m = MUL32(_i1, _i2>>32) + MUL32(_i1>>32, _i2); \
119f1939f7cSShane Wang rh = MUL32(_i1>>32, _i2>>32); \
120f1939f7cSShane Wang rl = MUL32(_i1, _i2); \
121f1939f7cSShane Wang ADD128(rh, rl, (m >> 32), (m << 32)); \
122f1939f7cSShane Wang } while (0)
123f1939f7cSShane Wang
124f1939f7cSShane Wang #define MUL64(rh, rl, i1, i2) \
125f1939f7cSShane Wang do { \
126f1939f7cSShane Wang u64 _i1 = (i1), _i2 = (i2); \
127f1939f7cSShane Wang u64 m1 = MUL32(_i1, _i2>>32); \
128f1939f7cSShane Wang u64 m2 = MUL32(_i1>>32, _i2); \
129f1939f7cSShane Wang rh = MUL32(_i1>>32, _i2>>32); \
130f1939f7cSShane Wang rl = MUL32(_i1, _i2); \
131f1939f7cSShane Wang ADD128(rh, rl, (m1 >> 32), (m1 << 32)); \
132f1939f7cSShane Wang ADD128(rh, rl, (m2 >> 32), (m2 << 32)); \
133f1939f7cSShane Wang } while (0)
134f1939f7cSShane Wang
135f1939f7cSShane Wang /*
136f1939f7cSShane Wang * For highest performance the L1 NH and L2 polynomial hashes should be
13725985edcSLucas De Marchi * carefully implemented to take advantage of one's target architecture.
138f1939f7cSShane Wang * Here these two hash functions are defined multiple time; once for
139f1939f7cSShane Wang * 64-bit architectures, once for 32-bit SSE2 architectures, and once
140f1939f7cSShane Wang * for the rest (32-bit) architectures.
141f1939f7cSShane Wang * For each, nh_16 *must* be defined (works on multiples of 16 bytes).
142f1939f7cSShane Wang * Optionally, nh_vmac_nhbytes can be defined (for multiples of
143f1939f7cSShane Wang * VMAC_NHBYTES), and nh_16_2 and nh_vmac_nhbytes_2 (versions that do two
144f1939f7cSShane Wang * NH computations at once).
145f1939f7cSShane Wang */
146f1939f7cSShane Wang
147f1939f7cSShane Wang #ifdef CONFIG_64BIT
148f1939f7cSShane Wang
149f1939f7cSShane Wang #define nh_16(mp, kp, nw, rh, rl) \
150f1939f7cSShane Wang do { \
151f1939f7cSShane Wang int i; u64 th, tl; \
152f1939f7cSShane Wang rh = rl = 0; \
153f1939f7cSShane Wang for (i = 0; i < nw; i += 2) { \
154304a204eSShane Wang MUL64(th, tl, pe64_to_cpup((mp)+i)+(kp)[i], \
155304a204eSShane Wang pe64_to_cpup((mp)+i+1)+(kp)[i+1]); \
156f1939f7cSShane Wang ADD128(rh, rl, th, tl); \
157f1939f7cSShane Wang } \
158f1939f7cSShane Wang } while (0)
159f1939f7cSShane Wang
160f1939f7cSShane Wang #define nh_16_2(mp, kp, nw, rh, rl, rh1, rl1) \
161f1939f7cSShane Wang do { \
162f1939f7cSShane Wang int i; u64 th, tl; \
163f1939f7cSShane Wang rh1 = rl1 = rh = rl = 0; \
164f1939f7cSShane Wang for (i = 0; i < nw; i += 2) { \
165304a204eSShane Wang MUL64(th, tl, pe64_to_cpup((mp)+i)+(kp)[i], \
166304a204eSShane Wang pe64_to_cpup((mp)+i+1)+(kp)[i+1]); \
167f1939f7cSShane Wang ADD128(rh, rl, th, tl); \
168304a204eSShane Wang MUL64(th, tl, pe64_to_cpup((mp)+i)+(kp)[i+2], \
169304a204eSShane Wang pe64_to_cpup((mp)+i+1)+(kp)[i+3]); \
170f1939f7cSShane Wang ADD128(rh1, rl1, th, tl); \
171f1939f7cSShane Wang } \
172f1939f7cSShane Wang } while (0)
173f1939f7cSShane Wang
174f1939f7cSShane Wang #if (VMAC_NHBYTES >= 64) /* These versions do 64-bytes of message at a time */
175f1939f7cSShane Wang #define nh_vmac_nhbytes(mp, kp, nw, rh, rl) \
176f1939f7cSShane Wang do { \
177f1939f7cSShane Wang int i; u64 th, tl; \
178f1939f7cSShane Wang rh = rl = 0; \
179f1939f7cSShane Wang for (i = 0; i < nw; i += 8) { \
180304a204eSShane Wang MUL64(th, tl, pe64_to_cpup((mp)+i)+(kp)[i], \
181304a204eSShane Wang pe64_to_cpup((mp)+i+1)+(kp)[i+1]); \
182f1939f7cSShane Wang ADD128(rh, rl, th, tl); \
183304a204eSShane Wang MUL64(th, tl, pe64_to_cpup((mp)+i+2)+(kp)[i+2], \
184304a204eSShane Wang pe64_to_cpup((mp)+i+3)+(kp)[i+3]); \
185f1939f7cSShane Wang ADD128(rh, rl, th, tl); \
186304a204eSShane Wang MUL64(th, tl, pe64_to_cpup((mp)+i+4)+(kp)[i+4], \
187304a204eSShane Wang pe64_to_cpup((mp)+i+5)+(kp)[i+5]); \
188f1939f7cSShane Wang ADD128(rh, rl, th, tl); \
189304a204eSShane Wang MUL64(th, tl, pe64_to_cpup((mp)+i+6)+(kp)[i+6], \
190304a204eSShane Wang pe64_to_cpup((mp)+i+7)+(kp)[i+7]); \
191f1939f7cSShane Wang ADD128(rh, rl, th, tl); \
192f1939f7cSShane Wang } \
193f1939f7cSShane Wang } while (0)
194f1939f7cSShane Wang
195f1939f7cSShane Wang #define nh_vmac_nhbytes_2(mp, kp, nw, rh, rl, rh1, rl1) \
196f1939f7cSShane Wang do { \
197f1939f7cSShane Wang int i; u64 th, tl; \
198f1939f7cSShane Wang rh1 = rl1 = rh = rl = 0; \
199f1939f7cSShane Wang for (i = 0; i < nw; i += 8) { \
200304a204eSShane Wang MUL64(th, tl, pe64_to_cpup((mp)+i)+(kp)[i], \
201304a204eSShane Wang pe64_to_cpup((mp)+i+1)+(kp)[i+1]); \
202f1939f7cSShane Wang ADD128(rh, rl, th, tl); \
203304a204eSShane Wang MUL64(th, tl, pe64_to_cpup((mp)+i)+(kp)[i+2], \
204304a204eSShane Wang pe64_to_cpup((mp)+i+1)+(kp)[i+3]); \
205f1939f7cSShane Wang ADD128(rh1, rl1, th, tl); \
206304a204eSShane Wang MUL64(th, tl, pe64_to_cpup((mp)+i+2)+(kp)[i+2], \
207304a204eSShane Wang pe64_to_cpup((mp)+i+3)+(kp)[i+3]); \
208f1939f7cSShane Wang ADD128(rh, rl, th, tl); \
209304a204eSShane Wang MUL64(th, tl, pe64_to_cpup((mp)+i+2)+(kp)[i+4], \
210304a204eSShane Wang pe64_to_cpup((mp)+i+3)+(kp)[i+5]); \
211f1939f7cSShane Wang ADD128(rh1, rl1, th, tl); \
212304a204eSShane Wang MUL64(th, tl, pe64_to_cpup((mp)+i+4)+(kp)[i+4], \
213304a204eSShane Wang pe64_to_cpup((mp)+i+5)+(kp)[i+5]); \
214f1939f7cSShane Wang ADD128(rh, rl, th, tl); \
215304a204eSShane Wang MUL64(th, tl, pe64_to_cpup((mp)+i+4)+(kp)[i+6], \
216304a204eSShane Wang pe64_to_cpup((mp)+i+5)+(kp)[i+7]); \
217f1939f7cSShane Wang ADD128(rh1, rl1, th, tl); \
218304a204eSShane Wang MUL64(th, tl, pe64_to_cpup((mp)+i+6)+(kp)[i+6], \
219304a204eSShane Wang pe64_to_cpup((mp)+i+7)+(kp)[i+7]); \
220f1939f7cSShane Wang ADD128(rh, rl, th, tl); \
221304a204eSShane Wang MUL64(th, tl, pe64_to_cpup((mp)+i+6)+(kp)[i+8], \
222304a204eSShane Wang pe64_to_cpup((mp)+i+7)+(kp)[i+9]); \
223f1939f7cSShane Wang ADD128(rh1, rl1, th, tl); \
224f1939f7cSShane Wang } \
225f1939f7cSShane Wang } while (0)
226f1939f7cSShane Wang #endif
227f1939f7cSShane Wang
228f1939f7cSShane Wang #define poly_step(ah, al, kh, kl, mh, ml) \
229f1939f7cSShane Wang do { \
230f1939f7cSShane Wang u64 t1h, t1l, t2h, t2l, t3h, t3l, z = 0; \
231f1939f7cSShane Wang /* compute ab*cd, put bd into result registers */ \
232f1939f7cSShane Wang PMUL64(t3h, t3l, al, kh); \
233f1939f7cSShane Wang PMUL64(t2h, t2l, ah, kl); \
234f1939f7cSShane Wang PMUL64(t1h, t1l, ah, 2*kh); \
235f1939f7cSShane Wang PMUL64(ah, al, al, kl); \
236f1939f7cSShane Wang /* add 2 * ac to result */ \
237f1939f7cSShane Wang ADD128(ah, al, t1h, t1l); \
238f1939f7cSShane Wang /* add together ad + bc */ \
239f1939f7cSShane Wang ADD128(t2h, t2l, t3h, t3l); \
240f1939f7cSShane Wang /* now (ah,al), (t2l,2*t2h) need summing */ \
241f1939f7cSShane Wang /* first add the high registers, carrying into t2h */ \
242f1939f7cSShane Wang ADD128(t2h, ah, z, t2l); \
243f1939f7cSShane Wang /* double t2h and add top bit of ah */ \
244f1939f7cSShane Wang t2h = 2 * t2h + (ah >> 63); \
245f1939f7cSShane Wang ah &= m63; \
246f1939f7cSShane Wang /* now add the low registers */ \
247f1939f7cSShane Wang ADD128(ah, al, mh, ml); \
248f1939f7cSShane Wang ADD128(ah, al, z, t2h); \
249f1939f7cSShane Wang } while (0)
250f1939f7cSShane Wang
251f1939f7cSShane Wang #else /* ! CONFIG_64BIT */
252f1939f7cSShane Wang
253f1939f7cSShane Wang #ifndef nh_16
254f1939f7cSShane Wang #define nh_16(mp, kp, nw, rh, rl) \
255f1939f7cSShane Wang do { \
256f1939f7cSShane Wang u64 t1, t2, m1, m2, t; \
257f1939f7cSShane Wang int i; \
258f1939f7cSShane Wang rh = rl = t = 0; \
259f1939f7cSShane Wang for (i = 0; i < nw; i += 2) { \
260304a204eSShane Wang t1 = pe64_to_cpup(mp+i) + kp[i]; \
261304a204eSShane Wang t2 = pe64_to_cpup(mp+i+1) + kp[i+1]; \
262f1939f7cSShane Wang m2 = MUL32(t1 >> 32, t2); \
263f1939f7cSShane Wang m1 = MUL32(t1, t2 >> 32); \
264f1939f7cSShane Wang ADD128(rh, rl, MUL32(t1 >> 32, t2 >> 32), \
265f1939f7cSShane Wang MUL32(t1, t2)); \
266f1939f7cSShane Wang rh += (u64)(u32)(m1 >> 32) \
267f1939f7cSShane Wang + (u32)(m2 >> 32); \
268f1939f7cSShane Wang t += (u64)(u32)m1 + (u32)m2; \
269f1939f7cSShane Wang } \
270f1939f7cSShane Wang ADD128(rh, rl, (t >> 32), (t << 32)); \
271f1939f7cSShane Wang } while (0)
272f1939f7cSShane Wang #endif
273f1939f7cSShane Wang
poly_step_func(u64 * ahi,u64 * alo,const u64 * kh,const u64 * kl,const u64 * mh,const u64 * ml)274f1939f7cSShane Wang static void poly_step_func(u64 *ahi, u64 *alo,
275f1939f7cSShane Wang const u64 *kh, const u64 *kl,
276f1939f7cSShane Wang const u64 *mh, const u64 *ml)
277f1939f7cSShane Wang {
278f1939f7cSShane Wang #define a0 (*(((u32 *)alo)+INDEX_LOW))
279f1939f7cSShane Wang #define a1 (*(((u32 *)alo)+INDEX_HIGH))
280f1939f7cSShane Wang #define a2 (*(((u32 *)ahi)+INDEX_LOW))
281f1939f7cSShane Wang #define a3 (*(((u32 *)ahi)+INDEX_HIGH))
282f1939f7cSShane Wang #define k0 (*(((u32 *)kl)+INDEX_LOW))
283f1939f7cSShane Wang #define k1 (*(((u32 *)kl)+INDEX_HIGH))
284f1939f7cSShane Wang #define k2 (*(((u32 *)kh)+INDEX_LOW))
285f1939f7cSShane Wang #define k3 (*(((u32 *)kh)+INDEX_HIGH))
286f1939f7cSShane Wang
287f1939f7cSShane Wang u64 p, q, t;
288f1939f7cSShane Wang u32 t2;
289f1939f7cSShane Wang
290f1939f7cSShane Wang p = MUL32(a3, k3);
291f1939f7cSShane Wang p += p;
292f1939f7cSShane Wang p += *(u64 *)mh;
293f1939f7cSShane Wang p += MUL32(a0, k2);
294f1939f7cSShane Wang p += MUL32(a1, k1);
295f1939f7cSShane Wang p += MUL32(a2, k0);
296f1939f7cSShane Wang t = (u32)(p);
297f1939f7cSShane Wang p >>= 32;
298f1939f7cSShane Wang p += MUL32(a0, k3);
299f1939f7cSShane Wang p += MUL32(a1, k2);
300f1939f7cSShane Wang p += MUL32(a2, k1);
301f1939f7cSShane Wang p += MUL32(a3, k0);
302f1939f7cSShane Wang t |= ((u64)((u32)p & 0x7fffffff)) << 32;
303f1939f7cSShane Wang p >>= 31;
304f1939f7cSShane Wang p += (u64)(((u32 *)ml)[INDEX_LOW]);
305f1939f7cSShane Wang p += MUL32(a0, k0);
306f1939f7cSShane Wang q = MUL32(a1, k3);
307f1939f7cSShane Wang q += MUL32(a2, k2);
308f1939f7cSShane Wang q += MUL32(a3, k1);
309f1939f7cSShane Wang q += q;
310f1939f7cSShane Wang p += q;
311f1939f7cSShane Wang t2 = (u32)(p);
312f1939f7cSShane Wang p >>= 32;
313f1939f7cSShane Wang p += (u64)(((u32 *)ml)[INDEX_HIGH]);
314f1939f7cSShane Wang p += MUL32(a0, k1);
315f1939f7cSShane Wang p += MUL32(a1, k0);
316f1939f7cSShane Wang q = MUL32(a2, k3);
317f1939f7cSShane Wang q += MUL32(a3, k2);
318f1939f7cSShane Wang q += q;
319f1939f7cSShane Wang p += q;
320f1939f7cSShane Wang *(u64 *)(alo) = (p << 32) | t2;
321f1939f7cSShane Wang p >>= 32;
322f1939f7cSShane Wang *(u64 *)(ahi) = p + t;
323f1939f7cSShane Wang
324f1939f7cSShane Wang #undef a0
325f1939f7cSShane Wang #undef a1
326f1939f7cSShane Wang #undef a2
327f1939f7cSShane Wang #undef a3
328f1939f7cSShane Wang #undef k0
329f1939f7cSShane Wang #undef k1
330f1939f7cSShane Wang #undef k2
331f1939f7cSShane Wang #undef k3
332f1939f7cSShane Wang }
333f1939f7cSShane Wang
334f1939f7cSShane Wang #define poly_step(ah, al, kh, kl, mh, ml) \
335f1939f7cSShane Wang poly_step_func(&(ah), &(al), &(kh), &(kl), &(mh), &(ml))
336f1939f7cSShane Wang
337f1939f7cSShane Wang #endif /* end of specialized NH and poly definitions */
338f1939f7cSShane Wang
339f1939f7cSShane Wang /* At least nh_16 is defined. Defined others as needed here */
340f1939f7cSShane Wang #ifndef nh_16_2
341f1939f7cSShane Wang #define nh_16_2(mp, kp, nw, rh, rl, rh2, rl2) \
342f1939f7cSShane Wang do { \
343f1939f7cSShane Wang nh_16(mp, kp, nw, rh, rl); \
344f1939f7cSShane Wang nh_16(mp, ((kp)+2), nw, rh2, rl2); \
345f1939f7cSShane Wang } while (0)
346f1939f7cSShane Wang #endif
347f1939f7cSShane Wang #ifndef nh_vmac_nhbytes
348f1939f7cSShane Wang #define nh_vmac_nhbytes(mp, kp, nw, rh, rl) \
349f1939f7cSShane Wang nh_16(mp, kp, nw, rh, rl)
350f1939f7cSShane Wang #endif
351f1939f7cSShane Wang #ifndef nh_vmac_nhbytes_2
352f1939f7cSShane Wang #define nh_vmac_nhbytes_2(mp, kp, nw, rh, rl, rh2, rl2) \
353f1939f7cSShane Wang do { \
354f1939f7cSShane Wang nh_vmac_nhbytes(mp, kp, nw, rh, rl); \
355f1939f7cSShane Wang nh_vmac_nhbytes(mp, ((kp)+2), nw, rh2, rl2); \
356f1939f7cSShane Wang } while (0)
357f1939f7cSShane Wang #endif
358f1939f7cSShane Wang
l3hash(u64 p1,u64 p2,u64 k1,u64 k2,u64 len)359304a204eSShane Wang static u64 l3hash(u64 p1, u64 p2, u64 k1, u64 k2, u64 len)
360f1939f7cSShane Wang {
361f1939f7cSShane Wang u64 rh, rl, t, z = 0;
362f1939f7cSShane Wang
363f1939f7cSShane Wang /* fully reduce (p1,p2)+(len,0) mod p127 */
364f1939f7cSShane Wang t = p1 >> 63;
365f1939f7cSShane Wang p1 &= m63;
366f1939f7cSShane Wang ADD128(p1, p2, len, t);
367f1939f7cSShane Wang /* At this point, (p1,p2) is at most 2^127+(len<<64) */
368f1939f7cSShane Wang t = (p1 > m63) + ((p1 == m63) && (p2 == m64));
369f1939f7cSShane Wang ADD128(p1, p2, z, t);
370f1939f7cSShane Wang p1 &= m63;
371f1939f7cSShane Wang
372f1939f7cSShane Wang /* compute (p1,p2)/(2^64-2^32) and (p1,p2)%(2^64-2^32) */
373f1939f7cSShane Wang t = p1 + (p2 >> 32);
374f1939f7cSShane Wang t += (t >> 32);
375f1939f7cSShane Wang t += (u32)t > 0xfffffffeu;
376f1939f7cSShane Wang p1 += (t >> 32);
377f1939f7cSShane Wang p2 += (p1 << 32);
378f1939f7cSShane Wang
379f1939f7cSShane Wang /* compute (p1+k1)%p64 and (p2+k2)%p64 */
380f1939f7cSShane Wang p1 += k1;
381f1939f7cSShane Wang p1 += (0 - (p1 < k1)) & 257;
382f1939f7cSShane Wang p2 += k2;
383f1939f7cSShane Wang p2 += (0 - (p2 < k2)) & 257;
384f1939f7cSShane Wang
385f1939f7cSShane Wang /* compute (p1+k1)*(p2+k2)%p64 */
386f1939f7cSShane Wang MUL64(rh, rl, p1, p2);
387f1939f7cSShane Wang t = rh >> 56;
388f1939f7cSShane Wang ADD128(t, rl, z, rh);
389f1939f7cSShane Wang rh <<= 8;
390f1939f7cSShane Wang ADD128(t, rl, z, rh);
391f1939f7cSShane Wang t += t << 8;
392f1939f7cSShane Wang rl += t;
393f1939f7cSShane Wang rl += (0 - (rl < t)) & 257;
394f1939f7cSShane Wang rl += (0 - (rl > p64-1)) & 257;
395f1939f7cSShane Wang return rl;
396f1939f7cSShane Wang }
397f1939f7cSShane Wang
398bb296481SEric Biggers /* L1 and L2-hash one or more VMAC_NHBYTES-byte blocks */
vhash_blocks(const struct vmac_tfm_ctx * tctx,struct vmac_desc_ctx * dctx,const __le64 * mptr,unsigned int blocks)399bb296481SEric Biggers static void vhash_blocks(const struct vmac_tfm_ctx *tctx,
400bb296481SEric Biggers struct vmac_desc_ctx *dctx,
401bb296481SEric Biggers const __le64 *mptr, unsigned int blocks)
402f1939f7cSShane Wang {
403bb296481SEric Biggers const u64 *kptr = tctx->nhkey;
404bb296481SEric Biggers const u64 pkh = tctx->polykey[0];
405bb296481SEric Biggers const u64 pkl = tctx->polykey[1];
406bb296481SEric Biggers u64 ch = dctx->polytmp[0];
407bb296481SEric Biggers u64 cl = dctx->polytmp[1];
408bb296481SEric Biggers u64 rh, rl;
409f1939f7cSShane Wang
410bb296481SEric Biggers if (!dctx->first_block_processed) {
411bb296481SEric Biggers dctx->first_block_processed = true;
412f1939f7cSShane Wang nh_vmac_nhbytes(mptr, kptr, VMAC_NHBYTES/8, rh, rl);
413f1939f7cSShane Wang rh &= m62;
414f1939f7cSShane Wang ADD128(ch, cl, rh, rl);
415f1939f7cSShane Wang mptr += (VMAC_NHBYTES/sizeof(u64));
416bb296481SEric Biggers blocks--;
417f1939f7cSShane Wang }
418f1939f7cSShane Wang
419bb296481SEric Biggers while (blocks--) {
420f1939f7cSShane Wang nh_vmac_nhbytes(mptr, kptr, VMAC_NHBYTES/8, rh, rl);
421f1939f7cSShane Wang rh &= m62;
422f1939f7cSShane Wang poly_step(ch, cl, pkh, pkl, rh, rl);
423f1939f7cSShane Wang mptr += (VMAC_NHBYTES/sizeof(u64));
424f1939f7cSShane Wang }
425f1939f7cSShane Wang
426bb296481SEric Biggers dctx->polytmp[0] = ch;
427bb296481SEric Biggers dctx->polytmp[1] = cl;
428f1939f7cSShane Wang }
429f1939f7cSShane Wang
vmac_setkey(struct crypto_shash * tfm,const u8 * key,unsigned int keylen)430bb296481SEric Biggers static int vmac_setkey(struct crypto_shash *tfm,
431bb296481SEric Biggers const u8 *key, unsigned int keylen)
432f1939f7cSShane Wang {
433bb296481SEric Biggers struct vmac_tfm_ctx *tctx = crypto_shash_ctx(tfm);
434bb296481SEric Biggers __be64 out[2];
435bb296481SEric Biggers u8 in[16] = { 0 };
436bb296481SEric Biggers unsigned int i;
437bb296481SEric Biggers int err;
438f1939f7cSShane Wang
439674f368aSEric Biggers if (keylen != VMAC_KEY_LEN)
440bb296481SEric Biggers return -EINVAL;
441f1939f7cSShane Wang
442bb296481SEric Biggers err = crypto_cipher_setkey(tctx->cipher, key, keylen);
443f1939f7cSShane Wang if (err)
444f1939f7cSShane Wang return err;
445f1939f7cSShane Wang
446f1939f7cSShane Wang /* Fill nh key */
447bb296481SEric Biggers in[0] = 0x80;
448bb296481SEric Biggers for (i = 0; i < ARRAY_SIZE(tctx->nhkey); i += 2) {
449bb296481SEric Biggers crypto_cipher_encrypt_one(tctx->cipher, (u8 *)out, in);
450bb296481SEric Biggers tctx->nhkey[i] = be64_to_cpu(out[0]);
451bb296481SEric Biggers tctx->nhkey[i+1] = be64_to_cpu(out[1]);
452bb296481SEric Biggers in[15]++;
453f1939f7cSShane Wang }
454f1939f7cSShane Wang
455f1939f7cSShane Wang /* Fill poly key */
456bb296481SEric Biggers in[0] = 0xC0;
457bb296481SEric Biggers in[15] = 0;
458bb296481SEric Biggers for (i = 0; i < ARRAY_SIZE(tctx->polykey); i += 2) {
459bb296481SEric Biggers crypto_cipher_encrypt_one(tctx->cipher, (u8 *)out, in);
460bb296481SEric Biggers tctx->polykey[i] = be64_to_cpu(out[0]) & mpoly;
461bb296481SEric Biggers tctx->polykey[i+1] = be64_to_cpu(out[1]) & mpoly;
462bb296481SEric Biggers in[15]++;
463f1939f7cSShane Wang }
464f1939f7cSShane Wang
465f1939f7cSShane Wang /* Fill ip key */
466bb296481SEric Biggers in[0] = 0xE0;
467bb296481SEric Biggers in[15] = 0;
468bb296481SEric Biggers for (i = 0; i < ARRAY_SIZE(tctx->l3key); i += 2) {
469f1939f7cSShane Wang do {
470bb296481SEric Biggers crypto_cipher_encrypt_one(tctx->cipher, (u8 *)out, in);
471bb296481SEric Biggers tctx->l3key[i] = be64_to_cpu(out[0]);
472bb296481SEric Biggers tctx->l3key[i+1] = be64_to_cpu(out[1]);
473bb296481SEric Biggers in[15]++;
474bb296481SEric Biggers } while (tctx->l3key[i] >= p64 || tctx->l3key[i+1] >= p64);
475f1939f7cSShane Wang }
476f1939f7cSShane Wang
477f1939f7cSShane Wang return 0;
478f1939f7cSShane Wang }
479f1939f7cSShane Wang
vmac_init(struct shash_desc * desc)480bb296481SEric Biggers static int vmac_init(struct shash_desc *desc)
481f1939f7cSShane Wang {
482bb296481SEric Biggers const struct vmac_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm);
483bb296481SEric Biggers struct vmac_desc_ctx *dctx = shash_desc_ctx(desc);
484f1939f7cSShane Wang
485bb296481SEric Biggers dctx->partial_size = 0;
486bb296481SEric Biggers dctx->first_block_processed = false;
487bb296481SEric Biggers memcpy(dctx->polytmp, tctx->polykey, sizeof(dctx->polytmp));
488ed331adaSEric Biggers dctx->nonce_size = 0;
489ed331adaSEric Biggers return 0;
490ed331adaSEric Biggers }
491ed331adaSEric Biggers
vmac_update(struct shash_desc * desc,const u8 * p,unsigned int len)492bb296481SEric Biggers static int vmac_update(struct shash_desc *desc, const u8 *p, unsigned int len)
493bb296481SEric Biggers {
494bb296481SEric Biggers const struct vmac_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm);
495bb296481SEric Biggers struct vmac_desc_ctx *dctx = shash_desc_ctx(desc);
496bb296481SEric Biggers unsigned int n;
497bb296481SEric Biggers
498ed331adaSEric Biggers /* Nonce is passed as first VMAC_NONCEBYTES bytes of data */
499ed331adaSEric Biggers if (dctx->nonce_size < VMAC_NONCEBYTES) {
500ed331adaSEric Biggers n = min(len, VMAC_NONCEBYTES - dctx->nonce_size);
501ed331adaSEric Biggers memcpy(&dctx->nonce.bytes[dctx->nonce_size], p, n);
502ed331adaSEric Biggers dctx->nonce_size += n;
503ed331adaSEric Biggers p += n;
504ed331adaSEric Biggers len -= n;
505ed331adaSEric Biggers }
506ed331adaSEric Biggers
507bb296481SEric Biggers if (dctx->partial_size) {
508bb296481SEric Biggers n = min(len, VMAC_NHBYTES - dctx->partial_size);
509bb296481SEric Biggers memcpy(&dctx->partial[dctx->partial_size], p, n);
510bb296481SEric Biggers dctx->partial_size += n;
511bb296481SEric Biggers p += n;
512bb296481SEric Biggers len -= n;
513bb296481SEric Biggers if (dctx->partial_size == VMAC_NHBYTES) {
514bb296481SEric Biggers vhash_blocks(tctx, dctx, dctx->partial_words, 1);
515bb296481SEric Biggers dctx->partial_size = 0;
516bb296481SEric Biggers }
517bb296481SEric Biggers }
518bb296481SEric Biggers
519bb296481SEric Biggers if (len >= VMAC_NHBYTES) {
520bb296481SEric Biggers n = round_down(len, VMAC_NHBYTES);
521bb296481SEric Biggers /* TODO: 'p' may be misaligned here */
522bb296481SEric Biggers vhash_blocks(tctx, dctx, (const __le64 *)p, n / VMAC_NHBYTES);
523bb296481SEric Biggers p += n;
524bb296481SEric Biggers len -= n;
525bb296481SEric Biggers }
526bb296481SEric Biggers
527bb296481SEric Biggers if (len) {
528bb296481SEric Biggers memcpy(dctx->partial, p, len);
529bb296481SEric Biggers dctx->partial_size = len;
530bb296481SEric Biggers }
531bb296481SEric Biggers
532bb296481SEric Biggers return 0;
533bb296481SEric Biggers }
534bb296481SEric Biggers
vhash_final(const struct vmac_tfm_ctx * tctx,struct vmac_desc_ctx * dctx)535bb296481SEric Biggers static u64 vhash_final(const struct vmac_tfm_ctx *tctx,
536bb296481SEric Biggers struct vmac_desc_ctx *dctx)
537bb296481SEric Biggers {
538bb296481SEric Biggers unsigned int partial = dctx->partial_size;
539bb296481SEric Biggers u64 ch = dctx->polytmp[0];
540bb296481SEric Biggers u64 cl = dctx->polytmp[1];
541bb296481SEric Biggers
542bb296481SEric Biggers /* L1 and L2-hash the final block if needed */
543bb296481SEric Biggers if (partial) {
544bb296481SEric Biggers /* Zero-pad to next 128-bit boundary */
545bb296481SEric Biggers unsigned int n = round_up(partial, 16);
546bb296481SEric Biggers u64 rh, rl;
547bb296481SEric Biggers
548bb296481SEric Biggers memset(&dctx->partial[partial], 0, n - partial);
549bb296481SEric Biggers nh_16(dctx->partial_words, tctx->nhkey, n / 8, rh, rl);
550bb296481SEric Biggers rh &= m62;
551bb296481SEric Biggers if (dctx->first_block_processed)
552bb296481SEric Biggers poly_step(ch, cl, tctx->polykey[0], tctx->polykey[1],
553bb296481SEric Biggers rh, rl);
554bb296481SEric Biggers else
555bb296481SEric Biggers ADD128(ch, cl, rh, rl);
556bb296481SEric Biggers }
557bb296481SEric Biggers
558bb296481SEric Biggers /* L3-hash the 128-bit output of L2-hash */
559bb296481SEric Biggers return l3hash(ch, cl, tctx->l3key[0], tctx->l3key[1], partial * 8);
560bb296481SEric Biggers }
561bb296481SEric Biggers
vmac_final(struct shash_desc * desc,u8 * out)5620917b873SEric Biggers static int vmac_final(struct shash_desc *desc, u8 *out)
563bb296481SEric Biggers {
564bb296481SEric Biggers const struct vmac_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm);
565bb296481SEric Biggers struct vmac_desc_ctx *dctx = shash_desc_ctx(desc);
566bb296481SEric Biggers int index;
567bb296481SEric Biggers u64 hash, pad;
568bb296481SEric Biggers
569ed331adaSEric Biggers if (dctx->nonce_size != VMAC_NONCEBYTES)
570ed331adaSEric Biggers return -EINVAL;
571ed331adaSEric Biggers
572ed331adaSEric Biggers /*
573ed331adaSEric Biggers * The VMAC specification requires a nonce at least 1 bit shorter than
574ed331adaSEric Biggers * the block cipher's block length, so we actually only accept a 127-bit
575ed331adaSEric Biggers * nonce. We define the unused bit to be the first one and require that
576ed331adaSEric Biggers * it be 0, so the needed prepending of a 0 bit is implicit.
577ed331adaSEric Biggers */
578ed331adaSEric Biggers if (dctx->nonce.bytes[0] & 0x80)
579ed331adaSEric Biggers return -EINVAL;
580ed331adaSEric Biggers
581bb296481SEric Biggers /* Finish calculating the VHASH of the message */
582bb296481SEric Biggers hash = vhash_final(tctx, dctx);
583bb296481SEric Biggers
584bb296481SEric Biggers /* Generate pseudorandom pad by encrypting the nonce */
585ed331adaSEric Biggers BUILD_BUG_ON(VMAC_NONCEBYTES != 2 * (VMAC_TAG_LEN / 8));
586ed331adaSEric Biggers index = dctx->nonce.bytes[VMAC_NONCEBYTES - 1] & 1;
587ed331adaSEric Biggers dctx->nonce.bytes[VMAC_NONCEBYTES - 1] &= ~1;
588ed331adaSEric Biggers crypto_cipher_encrypt_one(tctx->cipher, dctx->nonce.bytes,
589ed331adaSEric Biggers dctx->nonce.bytes);
590ed331adaSEric Biggers pad = be64_to_cpu(dctx->nonce.pads[index]);
591bb296481SEric Biggers
592bb296481SEric Biggers /* The VMAC is the sum of VHASH and the pseudorandom pad */
5930917b873SEric Biggers put_unaligned_be64(hash + pad, out);
594f1939f7cSShane Wang return 0;
595f1939f7cSShane Wang }
596f1939f7cSShane Wang
vmac_init_tfm(struct crypto_tfm * tfm)597f1939f7cSShane Wang static int vmac_init_tfm(struct crypto_tfm *tfm)
598f1939f7cSShane Wang {
599bb296481SEric Biggers struct crypto_instance *inst = crypto_tfm_alg_instance(tfm);
600d5ed3b65SEric Biggers struct crypto_cipher_spawn *spawn = crypto_instance_ctx(inst);
601bb296481SEric Biggers struct vmac_tfm_ctx *tctx = crypto_tfm_ctx(tfm);
602bb296481SEric Biggers struct crypto_cipher *cipher;
603f1939f7cSShane Wang
604f1939f7cSShane Wang cipher = crypto_spawn_cipher(spawn);
605f1939f7cSShane Wang if (IS_ERR(cipher))
606f1939f7cSShane Wang return PTR_ERR(cipher);
607f1939f7cSShane Wang
608bb296481SEric Biggers tctx->cipher = cipher;
609f1939f7cSShane Wang return 0;
610f1939f7cSShane Wang }
611f1939f7cSShane Wang
vmac_exit_tfm(struct crypto_tfm * tfm)612f1939f7cSShane Wang static void vmac_exit_tfm(struct crypto_tfm *tfm)
613f1939f7cSShane Wang {
614bb296481SEric Biggers struct vmac_tfm_ctx *tctx = crypto_tfm_ctx(tfm);
615bb296481SEric Biggers
616bb296481SEric Biggers crypto_free_cipher(tctx->cipher);
617f1939f7cSShane Wang }
618f1939f7cSShane Wang
vmac_create(struct crypto_template * tmpl,struct rtattr ** tb)6190917b873SEric Biggers static int vmac_create(struct crypto_template *tmpl, struct rtattr **tb)
620f1939f7cSShane Wang {
621f1939f7cSShane Wang struct shash_instance *inst;
6223b4e73d8SEric Biggers struct crypto_cipher_spawn *spawn;
623f1939f7cSShane Wang struct crypto_alg *alg;
6247bcb2c99SEric Biggers u32 mask;
625f1939f7cSShane Wang int err;
626f1939f7cSShane Wang
6277bcb2c99SEric Biggers err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_SHASH, &mask);
628f1939f7cSShane Wang if (err)
629f1939f7cSShane Wang return err;
630f1939f7cSShane Wang
6313b4e73d8SEric Biggers inst = kzalloc(sizeof(*inst) + sizeof(*spawn), GFP_KERNEL);
6323b4e73d8SEric Biggers if (!inst)
6333b4e73d8SEric Biggers return -ENOMEM;
6343b4e73d8SEric Biggers spawn = shash_instance_ctx(inst);
6353b4e73d8SEric Biggers
6363b4e73d8SEric Biggers err = crypto_grab_cipher(spawn, shash_crypto_instance(inst),
6377bcb2c99SEric Biggers crypto_attr_alg_name(tb[1]), 0, mask);
6383b4e73d8SEric Biggers if (err)
6393b4e73d8SEric Biggers goto err_free_inst;
6403b4e73d8SEric Biggers alg = crypto_spawn_cipher_alg(spawn);
641f1939f7cSShane Wang
64273bf20efSEric Biggers err = -EINVAL;
643ed331adaSEric Biggers if (alg->cra_blocksize != VMAC_NONCEBYTES)
6443b4e73d8SEric Biggers goto err_free_inst;
64573bf20efSEric Biggers
6463b4e73d8SEric Biggers err = crypto_inst_setname(shash_crypto_instance(inst), tmpl->name, alg);
647f1939f7cSShane Wang if (err)
6483b4e73d8SEric Biggers goto err_free_inst;
649f1939f7cSShane Wang
650f1939f7cSShane Wang inst->alg.base.cra_priority = alg->cra_priority;
651f1939f7cSShane Wang inst->alg.base.cra_blocksize = alg->cra_blocksize;
652f1939f7cSShane Wang inst->alg.base.cra_alignmask = alg->cra_alignmask;
653f1939f7cSShane Wang
654bb296481SEric Biggers inst->alg.base.cra_ctxsize = sizeof(struct vmac_tfm_ctx);
655f1939f7cSShane Wang inst->alg.base.cra_init = vmac_init_tfm;
656f1939f7cSShane Wang inst->alg.base.cra_exit = vmac_exit_tfm;
657f1939f7cSShane Wang
658bb296481SEric Biggers inst->alg.descsize = sizeof(struct vmac_desc_ctx);
659bb296481SEric Biggers inst->alg.digestsize = VMAC_TAG_LEN / 8;
660f1939f7cSShane Wang inst->alg.init = vmac_init;
661f1939f7cSShane Wang inst->alg.update = vmac_update;
6620917b873SEric Biggers inst->alg.final = vmac_final;
663f1939f7cSShane Wang inst->alg.setkey = vmac_setkey;
664f1939f7cSShane Wang
665a39c66ccSEric Biggers inst->free = shash_free_singlespawn_instance;
666a39c66ccSEric Biggers
667f1939f7cSShane Wang err = shash_register_instance(tmpl, inst);
668f1939f7cSShane Wang if (err) {
6693b4e73d8SEric Biggers err_free_inst:
670a39c66ccSEric Biggers shash_free_singlespawn_instance(inst);
671f1939f7cSShane Wang }
672f1939f7cSShane Wang return err;
673f1939f7cSShane Wang }
674f1939f7cSShane Wang
675ed331adaSEric Biggers static struct crypto_template vmac64_tmpl = {
676ed331adaSEric Biggers .name = "vmac64",
6770917b873SEric Biggers .create = vmac_create,
678ed331adaSEric Biggers .module = THIS_MODULE,
679ed331adaSEric Biggers };
680ed331adaSEric Biggers
vmac_module_init(void)681f1939f7cSShane Wang static int __init vmac_module_init(void)
682f1939f7cSShane Wang {
6830917b873SEric Biggers return crypto_register_template(&vmac64_tmpl);
684f1939f7cSShane Wang }
685f1939f7cSShane Wang
vmac_module_exit(void)686f1939f7cSShane Wang static void __exit vmac_module_exit(void)
687f1939f7cSShane Wang {
688ed331adaSEric Biggers crypto_unregister_template(&vmac64_tmpl);
689f1939f7cSShane Wang }
690f1939f7cSShane Wang
691c4741b23SEric Biggers subsys_initcall(vmac_module_init);
692f1939f7cSShane Wang module_exit(vmac_module_exit);
693f1939f7cSShane Wang
694f1939f7cSShane Wang MODULE_LICENSE("GPL");
695f1939f7cSShane Wang MODULE_DESCRIPTION("VMAC hash algorithm");
696ed331adaSEric Biggers MODULE_ALIAS_CRYPTO("vmac64");
697*0eb76ba2SArd Biesheuvel MODULE_IMPORT_NS(CRYPTO_INTERNAL);
698