1ed20078bSArd Biesheuvel // SPDX-License-Identifier: GPL-2.0 OR MIT
2ed20078bSArd Biesheuvel /*
3ed20078bSArd Biesheuvel * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
4ed20078bSArd Biesheuvel *
5ed20078bSArd Biesheuvel * This is an implementation of the ChaCha20Poly1305 AEAD construction.
6ed20078bSArd Biesheuvel *
7ed20078bSArd Biesheuvel * Information: https://tools.ietf.org/html/rfc8439
8ed20078bSArd Biesheuvel */
9ed20078bSArd Biesheuvel
10ed20078bSArd Biesheuvel #include <crypto/algapi.h>
11ed20078bSArd Biesheuvel #include <crypto/chacha20poly1305.h>
12ed20078bSArd Biesheuvel #include <crypto/chacha.h>
13ed20078bSArd Biesheuvel #include <crypto/poly1305.h>
14d95312a3SArd Biesheuvel #include <crypto/scatterwalk.h>
15ed20078bSArd Biesheuvel
16ed20078bSArd Biesheuvel #include <asm/unaligned.h>
17ed20078bSArd Biesheuvel #include <linux/kernel.h>
18ed20078bSArd Biesheuvel #include <linux/init.h>
19ed20078bSArd Biesheuvel #include <linux/mm.h>
20ed20078bSArd Biesheuvel #include <linux/module.h>
21ed20078bSArd Biesheuvel
22ed20078bSArd Biesheuvel #define CHACHA_KEY_WORDS (CHACHA_KEY_SIZE / sizeof(u32))
23ed20078bSArd Biesheuvel
chacha_load_key(u32 * k,const u8 * in)24ed20078bSArd Biesheuvel static void chacha_load_key(u32 *k, const u8 *in)
25ed20078bSArd Biesheuvel {
26ed20078bSArd Biesheuvel k[0] = get_unaligned_le32(in);
27ed20078bSArd Biesheuvel k[1] = get_unaligned_le32(in + 4);
28ed20078bSArd Biesheuvel k[2] = get_unaligned_le32(in + 8);
29ed20078bSArd Biesheuvel k[3] = get_unaligned_le32(in + 12);
30ed20078bSArd Biesheuvel k[4] = get_unaligned_le32(in + 16);
31ed20078bSArd Biesheuvel k[5] = get_unaligned_le32(in + 20);
32ed20078bSArd Biesheuvel k[6] = get_unaligned_le32(in + 24);
33ed20078bSArd Biesheuvel k[7] = get_unaligned_le32(in + 28);
34ed20078bSArd Biesheuvel }
35ed20078bSArd Biesheuvel
xchacha_init(u32 * chacha_state,const u8 * key,const u8 * nonce)36ed20078bSArd Biesheuvel static void xchacha_init(u32 *chacha_state, const u8 *key, const u8 *nonce)
37ed20078bSArd Biesheuvel {
38ed20078bSArd Biesheuvel u32 k[CHACHA_KEY_WORDS];
39ed20078bSArd Biesheuvel u8 iv[CHACHA_IV_SIZE];
40ed20078bSArd Biesheuvel
41ed20078bSArd Biesheuvel memset(iv, 0, 8);
42ed20078bSArd Biesheuvel memcpy(iv + 8, nonce + 16, 8);
43ed20078bSArd Biesheuvel
44ed20078bSArd Biesheuvel chacha_load_key(k, key);
45ed20078bSArd Biesheuvel
46ed20078bSArd Biesheuvel /* Compute the subkey given the original key and first 128 nonce bits */
47ed20078bSArd Biesheuvel chacha_init(chacha_state, k, nonce);
48ed20078bSArd Biesheuvel hchacha_block(chacha_state, k, 20);
49ed20078bSArd Biesheuvel
50ed20078bSArd Biesheuvel chacha_init(chacha_state, k, iv);
51ed20078bSArd Biesheuvel
52ed20078bSArd Biesheuvel memzero_explicit(k, sizeof(k));
53ed20078bSArd Biesheuvel memzero_explicit(iv, sizeof(iv));
54ed20078bSArd Biesheuvel }
55ed20078bSArd Biesheuvel
56ed20078bSArd Biesheuvel static void
__chacha20poly1305_encrypt(u8 * dst,const u8 * src,const size_t src_len,const u8 * ad,const size_t ad_len,u32 * chacha_state)57ed20078bSArd Biesheuvel __chacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len,
58ed20078bSArd Biesheuvel const u8 *ad, const size_t ad_len, u32 *chacha_state)
59ed20078bSArd Biesheuvel {
60ed20078bSArd Biesheuvel const u8 *pad0 = page_address(ZERO_PAGE(0));
61ed20078bSArd Biesheuvel struct poly1305_desc_ctx poly1305_state;
62ed20078bSArd Biesheuvel union {
63ed20078bSArd Biesheuvel u8 block0[POLY1305_KEY_SIZE];
64ed20078bSArd Biesheuvel __le64 lens[2];
65ed20078bSArd Biesheuvel } b;
66ed20078bSArd Biesheuvel
67413808b7SEric Biggers chacha20_crypt(chacha_state, b.block0, pad0, sizeof(b.block0));
68ed20078bSArd Biesheuvel poly1305_init(&poly1305_state, b.block0);
69ed20078bSArd Biesheuvel
70ed20078bSArd Biesheuvel poly1305_update(&poly1305_state, ad, ad_len);
71ed20078bSArd Biesheuvel if (ad_len & 0xf)
72ed20078bSArd Biesheuvel poly1305_update(&poly1305_state, pad0, 0x10 - (ad_len & 0xf));
73ed20078bSArd Biesheuvel
74413808b7SEric Biggers chacha20_crypt(chacha_state, dst, src, src_len);
75ed20078bSArd Biesheuvel
76ed20078bSArd Biesheuvel poly1305_update(&poly1305_state, dst, src_len);
77ed20078bSArd Biesheuvel if (src_len & 0xf)
78ed20078bSArd Biesheuvel poly1305_update(&poly1305_state, pad0, 0x10 - (src_len & 0xf));
79ed20078bSArd Biesheuvel
80ed20078bSArd Biesheuvel b.lens[0] = cpu_to_le64(ad_len);
81ed20078bSArd Biesheuvel b.lens[1] = cpu_to_le64(src_len);
82ed20078bSArd Biesheuvel poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens));
83ed20078bSArd Biesheuvel
84ed20078bSArd Biesheuvel poly1305_final(&poly1305_state, dst + src_len);
85ed20078bSArd Biesheuvel
86ed20078bSArd Biesheuvel memzero_explicit(chacha_state, CHACHA_STATE_WORDS * sizeof(u32));
87ed20078bSArd Biesheuvel memzero_explicit(&b, sizeof(b));
88ed20078bSArd Biesheuvel }
89ed20078bSArd Biesheuvel
chacha20poly1305_encrypt(u8 * dst,const u8 * src,const size_t src_len,const u8 * ad,const size_t ad_len,const u64 nonce,const u8 key[CHACHA20POLY1305_KEY_SIZE])90ed20078bSArd Biesheuvel void chacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len,
91ed20078bSArd Biesheuvel const u8 *ad, const size_t ad_len,
92ed20078bSArd Biesheuvel const u64 nonce,
93ed20078bSArd Biesheuvel const u8 key[CHACHA20POLY1305_KEY_SIZE])
94ed20078bSArd Biesheuvel {
95ed20078bSArd Biesheuvel u32 chacha_state[CHACHA_STATE_WORDS];
96ed20078bSArd Biesheuvel u32 k[CHACHA_KEY_WORDS];
97ed20078bSArd Biesheuvel __le64 iv[2];
98ed20078bSArd Biesheuvel
99ed20078bSArd Biesheuvel chacha_load_key(k, key);
100ed20078bSArd Biesheuvel
101ed20078bSArd Biesheuvel iv[0] = 0;
102ed20078bSArd Biesheuvel iv[1] = cpu_to_le64(nonce);
103ed20078bSArd Biesheuvel
104ed20078bSArd Biesheuvel chacha_init(chacha_state, k, (u8 *)iv);
105ed20078bSArd Biesheuvel __chacha20poly1305_encrypt(dst, src, src_len, ad, ad_len, chacha_state);
106ed20078bSArd Biesheuvel
107ed20078bSArd Biesheuvel memzero_explicit(iv, sizeof(iv));
108ed20078bSArd Biesheuvel memzero_explicit(k, sizeof(k));
109ed20078bSArd Biesheuvel }
110ed20078bSArd Biesheuvel EXPORT_SYMBOL(chacha20poly1305_encrypt);
111ed20078bSArd Biesheuvel
xchacha20poly1305_encrypt(u8 * dst,const u8 * src,const size_t src_len,const u8 * ad,const size_t ad_len,const u8 nonce[XCHACHA20POLY1305_NONCE_SIZE],const u8 key[CHACHA20POLY1305_KEY_SIZE])112ed20078bSArd Biesheuvel void xchacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len,
113ed20078bSArd Biesheuvel const u8 *ad, const size_t ad_len,
114ed20078bSArd Biesheuvel const u8 nonce[XCHACHA20POLY1305_NONCE_SIZE],
115ed20078bSArd Biesheuvel const u8 key[CHACHA20POLY1305_KEY_SIZE])
116ed20078bSArd Biesheuvel {
117ed20078bSArd Biesheuvel u32 chacha_state[CHACHA_STATE_WORDS];
118ed20078bSArd Biesheuvel
119ed20078bSArd Biesheuvel xchacha_init(chacha_state, key, nonce);
120ed20078bSArd Biesheuvel __chacha20poly1305_encrypt(dst, src, src_len, ad, ad_len, chacha_state);
121ed20078bSArd Biesheuvel }
122ed20078bSArd Biesheuvel EXPORT_SYMBOL(xchacha20poly1305_encrypt);
123ed20078bSArd Biesheuvel
124ed20078bSArd Biesheuvel static bool
__chacha20poly1305_decrypt(u8 * dst,const u8 * src,const size_t src_len,const u8 * ad,const size_t ad_len,u32 * chacha_state)125ed20078bSArd Biesheuvel __chacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len,
126ed20078bSArd Biesheuvel const u8 *ad, const size_t ad_len, u32 *chacha_state)
127ed20078bSArd Biesheuvel {
128ed20078bSArd Biesheuvel const u8 *pad0 = page_address(ZERO_PAGE(0));
129ed20078bSArd Biesheuvel struct poly1305_desc_ctx poly1305_state;
130ed20078bSArd Biesheuvel size_t dst_len;
131ed20078bSArd Biesheuvel int ret;
132ed20078bSArd Biesheuvel union {
133ed20078bSArd Biesheuvel u8 block0[POLY1305_KEY_SIZE];
134ed20078bSArd Biesheuvel u8 mac[POLY1305_DIGEST_SIZE];
135ed20078bSArd Biesheuvel __le64 lens[2];
136ed20078bSArd Biesheuvel } b;
137ed20078bSArd Biesheuvel
138ed20078bSArd Biesheuvel if (unlikely(src_len < POLY1305_DIGEST_SIZE))
139ed20078bSArd Biesheuvel return false;
140ed20078bSArd Biesheuvel
141413808b7SEric Biggers chacha20_crypt(chacha_state, b.block0, pad0, sizeof(b.block0));
142ed20078bSArd Biesheuvel poly1305_init(&poly1305_state, b.block0);
143ed20078bSArd Biesheuvel
144ed20078bSArd Biesheuvel poly1305_update(&poly1305_state, ad, ad_len);
145ed20078bSArd Biesheuvel if (ad_len & 0xf)
146ed20078bSArd Biesheuvel poly1305_update(&poly1305_state, pad0, 0x10 - (ad_len & 0xf));
147ed20078bSArd Biesheuvel
148ed20078bSArd Biesheuvel dst_len = src_len - POLY1305_DIGEST_SIZE;
149ed20078bSArd Biesheuvel poly1305_update(&poly1305_state, src, dst_len);
150ed20078bSArd Biesheuvel if (dst_len & 0xf)
151ed20078bSArd Biesheuvel poly1305_update(&poly1305_state, pad0, 0x10 - (dst_len & 0xf));
152ed20078bSArd Biesheuvel
153ed20078bSArd Biesheuvel b.lens[0] = cpu_to_le64(ad_len);
154ed20078bSArd Biesheuvel b.lens[1] = cpu_to_le64(dst_len);
155ed20078bSArd Biesheuvel poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens));
156ed20078bSArd Biesheuvel
157ed20078bSArd Biesheuvel poly1305_final(&poly1305_state, b.mac);
158ed20078bSArd Biesheuvel
159ed20078bSArd Biesheuvel ret = crypto_memneq(b.mac, src + dst_len, POLY1305_DIGEST_SIZE);
160ed20078bSArd Biesheuvel if (likely(!ret))
161413808b7SEric Biggers chacha20_crypt(chacha_state, dst, src, dst_len);
162ed20078bSArd Biesheuvel
163ed20078bSArd Biesheuvel memzero_explicit(&b, sizeof(b));
164ed20078bSArd Biesheuvel
165ed20078bSArd Biesheuvel return !ret;
166ed20078bSArd Biesheuvel }
167ed20078bSArd Biesheuvel
chacha20poly1305_decrypt(u8 * dst,const u8 * src,const size_t src_len,const u8 * ad,const size_t ad_len,const u64 nonce,const u8 key[CHACHA20POLY1305_KEY_SIZE])168ed20078bSArd Biesheuvel bool chacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len,
169ed20078bSArd Biesheuvel const u8 *ad, const size_t ad_len,
170ed20078bSArd Biesheuvel const u64 nonce,
171ed20078bSArd Biesheuvel const u8 key[CHACHA20POLY1305_KEY_SIZE])
172ed20078bSArd Biesheuvel {
173ed20078bSArd Biesheuvel u32 chacha_state[CHACHA_STATE_WORDS];
174ed20078bSArd Biesheuvel u32 k[CHACHA_KEY_WORDS];
175ed20078bSArd Biesheuvel __le64 iv[2];
176ed20078bSArd Biesheuvel bool ret;
177ed20078bSArd Biesheuvel
178ed20078bSArd Biesheuvel chacha_load_key(k, key);
179ed20078bSArd Biesheuvel
180ed20078bSArd Biesheuvel iv[0] = 0;
181ed20078bSArd Biesheuvel iv[1] = cpu_to_le64(nonce);
182ed20078bSArd Biesheuvel
183ed20078bSArd Biesheuvel chacha_init(chacha_state, k, (u8 *)iv);
184ed20078bSArd Biesheuvel ret = __chacha20poly1305_decrypt(dst, src, src_len, ad, ad_len,
185ed20078bSArd Biesheuvel chacha_state);
186ed20078bSArd Biesheuvel
187ed20078bSArd Biesheuvel memzero_explicit(chacha_state, sizeof(chacha_state));
188ed20078bSArd Biesheuvel memzero_explicit(iv, sizeof(iv));
189ed20078bSArd Biesheuvel memzero_explicit(k, sizeof(k));
190ed20078bSArd Biesheuvel return ret;
191ed20078bSArd Biesheuvel }
192ed20078bSArd Biesheuvel EXPORT_SYMBOL(chacha20poly1305_decrypt);
193ed20078bSArd Biesheuvel
xchacha20poly1305_decrypt(u8 * dst,const u8 * src,const size_t src_len,const u8 * ad,const size_t ad_len,const u8 nonce[XCHACHA20POLY1305_NONCE_SIZE],const u8 key[CHACHA20POLY1305_KEY_SIZE])194ed20078bSArd Biesheuvel bool xchacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len,
195ed20078bSArd Biesheuvel const u8 *ad, const size_t ad_len,
196ed20078bSArd Biesheuvel const u8 nonce[XCHACHA20POLY1305_NONCE_SIZE],
197ed20078bSArd Biesheuvel const u8 key[CHACHA20POLY1305_KEY_SIZE])
198ed20078bSArd Biesheuvel {
199ed20078bSArd Biesheuvel u32 chacha_state[CHACHA_STATE_WORDS];
200ed20078bSArd Biesheuvel
201ed20078bSArd Biesheuvel xchacha_init(chacha_state, key, nonce);
202ed20078bSArd Biesheuvel return __chacha20poly1305_decrypt(dst, src, src_len, ad, ad_len,
203ed20078bSArd Biesheuvel chacha_state);
204ed20078bSArd Biesheuvel }
205ed20078bSArd Biesheuvel EXPORT_SYMBOL(xchacha20poly1305_decrypt);
206ed20078bSArd Biesheuvel
207d95312a3SArd Biesheuvel static
chacha20poly1305_crypt_sg_inplace(struct scatterlist * src,const size_t src_len,const u8 * ad,const size_t ad_len,const u64 nonce,const u8 key[CHACHA20POLY1305_KEY_SIZE],int encrypt)208d95312a3SArd Biesheuvel bool chacha20poly1305_crypt_sg_inplace(struct scatterlist *src,
209d95312a3SArd Biesheuvel const size_t src_len,
210d95312a3SArd Biesheuvel const u8 *ad, const size_t ad_len,
211d95312a3SArd Biesheuvel const u64 nonce,
212d95312a3SArd Biesheuvel const u8 key[CHACHA20POLY1305_KEY_SIZE],
213d95312a3SArd Biesheuvel int encrypt)
214d95312a3SArd Biesheuvel {
215d95312a3SArd Biesheuvel const u8 *pad0 = page_address(ZERO_PAGE(0));
216d95312a3SArd Biesheuvel struct poly1305_desc_ctx poly1305_state;
217d95312a3SArd Biesheuvel u32 chacha_state[CHACHA_STATE_WORDS];
218d95312a3SArd Biesheuvel struct sg_mapping_iter miter;
219d95312a3SArd Biesheuvel size_t partial = 0;
220d95312a3SArd Biesheuvel unsigned int flags;
221d95312a3SArd Biesheuvel bool ret = true;
222d95312a3SArd Biesheuvel int sl;
223d95312a3SArd Biesheuvel union {
224d95312a3SArd Biesheuvel struct {
225d95312a3SArd Biesheuvel u32 k[CHACHA_KEY_WORDS];
226d95312a3SArd Biesheuvel __le64 iv[2];
227d95312a3SArd Biesheuvel };
228d95312a3SArd Biesheuvel u8 block0[POLY1305_KEY_SIZE];
229d95312a3SArd Biesheuvel u8 chacha_stream[CHACHA_BLOCK_SIZE];
230d95312a3SArd Biesheuvel struct {
231d95312a3SArd Biesheuvel u8 mac[2][POLY1305_DIGEST_SIZE];
232d95312a3SArd Biesheuvel };
233d95312a3SArd Biesheuvel __le64 lens[2];
234d95312a3SArd Biesheuvel } b __aligned(16);
235d95312a3SArd Biesheuvel
236c9cc0517SJason A. Donenfeld if (WARN_ON(src_len > INT_MAX))
237c9cc0517SJason A. Donenfeld return false;
238c9cc0517SJason A. Donenfeld
239d95312a3SArd Biesheuvel chacha_load_key(b.k, key);
240d95312a3SArd Biesheuvel
241d95312a3SArd Biesheuvel b.iv[0] = 0;
242d95312a3SArd Biesheuvel b.iv[1] = cpu_to_le64(nonce);
243d95312a3SArd Biesheuvel
244d95312a3SArd Biesheuvel chacha_init(chacha_state, b.k, (u8 *)b.iv);
245413808b7SEric Biggers chacha20_crypt(chacha_state, b.block0, pad0, sizeof(b.block0));
246d95312a3SArd Biesheuvel poly1305_init(&poly1305_state, b.block0);
247d95312a3SArd Biesheuvel
248d95312a3SArd Biesheuvel if (unlikely(ad_len)) {
249d95312a3SArd Biesheuvel poly1305_update(&poly1305_state, ad, ad_len);
250d95312a3SArd Biesheuvel if (ad_len & 0xf)
251d95312a3SArd Biesheuvel poly1305_update(&poly1305_state, pad0, 0x10 - (ad_len & 0xf));
252d95312a3SArd Biesheuvel }
253d95312a3SArd Biesheuvel
254255f6c2eSHerbert Xu flags = SG_MITER_TO_SG | SG_MITER_ATOMIC;
255d95312a3SArd Biesheuvel
256d95312a3SArd Biesheuvel sg_miter_start(&miter, src, sg_nents(src), flags);
257d95312a3SArd Biesheuvel
258d95312a3SArd Biesheuvel for (sl = src_len; sl > 0 && sg_miter_next(&miter); sl -= miter.length) {
259d95312a3SArd Biesheuvel u8 *addr = miter.addr;
260d95312a3SArd Biesheuvel size_t length = min_t(size_t, sl, miter.length);
261d95312a3SArd Biesheuvel
262d95312a3SArd Biesheuvel if (!encrypt)
263d95312a3SArd Biesheuvel poly1305_update(&poly1305_state, addr, length);
264d95312a3SArd Biesheuvel
265d95312a3SArd Biesheuvel if (unlikely(partial)) {
266d95312a3SArd Biesheuvel size_t l = min(length, CHACHA_BLOCK_SIZE - partial);
267d95312a3SArd Biesheuvel
268d95312a3SArd Biesheuvel crypto_xor(addr, b.chacha_stream + partial, l);
269d95312a3SArd Biesheuvel partial = (partial + l) & (CHACHA_BLOCK_SIZE - 1);
270d95312a3SArd Biesheuvel
271d95312a3SArd Biesheuvel addr += l;
272d95312a3SArd Biesheuvel length -= l;
273d95312a3SArd Biesheuvel }
274d95312a3SArd Biesheuvel
275d95312a3SArd Biesheuvel if (likely(length >= CHACHA_BLOCK_SIZE || length == sl)) {
276d95312a3SArd Biesheuvel size_t l = length;
277d95312a3SArd Biesheuvel
278d95312a3SArd Biesheuvel if (unlikely(length < sl))
279d95312a3SArd Biesheuvel l &= ~(CHACHA_BLOCK_SIZE - 1);
280413808b7SEric Biggers chacha20_crypt(chacha_state, addr, addr, l);
281d95312a3SArd Biesheuvel addr += l;
282d95312a3SArd Biesheuvel length -= l;
283d95312a3SArd Biesheuvel }
284d95312a3SArd Biesheuvel
285d95312a3SArd Biesheuvel if (unlikely(length > 0)) {
286413808b7SEric Biggers chacha20_crypt(chacha_state, b.chacha_stream, pad0,
287413808b7SEric Biggers CHACHA_BLOCK_SIZE);
288d95312a3SArd Biesheuvel crypto_xor(addr, b.chacha_stream, length);
289d95312a3SArd Biesheuvel partial = length;
290d95312a3SArd Biesheuvel }
291d95312a3SArd Biesheuvel
292d95312a3SArd Biesheuvel if (encrypt)
293d95312a3SArd Biesheuvel poly1305_update(&poly1305_state, miter.addr,
294d95312a3SArd Biesheuvel min_t(size_t, sl, miter.length));
295d95312a3SArd Biesheuvel }
296d95312a3SArd Biesheuvel
297d95312a3SArd Biesheuvel if (src_len & 0xf)
298d95312a3SArd Biesheuvel poly1305_update(&poly1305_state, pad0, 0x10 - (src_len & 0xf));
299d95312a3SArd Biesheuvel
300d95312a3SArd Biesheuvel b.lens[0] = cpu_to_le64(ad_len);
301d95312a3SArd Biesheuvel b.lens[1] = cpu_to_le64(src_len);
302d95312a3SArd Biesheuvel poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens));
303d95312a3SArd Biesheuvel
304d95312a3SArd Biesheuvel if (likely(sl <= -POLY1305_DIGEST_SIZE)) {
305d95312a3SArd Biesheuvel if (encrypt) {
306d95312a3SArd Biesheuvel poly1305_final(&poly1305_state,
307d95312a3SArd Biesheuvel miter.addr + miter.length + sl);
308d95312a3SArd Biesheuvel ret = true;
309d95312a3SArd Biesheuvel } else {
310d95312a3SArd Biesheuvel poly1305_final(&poly1305_state, b.mac[0]);
311d95312a3SArd Biesheuvel ret = !crypto_memneq(b.mac[0],
312d95312a3SArd Biesheuvel miter.addr + miter.length + sl,
313d95312a3SArd Biesheuvel POLY1305_DIGEST_SIZE);
314d95312a3SArd Biesheuvel }
315d95312a3SArd Biesheuvel }
316d95312a3SArd Biesheuvel
317d95312a3SArd Biesheuvel sg_miter_stop(&miter);
318d95312a3SArd Biesheuvel
319d95312a3SArd Biesheuvel if (unlikely(sl > -POLY1305_DIGEST_SIZE)) {
320d95312a3SArd Biesheuvel poly1305_final(&poly1305_state, b.mac[1]);
321d95312a3SArd Biesheuvel scatterwalk_map_and_copy(b.mac[encrypt], src, src_len,
322d95312a3SArd Biesheuvel sizeof(b.mac[1]), encrypt);
323d95312a3SArd Biesheuvel ret = encrypt ||
324d95312a3SArd Biesheuvel !crypto_memneq(b.mac[0], b.mac[1], POLY1305_DIGEST_SIZE);
325d95312a3SArd Biesheuvel }
326d95312a3SArd Biesheuvel
327d95312a3SArd Biesheuvel memzero_explicit(chacha_state, sizeof(chacha_state));
328d95312a3SArd Biesheuvel memzero_explicit(&b, sizeof(b));
329d95312a3SArd Biesheuvel
330d95312a3SArd Biesheuvel return ret;
331d95312a3SArd Biesheuvel }
332d95312a3SArd Biesheuvel
chacha20poly1305_encrypt_sg_inplace(struct scatterlist * src,size_t src_len,const u8 * ad,const size_t ad_len,const u64 nonce,const u8 key[CHACHA20POLY1305_KEY_SIZE])333d95312a3SArd Biesheuvel bool chacha20poly1305_encrypt_sg_inplace(struct scatterlist *src, size_t src_len,
334d95312a3SArd Biesheuvel const u8 *ad, const size_t ad_len,
335d95312a3SArd Biesheuvel const u64 nonce,
336d95312a3SArd Biesheuvel const u8 key[CHACHA20POLY1305_KEY_SIZE])
337d95312a3SArd Biesheuvel {
338d95312a3SArd Biesheuvel return chacha20poly1305_crypt_sg_inplace(src, src_len, ad, ad_len,
339d95312a3SArd Biesheuvel nonce, key, 1);
340d95312a3SArd Biesheuvel }
341d95312a3SArd Biesheuvel EXPORT_SYMBOL(chacha20poly1305_encrypt_sg_inplace);
342d95312a3SArd Biesheuvel
chacha20poly1305_decrypt_sg_inplace(struct scatterlist * src,size_t src_len,const u8 * ad,const size_t ad_len,const u64 nonce,const u8 key[CHACHA20POLY1305_KEY_SIZE])343d95312a3SArd Biesheuvel bool chacha20poly1305_decrypt_sg_inplace(struct scatterlist *src, size_t src_len,
344d95312a3SArd Biesheuvel const u8 *ad, const size_t ad_len,
345d95312a3SArd Biesheuvel const u64 nonce,
346d95312a3SArd Biesheuvel const u8 key[CHACHA20POLY1305_KEY_SIZE])
347d95312a3SArd Biesheuvel {
348d95312a3SArd Biesheuvel if (unlikely(src_len < POLY1305_DIGEST_SIZE))
349d95312a3SArd Biesheuvel return false;
350d95312a3SArd Biesheuvel
351d95312a3SArd Biesheuvel return chacha20poly1305_crypt_sg_inplace(src,
352d95312a3SArd Biesheuvel src_len - POLY1305_DIGEST_SIZE,
353d95312a3SArd Biesheuvel ad, ad_len, nonce, key, 0);
354d95312a3SArd Biesheuvel }
355d95312a3SArd Biesheuvel EXPORT_SYMBOL(chacha20poly1305_decrypt_sg_inplace);
356d95312a3SArd Biesheuvel
chacha20poly1305_init(void)357*f03a3cabSRandy Dunlap static int __init chacha20poly1305_init(void)
358ed20078bSArd Biesheuvel {
359ed20078bSArd Biesheuvel if (!IS_ENABLED(CONFIG_CRYPTO_MANAGER_DISABLE_TESTS) &&
360ed20078bSArd Biesheuvel WARN_ON(!chacha20poly1305_selftest()))
361ed20078bSArd Biesheuvel return -ENODEV;
362ed20078bSArd Biesheuvel return 0;
363ed20078bSArd Biesheuvel }
364ed20078bSArd Biesheuvel
chacha20poly1305_exit(void)365*f03a3cabSRandy Dunlap static void __exit chacha20poly1305_exit(void)
366ac88c322SJason A. Donenfeld {
367ac88c322SJason A. Donenfeld }
368ac88c322SJason A. Donenfeld
369*f03a3cabSRandy Dunlap module_init(chacha20poly1305_init);
370*f03a3cabSRandy Dunlap module_exit(chacha20poly1305_exit);
371ed20078bSArd Biesheuvel MODULE_LICENSE("GPL v2");
372ed20078bSArd Biesheuvel MODULE_DESCRIPTION("ChaCha20Poly1305 AEAD construction");
373ed20078bSArd Biesheuvel MODULE_AUTHOR("Jason A. Donenfeld <Jason@zx2c4.com>");
374