1 // SPDX-License-Identifier: GPL-2.0 OR MIT 2 /* 3 * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. 4 * 5 * This is an implementation of the ChaCha20Poly1305 AEAD construction. 6 * 7 * Information: https://tools.ietf.org/html/rfc8439 8 */ 9 10 #include <crypto/algapi.h> 11 #include <crypto/chacha20poly1305.h> 12 #include <crypto/chacha.h> 13 #include <crypto/poly1305.h> 14 #include <crypto/scatterwalk.h> 15 16 #include <asm/unaligned.h> 17 #include <linux/kernel.h> 18 #include <linux/init.h> 19 #include <linux/mm.h> 20 #include <linux/module.h> 21 22 #define CHACHA_KEY_WORDS (CHACHA_KEY_SIZE / sizeof(u32)) 23 24 bool __init chacha20poly1305_selftest(void); 25 26 static void chacha_load_key(u32 *k, const u8 *in) 27 { 28 k[0] = get_unaligned_le32(in); 29 k[1] = get_unaligned_le32(in + 4); 30 k[2] = get_unaligned_le32(in + 8); 31 k[3] = get_unaligned_le32(in + 12); 32 k[4] = get_unaligned_le32(in + 16); 33 k[5] = get_unaligned_le32(in + 20); 34 k[6] = get_unaligned_le32(in + 24); 35 k[7] = get_unaligned_le32(in + 28); 36 } 37 38 static void xchacha_init(u32 *chacha_state, const u8 *key, const u8 *nonce) 39 { 40 u32 k[CHACHA_KEY_WORDS]; 41 u8 iv[CHACHA_IV_SIZE]; 42 43 memset(iv, 0, 8); 44 memcpy(iv + 8, nonce + 16, 8); 45 46 chacha_load_key(k, key); 47 48 /* Compute the subkey given the original key and first 128 nonce bits */ 49 chacha_init(chacha_state, k, nonce); 50 hchacha_block(chacha_state, k, 20); 51 52 chacha_init(chacha_state, k, iv); 53 54 memzero_explicit(k, sizeof(k)); 55 memzero_explicit(iv, sizeof(iv)); 56 } 57 58 static void 59 __chacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len, 60 const u8 *ad, const size_t ad_len, u32 *chacha_state) 61 { 62 const u8 *pad0 = page_address(ZERO_PAGE(0)); 63 struct poly1305_desc_ctx poly1305_state; 64 union { 65 u8 block0[POLY1305_KEY_SIZE]; 66 __le64 lens[2]; 67 } b; 68 69 chacha20_crypt(chacha_state, b.block0, pad0, sizeof(b.block0)); 70 poly1305_init(&poly1305_state, b.block0); 71 72 poly1305_update(&poly1305_state, ad, ad_len); 73 if (ad_len & 0xf) 74 poly1305_update(&poly1305_state, pad0, 0x10 - (ad_len & 0xf)); 75 76 chacha20_crypt(chacha_state, dst, src, src_len); 77 78 poly1305_update(&poly1305_state, dst, src_len); 79 if (src_len & 0xf) 80 poly1305_update(&poly1305_state, pad0, 0x10 - (src_len & 0xf)); 81 82 b.lens[0] = cpu_to_le64(ad_len); 83 b.lens[1] = cpu_to_le64(src_len); 84 poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens)); 85 86 poly1305_final(&poly1305_state, dst + src_len); 87 88 memzero_explicit(chacha_state, CHACHA_STATE_WORDS * sizeof(u32)); 89 memzero_explicit(&b, sizeof(b)); 90 } 91 92 void chacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len, 93 const u8 *ad, const size_t ad_len, 94 const u64 nonce, 95 const u8 key[CHACHA20POLY1305_KEY_SIZE]) 96 { 97 u32 chacha_state[CHACHA_STATE_WORDS]; 98 u32 k[CHACHA_KEY_WORDS]; 99 __le64 iv[2]; 100 101 chacha_load_key(k, key); 102 103 iv[0] = 0; 104 iv[1] = cpu_to_le64(nonce); 105 106 chacha_init(chacha_state, k, (u8 *)iv); 107 __chacha20poly1305_encrypt(dst, src, src_len, ad, ad_len, chacha_state); 108 109 memzero_explicit(iv, sizeof(iv)); 110 memzero_explicit(k, sizeof(k)); 111 } 112 EXPORT_SYMBOL(chacha20poly1305_encrypt); 113 114 void xchacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len, 115 const u8 *ad, const size_t ad_len, 116 const u8 nonce[XCHACHA20POLY1305_NONCE_SIZE], 117 const u8 key[CHACHA20POLY1305_KEY_SIZE]) 118 { 119 u32 chacha_state[CHACHA_STATE_WORDS]; 120 121 xchacha_init(chacha_state, key, nonce); 122 __chacha20poly1305_encrypt(dst, src, src_len, ad, ad_len, chacha_state); 123 } 124 EXPORT_SYMBOL(xchacha20poly1305_encrypt); 125 126 static bool 127 __chacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len, 128 const u8 *ad, const size_t ad_len, u32 *chacha_state) 129 { 130 const u8 *pad0 = page_address(ZERO_PAGE(0)); 131 struct poly1305_desc_ctx poly1305_state; 132 size_t dst_len; 133 int ret; 134 union { 135 u8 block0[POLY1305_KEY_SIZE]; 136 u8 mac[POLY1305_DIGEST_SIZE]; 137 __le64 lens[2]; 138 } b; 139 140 if (unlikely(src_len < POLY1305_DIGEST_SIZE)) 141 return false; 142 143 chacha20_crypt(chacha_state, b.block0, pad0, sizeof(b.block0)); 144 poly1305_init(&poly1305_state, b.block0); 145 146 poly1305_update(&poly1305_state, ad, ad_len); 147 if (ad_len & 0xf) 148 poly1305_update(&poly1305_state, pad0, 0x10 - (ad_len & 0xf)); 149 150 dst_len = src_len - POLY1305_DIGEST_SIZE; 151 poly1305_update(&poly1305_state, src, dst_len); 152 if (dst_len & 0xf) 153 poly1305_update(&poly1305_state, pad0, 0x10 - (dst_len & 0xf)); 154 155 b.lens[0] = cpu_to_le64(ad_len); 156 b.lens[1] = cpu_to_le64(dst_len); 157 poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens)); 158 159 poly1305_final(&poly1305_state, b.mac); 160 161 ret = crypto_memneq(b.mac, src + dst_len, POLY1305_DIGEST_SIZE); 162 if (likely(!ret)) 163 chacha20_crypt(chacha_state, dst, src, dst_len); 164 165 memzero_explicit(&b, sizeof(b)); 166 167 return !ret; 168 } 169 170 bool chacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len, 171 const u8 *ad, const size_t ad_len, 172 const u64 nonce, 173 const u8 key[CHACHA20POLY1305_KEY_SIZE]) 174 { 175 u32 chacha_state[CHACHA_STATE_WORDS]; 176 u32 k[CHACHA_KEY_WORDS]; 177 __le64 iv[2]; 178 bool ret; 179 180 chacha_load_key(k, key); 181 182 iv[0] = 0; 183 iv[1] = cpu_to_le64(nonce); 184 185 chacha_init(chacha_state, k, (u8 *)iv); 186 ret = __chacha20poly1305_decrypt(dst, src, src_len, ad, ad_len, 187 chacha_state); 188 189 memzero_explicit(chacha_state, sizeof(chacha_state)); 190 memzero_explicit(iv, sizeof(iv)); 191 memzero_explicit(k, sizeof(k)); 192 return ret; 193 } 194 EXPORT_SYMBOL(chacha20poly1305_decrypt); 195 196 bool xchacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len, 197 const u8 *ad, const size_t ad_len, 198 const u8 nonce[XCHACHA20POLY1305_NONCE_SIZE], 199 const u8 key[CHACHA20POLY1305_KEY_SIZE]) 200 { 201 u32 chacha_state[CHACHA_STATE_WORDS]; 202 203 xchacha_init(chacha_state, key, nonce); 204 return __chacha20poly1305_decrypt(dst, src, src_len, ad, ad_len, 205 chacha_state); 206 } 207 EXPORT_SYMBOL(xchacha20poly1305_decrypt); 208 209 static 210 bool chacha20poly1305_crypt_sg_inplace(struct scatterlist *src, 211 const size_t src_len, 212 const u8 *ad, const size_t ad_len, 213 const u64 nonce, 214 const u8 key[CHACHA20POLY1305_KEY_SIZE], 215 int encrypt) 216 { 217 const u8 *pad0 = page_address(ZERO_PAGE(0)); 218 struct poly1305_desc_ctx poly1305_state; 219 u32 chacha_state[CHACHA_STATE_WORDS]; 220 struct sg_mapping_iter miter; 221 size_t partial = 0; 222 unsigned int flags; 223 bool ret = true; 224 int sl; 225 union { 226 struct { 227 u32 k[CHACHA_KEY_WORDS]; 228 __le64 iv[2]; 229 }; 230 u8 block0[POLY1305_KEY_SIZE]; 231 u8 chacha_stream[CHACHA_BLOCK_SIZE]; 232 struct { 233 u8 mac[2][POLY1305_DIGEST_SIZE]; 234 }; 235 __le64 lens[2]; 236 } b __aligned(16); 237 238 chacha_load_key(b.k, key); 239 240 b.iv[0] = 0; 241 b.iv[1] = cpu_to_le64(nonce); 242 243 chacha_init(chacha_state, b.k, (u8 *)b.iv); 244 chacha20_crypt(chacha_state, b.block0, pad0, sizeof(b.block0)); 245 poly1305_init(&poly1305_state, b.block0); 246 247 if (unlikely(ad_len)) { 248 poly1305_update(&poly1305_state, ad, ad_len); 249 if (ad_len & 0xf) 250 poly1305_update(&poly1305_state, pad0, 0x10 - (ad_len & 0xf)); 251 } 252 253 flags = SG_MITER_TO_SG; 254 if (!preemptible()) 255 flags |= SG_MITER_ATOMIC; 256 257 sg_miter_start(&miter, src, sg_nents(src), flags); 258 259 for (sl = src_len; sl > 0 && sg_miter_next(&miter); sl -= miter.length) { 260 u8 *addr = miter.addr; 261 size_t length = min_t(size_t, sl, miter.length); 262 263 if (!encrypt) 264 poly1305_update(&poly1305_state, addr, length); 265 266 if (unlikely(partial)) { 267 size_t l = min(length, CHACHA_BLOCK_SIZE - partial); 268 269 crypto_xor(addr, b.chacha_stream + partial, l); 270 partial = (partial + l) & (CHACHA_BLOCK_SIZE - 1); 271 272 addr += l; 273 length -= l; 274 } 275 276 if (likely(length >= CHACHA_BLOCK_SIZE || length == sl)) { 277 size_t l = length; 278 279 if (unlikely(length < sl)) 280 l &= ~(CHACHA_BLOCK_SIZE - 1); 281 chacha20_crypt(chacha_state, addr, addr, l); 282 addr += l; 283 length -= l; 284 } 285 286 if (unlikely(length > 0)) { 287 chacha20_crypt(chacha_state, b.chacha_stream, pad0, 288 CHACHA_BLOCK_SIZE); 289 crypto_xor(addr, b.chacha_stream, length); 290 partial = length; 291 } 292 293 if (encrypt) 294 poly1305_update(&poly1305_state, miter.addr, 295 min_t(size_t, sl, miter.length)); 296 } 297 298 if (src_len & 0xf) 299 poly1305_update(&poly1305_state, pad0, 0x10 - (src_len & 0xf)); 300 301 b.lens[0] = cpu_to_le64(ad_len); 302 b.lens[1] = cpu_to_le64(src_len); 303 poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens)); 304 305 if (likely(sl <= -POLY1305_DIGEST_SIZE)) { 306 if (encrypt) { 307 poly1305_final(&poly1305_state, 308 miter.addr + miter.length + sl); 309 ret = true; 310 } else { 311 poly1305_final(&poly1305_state, b.mac[0]); 312 ret = !crypto_memneq(b.mac[0], 313 miter.addr + miter.length + sl, 314 POLY1305_DIGEST_SIZE); 315 } 316 } 317 318 sg_miter_stop(&miter); 319 320 if (unlikely(sl > -POLY1305_DIGEST_SIZE)) { 321 poly1305_final(&poly1305_state, b.mac[1]); 322 scatterwalk_map_and_copy(b.mac[encrypt], src, src_len, 323 sizeof(b.mac[1]), encrypt); 324 ret = encrypt || 325 !crypto_memneq(b.mac[0], b.mac[1], POLY1305_DIGEST_SIZE); 326 } 327 328 memzero_explicit(chacha_state, sizeof(chacha_state)); 329 memzero_explicit(&b, sizeof(b)); 330 331 return ret; 332 } 333 334 bool chacha20poly1305_encrypt_sg_inplace(struct scatterlist *src, size_t src_len, 335 const u8 *ad, const size_t ad_len, 336 const u64 nonce, 337 const u8 key[CHACHA20POLY1305_KEY_SIZE]) 338 { 339 return chacha20poly1305_crypt_sg_inplace(src, src_len, ad, ad_len, 340 nonce, key, 1); 341 } 342 EXPORT_SYMBOL(chacha20poly1305_encrypt_sg_inplace); 343 344 bool chacha20poly1305_decrypt_sg_inplace(struct scatterlist *src, size_t src_len, 345 const u8 *ad, const size_t ad_len, 346 const u64 nonce, 347 const u8 key[CHACHA20POLY1305_KEY_SIZE]) 348 { 349 if (unlikely(src_len < POLY1305_DIGEST_SIZE)) 350 return false; 351 352 return chacha20poly1305_crypt_sg_inplace(src, 353 src_len - POLY1305_DIGEST_SIZE, 354 ad, ad_len, nonce, key, 0); 355 } 356 EXPORT_SYMBOL(chacha20poly1305_decrypt_sg_inplace); 357 358 static int __init mod_init(void) 359 { 360 if (!IS_ENABLED(CONFIG_CRYPTO_MANAGER_DISABLE_TESTS) && 361 WARN_ON(!chacha20poly1305_selftest())) 362 return -ENODEV; 363 return 0; 364 } 365 366 module_init(mod_init); 367 MODULE_LICENSE("GPL v2"); 368 MODULE_DESCRIPTION("ChaCha20Poly1305 AEAD construction"); 369 MODULE_AUTHOR("Jason A. Donenfeld <Jason@zx2c4.com>"); 370