1 // SPDX-License-Identifier: GPL-2.0 OR MIT 2 /* 3 * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. 4 * 5 * This is an implementation of the ChaCha20Poly1305 AEAD construction. 6 * 7 * Information: https://tools.ietf.org/html/rfc8439 8 */ 9 10 #include <crypto/algapi.h> 11 #include <crypto/chacha20poly1305.h> 12 #include <crypto/chacha.h> 13 #include <crypto/poly1305.h> 14 #include <crypto/scatterwalk.h> 15 16 #include <asm/unaligned.h> 17 #include <linux/kernel.h> 18 #include <linux/init.h> 19 #include <linux/mm.h> 20 #include <linux/module.h> 21 22 #define CHACHA_KEY_WORDS (CHACHA_KEY_SIZE / sizeof(u32)) 23 24 static void chacha_load_key(u32 *k, const u8 *in) 25 { 26 k[0] = get_unaligned_le32(in); 27 k[1] = get_unaligned_le32(in + 4); 28 k[2] = get_unaligned_le32(in + 8); 29 k[3] = get_unaligned_le32(in + 12); 30 k[4] = get_unaligned_le32(in + 16); 31 k[5] = get_unaligned_le32(in + 20); 32 k[6] = get_unaligned_le32(in + 24); 33 k[7] = get_unaligned_le32(in + 28); 34 } 35 36 static void xchacha_init(u32 *chacha_state, const u8 *key, const u8 *nonce) 37 { 38 u32 k[CHACHA_KEY_WORDS]; 39 u8 iv[CHACHA_IV_SIZE]; 40 41 memset(iv, 0, 8); 42 memcpy(iv + 8, nonce + 16, 8); 43 44 chacha_load_key(k, key); 45 46 /* Compute the subkey given the original key and first 128 nonce bits */ 47 chacha_init(chacha_state, k, nonce); 48 hchacha_block(chacha_state, k, 20); 49 50 chacha_init(chacha_state, k, iv); 51 52 memzero_explicit(k, sizeof(k)); 53 memzero_explicit(iv, sizeof(iv)); 54 } 55 56 static void 57 __chacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len, 58 const u8 *ad, const size_t ad_len, u32 *chacha_state) 59 { 60 const u8 *pad0 = page_address(ZERO_PAGE(0)); 61 struct poly1305_desc_ctx poly1305_state; 62 union { 63 u8 block0[POLY1305_KEY_SIZE]; 64 __le64 lens[2]; 65 } b; 66 67 chacha20_crypt(chacha_state, b.block0, pad0, sizeof(b.block0)); 68 poly1305_init(&poly1305_state, b.block0); 69 70 poly1305_update(&poly1305_state, ad, ad_len); 71 if (ad_len & 0xf) 72 poly1305_update(&poly1305_state, pad0, 0x10 - (ad_len & 0xf)); 73 74 chacha20_crypt(chacha_state, dst, src, src_len); 75 76 poly1305_update(&poly1305_state, dst, src_len); 77 if (src_len & 0xf) 78 poly1305_update(&poly1305_state, pad0, 0x10 - (src_len & 0xf)); 79 80 b.lens[0] = cpu_to_le64(ad_len); 81 b.lens[1] = cpu_to_le64(src_len); 82 poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens)); 83 84 poly1305_final(&poly1305_state, dst + src_len); 85 86 memzero_explicit(chacha_state, CHACHA_STATE_WORDS * sizeof(u32)); 87 memzero_explicit(&b, sizeof(b)); 88 } 89 90 void chacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len, 91 const u8 *ad, const size_t ad_len, 92 const u64 nonce, 93 const u8 key[CHACHA20POLY1305_KEY_SIZE]) 94 { 95 u32 chacha_state[CHACHA_STATE_WORDS]; 96 u32 k[CHACHA_KEY_WORDS]; 97 __le64 iv[2]; 98 99 chacha_load_key(k, key); 100 101 iv[0] = 0; 102 iv[1] = cpu_to_le64(nonce); 103 104 chacha_init(chacha_state, k, (u8 *)iv); 105 __chacha20poly1305_encrypt(dst, src, src_len, ad, ad_len, chacha_state); 106 107 memzero_explicit(iv, sizeof(iv)); 108 memzero_explicit(k, sizeof(k)); 109 } 110 EXPORT_SYMBOL(chacha20poly1305_encrypt); 111 112 void xchacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len, 113 const u8 *ad, const size_t ad_len, 114 const u8 nonce[XCHACHA20POLY1305_NONCE_SIZE], 115 const u8 key[CHACHA20POLY1305_KEY_SIZE]) 116 { 117 u32 chacha_state[CHACHA_STATE_WORDS]; 118 119 xchacha_init(chacha_state, key, nonce); 120 __chacha20poly1305_encrypt(dst, src, src_len, ad, ad_len, chacha_state); 121 } 122 EXPORT_SYMBOL(xchacha20poly1305_encrypt); 123 124 static bool 125 __chacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len, 126 const u8 *ad, const size_t ad_len, u32 *chacha_state) 127 { 128 const u8 *pad0 = page_address(ZERO_PAGE(0)); 129 struct poly1305_desc_ctx poly1305_state; 130 size_t dst_len; 131 int ret; 132 union { 133 u8 block0[POLY1305_KEY_SIZE]; 134 u8 mac[POLY1305_DIGEST_SIZE]; 135 __le64 lens[2]; 136 } b; 137 138 if (unlikely(src_len < POLY1305_DIGEST_SIZE)) 139 return false; 140 141 chacha20_crypt(chacha_state, b.block0, pad0, sizeof(b.block0)); 142 poly1305_init(&poly1305_state, b.block0); 143 144 poly1305_update(&poly1305_state, ad, ad_len); 145 if (ad_len & 0xf) 146 poly1305_update(&poly1305_state, pad0, 0x10 - (ad_len & 0xf)); 147 148 dst_len = src_len - POLY1305_DIGEST_SIZE; 149 poly1305_update(&poly1305_state, src, dst_len); 150 if (dst_len & 0xf) 151 poly1305_update(&poly1305_state, pad0, 0x10 - (dst_len & 0xf)); 152 153 b.lens[0] = cpu_to_le64(ad_len); 154 b.lens[1] = cpu_to_le64(dst_len); 155 poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens)); 156 157 poly1305_final(&poly1305_state, b.mac); 158 159 ret = crypto_memneq(b.mac, src + dst_len, POLY1305_DIGEST_SIZE); 160 if (likely(!ret)) 161 chacha20_crypt(chacha_state, dst, src, dst_len); 162 163 memzero_explicit(&b, sizeof(b)); 164 165 return !ret; 166 } 167 168 bool chacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len, 169 const u8 *ad, const size_t ad_len, 170 const u64 nonce, 171 const u8 key[CHACHA20POLY1305_KEY_SIZE]) 172 { 173 u32 chacha_state[CHACHA_STATE_WORDS]; 174 u32 k[CHACHA_KEY_WORDS]; 175 __le64 iv[2]; 176 bool ret; 177 178 chacha_load_key(k, key); 179 180 iv[0] = 0; 181 iv[1] = cpu_to_le64(nonce); 182 183 chacha_init(chacha_state, k, (u8 *)iv); 184 ret = __chacha20poly1305_decrypt(dst, src, src_len, ad, ad_len, 185 chacha_state); 186 187 memzero_explicit(chacha_state, sizeof(chacha_state)); 188 memzero_explicit(iv, sizeof(iv)); 189 memzero_explicit(k, sizeof(k)); 190 return ret; 191 } 192 EXPORT_SYMBOL(chacha20poly1305_decrypt); 193 194 bool xchacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len, 195 const u8 *ad, const size_t ad_len, 196 const u8 nonce[XCHACHA20POLY1305_NONCE_SIZE], 197 const u8 key[CHACHA20POLY1305_KEY_SIZE]) 198 { 199 u32 chacha_state[CHACHA_STATE_WORDS]; 200 201 xchacha_init(chacha_state, key, nonce); 202 return __chacha20poly1305_decrypt(dst, src, src_len, ad, ad_len, 203 chacha_state); 204 } 205 EXPORT_SYMBOL(xchacha20poly1305_decrypt); 206 207 static 208 bool chacha20poly1305_crypt_sg_inplace(struct scatterlist *src, 209 const size_t src_len, 210 const u8 *ad, const size_t ad_len, 211 const u64 nonce, 212 const u8 key[CHACHA20POLY1305_KEY_SIZE], 213 int encrypt) 214 { 215 const u8 *pad0 = page_address(ZERO_PAGE(0)); 216 struct poly1305_desc_ctx poly1305_state; 217 u32 chacha_state[CHACHA_STATE_WORDS]; 218 struct sg_mapping_iter miter; 219 size_t partial = 0; 220 unsigned int flags; 221 bool ret = true; 222 int sl; 223 union { 224 struct { 225 u32 k[CHACHA_KEY_WORDS]; 226 __le64 iv[2]; 227 }; 228 u8 block0[POLY1305_KEY_SIZE]; 229 u8 chacha_stream[CHACHA_BLOCK_SIZE]; 230 struct { 231 u8 mac[2][POLY1305_DIGEST_SIZE]; 232 }; 233 __le64 lens[2]; 234 } b __aligned(16); 235 236 if (WARN_ON(src_len > INT_MAX)) 237 return false; 238 239 chacha_load_key(b.k, key); 240 241 b.iv[0] = 0; 242 b.iv[1] = cpu_to_le64(nonce); 243 244 chacha_init(chacha_state, b.k, (u8 *)b.iv); 245 chacha20_crypt(chacha_state, b.block0, pad0, sizeof(b.block0)); 246 poly1305_init(&poly1305_state, b.block0); 247 248 if (unlikely(ad_len)) { 249 poly1305_update(&poly1305_state, ad, ad_len); 250 if (ad_len & 0xf) 251 poly1305_update(&poly1305_state, pad0, 0x10 - (ad_len & 0xf)); 252 } 253 254 flags = SG_MITER_TO_SG | SG_MITER_ATOMIC; 255 256 sg_miter_start(&miter, src, sg_nents(src), flags); 257 258 for (sl = src_len; sl > 0 && sg_miter_next(&miter); sl -= miter.length) { 259 u8 *addr = miter.addr; 260 size_t length = min_t(size_t, sl, miter.length); 261 262 if (!encrypt) 263 poly1305_update(&poly1305_state, addr, length); 264 265 if (unlikely(partial)) { 266 size_t l = min(length, CHACHA_BLOCK_SIZE - partial); 267 268 crypto_xor(addr, b.chacha_stream + partial, l); 269 partial = (partial + l) & (CHACHA_BLOCK_SIZE - 1); 270 271 addr += l; 272 length -= l; 273 } 274 275 if (likely(length >= CHACHA_BLOCK_SIZE || length == sl)) { 276 size_t l = length; 277 278 if (unlikely(length < sl)) 279 l &= ~(CHACHA_BLOCK_SIZE - 1); 280 chacha20_crypt(chacha_state, addr, addr, l); 281 addr += l; 282 length -= l; 283 } 284 285 if (unlikely(length > 0)) { 286 chacha20_crypt(chacha_state, b.chacha_stream, pad0, 287 CHACHA_BLOCK_SIZE); 288 crypto_xor(addr, b.chacha_stream, length); 289 partial = length; 290 } 291 292 if (encrypt) 293 poly1305_update(&poly1305_state, miter.addr, 294 min_t(size_t, sl, miter.length)); 295 } 296 297 if (src_len & 0xf) 298 poly1305_update(&poly1305_state, pad0, 0x10 - (src_len & 0xf)); 299 300 b.lens[0] = cpu_to_le64(ad_len); 301 b.lens[1] = cpu_to_le64(src_len); 302 poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens)); 303 304 if (likely(sl <= -POLY1305_DIGEST_SIZE)) { 305 if (encrypt) { 306 poly1305_final(&poly1305_state, 307 miter.addr + miter.length + sl); 308 ret = true; 309 } else { 310 poly1305_final(&poly1305_state, b.mac[0]); 311 ret = !crypto_memneq(b.mac[0], 312 miter.addr + miter.length + sl, 313 POLY1305_DIGEST_SIZE); 314 } 315 } 316 317 sg_miter_stop(&miter); 318 319 if (unlikely(sl > -POLY1305_DIGEST_SIZE)) { 320 poly1305_final(&poly1305_state, b.mac[1]); 321 scatterwalk_map_and_copy(b.mac[encrypt], src, src_len, 322 sizeof(b.mac[1]), encrypt); 323 ret = encrypt || 324 !crypto_memneq(b.mac[0], b.mac[1], POLY1305_DIGEST_SIZE); 325 } 326 327 memzero_explicit(chacha_state, sizeof(chacha_state)); 328 memzero_explicit(&b, sizeof(b)); 329 330 return ret; 331 } 332 333 bool chacha20poly1305_encrypt_sg_inplace(struct scatterlist *src, size_t src_len, 334 const u8 *ad, const size_t ad_len, 335 const u64 nonce, 336 const u8 key[CHACHA20POLY1305_KEY_SIZE]) 337 { 338 return chacha20poly1305_crypt_sg_inplace(src, src_len, ad, ad_len, 339 nonce, key, 1); 340 } 341 EXPORT_SYMBOL(chacha20poly1305_encrypt_sg_inplace); 342 343 bool chacha20poly1305_decrypt_sg_inplace(struct scatterlist *src, size_t src_len, 344 const u8 *ad, const size_t ad_len, 345 const u64 nonce, 346 const u8 key[CHACHA20POLY1305_KEY_SIZE]) 347 { 348 if (unlikely(src_len < POLY1305_DIGEST_SIZE)) 349 return false; 350 351 return chacha20poly1305_crypt_sg_inplace(src, 352 src_len - POLY1305_DIGEST_SIZE, 353 ad, ad_len, nonce, key, 0); 354 } 355 EXPORT_SYMBOL(chacha20poly1305_decrypt_sg_inplace); 356 357 static int __init chacha20poly1305_init(void) 358 { 359 if (!IS_ENABLED(CONFIG_CRYPTO_MANAGER_DISABLE_TESTS) && 360 WARN_ON(!chacha20poly1305_selftest())) 361 return -ENODEV; 362 return 0; 363 } 364 365 static void __exit chacha20poly1305_exit(void) 366 { 367 } 368 369 module_init(chacha20poly1305_init); 370 module_exit(chacha20poly1305_exit); 371 MODULE_LICENSE("GPL v2"); 372 MODULE_DESCRIPTION("ChaCha20Poly1305 AEAD construction"); 373 MODULE_AUTHOR("Jason A. Donenfeld <Jason@zx2c4.com>"); 374