1 // SPDX-License-Identifier: GPL-2.0 OR MIT 2 /* 3 * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. 4 * 5 * This is an implementation of the ChaCha20Poly1305 AEAD construction. 6 * 7 * Information: https://tools.ietf.org/html/rfc8439 8 */ 9 10 #include <crypto/algapi.h> 11 #include <crypto/chacha20poly1305.h> 12 #include <crypto/chacha.h> 13 #include <crypto/poly1305.h> 14 #include <crypto/scatterwalk.h> 15 16 #include <asm/unaligned.h> 17 #include <linux/kernel.h> 18 #include <linux/init.h> 19 #include <linux/mm.h> 20 #include <linux/module.h> 21 22 #define CHACHA_KEY_WORDS (CHACHA_KEY_SIZE / sizeof(u32)) 23 24 bool __init chacha20poly1305_selftest(void); 25 26 static void chacha_load_key(u32 *k, const u8 *in) 27 { 28 k[0] = get_unaligned_le32(in); 29 k[1] = get_unaligned_le32(in + 4); 30 k[2] = get_unaligned_le32(in + 8); 31 k[3] = get_unaligned_le32(in + 12); 32 k[4] = get_unaligned_le32(in + 16); 33 k[5] = get_unaligned_le32(in + 20); 34 k[6] = get_unaligned_le32(in + 24); 35 k[7] = get_unaligned_le32(in + 28); 36 } 37 38 static void xchacha_init(u32 *chacha_state, const u8 *key, const u8 *nonce) 39 { 40 u32 k[CHACHA_KEY_WORDS]; 41 u8 iv[CHACHA_IV_SIZE]; 42 43 memset(iv, 0, 8); 44 memcpy(iv + 8, nonce + 16, 8); 45 46 chacha_load_key(k, key); 47 48 /* Compute the subkey given the original key and first 128 nonce bits */ 49 chacha_init(chacha_state, k, nonce); 50 hchacha_block(chacha_state, k, 20); 51 52 chacha_init(chacha_state, k, iv); 53 54 memzero_explicit(k, sizeof(k)); 55 memzero_explicit(iv, sizeof(iv)); 56 } 57 58 static void 59 __chacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len, 60 const u8 *ad, const size_t ad_len, u32 *chacha_state) 61 { 62 const u8 *pad0 = page_address(ZERO_PAGE(0)); 63 struct poly1305_desc_ctx poly1305_state; 64 union { 65 u8 block0[POLY1305_KEY_SIZE]; 66 __le64 lens[2]; 67 } b; 68 69 chacha20_crypt(chacha_state, b.block0, pad0, sizeof(b.block0)); 70 poly1305_init(&poly1305_state, b.block0); 71 72 poly1305_update(&poly1305_state, ad, ad_len); 73 if (ad_len & 0xf) 74 poly1305_update(&poly1305_state, pad0, 0x10 - (ad_len & 0xf)); 75 76 chacha20_crypt(chacha_state, dst, src, src_len); 77 78 poly1305_update(&poly1305_state, dst, src_len); 79 if (src_len & 0xf) 80 poly1305_update(&poly1305_state, pad0, 0x10 - (src_len & 0xf)); 81 82 b.lens[0] = cpu_to_le64(ad_len); 83 b.lens[1] = cpu_to_le64(src_len); 84 poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens)); 85 86 poly1305_final(&poly1305_state, dst + src_len); 87 88 memzero_explicit(chacha_state, CHACHA_STATE_WORDS * sizeof(u32)); 89 memzero_explicit(&b, sizeof(b)); 90 } 91 92 void chacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len, 93 const u8 *ad, const size_t ad_len, 94 const u64 nonce, 95 const u8 key[CHACHA20POLY1305_KEY_SIZE]) 96 { 97 u32 chacha_state[CHACHA_STATE_WORDS]; 98 u32 k[CHACHA_KEY_WORDS]; 99 __le64 iv[2]; 100 101 chacha_load_key(k, key); 102 103 iv[0] = 0; 104 iv[1] = cpu_to_le64(nonce); 105 106 chacha_init(chacha_state, k, (u8 *)iv); 107 __chacha20poly1305_encrypt(dst, src, src_len, ad, ad_len, chacha_state); 108 109 memzero_explicit(iv, sizeof(iv)); 110 memzero_explicit(k, sizeof(k)); 111 } 112 EXPORT_SYMBOL(chacha20poly1305_encrypt); 113 114 void xchacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len, 115 const u8 *ad, const size_t ad_len, 116 const u8 nonce[XCHACHA20POLY1305_NONCE_SIZE], 117 const u8 key[CHACHA20POLY1305_KEY_SIZE]) 118 { 119 u32 chacha_state[CHACHA_STATE_WORDS]; 120 121 xchacha_init(chacha_state, key, nonce); 122 __chacha20poly1305_encrypt(dst, src, src_len, ad, ad_len, chacha_state); 123 } 124 EXPORT_SYMBOL(xchacha20poly1305_encrypt); 125 126 static bool 127 __chacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len, 128 const u8 *ad, const size_t ad_len, u32 *chacha_state) 129 { 130 const u8 *pad0 = page_address(ZERO_PAGE(0)); 131 struct poly1305_desc_ctx poly1305_state; 132 size_t dst_len; 133 int ret; 134 union { 135 u8 block0[POLY1305_KEY_SIZE]; 136 u8 mac[POLY1305_DIGEST_SIZE]; 137 __le64 lens[2]; 138 } b; 139 140 if (unlikely(src_len < POLY1305_DIGEST_SIZE)) 141 return false; 142 143 chacha20_crypt(chacha_state, b.block0, pad0, sizeof(b.block0)); 144 poly1305_init(&poly1305_state, b.block0); 145 146 poly1305_update(&poly1305_state, ad, ad_len); 147 if (ad_len & 0xf) 148 poly1305_update(&poly1305_state, pad0, 0x10 - (ad_len & 0xf)); 149 150 dst_len = src_len - POLY1305_DIGEST_SIZE; 151 poly1305_update(&poly1305_state, src, dst_len); 152 if (dst_len & 0xf) 153 poly1305_update(&poly1305_state, pad0, 0x10 - (dst_len & 0xf)); 154 155 b.lens[0] = cpu_to_le64(ad_len); 156 b.lens[1] = cpu_to_le64(dst_len); 157 poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens)); 158 159 poly1305_final(&poly1305_state, b.mac); 160 161 ret = crypto_memneq(b.mac, src + dst_len, POLY1305_DIGEST_SIZE); 162 if (likely(!ret)) 163 chacha20_crypt(chacha_state, dst, src, dst_len); 164 165 memzero_explicit(&b, sizeof(b)); 166 167 return !ret; 168 } 169 170 bool chacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len, 171 const u8 *ad, const size_t ad_len, 172 const u64 nonce, 173 const u8 key[CHACHA20POLY1305_KEY_SIZE]) 174 { 175 u32 chacha_state[CHACHA_STATE_WORDS]; 176 u32 k[CHACHA_KEY_WORDS]; 177 __le64 iv[2]; 178 bool ret; 179 180 chacha_load_key(k, key); 181 182 iv[0] = 0; 183 iv[1] = cpu_to_le64(nonce); 184 185 chacha_init(chacha_state, k, (u8 *)iv); 186 ret = __chacha20poly1305_decrypt(dst, src, src_len, ad, ad_len, 187 chacha_state); 188 189 memzero_explicit(chacha_state, sizeof(chacha_state)); 190 memzero_explicit(iv, sizeof(iv)); 191 memzero_explicit(k, sizeof(k)); 192 return ret; 193 } 194 EXPORT_SYMBOL(chacha20poly1305_decrypt); 195 196 bool xchacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len, 197 const u8 *ad, const size_t ad_len, 198 const u8 nonce[XCHACHA20POLY1305_NONCE_SIZE], 199 const u8 key[CHACHA20POLY1305_KEY_SIZE]) 200 { 201 u32 chacha_state[CHACHA_STATE_WORDS]; 202 203 xchacha_init(chacha_state, key, nonce); 204 return __chacha20poly1305_decrypt(dst, src, src_len, ad, ad_len, 205 chacha_state); 206 } 207 EXPORT_SYMBOL(xchacha20poly1305_decrypt); 208 209 static 210 bool chacha20poly1305_crypt_sg_inplace(struct scatterlist *src, 211 const size_t src_len, 212 const u8 *ad, const size_t ad_len, 213 const u64 nonce, 214 const u8 key[CHACHA20POLY1305_KEY_SIZE], 215 int encrypt) 216 { 217 const u8 *pad0 = page_address(ZERO_PAGE(0)); 218 struct poly1305_desc_ctx poly1305_state; 219 u32 chacha_state[CHACHA_STATE_WORDS]; 220 struct sg_mapping_iter miter; 221 size_t partial = 0; 222 unsigned int flags; 223 bool ret = true; 224 int sl; 225 union { 226 struct { 227 u32 k[CHACHA_KEY_WORDS]; 228 __le64 iv[2]; 229 }; 230 u8 block0[POLY1305_KEY_SIZE]; 231 u8 chacha_stream[CHACHA_BLOCK_SIZE]; 232 struct { 233 u8 mac[2][POLY1305_DIGEST_SIZE]; 234 }; 235 __le64 lens[2]; 236 } b __aligned(16); 237 238 if (WARN_ON(src_len > INT_MAX)) 239 return false; 240 241 chacha_load_key(b.k, key); 242 243 b.iv[0] = 0; 244 b.iv[1] = cpu_to_le64(nonce); 245 246 chacha_init(chacha_state, b.k, (u8 *)b.iv); 247 chacha20_crypt(chacha_state, b.block0, pad0, sizeof(b.block0)); 248 poly1305_init(&poly1305_state, b.block0); 249 250 if (unlikely(ad_len)) { 251 poly1305_update(&poly1305_state, ad, ad_len); 252 if (ad_len & 0xf) 253 poly1305_update(&poly1305_state, pad0, 0x10 - (ad_len & 0xf)); 254 } 255 256 flags = SG_MITER_TO_SG; 257 if (!preemptible()) 258 flags |= SG_MITER_ATOMIC; 259 260 sg_miter_start(&miter, src, sg_nents(src), flags); 261 262 for (sl = src_len; sl > 0 && sg_miter_next(&miter); sl -= miter.length) { 263 u8 *addr = miter.addr; 264 size_t length = min_t(size_t, sl, miter.length); 265 266 if (!encrypt) 267 poly1305_update(&poly1305_state, addr, length); 268 269 if (unlikely(partial)) { 270 size_t l = min(length, CHACHA_BLOCK_SIZE - partial); 271 272 crypto_xor(addr, b.chacha_stream + partial, l); 273 partial = (partial + l) & (CHACHA_BLOCK_SIZE - 1); 274 275 addr += l; 276 length -= l; 277 } 278 279 if (likely(length >= CHACHA_BLOCK_SIZE || length == sl)) { 280 size_t l = length; 281 282 if (unlikely(length < sl)) 283 l &= ~(CHACHA_BLOCK_SIZE - 1); 284 chacha20_crypt(chacha_state, addr, addr, l); 285 addr += l; 286 length -= l; 287 } 288 289 if (unlikely(length > 0)) { 290 chacha20_crypt(chacha_state, b.chacha_stream, pad0, 291 CHACHA_BLOCK_SIZE); 292 crypto_xor(addr, b.chacha_stream, length); 293 partial = length; 294 } 295 296 if (encrypt) 297 poly1305_update(&poly1305_state, miter.addr, 298 min_t(size_t, sl, miter.length)); 299 } 300 301 if (src_len & 0xf) 302 poly1305_update(&poly1305_state, pad0, 0x10 - (src_len & 0xf)); 303 304 b.lens[0] = cpu_to_le64(ad_len); 305 b.lens[1] = cpu_to_le64(src_len); 306 poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens)); 307 308 if (likely(sl <= -POLY1305_DIGEST_SIZE)) { 309 if (encrypt) { 310 poly1305_final(&poly1305_state, 311 miter.addr + miter.length + sl); 312 ret = true; 313 } else { 314 poly1305_final(&poly1305_state, b.mac[0]); 315 ret = !crypto_memneq(b.mac[0], 316 miter.addr + miter.length + sl, 317 POLY1305_DIGEST_SIZE); 318 } 319 } 320 321 sg_miter_stop(&miter); 322 323 if (unlikely(sl > -POLY1305_DIGEST_SIZE)) { 324 poly1305_final(&poly1305_state, b.mac[1]); 325 scatterwalk_map_and_copy(b.mac[encrypt], src, src_len, 326 sizeof(b.mac[1]), encrypt); 327 ret = encrypt || 328 !crypto_memneq(b.mac[0], b.mac[1], POLY1305_DIGEST_SIZE); 329 } 330 331 memzero_explicit(chacha_state, sizeof(chacha_state)); 332 memzero_explicit(&b, sizeof(b)); 333 334 return ret; 335 } 336 337 bool chacha20poly1305_encrypt_sg_inplace(struct scatterlist *src, size_t src_len, 338 const u8 *ad, const size_t ad_len, 339 const u64 nonce, 340 const u8 key[CHACHA20POLY1305_KEY_SIZE]) 341 { 342 return chacha20poly1305_crypt_sg_inplace(src, src_len, ad, ad_len, 343 nonce, key, 1); 344 } 345 EXPORT_SYMBOL(chacha20poly1305_encrypt_sg_inplace); 346 347 bool chacha20poly1305_decrypt_sg_inplace(struct scatterlist *src, size_t src_len, 348 const u8 *ad, const size_t ad_len, 349 const u64 nonce, 350 const u8 key[CHACHA20POLY1305_KEY_SIZE]) 351 { 352 if (unlikely(src_len < POLY1305_DIGEST_SIZE)) 353 return false; 354 355 return chacha20poly1305_crypt_sg_inplace(src, 356 src_len - POLY1305_DIGEST_SIZE, 357 ad, ad_len, nonce, key, 0); 358 } 359 EXPORT_SYMBOL(chacha20poly1305_decrypt_sg_inplace); 360 361 static int __init mod_init(void) 362 { 363 if (!IS_ENABLED(CONFIG_CRYPTO_MANAGER_DISABLE_TESTS) && 364 WARN_ON(!chacha20poly1305_selftest())) 365 return -ENODEV; 366 return 0; 367 } 368 369 module_init(mod_init); 370 MODULE_LICENSE("GPL v2"); 371 MODULE_DESCRIPTION("ChaCha20Poly1305 AEAD construction"); 372 MODULE_AUTHOR("Jason A. Donenfeld <Jason@zx2c4.com>"); 373