1 // SPDX-License-Identifier: GPL-2.0 OR MIT 2 /* 3 * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. 4 * 5 * This is an implementation of the ChaCha20Poly1305 AEAD construction. 6 * 7 * Information: https://tools.ietf.org/html/rfc8439 8 */ 9 10 #include <crypto/algapi.h> 11 #include <crypto/chacha20poly1305.h> 12 #include <crypto/chacha.h> 13 #include <crypto/poly1305.h> 14 #include <crypto/scatterwalk.h> 15 16 #include <asm/unaligned.h> 17 #include <linux/kernel.h> 18 #include <linux/init.h> 19 #include <linux/mm.h> 20 #include <linux/module.h> 21 22 #define CHACHA_KEY_WORDS (CHACHA_KEY_SIZE / sizeof(u32)) 23 24 static void chacha_load_key(u32 *k, const u8 *in) 25 { 26 k[0] = get_unaligned_le32(in); 27 k[1] = get_unaligned_le32(in + 4); 28 k[2] = get_unaligned_le32(in + 8); 29 k[3] = get_unaligned_le32(in + 12); 30 k[4] = get_unaligned_le32(in + 16); 31 k[5] = get_unaligned_le32(in + 20); 32 k[6] = get_unaligned_le32(in + 24); 33 k[7] = get_unaligned_le32(in + 28); 34 } 35 36 static void xchacha_init(u32 *chacha_state, const u8 *key, const u8 *nonce) 37 { 38 u32 k[CHACHA_KEY_WORDS]; 39 u8 iv[CHACHA_IV_SIZE]; 40 41 memset(iv, 0, 8); 42 memcpy(iv + 8, nonce + 16, 8); 43 44 chacha_load_key(k, key); 45 46 /* Compute the subkey given the original key and first 128 nonce bits */ 47 chacha_init(chacha_state, k, nonce); 48 hchacha_block(chacha_state, k, 20); 49 50 chacha_init(chacha_state, k, iv); 51 52 memzero_explicit(k, sizeof(k)); 53 memzero_explicit(iv, sizeof(iv)); 54 } 55 56 static void 57 __chacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len, 58 const u8 *ad, const size_t ad_len, u32 *chacha_state) 59 { 60 const u8 *pad0 = page_address(ZERO_PAGE(0)); 61 struct poly1305_desc_ctx poly1305_state; 62 union { 63 u8 block0[POLY1305_KEY_SIZE]; 64 __le64 lens[2]; 65 } b; 66 67 chacha20_crypt(chacha_state, b.block0, pad0, sizeof(b.block0)); 68 poly1305_init(&poly1305_state, b.block0); 69 70 poly1305_update(&poly1305_state, ad, ad_len); 71 if (ad_len & 0xf) 72 poly1305_update(&poly1305_state, pad0, 0x10 - (ad_len & 0xf)); 73 74 chacha20_crypt(chacha_state, dst, src, src_len); 75 76 poly1305_update(&poly1305_state, dst, src_len); 77 if (src_len & 0xf) 78 poly1305_update(&poly1305_state, pad0, 0x10 - (src_len & 0xf)); 79 80 b.lens[0] = cpu_to_le64(ad_len); 81 b.lens[1] = cpu_to_le64(src_len); 82 poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens)); 83 84 poly1305_final(&poly1305_state, dst + src_len); 85 86 memzero_explicit(chacha_state, CHACHA_STATE_WORDS * sizeof(u32)); 87 memzero_explicit(&b, sizeof(b)); 88 } 89 90 void chacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len, 91 const u8 *ad, const size_t ad_len, 92 const u64 nonce, 93 const u8 key[CHACHA20POLY1305_KEY_SIZE]) 94 { 95 u32 chacha_state[CHACHA_STATE_WORDS]; 96 u32 k[CHACHA_KEY_WORDS]; 97 __le64 iv[2]; 98 99 chacha_load_key(k, key); 100 101 iv[0] = 0; 102 iv[1] = cpu_to_le64(nonce); 103 104 chacha_init(chacha_state, k, (u8 *)iv); 105 __chacha20poly1305_encrypt(dst, src, src_len, ad, ad_len, chacha_state); 106 107 memzero_explicit(iv, sizeof(iv)); 108 memzero_explicit(k, sizeof(k)); 109 } 110 EXPORT_SYMBOL(chacha20poly1305_encrypt); 111 112 void xchacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len, 113 const u8 *ad, const size_t ad_len, 114 const u8 nonce[XCHACHA20POLY1305_NONCE_SIZE], 115 const u8 key[CHACHA20POLY1305_KEY_SIZE]) 116 { 117 u32 chacha_state[CHACHA_STATE_WORDS]; 118 119 xchacha_init(chacha_state, key, nonce); 120 __chacha20poly1305_encrypt(dst, src, src_len, ad, ad_len, chacha_state); 121 } 122 EXPORT_SYMBOL(xchacha20poly1305_encrypt); 123 124 static bool 125 __chacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len, 126 const u8 *ad, const size_t ad_len, u32 *chacha_state) 127 { 128 const u8 *pad0 = page_address(ZERO_PAGE(0)); 129 struct poly1305_desc_ctx poly1305_state; 130 size_t dst_len; 131 int ret; 132 union { 133 u8 block0[POLY1305_KEY_SIZE]; 134 u8 mac[POLY1305_DIGEST_SIZE]; 135 __le64 lens[2]; 136 } b; 137 138 if (unlikely(src_len < POLY1305_DIGEST_SIZE)) 139 return false; 140 141 chacha20_crypt(chacha_state, b.block0, pad0, sizeof(b.block0)); 142 poly1305_init(&poly1305_state, b.block0); 143 144 poly1305_update(&poly1305_state, ad, ad_len); 145 if (ad_len & 0xf) 146 poly1305_update(&poly1305_state, pad0, 0x10 - (ad_len & 0xf)); 147 148 dst_len = src_len - POLY1305_DIGEST_SIZE; 149 poly1305_update(&poly1305_state, src, dst_len); 150 if (dst_len & 0xf) 151 poly1305_update(&poly1305_state, pad0, 0x10 - (dst_len & 0xf)); 152 153 b.lens[0] = cpu_to_le64(ad_len); 154 b.lens[1] = cpu_to_le64(dst_len); 155 poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens)); 156 157 poly1305_final(&poly1305_state, b.mac); 158 159 ret = crypto_memneq(b.mac, src + dst_len, POLY1305_DIGEST_SIZE); 160 if (likely(!ret)) 161 chacha20_crypt(chacha_state, dst, src, dst_len); 162 163 memzero_explicit(&b, sizeof(b)); 164 165 return !ret; 166 } 167 168 bool chacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len, 169 const u8 *ad, const size_t ad_len, 170 const u64 nonce, 171 const u8 key[CHACHA20POLY1305_KEY_SIZE]) 172 { 173 u32 chacha_state[CHACHA_STATE_WORDS]; 174 u32 k[CHACHA_KEY_WORDS]; 175 __le64 iv[2]; 176 bool ret; 177 178 chacha_load_key(k, key); 179 180 iv[0] = 0; 181 iv[1] = cpu_to_le64(nonce); 182 183 chacha_init(chacha_state, k, (u8 *)iv); 184 ret = __chacha20poly1305_decrypt(dst, src, src_len, ad, ad_len, 185 chacha_state); 186 187 memzero_explicit(chacha_state, sizeof(chacha_state)); 188 memzero_explicit(iv, sizeof(iv)); 189 memzero_explicit(k, sizeof(k)); 190 return ret; 191 } 192 EXPORT_SYMBOL(chacha20poly1305_decrypt); 193 194 bool xchacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len, 195 const u8 *ad, const size_t ad_len, 196 const u8 nonce[XCHACHA20POLY1305_NONCE_SIZE], 197 const u8 key[CHACHA20POLY1305_KEY_SIZE]) 198 { 199 u32 chacha_state[CHACHA_STATE_WORDS]; 200 201 xchacha_init(chacha_state, key, nonce); 202 return __chacha20poly1305_decrypt(dst, src, src_len, ad, ad_len, 203 chacha_state); 204 } 205 EXPORT_SYMBOL(xchacha20poly1305_decrypt); 206 207 static 208 bool chacha20poly1305_crypt_sg_inplace(struct scatterlist *src, 209 const size_t src_len, 210 const u8 *ad, const size_t ad_len, 211 const u64 nonce, 212 const u8 key[CHACHA20POLY1305_KEY_SIZE], 213 int encrypt) 214 { 215 const u8 *pad0 = page_address(ZERO_PAGE(0)); 216 struct poly1305_desc_ctx poly1305_state; 217 u32 chacha_state[CHACHA_STATE_WORDS]; 218 struct sg_mapping_iter miter; 219 size_t partial = 0; 220 unsigned int flags; 221 bool ret = true; 222 int sl; 223 union { 224 struct { 225 u32 k[CHACHA_KEY_WORDS]; 226 __le64 iv[2]; 227 }; 228 u8 block0[POLY1305_KEY_SIZE]; 229 u8 chacha_stream[CHACHA_BLOCK_SIZE]; 230 struct { 231 u8 mac[2][POLY1305_DIGEST_SIZE]; 232 }; 233 __le64 lens[2]; 234 } b __aligned(16); 235 236 if (WARN_ON(src_len > INT_MAX)) 237 return false; 238 239 chacha_load_key(b.k, key); 240 241 b.iv[0] = 0; 242 b.iv[1] = cpu_to_le64(nonce); 243 244 chacha_init(chacha_state, b.k, (u8 *)b.iv); 245 chacha20_crypt(chacha_state, b.block0, pad0, sizeof(b.block0)); 246 poly1305_init(&poly1305_state, b.block0); 247 248 if (unlikely(ad_len)) { 249 poly1305_update(&poly1305_state, ad, ad_len); 250 if (ad_len & 0xf) 251 poly1305_update(&poly1305_state, pad0, 0x10 - (ad_len & 0xf)); 252 } 253 254 flags = SG_MITER_TO_SG; 255 if (!preemptible()) 256 flags |= SG_MITER_ATOMIC; 257 258 sg_miter_start(&miter, src, sg_nents(src), flags); 259 260 for (sl = src_len; sl > 0 && sg_miter_next(&miter); sl -= miter.length) { 261 u8 *addr = miter.addr; 262 size_t length = min_t(size_t, sl, miter.length); 263 264 if (!encrypt) 265 poly1305_update(&poly1305_state, addr, length); 266 267 if (unlikely(partial)) { 268 size_t l = min(length, CHACHA_BLOCK_SIZE - partial); 269 270 crypto_xor(addr, b.chacha_stream + partial, l); 271 partial = (partial + l) & (CHACHA_BLOCK_SIZE - 1); 272 273 addr += l; 274 length -= l; 275 } 276 277 if (likely(length >= CHACHA_BLOCK_SIZE || length == sl)) { 278 size_t l = length; 279 280 if (unlikely(length < sl)) 281 l &= ~(CHACHA_BLOCK_SIZE - 1); 282 chacha20_crypt(chacha_state, addr, addr, l); 283 addr += l; 284 length -= l; 285 } 286 287 if (unlikely(length > 0)) { 288 chacha20_crypt(chacha_state, b.chacha_stream, pad0, 289 CHACHA_BLOCK_SIZE); 290 crypto_xor(addr, b.chacha_stream, length); 291 partial = length; 292 } 293 294 if (encrypt) 295 poly1305_update(&poly1305_state, miter.addr, 296 min_t(size_t, sl, miter.length)); 297 } 298 299 if (src_len & 0xf) 300 poly1305_update(&poly1305_state, pad0, 0x10 - (src_len & 0xf)); 301 302 b.lens[0] = cpu_to_le64(ad_len); 303 b.lens[1] = cpu_to_le64(src_len); 304 poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens)); 305 306 if (likely(sl <= -POLY1305_DIGEST_SIZE)) { 307 if (encrypt) { 308 poly1305_final(&poly1305_state, 309 miter.addr + miter.length + sl); 310 ret = true; 311 } else { 312 poly1305_final(&poly1305_state, b.mac[0]); 313 ret = !crypto_memneq(b.mac[0], 314 miter.addr + miter.length + sl, 315 POLY1305_DIGEST_SIZE); 316 } 317 } 318 319 sg_miter_stop(&miter); 320 321 if (unlikely(sl > -POLY1305_DIGEST_SIZE)) { 322 poly1305_final(&poly1305_state, b.mac[1]); 323 scatterwalk_map_and_copy(b.mac[encrypt], src, src_len, 324 sizeof(b.mac[1]), encrypt); 325 ret = encrypt || 326 !crypto_memneq(b.mac[0], b.mac[1], POLY1305_DIGEST_SIZE); 327 } 328 329 memzero_explicit(chacha_state, sizeof(chacha_state)); 330 memzero_explicit(&b, sizeof(b)); 331 332 return ret; 333 } 334 335 bool chacha20poly1305_encrypt_sg_inplace(struct scatterlist *src, size_t src_len, 336 const u8 *ad, const size_t ad_len, 337 const u64 nonce, 338 const u8 key[CHACHA20POLY1305_KEY_SIZE]) 339 { 340 return chacha20poly1305_crypt_sg_inplace(src, src_len, ad, ad_len, 341 nonce, key, 1); 342 } 343 EXPORT_SYMBOL(chacha20poly1305_encrypt_sg_inplace); 344 345 bool chacha20poly1305_decrypt_sg_inplace(struct scatterlist *src, size_t src_len, 346 const u8 *ad, const size_t ad_len, 347 const u64 nonce, 348 const u8 key[CHACHA20POLY1305_KEY_SIZE]) 349 { 350 if (unlikely(src_len < POLY1305_DIGEST_SIZE)) 351 return false; 352 353 return chacha20poly1305_crypt_sg_inplace(src, 354 src_len - POLY1305_DIGEST_SIZE, 355 ad, ad_len, nonce, key, 0); 356 } 357 EXPORT_SYMBOL(chacha20poly1305_decrypt_sg_inplace); 358 359 static int __init mod_init(void) 360 { 361 if (!IS_ENABLED(CONFIG_CRYPTO_MANAGER_DISABLE_TESTS) && 362 WARN_ON(!chacha20poly1305_selftest())) 363 return -ENODEV; 364 return 0; 365 } 366 367 module_init(mod_init); 368 MODULE_LICENSE("GPL v2"); 369 MODULE_DESCRIPTION("ChaCha20Poly1305 AEAD construction"); 370 MODULE_AUTHOR("Jason A. Donenfeld <Jason@zx2c4.com>"); 371