1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. 4 */ 5 6 #include "noise.h" 7 #include "device.h" 8 #include "peer.h" 9 #include "messages.h" 10 #include "queueing.h" 11 #include "peerlookup.h" 12 13 #include <linux/rcupdate.h> 14 #include <linux/slab.h> 15 #include <linux/bitmap.h> 16 #include <linux/scatterlist.h> 17 #include <linux/highmem.h> 18 #include <crypto/algapi.h> 19 20 /* This implements Noise_IKpsk2: 21 * 22 * <- s 23 * ****** 24 * -> e, es, s, ss, {t} 25 * <- e, ee, se, psk, {} 26 */ 27 28 static const u8 handshake_name[37] = "Noise_IKpsk2_25519_ChaChaPoly_BLAKE2s"; 29 static const u8 identifier_name[34] = "WireGuard v1 zx2c4 Jason@zx2c4.com"; 30 static u8 handshake_init_hash[NOISE_HASH_LEN] __ro_after_init; 31 static u8 handshake_init_chaining_key[NOISE_HASH_LEN] __ro_after_init; 32 static atomic64_t keypair_counter = ATOMIC64_INIT(0); 33 34 void __init wg_noise_init(void) 35 { 36 struct blake2s_state blake; 37 38 blake2s(handshake_init_chaining_key, handshake_name, NULL, 39 NOISE_HASH_LEN, sizeof(handshake_name), 0); 40 blake2s_init(&blake, NOISE_HASH_LEN); 41 blake2s_update(&blake, handshake_init_chaining_key, NOISE_HASH_LEN); 42 blake2s_update(&blake, identifier_name, sizeof(identifier_name)); 43 blake2s_final(&blake, handshake_init_hash); 44 } 45 46 /* Must hold peer->handshake.static_identity->lock */ 47 void wg_noise_precompute_static_static(struct wg_peer *peer) 48 { 49 down_write(&peer->handshake.lock); 50 if (!peer->handshake.static_identity->has_identity || 51 !curve25519(peer->handshake.precomputed_static_static, 52 peer->handshake.static_identity->static_private, 53 peer->handshake.remote_static)) 54 memset(peer->handshake.precomputed_static_static, 0, 55 NOISE_PUBLIC_KEY_LEN); 56 up_write(&peer->handshake.lock); 57 } 58 59 void wg_noise_handshake_init(struct noise_handshake *handshake, 60 struct noise_static_identity *static_identity, 61 const u8 peer_public_key[NOISE_PUBLIC_KEY_LEN], 62 const u8 peer_preshared_key[NOISE_SYMMETRIC_KEY_LEN], 63 struct wg_peer *peer) 64 { 65 memset(handshake, 0, sizeof(*handshake)); 66 init_rwsem(&handshake->lock); 67 handshake->entry.type = INDEX_HASHTABLE_HANDSHAKE; 68 handshake->entry.peer = peer; 69 memcpy(handshake->remote_static, peer_public_key, NOISE_PUBLIC_KEY_LEN); 70 if (peer_preshared_key) 71 memcpy(handshake->preshared_key, peer_preshared_key, 72 NOISE_SYMMETRIC_KEY_LEN); 73 handshake->static_identity = static_identity; 74 handshake->state = HANDSHAKE_ZEROED; 75 wg_noise_precompute_static_static(peer); 76 } 77 78 static void handshake_zero(struct noise_handshake *handshake) 79 { 80 memset(&handshake->ephemeral_private, 0, NOISE_PUBLIC_KEY_LEN); 81 memset(&handshake->remote_ephemeral, 0, NOISE_PUBLIC_KEY_LEN); 82 memset(&handshake->hash, 0, NOISE_HASH_LEN); 83 memset(&handshake->chaining_key, 0, NOISE_HASH_LEN); 84 handshake->remote_index = 0; 85 handshake->state = HANDSHAKE_ZEROED; 86 } 87 88 void wg_noise_handshake_clear(struct noise_handshake *handshake) 89 { 90 down_write(&handshake->lock); 91 wg_index_hashtable_remove( 92 handshake->entry.peer->device->index_hashtable, 93 &handshake->entry); 94 handshake_zero(handshake); 95 up_write(&handshake->lock); 96 } 97 98 static struct noise_keypair *keypair_create(struct wg_peer *peer) 99 { 100 struct noise_keypair *keypair = kzalloc(sizeof(*keypair), GFP_KERNEL); 101 102 if (unlikely(!keypair)) 103 return NULL; 104 spin_lock_init(&keypair->receiving_counter.lock); 105 keypair->internal_id = atomic64_inc_return(&keypair_counter); 106 keypair->entry.type = INDEX_HASHTABLE_KEYPAIR; 107 keypair->entry.peer = peer; 108 kref_init(&keypair->refcount); 109 return keypair; 110 } 111 112 static void keypair_free_rcu(struct rcu_head *rcu) 113 { 114 kfree_sensitive(container_of(rcu, struct noise_keypair, rcu)); 115 } 116 117 static void keypair_free_kref(struct kref *kref) 118 { 119 struct noise_keypair *keypair = 120 container_of(kref, struct noise_keypair, refcount); 121 122 net_dbg_ratelimited("%s: Keypair %llu destroyed for peer %llu\n", 123 keypair->entry.peer->device->dev->name, 124 keypair->internal_id, 125 keypair->entry.peer->internal_id); 126 wg_index_hashtable_remove(keypair->entry.peer->device->index_hashtable, 127 &keypair->entry); 128 call_rcu(&keypair->rcu, keypair_free_rcu); 129 } 130 131 void wg_noise_keypair_put(struct noise_keypair *keypair, bool unreference_now) 132 { 133 if (unlikely(!keypair)) 134 return; 135 if (unlikely(unreference_now)) 136 wg_index_hashtable_remove( 137 keypair->entry.peer->device->index_hashtable, 138 &keypair->entry); 139 kref_put(&keypair->refcount, keypair_free_kref); 140 } 141 142 struct noise_keypair *wg_noise_keypair_get(struct noise_keypair *keypair) 143 { 144 RCU_LOCKDEP_WARN(!rcu_read_lock_bh_held(), 145 "Taking noise keypair reference without holding the RCU BH read lock"); 146 if (unlikely(!keypair || !kref_get_unless_zero(&keypair->refcount))) 147 return NULL; 148 return keypair; 149 } 150 151 void wg_noise_keypairs_clear(struct noise_keypairs *keypairs) 152 { 153 struct noise_keypair *old; 154 155 spin_lock_bh(&keypairs->keypair_update_lock); 156 157 /* We zero the next_keypair before zeroing the others, so that 158 * wg_noise_received_with_keypair returns early before subsequent ones 159 * are zeroed. 160 */ 161 old = rcu_dereference_protected(keypairs->next_keypair, 162 lockdep_is_held(&keypairs->keypair_update_lock)); 163 RCU_INIT_POINTER(keypairs->next_keypair, NULL); 164 wg_noise_keypair_put(old, true); 165 166 old = rcu_dereference_protected(keypairs->previous_keypair, 167 lockdep_is_held(&keypairs->keypair_update_lock)); 168 RCU_INIT_POINTER(keypairs->previous_keypair, NULL); 169 wg_noise_keypair_put(old, true); 170 171 old = rcu_dereference_protected(keypairs->current_keypair, 172 lockdep_is_held(&keypairs->keypair_update_lock)); 173 RCU_INIT_POINTER(keypairs->current_keypair, NULL); 174 wg_noise_keypair_put(old, true); 175 176 spin_unlock_bh(&keypairs->keypair_update_lock); 177 } 178 179 void wg_noise_expire_current_peer_keypairs(struct wg_peer *peer) 180 { 181 struct noise_keypair *keypair; 182 183 wg_noise_handshake_clear(&peer->handshake); 184 wg_noise_reset_last_sent_handshake(&peer->last_sent_handshake); 185 186 spin_lock_bh(&peer->keypairs.keypair_update_lock); 187 keypair = rcu_dereference_protected(peer->keypairs.next_keypair, 188 lockdep_is_held(&peer->keypairs.keypair_update_lock)); 189 if (keypair) 190 keypair->sending.is_valid = false; 191 keypair = rcu_dereference_protected(peer->keypairs.current_keypair, 192 lockdep_is_held(&peer->keypairs.keypair_update_lock)); 193 if (keypair) 194 keypair->sending.is_valid = false; 195 spin_unlock_bh(&peer->keypairs.keypair_update_lock); 196 } 197 198 static void add_new_keypair(struct noise_keypairs *keypairs, 199 struct noise_keypair *new_keypair) 200 { 201 struct noise_keypair *previous_keypair, *next_keypair, *current_keypair; 202 203 spin_lock_bh(&keypairs->keypair_update_lock); 204 previous_keypair = rcu_dereference_protected(keypairs->previous_keypair, 205 lockdep_is_held(&keypairs->keypair_update_lock)); 206 next_keypair = rcu_dereference_protected(keypairs->next_keypair, 207 lockdep_is_held(&keypairs->keypair_update_lock)); 208 current_keypair = rcu_dereference_protected(keypairs->current_keypair, 209 lockdep_is_held(&keypairs->keypair_update_lock)); 210 if (new_keypair->i_am_the_initiator) { 211 /* If we're the initiator, it means we've sent a handshake, and 212 * received a confirmation response, which means this new 213 * keypair can now be used. 214 */ 215 if (next_keypair) { 216 /* If there already was a next keypair pending, we 217 * demote it to be the previous keypair, and free the 218 * existing current. Note that this means KCI can result 219 * in this transition. It would perhaps be more sound to 220 * always just get rid of the unused next keypair 221 * instead of putting it in the previous slot, but this 222 * might be a bit less robust. Something to think about 223 * for the future. 224 */ 225 RCU_INIT_POINTER(keypairs->next_keypair, NULL); 226 rcu_assign_pointer(keypairs->previous_keypair, 227 next_keypair); 228 wg_noise_keypair_put(current_keypair, true); 229 } else /* If there wasn't an existing next keypair, we replace 230 * the previous with the current one. 231 */ 232 rcu_assign_pointer(keypairs->previous_keypair, 233 current_keypair); 234 /* At this point we can get rid of the old previous keypair, and 235 * set up the new keypair. 236 */ 237 wg_noise_keypair_put(previous_keypair, true); 238 rcu_assign_pointer(keypairs->current_keypair, new_keypair); 239 } else { 240 /* If we're the responder, it means we can't use the new keypair 241 * until we receive confirmation via the first data packet, so 242 * we get rid of the existing previous one, the possibly 243 * existing next one, and slide in the new next one. 244 */ 245 rcu_assign_pointer(keypairs->next_keypair, new_keypair); 246 wg_noise_keypair_put(next_keypair, true); 247 RCU_INIT_POINTER(keypairs->previous_keypair, NULL); 248 wg_noise_keypair_put(previous_keypair, true); 249 } 250 spin_unlock_bh(&keypairs->keypair_update_lock); 251 } 252 253 bool wg_noise_received_with_keypair(struct noise_keypairs *keypairs, 254 struct noise_keypair *received_keypair) 255 { 256 struct noise_keypair *old_keypair; 257 bool key_is_new; 258 259 /* We first check without taking the spinlock. */ 260 key_is_new = received_keypair == 261 rcu_access_pointer(keypairs->next_keypair); 262 if (likely(!key_is_new)) 263 return false; 264 265 spin_lock_bh(&keypairs->keypair_update_lock); 266 /* After locking, we double check that things didn't change from 267 * beneath us. 268 */ 269 if (unlikely(received_keypair != 270 rcu_dereference_protected(keypairs->next_keypair, 271 lockdep_is_held(&keypairs->keypair_update_lock)))) { 272 spin_unlock_bh(&keypairs->keypair_update_lock); 273 return false; 274 } 275 276 /* When we've finally received the confirmation, we slide the next 277 * into the current, the current into the previous, and get rid of 278 * the old previous. 279 */ 280 old_keypair = rcu_dereference_protected(keypairs->previous_keypair, 281 lockdep_is_held(&keypairs->keypair_update_lock)); 282 rcu_assign_pointer(keypairs->previous_keypair, 283 rcu_dereference_protected(keypairs->current_keypair, 284 lockdep_is_held(&keypairs->keypair_update_lock))); 285 wg_noise_keypair_put(old_keypair, true); 286 rcu_assign_pointer(keypairs->current_keypair, received_keypair); 287 RCU_INIT_POINTER(keypairs->next_keypair, NULL); 288 289 spin_unlock_bh(&keypairs->keypair_update_lock); 290 return true; 291 } 292 293 /* Must hold static_identity->lock */ 294 void wg_noise_set_static_identity_private_key( 295 struct noise_static_identity *static_identity, 296 const u8 private_key[NOISE_PUBLIC_KEY_LEN]) 297 { 298 memcpy(static_identity->static_private, private_key, 299 NOISE_PUBLIC_KEY_LEN); 300 curve25519_clamp_secret(static_identity->static_private); 301 static_identity->has_identity = curve25519_generate_public( 302 static_identity->static_public, private_key); 303 } 304 305 static void hmac(u8 *out, const u8 *in, const u8 *key, const size_t inlen, const size_t keylen) 306 { 307 struct blake2s_state state; 308 u8 x_key[BLAKE2S_BLOCK_SIZE] __aligned(__alignof__(u32)) = { 0 }; 309 u8 i_hash[BLAKE2S_HASH_SIZE] __aligned(__alignof__(u32)); 310 int i; 311 312 if (keylen > BLAKE2S_BLOCK_SIZE) { 313 blake2s_init(&state, BLAKE2S_HASH_SIZE); 314 blake2s_update(&state, key, keylen); 315 blake2s_final(&state, x_key); 316 } else 317 memcpy(x_key, key, keylen); 318 319 for (i = 0; i < BLAKE2S_BLOCK_SIZE; ++i) 320 x_key[i] ^= 0x36; 321 322 blake2s_init(&state, BLAKE2S_HASH_SIZE); 323 blake2s_update(&state, x_key, BLAKE2S_BLOCK_SIZE); 324 blake2s_update(&state, in, inlen); 325 blake2s_final(&state, i_hash); 326 327 for (i = 0; i < BLAKE2S_BLOCK_SIZE; ++i) 328 x_key[i] ^= 0x5c ^ 0x36; 329 330 blake2s_init(&state, BLAKE2S_HASH_SIZE); 331 blake2s_update(&state, x_key, BLAKE2S_BLOCK_SIZE); 332 blake2s_update(&state, i_hash, BLAKE2S_HASH_SIZE); 333 blake2s_final(&state, i_hash); 334 335 memcpy(out, i_hash, BLAKE2S_HASH_SIZE); 336 memzero_explicit(x_key, BLAKE2S_BLOCK_SIZE); 337 memzero_explicit(i_hash, BLAKE2S_HASH_SIZE); 338 } 339 340 /* This is Hugo Krawczyk's HKDF: 341 * - https://eprint.iacr.org/2010/264.pdf 342 * - https://tools.ietf.org/html/rfc5869 343 */ 344 static void kdf(u8 *first_dst, u8 *second_dst, u8 *third_dst, const u8 *data, 345 size_t first_len, size_t second_len, size_t third_len, 346 size_t data_len, const u8 chaining_key[NOISE_HASH_LEN]) 347 { 348 u8 output[BLAKE2S_HASH_SIZE + 1]; 349 u8 secret[BLAKE2S_HASH_SIZE]; 350 351 WARN_ON(IS_ENABLED(DEBUG) && 352 (first_len > BLAKE2S_HASH_SIZE || 353 second_len > BLAKE2S_HASH_SIZE || 354 third_len > BLAKE2S_HASH_SIZE || 355 ((second_len || second_dst || third_len || third_dst) && 356 (!first_len || !first_dst)) || 357 ((third_len || third_dst) && (!second_len || !second_dst)))); 358 359 /* Extract entropy from data into secret */ 360 hmac(secret, data, chaining_key, data_len, NOISE_HASH_LEN); 361 362 if (!first_dst || !first_len) 363 goto out; 364 365 /* Expand first key: key = secret, data = 0x1 */ 366 output[0] = 1; 367 hmac(output, output, secret, 1, BLAKE2S_HASH_SIZE); 368 memcpy(first_dst, output, first_len); 369 370 if (!second_dst || !second_len) 371 goto out; 372 373 /* Expand second key: key = secret, data = first-key || 0x2 */ 374 output[BLAKE2S_HASH_SIZE] = 2; 375 hmac(output, output, secret, BLAKE2S_HASH_SIZE + 1, BLAKE2S_HASH_SIZE); 376 memcpy(second_dst, output, second_len); 377 378 if (!third_dst || !third_len) 379 goto out; 380 381 /* Expand third key: key = secret, data = second-key || 0x3 */ 382 output[BLAKE2S_HASH_SIZE] = 3; 383 hmac(output, output, secret, BLAKE2S_HASH_SIZE + 1, BLAKE2S_HASH_SIZE); 384 memcpy(third_dst, output, third_len); 385 386 out: 387 /* Clear sensitive data from stack */ 388 memzero_explicit(secret, BLAKE2S_HASH_SIZE); 389 memzero_explicit(output, BLAKE2S_HASH_SIZE + 1); 390 } 391 392 static void derive_keys(struct noise_symmetric_key *first_dst, 393 struct noise_symmetric_key *second_dst, 394 const u8 chaining_key[NOISE_HASH_LEN]) 395 { 396 u64 birthdate = ktime_get_coarse_boottime_ns(); 397 kdf(first_dst->key, second_dst->key, NULL, NULL, 398 NOISE_SYMMETRIC_KEY_LEN, NOISE_SYMMETRIC_KEY_LEN, 0, 0, 399 chaining_key); 400 first_dst->birthdate = second_dst->birthdate = birthdate; 401 first_dst->is_valid = second_dst->is_valid = true; 402 } 403 404 static bool __must_check mix_dh(u8 chaining_key[NOISE_HASH_LEN], 405 u8 key[NOISE_SYMMETRIC_KEY_LEN], 406 const u8 private[NOISE_PUBLIC_KEY_LEN], 407 const u8 public[NOISE_PUBLIC_KEY_LEN]) 408 { 409 u8 dh_calculation[NOISE_PUBLIC_KEY_LEN]; 410 411 if (unlikely(!curve25519(dh_calculation, private, public))) 412 return false; 413 kdf(chaining_key, key, NULL, dh_calculation, NOISE_HASH_LEN, 414 NOISE_SYMMETRIC_KEY_LEN, 0, NOISE_PUBLIC_KEY_LEN, chaining_key); 415 memzero_explicit(dh_calculation, NOISE_PUBLIC_KEY_LEN); 416 return true; 417 } 418 419 static bool __must_check mix_precomputed_dh(u8 chaining_key[NOISE_HASH_LEN], 420 u8 key[NOISE_SYMMETRIC_KEY_LEN], 421 const u8 precomputed[NOISE_PUBLIC_KEY_LEN]) 422 { 423 static u8 zero_point[NOISE_PUBLIC_KEY_LEN]; 424 if (unlikely(!crypto_memneq(precomputed, zero_point, NOISE_PUBLIC_KEY_LEN))) 425 return false; 426 kdf(chaining_key, key, NULL, precomputed, NOISE_HASH_LEN, 427 NOISE_SYMMETRIC_KEY_LEN, 0, NOISE_PUBLIC_KEY_LEN, 428 chaining_key); 429 return true; 430 } 431 432 static void mix_hash(u8 hash[NOISE_HASH_LEN], const u8 *src, size_t src_len) 433 { 434 struct blake2s_state blake; 435 436 blake2s_init(&blake, NOISE_HASH_LEN); 437 blake2s_update(&blake, hash, NOISE_HASH_LEN); 438 blake2s_update(&blake, src, src_len); 439 blake2s_final(&blake, hash); 440 } 441 442 static void mix_psk(u8 chaining_key[NOISE_HASH_LEN], u8 hash[NOISE_HASH_LEN], 443 u8 key[NOISE_SYMMETRIC_KEY_LEN], 444 const u8 psk[NOISE_SYMMETRIC_KEY_LEN]) 445 { 446 u8 temp_hash[NOISE_HASH_LEN]; 447 448 kdf(chaining_key, temp_hash, key, psk, NOISE_HASH_LEN, NOISE_HASH_LEN, 449 NOISE_SYMMETRIC_KEY_LEN, NOISE_SYMMETRIC_KEY_LEN, chaining_key); 450 mix_hash(hash, temp_hash, NOISE_HASH_LEN); 451 memzero_explicit(temp_hash, NOISE_HASH_LEN); 452 } 453 454 static void handshake_init(u8 chaining_key[NOISE_HASH_LEN], 455 u8 hash[NOISE_HASH_LEN], 456 const u8 remote_static[NOISE_PUBLIC_KEY_LEN]) 457 { 458 memcpy(hash, handshake_init_hash, NOISE_HASH_LEN); 459 memcpy(chaining_key, handshake_init_chaining_key, NOISE_HASH_LEN); 460 mix_hash(hash, remote_static, NOISE_PUBLIC_KEY_LEN); 461 } 462 463 static void message_encrypt(u8 *dst_ciphertext, const u8 *src_plaintext, 464 size_t src_len, u8 key[NOISE_SYMMETRIC_KEY_LEN], 465 u8 hash[NOISE_HASH_LEN]) 466 { 467 chacha20poly1305_encrypt(dst_ciphertext, src_plaintext, src_len, hash, 468 NOISE_HASH_LEN, 469 0 /* Always zero for Noise_IK */, key); 470 mix_hash(hash, dst_ciphertext, noise_encrypted_len(src_len)); 471 } 472 473 static bool message_decrypt(u8 *dst_plaintext, const u8 *src_ciphertext, 474 size_t src_len, u8 key[NOISE_SYMMETRIC_KEY_LEN], 475 u8 hash[NOISE_HASH_LEN]) 476 { 477 if (!chacha20poly1305_decrypt(dst_plaintext, src_ciphertext, src_len, 478 hash, NOISE_HASH_LEN, 479 0 /* Always zero for Noise_IK */, key)) 480 return false; 481 mix_hash(hash, src_ciphertext, src_len); 482 return true; 483 } 484 485 static void message_ephemeral(u8 ephemeral_dst[NOISE_PUBLIC_KEY_LEN], 486 const u8 ephemeral_src[NOISE_PUBLIC_KEY_LEN], 487 u8 chaining_key[NOISE_HASH_LEN], 488 u8 hash[NOISE_HASH_LEN]) 489 { 490 if (ephemeral_dst != ephemeral_src) 491 memcpy(ephemeral_dst, ephemeral_src, NOISE_PUBLIC_KEY_LEN); 492 mix_hash(hash, ephemeral_src, NOISE_PUBLIC_KEY_LEN); 493 kdf(chaining_key, NULL, NULL, ephemeral_src, NOISE_HASH_LEN, 0, 0, 494 NOISE_PUBLIC_KEY_LEN, chaining_key); 495 } 496 497 static void tai64n_now(u8 output[NOISE_TIMESTAMP_LEN]) 498 { 499 struct timespec64 now; 500 501 ktime_get_real_ts64(&now); 502 503 /* In order to prevent some sort of infoleak from precise timers, we 504 * round down the nanoseconds part to the closest rounded-down power of 505 * two to the maximum initiations per second allowed anyway by the 506 * implementation. 507 */ 508 now.tv_nsec = ALIGN_DOWN(now.tv_nsec, 509 rounddown_pow_of_two(NSEC_PER_SEC / INITIATIONS_PER_SECOND)); 510 511 /* https://cr.yp.to/libtai/tai64.html */ 512 *(__be64 *)output = cpu_to_be64(0x400000000000000aULL + now.tv_sec); 513 *(__be32 *)(output + sizeof(__be64)) = cpu_to_be32(now.tv_nsec); 514 } 515 516 bool 517 wg_noise_handshake_create_initiation(struct message_handshake_initiation *dst, 518 struct noise_handshake *handshake) 519 { 520 u8 timestamp[NOISE_TIMESTAMP_LEN]; 521 u8 key[NOISE_SYMMETRIC_KEY_LEN]; 522 bool ret = false; 523 524 /* We need to wait for crng _before_ taking any locks, since 525 * curve25519_generate_secret uses get_random_bytes_wait. 526 */ 527 wait_for_random_bytes(); 528 529 down_read(&handshake->static_identity->lock); 530 down_write(&handshake->lock); 531 532 if (unlikely(!handshake->static_identity->has_identity)) 533 goto out; 534 535 dst->header.type = cpu_to_le32(MESSAGE_HANDSHAKE_INITIATION); 536 537 handshake_init(handshake->chaining_key, handshake->hash, 538 handshake->remote_static); 539 540 /* e */ 541 curve25519_generate_secret(handshake->ephemeral_private); 542 if (!curve25519_generate_public(dst->unencrypted_ephemeral, 543 handshake->ephemeral_private)) 544 goto out; 545 message_ephemeral(dst->unencrypted_ephemeral, 546 dst->unencrypted_ephemeral, handshake->chaining_key, 547 handshake->hash); 548 549 /* es */ 550 if (!mix_dh(handshake->chaining_key, key, handshake->ephemeral_private, 551 handshake->remote_static)) 552 goto out; 553 554 /* s */ 555 message_encrypt(dst->encrypted_static, 556 handshake->static_identity->static_public, 557 NOISE_PUBLIC_KEY_LEN, key, handshake->hash); 558 559 /* ss */ 560 if (!mix_precomputed_dh(handshake->chaining_key, key, 561 handshake->precomputed_static_static)) 562 goto out; 563 564 /* {t} */ 565 tai64n_now(timestamp); 566 message_encrypt(dst->encrypted_timestamp, timestamp, 567 NOISE_TIMESTAMP_LEN, key, handshake->hash); 568 569 dst->sender_index = wg_index_hashtable_insert( 570 handshake->entry.peer->device->index_hashtable, 571 &handshake->entry); 572 573 handshake->state = HANDSHAKE_CREATED_INITIATION; 574 ret = true; 575 576 out: 577 up_write(&handshake->lock); 578 up_read(&handshake->static_identity->lock); 579 memzero_explicit(key, NOISE_SYMMETRIC_KEY_LEN); 580 return ret; 581 } 582 583 struct wg_peer * 584 wg_noise_handshake_consume_initiation(struct message_handshake_initiation *src, 585 struct wg_device *wg) 586 { 587 struct wg_peer *peer = NULL, *ret_peer = NULL; 588 struct noise_handshake *handshake; 589 bool replay_attack, flood_attack; 590 u8 key[NOISE_SYMMETRIC_KEY_LEN]; 591 u8 chaining_key[NOISE_HASH_LEN]; 592 u8 hash[NOISE_HASH_LEN]; 593 u8 s[NOISE_PUBLIC_KEY_LEN]; 594 u8 e[NOISE_PUBLIC_KEY_LEN]; 595 u8 t[NOISE_TIMESTAMP_LEN]; 596 u64 initiation_consumption; 597 598 down_read(&wg->static_identity.lock); 599 if (unlikely(!wg->static_identity.has_identity)) 600 goto out; 601 602 handshake_init(chaining_key, hash, wg->static_identity.static_public); 603 604 /* e */ 605 message_ephemeral(e, src->unencrypted_ephemeral, chaining_key, hash); 606 607 /* es */ 608 if (!mix_dh(chaining_key, key, wg->static_identity.static_private, e)) 609 goto out; 610 611 /* s */ 612 if (!message_decrypt(s, src->encrypted_static, 613 sizeof(src->encrypted_static), key, hash)) 614 goto out; 615 616 /* Lookup which peer we're actually talking to */ 617 peer = wg_pubkey_hashtable_lookup(wg->peer_hashtable, s); 618 if (!peer) 619 goto out; 620 handshake = &peer->handshake; 621 622 /* ss */ 623 if (!mix_precomputed_dh(chaining_key, key, 624 handshake->precomputed_static_static)) 625 goto out; 626 627 /* {t} */ 628 if (!message_decrypt(t, src->encrypted_timestamp, 629 sizeof(src->encrypted_timestamp), key, hash)) 630 goto out; 631 632 down_read(&handshake->lock); 633 replay_attack = memcmp(t, handshake->latest_timestamp, 634 NOISE_TIMESTAMP_LEN) <= 0; 635 flood_attack = (s64)handshake->last_initiation_consumption + 636 NSEC_PER_SEC / INITIATIONS_PER_SECOND > 637 (s64)ktime_get_coarse_boottime_ns(); 638 up_read(&handshake->lock); 639 if (replay_attack || flood_attack) 640 goto out; 641 642 /* Success! Copy everything to peer */ 643 down_write(&handshake->lock); 644 memcpy(handshake->remote_ephemeral, e, NOISE_PUBLIC_KEY_LEN); 645 if (memcmp(t, handshake->latest_timestamp, NOISE_TIMESTAMP_LEN) > 0) 646 memcpy(handshake->latest_timestamp, t, NOISE_TIMESTAMP_LEN); 647 memcpy(handshake->hash, hash, NOISE_HASH_LEN); 648 memcpy(handshake->chaining_key, chaining_key, NOISE_HASH_LEN); 649 handshake->remote_index = src->sender_index; 650 initiation_consumption = ktime_get_coarse_boottime_ns(); 651 if ((s64)(handshake->last_initiation_consumption - initiation_consumption) < 0) 652 handshake->last_initiation_consumption = initiation_consumption; 653 handshake->state = HANDSHAKE_CONSUMED_INITIATION; 654 up_write(&handshake->lock); 655 ret_peer = peer; 656 657 out: 658 memzero_explicit(key, NOISE_SYMMETRIC_KEY_LEN); 659 memzero_explicit(hash, NOISE_HASH_LEN); 660 memzero_explicit(chaining_key, NOISE_HASH_LEN); 661 up_read(&wg->static_identity.lock); 662 if (!ret_peer) 663 wg_peer_put(peer); 664 return ret_peer; 665 } 666 667 bool wg_noise_handshake_create_response(struct message_handshake_response *dst, 668 struct noise_handshake *handshake) 669 { 670 u8 key[NOISE_SYMMETRIC_KEY_LEN]; 671 bool ret = false; 672 673 /* We need to wait for crng _before_ taking any locks, since 674 * curve25519_generate_secret uses get_random_bytes_wait. 675 */ 676 wait_for_random_bytes(); 677 678 down_read(&handshake->static_identity->lock); 679 down_write(&handshake->lock); 680 681 if (handshake->state != HANDSHAKE_CONSUMED_INITIATION) 682 goto out; 683 684 dst->header.type = cpu_to_le32(MESSAGE_HANDSHAKE_RESPONSE); 685 dst->receiver_index = handshake->remote_index; 686 687 /* e */ 688 curve25519_generate_secret(handshake->ephemeral_private); 689 if (!curve25519_generate_public(dst->unencrypted_ephemeral, 690 handshake->ephemeral_private)) 691 goto out; 692 message_ephemeral(dst->unencrypted_ephemeral, 693 dst->unencrypted_ephemeral, handshake->chaining_key, 694 handshake->hash); 695 696 /* ee */ 697 if (!mix_dh(handshake->chaining_key, NULL, handshake->ephemeral_private, 698 handshake->remote_ephemeral)) 699 goto out; 700 701 /* se */ 702 if (!mix_dh(handshake->chaining_key, NULL, handshake->ephemeral_private, 703 handshake->remote_static)) 704 goto out; 705 706 /* psk */ 707 mix_psk(handshake->chaining_key, handshake->hash, key, 708 handshake->preshared_key); 709 710 /* {} */ 711 message_encrypt(dst->encrypted_nothing, NULL, 0, key, handshake->hash); 712 713 dst->sender_index = wg_index_hashtable_insert( 714 handshake->entry.peer->device->index_hashtable, 715 &handshake->entry); 716 717 handshake->state = HANDSHAKE_CREATED_RESPONSE; 718 ret = true; 719 720 out: 721 up_write(&handshake->lock); 722 up_read(&handshake->static_identity->lock); 723 memzero_explicit(key, NOISE_SYMMETRIC_KEY_LEN); 724 return ret; 725 } 726 727 struct wg_peer * 728 wg_noise_handshake_consume_response(struct message_handshake_response *src, 729 struct wg_device *wg) 730 { 731 enum noise_handshake_state state = HANDSHAKE_ZEROED; 732 struct wg_peer *peer = NULL, *ret_peer = NULL; 733 struct noise_handshake *handshake; 734 u8 key[NOISE_SYMMETRIC_KEY_LEN]; 735 u8 hash[NOISE_HASH_LEN]; 736 u8 chaining_key[NOISE_HASH_LEN]; 737 u8 e[NOISE_PUBLIC_KEY_LEN]; 738 u8 ephemeral_private[NOISE_PUBLIC_KEY_LEN]; 739 u8 static_private[NOISE_PUBLIC_KEY_LEN]; 740 u8 preshared_key[NOISE_SYMMETRIC_KEY_LEN]; 741 742 down_read(&wg->static_identity.lock); 743 744 if (unlikely(!wg->static_identity.has_identity)) 745 goto out; 746 747 handshake = (struct noise_handshake *)wg_index_hashtable_lookup( 748 wg->index_hashtable, INDEX_HASHTABLE_HANDSHAKE, 749 src->receiver_index, &peer); 750 if (unlikely(!handshake)) 751 goto out; 752 753 down_read(&handshake->lock); 754 state = handshake->state; 755 memcpy(hash, handshake->hash, NOISE_HASH_LEN); 756 memcpy(chaining_key, handshake->chaining_key, NOISE_HASH_LEN); 757 memcpy(ephemeral_private, handshake->ephemeral_private, 758 NOISE_PUBLIC_KEY_LEN); 759 memcpy(preshared_key, handshake->preshared_key, 760 NOISE_SYMMETRIC_KEY_LEN); 761 up_read(&handshake->lock); 762 763 if (state != HANDSHAKE_CREATED_INITIATION) 764 goto fail; 765 766 /* e */ 767 message_ephemeral(e, src->unencrypted_ephemeral, chaining_key, hash); 768 769 /* ee */ 770 if (!mix_dh(chaining_key, NULL, ephemeral_private, e)) 771 goto fail; 772 773 /* se */ 774 if (!mix_dh(chaining_key, NULL, wg->static_identity.static_private, e)) 775 goto fail; 776 777 /* psk */ 778 mix_psk(chaining_key, hash, key, preshared_key); 779 780 /* {} */ 781 if (!message_decrypt(NULL, src->encrypted_nothing, 782 sizeof(src->encrypted_nothing), key, hash)) 783 goto fail; 784 785 /* Success! Copy everything to peer */ 786 down_write(&handshake->lock); 787 /* It's important to check that the state is still the same, while we 788 * have an exclusive lock. 789 */ 790 if (handshake->state != state) { 791 up_write(&handshake->lock); 792 goto fail; 793 } 794 memcpy(handshake->remote_ephemeral, e, NOISE_PUBLIC_KEY_LEN); 795 memcpy(handshake->hash, hash, NOISE_HASH_LEN); 796 memcpy(handshake->chaining_key, chaining_key, NOISE_HASH_LEN); 797 handshake->remote_index = src->sender_index; 798 handshake->state = HANDSHAKE_CONSUMED_RESPONSE; 799 up_write(&handshake->lock); 800 ret_peer = peer; 801 goto out; 802 803 fail: 804 wg_peer_put(peer); 805 out: 806 memzero_explicit(key, NOISE_SYMMETRIC_KEY_LEN); 807 memzero_explicit(hash, NOISE_HASH_LEN); 808 memzero_explicit(chaining_key, NOISE_HASH_LEN); 809 memzero_explicit(ephemeral_private, NOISE_PUBLIC_KEY_LEN); 810 memzero_explicit(static_private, NOISE_PUBLIC_KEY_LEN); 811 memzero_explicit(preshared_key, NOISE_SYMMETRIC_KEY_LEN); 812 up_read(&wg->static_identity.lock); 813 return ret_peer; 814 } 815 816 bool wg_noise_handshake_begin_session(struct noise_handshake *handshake, 817 struct noise_keypairs *keypairs) 818 { 819 struct noise_keypair *new_keypair; 820 bool ret = false; 821 822 down_write(&handshake->lock); 823 if (handshake->state != HANDSHAKE_CREATED_RESPONSE && 824 handshake->state != HANDSHAKE_CONSUMED_RESPONSE) 825 goto out; 826 827 new_keypair = keypair_create(handshake->entry.peer); 828 if (!new_keypair) 829 goto out; 830 new_keypair->i_am_the_initiator = handshake->state == 831 HANDSHAKE_CONSUMED_RESPONSE; 832 new_keypair->remote_index = handshake->remote_index; 833 834 if (new_keypair->i_am_the_initiator) 835 derive_keys(&new_keypair->sending, &new_keypair->receiving, 836 handshake->chaining_key); 837 else 838 derive_keys(&new_keypair->receiving, &new_keypair->sending, 839 handshake->chaining_key); 840 841 handshake_zero(handshake); 842 rcu_read_lock_bh(); 843 if (likely(!READ_ONCE(container_of(handshake, struct wg_peer, 844 handshake)->is_dead))) { 845 add_new_keypair(keypairs, new_keypair); 846 net_dbg_ratelimited("%s: Keypair %llu created for peer %llu\n", 847 handshake->entry.peer->device->dev->name, 848 new_keypair->internal_id, 849 handshake->entry.peer->internal_id); 850 ret = wg_index_hashtable_replace( 851 handshake->entry.peer->device->index_hashtable, 852 &handshake->entry, &new_keypair->entry); 853 } else { 854 kfree_sensitive(new_keypair); 855 } 856 rcu_read_unlock_bh(); 857 858 out: 859 up_write(&handshake->lock); 860 return ret; 861 } 862