1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. 4 */ 5 6 #include "queueing.h" 7 #include "device.h" 8 #include "peer.h" 9 #include "timers.h" 10 #include "messages.h" 11 #include "cookie.h" 12 #include "socket.h" 13 14 #include <linux/ip.h> 15 #include <linux/ipv6.h> 16 #include <linux/udp.h> 17 #include <net/ip_tunnels.h> 18 19 /* Must be called with bh disabled. */ 20 static void update_rx_stats(struct wg_peer *peer, size_t len) 21 { 22 struct pcpu_sw_netstats *tstats = 23 get_cpu_ptr(peer->device->dev->tstats); 24 25 u64_stats_update_begin(&tstats->syncp); 26 ++tstats->rx_packets; 27 tstats->rx_bytes += len; 28 peer->rx_bytes += len; 29 u64_stats_update_end(&tstats->syncp); 30 put_cpu_ptr(tstats); 31 } 32 33 #define SKB_TYPE_LE32(skb) (((struct message_header *)(skb)->data)->type) 34 35 static size_t validate_header_len(struct sk_buff *skb) 36 { 37 if (unlikely(skb->len < sizeof(struct message_header))) 38 return 0; 39 if (SKB_TYPE_LE32(skb) == cpu_to_le32(MESSAGE_DATA) && 40 skb->len >= MESSAGE_MINIMUM_LENGTH) 41 return sizeof(struct message_data); 42 if (SKB_TYPE_LE32(skb) == cpu_to_le32(MESSAGE_HANDSHAKE_INITIATION) && 43 skb->len == sizeof(struct message_handshake_initiation)) 44 return sizeof(struct message_handshake_initiation); 45 if (SKB_TYPE_LE32(skb) == cpu_to_le32(MESSAGE_HANDSHAKE_RESPONSE) && 46 skb->len == sizeof(struct message_handshake_response)) 47 return sizeof(struct message_handshake_response); 48 if (SKB_TYPE_LE32(skb) == cpu_to_le32(MESSAGE_HANDSHAKE_COOKIE) && 49 skb->len == sizeof(struct message_handshake_cookie)) 50 return sizeof(struct message_handshake_cookie); 51 return 0; 52 } 53 54 static int prepare_skb_header(struct sk_buff *skb, struct wg_device *wg) 55 { 56 size_t data_offset, data_len, header_len; 57 struct udphdr *udp; 58 59 if (unlikely(wg_skb_examine_untrusted_ip_hdr(skb) != skb->protocol || 60 skb_transport_header(skb) < skb->head || 61 (skb_transport_header(skb) + sizeof(struct udphdr)) > 62 skb_tail_pointer(skb))) 63 return -EINVAL; /* Bogus IP header */ 64 udp = udp_hdr(skb); 65 data_offset = (u8 *)udp - skb->data; 66 if (unlikely(data_offset > U16_MAX || 67 data_offset + sizeof(struct udphdr) > skb->len)) 68 /* Packet has offset at impossible location or isn't big enough 69 * to have UDP fields. 70 */ 71 return -EINVAL; 72 data_len = ntohs(udp->len); 73 if (unlikely(data_len < sizeof(struct udphdr) || 74 data_len > skb->len - data_offset)) 75 /* UDP packet is reporting too small of a size or lying about 76 * its size. 77 */ 78 return -EINVAL; 79 data_len -= sizeof(struct udphdr); 80 data_offset = (u8 *)udp + sizeof(struct udphdr) - skb->data; 81 if (unlikely(!pskb_may_pull(skb, 82 data_offset + sizeof(struct message_header)) || 83 pskb_trim(skb, data_len + data_offset) < 0)) 84 return -EINVAL; 85 skb_pull(skb, data_offset); 86 if (unlikely(skb->len != data_len)) 87 /* Final len does not agree with calculated len */ 88 return -EINVAL; 89 header_len = validate_header_len(skb); 90 if (unlikely(!header_len)) 91 return -EINVAL; 92 __skb_push(skb, data_offset); 93 if (unlikely(!pskb_may_pull(skb, data_offset + header_len))) 94 return -EINVAL; 95 __skb_pull(skb, data_offset); 96 return 0; 97 } 98 99 static void wg_receive_handshake_packet(struct wg_device *wg, 100 struct sk_buff *skb) 101 { 102 enum cookie_mac_state mac_state; 103 struct wg_peer *peer = NULL; 104 /* This is global, so that our load calculation applies to the whole 105 * system. We don't care about races with it at all. 106 */ 107 static u64 last_under_load; 108 bool packet_needs_cookie; 109 bool under_load; 110 111 if (SKB_TYPE_LE32(skb) == cpu_to_le32(MESSAGE_HANDSHAKE_COOKIE)) { 112 net_dbg_skb_ratelimited("%s: Receiving cookie response from %pISpfsc\n", 113 wg->dev->name, skb); 114 wg_cookie_message_consume( 115 (struct message_handshake_cookie *)skb->data, wg); 116 return; 117 } 118 119 under_load = skb_queue_len(&wg->incoming_handshakes) >= 120 MAX_QUEUED_INCOMING_HANDSHAKES / 8; 121 if (under_load) 122 last_under_load = ktime_get_coarse_boottime_ns(); 123 else if (last_under_load) 124 under_load = !wg_birthdate_has_expired(last_under_load, 1); 125 mac_state = wg_cookie_validate_packet(&wg->cookie_checker, skb, 126 under_load); 127 if ((under_load && mac_state == VALID_MAC_WITH_COOKIE) || 128 (!under_load && mac_state == VALID_MAC_BUT_NO_COOKIE)) { 129 packet_needs_cookie = false; 130 } else if (under_load && mac_state == VALID_MAC_BUT_NO_COOKIE) { 131 packet_needs_cookie = true; 132 } else { 133 net_dbg_skb_ratelimited("%s: Invalid MAC of handshake, dropping packet from %pISpfsc\n", 134 wg->dev->name, skb); 135 return; 136 } 137 138 switch (SKB_TYPE_LE32(skb)) { 139 case cpu_to_le32(MESSAGE_HANDSHAKE_INITIATION): { 140 struct message_handshake_initiation *message = 141 (struct message_handshake_initiation *)skb->data; 142 143 if (packet_needs_cookie) { 144 wg_packet_send_handshake_cookie(wg, skb, 145 message->sender_index); 146 return; 147 } 148 peer = wg_noise_handshake_consume_initiation(message, wg); 149 if (unlikely(!peer)) { 150 net_dbg_skb_ratelimited("%s: Invalid handshake initiation from %pISpfsc\n", 151 wg->dev->name, skb); 152 return; 153 } 154 wg_socket_set_peer_endpoint_from_skb(peer, skb); 155 net_dbg_ratelimited("%s: Receiving handshake initiation from peer %llu (%pISpfsc)\n", 156 wg->dev->name, peer->internal_id, 157 &peer->endpoint.addr); 158 wg_packet_send_handshake_response(peer); 159 break; 160 } 161 case cpu_to_le32(MESSAGE_HANDSHAKE_RESPONSE): { 162 struct message_handshake_response *message = 163 (struct message_handshake_response *)skb->data; 164 165 if (packet_needs_cookie) { 166 wg_packet_send_handshake_cookie(wg, skb, 167 message->sender_index); 168 return; 169 } 170 peer = wg_noise_handshake_consume_response(message, wg); 171 if (unlikely(!peer)) { 172 net_dbg_skb_ratelimited("%s: Invalid handshake response from %pISpfsc\n", 173 wg->dev->name, skb); 174 return; 175 } 176 wg_socket_set_peer_endpoint_from_skb(peer, skb); 177 net_dbg_ratelimited("%s: Receiving handshake response from peer %llu (%pISpfsc)\n", 178 wg->dev->name, peer->internal_id, 179 &peer->endpoint.addr); 180 if (wg_noise_handshake_begin_session(&peer->handshake, 181 &peer->keypairs)) { 182 wg_timers_session_derived(peer); 183 wg_timers_handshake_complete(peer); 184 /* Calling this function will either send any existing 185 * packets in the queue and not send a keepalive, which 186 * is the best case, Or, if there's nothing in the 187 * queue, it will send a keepalive, in order to give 188 * immediate confirmation of the session. 189 */ 190 wg_packet_send_keepalive(peer); 191 } 192 break; 193 } 194 } 195 196 if (unlikely(!peer)) { 197 WARN(1, "Somehow a wrong type of packet wound up in the handshake queue!\n"); 198 return; 199 } 200 201 local_bh_disable(); 202 update_rx_stats(peer, skb->len); 203 local_bh_enable(); 204 205 wg_timers_any_authenticated_packet_received(peer); 206 wg_timers_any_authenticated_packet_traversal(peer); 207 wg_peer_put(peer); 208 } 209 210 void wg_packet_handshake_receive_worker(struct work_struct *work) 211 { 212 struct wg_device *wg = container_of(work, struct multicore_worker, 213 work)->ptr; 214 struct sk_buff *skb; 215 216 while ((skb = skb_dequeue(&wg->incoming_handshakes)) != NULL) { 217 wg_receive_handshake_packet(wg, skb); 218 dev_kfree_skb(skb); 219 cond_resched(); 220 } 221 } 222 223 static void keep_key_fresh(struct wg_peer *peer) 224 { 225 struct noise_keypair *keypair; 226 bool send = false; 227 228 if (peer->sent_lastminute_handshake) 229 return; 230 231 rcu_read_lock_bh(); 232 keypair = rcu_dereference_bh(peer->keypairs.current_keypair); 233 if (likely(keypair && READ_ONCE(keypair->sending.is_valid)) && 234 keypair->i_am_the_initiator && 235 unlikely(wg_birthdate_has_expired(keypair->sending.birthdate, 236 REJECT_AFTER_TIME - KEEPALIVE_TIMEOUT - REKEY_TIMEOUT))) 237 send = true; 238 rcu_read_unlock_bh(); 239 240 if (send) { 241 peer->sent_lastminute_handshake = true; 242 wg_packet_send_queued_handshake_initiation(peer, false); 243 } 244 } 245 246 static bool decrypt_packet(struct sk_buff *skb, struct noise_symmetric_key *key) 247 { 248 struct scatterlist sg[MAX_SKB_FRAGS + 8]; 249 struct sk_buff *trailer; 250 unsigned int offset; 251 int num_frags; 252 253 if (unlikely(!key)) 254 return false; 255 256 if (unlikely(!READ_ONCE(key->is_valid) || 257 wg_birthdate_has_expired(key->birthdate, REJECT_AFTER_TIME) || 258 key->counter.receive.counter >= REJECT_AFTER_MESSAGES)) { 259 WRITE_ONCE(key->is_valid, false); 260 return false; 261 } 262 263 PACKET_CB(skb)->nonce = 264 le64_to_cpu(((struct message_data *)skb->data)->counter); 265 266 /* We ensure that the network header is part of the packet before we 267 * call skb_cow_data, so that there's no chance that data is removed 268 * from the skb, so that later we can extract the original endpoint. 269 */ 270 offset = skb->data - skb_network_header(skb); 271 skb_push(skb, offset); 272 num_frags = skb_cow_data(skb, 0, &trailer); 273 offset += sizeof(struct message_data); 274 skb_pull(skb, offset); 275 if (unlikely(num_frags < 0 || num_frags > ARRAY_SIZE(sg))) 276 return false; 277 278 sg_init_table(sg, num_frags); 279 if (skb_to_sgvec(skb, sg, 0, skb->len) <= 0) 280 return false; 281 282 if (!chacha20poly1305_decrypt_sg_inplace(sg, skb->len, NULL, 0, 283 PACKET_CB(skb)->nonce, 284 key->key)) 285 return false; 286 287 /* Another ugly situation of pushing and pulling the header so as to 288 * keep endpoint information intact. 289 */ 290 skb_push(skb, offset); 291 if (pskb_trim(skb, skb->len - noise_encrypted_len(0))) 292 return false; 293 skb_pull(skb, offset); 294 295 return true; 296 } 297 298 /* This is RFC6479, a replay detection bitmap algorithm that avoids bitshifts */ 299 static bool counter_validate(union noise_counter *counter, u64 their_counter) 300 { 301 unsigned long index, index_current, top, i; 302 bool ret = false; 303 304 spin_lock_bh(&counter->receive.lock); 305 306 if (unlikely(counter->receive.counter >= REJECT_AFTER_MESSAGES + 1 || 307 their_counter >= REJECT_AFTER_MESSAGES)) 308 goto out; 309 310 ++their_counter; 311 312 if (unlikely((COUNTER_WINDOW_SIZE + their_counter) < 313 counter->receive.counter)) 314 goto out; 315 316 index = their_counter >> ilog2(BITS_PER_LONG); 317 318 if (likely(their_counter > counter->receive.counter)) { 319 index_current = counter->receive.counter >> ilog2(BITS_PER_LONG); 320 top = min_t(unsigned long, index - index_current, 321 COUNTER_BITS_TOTAL / BITS_PER_LONG); 322 for (i = 1; i <= top; ++i) 323 counter->receive.backtrack[(i + index_current) & 324 ((COUNTER_BITS_TOTAL / BITS_PER_LONG) - 1)] = 0; 325 counter->receive.counter = their_counter; 326 } 327 328 index &= (COUNTER_BITS_TOTAL / BITS_PER_LONG) - 1; 329 ret = !test_and_set_bit(their_counter & (BITS_PER_LONG - 1), 330 &counter->receive.backtrack[index]); 331 332 out: 333 spin_unlock_bh(&counter->receive.lock); 334 return ret; 335 } 336 337 #include "selftest/counter.c" 338 339 static void wg_packet_consume_data_done(struct wg_peer *peer, 340 struct sk_buff *skb, 341 struct endpoint *endpoint) 342 { 343 struct net_device *dev = peer->device->dev; 344 unsigned int len, len_before_trim; 345 struct wg_peer *routed_peer; 346 347 wg_socket_set_peer_endpoint(peer, endpoint); 348 349 if (unlikely(wg_noise_received_with_keypair(&peer->keypairs, 350 PACKET_CB(skb)->keypair))) { 351 wg_timers_handshake_complete(peer); 352 wg_packet_send_staged_packets(peer); 353 } 354 355 keep_key_fresh(peer); 356 357 wg_timers_any_authenticated_packet_received(peer); 358 wg_timers_any_authenticated_packet_traversal(peer); 359 360 /* A packet with length 0 is a keepalive packet */ 361 if (unlikely(!skb->len)) { 362 update_rx_stats(peer, message_data_len(0)); 363 net_dbg_ratelimited("%s: Receiving keepalive packet from peer %llu (%pISpfsc)\n", 364 dev->name, peer->internal_id, 365 &peer->endpoint.addr); 366 goto packet_processed; 367 } 368 369 wg_timers_data_received(peer); 370 371 if (unlikely(skb_network_header(skb) < skb->head)) 372 goto dishonest_packet_size; 373 if (unlikely(!(pskb_network_may_pull(skb, sizeof(struct iphdr)) && 374 (ip_hdr(skb)->version == 4 || 375 (ip_hdr(skb)->version == 6 && 376 pskb_network_may_pull(skb, sizeof(struct ipv6hdr))))))) 377 goto dishonest_packet_type; 378 379 skb->dev = dev; 380 /* We've already verified the Poly1305 auth tag, which means this packet 381 * was not modified in transit. We can therefore tell the networking 382 * stack that all checksums of every layer of encapsulation have already 383 * been checked "by the hardware" and therefore is unnecessary to check 384 * again in software. 385 */ 386 skb->ip_summed = CHECKSUM_UNNECESSARY; 387 skb->csum_level = ~0; /* All levels */ 388 skb->protocol = wg_skb_examine_untrusted_ip_hdr(skb); 389 if (skb->protocol == htons(ETH_P_IP)) { 390 len = ntohs(ip_hdr(skb)->tot_len); 391 if (unlikely(len < sizeof(struct iphdr))) 392 goto dishonest_packet_size; 393 if (INET_ECN_is_ce(PACKET_CB(skb)->ds)) 394 IP_ECN_set_ce(ip_hdr(skb)); 395 } else if (skb->protocol == htons(ETH_P_IPV6)) { 396 len = ntohs(ipv6_hdr(skb)->payload_len) + 397 sizeof(struct ipv6hdr); 398 if (INET_ECN_is_ce(PACKET_CB(skb)->ds)) 399 IP6_ECN_set_ce(skb, ipv6_hdr(skb)); 400 } else { 401 goto dishonest_packet_type; 402 } 403 404 if (unlikely(len > skb->len)) 405 goto dishonest_packet_size; 406 len_before_trim = skb->len; 407 if (unlikely(pskb_trim(skb, len))) 408 goto packet_processed; 409 410 routed_peer = wg_allowedips_lookup_src(&peer->device->peer_allowedips, 411 skb); 412 wg_peer_put(routed_peer); /* We don't need the extra reference. */ 413 414 if (unlikely(routed_peer != peer)) 415 goto dishonest_packet_peer; 416 417 if (unlikely(napi_gro_receive(&peer->napi, skb) == GRO_DROP)) { 418 ++dev->stats.rx_dropped; 419 net_dbg_ratelimited("%s: Failed to give packet to userspace from peer %llu (%pISpfsc)\n", 420 dev->name, peer->internal_id, 421 &peer->endpoint.addr); 422 } else { 423 update_rx_stats(peer, message_data_len(len_before_trim)); 424 } 425 return; 426 427 dishonest_packet_peer: 428 net_dbg_skb_ratelimited("%s: Packet has unallowed src IP (%pISc) from peer %llu (%pISpfsc)\n", 429 dev->name, skb, peer->internal_id, 430 &peer->endpoint.addr); 431 ++dev->stats.rx_errors; 432 ++dev->stats.rx_frame_errors; 433 goto packet_processed; 434 dishonest_packet_type: 435 net_dbg_ratelimited("%s: Packet is neither ipv4 nor ipv6 from peer %llu (%pISpfsc)\n", 436 dev->name, peer->internal_id, &peer->endpoint.addr); 437 ++dev->stats.rx_errors; 438 ++dev->stats.rx_frame_errors; 439 goto packet_processed; 440 dishonest_packet_size: 441 net_dbg_ratelimited("%s: Packet has incorrect size from peer %llu (%pISpfsc)\n", 442 dev->name, peer->internal_id, &peer->endpoint.addr); 443 ++dev->stats.rx_errors; 444 ++dev->stats.rx_length_errors; 445 goto packet_processed; 446 packet_processed: 447 dev_kfree_skb(skb); 448 } 449 450 int wg_packet_rx_poll(struct napi_struct *napi, int budget) 451 { 452 struct wg_peer *peer = container_of(napi, struct wg_peer, napi); 453 struct crypt_queue *queue = &peer->rx_queue; 454 struct noise_keypair *keypair; 455 struct endpoint endpoint; 456 enum packet_state state; 457 struct sk_buff *skb; 458 int work_done = 0; 459 bool free; 460 461 if (unlikely(budget <= 0)) 462 return 0; 463 464 while ((skb = __ptr_ring_peek(&queue->ring)) != NULL && 465 (state = atomic_read_acquire(&PACKET_CB(skb)->state)) != 466 PACKET_STATE_UNCRYPTED) { 467 __ptr_ring_discard_one(&queue->ring); 468 peer = PACKET_PEER(skb); 469 keypair = PACKET_CB(skb)->keypair; 470 free = true; 471 472 if (unlikely(state != PACKET_STATE_CRYPTED)) 473 goto next; 474 475 if (unlikely(!counter_validate(&keypair->receiving.counter, 476 PACKET_CB(skb)->nonce))) { 477 net_dbg_ratelimited("%s: Packet has invalid nonce %llu (max %llu)\n", 478 peer->device->dev->name, 479 PACKET_CB(skb)->nonce, 480 keypair->receiving.counter.receive.counter); 481 goto next; 482 } 483 484 if (unlikely(wg_socket_endpoint_from_skb(&endpoint, skb))) 485 goto next; 486 487 wg_reset_packet(skb); 488 wg_packet_consume_data_done(peer, skb, &endpoint); 489 free = false; 490 491 next: 492 wg_noise_keypair_put(keypair, false); 493 wg_peer_put(peer); 494 if (unlikely(free)) 495 dev_kfree_skb(skb); 496 497 if (++work_done >= budget) 498 break; 499 } 500 501 if (work_done < budget) 502 napi_complete_done(napi, work_done); 503 504 return work_done; 505 } 506 507 void wg_packet_decrypt_worker(struct work_struct *work) 508 { 509 struct crypt_queue *queue = container_of(work, struct multicore_worker, 510 work)->ptr; 511 struct sk_buff *skb; 512 513 while ((skb = ptr_ring_consume_bh(&queue->ring)) != NULL) { 514 enum packet_state state = likely(decrypt_packet(skb, 515 &PACKET_CB(skb)->keypair->receiving)) ? 516 PACKET_STATE_CRYPTED : PACKET_STATE_DEAD; 517 wg_queue_enqueue_per_peer_napi(skb, state); 518 } 519 } 520 521 static void wg_packet_consume_data(struct wg_device *wg, struct sk_buff *skb) 522 { 523 __le32 idx = ((struct message_data *)skb->data)->key_idx; 524 struct wg_peer *peer = NULL; 525 int ret; 526 527 rcu_read_lock_bh(); 528 PACKET_CB(skb)->keypair = 529 (struct noise_keypair *)wg_index_hashtable_lookup( 530 wg->index_hashtable, INDEX_HASHTABLE_KEYPAIR, idx, 531 &peer); 532 if (unlikely(!wg_noise_keypair_get(PACKET_CB(skb)->keypair))) 533 goto err_keypair; 534 535 if (unlikely(READ_ONCE(peer->is_dead))) 536 goto err; 537 538 ret = wg_queue_enqueue_per_device_and_peer(&wg->decrypt_queue, 539 &peer->rx_queue, skb, 540 wg->packet_crypt_wq, 541 &wg->decrypt_queue.last_cpu); 542 if (unlikely(ret == -EPIPE)) 543 wg_queue_enqueue_per_peer_napi(skb, PACKET_STATE_DEAD); 544 if (likely(!ret || ret == -EPIPE)) { 545 rcu_read_unlock_bh(); 546 return; 547 } 548 err: 549 wg_noise_keypair_put(PACKET_CB(skb)->keypair, false); 550 err_keypair: 551 rcu_read_unlock_bh(); 552 wg_peer_put(peer); 553 dev_kfree_skb(skb); 554 } 555 556 void wg_packet_receive(struct wg_device *wg, struct sk_buff *skb) 557 { 558 if (unlikely(prepare_skb_header(skb, wg) < 0)) 559 goto err; 560 switch (SKB_TYPE_LE32(skb)) { 561 case cpu_to_le32(MESSAGE_HANDSHAKE_INITIATION): 562 case cpu_to_le32(MESSAGE_HANDSHAKE_RESPONSE): 563 case cpu_to_le32(MESSAGE_HANDSHAKE_COOKIE): { 564 int cpu; 565 566 if (skb_queue_len(&wg->incoming_handshakes) > 567 MAX_QUEUED_INCOMING_HANDSHAKES || 568 unlikely(!rng_is_initialized())) { 569 net_dbg_skb_ratelimited("%s: Dropping handshake packet from %pISpfsc\n", 570 wg->dev->name, skb); 571 goto err; 572 } 573 skb_queue_tail(&wg->incoming_handshakes, skb); 574 /* Queues up a call to packet_process_queued_handshake_ 575 * packets(skb): 576 */ 577 cpu = wg_cpumask_next_online(&wg->incoming_handshake_cpu); 578 queue_work_on(cpu, wg->handshake_receive_wq, 579 &per_cpu_ptr(wg->incoming_handshakes_worker, cpu)->work); 580 break; 581 } 582 case cpu_to_le32(MESSAGE_DATA): 583 PACKET_CB(skb)->ds = ip_tunnel_get_dsfield(ip_hdr(skb), skb); 584 wg_packet_consume_data(wg, skb); 585 break; 586 default: 587 net_dbg_skb_ratelimited("%s: Invalid packet from %pISpfsc\n", 588 wg->dev->name, skb); 589 goto err; 590 } 591 return; 592 593 err: 594 dev_kfree_skb(skb); 595 } 596