1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. 4 */ 5 6 #include "queueing.h" 7 #include "timers.h" 8 #include "device.h" 9 #include "peer.h" 10 #include "socket.h" 11 #include "messages.h" 12 #include "cookie.h" 13 14 #include <linux/uio.h> 15 #include <linux/inetdevice.h> 16 #include <linux/socket.h> 17 #include <net/ip_tunnels.h> 18 #include <net/udp.h> 19 #include <net/sock.h> 20 21 static void wg_packet_send_handshake_initiation(struct wg_peer *peer) 22 { 23 struct message_handshake_initiation packet; 24 25 if (!wg_birthdate_has_expired(atomic64_read(&peer->last_sent_handshake), 26 REKEY_TIMEOUT)) 27 return; /* This function is rate limited. */ 28 29 atomic64_set(&peer->last_sent_handshake, ktime_get_coarse_boottime_ns()); 30 net_dbg_ratelimited("%s: Sending handshake initiation to peer %llu (%pISpfsc)\n", 31 peer->device->dev->name, peer->internal_id, 32 &peer->endpoint.addr); 33 34 if (wg_noise_handshake_create_initiation(&packet, &peer->handshake)) { 35 wg_cookie_add_mac_to_packet(&packet, sizeof(packet), peer); 36 wg_timers_any_authenticated_packet_traversal(peer); 37 wg_timers_any_authenticated_packet_sent(peer); 38 atomic64_set(&peer->last_sent_handshake, 39 ktime_get_coarse_boottime_ns()); 40 wg_socket_send_buffer_to_peer(peer, &packet, sizeof(packet), 41 HANDSHAKE_DSCP); 42 wg_timers_handshake_initiated(peer); 43 } 44 } 45 46 void wg_packet_handshake_send_worker(struct work_struct *work) 47 { 48 struct wg_peer *peer = container_of(work, struct wg_peer, 49 transmit_handshake_work); 50 51 wg_packet_send_handshake_initiation(peer); 52 wg_peer_put(peer); 53 } 54 55 void wg_packet_send_queued_handshake_initiation(struct wg_peer *peer, 56 bool is_retry) 57 { 58 if (!is_retry) 59 peer->timer_handshake_attempts = 0; 60 61 rcu_read_lock_bh(); 62 /* We check last_sent_handshake here in addition to the actual function 63 * we're queueing up, so that we don't queue things if not strictly 64 * necessary: 65 */ 66 if (!wg_birthdate_has_expired(atomic64_read(&peer->last_sent_handshake), 67 REKEY_TIMEOUT) || 68 unlikely(READ_ONCE(peer->is_dead))) 69 goto out; 70 71 wg_peer_get(peer); 72 /* Queues up calling packet_send_queued_handshakes(peer), where we do a 73 * peer_put(peer) after: 74 */ 75 if (!queue_work(peer->device->handshake_send_wq, 76 &peer->transmit_handshake_work)) 77 /* If the work was already queued, we want to drop the 78 * extra reference: 79 */ 80 wg_peer_put(peer); 81 out: 82 rcu_read_unlock_bh(); 83 } 84 85 void wg_packet_send_handshake_response(struct wg_peer *peer) 86 { 87 struct message_handshake_response packet; 88 89 atomic64_set(&peer->last_sent_handshake, ktime_get_coarse_boottime_ns()); 90 net_dbg_ratelimited("%s: Sending handshake response to peer %llu (%pISpfsc)\n", 91 peer->device->dev->name, peer->internal_id, 92 &peer->endpoint.addr); 93 94 if (wg_noise_handshake_create_response(&packet, &peer->handshake)) { 95 wg_cookie_add_mac_to_packet(&packet, sizeof(packet), peer); 96 if (wg_noise_handshake_begin_session(&peer->handshake, 97 &peer->keypairs)) { 98 wg_timers_session_derived(peer); 99 wg_timers_any_authenticated_packet_traversal(peer); 100 wg_timers_any_authenticated_packet_sent(peer); 101 atomic64_set(&peer->last_sent_handshake, 102 ktime_get_coarse_boottime_ns()); 103 wg_socket_send_buffer_to_peer(peer, &packet, 104 sizeof(packet), 105 HANDSHAKE_DSCP); 106 } 107 } 108 } 109 110 void wg_packet_send_handshake_cookie(struct wg_device *wg, 111 struct sk_buff *initiating_skb, 112 __le32 sender_index) 113 { 114 struct message_handshake_cookie packet; 115 116 net_dbg_skb_ratelimited("%s: Sending cookie response for denied handshake message for %pISpfsc\n", 117 wg->dev->name, initiating_skb); 118 wg_cookie_message_create(&packet, initiating_skb, sender_index, 119 &wg->cookie_checker); 120 wg_socket_send_buffer_as_reply_to_skb(wg, initiating_skb, &packet, 121 sizeof(packet)); 122 } 123 124 static void keep_key_fresh(struct wg_peer *peer) 125 { 126 struct noise_keypair *keypair; 127 bool send; 128 129 rcu_read_lock_bh(); 130 keypair = rcu_dereference_bh(peer->keypairs.current_keypair); 131 send = keypair && READ_ONCE(keypair->sending.is_valid) && 132 (atomic64_read(&keypair->sending_counter) > REKEY_AFTER_MESSAGES || 133 (keypair->i_am_the_initiator && 134 wg_birthdate_has_expired(keypair->sending.birthdate, REKEY_AFTER_TIME))); 135 rcu_read_unlock_bh(); 136 137 if (unlikely(send)) 138 wg_packet_send_queued_handshake_initiation(peer, false); 139 } 140 141 static unsigned int calculate_skb_padding(struct sk_buff *skb) 142 { 143 unsigned int padded_size, last_unit = skb->len; 144 145 if (unlikely(!PACKET_CB(skb)->mtu)) 146 return ALIGN(last_unit, MESSAGE_PADDING_MULTIPLE) - last_unit; 147 148 /* We do this modulo business with the MTU, just in case the networking 149 * layer gives us a packet that's bigger than the MTU. In that case, we 150 * wouldn't want the final subtraction to overflow in the case of the 151 * padded_size being clamped. Fortunately, that's very rarely the case, 152 * so we optimize for that not happening. 153 */ 154 if (unlikely(last_unit > PACKET_CB(skb)->mtu)) 155 last_unit %= PACKET_CB(skb)->mtu; 156 157 padded_size = min(PACKET_CB(skb)->mtu, 158 ALIGN(last_unit, MESSAGE_PADDING_MULTIPLE)); 159 return padded_size - last_unit; 160 } 161 162 static bool encrypt_packet(struct sk_buff *skb, struct noise_keypair *keypair) 163 { 164 unsigned int padding_len, plaintext_len, trailer_len; 165 struct scatterlist sg[MAX_SKB_FRAGS + 8]; 166 struct message_data *header; 167 struct sk_buff *trailer; 168 int num_frags; 169 170 /* Force hash calculation before encryption so that flow analysis is 171 * consistent over the inner packet. 172 */ 173 skb_get_hash(skb); 174 175 /* Calculate lengths. */ 176 padding_len = calculate_skb_padding(skb); 177 trailer_len = padding_len + noise_encrypted_len(0); 178 plaintext_len = skb->len + padding_len; 179 180 /* Expand data section to have room for padding and auth tag. */ 181 num_frags = skb_cow_data(skb, trailer_len, &trailer); 182 if (unlikely(num_frags < 0 || num_frags > ARRAY_SIZE(sg))) 183 return false; 184 185 /* Set the padding to zeros, and make sure it and the auth tag are part 186 * of the skb. 187 */ 188 memset(skb_tail_pointer(trailer), 0, padding_len); 189 190 /* Expand head section to have room for our header and the network 191 * stack's headers. 192 */ 193 if (unlikely(skb_cow_head(skb, DATA_PACKET_HEAD_ROOM) < 0)) 194 return false; 195 196 /* Finalize checksum calculation for the inner packet, if required. */ 197 if (unlikely(skb->ip_summed == CHECKSUM_PARTIAL && 198 skb_checksum_help(skb))) 199 return false; 200 201 /* Only after checksumming can we safely add on the padding at the end 202 * and the header. 203 */ 204 skb_set_inner_network_header(skb, 0); 205 header = (struct message_data *)skb_push(skb, sizeof(*header)); 206 header->header.type = cpu_to_le32(MESSAGE_DATA); 207 header->key_idx = keypair->remote_index; 208 header->counter = cpu_to_le64(PACKET_CB(skb)->nonce); 209 pskb_put(skb, trailer, trailer_len); 210 211 /* Now we can encrypt the scattergather segments */ 212 sg_init_table(sg, num_frags); 213 if (skb_to_sgvec(skb, sg, sizeof(struct message_data), 214 noise_encrypted_len(plaintext_len)) <= 0) 215 return false; 216 return chacha20poly1305_encrypt_sg_inplace(sg, plaintext_len, NULL, 0, 217 PACKET_CB(skb)->nonce, 218 keypair->sending.key); 219 } 220 221 void wg_packet_send_keepalive(struct wg_peer *peer) 222 { 223 struct sk_buff *skb; 224 225 if (skb_queue_empty(&peer->staged_packet_queue)) { 226 skb = alloc_skb(DATA_PACKET_HEAD_ROOM + MESSAGE_MINIMUM_LENGTH, 227 GFP_ATOMIC); 228 if (unlikely(!skb)) 229 return; 230 skb_reserve(skb, DATA_PACKET_HEAD_ROOM); 231 skb->dev = peer->device->dev; 232 PACKET_CB(skb)->mtu = skb->dev->mtu; 233 skb_queue_tail(&peer->staged_packet_queue, skb); 234 net_dbg_ratelimited("%s: Sending keepalive packet to peer %llu (%pISpfsc)\n", 235 peer->device->dev->name, peer->internal_id, 236 &peer->endpoint.addr); 237 } 238 239 wg_packet_send_staged_packets(peer); 240 } 241 242 static void wg_packet_create_data_done(struct wg_peer *peer, struct sk_buff *first) 243 { 244 struct sk_buff *skb, *next; 245 bool is_keepalive, data_sent = false; 246 247 wg_timers_any_authenticated_packet_traversal(peer); 248 wg_timers_any_authenticated_packet_sent(peer); 249 skb_list_walk_safe(first, skb, next) { 250 is_keepalive = skb->len == message_data_len(0); 251 if (likely(!wg_socket_send_skb_to_peer(peer, skb, 252 PACKET_CB(skb)->ds) && !is_keepalive)) 253 data_sent = true; 254 } 255 256 if (likely(data_sent)) 257 wg_timers_data_sent(peer); 258 259 keep_key_fresh(peer); 260 } 261 262 void wg_packet_tx_worker(struct work_struct *work) 263 { 264 struct wg_peer *peer = container_of(work, struct wg_peer, transmit_packet_work); 265 struct noise_keypair *keypair; 266 enum packet_state state; 267 struct sk_buff *first; 268 269 while ((first = wg_prev_queue_peek(&peer->tx_queue)) != NULL && 270 (state = atomic_read_acquire(&PACKET_CB(first)->state)) != 271 PACKET_STATE_UNCRYPTED) { 272 wg_prev_queue_drop_peeked(&peer->tx_queue); 273 keypair = PACKET_CB(first)->keypair; 274 275 if (likely(state == PACKET_STATE_CRYPTED)) 276 wg_packet_create_data_done(peer, first); 277 else 278 kfree_skb_list(first); 279 280 wg_noise_keypair_put(keypair, false); 281 wg_peer_put(peer); 282 if (need_resched()) 283 cond_resched(); 284 } 285 } 286 287 void wg_packet_encrypt_worker(struct work_struct *work) 288 { 289 struct crypt_queue *queue = container_of(work, struct multicore_worker, 290 work)->ptr; 291 struct sk_buff *first, *skb, *next; 292 293 while ((first = ptr_ring_consume_bh(&queue->ring)) != NULL) { 294 enum packet_state state = PACKET_STATE_CRYPTED; 295 296 skb_list_walk_safe(first, skb, next) { 297 if (likely(encrypt_packet(skb, 298 PACKET_CB(first)->keypair))) { 299 wg_reset_packet(skb, true); 300 } else { 301 state = PACKET_STATE_DEAD; 302 break; 303 } 304 } 305 wg_queue_enqueue_per_peer_tx(first, state); 306 if (need_resched()) 307 cond_resched(); 308 } 309 } 310 311 static void wg_packet_create_data(struct wg_peer *peer, struct sk_buff *first) 312 { 313 struct wg_device *wg = peer->device; 314 int ret = -EINVAL; 315 316 rcu_read_lock_bh(); 317 if (unlikely(READ_ONCE(peer->is_dead))) 318 goto err; 319 320 ret = wg_queue_enqueue_per_device_and_peer(&wg->encrypt_queue, &peer->tx_queue, first, 321 wg->packet_crypt_wq, &wg->encrypt_queue.last_cpu); 322 if (unlikely(ret == -EPIPE)) 323 wg_queue_enqueue_per_peer_tx(first, PACKET_STATE_DEAD); 324 err: 325 rcu_read_unlock_bh(); 326 if (likely(!ret || ret == -EPIPE)) 327 return; 328 wg_noise_keypair_put(PACKET_CB(first)->keypair, false); 329 wg_peer_put(peer); 330 kfree_skb_list(first); 331 } 332 333 void wg_packet_purge_staged_packets(struct wg_peer *peer) 334 { 335 spin_lock_bh(&peer->staged_packet_queue.lock); 336 peer->device->dev->stats.tx_dropped += peer->staged_packet_queue.qlen; 337 __skb_queue_purge(&peer->staged_packet_queue); 338 spin_unlock_bh(&peer->staged_packet_queue.lock); 339 } 340 341 void wg_packet_send_staged_packets(struct wg_peer *peer) 342 { 343 struct noise_keypair *keypair; 344 struct sk_buff_head packets; 345 struct sk_buff *skb; 346 347 /* Steal the current queue into our local one. */ 348 __skb_queue_head_init(&packets); 349 spin_lock_bh(&peer->staged_packet_queue.lock); 350 skb_queue_splice_init(&peer->staged_packet_queue, &packets); 351 spin_unlock_bh(&peer->staged_packet_queue.lock); 352 if (unlikely(skb_queue_empty(&packets))) 353 return; 354 355 /* First we make sure we have a valid reference to a valid key. */ 356 rcu_read_lock_bh(); 357 keypair = wg_noise_keypair_get( 358 rcu_dereference_bh(peer->keypairs.current_keypair)); 359 rcu_read_unlock_bh(); 360 if (unlikely(!keypair)) 361 goto out_nokey; 362 if (unlikely(!READ_ONCE(keypair->sending.is_valid))) 363 goto out_nokey; 364 if (unlikely(wg_birthdate_has_expired(keypair->sending.birthdate, 365 REJECT_AFTER_TIME))) 366 goto out_invalid; 367 368 /* After we know we have a somewhat valid key, we now try to assign 369 * nonces to all of the packets in the queue. If we can't assign nonces 370 * for all of them, we just consider it a failure and wait for the next 371 * handshake. 372 */ 373 skb_queue_walk(&packets, skb) { 374 /* 0 for no outer TOS: no leak. TODO: at some later point, we 375 * might consider using flowi->tos as outer instead. 376 */ 377 PACKET_CB(skb)->ds = ip_tunnel_ecn_encap(0, ip_hdr(skb), skb); 378 PACKET_CB(skb)->nonce = 379 atomic64_inc_return(&keypair->sending_counter) - 1; 380 if (unlikely(PACKET_CB(skb)->nonce >= REJECT_AFTER_MESSAGES)) 381 goto out_invalid; 382 } 383 384 packets.prev->next = NULL; 385 wg_peer_get(keypair->entry.peer); 386 PACKET_CB(packets.next)->keypair = keypair; 387 wg_packet_create_data(peer, packets.next); 388 return; 389 390 out_invalid: 391 WRITE_ONCE(keypair->sending.is_valid, false); 392 out_nokey: 393 wg_noise_keypair_put(keypair, false); 394 395 /* We orphan the packets if we're waiting on a handshake, so that they 396 * don't block a socket's pool. 397 */ 398 skb_queue_walk(&packets, skb) 399 skb_orphan(skb); 400 /* Then we put them back on the top of the queue. We're not too 401 * concerned about accidentally getting things a little out of order if 402 * packets are being added really fast, because this queue is for before 403 * packets can even be sent and it's small anyway. 404 */ 405 spin_lock_bh(&peer->staged_packet_queue.lock); 406 skb_queue_splice(&packets, &peer->staged_packet_queue); 407 spin_unlock_bh(&peer->staged_packet_queue.lock); 408 409 /* If we're exiting because there's something wrong with the key, it 410 * means we should initiate a new handshake. 411 */ 412 wg_packet_send_queued_handshake_initiation(peer, false); 413 } 414