1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. 4 */ 5 6 #include "queueing.h" 7 #include "timers.h" 8 #include "device.h" 9 #include "peer.h" 10 #include "socket.h" 11 #include "messages.h" 12 #include "cookie.h" 13 14 #include <linux/uio.h> 15 #include <linux/inetdevice.h> 16 #include <linux/socket.h> 17 #include <net/ip_tunnels.h> 18 #include <net/udp.h> 19 #include <net/sock.h> 20 21 static void wg_packet_send_handshake_initiation(struct wg_peer *peer) 22 { 23 struct message_handshake_initiation packet; 24 25 if (!wg_birthdate_has_expired(atomic64_read(&peer->last_sent_handshake), 26 REKEY_TIMEOUT)) 27 return; /* This function is rate limited. */ 28 29 atomic64_set(&peer->last_sent_handshake, ktime_get_coarse_boottime_ns()); 30 net_dbg_ratelimited("%s: Sending handshake initiation to peer %llu (%pISpfsc)\n", 31 peer->device->dev->name, peer->internal_id, 32 &peer->endpoint.addr); 33 34 if (wg_noise_handshake_create_initiation(&packet, &peer->handshake)) { 35 wg_cookie_add_mac_to_packet(&packet, sizeof(packet), peer); 36 wg_timers_any_authenticated_packet_traversal(peer); 37 wg_timers_any_authenticated_packet_sent(peer); 38 atomic64_set(&peer->last_sent_handshake, 39 ktime_get_coarse_boottime_ns()); 40 wg_socket_send_buffer_to_peer(peer, &packet, sizeof(packet), 41 HANDSHAKE_DSCP); 42 wg_timers_handshake_initiated(peer); 43 } 44 } 45 46 void wg_packet_handshake_send_worker(struct work_struct *work) 47 { 48 struct wg_peer *peer = container_of(work, struct wg_peer, 49 transmit_handshake_work); 50 51 wg_packet_send_handshake_initiation(peer); 52 wg_peer_put(peer); 53 } 54 55 void wg_packet_send_queued_handshake_initiation(struct wg_peer *peer, 56 bool is_retry) 57 { 58 if (!is_retry) 59 peer->timer_handshake_attempts = 0; 60 61 rcu_read_lock_bh(); 62 /* We check last_sent_handshake here in addition to the actual function 63 * we're queueing up, so that we don't queue things if not strictly 64 * necessary: 65 */ 66 if (!wg_birthdate_has_expired(atomic64_read(&peer->last_sent_handshake), 67 REKEY_TIMEOUT) || 68 unlikely(READ_ONCE(peer->is_dead))) 69 goto out; 70 71 wg_peer_get(peer); 72 /* Queues up calling packet_send_queued_handshakes(peer), where we do a 73 * peer_put(peer) after: 74 */ 75 if (!queue_work(peer->device->handshake_send_wq, 76 &peer->transmit_handshake_work)) 77 /* If the work was already queued, we want to drop the 78 * extra reference: 79 */ 80 wg_peer_put(peer); 81 out: 82 rcu_read_unlock_bh(); 83 } 84 85 void wg_packet_send_handshake_response(struct wg_peer *peer) 86 { 87 struct message_handshake_response packet; 88 89 atomic64_set(&peer->last_sent_handshake, ktime_get_coarse_boottime_ns()); 90 net_dbg_ratelimited("%s: Sending handshake response to peer %llu (%pISpfsc)\n", 91 peer->device->dev->name, peer->internal_id, 92 &peer->endpoint.addr); 93 94 if (wg_noise_handshake_create_response(&packet, &peer->handshake)) { 95 wg_cookie_add_mac_to_packet(&packet, sizeof(packet), peer); 96 if (wg_noise_handshake_begin_session(&peer->handshake, 97 &peer->keypairs)) { 98 wg_timers_session_derived(peer); 99 wg_timers_any_authenticated_packet_traversal(peer); 100 wg_timers_any_authenticated_packet_sent(peer); 101 atomic64_set(&peer->last_sent_handshake, 102 ktime_get_coarse_boottime_ns()); 103 wg_socket_send_buffer_to_peer(peer, &packet, 104 sizeof(packet), 105 HANDSHAKE_DSCP); 106 } 107 } 108 } 109 110 void wg_packet_send_handshake_cookie(struct wg_device *wg, 111 struct sk_buff *initiating_skb, 112 __le32 sender_index) 113 { 114 struct message_handshake_cookie packet; 115 116 net_dbg_skb_ratelimited("%s: Sending cookie response for denied handshake message for %pISpfsc\n", 117 wg->dev->name, initiating_skb); 118 wg_cookie_message_create(&packet, initiating_skb, sender_index, 119 &wg->cookie_checker); 120 wg_socket_send_buffer_as_reply_to_skb(wg, initiating_skb, &packet, 121 sizeof(packet)); 122 } 123 124 static void keep_key_fresh(struct wg_peer *peer) 125 { 126 struct noise_keypair *keypair; 127 bool send = false; 128 129 rcu_read_lock_bh(); 130 keypair = rcu_dereference_bh(peer->keypairs.current_keypair); 131 if (likely(keypair && READ_ONCE(keypair->sending.is_valid)) && 132 (unlikely(atomic64_read(&keypair->sending.counter.counter) > 133 REKEY_AFTER_MESSAGES) || 134 (keypair->i_am_the_initiator && 135 unlikely(wg_birthdate_has_expired(keypair->sending.birthdate, 136 REKEY_AFTER_TIME))))) 137 send = true; 138 rcu_read_unlock_bh(); 139 140 if (send) 141 wg_packet_send_queued_handshake_initiation(peer, false); 142 } 143 144 static unsigned int calculate_skb_padding(struct sk_buff *skb) 145 { 146 unsigned int padded_size, last_unit = skb->len; 147 148 if (unlikely(!PACKET_CB(skb)->mtu)) 149 return ALIGN(last_unit, MESSAGE_PADDING_MULTIPLE) - last_unit; 150 151 /* We do this modulo business with the MTU, just in case the networking 152 * layer gives us a packet that's bigger than the MTU. In that case, we 153 * wouldn't want the final subtraction to overflow in the case of the 154 * padded_size being clamped. Fortunately, that's very rarely the case, 155 * so we optimize for that not happening. 156 */ 157 if (unlikely(last_unit > PACKET_CB(skb)->mtu)) 158 last_unit %= PACKET_CB(skb)->mtu; 159 160 padded_size = min(PACKET_CB(skb)->mtu, 161 ALIGN(last_unit, MESSAGE_PADDING_MULTIPLE)); 162 return padded_size - last_unit; 163 } 164 165 static bool encrypt_packet(struct sk_buff *skb, struct noise_keypair *keypair) 166 { 167 unsigned int padding_len, plaintext_len, trailer_len; 168 struct scatterlist sg[MAX_SKB_FRAGS + 8]; 169 struct message_data *header; 170 struct sk_buff *trailer; 171 int num_frags; 172 173 /* Calculate lengths. */ 174 padding_len = calculate_skb_padding(skb); 175 trailer_len = padding_len + noise_encrypted_len(0); 176 plaintext_len = skb->len + padding_len; 177 178 /* Expand data section to have room for padding and auth tag. */ 179 num_frags = skb_cow_data(skb, trailer_len, &trailer); 180 if (unlikely(num_frags < 0 || num_frags > ARRAY_SIZE(sg))) 181 return false; 182 183 /* Set the padding to zeros, and make sure it and the auth tag are part 184 * of the skb. 185 */ 186 memset(skb_tail_pointer(trailer), 0, padding_len); 187 188 /* Expand head section to have room for our header and the network 189 * stack's headers. 190 */ 191 if (unlikely(skb_cow_head(skb, DATA_PACKET_HEAD_ROOM) < 0)) 192 return false; 193 194 /* Finalize checksum calculation for the inner packet, if required. */ 195 if (unlikely(skb->ip_summed == CHECKSUM_PARTIAL && 196 skb_checksum_help(skb))) 197 return false; 198 199 /* Only after checksumming can we safely add on the padding at the end 200 * and the header. 201 */ 202 skb_set_inner_network_header(skb, 0); 203 header = (struct message_data *)skb_push(skb, sizeof(*header)); 204 header->header.type = cpu_to_le32(MESSAGE_DATA); 205 header->key_idx = keypair->remote_index; 206 header->counter = cpu_to_le64(PACKET_CB(skb)->nonce); 207 pskb_put(skb, trailer, trailer_len); 208 209 /* Now we can encrypt the scattergather segments */ 210 sg_init_table(sg, num_frags); 211 if (skb_to_sgvec(skb, sg, sizeof(struct message_data), 212 noise_encrypted_len(plaintext_len)) <= 0) 213 return false; 214 return chacha20poly1305_encrypt_sg_inplace(sg, plaintext_len, NULL, 0, 215 PACKET_CB(skb)->nonce, 216 keypair->sending.key); 217 } 218 219 void wg_packet_send_keepalive(struct wg_peer *peer) 220 { 221 struct sk_buff *skb; 222 223 if (skb_queue_empty(&peer->staged_packet_queue)) { 224 skb = alloc_skb(DATA_PACKET_HEAD_ROOM + MESSAGE_MINIMUM_LENGTH, 225 GFP_ATOMIC); 226 if (unlikely(!skb)) 227 return; 228 skb_reserve(skb, DATA_PACKET_HEAD_ROOM); 229 skb->dev = peer->device->dev; 230 PACKET_CB(skb)->mtu = skb->dev->mtu; 231 skb_queue_tail(&peer->staged_packet_queue, skb); 232 net_dbg_ratelimited("%s: Sending keepalive packet to peer %llu (%pISpfsc)\n", 233 peer->device->dev->name, peer->internal_id, 234 &peer->endpoint.addr); 235 } 236 237 wg_packet_send_staged_packets(peer); 238 } 239 240 static void wg_packet_create_data_done(struct sk_buff *first, 241 struct wg_peer *peer) 242 { 243 struct sk_buff *skb, *next; 244 bool is_keepalive, data_sent = false; 245 246 wg_timers_any_authenticated_packet_traversal(peer); 247 wg_timers_any_authenticated_packet_sent(peer); 248 skb_list_walk_safe(first, skb, next) { 249 is_keepalive = skb->len == message_data_len(0); 250 if (likely(!wg_socket_send_skb_to_peer(peer, skb, 251 PACKET_CB(skb)->ds) && !is_keepalive)) 252 data_sent = true; 253 } 254 255 if (likely(data_sent)) 256 wg_timers_data_sent(peer); 257 258 keep_key_fresh(peer); 259 } 260 261 void wg_packet_tx_worker(struct work_struct *work) 262 { 263 struct crypt_queue *queue = container_of(work, struct crypt_queue, 264 work); 265 struct noise_keypair *keypair; 266 enum packet_state state; 267 struct sk_buff *first; 268 struct wg_peer *peer; 269 270 while ((first = __ptr_ring_peek(&queue->ring)) != NULL && 271 (state = atomic_read_acquire(&PACKET_CB(first)->state)) != 272 PACKET_STATE_UNCRYPTED) { 273 __ptr_ring_discard_one(&queue->ring); 274 peer = PACKET_PEER(first); 275 keypair = PACKET_CB(first)->keypair; 276 277 if (likely(state == PACKET_STATE_CRYPTED)) 278 wg_packet_create_data_done(first, peer); 279 else 280 kfree_skb_list(first); 281 282 wg_noise_keypair_put(keypair, false); 283 wg_peer_put(peer); 284 } 285 } 286 287 void wg_packet_encrypt_worker(struct work_struct *work) 288 { 289 struct crypt_queue *queue = container_of(work, struct multicore_worker, 290 work)->ptr; 291 struct sk_buff *first, *skb, *next; 292 293 while ((first = ptr_ring_consume_bh(&queue->ring)) != NULL) { 294 enum packet_state state = PACKET_STATE_CRYPTED; 295 296 skb_list_walk_safe(first, skb, next) { 297 if (likely(encrypt_packet(skb, 298 PACKET_CB(first)->keypair))) { 299 wg_reset_packet(skb); 300 } else { 301 state = PACKET_STATE_DEAD; 302 break; 303 } 304 } 305 wg_queue_enqueue_per_peer(&PACKET_PEER(first)->tx_queue, first, 306 state); 307 308 } 309 } 310 311 static void wg_packet_create_data(struct sk_buff *first) 312 { 313 struct wg_peer *peer = PACKET_PEER(first); 314 struct wg_device *wg = peer->device; 315 int ret = -EINVAL; 316 317 rcu_read_lock_bh(); 318 if (unlikely(READ_ONCE(peer->is_dead))) 319 goto err; 320 321 ret = wg_queue_enqueue_per_device_and_peer(&wg->encrypt_queue, 322 &peer->tx_queue, first, 323 wg->packet_crypt_wq, 324 &wg->encrypt_queue.last_cpu); 325 if (unlikely(ret == -EPIPE)) 326 wg_queue_enqueue_per_peer(&peer->tx_queue, first, 327 PACKET_STATE_DEAD); 328 err: 329 rcu_read_unlock_bh(); 330 if (likely(!ret || ret == -EPIPE)) 331 return; 332 wg_noise_keypair_put(PACKET_CB(first)->keypair, false); 333 wg_peer_put(peer); 334 kfree_skb_list(first); 335 } 336 337 void wg_packet_purge_staged_packets(struct wg_peer *peer) 338 { 339 spin_lock_bh(&peer->staged_packet_queue.lock); 340 peer->device->dev->stats.tx_dropped += peer->staged_packet_queue.qlen; 341 __skb_queue_purge(&peer->staged_packet_queue); 342 spin_unlock_bh(&peer->staged_packet_queue.lock); 343 } 344 345 void wg_packet_send_staged_packets(struct wg_peer *peer) 346 { 347 struct noise_symmetric_key *key; 348 struct noise_keypair *keypair; 349 struct sk_buff_head packets; 350 struct sk_buff *skb; 351 352 /* Steal the current queue into our local one. */ 353 __skb_queue_head_init(&packets); 354 spin_lock_bh(&peer->staged_packet_queue.lock); 355 skb_queue_splice_init(&peer->staged_packet_queue, &packets); 356 spin_unlock_bh(&peer->staged_packet_queue.lock); 357 if (unlikely(skb_queue_empty(&packets))) 358 return; 359 360 /* First we make sure we have a valid reference to a valid key. */ 361 rcu_read_lock_bh(); 362 keypair = wg_noise_keypair_get( 363 rcu_dereference_bh(peer->keypairs.current_keypair)); 364 rcu_read_unlock_bh(); 365 if (unlikely(!keypair)) 366 goto out_nokey; 367 key = &keypair->sending; 368 if (unlikely(!READ_ONCE(key->is_valid))) 369 goto out_nokey; 370 if (unlikely(wg_birthdate_has_expired(key->birthdate, 371 REJECT_AFTER_TIME))) 372 goto out_invalid; 373 374 /* After we know we have a somewhat valid key, we now try to assign 375 * nonces to all of the packets in the queue. If we can't assign nonces 376 * for all of them, we just consider it a failure and wait for the next 377 * handshake. 378 */ 379 skb_queue_walk(&packets, skb) { 380 /* 0 for no outer TOS: no leak. TODO: at some later point, we 381 * might consider using flowi->tos as outer instead. 382 */ 383 PACKET_CB(skb)->ds = ip_tunnel_ecn_encap(0, ip_hdr(skb), skb); 384 PACKET_CB(skb)->nonce = 385 atomic64_inc_return(&key->counter.counter) - 1; 386 if (unlikely(PACKET_CB(skb)->nonce >= REJECT_AFTER_MESSAGES)) 387 goto out_invalid; 388 } 389 390 packets.prev->next = NULL; 391 wg_peer_get(keypair->entry.peer); 392 PACKET_CB(packets.next)->keypair = keypair; 393 wg_packet_create_data(packets.next); 394 return; 395 396 out_invalid: 397 WRITE_ONCE(key->is_valid, false); 398 out_nokey: 399 wg_noise_keypair_put(keypair, false); 400 401 /* We orphan the packets if we're waiting on a handshake, so that they 402 * don't block a socket's pool. 403 */ 404 skb_queue_walk(&packets, skb) 405 skb_orphan(skb); 406 /* Then we put them back on the top of the queue. We're not too 407 * concerned about accidentally getting things a little out of order if 408 * packets are being added really fast, because this queue is for before 409 * packets can even be sent and it's small anyway. 410 */ 411 spin_lock_bh(&peer->staged_packet_queue.lock); 412 skb_queue_splice(&packets, &peer->staged_packet_queue); 413 spin_unlock_bh(&peer->staged_packet_queue.lock); 414 415 /* If we're exiting because there's something wrong with the key, it 416 * means we should initiate a new handshake. 417 */ 418 wg_packet_send_queued_handshake_initiation(peer, false); 419 } 420