1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. 4 */ 5 6 #include "queueing.h" 7 #include "socket.h" 8 #include "timers.h" 9 #include "device.h" 10 #include "ratelimiter.h" 11 #include "peer.h" 12 #include "messages.h" 13 14 #include <linux/module.h> 15 #include <linux/rtnetlink.h> 16 #include <linux/inet.h> 17 #include <linux/netdevice.h> 18 #include <linux/inetdevice.h> 19 #include <linux/if_arp.h> 20 #include <linux/icmp.h> 21 #include <linux/suspend.h> 22 #include <net/dst_metadata.h> 23 #include <net/gso.h> 24 #include <net/icmp.h> 25 #include <net/rtnetlink.h> 26 #include <net/ip_tunnels.h> 27 #include <net/addrconf.h> 28 29 static LIST_HEAD(device_list); 30 31 static int wg_open(struct net_device *dev) 32 { 33 struct in_device *dev_v4 = __in_dev_get_rtnl(dev); 34 struct inet6_dev *dev_v6 = __in6_dev_get(dev); 35 struct wg_device *wg = netdev_priv(dev); 36 struct wg_peer *peer; 37 int ret; 38 39 if (dev_v4) { 40 /* At some point we might put this check near the ip_rt_send_ 41 * redirect call of ip_forward in net/ipv4/ip_forward.c, similar 42 * to the current secpath check. 43 */ 44 IN_DEV_CONF_SET(dev_v4, SEND_REDIRECTS, false); 45 IPV4_DEVCONF_ALL(dev_net(dev), SEND_REDIRECTS) = false; 46 } 47 if (dev_v6) 48 dev_v6->cnf.addr_gen_mode = IN6_ADDR_GEN_MODE_NONE; 49 50 mutex_lock(&wg->device_update_lock); 51 ret = wg_socket_init(wg, wg->incoming_port); 52 if (ret < 0) 53 goto out; 54 list_for_each_entry(peer, &wg->peer_list, peer_list) { 55 wg_packet_send_staged_packets(peer); 56 if (peer->persistent_keepalive_interval) 57 wg_packet_send_keepalive(peer); 58 } 59 out: 60 mutex_unlock(&wg->device_update_lock); 61 return ret; 62 } 63 64 static int wg_pm_notification(struct notifier_block *nb, unsigned long action, void *data) 65 { 66 struct wg_device *wg; 67 struct wg_peer *peer; 68 69 /* If the machine is constantly suspending and resuming, as part of 70 * its normal operation rather than as a somewhat rare event, then we 71 * don't actually want to clear keys. 72 */ 73 if (IS_ENABLED(CONFIG_PM_AUTOSLEEP) || 74 IS_ENABLED(CONFIG_PM_USERSPACE_AUTOSLEEP)) 75 return 0; 76 77 if (action != PM_HIBERNATION_PREPARE && action != PM_SUSPEND_PREPARE) 78 return 0; 79 80 rtnl_lock(); 81 list_for_each_entry(wg, &device_list, device_list) { 82 mutex_lock(&wg->device_update_lock); 83 list_for_each_entry(peer, &wg->peer_list, peer_list) { 84 del_timer(&peer->timer_zero_key_material); 85 wg_noise_handshake_clear(&peer->handshake); 86 wg_noise_keypairs_clear(&peer->keypairs); 87 } 88 mutex_unlock(&wg->device_update_lock); 89 } 90 rtnl_unlock(); 91 rcu_barrier(); 92 return 0; 93 } 94 95 static struct notifier_block pm_notifier = { .notifier_call = wg_pm_notification }; 96 97 static int wg_vm_notification(struct notifier_block *nb, unsigned long action, void *data) 98 { 99 struct wg_device *wg; 100 struct wg_peer *peer; 101 102 rtnl_lock(); 103 list_for_each_entry(wg, &device_list, device_list) { 104 mutex_lock(&wg->device_update_lock); 105 list_for_each_entry(peer, &wg->peer_list, peer_list) 106 wg_noise_expire_current_peer_keypairs(peer); 107 mutex_unlock(&wg->device_update_lock); 108 } 109 rtnl_unlock(); 110 return 0; 111 } 112 113 static struct notifier_block vm_notifier = { .notifier_call = wg_vm_notification }; 114 115 static int wg_stop(struct net_device *dev) 116 { 117 struct wg_device *wg = netdev_priv(dev); 118 struct wg_peer *peer; 119 struct sk_buff *skb; 120 121 mutex_lock(&wg->device_update_lock); 122 list_for_each_entry(peer, &wg->peer_list, peer_list) { 123 wg_packet_purge_staged_packets(peer); 124 wg_timers_stop(peer); 125 wg_noise_handshake_clear(&peer->handshake); 126 wg_noise_keypairs_clear(&peer->keypairs); 127 wg_noise_reset_last_sent_handshake(&peer->last_sent_handshake); 128 } 129 mutex_unlock(&wg->device_update_lock); 130 while ((skb = ptr_ring_consume(&wg->handshake_queue.ring)) != NULL) 131 kfree_skb(skb); 132 atomic_set(&wg->handshake_queue_len, 0); 133 wg_socket_reinit(wg, NULL, NULL); 134 return 0; 135 } 136 137 static netdev_tx_t wg_xmit(struct sk_buff *skb, struct net_device *dev) 138 { 139 struct wg_device *wg = netdev_priv(dev); 140 struct sk_buff_head packets; 141 struct wg_peer *peer; 142 struct sk_buff *next; 143 sa_family_t family; 144 u32 mtu; 145 int ret; 146 147 if (unlikely(!wg_check_packet_protocol(skb))) { 148 ret = -EPROTONOSUPPORT; 149 net_dbg_ratelimited("%s: Invalid IP packet\n", dev->name); 150 goto err; 151 } 152 153 peer = wg_allowedips_lookup_dst(&wg->peer_allowedips, skb); 154 if (unlikely(!peer)) { 155 ret = -ENOKEY; 156 if (skb->protocol == htons(ETH_P_IP)) 157 net_dbg_ratelimited("%s: No peer has allowed IPs matching %pI4\n", 158 dev->name, &ip_hdr(skb)->daddr); 159 else if (skb->protocol == htons(ETH_P_IPV6)) 160 net_dbg_ratelimited("%s: No peer has allowed IPs matching %pI6\n", 161 dev->name, &ipv6_hdr(skb)->daddr); 162 goto err_icmp; 163 } 164 165 family = READ_ONCE(peer->endpoint.addr.sa_family); 166 if (unlikely(family != AF_INET && family != AF_INET6)) { 167 ret = -EDESTADDRREQ; 168 net_dbg_ratelimited("%s: No valid endpoint has been configured or discovered for peer %llu\n", 169 dev->name, peer->internal_id); 170 goto err_peer; 171 } 172 173 mtu = skb_valid_dst(skb) ? dst_mtu(skb_dst(skb)) : dev->mtu; 174 175 __skb_queue_head_init(&packets); 176 if (!skb_is_gso(skb)) { 177 skb_mark_not_on_list(skb); 178 } else { 179 struct sk_buff *segs = skb_gso_segment(skb, 0); 180 181 if (IS_ERR(segs)) { 182 ret = PTR_ERR(segs); 183 goto err_peer; 184 } 185 dev_kfree_skb(skb); 186 skb = segs; 187 } 188 189 skb_list_walk_safe(skb, skb, next) { 190 skb_mark_not_on_list(skb); 191 192 skb = skb_share_check(skb, GFP_ATOMIC); 193 if (unlikely(!skb)) 194 continue; 195 196 /* We only need to keep the original dst around for icmp, 197 * so at this point we're in a position to drop it. 198 */ 199 skb_dst_drop(skb); 200 201 PACKET_CB(skb)->mtu = mtu; 202 203 __skb_queue_tail(&packets, skb); 204 } 205 206 spin_lock_bh(&peer->staged_packet_queue.lock); 207 /* If the queue is getting too big, we start removing the oldest packets 208 * until it's small again. We do this before adding the new packet, so 209 * we don't remove GSO segments that are in excess. 210 */ 211 while (skb_queue_len(&peer->staged_packet_queue) > MAX_STAGED_PACKETS) { 212 dev_kfree_skb(__skb_dequeue(&peer->staged_packet_queue)); 213 DEV_STATS_INC(dev, tx_dropped); 214 } 215 skb_queue_splice_tail(&packets, &peer->staged_packet_queue); 216 spin_unlock_bh(&peer->staged_packet_queue.lock); 217 218 wg_packet_send_staged_packets(peer); 219 220 wg_peer_put(peer); 221 return NETDEV_TX_OK; 222 223 err_peer: 224 wg_peer_put(peer); 225 err_icmp: 226 if (skb->protocol == htons(ETH_P_IP)) 227 icmp_ndo_send(skb, ICMP_DEST_UNREACH, ICMP_HOST_UNREACH, 0); 228 else if (skb->protocol == htons(ETH_P_IPV6)) 229 icmpv6_ndo_send(skb, ICMPV6_DEST_UNREACH, ICMPV6_ADDR_UNREACH, 0); 230 err: 231 DEV_STATS_INC(dev, tx_errors); 232 kfree_skb(skb); 233 return ret; 234 } 235 236 static const struct net_device_ops netdev_ops = { 237 .ndo_open = wg_open, 238 .ndo_stop = wg_stop, 239 .ndo_start_xmit = wg_xmit, 240 .ndo_get_stats64 = dev_get_tstats64 241 }; 242 243 static void wg_destruct(struct net_device *dev) 244 { 245 struct wg_device *wg = netdev_priv(dev); 246 247 rtnl_lock(); 248 list_del(&wg->device_list); 249 rtnl_unlock(); 250 mutex_lock(&wg->device_update_lock); 251 rcu_assign_pointer(wg->creating_net, NULL); 252 wg->incoming_port = 0; 253 wg_socket_reinit(wg, NULL, NULL); 254 /* The final references are cleared in the below calls to destroy_workqueue. */ 255 wg_peer_remove_all(wg); 256 destroy_workqueue(wg->handshake_receive_wq); 257 destroy_workqueue(wg->handshake_send_wq); 258 destroy_workqueue(wg->packet_crypt_wq); 259 wg_packet_queue_free(&wg->handshake_queue, true); 260 wg_packet_queue_free(&wg->decrypt_queue, false); 261 wg_packet_queue_free(&wg->encrypt_queue, false); 262 rcu_barrier(); /* Wait for all the peers to be actually freed. */ 263 wg_ratelimiter_uninit(); 264 memzero_explicit(&wg->static_identity, sizeof(wg->static_identity)); 265 free_percpu(dev->tstats); 266 kvfree(wg->index_hashtable); 267 kvfree(wg->peer_hashtable); 268 mutex_unlock(&wg->device_update_lock); 269 270 pr_debug("%s: Interface destroyed\n", dev->name); 271 free_netdev(dev); 272 } 273 274 static const struct device_type device_type = { .name = KBUILD_MODNAME }; 275 276 static void wg_setup(struct net_device *dev) 277 { 278 struct wg_device *wg = netdev_priv(dev); 279 enum { WG_NETDEV_FEATURES = NETIF_F_HW_CSUM | NETIF_F_RXCSUM | 280 NETIF_F_SG | NETIF_F_GSO | 281 NETIF_F_GSO_SOFTWARE | NETIF_F_HIGHDMA }; 282 const int overhead = MESSAGE_MINIMUM_LENGTH + sizeof(struct udphdr) + 283 max(sizeof(struct ipv6hdr), sizeof(struct iphdr)); 284 285 dev->netdev_ops = &netdev_ops; 286 dev->header_ops = &ip_tunnel_header_ops; 287 dev->hard_header_len = 0; 288 dev->addr_len = 0; 289 dev->needed_headroom = DATA_PACKET_HEAD_ROOM; 290 dev->needed_tailroom = noise_encrypted_len(MESSAGE_PADDING_MULTIPLE); 291 dev->type = ARPHRD_NONE; 292 dev->flags = IFF_POINTOPOINT | IFF_NOARP; 293 dev->priv_flags |= IFF_NO_QUEUE; 294 dev->features |= NETIF_F_LLTX; 295 dev->features |= WG_NETDEV_FEATURES; 296 dev->hw_features |= WG_NETDEV_FEATURES; 297 dev->hw_enc_features |= WG_NETDEV_FEATURES; 298 dev->mtu = ETH_DATA_LEN - overhead; 299 dev->max_mtu = round_down(INT_MAX, MESSAGE_PADDING_MULTIPLE) - overhead; 300 301 SET_NETDEV_DEVTYPE(dev, &device_type); 302 303 /* We need to keep the dst around in case of icmp replies. */ 304 netif_keep_dst(dev); 305 306 memset(wg, 0, sizeof(*wg)); 307 wg->dev = dev; 308 } 309 310 static int wg_newlink(struct net *src_net, struct net_device *dev, 311 struct nlattr *tb[], struct nlattr *data[], 312 struct netlink_ext_ack *extack) 313 { 314 struct wg_device *wg = netdev_priv(dev); 315 int ret = -ENOMEM; 316 317 rcu_assign_pointer(wg->creating_net, src_net); 318 init_rwsem(&wg->static_identity.lock); 319 mutex_init(&wg->socket_update_lock); 320 mutex_init(&wg->device_update_lock); 321 wg_allowedips_init(&wg->peer_allowedips); 322 wg_cookie_checker_init(&wg->cookie_checker, wg); 323 INIT_LIST_HEAD(&wg->peer_list); 324 wg->device_update_gen = 1; 325 326 wg->peer_hashtable = wg_pubkey_hashtable_alloc(); 327 if (!wg->peer_hashtable) 328 return ret; 329 330 wg->index_hashtable = wg_index_hashtable_alloc(); 331 if (!wg->index_hashtable) 332 goto err_free_peer_hashtable; 333 334 dev->tstats = netdev_alloc_pcpu_stats(struct pcpu_sw_netstats); 335 if (!dev->tstats) 336 goto err_free_index_hashtable; 337 338 wg->handshake_receive_wq = alloc_workqueue("wg-kex-%s", 339 WQ_CPU_INTENSIVE | WQ_FREEZABLE, 0, dev->name); 340 if (!wg->handshake_receive_wq) 341 goto err_free_tstats; 342 343 wg->handshake_send_wq = alloc_workqueue("wg-kex-%s", 344 WQ_UNBOUND | WQ_FREEZABLE, 0, dev->name); 345 if (!wg->handshake_send_wq) 346 goto err_destroy_handshake_receive; 347 348 wg->packet_crypt_wq = alloc_workqueue("wg-crypt-%s", 349 WQ_CPU_INTENSIVE | WQ_MEM_RECLAIM, 0, dev->name); 350 if (!wg->packet_crypt_wq) 351 goto err_destroy_handshake_send; 352 353 ret = wg_packet_queue_init(&wg->encrypt_queue, wg_packet_encrypt_worker, 354 MAX_QUEUED_PACKETS); 355 if (ret < 0) 356 goto err_destroy_packet_crypt; 357 358 ret = wg_packet_queue_init(&wg->decrypt_queue, wg_packet_decrypt_worker, 359 MAX_QUEUED_PACKETS); 360 if (ret < 0) 361 goto err_free_encrypt_queue; 362 363 ret = wg_packet_queue_init(&wg->handshake_queue, wg_packet_handshake_receive_worker, 364 MAX_QUEUED_INCOMING_HANDSHAKES); 365 if (ret < 0) 366 goto err_free_decrypt_queue; 367 368 ret = wg_ratelimiter_init(); 369 if (ret < 0) 370 goto err_free_handshake_queue; 371 372 ret = register_netdevice(dev); 373 if (ret < 0) 374 goto err_uninit_ratelimiter; 375 376 list_add(&wg->device_list, &device_list); 377 378 /* We wait until the end to assign priv_destructor, so that 379 * register_netdevice doesn't call it for us if it fails. 380 */ 381 dev->priv_destructor = wg_destruct; 382 383 pr_debug("%s: Interface created\n", dev->name); 384 return ret; 385 386 err_uninit_ratelimiter: 387 wg_ratelimiter_uninit(); 388 err_free_handshake_queue: 389 wg_packet_queue_free(&wg->handshake_queue, false); 390 err_free_decrypt_queue: 391 wg_packet_queue_free(&wg->decrypt_queue, false); 392 err_free_encrypt_queue: 393 wg_packet_queue_free(&wg->encrypt_queue, false); 394 err_destroy_packet_crypt: 395 destroy_workqueue(wg->packet_crypt_wq); 396 err_destroy_handshake_send: 397 destroy_workqueue(wg->handshake_send_wq); 398 err_destroy_handshake_receive: 399 destroy_workqueue(wg->handshake_receive_wq); 400 err_free_tstats: 401 free_percpu(dev->tstats); 402 err_free_index_hashtable: 403 kvfree(wg->index_hashtable); 404 err_free_peer_hashtable: 405 kvfree(wg->peer_hashtable); 406 return ret; 407 } 408 409 static struct rtnl_link_ops link_ops __read_mostly = { 410 .kind = KBUILD_MODNAME, 411 .priv_size = sizeof(struct wg_device), 412 .setup = wg_setup, 413 .newlink = wg_newlink, 414 }; 415 416 static void wg_netns_pre_exit(struct net *net) 417 { 418 struct wg_device *wg; 419 struct wg_peer *peer; 420 421 rtnl_lock(); 422 list_for_each_entry(wg, &device_list, device_list) { 423 if (rcu_access_pointer(wg->creating_net) == net) { 424 pr_debug("%s: Creating namespace exiting\n", wg->dev->name); 425 netif_carrier_off(wg->dev); 426 mutex_lock(&wg->device_update_lock); 427 rcu_assign_pointer(wg->creating_net, NULL); 428 wg_socket_reinit(wg, NULL, NULL); 429 list_for_each_entry(peer, &wg->peer_list, peer_list) 430 wg_socket_clear_peer_endpoint_src(peer); 431 mutex_unlock(&wg->device_update_lock); 432 } 433 } 434 rtnl_unlock(); 435 } 436 437 static struct pernet_operations pernet_ops = { 438 .pre_exit = wg_netns_pre_exit 439 }; 440 441 int __init wg_device_init(void) 442 { 443 int ret; 444 445 ret = register_pm_notifier(&pm_notifier); 446 if (ret) 447 return ret; 448 449 ret = register_random_vmfork_notifier(&vm_notifier); 450 if (ret) 451 goto error_pm; 452 453 ret = register_pernet_device(&pernet_ops); 454 if (ret) 455 goto error_vm; 456 457 ret = rtnl_link_register(&link_ops); 458 if (ret) 459 goto error_pernet; 460 461 return 0; 462 463 error_pernet: 464 unregister_pernet_device(&pernet_ops); 465 error_vm: 466 unregister_random_vmfork_notifier(&vm_notifier); 467 error_pm: 468 unregister_pm_notifier(&pm_notifier); 469 return ret; 470 } 471 472 void wg_device_uninit(void) 473 { 474 rtnl_link_unregister(&link_ops); 475 unregister_pernet_device(&pernet_ops); 476 unregister_random_vmfork_notifier(&vm_notifier); 477 unregister_pm_notifier(&pm_notifier); 478 rcu_barrier(); 479 } 480