1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. 4 */ 5 6 #include "queueing.h" 7 #include "socket.h" 8 #include "timers.h" 9 #include "device.h" 10 #include "ratelimiter.h" 11 #include "peer.h" 12 #include "messages.h" 13 14 #include <linux/module.h> 15 #include <linux/rtnetlink.h> 16 #include <linux/inet.h> 17 #include <linux/netdevice.h> 18 #include <linux/inetdevice.h> 19 #include <linux/if_arp.h> 20 #include <linux/icmp.h> 21 #include <linux/suspend.h> 22 #include <net/icmp.h> 23 #include <net/rtnetlink.h> 24 #include <net/ip_tunnels.h> 25 #include <net/addrconf.h> 26 27 static LIST_HEAD(device_list); 28 29 static int wg_open(struct net_device *dev) 30 { 31 struct in_device *dev_v4 = __in_dev_get_rtnl(dev); 32 struct inet6_dev *dev_v6 = __in6_dev_get(dev); 33 struct wg_device *wg = netdev_priv(dev); 34 struct wg_peer *peer; 35 int ret; 36 37 if (dev_v4) { 38 /* At some point we might put this check near the ip_rt_send_ 39 * redirect call of ip_forward in net/ipv4/ip_forward.c, similar 40 * to the current secpath check. 41 */ 42 IN_DEV_CONF_SET(dev_v4, SEND_REDIRECTS, false); 43 IPV4_DEVCONF_ALL(dev_net(dev), SEND_REDIRECTS) = false; 44 } 45 if (dev_v6) 46 dev_v6->cnf.addr_gen_mode = IN6_ADDR_GEN_MODE_NONE; 47 48 mutex_lock(&wg->device_update_lock); 49 ret = wg_socket_init(wg, wg->incoming_port); 50 if (ret < 0) 51 goto out; 52 list_for_each_entry(peer, &wg->peer_list, peer_list) { 53 wg_packet_send_staged_packets(peer); 54 if (peer->persistent_keepalive_interval) 55 wg_packet_send_keepalive(peer); 56 } 57 out: 58 mutex_unlock(&wg->device_update_lock); 59 return ret; 60 } 61 62 static int wg_pm_notification(struct notifier_block *nb, unsigned long action, void *data) 63 { 64 struct wg_device *wg; 65 struct wg_peer *peer; 66 67 /* If the machine is constantly suspending and resuming, as part of 68 * its normal operation rather than as a somewhat rare event, then we 69 * don't actually want to clear keys. 70 */ 71 if (IS_ENABLED(CONFIG_PM_AUTOSLEEP) || IS_ENABLED(CONFIG_ANDROID)) 72 return 0; 73 74 if (action != PM_HIBERNATION_PREPARE && action != PM_SUSPEND_PREPARE) 75 return 0; 76 77 rtnl_lock(); 78 list_for_each_entry(wg, &device_list, device_list) { 79 mutex_lock(&wg->device_update_lock); 80 list_for_each_entry(peer, &wg->peer_list, peer_list) { 81 del_timer(&peer->timer_zero_key_material); 82 wg_noise_handshake_clear(&peer->handshake); 83 wg_noise_keypairs_clear(&peer->keypairs); 84 } 85 mutex_unlock(&wg->device_update_lock); 86 } 87 rtnl_unlock(); 88 rcu_barrier(); 89 return 0; 90 } 91 92 static struct notifier_block pm_notifier = { .notifier_call = wg_pm_notification }; 93 94 static int wg_vm_notification(struct notifier_block *nb, unsigned long action, void *data) 95 { 96 struct wg_device *wg; 97 struct wg_peer *peer; 98 99 rtnl_lock(); 100 list_for_each_entry(wg, &device_list, device_list) { 101 mutex_lock(&wg->device_update_lock); 102 list_for_each_entry(peer, &wg->peer_list, peer_list) 103 wg_noise_expire_current_peer_keypairs(peer); 104 mutex_unlock(&wg->device_update_lock); 105 } 106 rtnl_unlock(); 107 return 0; 108 } 109 110 static struct notifier_block vm_notifier = { .notifier_call = wg_vm_notification }; 111 112 static int wg_stop(struct net_device *dev) 113 { 114 struct wg_device *wg = netdev_priv(dev); 115 struct wg_peer *peer; 116 struct sk_buff *skb; 117 118 mutex_lock(&wg->device_update_lock); 119 list_for_each_entry(peer, &wg->peer_list, peer_list) { 120 wg_packet_purge_staged_packets(peer); 121 wg_timers_stop(peer); 122 wg_noise_handshake_clear(&peer->handshake); 123 wg_noise_keypairs_clear(&peer->keypairs); 124 wg_noise_reset_last_sent_handshake(&peer->last_sent_handshake); 125 } 126 mutex_unlock(&wg->device_update_lock); 127 while ((skb = ptr_ring_consume(&wg->handshake_queue.ring)) != NULL) 128 kfree_skb(skb); 129 atomic_set(&wg->handshake_queue_len, 0); 130 wg_socket_reinit(wg, NULL, NULL); 131 return 0; 132 } 133 134 static netdev_tx_t wg_xmit(struct sk_buff *skb, struct net_device *dev) 135 { 136 struct wg_device *wg = netdev_priv(dev); 137 struct sk_buff_head packets; 138 struct wg_peer *peer; 139 struct sk_buff *next; 140 sa_family_t family; 141 u32 mtu; 142 int ret; 143 144 if (unlikely(!wg_check_packet_protocol(skb))) { 145 ret = -EPROTONOSUPPORT; 146 net_dbg_ratelimited("%s: Invalid IP packet\n", dev->name); 147 goto err; 148 } 149 150 peer = wg_allowedips_lookup_dst(&wg->peer_allowedips, skb); 151 if (unlikely(!peer)) { 152 ret = -ENOKEY; 153 if (skb->protocol == htons(ETH_P_IP)) 154 net_dbg_ratelimited("%s: No peer has allowed IPs matching %pI4\n", 155 dev->name, &ip_hdr(skb)->daddr); 156 else if (skb->protocol == htons(ETH_P_IPV6)) 157 net_dbg_ratelimited("%s: No peer has allowed IPs matching %pI6\n", 158 dev->name, &ipv6_hdr(skb)->daddr); 159 goto err_icmp; 160 } 161 162 family = READ_ONCE(peer->endpoint.addr.sa_family); 163 if (unlikely(family != AF_INET && family != AF_INET6)) { 164 ret = -EDESTADDRREQ; 165 net_dbg_ratelimited("%s: No valid endpoint has been configured or discovered for peer %llu\n", 166 dev->name, peer->internal_id); 167 goto err_peer; 168 } 169 170 mtu = skb_dst(skb) ? dst_mtu(skb_dst(skb)) : dev->mtu; 171 172 __skb_queue_head_init(&packets); 173 if (!skb_is_gso(skb)) { 174 skb_mark_not_on_list(skb); 175 } else { 176 struct sk_buff *segs = skb_gso_segment(skb, 0); 177 178 if (IS_ERR(segs)) { 179 ret = PTR_ERR(segs); 180 goto err_peer; 181 } 182 dev_kfree_skb(skb); 183 skb = segs; 184 } 185 186 skb_list_walk_safe(skb, skb, next) { 187 skb_mark_not_on_list(skb); 188 189 skb = skb_share_check(skb, GFP_ATOMIC); 190 if (unlikely(!skb)) 191 continue; 192 193 /* We only need to keep the original dst around for icmp, 194 * so at this point we're in a position to drop it. 195 */ 196 skb_dst_drop(skb); 197 198 PACKET_CB(skb)->mtu = mtu; 199 200 __skb_queue_tail(&packets, skb); 201 } 202 203 spin_lock_bh(&peer->staged_packet_queue.lock); 204 /* If the queue is getting too big, we start removing the oldest packets 205 * until it's small again. We do this before adding the new packet, so 206 * we don't remove GSO segments that are in excess. 207 */ 208 while (skb_queue_len(&peer->staged_packet_queue) > MAX_STAGED_PACKETS) { 209 dev_kfree_skb(__skb_dequeue(&peer->staged_packet_queue)); 210 ++dev->stats.tx_dropped; 211 } 212 skb_queue_splice_tail(&packets, &peer->staged_packet_queue); 213 spin_unlock_bh(&peer->staged_packet_queue.lock); 214 215 wg_packet_send_staged_packets(peer); 216 217 wg_peer_put(peer); 218 return NETDEV_TX_OK; 219 220 err_peer: 221 wg_peer_put(peer); 222 err_icmp: 223 if (skb->protocol == htons(ETH_P_IP)) 224 icmp_ndo_send(skb, ICMP_DEST_UNREACH, ICMP_HOST_UNREACH, 0); 225 else if (skb->protocol == htons(ETH_P_IPV6)) 226 icmpv6_ndo_send(skb, ICMPV6_DEST_UNREACH, ICMPV6_ADDR_UNREACH, 0); 227 err: 228 ++dev->stats.tx_errors; 229 kfree_skb(skb); 230 return ret; 231 } 232 233 static const struct net_device_ops netdev_ops = { 234 .ndo_open = wg_open, 235 .ndo_stop = wg_stop, 236 .ndo_start_xmit = wg_xmit, 237 .ndo_get_stats64 = dev_get_tstats64 238 }; 239 240 static void wg_destruct(struct net_device *dev) 241 { 242 struct wg_device *wg = netdev_priv(dev); 243 244 rtnl_lock(); 245 list_del(&wg->device_list); 246 rtnl_unlock(); 247 mutex_lock(&wg->device_update_lock); 248 rcu_assign_pointer(wg->creating_net, NULL); 249 wg->incoming_port = 0; 250 wg_socket_reinit(wg, NULL, NULL); 251 /* The final references are cleared in the below calls to destroy_workqueue. */ 252 wg_peer_remove_all(wg); 253 destroy_workqueue(wg->handshake_receive_wq); 254 destroy_workqueue(wg->handshake_send_wq); 255 destroy_workqueue(wg->packet_crypt_wq); 256 wg_packet_queue_free(&wg->handshake_queue, true); 257 wg_packet_queue_free(&wg->decrypt_queue, false); 258 wg_packet_queue_free(&wg->encrypt_queue, false); 259 rcu_barrier(); /* Wait for all the peers to be actually freed. */ 260 wg_ratelimiter_uninit(); 261 memzero_explicit(&wg->static_identity, sizeof(wg->static_identity)); 262 free_percpu(dev->tstats); 263 kvfree(wg->index_hashtable); 264 kvfree(wg->peer_hashtable); 265 mutex_unlock(&wg->device_update_lock); 266 267 pr_debug("%s: Interface destroyed\n", dev->name); 268 free_netdev(dev); 269 } 270 271 static const struct device_type device_type = { .name = KBUILD_MODNAME }; 272 273 static void wg_setup(struct net_device *dev) 274 { 275 struct wg_device *wg = netdev_priv(dev); 276 enum { WG_NETDEV_FEATURES = NETIF_F_HW_CSUM | NETIF_F_RXCSUM | 277 NETIF_F_SG | NETIF_F_GSO | 278 NETIF_F_GSO_SOFTWARE | NETIF_F_HIGHDMA }; 279 const int overhead = MESSAGE_MINIMUM_LENGTH + sizeof(struct udphdr) + 280 max(sizeof(struct ipv6hdr), sizeof(struct iphdr)); 281 282 dev->netdev_ops = &netdev_ops; 283 dev->header_ops = &ip_tunnel_header_ops; 284 dev->hard_header_len = 0; 285 dev->addr_len = 0; 286 dev->needed_headroom = DATA_PACKET_HEAD_ROOM; 287 dev->needed_tailroom = noise_encrypted_len(MESSAGE_PADDING_MULTIPLE); 288 dev->type = ARPHRD_NONE; 289 dev->flags = IFF_POINTOPOINT | IFF_NOARP; 290 dev->priv_flags |= IFF_NO_QUEUE; 291 dev->features |= NETIF_F_LLTX; 292 dev->features |= WG_NETDEV_FEATURES; 293 dev->hw_features |= WG_NETDEV_FEATURES; 294 dev->hw_enc_features |= WG_NETDEV_FEATURES; 295 dev->mtu = ETH_DATA_LEN - overhead; 296 dev->max_mtu = round_down(INT_MAX, MESSAGE_PADDING_MULTIPLE) - overhead; 297 298 SET_NETDEV_DEVTYPE(dev, &device_type); 299 300 /* We need to keep the dst around in case of icmp replies. */ 301 netif_keep_dst(dev); 302 303 memset(wg, 0, sizeof(*wg)); 304 wg->dev = dev; 305 } 306 307 static int wg_newlink(struct net *src_net, struct net_device *dev, 308 struct nlattr *tb[], struct nlattr *data[], 309 struct netlink_ext_ack *extack) 310 { 311 struct wg_device *wg = netdev_priv(dev); 312 int ret = -ENOMEM; 313 314 rcu_assign_pointer(wg->creating_net, src_net); 315 init_rwsem(&wg->static_identity.lock); 316 mutex_init(&wg->socket_update_lock); 317 mutex_init(&wg->device_update_lock); 318 wg_allowedips_init(&wg->peer_allowedips); 319 wg_cookie_checker_init(&wg->cookie_checker, wg); 320 INIT_LIST_HEAD(&wg->peer_list); 321 wg->device_update_gen = 1; 322 323 wg->peer_hashtable = wg_pubkey_hashtable_alloc(); 324 if (!wg->peer_hashtable) 325 return ret; 326 327 wg->index_hashtable = wg_index_hashtable_alloc(); 328 if (!wg->index_hashtable) 329 goto err_free_peer_hashtable; 330 331 dev->tstats = netdev_alloc_pcpu_stats(struct pcpu_sw_netstats); 332 if (!dev->tstats) 333 goto err_free_index_hashtable; 334 335 wg->handshake_receive_wq = alloc_workqueue("wg-kex-%s", 336 WQ_CPU_INTENSIVE | WQ_FREEZABLE, 0, dev->name); 337 if (!wg->handshake_receive_wq) 338 goto err_free_tstats; 339 340 wg->handshake_send_wq = alloc_workqueue("wg-kex-%s", 341 WQ_UNBOUND | WQ_FREEZABLE, 0, dev->name); 342 if (!wg->handshake_send_wq) 343 goto err_destroy_handshake_receive; 344 345 wg->packet_crypt_wq = alloc_workqueue("wg-crypt-%s", 346 WQ_CPU_INTENSIVE | WQ_MEM_RECLAIM, 0, dev->name); 347 if (!wg->packet_crypt_wq) 348 goto err_destroy_handshake_send; 349 350 ret = wg_packet_queue_init(&wg->encrypt_queue, wg_packet_encrypt_worker, 351 MAX_QUEUED_PACKETS); 352 if (ret < 0) 353 goto err_destroy_packet_crypt; 354 355 ret = wg_packet_queue_init(&wg->decrypt_queue, wg_packet_decrypt_worker, 356 MAX_QUEUED_PACKETS); 357 if (ret < 0) 358 goto err_free_encrypt_queue; 359 360 ret = wg_packet_queue_init(&wg->handshake_queue, wg_packet_handshake_receive_worker, 361 MAX_QUEUED_INCOMING_HANDSHAKES); 362 if (ret < 0) 363 goto err_free_decrypt_queue; 364 365 ret = wg_ratelimiter_init(); 366 if (ret < 0) 367 goto err_free_handshake_queue; 368 369 ret = register_netdevice(dev); 370 if (ret < 0) 371 goto err_uninit_ratelimiter; 372 373 list_add(&wg->device_list, &device_list); 374 375 /* We wait until the end to assign priv_destructor, so that 376 * register_netdevice doesn't call it for us if it fails. 377 */ 378 dev->priv_destructor = wg_destruct; 379 380 pr_debug("%s: Interface created\n", dev->name); 381 return ret; 382 383 err_uninit_ratelimiter: 384 wg_ratelimiter_uninit(); 385 err_free_handshake_queue: 386 wg_packet_queue_free(&wg->handshake_queue, false); 387 err_free_decrypt_queue: 388 wg_packet_queue_free(&wg->decrypt_queue, false); 389 err_free_encrypt_queue: 390 wg_packet_queue_free(&wg->encrypt_queue, false); 391 err_destroy_packet_crypt: 392 destroy_workqueue(wg->packet_crypt_wq); 393 err_destroy_handshake_send: 394 destroy_workqueue(wg->handshake_send_wq); 395 err_destroy_handshake_receive: 396 destroy_workqueue(wg->handshake_receive_wq); 397 err_free_tstats: 398 free_percpu(dev->tstats); 399 err_free_index_hashtable: 400 kvfree(wg->index_hashtable); 401 err_free_peer_hashtable: 402 kvfree(wg->peer_hashtable); 403 return ret; 404 } 405 406 static struct rtnl_link_ops link_ops __read_mostly = { 407 .kind = KBUILD_MODNAME, 408 .priv_size = sizeof(struct wg_device), 409 .setup = wg_setup, 410 .newlink = wg_newlink, 411 }; 412 413 static void wg_netns_pre_exit(struct net *net) 414 { 415 struct wg_device *wg; 416 struct wg_peer *peer; 417 418 rtnl_lock(); 419 list_for_each_entry(wg, &device_list, device_list) { 420 if (rcu_access_pointer(wg->creating_net) == net) { 421 pr_debug("%s: Creating namespace exiting\n", wg->dev->name); 422 netif_carrier_off(wg->dev); 423 mutex_lock(&wg->device_update_lock); 424 rcu_assign_pointer(wg->creating_net, NULL); 425 wg_socket_reinit(wg, NULL, NULL); 426 list_for_each_entry(peer, &wg->peer_list, peer_list) 427 wg_socket_clear_peer_endpoint_src(peer); 428 mutex_unlock(&wg->device_update_lock); 429 } 430 } 431 rtnl_unlock(); 432 } 433 434 static struct pernet_operations pernet_ops = { 435 .pre_exit = wg_netns_pre_exit 436 }; 437 438 int __init wg_device_init(void) 439 { 440 int ret; 441 442 ret = register_pm_notifier(&pm_notifier); 443 if (ret) 444 return ret; 445 446 ret = register_random_vmfork_notifier(&vm_notifier); 447 if (ret) 448 goto error_pm; 449 450 ret = register_pernet_device(&pernet_ops); 451 if (ret) 452 goto error_vm; 453 454 ret = rtnl_link_register(&link_ops); 455 if (ret) 456 goto error_pernet; 457 458 return 0; 459 460 error_pernet: 461 unregister_pernet_device(&pernet_ops); 462 error_vm: 463 unregister_random_vmfork_notifier(&vm_notifier); 464 error_pm: 465 unregister_pm_notifier(&pm_notifier); 466 return ret; 467 } 468 469 void wg_device_uninit(void) 470 { 471 rtnl_link_unregister(&link_ops); 472 unregister_pernet_device(&pernet_ops); 473 unregister_random_vmfork_notifier(&vm_notifier); 474 unregister_pm_notifier(&pm_notifier); 475 rcu_barrier(); 476 } 477