xref: /openbmc/linux/drivers/net/wireguard/device.c (revision b58c6630)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
4  */
5 
6 #include "queueing.h"
7 #include "socket.h"
8 #include "timers.h"
9 #include "device.h"
10 #include "ratelimiter.h"
11 #include "peer.h"
12 #include "messages.h"
13 
14 #include <linux/module.h>
15 #include <linux/rtnetlink.h>
16 #include <linux/inet.h>
17 #include <linux/netdevice.h>
18 #include <linux/inetdevice.h>
19 #include <linux/if_arp.h>
20 #include <linux/icmp.h>
21 #include <linux/suspend.h>
22 #include <net/icmp.h>
23 #include <net/rtnetlink.h>
24 #include <net/ip_tunnels.h>
25 #include <net/addrconf.h>
26 
27 static LIST_HEAD(device_list);
28 
29 static int wg_open(struct net_device *dev)
30 {
31 	struct in_device *dev_v4 = __in_dev_get_rtnl(dev);
32 	struct inet6_dev *dev_v6 = __in6_dev_get(dev);
33 	struct wg_device *wg = netdev_priv(dev);
34 	struct wg_peer *peer;
35 	int ret;
36 
37 	if (dev_v4) {
38 		/* At some point we might put this check near the ip_rt_send_
39 		 * redirect call of ip_forward in net/ipv4/ip_forward.c, similar
40 		 * to the current secpath check.
41 		 */
42 		IN_DEV_CONF_SET(dev_v4, SEND_REDIRECTS, false);
43 		IPV4_DEVCONF_ALL(dev_net(dev), SEND_REDIRECTS) = false;
44 	}
45 	if (dev_v6)
46 		dev_v6->cnf.addr_gen_mode = IN6_ADDR_GEN_MODE_NONE;
47 
48 	ret = wg_socket_init(wg, wg->incoming_port);
49 	if (ret < 0)
50 		return ret;
51 	mutex_lock(&wg->device_update_lock);
52 	list_for_each_entry(peer, &wg->peer_list, peer_list) {
53 		wg_packet_send_staged_packets(peer);
54 		if (peer->persistent_keepalive_interval)
55 			wg_packet_send_keepalive(peer);
56 	}
57 	mutex_unlock(&wg->device_update_lock);
58 	return 0;
59 }
60 
61 #ifdef CONFIG_PM_SLEEP
62 static int wg_pm_notification(struct notifier_block *nb, unsigned long action,
63 			      void *data)
64 {
65 	struct wg_device *wg;
66 	struct wg_peer *peer;
67 
68 	/* If the machine is constantly suspending and resuming, as part of
69 	 * its normal operation rather than as a somewhat rare event, then we
70 	 * don't actually want to clear keys.
71 	 */
72 	if (IS_ENABLED(CONFIG_PM_AUTOSLEEP) || IS_ENABLED(CONFIG_ANDROID))
73 		return 0;
74 
75 	if (action != PM_HIBERNATION_PREPARE && action != PM_SUSPEND_PREPARE)
76 		return 0;
77 
78 	rtnl_lock();
79 	list_for_each_entry(wg, &device_list, device_list) {
80 		mutex_lock(&wg->device_update_lock);
81 		list_for_each_entry(peer, &wg->peer_list, peer_list) {
82 			del_timer(&peer->timer_zero_key_material);
83 			wg_noise_handshake_clear(&peer->handshake);
84 			wg_noise_keypairs_clear(&peer->keypairs);
85 		}
86 		mutex_unlock(&wg->device_update_lock);
87 	}
88 	rtnl_unlock();
89 	rcu_barrier();
90 	return 0;
91 }
92 
93 static struct notifier_block pm_notifier = { .notifier_call = wg_pm_notification };
94 #endif
95 
96 static int wg_stop(struct net_device *dev)
97 {
98 	struct wg_device *wg = netdev_priv(dev);
99 	struct wg_peer *peer;
100 
101 	mutex_lock(&wg->device_update_lock);
102 	list_for_each_entry(peer, &wg->peer_list, peer_list) {
103 		wg_packet_purge_staged_packets(peer);
104 		wg_timers_stop(peer);
105 		wg_noise_handshake_clear(&peer->handshake);
106 		wg_noise_keypairs_clear(&peer->keypairs);
107 		wg_noise_reset_last_sent_handshake(&peer->last_sent_handshake);
108 	}
109 	mutex_unlock(&wg->device_update_lock);
110 	skb_queue_purge(&wg->incoming_handshakes);
111 	wg_socket_reinit(wg, NULL, NULL);
112 	return 0;
113 }
114 
115 static netdev_tx_t wg_xmit(struct sk_buff *skb, struct net_device *dev)
116 {
117 	struct wg_device *wg = netdev_priv(dev);
118 	struct sk_buff_head packets;
119 	struct wg_peer *peer;
120 	struct sk_buff *next;
121 	sa_family_t family;
122 	u32 mtu;
123 	int ret;
124 
125 	if (unlikely(!wg_check_packet_protocol(skb))) {
126 		ret = -EPROTONOSUPPORT;
127 		net_dbg_ratelimited("%s: Invalid IP packet\n", dev->name);
128 		goto err;
129 	}
130 
131 	peer = wg_allowedips_lookup_dst(&wg->peer_allowedips, skb);
132 	if (unlikely(!peer)) {
133 		ret = -ENOKEY;
134 		if (skb->protocol == htons(ETH_P_IP))
135 			net_dbg_ratelimited("%s: No peer has allowed IPs matching %pI4\n",
136 					    dev->name, &ip_hdr(skb)->daddr);
137 		else if (skb->protocol == htons(ETH_P_IPV6))
138 			net_dbg_ratelimited("%s: No peer has allowed IPs matching %pI6\n",
139 					    dev->name, &ipv6_hdr(skb)->daddr);
140 		goto err;
141 	}
142 
143 	family = READ_ONCE(peer->endpoint.addr.sa_family);
144 	if (unlikely(family != AF_INET && family != AF_INET6)) {
145 		ret = -EDESTADDRREQ;
146 		net_dbg_ratelimited("%s: No valid endpoint has been configured or discovered for peer %llu\n",
147 				    dev->name, peer->internal_id);
148 		goto err_peer;
149 	}
150 
151 	mtu = skb_dst(skb) ? dst_mtu(skb_dst(skb)) : dev->mtu;
152 
153 	__skb_queue_head_init(&packets);
154 	if (!skb_is_gso(skb)) {
155 		skb_mark_not_on_list(skb);
156 	} else {
157 		struct sk_buff *segs = skb_gso_segment(skb, 0);
158 
159 		if (unlikely(IS_ERR(segs))) {
160 			ret = PTR_ERR(segs);
161 			goto err_peer;
162 		}
163 		dev_kfree_skb(skb);
164 		skb = segs;
165 	}
166 
167 	skb_list_walk_safe(skb, skb, next) {
168 		skb_mark_not_on_list(skb);
169 
170 		skb = skb_share_check(skb, GFP_ATOMIC);
171 		if (unlikely(!skb))
172 			continue;
173 
174 		/* We only need to keep the original dst around for icmp,
175 		 * so at this point we're in a position to drop it.
176 		 */
177 		skb_dst_drop(skb);
178 
179 		PACKET_CB(skb)->mtu = mtu;
180 
181 		__skb_queue_tail(&packets, skb);
182 	}
183 
184 	spin_lock_bh(&peer->staged_packet_queue.lock);
185 	/* If the queue is getting too big, we start removing the oldest packets
186 	 * until it's small again. We do this before adding the new packet, so
187 	 * we don't remove GSO segments that are in excess.
188 	 */
189 	while (skb_queue_len(&peer->staged_packet_queue) > MAX_STAGED_PACKETS) {
190 		dev_kfree_skb(__skb_dequeue(&peer->staged_packet_queue));
191 		++dev->stats.tx_dropped;
192 	}
193 	skb_queue_splice_tail(&packets, &peer->staged_packet_queue);
194 	spin_unlock_bh(&peer->staged_packet_queue.lock);
195 
196 	wg_packet_send_staged_packets(peer);
197 
198 	wg_peer_put(peer);
199 	return NETDEV_TX_OK;
200 
201 err_peer:
202 	wg_peer_put(peer);
203 err:
204 	++dev->stats.tx_errors;
205 	if (skb->protocol == htons(ETH_P_IP))
206 		icmp_ndo_send(skb, ICMP_DEST_UNREACH, ICMP_HOST_UNREACH, 0);
207 	else if (skb->protocol == htons(ETH_P_IPV6))
208 		icmpv6_ndo_send(skb, ICMPV6_DEST_UNREACH, ICMPV6_ADDR_UNREACH, 0);
209 	kfree_skb(skb);
210 	return ret;
211 }
212 
213 static const struct net_device_ops netdev_ops = {
214 	.ndo_open		= wg_open,
215 	.ndo_stop		= wg_stop,
216 	.ndo_start_xmit		= wg_xmit,
217 	.ndo_get_stats64	= ip_tunnel_get_stats64
218 };
219 
220 static void wg_destruct(struct net_device *dev)
221 {
222 	struct wg_device *wg = netdev_priv(dev);
223 
224 	rtnl_lock();
225 	list_del(&wg->device_list);
226 	rtnl_unlock();
227 	mutex_lock(&wg->device_update_lock);
228 	wg->incoming_port = 0;
229 	wg_socket_reinit(wg, NULL, NULL);
230 	/* The final references are cleared in the below calls to destroy_workqueue. */
231 	wg_peer_remove_all(wg);
232 	destroy_workqueue(wg->handshake_receive_wq);
233 	destroy_workqueue(wg->handshake_send_wq);
234 	destroy_workqueue(wg->packet_crypt_wq);
235 	wg_packet_queue_free(&wg->decrypt_queue, true);
236 	wg_packet_queue_free(&wg->encrypt_queue, true);
237 	rcu_barrier(); /* Wait for all the peers to be actually freed. */
238 	wg_ratelimiter_uninit();
239 	memzero_explicit(&wg->static_identity, sizeof(wg->static_identity));
240 	skb_queue_purge(&wg->incoming_handshakes);
241 	free_percpu(dev->tstats);
242 	free_percpu(wg->incoming_handshakes_worker);
243 	if (wg->have_creating_net_ref)
244 		put_net(wg->creating_net);
245 	kvfree(wg->index_hashtable);
246 	kvfree(wg->peer_hashtable);
247 	mutex_unlock(&wg->device_update_lock);
248 
249 	pr_debug("%s: Interface deleted\n", dev->name);
250 	free_netdev(dev);
251 }
252 
253 static const struct device_type device_type = { .name = KBUILD_MODNAME };
254 
255 static void wg_setup(struct net_device *dev)
256 {
257 	struct wg_device *wg = netdev_priv(dev);
258 	enum { WG_NETDEV_FEATURES = NETIF_F_HW_CSUM | NETIF_F_RXCSUM |
259 				    NETIF_F_SG | NETIF_F_GSO |
260 				    NETIF_F_GSO_SOFTWARE | NETIF_F_HIGHDMA };
261 	const int overhead = MESSAGE_MINIMUM_LENGTH + sizeof(struct udphdr) +
262 			     max(sizeof(struct ipv6hdr), sizeof(struct iphdr));
263 
264 	dev->netdev_ops = &netdev_ops;
265 	dev->hard_header_len = 0;
266 	dev->addr_len = 0;
267 	dev->needed_headroom = DATA_PACKET_HEAD_ROOM;
268 	dev->needed_tailroom = noise_encrypted_len(MESSAGE_PADDING_MULTIPLE);
269 	dev->type = ARPHRD_NONE;
270 	dev->flags = IFF_POINTOPOINT | IFF_NOARP;
271 	dev->priv_flags |= IFF_NO_QUEUE;
272 	dev->features |= NETIF_F_LLTX;
273 	dev->features |= WG_NETDEV_FEATURES;
274 	dev->hw_features |= WG_NETDEV_FEATURES;
275 	dev->hw_enc_features |= WG_NETDEV_FEATURES;
276 	dev->mtu = ETH_DATA_LEN - overhead;
277 	dev->max_mtu = round_down(INT_MAX, MESSAGE_PADDING_MULTIPLE) - overhead;
278 
279 	SET_NETDEV_DEVTYPE(dev, &device_type);
280 
281 	/* We need to keep the dst around in case of icmp replies. */
282 	netif_keep_dst(dev);
283 
284 	memset(wg, 0, sizeof(*wg));
285 	wg->dev = dev;
286 }
287 
288 static int wg_newlink(struct net *src_net, struct net_device *dev,
289 		      struct nlattr *tb[], struct nlattr *data[],
290 		      struct netlink_ext_ack *extack)
291 {
292 	struct wg_device *wg = netdev_priv(dev);
293 	int ret = -ENOMEM;
294 
295 	wg->creating_net = src_net;
296 	init_rwsem(&wg->static_identity.lock);
297 	mutex_init(&wg->socket_update_lock);
298 	mutex_init(&wg->device_update_lock);
299 	skb_queue_head_init(&wg->incoming_handshakes);
300 	wg_allowedips_init(&wg->peer_allowedips);
301 	wg_cookie_checker_init(&wg->cookie_checker, wg);
302 	INIT_LIST_HEAD(&wg->peer_list);
303 	wg->device_update_gen = 1;
304 
305 	wg->peer_hashtable = wg_pubkey_hashtable_alloc();
306 	if (!wg->peer_hashtable)
307 		return ret;
308 
309 	wg->index_hashtable = wg_index_hashtable_alloc();
310 	if (!wg->index_hashtable)
311 		goto err_free_peer_hashtable;
312 
313 	dev->tstats = netdev_alloc_pcpu_stats(struct pcpu_sw_netstats);
314 	if (!dev->tstats)
315 		goto err_free_index_hashtable;
316 
317 	wg->incoming_handshakes_worker =
318 		wg_packet_percpu_multicore_worker_alloc(
319 				wg_packet_handshake_receive_worker, wg);
320 	if (!wg->incoming_handshakes_worker)
321 		goto err_free_tstats;
322 
323 	wg->handshake_receive_wq = alloc_workqueue("wg-kex-%s",
324 			WQ_CPU_INTENSIVE | WQ_FREEZABLE, 0, dev->name);
325 	if (!wg->handshake_receive_wq)
326 		goto err_free_incoming_handshakes;
327 
328 	wg->handshake_send_wq = alloc_workqueue("wg-kex-%s",
329 			WQ_UNBOUND | WQ_FREEZABLE, 0, dev->name);
330 	if (!wg->handshake_send_wq)
331 		goto err_destroy_handshake_receive;
332 
333 	wg->packet_crypt_wq = alloc_workqueue("wg-crypt-%s",
334 			WQ_CPU_INTENSIVE | WQ_MEM_RECLAIM, 0, dev->name);
335 	if (!wg->packet_crypt_wq)
336 		goto err_destroy_handshake_send;
337 
338 	ret = wg_packet_queue_init(&wg->encrypt_queue, wg_packet_encrypt_worker,
339 				   true, MAX_QUEUED_PACKETS);
340 	if (ret < 0)
341 		goto err_destroy_packet_crypt;
342 
343 	ret = wg_packet_queue_init(&wg->decrypt_queue, wg_packet_decrypt_worker,
344 				   true, MAX_QUEUED_PACKETS);
345 	if (ret < 0)
346 		goto err_free_encrypt_queue;
347 
348 	ret = wg_ratelimiter_init();
349 	if (ret < 0)
350 		goto err_free_decrypt_queue;
351 
352 	ret = register_netdevice(dev);
353 	if (ret < 0)
354 		goto err_uninit_ratelimiter;
355 
356 	list_add(&wg->device_list, &device_list);
357 
358 	/* We wait until the end to assign priv_destructor, so that
359 	 * register_netdevice doesn't call it for us if it fails.
360 	 */
361 	dev->priv_destructor = wg_destruct;
362 
363 	pr_debug("%s: Interface created\n", dev->name);
364 	return ret;
365 
366 err_uninit_ratelimiter:
367 	wg_ratelimiter_uninit();
368 err_free_decrypt_queue:
369 	wg_packet_queue_free(&wg->decrypt_queue, true);
370 err_free_encrypt_queue:
371 	wg_packet_queue_free(&wg->encrypt_queue, true);
372 err_destroy_packet_crypt:
373 	destroy_workqueue(wg->packet_crypt_wq);
374 err_destroy_handshake_send:
375 	destroy_workqueue(wg->handshake_send_wq);
376 err_destroy_handshake_receive:
377 	destroy_workqueue(wg->handshake_receive_wq);
378 err_free_incoming_handshakes:
379 	free_percpu(wg->incoming_handshakes_worker);
380 err_free_tstats:
381 	free_percpu(dev->tstats);
382 err_free_index_hashtable:
383 	kvfree(wg->index_hashtable);
384 err_free_peer_hashtable:
385 	kvfree(wg->peer_hashtable);
386 	return ret;
387 }
388 
389 static struct rtnl_link_ops link_ops __read_mostly = {
390 	.kind			= KBUILD_MODNAME,
391 	.priv_size		= sizeof(struct wg_device),
392 	.setup			= wg_setup,
393 	.newlink		= wg_newlink,
394 };
395 
396 static int wg_netdevice_notification(struct notifier_block *nb,
397 				     unsigned long action, void *data)
398 {
399 	struct net_device *dev = ((struct netdev_notifier_info *)data)->dev;
400 	struct wg_device *wg = netdev_priv(dev);
401 
402 	ASSERT_RTNL();
403 
404 	if (action != NETDEV_REGISTER || dev->netdev_ops != &netdev_ops)
405 		return 0;
406 
407 	if (dev_net(dev) == wg->creating_net && wg->have_creating_net_ref) {
408 		put_net(wg->creating_net);
409 		wg->have_creating_net_ref = false;
410 	} else if (dev_net(dev) != wg->creating_net &&
411 		   !wg->have_creating_net_ref) {
412 		wg->have_creating_net_ref = true;
413 		get_net(wg->creating_net);
414 	}
415 	return 0;
416 }
417 
418 static struct notifier_block netdevice_notifier = {
419 	.notifier_call = wg_netdevice_notification
420 };
421 
422 int __init wg_device_init(void)
423 {
424 	int ret;
425 
426 #ifdef CONFIG_PM_SLEEP
427 	ret = register_pm_notifier(&pm_notifier);
428 	if (ret)
429 		return ret;
430 #endif
431 
432 	ret = register_netdevice_notifier(&netdevice_notifier);
433 	if (ret)
434 		goto error_pm;
435 
436 	ret = rtnl_link_register(&link_ops);
437 	if (ret)
438 		goto error_netdevice;
439 
440 	return 0;
441 
442 error_netdevice:
443 	unregister_netdevice_notifier(&netdevice_notifier);
444 error_pm:
445 #ifdef CONFIG_PM_SLEEP
446 	unregister_pm_notifier(&pm_notifier);
447 #endif
448 	return ret;
449 }
450 
451 void wg_device_uninit(void)
452 {
453 	rtnl_link_unregister(&link_ops);
454 	unregister_netdevice_notifier(&netdevice_notifier);
455 #ifdef CONFIG_PM_SLEEP
456 	unregister_pm_notifier(&pm_notifier);
457 #endif
458 	rcu_barrier();
459 }
460