xref: /openbmc/linux/net/mpls/af_mpls.c (revision 4da722ca)
1 #include <linux/types.h>
2 #include <linux/skbuff.h>
3 #include <linux/socket.h>
4 #include <linux/sysctl.h>
5 #include <linux/net.h>
6 #include <linux/module.h>
7 #include <linux/if_arp.h>
8 #include <linux/ipv6.h>
9 #include <linux/mpls.h>
10 #include <linux/netconf.h>
11 #include <linux/vmalloc.h>
12 #include <linux/percpu.h>
13 #include <net/ip.h>
14 #include <net/dst.h>
15 #include <net/sock.h>
16 #include <net/arp.h>
17 #include <net/ip_fib.h>
18 #include <net/netevent.h>
19 #include <net/netns/generic.h>
20 #if IS_ENABLED(CONFIG_IPV6)
21 #include <net/ipv6.h>
22 #endif
23 #include <net/addrconf.h>
24 #include <net/nexthop.h>
25 #include "internal.h"
26 
27 /* max memory we will use for mpls_route */
28 #define MAX_MPLS_ROUTE_MEM	4096
29 
30 /* Maximum number of labels to look ahead at when selecting a path of
31  * a multipath route
32  */
33 #define MAX_MP_SELECT_LABELS 4
34 
35 #define MPLS_NEIGH_TABLE_UNSPEC (NEIGH_LINK_TABLE + 1)
36 
37 static int zero = 0;
38 static int one = 1;
39 static int label_limit = (1 << 20) - 1;
40 static int ttl_max = 255;
41 
42 static void rtmsg_lfib(int event, u32 label, struct mpls_route *rt,
43 		       struct nlmsghdr *nlh, struct net *net, u32 portid,
44 		       unsigned int nlm_flags);
45 
46 static struct mpls_route *mpls_route_input_rcu(struct net *net, unsigned index)
47 {
48 	struct mpls_route *rt = NULL;
49 
50 	if (index < net->mpls.platform_labels) {
51 		struct mpls_route __rcu **platform_label =
52 			rcu_dereference(net->mpls.platform_label);
53 		rt = rcu_dereference(platform_label[index]);
54 	}
55 	return rt;
56 }
57 
58 bool mpls_output_possible(const struct net_device *dev)
59 {
60 	return dev && (dev->flags & IFF_UP) && netif_carrier_ok(dev);
61 }
62 EXPORT_SYMBOL_GPL(mpls_output_possible);
63 
64 static u8 *__mpls_nh_via(struct mpls_route *rt, struct mpls_nh *nh)
65 {
66 	return (u8 *)nh + rt->rt_via_offset;
67 }
68 
69 static const u8 *mpls_nh_via(const struct mpls_route *rt,
70 			     const struct mpls_nh *nh)
71 {
72 	return __mpls_nh_via((struct mpls_route *)rt, (struct mpls_nh *)nh);
73 }
74 
75 static unsigned int mpls_nh_header_size(const struct mpls_nh *nh)
76 {
77 	/* The size of the layer 2.5 labels to be added for this route */
78 	return nh->nh_labels * sizeof(struct mpls_shim_hdr);
79 }
80 
81 unsigned int mpls_dev_mtu(const struct net_device *dev)
82 {
83 	/* The amount of data the layer 2 frame can hold */
84 	return dev->mtu;
85 }
86 EXPORT_SYMBOL_GPL(mpls_dev_mtu);
87 
88 bool mpls_pkt_too_big(const struct sk_buff *skb, unsigned int mtu)
89 {
90 	if (skb->len <= mtu)
91 		return false;
92 
93 	if (skb_is_gso(skb) && skb_gso_validate_mtu(skb, mtu))
94 		return false;
95 
96 	return true;
97 }
98 EXPORT_SYMBOL_GPL(mpls_pkt_too_big);
99 
100 void mpls_stats_inc_outucastpkts(struct net_device *dev,
101 				 const struct sk_buff *skb)
102 {
103 	struct mpls_dev *mdev;
104 
105 	if (skb->protocol == htons(ETH_P_MPLS_UC)) {
106 		mdev = mpls_dev_get(dev);
107 		if (mdev)
108 			MPLS_INC_STATS_LEN(mdev, skb->len,
109 					   tx_packets,
110 					   tx_bytes);
111 	} else if (skb->protocol == htons(ETH_P_IP)) {
112 		IP_UPD_PO_STATS(dev_net(dev), IPSTATS_MIB_OUT, skb->len);
113 #if IS_ENABLED(CONFIG_IPV6)
114 	} else if (skb->protocol == htons(ETH_P_IPV6)) {
115 		struct inet6_dev *in6dev = __in6_dev_get(dev);
116 
117 		if (in6dev)
118 			IP6_UPD_PO_STATS(dev_net(dev), in6dev,
119 					 IPSTATS_MIB_OUT, skb->len);
120 #endif
121 	}
122 }
123 EXPORT_SYMBOL_GPL(mpls_stats_inc_outucastpkts);
124 
125 static u32 mpls_multipath_hash(struct mpls_route *rt, struct sk_buff *skb)
126 {
127 	struct mpls_entry_decoded dec;
128 	unsigned int mpls_hdr_len = 0;
129 	struct mpls_shim_hdr *hdr;
130 	bool eli_seen = false;
131 	int label_index;
132 	u32 hash = 0;
133 
134 	for (label_index = 0; label_index < MAX_MP_SELECT_LABELS;
135 	     label_index++) {
136 		mpls_hdr_len += sizeof(*hdr);
137 		if (!pskb_may_pull(skb, mpls_hdr_len))
138 			break;
139 
140 		/* Read and decode the current label */
141 		hdr = mpls_hdr(skb) + label_index;
142 		dec = mpls_entry_decode(hdr);
143 
144 		/* RFC6790 - reserved labels MUST NOT be used as keys
145 		 * for the load-balancing function
146 		 */
147 		if (likely(dec.label >= MPLS_LABEL_FIRST_UNRESERVED)) {
148 			hash = jhash_1word(dec.label, hash);
149 
150 			/* The entropy label follows the entropy label
151 			 * indicator, so this means that the entropy
152 			 * label was just added to the hash - no need to
153 			 * go any deeper either in the label stack or in the
154 			 * payload
155 			 */
156 			if (eli_seen)
157 				break;
158 		} else if (dec.label == MPLS_LABEL_ENTROPY) {
159 			eli_seen = true;
160 		}
161 
162 		if (!dec.bos)
163 			continue;
164 
165 		/* found bottom label; does skb have room for a header? */
166 		if (pskb_may_pull(skb, mpls_hdr_len + sizeof(struct iphdr))) {
167 			const struct iphdr *v4hdr;
168 
169 			v4hdr = (const struct iphdr *)(hdr + 1);
170 			if (v4hdr->version == 4) {
171 				hash = jhash_3words(ntohl(v4hdr->saddr),
172 						    ntohl(v4hdr->daddr),
173 						    v4hdr->protocol, hash);
174 			} else if (v4hdr->version == 6 &&
175 				   pskb_may_pull(skb, mpls_hdr_len +
176 						 sizeof(struct ipv6hdr))) {
177 				const struct ipv6hdr *v6hdr;
178 
179 				v6hdr = (const struct ipv6hdr *)(hdr + 1);
180 				hash = __ipv6_addr_jhash(&v6hdr->saddr, hash);
181 				hash = __ipv6_addr_jhash(&v6hdr->daddr, hash);
182 				hash = jhash_1word(v6hdr->nexthdr, hash);
183 			}
184 		}
185 
186 		break;
187 	}
188 
189 	return hash;
190 }
191 
192 static struct mpls_nh *mpls_get_nexthop(struct mpls_route *rt, u8 index)
193 {
194 	return (struct mpls_nh *)((u8 *)rt->rt_nh + index * rt->rt_nh_size);
195 }
196 
197 /* number of alive nexthops (rt->rt_nhn_alive) and the flags for
198  * a next hop (nh->nh_flags) are modified by netdev event handlers.
199  * Since those fields can change at any moment, use READ_ONCE to
200  * access both.
201  */
202 static struct mpls_nh *mpls_select_multipath(struct mpls_route *rt,
203 					     struct sk_buff *skb)
204 {
205 	u32 hash = 0;
206 	int nh_index = 0;
207 	int n = 0;
208 	u8 alive;
209 
210 	/* No need to look further into packet if there's only
211 	 * one path
212 	 */
213 	if (rt->rt_nhn == 1)
214 		return rt->rt_nh;
215 
216 	alive = READ_ONCE(rt->rt_nhn_alive);
217 	if (alive == 0)
218 		return NULL;
219 
220 	hash = mpls_multipath_hash(rt, skb);
221 	nh_index = hash % alive;
222 	if (alive == rt->rt_nhn)
223 		goto out;
224 	for_nexthops(rt) {
225 		unsigned int nh_flags = READ_ONCE(nh->nh_flags);
226 
227 		if (nh_flags & (RTNH_F_DEAD | RTNH_F_LINKDOWN))
228 			continue;
229 		if (n == nh_index)
230 			return nh;
231 		n++;
232 	} endfor_nexthops(rt);
233 
234 out:
235 	return mpls_get_nexthop(rt, nh_index);
236 }
237 
238 static bool mpls_egress(struct net *net, struct mpls_route *rt,
239 			struct sk_buff *skb, struct mpls_entry_decoded dec)
240 {
241 	enum mpls_payload_type payload_type;
242 	bool success = false;
243 
244 	/* The IPv4 code below accesses through the IPv4 header
245 	 * checksum, which is 12 bytes into the packet.
246 	 * The IPv6 code below accesses through the IPv6 hop limit
247 	 * which is 8 bytes into the packet.
248 	 *
249 	 * For all supported cases there should always be at least 12
250 	 * bytes of packet data present.  The IPv4 header is 20 bytes
251 	 * without options and the IPv6 header is always 40 bytes
252 	 * long.
253 	 */
254 	if (!pskb_may_pull(skb, 12))
255 		return false;
256 
257 	payload_type = rt->rt_payload_type;
258 	if (payload_type == MPT_UNSPEC)
259 		payload_type = ip_hdr(skb)->version;
260 
261 	switch (payload_type) {
262 	case MPT_IPV4: {
263 		struct iphdr *hdr4 = ip_hdr(skb);
264 		u8 new_ttl;
265 		skb->protocol = htons(ETH_P_IP);
266 
267 		/* If propagating TTL, take the decremented TTL from
268 		 * the incoming MPLS header, otherwise decrement the
269 		 * TTL, but only if not 0 to avoid underflow.
270 		 */
271 		if (rt->rt_ttl_propagate == MPLS_TTL_PROP_ENABLED ||
272 		    (rt->rt_ttl_propagate == MPLS_TTL_PROP_DEFAULT &&
273 		     net->mpls.ip_ttl_propagate))
274 			new_ttl = dec.ttl;
275 		else
276 			new_ttl = hdr4->ttl ? hdr4->ttl - 1 : 0;
277 
278 		csum_replace2(&hdr4->check,
279 			      htons(hdr4->ttl << 8),
280 			      htons(new_ttl << 8));
281 		hdr4->ttl = new_ttl;
282 		success = true;
283 		break;
284 	}
285 	case MPT_IPV6: {
286 		struct ipv6hdr *hdr6 = ipv6_hdr(skb);
287 		skb->protocol = htons(ETH_P_IPV6);
288 
289 		/* If propagating TTL, take the decremented TTL from
290 		 * the incoming MPLS header, otherwise decrement the
291 		 * hop limit, but only if not 0 to avoid underflow.
292 		 */
293 		if (rt->rt_ttl_propagate == MPLS_TTL_PROP_ENABLED ||
294 		    (rt->rt_ttl_propagate == MPLS_TTL_PROP_DEFAULT &&
295 		     net->mpls.ip_ttl_propagate))
296 			hdr6->hop_limit = dec.ttl;
297 		else if (hdr6->hop_limit)
298 			hdr6->hop_limit = hdr6->hop_limit - 1;
299 		success = true;
300 		break;
301 	}
302 	case MPT_UNSPEC:
303 		/* Should have decided which protocol it is by now */
304 		break;
305 	}
306 
307 	return success;
308 }
309 
310 static int mpls_forward(struct sk_buff *skb, struct net_device *dev,
311 			struct packet_type *pt, struct net_device *orig_dev)
312 {
313 	struct net *net = dev_net(dev);
314 	struct mpls_shim_hdr *hdr;
315 	struct mpls_route *rt;
316 	struct mpls_nh *nh;
317 	struct mpls_entry_decoded dec;
318 	struct net_device *out_dev;
319 	struct mpls_dev *out_mdev;
320 	struct mpls_dev *mdev;
321 	unsigned int hh_len;
322 	unsigned int new_header_size;
323 	unsigned int mtu;
324 	int err;
325 
326 	/* Careful this entire function runs inside of an rcu critical section */
327 
328 	mdev = mpls_dev_get(dev);
329 	if (!mdev)
330 		goto drop;
331 
332 	MPLS_INC_STATS_LEN(mdev, skb->len, rx_packets,
333 			   rx_bytes);
334 
335 	if (!mdev->input_enabled) {
336 		MPLS_INC_STATS(mdev, rx_dropped);
337 		goto drop;
338 	}
339 
340 	if (skb->pkt_type != PACKET_HOST)
341 		goto err;
342 
343 	if ((skb = skb_share_check(skb, GFP_ATOMIC)) == NULL)
344 		goto err;
345 
346 	if (!pskb_may_pull(skb, sizeof(*hdr)))
347 		goto err;
348 
349 	/* Read and decode the label */
350 	hdr = mpls_hdr(skb);
351 	dec = mpls_entry_decode(hdr);
352 
353 	rt = mpls_route_input_rcu(net, dec.label);
354 	if (!rt) {
355 		MPLS_INC_STATS(mdev, rx_noroute);
356 		goto drop;
357 	}
358 
359 	nh = mpls_select_multipath(rt, skb);
360 	if (!nh)
361 		goto err;
362 
363 	/* Pop the label */
364 	skb_pull(skb, sizeof(*hdr));
365 	skb_reset_network_header(skb);
366 
367 	skb_orphan(skb);
368 
369 	if (skb_warn_if_lro(skb))
370 		goto err;
371 
372 	skb_forward_csum(skb);
373 
374 	/* Verify ttl is valid */
375 	if (dec.ttl <= 1)
376 		goto err;
377 	dec.ttl -= 1;
378 
379 	/* Find the output device */
380 	out_dev = rcu_dereference(nh->nh_dev);
381 	if (!mpls_output_possible(out_dev))
382 		goto tx_err;
383 
384 	/* Verify the destination can hold the packet */
385 	new_header_size = mpls_nh_header_size(nh);
386 	mtu = mpls_dev_mtu(out_dev);
387 	if (mpls_pkt_too_big(skb, mtu - new_header_size))
388 		goto tx_err;
389 
390 	hh_len = LL_RESERVED_SPACE(out_dev);
391 	if (!out_dev->header_ops)
392 		hh_len = 0;
393 
394 	/* Ensure there is enough space for the headers in the skb */
395 	if (skb_cow(skb, hh_len + new_header_size))
396 		goto tx_err;
397 
398 	skb->dev = out_dev;
399 	skb->protocol = htons(ETH_P_MPLS_UC);
400 
401 	if (unlikely(!new_header_size && dec.bos)) {
402 		/* Penultimate hop popping */
403 		if (!mpls_egress(dev_net(out_dev), rt, skb, dec))
404 			goto err;
405 	} else {
406 		bool bos;
407 		int i;
408 		skb_push(skb, new_header_size);
409 		skb_reset_network_header(skb);
410 		/* Push the new labels */
411 		hdr = mpls_hdr(skb);
412 		bos = dec.bos;
413 		for (i = nh->nh_labels - 1; i >= 0; i--) {
414 			hdr[i] = mpls_entry_encode(nh->nh_label[i],
415 						   dec.ttl, 0, bos);
416 			bos = false;
417 		}
418 	}
419 
420 	mpls_stats_inc_outucastpkts(out_dev, skb);
421 
422 	/* If via wasn't specified then send out using device address */
423 	if (nh->nh_via_table == MPLS_NEIGH_TABLE_UNSPEC)
424 		err = neigh_xmit(NEIGH_LINK_TABLE, out_dev,
425 				 out_dev->dev_addr, skb);
426 	else
427 		err = neigh_xmit(nh->nh_via_table, out_dev,
428 				 mpls_nh_via(rt, nh), skb);
429 	if (err)
430 		net_dbg_ratelimited("%s: packet transmission failed: %d\n",
431 				    __func__, err);
432 	return 0;
433 
434 tx_err:
435 	out_mdev = out_dev ? mpls_dev_get(out_dev) : NULL;
436 	if (out_mdev)
437 		MPLS_INC_STATS(out_mdev, tx_errors);
438 	goto drop;
439 err:
440 	MPLS_INC_STATS(mdev, rx_errors);
441 drop:
442 	kfree_skb(skb);
443 	return NET_RX_DROP;
444 }
445 
446 static struct packet_type mpls_packet_type __read_mostly = {
447 	.type = cpu_to_be16(ETH_P_MPLS_UC),
448 	.func = mpls_forward,
449 };
450 
451 static const struct nla_policy rtm_mpls_policy[RTA_MAX+1] = {
452 	[RTA_DST]		= { .type = NLA_U32 },
453 	[RTA_OIF]		= { .type = NLA_U32 },
454 	[RTA_TTL_PROPAGATE]	= { .type = NLA_U8 },
455 };
456 
457 struct mpls_route_config {
458 	u32			rc_protocol;
459 	u32			rc_ifindex;
460 	u8			rc_via_table;
461 	u8			rc_via_alen;
462 	u8			rc_via[MAX_VIA_ALEN];
463 	u32			rc_label;
464 	u8			rc_ttl_propagate;
465 	u8			rc_output_labels;
466 	u32			rc_output_label[MAX_NEW_LABELS];
467 	u32			rc_nlflags;
468 	enum mpls_payload_type	rc_payload_type;
469 	struct nl_info		rc_nlinfo;
470 	struct rtnexthop	*rc_mp;
471 	int			rc_mp_len;
472 };
473 
474 /* all nexthops within a route have the same size based on max
475  * number of labels and max via length for a hop
476  */
477 static struct mpls_route *mpls_rt_alloc(u8 num_nh, u8 max_alen, u8 max_labels)
478 {
479 	u8 nh_size = MPLS_NH_SIZE(max_labels, max_alen);
480 	struct mpls_route *rt;
481 	size_t size;
482 
483 	size = sizeof(*rt) + num_nh * nh_size;
484 	if (size > MAX_MPLS_ROUTE_MEM)
485 		return ERR_PTR(-EINVAL);
486 
487 	rt = kzalloc(size, GFP_KERNEL);
488 	if (!rt)
489 		return ERR_PTR(-ENOMEM);
490 
491 	rt->rt_nhn = num_nh;
492 	rt->rt_nhn_alive = num_nh;
493 	rt->rt_nh_size = nh_size;
494 	rt->rt_via_offset = MPLS_NH_VIA_OFF(max_labels);
495 
496 	return rt;
497 }
498 
499 static void mpls_rt_free(struct mpls_route *rt)
500 {
501 	if (rt)
502 		kfree_rcu(rt, rt_rcu);
503 }
504 
505 static void mpls_notify_route(struct net *net, unsigned index,
506 			      struct mpls_route *old, struct mpls_route *new,
507 			      const struct nl_info *info)
508 {
509 	struct nlmsghdr *nlh = info ? info->nlh : NULL;
510 	unsigned portid = info ? info->portid : 0;
511 	int event = new ? RTM_NEWROUTE : RTM_DELROUTE;
512 	struct mpls_route *rt = new ? new : old;
513 	unsigned nlm_flags = (old && new) ? NLM_F_REPLACE : 0;
514 	/* Ignore reserved labels for now */
515 	if (rt && (index >= MPLS_LABEL_FIRST_UNRESERVED))
516 		rtmsg_lfib(event, index, rt, nlh, net, portid, nlm_flags);
517 }
518 
519 static void mpls_route_update(struct net *net, unsigned index,
520 			      struct mpls_route *new,
521 			      const struct nl_info *info)
522 {
523 	struct mpls_route __rcu **platform_label;
524 	struct mpls_route *rt;
525 
526 	ASSERT_RTNL();
527 
528 	platform_label = rtnl_dereference(net->mpls.platform_label);
529 	rt = rtnl_dereference(platform_label[index]);
530 	rcu_assign_pointer(platform_label[index], new);
531 
532 	mpls_notify_route(net, index, rt, new, info);
533 
534 	/* If we removed a route free it now */
535 	mpls_rt_free(rt);
536 }
537 
538 static unsigned find_free_label(struct net *net)
539 {
540 	struct mpls_route __rcu **platform_label;
541 	size_t platform_labels;
542 	unsigned index;
543 
544 	platform_label = rtnl_dereference(net->mpls.platform_label);
545 	platform_labels = net->mpls.platform_labels;
546 	for (index = MPLS_LABEL_FIRST_UNRESERVED; index < platform_labels;
547 	     index++) {
548 		if (!rtnl_dereference(platform_label[index]))
549 			return index;
550 	}
551 	return LABEL_NOT_SPECIFIED;
552 }
553 
554 #if IS_ENABLED(CONFIG_INET)
555 static struct net_device *inet_fib_lookup_dev(struct net *net,
556 					      const void *addr)
557 {
558 	struct net_device *dev;
559 	struct rtable *rt;
560 	struct in_addr daddr;
561 
562 	memcpy(&daddr, addr, sizeof(struct in_addr));
563 	rt = ip_route_output(net, daddr.s_addr, 0, 0, 0);
564 	if (IS_ERR(rt))
565 		return ERR_CAST(rt);
566 
567 	dev = rt->dst.dev;
568 	dev_hold(dev);
569 
570 	ip_rt_put(rt);
571 
572 	return dev;
573 }
574 #else
575 static struct net_device *inet_fib_lookup_dev(struct net *net,
576 					      const void *addr)
577 {
578 	return ERR_PTR(-EAFNOSUPPORT);
579 }
580 #endif
581 
582 #if IS_ENABLED(CONFIG_IPV6)
583 static struct net_device *inet6_fib_lookup_dev(struct net *net,
584 					       const void *addr)
585 {
586 	struct net_device *dev;
587 	struct dst_entry *dst;
588 	struct flowi6 fl6;
589 	int err;
590 
591 	if (!ipv6_stub)
592 		return ERR_PTR(-EAFNOSUPPORT);
593 
594 	memset(&fl6, 0, sizeof(fl6));
595 	memcpy(&fl6.daddr, addr, sizeof(struct in6_addr));
596 	err = ipv6_stub->ipv6_dst_lookup(net, NULL, &dst, &fl6);
597 	if (err)
598 		return ERR_PTR(err);
599 
600 	dev = dst->dev;
601 	dev_hold(dev);
602 	dst_release(dst);
603 
604 	return dev;
605 }
606 #else
607 static struct net_device *inet6_fib_lookup_dev(struct net *net,
608 					       const void *addr)
609 {
610 	return ERR_PTR(-EAFNOSUPPORT);
611 }
612 #endif
613 
614 static struct net_device *find_outdev(struct net *net,
615 				      struct mpls_route *rt,
616 				      struct mpls_nh *nh, int oif)
617 {
618 	struct net_device *dev = NULL;
619 
620 	if (!oif) {
621 		switch (nh->nh_via_table) {
622 		case NEIGH_ARP_TABLE:
623 			dev = inet_fib_lookup_dev(net, mpls_nh_via(rt, nh));
624 			break;
625 		case NEIGH_ND_TABLE:
626 			dev = inet6_fib_lookup_dev(net, mpls_nh_via(rt, nh));
627 			break;
628 		case NEIGH_LINK_TABLE:
629 			break;
630 		}
631 	} else {
632 		dev = dev_get_by_index(net, oif);
633 	}
634 
635 	if (!dev)
636 		return ERR_PTR(-ENODEV);
637 
638 	if (IS_ERR(dev))
639 		return dev;
640 
641 	/* The caller is holding rtnl anyways, so release the dev reference */
642 	dev_put(dev);
643 
644 	return dev;
645 }
646 
647 static int mpls_nh_assign_dev(struct net *net, struct mpls_route *rt,
648 			      struct mpls_nh *nh, int oif)
649 {
650 	struct net_device *dev = NULL;
651 	int err = -ENODEV;
652 
653 	dev = find_outdev(net, rt, nh, oif);
654 	if (IS_ERR(dev)) {
655 		err = PTR_ERR(dev);
656 		dev = NULL;
657 		goto errout;
658 	}
659 
660 	/* Ensure this is a supported device */
661 	err = -EINVAL;
662 	if (!mpls_dev_get(dev))
663 		goto errout;
664 
665 	if ((nh->nh_via_table == NEIGH_LINK_TABLE) &&
666 	    (dev->addr_len != nh->nh_via_alen))
667 		goto errout;
668 
669 	RCU_INIT_POINTER(nh->nh_dev, dev);
670 
671 	if (!(dev->flags & IFF_UP)) {
672 		nh->nh_flags |= RTNH_F_DEAD;
673 	} else {
674 		unsigned int flags;
675 
676 		flags = dev_get_flags(dev);
677 		if (!(flags & (IFF_RUNNING | IFF_LOWER_UP)))
678 			nh->nh_flags |= RTNH_F_LINKDOWN;
679 	}
680 
681 	return 0;
682 
683 errout:
684 	return err;
685 }
686 
687 static int nla_get_via(const struct nlattr *nla, u8 *via_alen, u8 *via_table,
688 		       u8 via_addr[], struct netlink_ext_ack *extack)
689 {
690 	struct rtvia *via = nla_data(nla);
691 	int err = -EINVAL;
692 	int alen;
693 
694 	if (nla_len(nla) < offsetof(struct rtvia, rtvia_addr)) {
695 		NL_SET_ERR_MSG_ATTR(extack, nla,
696 				    "Invalid attribute length for RTA_VIA");
697 		goto errout;
698 	}
699 	alen = nla_len(nla) -
700 			offsetof(struct rtvia, rtvia_addr);
701 	if (alen > MAX_VIA_ALEN) {
702 		NL_SET_ERR_MSG_ATTR(extack, nla,
703 				    "Invalid address length for RTA_VIA");
704 		goto errout;
705 	}
706 
707 	/* Validate the address family */
708 	switch (via->rtvia_family) {
709 	case AF_PACKET:
710 		*via_table = NEIGH_LINK_TABLE;
711 		break;
712 	case AF_INET:
713 		*via_table = NEIGH_ARP_TABLE;
714 		if (alen != 4)
715 			goto errout;
716 		break;
717 	case AF_INET6:
718 		*via_table = NEIGH_ND_TABLE;
719 		if (alen != 16)
720 			goto errout;
721 		break;
722 	default:
723 		/* Unsupported address family */
724 		goto errout;
725 	}
726 
727 	memcpy(via_addr, via->rtvia_addr, alen);
728 	*via_alen = alen;
729 	err = 0;
730 
731 errout:
732 	return err;
733 }
734 
735 static int mpls_nh_build_from_cfg(struct mpls_route_config *cfg,
736 				  struct mpls_route *rt)
737 {
738 	struct net *net = cfg->rc_nlinfo.nl_net;
739 	struct mpls_nh *nh = rt->rt_nh;
740 	int err;
741 	int i;
742 
743 	if (!nh)
744 		return -ENOMEM;
745 
746 	nh->nh_labels = cfg->rc_output_labels;
747 	for (i = 0; i < nh->nh_labels; i++)
748 		nh->nh_label[i] = cfg->rc_output_label[i];
749 
750 	nh->nh_via_table = cfg->rc_via_table;
751 	memcpy(__mpls_nh_via(rt, nh), cfg->rc_via, cfg->rc_via_alen);
752 	nh->nh_via_alen = cfg->rc_via_alen;
753 
754 	err = mpls_nh_assign_dev(net, rt, nh, cfg->rc_ifindex);
755 	if (err)
756 		goto errout;
757 
758 	if (nh->nh_flags & (RTNH_F_DEAD | RTNH_F_LINKDOWN))
759 		rt->rt_nhn_alive--;
760 
761 	return 0;
762 
763 errout:
764 	return err;
765 }
766 
767 static int mpls_nh_build(struct net *net, struct mpls_route *rt,
768 			 struct mpls_nh *nh, int oif, struct nlattr *via,
769 			 struct nlattr *newdst, u8 max_labels,
770 			 struct netlink_ext_ack *extack)
771 {
772 	int err = -ENOMEM;
773 
774 	if (!nh)
775 		goto errout;
776 
777 	if (newdst) {
778 		err = nla_get_labels(newdst, max_labels, &nh->nh_labels,
779 				     nh->nh_label, extack);
780 		if (err)
781 			goto errout;
782 	}
783 
784 	if (via) {
785 		err = nla_get_via(via, &nh->nh_via_alen, &nh->nh_via_table,
786 				  __mpls_nh_via(rt, nh), extack);
787 		if (err)
788 			goto errout;
789 	} else {
790 		nh->nh_via_table = MPLS_NEIGH_TABLE_UNSPEC;
791 	}
792 
793 	err = mpls_nh_assign_dev(net, rt, nh, oif);
794 	if (err)
795 		goto errout;
796 
797 	return 0;
798 
799 errout:
800 	return err;
801 }
802 
803 static u8 mpls_count_nexthops(struct rtnexthop *rtnh, int len,
804 			      u8 cfg_via_alen, u8 *max_via_alen,
805 			      u8 *max_labels)
806 {
807 	int remaining = len;
808 	u8 nhs = 0;
809 
810 	*max_via_alen = 0;
811 	*max_labels = 0;
812 
813 	while (rtnh_ok(rtnh, remaining)) {
814 		struct nlattr *nla, *attrs = rtnh_attrs(rtnh);
815 		int attrlen;
816 		u8 n_labels = 0;
817 
818 		attrlen = rtnh_attrlen(rtnh);
819 		nla = nla_find(attrs, attrlen, RTA_VIA);
820 		if (nla && nla_len(nla) >=
821 		    offsetof(struct rtvia, rtvia_addr)) {
822 			int via_alen = nla_len(nla) -
823 				offsetof(struct rtvia, rtvia_addr);
824 
825 			if (via_alen <= MAX_VIA_ALEN)
826 				*max_via_alen = max_t(u16, *max_via_alen,
827 						      via_alen);
828 		}
829 
830 		nla = nla_find(attrs, attrlen, RTA_NEWDST);
831 		if (nla &&
832 		    nla_get_labels(nla, MAX_NEW_LABELS, &n_labels,
833 				   NULL, NULL) != 0)
834 			return 0;
835 
836 		*max_labels = max_t(u8, *max_labels, n_labels);
837 
838 		/* number of nexthops is tracked by a u8.
839 		 * Check for overflow.
840 		 */
841 		if (nhs == 255)
842 			return 0;
843 		nhs++;
844 
845 		rtnh = rtnh_next(rtnh, &remaining);
846 	}
847 
848 	/* leftover implies invalid nexthop configuration, discard it */
849 	return remaining > 0 ? 0 : nhs;
850 }
851 
852 static int mpls_nh_build_multi(struct mpls_route_config *cfg,
853 			       struct mpls_route *rt, u8 max_labels,
854 			       struct netlink_ext_ack *extack)
855 {
856 	struct rtnexthop *rtnh = cfg->rc_mp;
857 	struct nlattr *nla_via, *nla_newdst;
858 	int remaining = cfg->rc_mp_len;
859 	int err = 0;
860 	u8 nhs = 0;
861 
862 	change_nexthops(rt) {
863 		int attrlen;
864 
865 		nla_via = NULL;
866 		nla_newdst = NULL;
867 
868 		err = -EINVAL;
869 		if (!rtnh_ok(rtnh, remaining))
870 			goto errout;
871 
872 		/* neither weighted multipath nor any flags
873 		 * are supported
874 		 */
875 		if (rtnh->rtnh_hops || rtnh->rtnh_flags)
876 			goto errout;
877 
878 		attrlen = rtnh_attrlen(rtnh);
879 		if (attrlen > 0) {
880 			struct nlattr *attrs = rtnh_attrs(rtnh);
881 
882 			nla_via = nla_find(attrs, attrlen, RTA_VIA);
883 			nla_newdst = nla_find(attrs, attrlen, RTA_NEWDST);
884 		}
885 
886 		err = mpls_nh_build(cfg->rc_nlinfo.nl_net, rt, nh,
887 				    rtnh->rtnh_ifindex, nla_via, nla_newdst,
888 				    max_labels, extack);
889 		if (err)
890 			goto errout;
891 
892 		if (nh->nh_flags & (RTNH_F_DEAD | RTNH_F_LINKDOWN))
893 			rt->rt_nhn_alive--;
894 
895 		rtnh = rtnh_next(rtnh, &remaining);
896 		nhs++;
897 	} endfor_nexthops(rt);
898 
899 	rt->rt_nhn = nhs;
900 
901 	return 0;
902 
903 errout:
904 	return err;
905 }
906 
907 static bool mpls_label_ok(struct net *net, unsigned int index,
908 			  struct netlink_ext_ack *extack)
909 {
910 	/* Reserved labels may not be set */
911 	if (index < MPLS_LABEL_FIRST_UNRESERVED) {
912 		NL_SET_ERR_MSG(extack,
913 			       "Invalid label - must be MPLS_LABEL_FIRST_UNRESERVED or higher");
914 		return false;
915 	}
916 
917 	/* The full 20 bit range may not be supported. */
918 	if (index >= net->mpls.platform_labels) {
919 		NL_SET_ERR_MSG(extack,
920 			       "Label >= configured maximum in platform_labels");
921 		return false;
922 	}
923 
924 	return true;
925 }
926 
927 static int mpls_route_add(struct mpls_route_config *cfg,
928 			  struct netlink_ext_ack *extack)
929 {
930 	struct mpls_route __rcu **platform_label;
931 	struct net *net = cfg->rc_nlinfo.nl_net;
932 	struct mpls_route *rt, *old;
933 	int err = -EINVAL;
934 	u8 max_via_alen;
935 	unsigned index;
936 	u8 max_labels;
937 	u8 nhs;
938 
939 	index = cfg->rc_label;
940 
941 	/* If a label was not specified during insert pick one */
942 	if ((index == LABEL_NOT_SPECIFIED) &&
943 	    (cfg->rc_nlflags & NLM_F_CREATE)) {
944 		index = find_free_label(net);
945 	}
946 
947 	if (!mpls_label_ok(net, index, extack))
948 		goto errout;
949 
950 	/* Append makes no sense with mpls */
951 	err = -EOPNOTSUPP;
952 	if (cfg->rc_nlflags & NLM_F_APPEND) {
953 		NL_SET_ERR_MSG(extack, "MPLS does not support route append");
954 		goto errout;
955 	}
956 
957 	err = -EEXIST;
958 	platform_label = rtnl_dereference(net->mpls.platform_label);
959 	old = rtnl_dereference(platform_label[index]);
960 	if ((cfg->rc_nlflags & NLM_F_EXCL) && old)
961 		goto errout;
962 
963 	err = -EEXIST;
964 	if (!(cfg->rc_nlflags & NLM_F_REPLACE) && old)
965 		goto errout;
966 
967 	err = -ENOENT;
968 	if (!(cfg->rc_nlflags & NLM_F_CREATE) && !old)
969 		goto errout;
970 
971 	err = -EINVAL;
972 	if (cfg->rc_mp) {
973 		nhs = mpls_count_nexthops(cfg->rc_mp, cfg->rc_mp_len,
974 					  cfg->rc_via_alen, &max_via_alen,
975 					  &max_labels);
976 	} else {
977 		max_via_alen = cfg->rc_via_alen;
978 		max_labels = cfg->rc_output_labels;
979 		nhs = 1;
980 	}
981 
982 	if (nhs == 0) {
983 		NL_SET_ERR_MSG(extack, "Route does not contain a nexthop");
984 		goto errout;
985 	}
986 
987 	err = -ENOMEM;
988 	rt = mpls_rt_alloc(nhs, max_via_alen, max_labels);
989 	if (IS_ERR(rt)) {
990 		err = PTR_ERR(rt);
991 		goto errout;
992 	}
993 
994 	rt->rt_protocol = cfg->rc_protocol;
995 	rt->rt_payload_type = cfg->rc_payload_type;
996 	rt->rt_ttl_propagate = cfg->rc_ttl_propagate;
997 
998 	if (cfg->rc_mp)
999 		err = mpls_nh_build_multi(cfg, rt, max_labels, extack);
1000 	else
1001 		err = mpls_nh_build_from_cfg(cfg, rt);
1002 	if (err)
1003 		goto freert;
1004 
1005 	mpls_route_update(net, index, rt, &cfg->rc_nlinfo);
1006 
1007 	return 0;
1008 
1009 freert:
1010 	mpls_rt_free(rt);
1011 errout:
1012 	return err;
1013 }
1014 
1015 static int mpls_route_del(struct mpls_route_config *cfg,
1016 			  struct netlink_ext_ack *extack)
1017 {
1018 	struct net *net = cfg->rc_nlinfo.nl_net;
1019 	unsigned index;
1020 	int err = -EINVAL;
1021 
1022 	index = cfg->rc_label;
1023 
1024 	if (!mpls_label_ok(net, index, extack))
1025 		goto errout;
1026 
1027 	mpls_route_update(net, index, NULL, &cfg->rc_nlinfo);
1028 
1029 	err = 0;
1030 errout:
1031 	return err;
1032 }
1033 
1034 static void mpls_get_stats(struct mpls_dev *mdev,
1035 			   struct mpls_link_stats *stats)
1036 {
1037 	struct mpls_pcpu_stats *p;
1038 	int i;
1039 
1040 	memset(stats, 0, sizeof(*stats));
1041 
1042 	for_each_possible_cpu(i) {
1043 		struct mpls_link_stats local;
1044 		unsigned int start;
1045 
1046 		p = per_cpu_ptr(mdev->stats, i);
1047 		do {
1048 			start = u64_stats_fetch_begin(&p->syncp);
1049 			local = p->stats;
1050 		} while (u64_stats_fetch_retry(&p->syncp, start));
1051 
1052 		stats->rx_packets	+= local.rx_packets;
1053 		stats->rx_bytes		+= local.rx_bytes;
1054 		stats->tx_packets	+= local.tx_packets;
1055 		stats->tx_bytes		+= local.tx_bytes;
1056 		stats->rx_errors	+= local.rx_errors;
1057 		stats->tx_errors	+= local.tx_errors;
1058 		stats->rx_dropped	+= local.rx_dropped;
1059 		stats->tx_dropped	+= local.tx_dropped;
1060 		stats->rx_noroute	+= local.rx_noroute;
1061 	}
1062 }
1063 
1064 static int mpls_fill_stats_af(struct sk_buff *skb,
1065 			      const struct net_device *dev)
1066 {
1067 	struct mpls_link_stats *stats;
1068 	struct mpls_dev *mdev;
1069 	struct nlattr *nla;
1070 
1071 	mdev = mpls_dev_get(dev);
1072 	if (!mdev)
1073 		return -ENODATA;
1074 
1075 	nla = nla_reserve_64bit(skb, MPLS_STATS_LINK,
1076 				sizeof(struct mpls_link_stats),
1077 				MPLS_STATS_UNSPEC);
1078 	if (!nla)
1079 		return -EMSGSIZE;
1080 
1081 	stats = nla_data(nla);
1082 	mpls_get_stats(mdev, stats);
1083 
1084 	return 0;
1085 }
1086 
1087 static size_t mpls_get_stats_af_size(const struct net_device *dev)
1088 {
1089 	struct mpls_dev *mdev;
1090 
1091 	mdev = mpls_dev_get(dev);
1092 	if (!mdev)
1093 		return 0;
1094 
1095 	return nla_total_size_64bit(sizeof(struct mpls_link_stats));
1096 }
1097 
1098 static int mpls_netconf_fill_devconf(struct sk_buff *skb, struct mpls_dev *mdev,
1099 				     u32 portid, u32 seq, int event,
1100 				     unsigned int flags, int type)
1101 {
1102 	struct nlmsghdr  *nlh;
1103 	struct netconfmsg *ncm;
1104 	bool all = false;
1105 
1106 	nlh = nlmsg_put(skb, portid, seq, event, sizeof(struct netconfmsg),
1107 			flags);
1108 	if (!nlh)
1109 		return -EMSGSIZE;
1110 
1111 	if (type == NETCONFA_ALL)
1112 		all = true;
1113 
1114 	ncm = nlmsg_data(nlh);
1115 	ncm->ncm_family = AF_MPLS;
1116 
1117 	if (nla_put_s32(skb, NETCONFA_IFINDEX, mdev->dev->ifindex) < 0)
1118 		goto nla_put_failure;
1119 
1120 	if ((all || type == NETCONFA_INPUT) &&
1121 	    nla_put_s32(skb, NETCONFA_INPUT,
1122 			mdev->input_enabled) < 0)
1123 		goto nla_put_failure;
1124 
1125 	nlmsg_end(skb, nlh);
1126 	return 0;
1127 
1128 nla_put_failure:
1129 	nlmsg_cancel(skb, nlh);
1130 	return -EMSGSIZE;
1131 }
1132 
1133 static int mpls_netconf_msgsize_devconf(int type)
1134 {
1135 	int size = NLMSG_ALIGN(sizeof(struct netconfmsg))
1136 			+ nla_total_size(4); /* NETCONFA_IFINDEX */
1137 	bool all = false;
1138 
1139 	if (type == NETCONFA_ALL)
1140 		all = true;
1141 
1142 	if (all || type == NETCONFA_INPUT)
1143 		size += nla_total_size(4);
1144 
1145 	return size;
1146 }
1147 
1148 static void mpls_netconf_notify_devconf(struct net *net, int event,
1149 					int type, struct mpls_dev *mdev)
1150 {
1151 	struct sk_buff *skb;
1152 	int err = -ENOBUFS;
1153 
1154 	skb = nlmsg_new(mpls_netconf_msgsize_devconf(type), GFP_KERNEL);
1155 	if (!skb)
1156 		goto errout;
1157 
1158 	err = mpls_netconf_fill_devconf(skb, mdev, 0, 0, event, 0, type);
1159 	if (err < 0) {
1160 		/* -EMSGSIZE implies BUG in mpls_netconf_msgsize_devconf() */
1161 		WARN_ON(err == -EMSGSIZE);
1162 		kfree_skb(skb);
1163 		goto errout;
1164 	}
1165 
1166 	rtnl_notify(skb, net, 0, RTNLGRP_MPLS_NETCONF, NULL, GFP_KERNEL);
1167 	return;
1168 errout:
1169 	if (err < 0)
1170 		rtnl_set_sk_err(net, RTNLGRP_MPLS_NETCONF, err);
1171 }
1172 
1173 static const struct nla_policy devconf_mpls_policy[NETCONFA_MAX + 1] = {
1174 	[NETCONFA_IFINDEX]	= { .len = sizeof(int) },
1175 };
1176 
1177 static int mpls_netconf_get_devconf(struct sk_buff *in_skb,
1178 				    struct nlmsghdr *nlh,
1179 				    struct netlink_ext_ack *extack)
1180 {
1181 	struct net *net = sock_net(in_skb->sk);
1182 	struct nlattr *tb[NETCONFA_MAX + 1];
1183 	struct netconfmsg *ncm;
1184 	struct net_device *dev;
1185 	struct mpls_dev *mdev;
1186 	struct sk_buff *skb;
1187 	int ifindex;
1188 	int err;
1189 
1190 	err = nlmsg_parse(nlh, sizeof(*ncm), tb, NETCONFA_MAX,
1191 			  devconf_mpls_policy, NULL);
1192 	if (err < 0)
1193 		goto errout;
1194 
1195 	err = -EINVAL;
1196 	if (!tb[NETCONFA_IFINDEX])
1197 		goto errout;
1198 
1199 	ifindex = nla_get_s32(tb[NETCONFA_IFINDEX]);
1200 	dev = __dev_get_by_index(net, ifindex);
1201 	if (!dev)
1202 		goto errout;
1203 
1204 	mdev = mpls_dev_get(dev);
1205 	if (!mdev)
1206 		goto errout;
1207 
1208 	err = -ENOBUFS;
1209 	skb = nlmsg_new(mpls_netconf_msgsize_devconf(NETCONFA_ALL), GFP_KERNEL);
1210 	if (!skb)
1211 		goto errout;
1212 
1213 	err = mpls_netconf_fill_devconf(skb, mdev,
1214 					NETLINK_CB(in_skb).portid,
1215 					nlh->nlmsg_seq, RTM_NEWNETCONF, 0,
1216 					NETCONFA_ALL);
1217 	if (err < 0) {
1218 		/* -EMSGSIZE implies BUG in mpls_netconf_msgsize_devconf() */
1219 		WARN_ON(err == -EMSGSIZE);
1220 		kfree_skb(skb);
1221 		goto errout;
1222 	}
1223 	err = rtnl_unicast(skb, net, NETLINK_CB(in_skb).portid);
1224 errout:
1225 	return err;
1226 }
1227 
1228 static int mpls_netconf_dump_devconf(struct sk_buff *skb,
1229 				     struct netlink_callback *cb)
1230 {
1231 	struct net *net = sock_net(skb->sk);
1232 	struct hlist_head *head;
1233 	struct net_device *dev;
1234 	struct mpls_dev *mdev;
1235 	int idx, s_idx;
1236 	int h, s_h;
1237 
1238 	s_h = cb->args[0];
1239 	s_idx = idx = cb->args[1];
1240 
1241 	for (h = s_h; h < NETDEV_HASHENTRIES; h++, s_idx = 0) {
1242 		idx = 0;
1243 		head = &net->dev_index_head[h];
1244 		rcu_read_lock();
1245 		cb->seq = net->dev_base_seq;
1246 		hlist_for_each_entry_rcu(dev, head, index_hlist) {
1247 			if (idx < s_idx)
1248 				goto cont;
1249 			mdev = mpls_dev_get(dev);
1250 			if (!mdev)
1251 				goto cont;
1252 			if (mpls_netconf_fill_devconf(skb, mdev,
1253 						      NETLINK_CB(cb->skb).portid,
1254 						      cb->nlh->nlmsg_seq,
1255 						      RTM_NEWNETCONF,
1256 						      NLM_F_MULTI,
1257 						      NETCONFA_ALL) < 0) {
1258 				rcu_read_unlock();
1259 				goto done;
1260 			}
1261 			nl_dump_check_consistent(cb, nlmsg_hdr(skb));
1262 cont:
1263 			idx++;
1264 		}
1265 		rcu_read_unlock();
1266 	}
1267 done:
1268 	cb->args[0] = h;
1269 	cb->args[1] = idx;
1270 
1271 	return skb->len;
1272 }
1273 
1274 #define MPLS_PERDEV_SYSCTL_OFFSET(field)	\
1275 	(&((struct mpls_dev *)0)->field)
1276 
1277 static int mpls_conf_proc(struct ctl_table *ctl, int write,
1278 			  void __user *buffer,
1279 			  size_t *lenp, loff_t *ppos)
1280 {
1281 	int oval = *(int *)ctl->data;
1282 	int ret = proc_dointvec(ctl, write, buffer, lenp, ppos);
1283 
1284 	if (write) {
1285 		struct mpls_dev *mdev = ctl->extra1;
1286 		int i = (int *)ctl->data - (int *)mdev;
1287 		struct net *net = ctl->extra2;
1288 		int val = *(int *)ctl->data;
1289 
1290 		if (i == offsetof(struct mpls_dev, input_enabled) &&
1291 		    val != oval) {
1292 			mpls_netconf_notify_devconf(net, RTM_NEWNETCONF,
1293 						    NETCONFA_INPUT, mdev);
1294 		}
1295 	}
1296 
1297 	return ret;
1298 }
1299 
1300 static const struct ctl_table mpls_dev_table[] = {
1301 	{
1302 		.procname	= "input",
1303 		.maxlen		= sizeof(int),
1304 		.mode		= 0644,
1305 		.proc_handler	= mpls_conf_proc,
1306 		.data		= MPLS_PERDEV_SYSCTL_OFFSET(input_enabled),
1307 	},
1308 	{ }
1309 };
1310 
1311 static int mpls_dev_sysctl_register(struct net_device *dev,
1312 				    struct mpls_dev *mdev)
1313 {
1314 	char path[sizeof("net/mpls/conf/") + IFNAMSIZ];
1315 	struct net *net = dev_net(dev);
1316 	struct ctl_table *table;
1317 	int i;
1318 
1319 	table = kmemdup(&mpls_dev_table, sizeof(mpls_dev_table), GFP_KERNEL);
1320 	if (!table)
1321 		goto out;
1322 
1323 	/* Table data contains only offsets relative to the base of
1324 	 * the mdev at this point, so make them absolute.
1325 	 */
1326 	for (i = 0; i < ARRAY_SIZE(mpls_dev_table); i++) {
1327 		table[i].data = (char *)mdev + (uintptr_t)table[i].data;
1328 		table[i].extra1 = mdev;
1329 		table[i].extra2 = net;
1330 	}
1331 
1332 	snprintf(path, sizeof(path), "net/mpls/conf/%s", dev->name);
1333 
1334 	mdev->sysctl = register_net_sysctl(net, path, table);
1335 	if (!mdev->sysctl)
1336 		goto free;
1337 
1338 	mpls_netconf_notify_devconf(net, RTM_NEWNETCONF, NETCONFA_ALL, mdev);
1339 	return 0;
1340 
1341 free:
1342 	kfree(table);
1343 out:
1344 	return -ENOBUFS;
1345 }
1346 
1347 static void mpls_dev_sysctl_unregister(struct net_device *dev,
1348 				       struct mpls_dev *mdev)
1349 {
1350 	struct net *net = dev_net(dev);
1351 	struct ctl_table *table;
1352 
1353 	table = mdev->sysctl->ctl_table_arg;
1354 	unregister_net_sysctl_table(mdev->sysctl);
1355 	kfree(table);
1356 
1357 	mpls_netconf_notify_devconf(net, RTM_DELNETCONF, 0, mdev);
1358 }
1359 
1360 static struct mpls_dev *mpls_add_dev(struct net_device *dev)
1361 {
1362 	struct mpls_dev *mdev;
1363 	int err = -ENOMEM;
1364 	int i;
1365 
1366 	ASSERT_RTNL();
1367 
1368 	mdev = kzalloc(sizeof(*mdev), GFP_KERNEL);
1369 	if (!mdev)
1370 		return ERR_PTR(err);
1371 
1372 	mdev->stats = alloc_percpu(struct mpls_pcpu_stats);
1373 	if (!mdev->stats)
1374 		goto free;
1375 
1376 	for_each_possible_cpu(i) {
1377 		struct mpls_pcpu_stats *mpls_stats;
1378 
1379 		mpls_stats = per_cpu_ptr(mdev->stats, i);
1380 		u64_stats_init(&mpls_stats->syncp);
1381 	}
1382 
1383 	mdev->dev = dev;
1384 
1385 	err = mpls_dev_sysctl_register(dev, mdev);
1386 	if (err)
1387 		goto free;
1388 
1389 	rcu_assign_pointer(dev->mpls_ptr, mdev);
1390 
1391 	return mdev;
1392 
1393 free:
1394 	free_percpu(mdev->stats);
1395 	kfree(mdev);
1396 	return ERR_PTR(err);
1397 }
1398 
1399 static void mpls_dev_destroy_rcu(struct rcu_head *head)
1400 {
1401 	struct mpls_dev *mdev = container_of(head, struct mpls_dev, rcu);
1402 
1403 	free_percpu(mdev->stats);
1404 	kfree(mdev);
1405 }
1406 
1407 static void mpls_ifdown(struct net_device *dev, int event)
1408 {
1409 	struct mpls_route __rcu **platform_label;
1410 	struct net *net = dev_net(dev);
1411 	u8 alive, deleted;
1412 	unsigned index;
1413 
1414 	platform_label = rtnl_dereference(net->mpls.platform_label);
1415 	for (index = 0; index < net->mpls.platform_labels; index++) {
1416 		struct mpls_route *rt = rtnl_dereference(platform_label[index]);
1417 
1418 		if (!rt)
1419 			continue;
1420 
1421 		alive = 0;
1422 		deleted = 0;
1423 		change_nexthops(rt) {
1424 			unsigned int nh_flags = nh->nh_flags;
1425 
1426 			if (rtnl_dereference(nh->nh_dev) != dev)
1427 				goto next;
1428 
1429 			switch (event) {
1430 			case NETDEV_DOWN:
1431 			case NETDEV_UNREGISTER:
1432 				nh_flags |= RTNH_F_DEAD;
1433 				/* fall through */
1434 			case NETDEV_CHANGE:
1435 				nh_flags |= RTNH_F_LINKDOWN;
1436 				break;
1437 			}
1438 			if (event == NETDEV_UNREGISTER)
1439 				RCU_INIT_POINTER(nh->nh_dev, NULL);
1440 
1441 			if (nh->nh_flags != nh_flags)
1442 				WRITE_ONCE(nh->nh_flags, nh_flags);
1443 next:
1444 			if (!(nh_flags & (RTNH_F_DEAD | RTNH_F_LINKDOWN)))
1445 				alive++;
1446 			if (!rtnl_dereference(nh->nh_dev))
1447 				deleted++;
1448 		} endfor_nexthops(rt);
1449 
1450 		WRITE_ONCE(rt->rt_nhn_alive, alive);
1451 
1452 		/* if there are no more nexthops, delete the route */
1453 		if (event == NETDEV_UNREGISTER && deleted == rt->rt_nhn)
1454 			mpls_route_update(net, index, NULL, NULL);
1455 	}
1456 }
1457 
1458 static void mpls_ifup(struct net_device *dev, unsigned int flags)
1459 {
1460 	struct mpls_route __rcu **platform_label;
1461 	struct net *net = dev_net(dev);
1462 	unsigned index;
1463 	u8 alive;
1464 
1465 	platform_label = rtnl_dereference(net->mpls.platform_label);
1466 	for (index = 0; index < net->mpls.platform_labels; index++) {
1467 		struct mpls_route *rt = rtnl_dereference(platform_label[index]);
1468 
1469 		if (!rt)
1470 			continue;
1471 
1472 		alive = 0;
1473 		change_nexthops(rt) {
1474 			unsigned int nh_flags = nh->nh_flags;
1475 			struct net_device *nh_dev =
1476 				rtnl_dereference(nh->nh_dev);
1477 
1478 			if (!(nh_flags & flags)) {
1479 				alive++;
1480 				continue;
1481 			}
1482 			if (nh_dev != dev)
1483 				continue;
1484 			alive++;
1485 			nh_flags &= ~flags;
1486 			WRITE_ONCE(nh->nh_flags, nh_flags);
1487 		} endfor_nexthops(rt);
1488 
1489 		WRITE_ONCE(rt->rt_nhn_alive, alive);
1490 	}
1491 }
1492 
1493 static int mpls_dev_notify(struct notifier_block *this, unsigned long event,
1494 			   void *ptr)
1495 {
1496 	struct net_device *dev = netdev_notifier_info_to_dev(ptr);
1497 	struct mpls_dev *mdev;
1498 	unsigned int flags;
1499 
1500 	if (event == NETDEV_REGISTER) {
1501 		/* For now just support Ethernet, IPGRE, SIT and IPIP devices */
1502 		if (dev->type == ARPHRD_ETHER ||
1503 		    dev->type == ARPHRD_LOOPBACK ||
1504 		    dev->type == ARPHRD_IPGRE ||
1505 		    dev->type == ARPHRD_SIT ||
1506 		    dev->type == ARPHRD_TUNNEL) {
1507 			mdev = mpls_add_dev(dev);
1508 			if (IS_ERR(mdev))
1509 				return notifier_from_errno(PTR_ERR(mdev));
1510 		}
1511 		return NOTIFY_OK;
1512 	}
1513 
1514 	mdev = mpls_dev_get(dev);
1515 	if (!mdev)
1516 		return NOTIFY_OK;
1517 
1518 	switch (event) {
1519 	case NETDEV_DOWN:
1520 		mpls_ifdown(dev, event);
1521 		break;
1522 	case NETDEV_UP:
1523 		flags = dev_get_flags(dev);
1524 		if (flags & (IFF_RUNNING | IFF_LOWER_UP))
1525 			mpls_ifup(dev, RTNH_F_DEAD | RTNH_F_LINKDOWN);
1526 		else
1527 			mpls_ifup(dev, RTNH_F_DEAD);
1528 		break;
1529 	case NETDEV_CHANGE:
1530 		flags = dev_get_flags(dev);
1531 		if (flags & (IFF_RUNNING | IFF_LOWER_UP))
1532 			mpls_ifup(dev, RTNH_F_DEAD | RTNH_F_LINKDOWN);
1533 		else
1534 			mpls_ifdown(dev, event);
1535 		break;
1536 	case NETDEV_UNREGISTER:
1537 		mpls_ifdown(dev, event);
1538 		mdev = mpls_dev_get(dev);
1539 		if (mdev) {
1540 			mpls_dev_sysctl_unregister(dev, mdev);
1541 			RCU_INIT_POINTER(dev->mpls_ptr, NULL);
1542 			call_rcu(&mdev->rcu, mpls_dev_destroy_rcu);
1543 		}
1544 		break;
1545 	case NETDEV_CHANGENAME:
1546 		mdev = mpls_dev_get(dev);
1547 		if (mdev) {
1548 			int err;
1549 
1550 			mpls_dev_sysctl_unregister(dev, mdev);
1551 			err = mpls_dev_sysctl_register(dev, mdev);
1552 			if (err)
1553 				return notifier_from_errno(err);
1554 		}
1555 		break;
1556 	}
1557 	return NOTIFY_OK;
1558 }
1559 
1560 static struct notifier_block mpls_dev_notifier = {
1561 	.notifier_call = mpls_dev_notify,
1562 };
1563 
1564 static int nla_put_via(struct sk_buff *skb,
1565 		       u8 table, const void *addr, int alen)
1566 {
1567 	static const int table_to_family[NEIGH_NR_TABLES + 1] = {
1568 		AF_INET, AF_INET6, AF_DECnet, AF_PACKET,
1569 	};
1570 	struct nlattr *nla;
1571 	struct rtvia *via;
1572 	int family = AF_UNSPEC;
1573 
1574 	nla = nla_reserve(skb, RTA_VIA, alen + 2);
1575 	if (!nla)
1576 		return -EMSGSIZE;
1577 
1578 	if (table <= NEIGH_NR_TABLES)
1579 		family = table_to_family[table];
1580 
1581 	via = nla_data(nla);
1582 	via->rtvia_family = family;
1583 	memcpy(via->rtvia_addr, addr, alen);
1584 	return 0;
1585 }
1586 
1587 int nla_put_labels(struct sk_buff *skb, int attrtype,
1588 		   u8 labels, const u32 label[])
1589 {
1590 	struct nlattr *nla;
1591 	struct mpls_shim_hdr *nla_label;
1592 	bool bos;
1593 	int i;
1594 	nla = nla_reserve(skb, attrtype, labels*4);
1595 	if (!nla)
1596 		return -EMSGSIZE;
1597 
1598 	nla_label = nla_data(nla);
1599 	bos = true;
1600 	for (i = labels - 1; i >= 0; i--) {
1601 		nla_label[i] = mpls_entry_encode(label[i], 0, 0, bos);
1602 		bos = false;
1603 	}
1604 
1605 	return 0;
1606 }
1607 EXPORT_SYMBOL_GPL(nla_put_labels);
1608 
1609 int nla_get_labels(const struct nlattr *nla, u8 max_labels, u8 *labels,
1610 		   u32 label[], struct netlink_ext_ack *extack)
1611 {
1612 	unsigned len = nla_len(nla);
1613 	struct mpls_shim_hdr *nla_label;
1614 	u8 nla_labels;
1615 	bool bos;
1616 	int i;
1617 
1618 	/* len needs to be an even multiple of 4 (the label size). Number
1619 	 * of labels is a u8 so check for overflow.
1620 	 */
1621 	if (len & 3 || len / 4 > 255) {
1622 		NL_SET_ERR_MSG_ATTR(extack, nla,
1623 				    "Invalid length for labels attribute");
1624 		return -EINVAL;
1625 	}
1626 
1627 	/* Limit the number of new labels allowed */
1628 	nla_labels = len/4;
1629 	if (nla_labels > max_labels) {
1630 		NL_SET_ERR_MSG(extack, "Too many labels");
1631 		return -EINVAL;
1632 	}
1633 
1634 	/* when label == NULL, caller wants number of labels */
1635 	if (!label)
1636 		goto out;
1637 
1638 	nla_label = nla_data(nla);
1639 	bos = true;
1640 	for (i = nla_labels - 1; i >= 0; i--, bos = false) {
1641 		struct mpls_entry_decoded dec;
1642 		dec = mpls_entry_decode(nla_label + i);
1643 
1644 		/* Ensure the bottom of stack flag is properly set
1645 		 * and ttl and tc are both clear.
1646 		 */
1647 		if (dec.ttl) {
1648 			NL_SET_ERR_MSG_ATTR(extack, nla,
1649 					    "TTL in label must be 0");
1650 			return -EINVAL;
1651 		}
1652 
1653 		if (dec.tc) {
1654 			NL_SET_ERR_MSG_ATTR(extack, nla,
1655 					    "Traffic class in label must be 0");
1656 			return -EINVAL;
1657 		}
1658 
1659 		if (dec.bos != bos) {
1660 			NL_SET_BAD_ATTR(extack, nla);
1661 			if (bos) {
1662 				NL_SET_ERR_MSG(extack,
1663 					       "BOS bit must be set in first label");
1664 			} else {
1665 				NL_SET_ERR_MSG(extack,
1666 					       "BOS bit can only be set in first label");
1667 			}
1668 			return -EINVAL;
1669 		}
1670 
1671 		switch (dec.label) {
1672 		case MPLS_LABEL_IMPLNULL:
1673 			/* RFC3032: This is a label that an LSR may
1674 			 * assign and distribute, but which never
1675 			 * actually appears in the encapsulation.
1676 			 */
1677 			NL_SET_ERR_MSG_ATTR(extack, nla,
1678 					    "Implicit NULL Label (3) can not be used in encapsulation");
1679 			return -EINVAL;
1680 		}
1681 
1682 		label[i] = dec.label;
1683 	}
1684 out:
1685 	*labels = nla_labels;
1686 	return 0;
1687 }
1688 EXPORT_SYMBOL_GPL(nla_get_labels);
1689 
1690 static int rtm_to_route_config(struct sk_buff *skb,
1691 			       struct nlmsghdr *nlh,
1692 			       struct mpls_route_config *cfg,
1693 			       struct netlink_ext_ack *extack)
1694 {
1695 	struct rtmsg *rtm;
1696 	struct nlattr *tb[RTA_MAX+1];
1697 	int index;
1698 	int err;
1699 
1700 	err = nlmsg_parse(nlh, sizeof(*rtm), tb, RTA_MAX, rtm_mpls_policy,
1701 			  extack);
1702 	if (err < 0)
1703 		goto errout;
1704 
1705 	err = -EINVAL;
1706 	rtm = nlmsg_data(nlh);
1707 
1708 	if (rtm->rtm_family != AF_MPLS) {
1709 		NL_SET_ERR_MSG(extack, "Invalid address family in rtmsg");
1710 		goto errout;
1711 	}
1712 	if (rtm->rtm_dst_len != 20) {
1713 		NL_SET_ERR_MSG(extack, "rtm_dst_len must be 20 for MPLS");
1714 		goto errout;
1715 	}
1716 	if (rtm->rtm_src_len != 0) {
1717 		NL_SET_ERR_MSG(extack, "rtm_src_len must be 0 for MPLS");
1718 		goto errout;
1719 	}
1720 	if (rtm->rtm_tos != 0) {
1721 		NL_SET_ERR_MSG(extack, "rtm_tos must be 0 for MPLS");
1722 		goto errout;
1723 	}
1724 	if (rtm->rtm_table != RT_TABLE_MAIN) {
1725 		NL_SET_ERR_MSG(extack,
1726 			       "MPLS only supports the main route table");
1727 		goto errout;
1728 	}
1729 	/* Any value is acceptable for rtm_protocol */
1730 
1731 	/* As mpls uses destination specific addresses
1732 	 * (or source specific address in the case of multicast)
1733 	 * all addresses have universal scope.
1734 	 */
1735 	if (rtm->rtm_scope != RT_SCOPE_UNIVERSE) {
1736 		NL_SET_ERR_MSG(extack,
1737 			       "Invalid route scope  - MPLS only supports UNIVERSE");
1738 		goto errout;
1739 	}
1740 	if (rtm->rtm_type != RTN_UNICAST) {
1741 		NL_SET_ERR_MSG(extack,
1742 			       "Invalid route type - MPLS only supports UNICAST");
1743 		goto errout;
1744 	}
1745 	if (rtm->rtm_flags != 0) {
1746 		NL_SET_ERR_MSG(extack, "rtm_flags must be 0 for MPLS");
1747 		goto errout;
1748 	}
1749 
1750 	cfg->rc_label		= LABEL_NOT_SPECIFIED;
1751 	cfg->rc_protocol	= rtm->rtm_protocol;
1752 	cfg->rc_via_table	= MPLS_NEIGH_TABLE_UNSPEC;
1753 	cfg->rc_ttl_propagate	= MPLS_TTL_PROP_DEFAULT;
1754 	cfg->rc_nlflags		= nlh->nlmsg_flags;
1755 	cfg->rc_nlinfo.portid	= NETLINK_CB(skb).portid;
1756 	cfg->rc_nlinfo.nlh	= nlh;
1757 	cfg->rc_nlinfo.nl_net	= sock_net(skb->sk);
1758 
1759 	for (index = 0; index <= RTA_MAX; index++) {
1760 		struct nlattr *nla = tb[index];
1761 		if (!nla)
1762 			continue;
1763 
1764 		switch (index) {
1765 		case RTA_OIF:
1766 			cfg->rc_ifindex = nla_get_u32(nla);
1767 			break;
1768 		case RTA_NEWDST:
1769 			if (nla_get_labels(nla, MAX_NEW_LABELS,
1770 					   &cfg->rc_output_labels,
1771 					   cfg->rc_output_label, extack))
1772 				goto errout;
1773 			break;
1774 		case RTA_DST:
1775 		{
1776 			u8 label_count;
1777 			if (nla_get_labels(nla, 1, &label_count,
1778 					   &cfg->rc_label, extack))
1779 				goto errout;
1780 
1781 			if (!mpls_label_ok(cfg->rc_nlinfo.nl_net,
1782 					   cfg->rc_label, extack))
1783 				goto errout;
1784 			break;
1785 		}
1786 		case RTA_VIA:
1787 		{
1788 			if (nla_get_via(nla, &cfg->rc_via_alen,
1789 					&cfg->rc_via_table, cfg->rc_via,
1790 					extack))
1791 				goto errout;
1792 			break;
1793 		}
1794 		case RTA_MULTIPATH:
1795 		{
1796 			cfg->rc_mp = nla_data(nla);
1797 			cfg->rc_mp_len = nla_len(nla);
1798 			break;
1799 		}
1800 		case RTA_TTL_PROPAGATE:
1801 		{
1802 			u8 ttl_propagate = nla_get_u8(nla);
1803 
1804 			if (ttl_propagate > 1) {
1805 				NL_SET_ERR_MSG_ATTR(extack, nla,
1806 						    "RTA_TTL_PROPAGATE can only be 0 or 1");
1807 				goto errout;
1808 			}
1809 			cfg->rc_ttl_propagate = ttl_propagate ?
1810 				MPLS_TTL_PROP_ENABLED :
1811 				MPLS_TTL_PROP_DISABLED;
1812 			break;
1813 		}
1814 		default:
1815 			NL_SET_ERR_MSG_ATTR(extack, nla, "Unknown attribute");
1816 			/* Unsupported attribute */
1817 			goto errout;
1818 		}
1819 	}
1820 
1821 	err = 0;
1822 errout:
1823 	return err;
1824 }
1825 
1826 static int mpls_rtm_delroute(struct sk_buff *skb, struct nlmsghdr *nlh,
1827 			     struct netlink_ext_ack *extack)
1828 {
1829 	struct mpls_route_config *cfg;
1830 	int err;
1831 
1832 	cfg = kzalloc(sizeof(*cfg), GFP_KERNEL);
1833 	if (!cfg)
1834 		return -ENOMEM;
1835 
1836 	err = rtm_to_route_config(skb, nlh, cfg, extack);
1837 	if (err < 0)
1838 		goto out;
1839 
1840 	err = mpls_route_del(cfg, extack);
1841 out:
1842 	kfree(cfg);
1843 
1844 	return err;
1845 }
1846 
1847 
1848 static int mpls_rtm_newroute(struct sk_buff *skb, struct nlmsghdr *nlh,
1849 			     struct netlink_ext_ack *extack)
1850 {
1851 	struct mpls_route_config *cfg;
1852 	int err;
1853 
1854 	cfg = kzalloc(sizeof(*cfg), GFP_KERNEL);
1855 	if (!cfg)
1856 		return -ENOMEM;
1857 
1858 	err = rtm_to_route_config(skb, nlh, cfg, extack);
1859 	if (err < 0)
1860 		goto out;
1861 
1862 	err = mpls_route_add(cfg, extack);
1863 out:
1864 	kfree(cfg);
1865 
1866 	return err;
1867 }
1868 
1869 static int mpls_dump_route(struct sk_buff *skb, u32 portid, u32 seq, int event,
1870 			   u32 label, struct mpls_route *rt, int flags)
1871 {
1872 	struct net_device *dev;
1873 	struct nlmsghdr *nlh;
1874 	struct rtmsg *rtm;
1875 
1876 	nlh = nlmsg_put(skb, portid, seq, event, sizeof(*rtm), flags);
1877 	if (nlh == NULL)
1878 		return -EMSGSIZE;
1879 
1880 	rtm = nlmsg_data(nlh);
1881 	rtm->rtm_family = AF_MPLS;
1882 	rtm->rtm_dst_len = 20;
1883 	rtm->rtm_src_len = 0;
1884 	rtm->rtm_tos = 0;
1885 	rtm->rtm_table = RT_TABLE_MAIN;
1886 	rtm->rtm_protocol = rt->rt_protocol;
1887 	rtm->rtm_scope = RT_SCOPE_UNIVERSE;
1888 	rtm->rtm_type = RTN_UNICAST;
1889 	rtm->rtm_flags = 0;
1890 
1891 	if (nla_put_labels(skb, RTA_DST, 1, &label))
1892 		goto nla_put_failure;
1893 
1894 	if (rt->rt_ttl_propagate != MPLS_TTL_PROP_DEFAULT) {
1895 		bool ttl_propagate =
1896 			rt->rt_ttl_propagate == MPLS_TTL_PROP_ENABLED;
1897 
1898 		if (nla_put_u8(skb, RTA_TTL_PROPAGATE,
1899 			       ttl_propagate))
1900 			goto nla_put_failure;
1901 	}
1902 	if (rt->rt_nhn == 1) {
1903 		const struct mpls_nh *nh = rt->rt_nh;
1904 
1905 		if (nh->nh_labels &&
1906 		    nla_put_labels(skb, RTA_NEWDST, nh->nh_labels,
1907 				   nh->nh_label))
1908 			goto nla_put_failure;
1909 		if (nh->nh_via_table != MPLS_NEIGH_TABLE_UNSPEC &&
1910 		    nla_put_via(skb, nh->nh_via_table, mpls_nh_via(rt, nh),
1911 				nh->nh_via_alen))
1912 			goto nla_put_failure;
1913 		dev = rtnl_dereference(nh->nh_dev);
1914 		if (dev && nla_put_u32(skb, RTA_OIF, dev->ifindex))
1915 			goto nla_put_failure;
1916 		if (nh->nh_flags & RTNH_F_LINKDOWN)
1917 			rtm->rtm_flags |= RTNH_F_LINKDOWN;
1918 		if (nh->nh_flags & RTNH_F_DEAD)
1919 			rtm->rtm_flags |= RTNH_F_DEAD;
1920 	} else {
1921 		struct rtnexthop *rtnh;
1922 		struct nlattr *mp;
1923 		u8 linkdown = 0;
1924 		u8 dead = 0;
1925 
1926 		mp = nla_nest_start(skb, RTA_MULTIPATH);
1927 		if (!mp)
1928 			goto nla_put_failure;
1929 
1930 		for_nexthops(rt) {
1931 			dev = rtnl_dereference(nh->nh_dev);
1932 			if (!dev)
1933 				continue;
1934 
1935 			rtnh = nla_reserve_nohdr(skb, sizeof(*rtnh));
1936 			if (!rtnh)
1937 				goto nla_put_failure;
1938 
1939 			rtnh->rtnh_ifindex = dev->ifindex;
1940 			if (nh->nh_flags & RTNH_F_LINKDOWN) {
1941 				rtnh->rtnh_flags |= RTNH_F_LINKDOWN;
1942 				linkdown++;
1943 			}
1944 			if (nh->nh_flags & RTNH_F_DEAD) {
1945 				rtnh->rtnh_flags |= RTNH_F_DEAD;
1946 				dead++;
1947 			}
1948 
1949 			if (nh->nh_labels && nla_put_labels(skb, RTA_NEWDST,
1950 							    nh->nh_labels,
1951 							    nh->nh_label))
1952 				goto nla_put_failure;
1953 			if (nh->nh_via_table != MPLS_NEIGH_TABLE_UNSPEC &&
1954 			    nla_put_via(skb, nh->nh_via_table,
1955 					mpls_nh_via(rt, nh),
1956 					nh->nh_via_alen))
1957 				goto nla_put_failure;
1958 
1959 			/* length of rtnetlink header + attributes */
1960 			rtnh->rtnh_len = nlmsg_get_pos(skb) - (void *)rtnh;
1961 		} endfor_nexthops(rt);
1962 
1963 		if (linkdown == rt->rt_nhn)
1964 			rtm->rtm_flags |= RTNH_F_LINKDOWN;
1965 		if (dead == rt->rt_nhn)
1966 			rtm->rtm_flags |= RTNH_F_DEAD;
1967 
1968 		nla_nest_end(skb, mp);
1969 	}
1970 
1971 	nlmsg_end(skb, nlh);
1972 	return 0;
1973 
1974 nla_put_failure:
1975 	nlmsg_cancel(skb, nlh);
1976 	return -EMSGSIZE;
1977 }
1978 
1979 static int mpls_dump_routes(struct sk_buff *skb, struct netlink_callback *cb)
1980 {
1981 	struct net *net = sock_net(skb->sk);
1982 	struct mpls_route __rcu **platform_label;
1983 	size_t platform_labels;
1984 	unsigned int index;
1985 
1986 	ASSERT_RTNL();
1987 
1988 	index = cb->args[0];
1989 	if (index < MPLS_LABEL_FIRST_UNRESERVED)
1990 		index = MPLS_LABEL_FIRST_UNRESERVED;
1991 
1992 	platform_label = rtnl_dereference(net->mpls.platform_label);
1993 	platform_labels = net->mpls.platform_labels;
1994 	for (; index < platform_labels; index++) {
1995 		struct mpls_route *rt;
1996 		rt = rtnl_dereference(platform_label[index]);
1997 		if (!rt)
1998 			continue;
1999 
2000 		if (mpls_dump_route(skb, NETLINK_CB(cb->skb).portid,
2001 				    cb->nlh->nlmsg_seq, RTM_NEWROUTE,
2002 				    index, rt, NLM_F_MULTI) < 0)
2003 			break;
2004 	}
2005 	cb->args[0] = index;
2006 
2007 	return skb->len;
2008 }
2009 
2010 static inline size_t lfib_nlmsg_size(struct mpls_route *rt)
2011 {
2012 	size_t payload =
2013 		NLMSG_ALIGN(sizeof(struct rtmsg))
2014 		+ nla_total_size(4)			/* RTA_DST */
2015 		+ nla_total_size(1);			/* RTA_TTL_PROPAGATE */
2016 
2017 	if (rt->rt_nhn == 1) {
2018 		struct mpls_nh *nh = rt->rt_nh;
2019 
2020 		if (nh->nh_dev)
2021 			payload += nla_total_size(4); /* RTA_OIF */
2022 		if (nh->nh_via_table != MPLS_NEIGH_TABLE_UNSPEC) /* RTA_VIA */
2023 			payload += nla_total_size(2 + nh->nh_via_alen);
2024 		if (nh->nh_labels) /* RTA_NEWDST */
2025 			payload += nla_total_size(nh->nh_labels * 4);
2026 	} else {
2027 		/* each nexthop is packed in an attribute */
2028 		size_t nhsize = 0;
2029 
2030 		for_nexthops(rt) {
2031 			if (!rtnl_dereference(nh->nh_dev))
2032 				continue;
2033 			nhsize += nla_total_size(sizeof(struct rtnexthop));
2034 			/* RTA_VIA */
2035 			if (nh->nh_via_table != MPLS_NEIGH_TABLE_UNSPEC)
2036 				nhsize += nla_total_size(2 + nh->nh_via_alen);
2037 			if (nh->nh_labels)
2038 				nhsize += nla_total_size(nh->nh_labels * 4);
2039 		} endfor_nexthops(rt);
2040 		/* nested attribute */
2041 		payload += nla_total_size(nhsize);
2042 	}
2043 
2044 	return payload;
2045 }
2046 
2047 static void rtmsg_lfib(int event, u32 label, struct mpls_route *rt,
2048 		       struct nlmsghdr *nlh, struct net *net, u32 portid,
2049 		       unsigned int nlm_flags)
2050 {
2051 	struct sk_buff *skb;
2052 	u32 seq = nlh ? nlh->nlmsg_seq : 0;
2053 	int err = -ENOBUFS;
2054 
2055 	skb = nlmsg_new(lfib_nlmsg_size(rt), GFP_KERNEL);
2056 	if (skb == NULL)
2057 		goto errout;
2058 
2059 	err = mpls_dump_route(skb, portid, seq, event, label, rt, nlm_flags);
2060 	if (err < 0) {
2061 		/* -EMSGSIZE implies BUG in lfib_nlmsg_size */
2062 		WARN_ON(err == -EMSGSIZE);
2063 		kfree_skb(skb);
2064 		goto errout;
2065 	}
2066 	rtnl_notify(skb, net, portid, RTNLGRP_MPLS_ROUTE, nlh, GFP_KERNEL);
2067 
2068 	return;
2069 errout:
2070 	if (err < 0)
2071 		rtnl_set_sk_err(net, RTNLGRP_MPLS_ROUTE, err);
2072 }
2073 
2074 static int mpls_getroute(struct sk_buff *in_skb, struct nlmsghdr *in_nlh,
2075 			 struct netlink_ext_ack *extack)
2076 {
2077 	struct net *net = sock_net(in_skb->sk);
2078 	u32 portid = NETLINK_CB(in_skb).portid;
2079 	u32 in_label = LABEL_NOT_SPECIFIED;
2080 	struct nlattr *tb[RTA_MAX + 1];
2081 	u32 labels[MAX_NEW_LABELS];
2082 	struct mpls_shim_hdr *hdr;
2083 	unsigned int hdr_size = 0;
2084 	struct net_device *dev;
2085 	struct mpls_route *rt;
2086 	struct rtmsg *rtm, *r;
2087 	struct nlmsghdr *nlh;
2088 	struct sk_buff *skb;
2089 	struct mpls_nh *nh;
2090 	u8 n_labels;
2091 	int err;
2092 
2093 	err = nlmsg_parse(in_nlh, sizeof(*rtm), tb, RTA_MAX,
2094 			  rtm_mpls_policy, extack);
2095 	if (err < 0)
2096 		goto errout;
2097 
2098 	rtm = nlmsg_data(in_nlh);
2099 
2100 	if (tb[RTA_DST]) {
2101 		u8 label_count;
2102 
2103 		if (nla_get_labels(tb[RTA_DST], 1, &label_count,
2104 				   &in_label, extack)) {
2105 			err = -EINVAL;
2106 			goto errout;
2107 		}
2108 
2109 		if (!mpls_label_ok(net, in_label, extack)) {
2110 			err = -EINVAL;
2111 			goto errout;
2112 		}
2113 	}
2114 
2115 	rt = mpls_route_input_rcu(net, in_label);
2116 	if (!rt) {
2117 		err = -ENETUNREACH;
2118 		goto errout;
2119 	}
2120 
2121 	if (rtm->rtm_flags & RTM_F_FIB_MATCH) {
2122 		skb = nlmsg_new(lfib_nlmsg_size(rt), GFP_KERNEL);
2123 		if (!skb) {
2124 			err = -ENOBUFS;
2125 			goto errout;
2126 		}
2127 
2128 		err = mpls_dump_route(skb, portid, in_nlh->nlmsg_seq,
2129 				      RTM_NEWROUTE, in_label, rt, 0);
2130 		if (err < 0) {
2131 			/* -EMSGSIZE implies BUG in lfib_nlmsg_size */
2132 			WARN_ON(err == -EMSGSIZE);
2133 			goto errout_free;
2134 		}
2135 
2136 		return rtnl_unicast(skb, net, portid);
2137 	}
2138 
2139 	if (tb[RTA_NEWDST]) {
2140 		if (nla_get_labels(tb[RTA_NEWDST], MAX_NEW_LABELS, &n_labels,
2141 				   labels, extack) != 0) {
2142 			err = -EINVAL;
2143 			goto errout;
2144 		}
2145 
2146 		hdr_size = n_labels * sizeof(struct mpls_shim_hdr);
2147 	}
2148 
2149 	skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
2150 	if (!skb) {
2151 		err = -ENOBUFS;
2152 		goto errout;
2153 	}
2154 
2155 	skb->protocol = htons(ETH_P_MPLS_UC);
2156 
2157 	if (hdr_size) {
2158 		bool bos;
2159 		int i;
2160 
2161 		if (skb_cow(skb, hdr_size)) {
2162 			err = -ENOBUFS;
2163 			goto errout_free;
2164 		}
2165 
2166 		skb_reserve(skb, hdr_size);
2167 		skb_push(skb, hdr_size);
2168 		skb_reset_network_header(skb);
2169 
2170 		/* Push new labels */
2171 		hdr = mpls_hdr(skb);
2172 		bos = true;
2173 		for (i = n_labels - 1; i >= 0; i--) {
2174 			hdr[i] = mpls_entry_encode(labels[i],
2175 						   1, 0, bos);
2176 			bos = false;
2177 		}
2178 	}
2179 
2180 	nh = mpls_select_multipath(rt, skb);
2181 	if (!nh) {
2182 		err = -ENETUNREACH;
2183 		goto errout_free;
2184 	}
2185 
2186 	if (hdr_size) {
2187 		skb_pull(skb, hdr_size);
2188 		skb_reset_network_header(skb);
2189 	}
2190 
2191 	nlh = nlmsg_put(skb, portid, in_nlh->nlmsg_seq,
2192 			RTM_NEWROUTE, sizeof(*r), 0);
2193 	if (!nlh) {
2194 		err = -EMSGSIZE;
2195 		goto errout_free;
2196 	}
2197 
2198 	r = nlmsg_data(nlh);
2199 	r->rtm_family	 = AF_MPLS;
2200 	r->rtm_dst_len	= 20;
2201 	r->rtm_src_len	= 0;
2202 	r->rtm_table	= RT_TABLE_MAIN;
2203 	r->rtm_type	= RTN_UNICAST;
2204 	r->rtm_scope	= RT_SCOPE_UNIVERSE;
2205 	r->rtm_protocol = rt->rt_protocol;
2206 	r->rtm_flags	= 0;
2207 
2208 	if (nla_put_labels(skb, RTA_DST, 1, &in_label))
2209 		goto nla_put_failure;
2210 
2211 	if (nh->nh_labels &&
2212 	    nla_put_labels(skb, RTA_NEWDST, nh->nh_labels,
2213 			   nh->nh_label))
2214 		goto nla_put_failure;
2215 
2216 	if (nh->nh_via_table != MPLS_NEIGH_TABLE_UNSPEC &&
2217 	    nla_put_via(skb, nh->nh_via_table, mpls_nh_via(rt, nh),
2218 			nh->nh_via_alen))
2219 		goto nla_put_failure;
2220 	dev = rtnl_dereference(nh->nh_dev);
2221 	if (dev && nla_put_u32(skb, RTA_OIF, dev->ifindex))
2222 		goto nla_put_failure;
2223 
2224 	nlmsg_end(skb, nlh);
2225 
2226 	err = rtnl_unicast(skb, net, portid);
2227 errout:
2228 	return err;
2229 
2230 nla_put_failure:
2231 	nlmsg_cancel(skb, nlh);
2232 	err = -EMSGSIZE;
2233 errout_free:
2234 	kfree_skb(skb);
2235 	return err;
2236 }
2237 
2238 static int resize_platform_label_table(struct net *net, size_t limit)
2239 {
2240 	size_t size = sizeof(struct mpls_route *) * limit;
2241 	size_t old_limit;
2242 	size_t cp_size;
2243 	struct mpls_route __rcu **labels = NULL, **old;
2244 	struct mpls_route *rt0 = NULL, *rt2 = NULL;
2245 	unsigned index;
2246 
2247 	if (size) {
2248 		labels = kvzalloc(size, GFP_KERNEL);
2249 		if (!labels)
2250 			goto nolabels;
2251 	}
2252 
2253 	/* In case the predefined labels need to be populated */
2254 	if (limit > MPLS_LABEL_IPV4NULL) {
2255 		struct net_device *lo = net->loopback_dev;
2256 		rt0 = mpls_rt_alloc(1, lo->addr_len, 0);
2257 		if (IS_ERR(rt0))
2258 			goto nort0;
2259 		RCU_INIT_POINTER(rt0->rt_nh->nh_dev, lo);
2260 		rt0->rt_protocol = RTPROT_KERNEL;
2261 		rt0->rt_payload_type = MPT_IPV4;
2262 		rt0->rt_ttl_propagate = MPLS_TTL_PROP_DEFAULT;
2263 		rt0->rt_nh->nh_via_table = NEIGH_LINK_TABLE;
2264 		rt0->rt_nh->nh_via_alen = lo->addr_len;
2265 		memcpy(__mpls_nh_via(rt0, rt0->rt_nh), lo->dev_addr,
2266 		       lo->addr_len);
2267 	}
2268 	if (limit > MPLS_LABEL_IPV6NULL) {
2269 		struct net_device *lo = net->loopback_dev;
2270 		rt2 = mpls_rt_alloc(1, lo->addr_len, 0);
2271 		if (IS_ERR(rt2))
2272 			goto nort2;
2273 		RCU_INIT_POINTER(rt2->rt_nh->nh_dev, lo);
2274 		rt2->rt_protocol = RTPROT_KERNEL;
2275 		rt2->rt_payload_type = MPT_IPV6;
2276 		rt2->rt_ttl_propagate = MPLS_TTL_PROP_DEFAULT;
2277 		rt2->rt_nh->nh_via_table = NEIGH_LINK_TABLE;
2278 		rt2->rt_nh->nh_via_alen = lo->addr_len;
2279 		memcpy(__mpls_nh_via(rt2, rt2->rt_nh), lo->dev_addr,
2280 		       lo->addr_len);
2281 	}
2282 
2283 	rtnl_lock();
2284 	/* Remember the original table */
2285 	old = rtnl_dereference(net->mpls.platform_label);
2286 	old_limit = net->mpls.platform_labels;
2287 
2288 	/* Free any labels beyond the new table */
2289 	for (index = limit; index < old_limit; index++)
2290 		mpls_route_update(net, index, NULL, NULL);
2291 
2292 	/* Copy over the old labels */
2293 	cp_size = size;
2294 	if (old_limit < limit)
2295 		cp_size = old_limit * sizeof(struct mpls_route *);
2296 
2297 	memcpy(labels, old, cp_size);
2298 
2299 	/* If needed set the predefined labels */
2300 	if ((old_limit <= MPLS_LABEL_IPV6NULL) &&
2301 	    (limit > MPLS_LABEL_IPV6NULL)) {
2302 		RCU_INIT_POINTER(labels[MPLS_LABEL_IPV6NULL], rt2);
2303 		rt2 = NULL;
2304 	}
2305 
2306 	if ((old_limit <= MPLS_LABEL_IPV4NULL) &&
2307 	    (limit > MPLS_LABEL_IPV4NULL)) {
2308 		RCU_INIT_POINTER(labels[MPLS_LABEL_IPV4NULL], rt0);
2309 		rt0 = NULL;
2310 	}
2311 
2312 	/* Update the global pointers */
2313 	net->mpls.platform_labels = limit;
2314 	rcu_assign_pointer(net->mpls.platform_label, labels);
2315 
2316 	rtnl_unlock();
2317 
2318 	mpls_rt_free(rt2);
2319 	mpls_rt_free(rt0);
2320 
2321 	if (old) {
2322 		synchronize_rcu();
2323 		kvfree(old);
2324 	}
2325 	return 0;
2326 
2327 nort2:
2328 	mpls_rt_free(rt0);
2329 nort0:
2330 	kvfree(labels);
2331 nolabels:
2332 	return -ENOMEM;
2333 }
2334 
2335 static int mpls_platform_labels(struct ctl_table *table, int write,
2336 				void __user *buffer, size_t *lenp, loff_t *ppos)
2337 {
2338 	struct net *net = table->data;
2339 	int platform_labels = net->mpls.platform_labels;
2340 	int ret;
2341 	struct ctl_table tmp = {
2342 		.procname	= table->procname,
2343 		.data		= &platform_labels,
2344 		.maxlen		= sizeof(int),
2345 		.mode		= table->mode,
2346 		.extra1		= &zero,
2347 		.extra2		= &label_limit,
2348 	};
2349 
2350 	ret = proc_dointvec_minmax(&tmp, write, buffer, lenp, ppos);
2351 
2352 	if (write && ret == 0)
2353 		ret = resize_platform_label_table(net, platform_labels);
2354 
2355 	return ret;
2356 }
2357 
2358 #define MPLS_NS_SYSCTL_OFFSET(field)		\
2359 	(&((struct net *)0)->field)
2360 
2361 static const struct ctl_table mpls_table[] = {
2362 	{
2363 		.procname	= "platform_labels",
2364 		.data		= NULL,
2365 		.maxlen		= sizeof(int),
2366 		.mode		= 0644,
2367 		.proc_handler	= mpls_platform_labels,
2368 	},
2369 	{
2370 		.procname	= "ip_ttl_propagate",
2371 		.data		= MPLS_NS_SYSCTL_OFFSET(mpls.ip_ttl_propagate),
2372 		.maxlen		= sizeof(int),
2373 		.mode		= 0644,
2374 		.proc_handler	= proc_dointvec_minmax,
2375 		.extra1		= &zero,
2376 		.extra2		= &one,
2377 	},
2378 	{
2379 		.procname	= "default_ttl",
2380 		.data		= MPLS_NS_SYSCTL_OFFSET(mpls.default_ttl),
2381 		.maxlen		= sizeof(int),
2382 		.mode		= 0644,
2383 		.proc_handler	= proc_dointvec_minmax,
2384 		.extra1		= &one,
2385 		.extra2		= &ttl_max,
2386 	},
2387 	{ }
2388 };
2389 
2390 static int mpls_net_init(struct net *net)
2391 {
2392 	struct ctl_table *table;
2393 	int i;
2394 
2395 	net->mpls.platform_labels = 0;
2396 	net->mpls.platform_label = NULL;
2397 	net->mpls.ip_ttl_propagate = 1;
2398 	net->mpls.default_ttl = 255;
2399 
2400 	table = kmemdup(mpls_table, sizeof(mpls_table), GFP_KERNEL);
2401 	if (table == NULL)
2402 		return -ENOMEM;
2403 
2404 	/* Table data contains only offsets relative to the base of
2405 	 * the mdev at this point, so make them absolute.
2406 	 */
2407 	for (i = 0; i < ARRAY_SIZE(mpls_table) - 1; i++)
2408 		table[i].data = (char *)net + (uintptr_t)table[i].data;
2409 
2410 	net->mpls.ctl = register_net_sysctl(net, "net/mpls", table);
2411 	if (net->mpls.ctl == NULL) {
2412 		kfree(table);
2413 		return -ENOMEM;
2414 	}
2415 
2416 	return 0;
2417 }
2418 
2419 static void mpls_net_exit(struct net *net)
2420 {
2421 	struct mpls_route __rcu **platform_label;
2422 	size_t platform_labels;
2423 	struct ctl_table *table;
2424 	unsigned int index;
2425 
2426 	table = net->mpls.ctl->ctl_table_arg;
2427 	unregister_net_sysctl_table(net->mpls.ctl);
2428 	kfree(table);
2429 
2430 	/* An rcu grace period has passed since there was a device in
2431 	 * the network namespace (and thus the last in flight packet)
2432 	 * left this network namespace.  This is because
2433 	 * unregister_netdevice_many and netdev_run_todo has completed
2434 	 * for each network device that was in this network namespace.
2435 	 *
2436 	 * As such no additional rcu synchronization is necessary when
2437 	 * freeing the platform_label table.
2438 	 */
2439 	rtnl_lock();
2440 	platform_label = rtnl_dereference(net->mpls.platform_label);
2441 	platform_labels = net->mpls.platform_labels;
2442 	for (index = 0; index < platform_labels; index++) {
2443 		struct mpls_route *rt = rtnl_dereference(platform_label[index]);
2444 		RCU_INIT_POINTER(platform_label[index], NULL);
2445 		mpls_notify_route(net, index, rt, NULL, NULL);
2446 		mpls_rt_free(rt);
2447 	}
2448 	rtnl_unlock();
2449 
2450 	kvfree(platform_label);
2451 }
2452 
2453 static struct pernet_operations mpls_net_ops = {
2454 	.init = mpls_net_init,
2455 	.exit = mpls_net_exit,
2456 };
2457 
2458 static struct rtnl_af_ops mpls_af_ops __read_mostly = {
2459 	.family		   = AF_MPLS,
2460 	.fill_stats_af	   = mpls_fill_stats_af,
2461 	.get_stats_af_size = mpls_get_stats_af_size,
2462 };
2463 
2464 static int __init mpls_init(void)
2465 {
2466 	int err;
2467 
2468 	BUILD_BUG_ON(sizeof(struct mpls_shim_hdr) != 4);
2469 
2470 	err = register_pernet_subsys(&mpls_net_ops);
2471 	if (err)
2472 		goto out;
2473 
2474 	err = register_netdevice_notifier(&mpls_dev_notifier);
2475 	if (err)
2476 		goto out_unregister_pernet;
2477 
2478 	dev_add_pack(&mpls_packet_type);
2479 
2480 	rtnl_af_register(&mpls_af_ops);
2481 
2482 	rtnl_register(PF_MPLS, RTM_NEWROUTE, mpls_rtm_newroute, NULL, NULL);
2483 	rtnl_register(PF_MPLS, RTM_DELROUTE, mpls_rtm_delroute, NULL, NULL);
2484 	rtnl_register(PF_MPLS, RTM_GETROUTE, mpls_getroute, mpls_dump_routes,
2485 		      NULL);
2486 	rtnl_register(PF_MPLS, RTM_GETNETCONF, mpls_netconf_get_devconf,
2487 		      mpls_netconf_dump_devconf, NULL);
2488 	err = 0;
2489 out:
2490 	return err;
2491 
2492 out_unregister_pernet:
2493 	unregister_pernet_subsys(&mpls_net_ops);
2494 	goto out;
2495 }
2496 module_init(mpls_init);
2497 
2498 static void __exit mpls_exit(void)
2499 {
2500 	rtnl_unregister_all(PF_MPLS);
2501 	rtnl_af_unregister(&mpls_af_ops);
2502 	dev_remove_pack(&mpls_packet_type);
2503 	unregister_netdevice_notifier(&mpls_dev_notifier);
2504 	unregister_pernet_subsys(&mpls_net_ops);
2505 }
2506 module_exit(mpls_exit);
2507 
2508 MODULE_DESCRIPTION("MultiProtocol Label Switching");
2509 MODULE_LICENSE("GPL v2");
2510 MODULE_ALIAS_NETPROTO(PF_MPLS);
2511