xref: /openbmc/linux/net/ipv4/fib_frontend.c (revision 4da722ca)
1 /*
2  * INET		An implementation of the TCP/IP protocol suite for the LINUX
3  *		operating system.  INET is implemented using the  BSD Socket
4  *		interface as the means of communication with the user level.
5  *
6  *		IPv4 Forwarding Information Base: FIB frontend.
7  *
8  * Authors:	Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
9  *
10  *		This program is free software; you can redistribute it and/or
11  *		modify it under the terms of the GNU General Public License
12  *		as published by the Free Software Foundation; either version
13  *		2 of the License, or (at your option) any later version.
14  */
15 
16 #include <linux/module.h>
17 #include <linux/uaccess.h>
18 #include <linux/bitops.h>
19 #include <linux/capability.h>
20 #include <linux/types.h>
21 #include <linux/kernel.h>
22 #include <linux/mm.h>
23 #include <linux/string.h>
24 #include <linux/socket.h>
25 #include <linux/sockios.h>
26 #include <linux/errno.h>
27 #include <linux/in.h>
28 #include <linux/inet.h>
29 #include <linux/inetdevice.h>
30 #include <linux/netdevice.h>
31 #include <linux/if_addr.h>
32 #include <linux/if_arp.h>
33 #include <linux/skbuff.h>
34 #include <linux/cache.h>
35 #include <linux/init.h>
36 #include <linux/list.h>
37 #include <linux/slab.h>
38 
39 #include <net/ip.h>
40 #include <net/protocol.h>
41 #include <net/route.h>
42 #include <net/tcp.h>
43 #include <net/sock.h>
44 #include <net/arp.h>
45 #include <net/ip_fib.h>
46 #include <net/rtnetlink.h>
47 #include <net/xfrm.h>
48 #include <net/l3mdev.h>
49 #include <net/lwtunnel.h>
50 #include <trace/events/fib.h>
51 
52 #ifndef CONFIG_IP_MULTIPLE_TABLES
53 
54 static int __net_init fib4_rules_init(struct net *net)
55 {
56 	struct fib_table *local_table, *main_table;
57 
58 	main_table  = fib_trie_table(RT_TABLE_MAIN, NULL);
59 	if (!main_table)
60 		return -ENOMEM;
61 
62 	local_table = fib_trie_table(RT_TABLE_LOCAL, main_table);
63 	if (!local_table)
64 		goto fail;
65 
66 	hlist_add_head_rcu(&local_table->tb_hlist,
67 				&net->ipv4.fib_table_hash[TABLE_LOCAL_INDEX]);
68 	hlist_add_head_rcu(&main_table->tb_hlist,
69 				&net->ipv4.fib_table_hash[TABLE_MAIN_INDEX]);
70 	return 0;
71 
72 fail:
73 	fib_free_table(main_table);
74 	return -ENOMEM;
75 }
76 #else
77 
78 struct fib_table *fib_new_table(struct net *net, u32 id)
79 {
80 	struct fib_table *tb, *alias = NULL;
81 	unsigned int h;
82 
83 	if (id == 0)
84 		id = RT_TABLE_MAIN;
85 	tb = fib_get_table(net, id);
86 	if (tb)
87 		return tb;
88 
89 	if (id == RT_TABLE_LOCAL && !net->ipv4.fib_has_custom_rules)
90 		alias = fib_new_table(net, RT_TABLE_MAIN);
91 
92 	tb = fib_trie_table(id, alias);
93 	if (!tb)
94 		return NULL;
95 
96 	switch (id) {
97 	case RT_TABLE_MAIN:
98 		rcu_assign_pointer(net->ipv4.fib_main, tb);
99 		break;
100 	case RT_TABLE_DEFAULT:
101 		rcu_assign_pointer(net->ipv4.fib_default, tb);
102 		break;
103 	default:
104 		break;
105 	}
106 
107 	h = id & (FIB_TABLE_HASHSZ - 1);
108 	hlist_add_head_rcu(&tb->tb_hlist, &net->ipv4.fib_table_hash[h]);
109 	return tb;
110 }
111 EXPORT_SYMBOL_GPL(fib_new_table);
112 
113 /* caller must hold either rtnl or rcu read lock */
114 struct fib_table *fib_get_table(struct net *net, u32 id)
115 {
116 	struct fib_table *tb;
117 	struct hlist_head *head;
118 	unsigned int h;
119 
120 	if (id == 0)
121 		id = RT_TABLE_MAIN;
122 	h = id & (FIB_TABLE_HASHSZ - 1);
123 
124 	head = &net->ipv4.fib_table_hash[h];
125 	hlist_for_each_entry_rcu(tb, head, tb_hlist) {
126 		if (tb->tb_id == id)
127 			return tb;
128 	}
129 	return NULL;
130 }
131 #endif /* CONFIG_IP_MULTIPLE_TABLES */
132 
133 static void fib_replace_table(struct net *net, struct fib_table *old,
134 			      struct fib_table *new)
135 {
136 #ifdef CONFIG_IP_MULTIPLE_TABLES
137 	switch (new->tb_id) {
138 	case RT_TABLE_MAIN:
139 		rcu_assign_pointer(net->ipv4.fib_main, new);
140 		break;
141 	case RT_TABLE_DEFAULT:
142 		rcu_assign_pointer(net->ipv4.fib_default, new);
143 		break;
144 	default:
145 		break;
146 	}
147 
148 #endif
149 	/* replace the old table in the hlist */
150 	hlist_replace_rcu(&old->tb_hlist, &new->tb_hlist);
151 }
152 
153 int fib_unmerge(struct net *net)
154 {
155 	struct fib_table *old, *new, *main_table;
156 
157 	/* attempt to fetch local table if it has been allocated */
158 	old = fib_get_table(net, RT_TABLE_LOCAL);
159 	if (!old)
160 		return 0;
161 
162 	new = fib_trie_unmerge(old);
163 	if (!new)
164 		return -ENOMEM;
165 
166 	/* table is already unmerged */
167 	if (new == old)
168 		return 0;
169 
170 	/* replace merged table with clean table */
171 	fib_replace_table(net, old, new);
172 	fib_free_table(old);
173 
174 	/* attempt to fetch main table if it has been allocated */
175 	main_table = fib_get_table(net, RT_TABLE_MAIN);
176 	if (!main_table)
177 		return 0;
178 
179 	/* flush local entries from main table */
180 	fib_table_flush_external(main_table);
181 
182 	return 0;
183 }
184 
185 static void fib_flush(struct net *net)
186 {
187 	int flushed = 0;
188 	unsigned int h;
189 
190 	for (h = 0; h < FIB_TABLE_HASHSZ; h++) {
191 		struct hlist_head *head = &net->ipv4.fib_table_hash[h];
192 		struct hlist_node *tmp;
193 		struct fib_table *tb;
194 
195 		hlist_for_each_entry_safe(tb, tmp, head, tb_hlist)
196 			flushed += fib_table_flush(net, tb);
197 	}
198 
199 	if (flushed)
200 		rt_cache_flush(net);
201 }
202 
203 /*
204  * Find address type as if only "dev" was present in the system. If
205  * on_dev is NULL then all interfaces are taken into consideration.
206  */
207 static inline unsigned int __inet_dev_addr_type(struct net *net,
208 						const struct net_device *dev,
209 						__be32 addr, u32 tb_id)
210 {
211 	struct flowi4		fl4 = { .daddr = addr };
212 	struct fib_result	res;
213 	unsigned int ret = RTN_BROADCAST;
214 	struct fib_table *table;
215 
216 	if (ipv4_is_zeronet(addr) || ipv4_is_lbcast(addr))
217 		return RTN_BROADCAST;
218 	if (ipv4_is_multicast(addr))
219 		return RTN_MULTICAST;
220 
221 	rcu_read_lock();
222 
223 	table = fib_get_table(net, tb_id);
224 	if (table) {
225 		ret = RTN_UNICAST;
226 		if (!fib_table_lookup(table, &fl4, &res, FIB_LOOKUP_NOREF)) {
227 			if (!dev || dev == res.fi->fib_dev)
228 				ret = res.type;
229 		}
230 	}
231 
232 	rcu_read_unlock();
233 	return ret;
234 }
235 
236 unsigned int inet_addr_type_table(struct net *net, __be32 addr, u32 tb_id)
237 {
238 	return __inet_dev_addr_type(net, NULL, addr, tb_id);
239 }
240 EXPORT_SYMBOL(inet_addr_type_table);
241 
242 unsigned int inet_addr_type(struct net *net, __be32 addr)
243 {
244 	return __inet_dev_addr_type(net, NULL, addr, RT_TABLE_LOCAL);
245 }
246 EXPORT_SYMBOL(inet_addr_type);
247 
248 unsigned int inet_dev_addr_type(struct net *net, const struct net_device *dev,
249 				__be32 addr)
250 {
251 	u32 rt_table = l3mdev_fib_table(dev) ? : RT_TABLE_LOCAL;
252 
253 	return __inet_dev_addr_type(net, dev, addr, rt_table);
254 }
255 EXPORT_SYMBOL(inet_dev_addr_type);
256 
257 /* inet_addr_type with dev == NULL but using the table from a dev
258  * if one is associated
259  */
260 unsigned int inet_addr_type_dev_table(struct net *net,
261 				      const struct net_device *dev,
262 				      __be32 addr)
263 {
264 	u32 rt_table = l3mdev_fib_table(dev) ? : RT_TABLE_LOCAL;
265 
266 	return __inet_dev_addr_type(net, NULL, addr, rt_table);
267 }
268 EXPORT_SYMBOL(inet_addr_type_dev_table);
269 
270 __be32 fib_compute_spec_dst(struct sk_buff *skb)
271 {
272 	struct net_device *dev = skb->dev;
273 	struct in_device *in_dev;
274 	struct fib_result res;
275 	struct rtable *rt;
276 	struct net *net;
277 	int scope;
278 
279 	rt = skb_rtable(skb);
280 	if ((rt->rt_flags & (RTCF_BROADCAST | RTCF_MULTICAST | RTCF_LOCAL)) ==
281 	    RTCF_LOCAL)
282 		return ip_hdr(skb)->daddr;
283 
284 	in_dev = __in_dev_get_rcu(dev);
285 	BUG_ON(!in_dev);
286 
287 	net = dev_net(dev);
288 
289 	scope = RT_SCOPE_UNIVERSE;
290 	if (!ipv4_is_zeronet(ip_hdr(skb)->saddr)) {
291 		struct flowi4 fl4 = {
292 			.flowi4_iif = LOOPBACK_IFINDEX,
293 			.daddr = ip_hdr(skb)->saddr,
294 			.flowi4_tos = RT_TOS(ip_hdr(skb)->tos),
295 			.flowi4_scope = scope,
296 			.flowi4_mark = IN_DEV_SRC_VMARK(in_dev) ? skb->mark : 0,
297 		};
298 		if (!fib_lookup(net, &fl4, &res, 0))
299 			return FIB_RES_PREFSRC(net, res);
300 	} else {
301 		scope = RT_SCOPE_LINK;
302 	}
303 
304 	return inet_select_addr(dev, ip_hdr(skb)->saddr, scope);
305 }
306 
307 /* Given (packet source, input interface) and optional (dst, oif, tos):
308  * - (main) check, that source is valid i.e. not broadcast or our local
309  *   address.
310  * - figure out what "logical" interface this packet arrived
311  *   and calculate "specific destination" address.
312  * - check, that packet arrived from expected physical interface.
313  * called with rcu_read_lock()
314  */
315 static int __fib_validate_source(struct sk_buff *skb, __be32 src, __be32 dst,
316 				 u8 tos, int oif, struct net_device *dev,
317 				 int rpf, struct in_device *idev, u32 *itag)
318 {
319 	int ret, no_addr;
320 	struct fib_result res;
321 	struct flowi4 fl4;
322 	struct net *net = dev_net(dev);
323 	bool dev_match;
324 
325 	fl4.flowi4_oif = 0;
326 	fl4.flowi4_iif = l3mdev_master_ifindex_rcu(dev);
327 	if (!fl4.flowi4_iif)
328 		fl4.flowi4_iif = oif ? : LOOPBACK_IFINDEX;
329 	fl4.daddr = src;
330 	fl4.saddr = dst;
331 	fl4.flowi4_tos = tos;
332 	fl4.flowi4_scope = RT_SCOPE_UNIVERSE;
333 	fl4.flowi4_tun_key.tun_id = 0;
334 	fl4.flowi4_flags = 0;
335 	fl4.flowi4_uid = sock_net_uid(net, NULL);
336 
337 	no_addr = idev->ifa_list == NULL;
338 
339 	fl4.flowi4_mark = IN_DEV_SRC_VMARK(idev) ? skb->mark : 0;
340 
341 	trace_fib_validate_source(dev, &fl4);
342 
343 	if (fib_lookup(net, &fl4, &res, 0))
344 		goto last_resort;
345 	if (res.type != RTN_UNICAST &&
346 	    (res.type != RTN_LOCAL || !IN_DEV_ACCEPT_LOCAL(idev)))
347 		goto e_inval;
348 	if (!rpf && !fib_num_tclassid_users(net) &&
349 	    (dev->ifindex != oif || !IN_DEV_TX_REDIRECTS(idev)))
350 		goto last_resort;
351 	fib_combine_itag(itag, &res);
352 	dev_match = false;
353 
354 #ifdef CONFIG_IP_ROUTE_MULTIPATH
355 	for (ret = 0; ret < res.fi->fib_nhs; ret++) {
356 		struct fib_nh *nh = &res.fi->fib_nh[ret];
357 
358 		if (nh->nh_dev == dev) {
359 			dev_match = true;
360 			break;
361 		} else if (l3mdev_master_ifindex_rcu(nh->nh_dev) == dev->ifindex) {
362 			dev_match = true;
363 			break;
364 		}
365 	}
366 #else
367 	if (FIB_RES_DEV(res) == dev)
368 		dev_match = true;
369 #endif
370 	if (dev_match) {
371 		ret = FIB_RES_NH(res).nh_scope >= RT_SCOPE_HOST;
372 		return ret;
373 	}
374 	if (no_addr)
375 		goto last_resort;
376 	if (rpf == 1)
377 		goto e_rpf;
378 	fl4.flowi4_oif = dev->ifindex;
379 
380 	ret = 0;
381 	if (fib_lookup(net, &fl4, &res, FIB_LOOKUP_IGNORE_LINKSTATE) == 0) {
382 		if (res.type == RTN_UNICAST)
383 			ret = FIB_RES_NH(res).nh_scope >= RT_SCOPE_HOST;
384 	}
385 	return ret;
386 
387 last_resort:
388 	if (rpf)
389 		goto e_rpf;
390 	*itag = 0;
391 	return 0;
392 
393 e_inval:
394 	return -EINVAL;
395 e_rpf:
396 	return -EXDEV;
397 }
398 
399 /* Ignore rp_filter for packets protected by IPsec. */
400 int fib_validate_source(struct sk_buff *skb, __be32 src, __be32 dst,
401 			u8 tos, int oif, struct net_device *dev,
402 			struct in_device *idev, u32 *itag)
403 {
404 	int r = secpath_exists(skb) ? 0 : IN_DEV_RPFILTER(idev);
405 
406 	if (!r && !fib_num_tclassid_users(dev_net(dev)) &&
407 	    IN_DEV_ACCEPT_LOCAL(idev) &&
408 	    (dev->ifindex != oif || !IN_DEV_TX_REDIRECTS(idev))) {
409 		*itag = 0;
410 		return 0;
411 	}
412 	return __fib_validate_source(skb, src, dst, tos, oif, dev, r, idev, itag);
413 }
414 
415 static inline __be32 sk_extract_addr(struct sockaddr *addr)
416 {
417 	return ((struct sockaddr_in *) addr)->sin_addr.s_addr;
418 }
419 
420 static int put_rtax(struct nlattr *mx, int len, int type, u32 value)
421 {
422 	struct nlattr *nla;
423 
424 	nla = (struct nlattr *) ((char *) mx + len);
425 	nla->nla_type = type;
426 	nla->nla_len = nla_attr_size(4);
427 	*(u32 *) nla_data(nla) = value;
428 
429 	return len + nla_total_size(4);
430 }
431 
432 static int rtentry_to_fib_config(struct net *net, int cmd, struct rtentry *rt,
433 				 struct fib_config *cfg)
434 {
435 	__be32 addr;
436 	int plen;
437 
438 	memset(cfg, 0, sizeof(*cfg));
439 	cfg->fc_nlinfo.nl_net = net;
440 
441 	if (rt->rt_dst.sa_family != AF_INET)
442 		return -EAFNOSUPPORT;
443 
444 	/*
445 	 * Check mask for validity:
446 	 * a) it must be contiguous.
447 	 * b) destination must have all host bits clear.
448 	 * c) if application forgot to set correct family (AF_INET),
449 	 *    reject request unless it is absolutely clear i.e.
450 	 *    both family and mask are zero.
451 	 */
452 	plen = 32;
453 	addr = sk_extract_addr(&rt->rt_dst);
454 	if (!(rt->rt_flags & RTF_HOST)) {
455 		__be32 mask = sk_extract_addr(&rt->rt_genmask);
456 
457 		if (rt->rt_genmask.sa_family != AF_INET) {
458 			if (mask || rt->rt_genmask.sa_family)
459 				return -EAFNOSUPPORT;
460 		}
461 
462 		if (bad_mask(mask, addr))
463 			return -EINVAL;
464 
465 		plen = inet_mask_len(mask);
466 	}
467 
468 	cfg->fc_dst_len = plen;
469 	cfg->fc_dst = addr;
470 
471 	if (cmd != SIOCDELRT) {
472 		cfg->fc_nlflags = NLM_F_CREATE;
473 		cfg->fc_protocol = RTPROT_BOOT;
474 	}
475 
476 	if (rt->rt_metric)
477 		cfg->fc_priority = rt->rt_metric - 1;
478 
479 	if (rt->rt_flags & RTF_REJECT) {
480 		cfg->fc_scope = RT_SCOPE_HOST;
481 		cfg->fc_type = RTN_UNREACHABLE;
482 		return 0;
483 	}
484 
485 	cfg->fc_scope = RT_SCOPE_NOWHERE;
486 	cfg->fc_type = RTN_UNICAST;
487 
488 	if (rt->rt_dev) {
489 		char *colon;
490 		struct net_device *dev;
491 		char devname[IFNAMSIZ];
492 
493 		if (copy_from_user(devname, rt->rt_dev, IFNAMSIZ-1))
494 			return -EFAULT;
495 
496 		devname[IFNAMSIZ-1] = 0;
497 		colon = strchr(devname, ':');
498 		if (colon)
499 			*colon = 0;
500 		dev = __dev_get_by_name(net, devname);
501 		if (!dev)
502 			return -ENODEV;
503 		cfg->fc_oif = dev->ifindex;
504 		cfg->fc_table = l3mdev_fib_table(dev);
505 		if (colon) {
506 			struct in_ifaddr *ifa;
507 			struct in_device *in_dev = __in_dev_get_rtnl(dev);
508 			if (!in_dev)
509 				return -ENODEV;
510 			*colon = ':';
511 			for (ifa = in_dev->ifa_list; ifa; ifa = ifa->ifa_next)
512 				if (strcmp(ifa->ifa_label, devname) == 0)
513 					break;
514 			if (!ifa)
515 				return -ENODEV;
516 			cfg->fc_prefsrc = ifa->ifa_local;
517 		}
518 	}
519 
520 	addr = sk_extract_addr(&rt->rt_gateway);
521 	if (rt->rt_gateway.sa_family == AF_INET && addr) {
522 		unsigned int addr_type;
523 
524 		cfg->fc_gw = addr;
525 		addr_type = inet_addr_type_table(net, addr, cfg->fc_table);
526 		if (rt->rt_flags & RTF_GATEWAY &&
527 		    addr_type == RTN_UNICAST)
528 			cfg->fc_scope = RT_SCOPE_UNIVERSE;
529 	}
530 
531 	if (cmd == SIOCDELRT)
532 		return 0;
533 
534 	if (rt->rt_flags & RTF_GATEWAY && !cfg->fc_gw)
535 		return -EINVAL;
536 
537 	if (cfg->fc_scope == RT_SCOPE_NOWHERE)
538 		cfg->fc_scope = RT_SCOPE_LINK;
539 
540 	if (rt->rt_flags & (RTF_MTU | RTF_WINDOW | RTF_IRTT)) {
541 		struct nlattr *mx;
542 		int len = 0;
543 
544 		mx = kzalloc(3 * nla_total_size(4), GFP_KERNEL);
545 		if (!mx)
546 			return -ENOMEM;
547 
548 		if (rt->rt_flags & RTF_MTU)
549 			len = put_rtax(mx, len, RTAX_ADVMSS, rt->rt_mtu - 40);
550 
551 		if (rt->rt_flags & RTF_WINDOW)
552 			len = put_rtax(mx, len, RTAX_WINDOW, rt->rt_window);
553 
554 		if (rt->rt_flags & RTF_IRTT)
555 			len = put_rtax(mx, len, RTAX_RTT, rt->rt_irtt << 3);
556 
557 		cfg->fc_mx = mx;
558 		cfg->fc_mx_len = len;
559 	}
560 
561 	return 0;
562 }
563 
564 /*
565  * Handle IP routing ioctl calls.
566  * These are used to manipulate the routing tables
567  */
568 int ip_rt_ioctl(struct net *net, unsigned int cmd, void __user *arg)
569 {
570 	struct fib_config cfg;
571 	struct rtentry rt;
572 	int err;
573 
574 	switch (cmd) {
575 	case SIOCADDRT:		/* Add a route */
576 	case SIOCDELRT:		/* Delete a route */
577 		if (!ns_capable(net->user_ns, CAP_NET_ADMIN))
578 			return -EPERM;
579 
580 		if (copy_from_user(&rt, arg, sizeof(rt)))
581 			return -EFAULT;
582 
583 		rtnl_lock();
584 		err = rtentry_to_fib_config(net, cmd, &rt, &cfg);
585 		if (err == 0) {
586 			struct fib_table *tb;
587 
588 			if (cmd == SIOCDELRT) {
589 				tb = fib_get_table(net, cfg.fc_table);
590 				if (tb)
591 					err = fib_table_delete(net, tb, &cfg,
592 							       NULL);
593 				else
594 					err = -ESRCH;
595 			} else {
596 				tb = fib_new_table(net, cfg.fc_table);
597 				if (tb)
598 					err = fib_table_insert(net, tb,
599 							       &cfg, NULL);
600 				else
601 					err = -ENOBUFS;
602 			}
603 
604 			/* allocated by rtentry_to_fib_config() */
605 			kfree(cfg.fc_mx);
606 		}
607 		rtnl_unlock();
608 		return err;
609 	}
610 	return -EINVAL;
611 }
612 
613 const struct nla_policy rtm_ipv4_policy[RTA_MAX + 1] = {
614 	[RTA_DST]		= { .type = NLA_U32 },
615 	[RTA_SRC]		= { .type = NLA_U32 },
616 	[RTA_IIF]		= { .type = NLA_U32 },
617 	[RTA_OIF]		= { .type = NLA_U32 },
618 	[RTA_GATEWAY]		= { .type = NLA_U32 },
619 	[RTA_PRIORITY]		= { .type = NLA_U32 },
620 	[RTA_PREFSRC]		= { .type = NLA_U32 },
621 	[RTA_METRICS]		= { .type = NLA_NESTED },
622 	[RTA_MULTIPATH]		= { .len = sizeof(struct rtnexthop) },
623 	[RTA_FLOW]		= { .type = NLA_U32 },
624 	[RTA_ENCAP_TYPE]	= { .type = NLA_U16 },
625 	[RTA_ENCAP]		= { .type = NLA_NESTED },
626 	[RTA_UID]		= { .type = NLA_U32 },
627 	[RTA_MARK]		= { .type = NLA_U32 },
628 };
629 
630 static int rtm_to_fib_config(struct net *net, struct sk_buff *skb,
631 			     struct nlmsghdr *nlh, struct fib_config *cfg,
632 			     struct netlink_ext_ack *extack)
633 {
634 	struct nlattr *attr;
635 	int err, remaining;
636 	struct rtmsg *rtm;
637 
638 	err = nlmsg_validate(nlh, sizeof(*rtm), RTA_MAX, rtm_ipv4_policy,
639 			     extack);
640 	if (err < 0)
641 		goto errout;
642 
643 	memset(cfg, 0, sizeof(*cfg));
644 
645 	rtm = nlmsg_data(nlh);
646 	cfg->fc_dst_len = rtm->rtm_dst_len;
647 	cfg->fc_tos = rtm->rtm_tos;
648 	cfg->fc_table = rtm->rtm_table;
649 	cfg->fc_protocol = rtm->rtm_protocol;
650 	cfg->fc_scope = rtm->rtm_scope;
651 	cfg->fc_type = rtm->rtm_type;
652 	cfg->fc_flags = rtm->rtm_flags;
653 	cfg->fc_nlflags = nlh->nlmsg_flags;
654 
655 	cfg->fc_nlinfo.portid = NETLINK_CB(skb).portid;
656 	cfg->fc_nlinfo.nlh = nlh;
657 	cfg->fc_nlinfo.nl_net = net;
658 
659 	if (cfg->fc_type > RTN_MAX) {
660 		NL_SET_ERR_MSG(extack, "Invalid route type");
661 		err = -EINVAL;
662 		goto errout;
663 	}
664 
665 	nlmsg_for_each_attr(attr, nlh, sizeof(struct rtmsg), remaining) {
666 		switch (nla_type(attr)) {
667 		case RTA_DST:
668 			cfg->fc_dst = nla_get_be32(attr);
669 			break;
670 		case RTA_OIF:
671 			cfg->fc_oif = nla_get_u32(attr);
672 			break;
673 		case RTA_GATEWAY:
674 			cfg->fc_gw = nla_get_be32(attr);
675 			break;
676 		case RTA_PRIORITY:
677 			cfg->fc_priority = nla_get_u32(attr);
678 			break;
679 		case RTA_PREFSRC:
680 			cfg->fc_prefsrc = nla_get_be32(attr);
681 			break;
682 		case RTA_METRICS:
683 			cfg->fc_mx = nla_data(attr);
684 			cfg->fc_mx_len = nla_len(attr);
685 			break;
686 		case RTA_MULTIPATH:
687 			err = lwtunnel_valid_encap_type_attr(nla_data(attr),
688 							     nla_len(attr),
689 							     extack);
690 			if (err < 0)
691 				goto errout;
692 			cfg->fc_mp = nla_data(attr);
693 			cfg->fc_mp_len = nla_len(attr);
694 			break;
695 		case RTA_FLOW:
696 			cfg->fc_flow = nla_get_u32(attr);
697 			break;
698 		case RTA_TABLE:
699 			cfg->fc_table = nla_get_u32(attr);
700 			break;
701 		case RTA_ENCAP:
702 			cfg->fc_encap = attr;
703 			break;
704 		case RTA_ENCAP_TYPE:
705 			cfg->fc_encap_type = nla_get_u16(attr);
706 			err = lwtunnel_valid_encap_type(cfg->fc_encap_type,
707 							extack);
708 			if (err < 0)
709 				goto errout;
710 			break;
711 		}
712 	}
713 
714 	return 0;
715 errout:
716 	return err;
717 }
718 
719 static int inet_rtm_delroute(struct sk_buff *skb, struct nlmsghdr *nlh,
720 			     struct netlink_ext_ack *extack)
721 {
722 	struct net *net = sock_net(skb->sk);
723 	struct fib_config cfg;
724 	struct fib_table *tb;
725 	int err;
726 
727 	err = rtm_to_fib_config(net, skb, nlh, &cfg, extack);
728 	if (err < 0)
729 		goto errout;
730 
731 	tb = fib_get_table(net, cfg.fc_table);
732 	if (!tb) {
733 		NL_SET_ERR_MSG(extack, "FIB table does not exist");
734 		err = -ESRCH;
735 		goto errout;
736 	}
737 
738 	err = fib_table_delete(net, tb, &cfg, extack);
739 errout:
740 	return err;
741 }
742 
743 static int inet_rtm_newroute(struct sk_buff *skb, struct nlmsghdr *nlh,
744 			     struct netlink_ext_ack *extack)
745 {
746 	struct net *net = sock_net(skb->sk);
747 	struct fib_config cfg;
748 	struct fib_table *tb;
749 	int err;
750 
751 	err = rtm_to_fib_config(net, skb, nlh, &cfg, extack);
752 	if (err < 0)
753 		goto errout;
754 
755 	tb = fib_new_table(net, cfg.fc_table);
756 	if (!tb) {
757 		err = -ENOBUFS;
758 		goto errout;
759 	}
760 
761 	err = fib_table_insert(net, tb, &cfg, extack);
762 errout:
763 	return err;
764 }
765 
766 static int inet_dump_fib(struct sk_buff *skb, struct netlink_callback *cb)
767 {
768 	struct net *net = sock_net(skb->sk);
769 	unsigned int h, s_h;
770 	unsigned int e = 0, s_e;
771 	struct fib_table *tb;
772 	struct hlist_head *head;
773 	int dumped = 0, err;
774 
775 	if (nlmsg_len(cb->nlh) >= sizeof(struct rtmsg) &&
776 	    ((struct rtmsg *) nlmsg_data(cb->nlh))->rtm_flags & RTM_F_CLONED)
777 		return skb->len;
778 
779 	s_h = cb->args[0];
780 	s_e = cb->args[1];
781 
782 	rcu_read_lock();
783 
784 	for (h = s_h; h < FIB_TABLE_HASHSZ; h++, s_e = 0) {
785 		e = 0;
786 		head = &net->ipv4.fib_table_hash[h];
787 		hlist_for_each_entry_rcu(tb, head, tb_hlist) {
788 			if (e < s_e)
789 				goto next;
790 			if (dumped)
791 				memset(&cb->args[2], 0, sizeof(cb->args) -
792 						 2 * sizeof(cb->args[0]));
793 			err = fib_table_dump(tb, skb, cb);
794 			if (err < 0) {
795 				if (likely(skb->len))
796 					goto out;
797 
798 				goto out_err;
799 			}
800 			dumped = 1;
801 next:
802 			e++;
803 		}
804 	}
805 out:
806 	err = skb->len;
807 out_err:
808 	rcu_read_unlock();
809 
810 	cb->args[1] = e;
811 	cb->args[0] = h;
812 
813 	return err;
814 }
815 
816 /* Prepare and feed intra-kernel routing request.
817  * Really, it should be netlink message, but :-( netlink
818  * can be not configured, so that we feed it directly
819  * to fib engine. It is legal, because all events occur
820  * only when netlink is already locked.
821  */
822 static void fib_magic(int cmd, int type, __be32 dst, int dst_len, struct in_ifaddr *ifa)
823 {
824 	struct net *net = dev_net(ifa->ifa_dev->dev);
825 	u32 tb_id = l3mdev_fib_table(ifa->ifa_dev->dev);
826 	struct fib_table *tb;
827 	struct fib_config cfg = {
828 		.fc_protocol = RTPROT_KERNEL,
829 		.fc_type = type,
830 		.fc_dst = dst,
831 		.fc_dst_len = dst_len,
832 		.fc_prefsrc = ifa->ifa_local,
833 		.fc_oif = ifa->ifa_dev->dev->ifindex,
834 		.fc_nlflags = NLM_F_CREATE | NLM_F_APPEND,
835 		.fc_nlinfo = {
836 			.nl_net = net,
837 		},
838 	};
839 
840 	if (!tb_id)
841 		tb_id = (type == RTN_UNICAST) ? RT_TABLE_MAIN : RT_TABLE_LOCAL;
842 
843 	tb = fib_new_table(net, tb_id);
844 	if (!tb)
845 		return;
846 
847 	cfg.fc_table = tb->tb_id;
848 
849 	if (type != RTN_LOCAL)
850 		cfg.fc_scope = RT_SCOPE_LINK;
851 	else
852 		cfg.fc_scope = RT_SCOPE_HOST;
853 
854 	if (cmd == RTM_NEWROUTE)
855 		fib_table_insert(net, tb, &cfg, NULL);
856 	else
857 		fib_table_delete(net, tb, &cfg, NULL);
858 }
859 
860 void fib_add_ifaddr(struct in_ifaddr *ifa)
861 {
862 	struct in_device *in_dev = ifa->ifa_dev;
863 	struct net_device *dev = in_dev->dev;
864 	struct in_ifaddr *prim = ifa;
865 	__be32 mask = ifa->ifa_mask;
866 	__be32 addr = ifa->ifa_local;
867 	__be32 prefix = ifa->ifa_address & mask;
868 
869 	if (ifa->ifa_flags & IFA_F_SECONDARY) {
870 		prim = inet_ifa_byprefix(in_dev, prefix, mask);
871 		if (!prim) {
872 			pr_warn("%s: bug: prim == NULL\n", __func__);
873 			return;
874 		}
875 	}
876 
877 	fib_magic(RTM_NEWROUTE, RTN_LOCAL, addr, 32, prim);
878 
879 	if (!(dev->flags & IFF_UP))
880 		return;
881 
882 	/* Add broadcast address, if it is explicitly assigned. */
883 	if (ifa->ifa_broadcast && ifa->ifa_broadcast != htonl(0xFFFFFFFF))
884 		fib_magic(RTM_NEWROUTE, RTN_BROADCAST, ifa->ifa_broadcast, 32, prim);
885 
886 	if (!ipv4_is_zeronet(prefix) && !(ifa->ifa_flags & IFA_F_SECONDARY) &&
887 	    (prefix != addr || ifa->ifa_prefixlen < 32)) {
888 		if (!(ifa->ifa_flags & IFA_F_NOPREFIXROUTE))
889 			fib_magic(RTM_NEWROUTE,
890 				  dev->flags & IFF_LOOPBACK ? RTN_LOCAL : RTN_UNICAST,
891 				  prefix, ifa->ifa_prefixlen, prim);
892 
893 		/* Add network specific broadcasts, when it takes a sense */
894 		if (ifa->ifa_prefixlen < 31) {
895 			fib_magic(RTM_NEWROUTE, RTN_BROADCAST, prefix, 32, prim);
896 			fib_magic(RTM_NEWROUTE, RTN_BROADCAST, prefix | ~mask,
897 				  32, prim);
898 		}
899 	}
900 }
901 
902 /* Delete primary or secondary address.
903  * Optionally, on secondary address promotion consider the addresses
904  * from subnet iprim as deleted, even if they are in device list.
905  * In this case the secondary ifa can be in device list.
906  */
907 void fib_del_ifaddr(struct in_ifaddr *ifa, struct in_ifaddr *iprim)
908 {
909 	struct in_device *in_dev = ifa->ifa_dev;
910 	struct net_device *dev = in_dev->dev;
911 	struct in_ifaddr *ifa1;
912 	struct in_ifaddr *prim = ifa, *prim1 = NULL;
913 	__be32 brd = ifa->ifa_address | ~ifa->ifa_mask;
914 	__be32 any = ifa->ifa_address & ifa->ifa_mask;
915 #define LOCAL_OK	1
916 #define BRD_OK		2
917 #define BRD0_OK		4
918 #define BRD1_OK		8
919 	unsigned int ok = 0;
920 	int subnet = 0;		/* Primary network */
921 	int gone = 1;		/* Address is missing */
922 	int same_prefsrc = 0;	/* Another primary with same IP */
923 
924 	if (ifa->ifa_flags & IFA_F_SECONDARY) {
925 		prim = inet_ifa_byprefix(in_dev, any, ifa->ifa_mask);
926 		if (!prim) {
927 			/* if the device has been deleted, we don't perform
928 			 * address promotion
929 			 */
930 			if (!in_dev->dead)
931 				pr_warn("%s: bug: prim == NULL\n", __func__);
932 			return;
933 		}
934 		if (iprim && iprim != prim) {
935 			pr_warn("%s: bug: iprim != prim\n", __func__);
936 			return;
937 		}
938 	} else if (!ipv4_is_zeronet(any) &&
939 		   (any != ifa->ifa_local || ifa->ifa_prefixlen < 32)) {
940 		if (!(ifa->ifa_flags & IFA_F_NOPREFIXROUTE))
941 			fib_magic(RTM_DELROUTE,
942 				  dev->flags & IFF_LOOPBACK ? RTN_LOCAL : RTN_UNICAST,
943 				  any, ifa->ifa_prefixlen, prim);
944 		subnet = 1;
945 	}
946 
947 	if (in_dev->dead)
948 		goto no_promotions;
949 
950 	/* Deletion is more complicated than add.
951 	 * We should take care of not to delete too much :-)
952 	 *
953 	 * Scan address list to be sure that addresses are really gone.
954 	 */
955 
956 	for (ifa1 = in_dev->ifa_list; ifa1; ifa1 = ifa1->ifa_next) {
957 		if (ifa1 == ifa) {
958 			/* promotion, keep the IP */
959 			gone = 0;
960 			continue;
961 		}
962 		/* Ignore IFAs from our subnet */
963 		if (iprim && ifa1->ifa_mask == iprim->ifa_mask &&
964 		    inet_ifa_match(ifa1->ifa_address, iprim))
965 			continue;
966 
967 		/* Ignore ifa1 if it uses different primary IP (prefsrc) */
968 		if (ifa1->ifa_flags & IFA_F_SECONDARY) {
969 			/* Another address from our subnet? */
970 			if (ifa1->ifa_mask == prim->ifa_mask &&
971 			    inet_ifa_match(ifa1->ifa_address, prim))
972 				prim1 = prim;
973 			else {
974 				/* We reached the secondaries, so
975 				 * same_prefsrc should be determined.
976 				 */
977 				if (!same_prefsrc)
978 					continue;
979 				/* Search new prim1 if ifa1 is not
980 				 * using the current prim1
981 				 */
982 				if (!prim1 ||
983 				    ifa1->ifa_mask != prim1->ifa_mask ||
984 				    !inet_ifa_match(ifa1->ifa_address, prim1))
985 					prim1 = inet_ifa_byprefix(in_dev,
986 							ifa1->ifa_address,
987 							ifa1->ifa_mask);
988 				if (!prim1)
989 					continue;
990 				if (prim1->ifa_local != prim->ifa_local)
991 					continue;
992 			}
993 		} else {
994 			if (prim->ifa_local != ifa1->ifa_local)
995 				continue;
996 			prim1 = ifa1;
997 			if (prim != prim1)
998 				same_prefsrc = 1;
999 		}
1000 		if (ifa->ifa_local == ifa1->ifa_local)
1001 			ok |= LOCAL_OK;
1002 		if (ifa->ifa_broadcast == ifa1->ifa_broadcast)
1003 			ok |= BRD_OK;
1004 		if (brd == ifa1->ifa_broadcast)
1005 			ok |= BRD1_OK;
1006 		if (any == ifa1->ifa_broadcast)
1007 			ok |= BRD0_OK;
1008 		/* primary has network specific broadcasts */
1009 		if (prim1 == ifa1 && ifa1->ifa_prefixlen < 31) {
1010 			__be32 brd1 = ifa1->ifa_address | ~ifa1->ifa_mask;
1011 			__be32 any1 = ifa1->ifa_address & ifa1->ifa_mask;
1012 
1013 			if (!ipv4_is_zeronet(any1)) {
1014 				if (ifa->ifa_broadcast == brd1 ||
1015 				    ifa->ifa_broadcast == any1)
1016 					ok |= BRD_OK;
1017 				if (brd == brd1 || brd == any1)
1018 					ok |= BRD1_OK;
1019 				if (any == brd1 || any == any1)
1020 					ok |= BRD0_OK;
1021 			}
1022 		}
1023 	}
1024 
1025 no_promotions:
1026 	if (!(ok & BRD_OK))
1027 		fib_magic(RTM_DELROUTE, RTN_BROADCAST, ifa->ifa_broadcast, 32, prim);
1028 	if (subnet && ifa->ifa_prefixlen < 31) {
1029 		if (!(ok & BRD1_OK))
1030 			fib_magic(RTM_DELROUTE, RTN_BROADCAST, brd, 32, prim);
1031 		if (!(ok & BRD0_OK))
1032 			fib_magic(RTM_DELROUTE, RTN_BROADCAST, any, 32, prim);
1033 	}
1034 	if (!(ok & LOCAL_OK)) {
1035 		unsigned int addr_type;
1036 
1037 		fib_magic(RTM_DELROUTE, RTN_LOCAL, ifa->ifa_local, 32, prim);
1038 
1039 		/* Check, that this local address finally disappeared. */
1040 		addr_type = inet_addr_type_dev_table(dev_net(dev), dev,
1041 						     ifa->ifa_local);
1042 		if (gone && addr_type != RTN_LOCAL) {
1043 			/* And the last, but not the least thing.
1044 			 * We must flush stray FIB entries.
1045 			 *
1046 			 * First of all, we scan fib_info list searching
1047 			 * for stray nexthop entries, then ignite fib_flush.
1048 			 */
1049 			if (fib_sync_down_addr(dev, ifa->ifa_local))
1050 				fib_flush(dev_net(dev));
1051 		}
1052 	}
1053 #undef LOCAL_OK
1054 #undef BRD_OK
1055 #undef BRD0_OK
1056 #undef BRD1_OK
1057 }
1058 
1059 static void nl_fib_lookup(struct net *net, struct fib_result_nl *frn)
1060 {
1061 
1062 	struct fib_result       res;
1063 	struct flowi4           fl4 = {
1064 		.flowi4_mark = frn->fl_mark,
1065 		.daddr = frn->fl_addr,
1066 		.flowi4_tos = frn->fl_tos,
1067 		.flowi4_scope = frn->fl_scope,
1068 	};
1069 	struct fib_table *tb;
1070 
1071 	rcu_read_lock();
1072 
1073 	tb = fib_get_table(net, frn->tb_id_in);
1074 
1075 	frn->err = -ENOENT;
1076 	if (tb) {
1077 		local_bh_disable();
1078 
1079 		frn->tb_id = tb->tb_id;
1080 		frn->err = fib_table_lookup(tb, &fl4, &res, FIB_LOOKUP_NOREF);
1081 
1082 		if (!frn->err) {
1083 			frn->prefixlen = res.prefixlen;
1084 			frn->nh_sel = res.nh_sel;
1085 			frn->type = res.type;
1086 			frn->scope = res.scope;
1087 		}
1088 		local_bh_enable();
1089 	}
1090 
1091 	rcu_read_unlock();
1092 }
1093 
1094 static void nl_fib_input(struct sk_buff *skb)
1095 {
1096 	struct net *net;
1097 	struct fib_result_nl *frn;
1098 	struct nlmsghdr *nlh;
1099 	u32 portid;
1100 
1101 	net = sock_net(skb->sk);
1102 	nlh = nlmsg_hdr(skb);
1103 	if (skb->len < nlmsg_total_size(sizeof(*frn)) ||
1104 	    skb->len < nlh->nlmsg_len ||
1105 	    nlmsg_len(nlh) < sizeof(*frn))
1106 		return;
1107 
1108 	skb = netlink_skb_clone(skb, GFP_KERNEL);
1109 	if (!skb)
1110 		return;
1111 	nlh = nlmsg_hdr(skb);
1112 
1113 	frn = (struct fib_result_nl *) nlmsg_data(nlh);
1114 	nl_fib_lookup(net, frn);
1115 
1116 	portid = NETLINK_CB(skb).portid;      /* netlink portid */
1117 	NETLINK_CB(skb).portid = 0;        /* from kernel */
1118 	NETLINK_CB(skb).dst_group = 0;  /* unicast */
1119 	netlink_unicast(net->ipv4.fibnl, skb, portid, MSG_DONTWAIT);
1120 }
1121 
1122 static int __net_init nl_fib_lookup_init(struct net *net)
1123 {
1124 	struct sock *sk;
1125 	struct netlink_kernel_cfg cfg = {
1126 		.input	= nl_fib_input,
1127 	};
1128 
1129 	sk = netlink_kernel_create(net, NETLINK_FIB_LOOKUP, &cfg);
1130 	if (!sk)
1131 		return -EAFNOSUPPORT;
1132 	net->ipv4.fibnl = sk;
1133 	return 0;
1134 }
1135 
1136 static void nl_fib_lookup_exit(struct net *net)
1137 {
1138 	netlink_kernel_release(net->ipv4.fibnl);
1139 	net->ipv4.fibnl = NULL;
1140 }
1141 
1142 static void fib_disable_ip(struct net_device *dev, unsigned long event,
1143 			   bool force)
1144 {
1145 	if (fib_sync_down_dev(dev, event, force))
1146 		fib_flush(dev_net(dev));
1147 	else
1148 		rt_cache_flush(dev_net(dev));
1149 	arp_ifdown(dev);
1150 }
1151 
1152 static int fib_inetaddr_event(struct notifier_block *this, unsigned long event, void *ptr)
1153 {
1154 	struct in_ifaddr *ifa = (struct in_ifaddr *)ptr;
1155 	struct net_device *dev = ifa->ifa_dev->dev;
1156 	struct net *net = dev_net(dev);
1157 
1158 	switch (event) {
1159 	case NETDEV_UP:
1160 		fib_add_ifaddr(ifa);
1161 #ifdef CONFIG_IP_ROUTE_MULTIPATH
1162 		fib_sync_up(dev, RTNH_F_DEAD);
1163 #endif
1164 		atomic_inc(&net->ipv4.dev_addr_genid);
1165 		rt_cache_flush(dev_net(dev));
1166 		break;
1167 	case NETDEV_DOWN:
1168 		fib_del_ifaddr(ifa, NULL);
1169 		atomic_inc(&net->ipv4.dev_addr_genid);
1170 		if (!ifa->ifa_dev->ifa_list) {
1171 			/* Last address was deleted from this interface.
1172 			 * Disable IP.
1173 			 */
1174 			fib_disable_ip(dev, event, true);
1175 		} else {
1176 			rt_cache_flush(dev_net(dev));
1177 		}
1178 		break;
1179 	}
1180 	return NOTIFY_DONE;
1181 }
1182 
1183 static int fib_netdev_event(struct notifier_block *this, unsigned long event, void *ptr)
1184 {
1185 	struct net_device *dev = netdev_notifier_info_to_dev(ptr);
1186 	struct netdev_notifier_changeupper_info *info;
1187 	struct in_device *in_dev;
1188 	struct net *net = dev_net(dev);
1189 	unsigned int flags;
1190 
1191 	if (event == NETDEV_UNREGISTER) {
1192 		fib_disable_ip(dev, event, true);
1193 		rt_flush_dev(dev);
1194 		return NOTIFY_DONE;
1195 	}
1196 
1197 	in_dev = __in_dev_get_rtnl(dev);
1198 	if (!in_dev)
1199 		return NOTIFY_DONE;
1200 
1201 	switch (event) {
1202 	case NETDEV_UP:
1203 		for_ifa(in_dev) {
1204 			fib_add_ifaddr(ifa);
1205 		} endfor_ifa(in_dev);
1206 #ifdef CONFIG_IP_ROUTE_MULTIPATH
1207 		fib_sync_up(dev, RTNH_F_DEAD);
1208 #endif
1209 		atomic_inc(&net->ipv4.dev_addr_genid);
1210 		rt_cache_flush(net);
1211 		break;
1212 	case NETDEV_DOWN:
1213 		fib_disable_ip(dev, event, false);
1214 		break;
1215 	case NETDEV_CHANGE:
1216 		flags = dev_get_flags(dev);
1217 		if (flags & (IFF_RUNNING | IFF_LOWER_UP))
1218 			fib_sync_up(dev, RTNH_F_LINKDOWN);
1219 		else
1220 			fib_sync_down_dev(dev, event, false);
1221 		/* fall through */
1222 	case NETDEV_CHANGEMTU:
1223 		rt_cache_flush(net);
1224 		break;
1225 	case NETDEV_CHANGEUPPER:
1226 		info = ptr;
1227 		/* flush all routes if dev is linked to or unlinked from
1228 		 * an L3 master device (e.g., VRF)
1229 		 */
1230 		if (info->upper_dev && netif_is_l3_master(info->upper_dev))
1231 			fib_disable_ip(dev, NETDEV_DOWN, true);
1232 		break;
1233 	}
1234 	return NOTIFY_DONE;
1235 }
1236 
1237 static struct notifier_block fib_inetaddr_notifier = {
1238 	.notifier_call = fib_inetaddr_event,
1239 };
1240 
1241 static struct notifier_block fib_netdev_notifier = {
1242 	.notifier_call = fib_netdev_event,
1243 };
1244 
1245 static int __net_init ip_fib_net_init(struct net *net)
1246 {
1247 	int err;
1248 	size_t size = sizeof(struct hlist_head) * FIB_TABLE_HASHSZ;
1249 
1250 	net->ipv4.fib_seq = 0;
1251 
1252 	/* Avoid false sharing : Use at least a full cache line */
1253 	size = max_t(size_t, size, L1_CACHE_BYTES);
1254 
1255 	net->ipv4.fib_table_hash = kzalloc(size, GFP_KERNEL);
1256 	if (!net->ipv4.fib_table_hash)
1257 		return -ENOMEM;
1258 
1259 	err = fib4_rules_init(net);
1260 	if (err < 0)
1261 		goto fail;
1262 	return 0;
1263 
1264 fail:
1265 	kfree(net->ipv4.fib_table_hash);
1266 	return err;
1267 }
1268 
1269 static void ip_fib_net_exit(struct net *net)
1270 {
1271 	unsigned int i;
1272 
1273 	rtnl_lock();
1274 #ifdef CONFIG_IP_MULTIPLE_TABLES
1275 	RCU_INIT_POINTER(net->ipv4.fib_main, NULL);
1276 	RCU_INIT_POINTER(net->ipv4.fib_default, NULL);
1277 #endif
1278 	for (i = 0; i < FIB_TABLE_HASHSZ; i++) {
1279 		struct hlist_head *head = &net->ipv4.fib_table_hash[i];
1280 		struct hlist_node *tmp;
1281 		struct fib_table *tb;
1282 
1283 		hlist_for_each_entry_safe(tb, tmp, head, tb_hlist) {
1284 			hlist_del(&tb->tb_hlist);
1285 			fib_table_flush(net, tb);
1286 			fib_free_table(tb);
1287 		}
1288 	}
1289 
1290 #ifdef CONFIG_IP_MULTIPLE_TABLES
1291 	fib4_rules_exit(net);
1292 #endif
1293 	rtnl_unlock();
1294 	kfree(net->ipv4.fib_table_hash);
1295 }
1296 
1297 static int __net_init fib_net_init(struct net *net)
1298 {
1299 	int error;
1300 
1301 #ifdef CONFIG_IP_ROUTE_CLASSID
1302 	net->ipv4.fib_num_tclassid_users = 0;
1303 #endif
1304 	error = ip_fib_net_init(net);
1305 	if (error < 0)
1306 		goto out;
1307 	error = nl_fib_lookup_init(net);
1308 	if (error < 0)
1309 		goto out_nlfl;
1310 	error = fib_proc_init(net);
1311 	if (error < 0)
1312 		goto out_proc;
1313 out:
1314 	return error;
1315 
1316 out_proc:
1317 	nl_fib_lookup_exit(net);
1318 out_nlfl:
1319 	ip_fib_net_exit(net);
1320 	goto out;
1321 }
1322 
1323 static void __net_exit fib_net_exit(struct net *net)
1324 {
1325 	fib_proc_exit(net);
1326 	nl_fib_lookup_exit(net);
1327 	ip_fib_net_exit(net);
1328 }
1329 
1330 static struct pernet_operations fib_net_ops = {
1331 	.init = fib_net_init,
1332 	.exit = fib_net_exit,
1333 };
1334 
1335 void __init ip_fib_init(void)
1336 {
1337 	rtnl_register(PF_INET, RTM_NEWROUTE, inet_rtm_newroute, NULL, NULL);
1338 	rtnl_register(PF_INET, RTM_DELROUTE, inet_rtm_delroute, NULL, NULL);
1339 	rtnl_register(PF_INET, RTM_GETROUTE, NULL, inet_dump_fib, NULL);
1340 
1341 	register_pernet_subsys(&fib_net_ops);
1342 	register_netdevice_notifier(&fib_netdev_notifier);
1343 	register_inetaddr_notifier(&fib_inetaddr_notifier);
1344 
1345 	fib_trie_init();
1346 }
1347