xref: /openbmc/linux/net/ipv4/route.c (revision 6aa7de05)
1 /*
2  * INET		An implementation of the TCP/IP protocol suite for the LINUX
3  *		operating system.  INET is implemented using the  BSD Socket
4  *		interface as the means of communication with the user level.
5  *
6  *		ROUTE - implementation of the IP router.
7  *
8  * Authors:	Ross Biro
9  *		Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
10  *		Alan Cox, <gw4pts@gw4pts.ampr.org>
11  *		Linus Torvalds, <Linus.Torvalds@helsinki.fi>
12  *		Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
13  *
14  * Fixes:
15  *		Alan Cox	:	Verify area fixes.
16  *		Alan Cox	:	cli() protects routing changes
17  *		Rui Oliveira	:	ICMP routing table updates
18  *		(rco@di.uminho.pt)	Routing table insertion and update
19  *		Linus Torvalds	:	Rewrote bits to be sensible
20  *		Alan Cox	:	Added BSD route gw semantics
21  *		Alan Cox	:	Super /proc >4K
22  *		Alan Cox	:	MTU in route table
23  *		Alan Cox	: 	MSS actually. Also added the window
24  *					clamper.
25  *		Sam Lantinga	:	Fixed route matching in rt_del()
26  *		Alan Cox	:	Routing cache support.
27  *		Alan Cox	:	Removed compatibility cruft.
28  *		Alan Cox	:	RTF_REJECT support.
29  *		Alan Cox	:	TCP irtt support.
30  *		Jonathan Naylor	:	Added Metric support.
31  *	Miquel van Smoorenburg	:	BSD API fixes.
32  *	Miquel van Smoorenburg	:	Metrics.
33  *		Alan Cox	:	Use __u32 properly
34  *		Alan Cox	:	Aligned routing errors more closely with BSD
35  *					our system is still very different.
36  *		Alan Cox	:	Faster /proc handling
37  *	Alexey Kuznetsov	:	Massive rework to support tree based routing,
38  *					routing caches and better behaviour.
39  *
40  *		Olaf Erb	:	irtt wasn't being copied right.
41  *		Bjorn Ekwall	:	Kerneld route support.
42  *		Alan Cox	:	Multicast fixed (I hope)
43  * 		Pavel Krauz	:	Limited broadcast fixed
44  *		Mike McLagan	:	Routing by source
45  *	Alexey Kuznetsov	:	End of old history. Split to fib.c and
46  *					route.c and rewritten from scratch.
47  *		Andi Kleen	:	Load-limit warning messages.
48  *	Vitaly E. Lavrov	:	Transparent proxy revived after year coma.
49  *	Vitaly E. Lavrov	:	Race condition in ip_route_input_slow.
50  *	Tobias Ringstrom	:	Uninitialized res.type in ip_route_output_slow.
51  *	Vladimir V. Ivanov	:	IP rule info (flowid) is really useful.
52  *		Marc Boucher	:	routing by fwmark
53  *	Robert Olsson		:	Added rt_cache statistics
54  *	Arnaldo C. Melo		:	Convert proc stuff to seq_file
55  *	Eric Dumazet		:	hashed spinlocks and rt_check_expire() fixes.
56  * 	Ilia Sotnikov		:	Ignore TOS on PMTUD and Redirect
57  * 	Ilia Sotnikov		:	Removed TOS from hash calculations
58  *
59  *		This program is free software; you can redistribute it and/or
60  *		modify it under the terms of the GNU General Public License
61  *		as published by the Free Software Foundation; either version
62  *		2 of the License, or (at your option) any later version.
63  */
64 
65 #define pr_fmt(fmt) "IPv4: " fmt
66 
67 #include <linux/module.h>
68 #include <linux/uaccess.h>
69 #include <linux/bitops.h>
70 #include <linux/types.h>
71 #include <linux/kernel.h>
72 #include <linux/mm.h>
73 #include <linux/string.h>
74 #include <linux/socket.h>
75 #include <linux/sockios.h>
76 #include <linux/errno.h>
77 #include <linux/in.h>
78 #include <linux/inet.h>
79 #include <linux/netdevice.h>
80 #include <linux/proc_fs.h>
81 #include <linux/init.h>
82 #include <linux/skbuff.h>
83 #include <linux/inetdevice.h>
84 #include <linux/igmp.h>
85 #include <linux/pkt_sched.h>
86 #include <linux/mroute.h>
87 #include <linux/netfilter_ipv4.h>
88 #include <linux/random.h>
89 #include <linux/rcupdate.h>
90 #include <linux/times.h>
91 #include <linux/slab.h>
92 #include <linux/jhash.h>
93 #include <net/dst.h>
94 #include <net/dst_metadata.h>
95 #include <net/net_namespace.h>
96 #include <net/protocol.h>
97 #include <net/ip.h>
98 #include <net/route.h>
99 #include <net/inetpeer.h>
100 #include <net/sock.h>
101 #include <net/ip_fib.h>
102 #include <net/arp.h>
103 #include <net/tcp.h>
104 #include <net/icmp.h>
105 #include <net/xfrm.h>
106 #include <net/lwtunnel.h>
107 #include <net/netevent.h>
108 #include <net/rtnetlink.h>
109 #ifdef CONFIG_SYSCTL
110 #include <linux/sysctl.h>
111 #include <linux/kmemleak.h>
112 #endif
113 #include <net/secure_seq.h>
114 #include <net/ip_tunnels.h>
115 #include <net/l3mdev.h>
116 
117 #include "fib_lookup.h"
118 
119 #define RT_FL_TOS(oldflp4) \
120 	((oldflp4)->flowi4_tos & (IPTOS_RT_MASK | RTO_ONLINK))
121 
122 #define RT_GC_TIMEOUT (300*HZ)
123 
124 static int ip_rt_max_size;
125 static int ip_rt_redirect_number __read_mostly	= 9;
126 static int ip_rt_redirect_load __read_mostly	= HZ / 50;
127 static int ip_rt_redirect_silence __read_mostly	= ((HZ / 50) << (9 + 1));
128 static int ip_rt_error_cost __read_mostly	= HZ;
129 static int ip_rt_error_burst __read_mostly	= 5 * HZ;
130 static int ip_rt_mtu_expires __read_mostly	= 10 * 60 * HZ;
131 static int ip_rt_min_pmtu __read_mostly		= 512 + 20 + 20;
132 static int ip_rt_min_advmss __read_mostly	= 256;
133 
134 static int ip_rt_gc_timeout __read_mostly	= RT_GC_TIMEOUT;
135 /*
136  *	Interface to generic destination cache.
137  */
138 
139 static struct dst_entry *ipv4_dst_check(struct dst_entry *dst, u32 cookie);
140 static unsigned int	 ipv4_default_advmss(const struct dst_entry *dst);
141 static unsigned int	 ipv4_mtu(const struct dst_entry *dst);
142 static struct dst_entry *ipv4_negative_advice(struct dst_entry *dst);
143 static void		 ipv4_link_failure(struct sk_buff *skb);
144 static void		 ip_rt_update_pmtu(struct dst_entry *dst, struct sock *sk,
145 					   struct sk_buff *skb, u32 mtu);
146 static void		 ip_do_redirect(struct dst_entry *dst, struct sock *sk,
147 					struct sk_buff *skb);
148 static void		ipv4_dst_destroy(struct dst_entry *dst);
149 
150 static u32 *ipv4_cow_metrics(struct dst_entry *dst, unsigned long old)
151 {
152 	WARN_ON(1);
153 	return NULL;
154 }
155 
156 static struct neighbour *ipv4_neigh_lookup(const struct dst_entry *dst,
157 					   struct sk_buff *skb,
158 					   const void *daddr);
159 static void ipv4_confirm_neigh(const struct dst_entry *dst, const void *daddr);
160 
161 static struct dst_ops ipv4_dst_ops = {
162 	.family =		AF_INET,
163 	.check =		ipv4_dst_check,
164 	.default_advmss =	ipv4_default_advmss,
165 	.mtu =			ipv4_mtu,
166 	.cow_metrics =		ipv4_cow_metrics,
167 	.destroy =		ipv4_dst_destroy,
168 	.negative_advice =	ipv4_negative_advice,
169 	.link_failure =		ipv4_link_failure,
170 	.update_pmtu =		ip_rt_update_pmtu,
171 	.redirect =		ip_do_redirect,
172 	.local_out =		__ip_local_out,
173 	.neigh_lookup =		ipv4_neigh_lookup,
174 	.confirm_neigh =	ipv4_confirm_neigh,
175 };
176 
177 #define ECN_OR_COST(class)	TC_PRIO_##class
178 
179 const __u8 ip_tos2prio[16] = {
180 	TC_PRIO_BESTEFFORT,
181 	ECN_OR_COST(BESTEFFORT),
182 	TC_PRIO_BESTEFFORT,
183 	ECN_OR_COST(BESTEFFORT),
184 	TC_PRIO_BULK,
185 	ECN_OR_COST(BULK),
186 	TC_PRIO_BULK,
187 	ECN_OR_COST(BULK),
188 	TC_PRIO_INTERACTIVE,
189 	ECN_OR_COST(INTERACTIVE),
190 	TC_PRIO_INTERACTIVE,
191 	ECN_OR_COST(INTERACTIVE),
192 	TC_PRIO_INTERACTIVE_BULK,
193 	ECN_OR_COST(INTERACTIVE_BULK),
194 	TC_PRIO_INTERACTIVE_BULK,
195 	ECN_OR_COST(INTERACTIVE_BULK)
196 };
197 EXPORT_SYMBOL(ip_tos2prio);
198 
199 static DEFINE_PER_CPU(struct rt_cache_stat, rt_cache_stat);
200 #define RT_CACHE_STAT_INC(field) raw_cpu_inc(rt_cache_stat.field)
201 
202 #ifdef CONFIG_PROC_FS
203 static void *rt_cache_seq_start(struct seq_file *seq, loff_t *pos)
204 {
205 	if (*pos)
206 		return NULL;
207 	return SEQ_START_TOKEN;
208 }
209 
210 static void *rt_cache_seq_next(struct seq_file *seq, void *v, loff_t *pos)
211 {
212 	++*pos;
213 	return NULL;
214 }
215 
216 static void rt_cache_seq_stop(struct seq_file *seq, void *v)
217 {
218 }
219 
220 static int rt_cache_seq_show(struct seq_file *seq, void *v)
221 {
222 	if (v == SEQ_START_TOKEN)
223 		seq_printf(seq, "%-127s\n",
224 			   "Iface\tDestination\tGateway \tFlags\t\tRefCnt\tUse\t"
225 			   "Metric\tSource\t\tMTU\tWindow\tIRTT\tTOS\tHHRef\t"
226 			   "HHUptod\tSpecDst");
227 	return 0;
228 }
229 
230 static const struct seq_operations rt_cache_seq_ops = {
231 	.start  = rt_cache_seq_start,
232 	.next   = rt_cache_seq_next,
233 	.stop   = rt_cache_seq_stop,
234 	.show   = rt_cache_seq_show,
235 };
236 
237 static int rt_cache_seq_open(struct inode *inode, struct file *file)
238 {
239 	return seq_open(file, &rt_cache_seq_ops);
240 }
241 
242 static const struct file_operations rt_cache_seq_fops = {
243 	.owner	 = THIS_MODULE,
244 	.open	 = rt_cache_seq_open,
245 	.read	 = seq_read,
246 	.llseek	 = seq_lseek,
247 	.release = seq_release,
248 };
249 
250 
251 static void *rt_cpu_seq_start(struct seq_file *seq, loff_t *pos)
252 {
253 	int cpu;
254 
255 	if (*pos == 0)
256 		return SEQ_START_TOKEN;
257 
258 	for (cpu = *pos-1; cpu < nr_cpu_ids; ++cpu) {
259 		if (!cpu_possible(cpu))
260 			continue;
261 		*pos = cpu+1;
262 		return &per_cpu(rt_cache_stat, cpu);
263 	}
264 	return NULL;
265 }
266 
267 static void *rt_cpu_seq_next(struct seq_file *seq, void *v, loff_t *pos)
268 {
269 	int cpu;
270 
271 	for (cpu = *pos; cpu < nr_cpu_ids; ++cpu) {
272 		if (!cpu_possible(cpu))
273 			continue;
274 		*pos = cpu+1;
275 		return &per_cpu(rt_cache_stat, cpu);
276 	}
277 	return NULL;
278 
279 }
280 
281 static void rt_cpu_seq_stop(struct seq_file *seq, void *v)
282 {
283 
284 }
285 
286 static int rt_cpu_seq_show(struct seq_file *seq, void *v)
287 {
288 	struct rt_cache_stat *st = v;
289 
290 	if (v == SEQ_START_TOKEN) {
291 		seq_printf(seq, "entries  in_hit in_slow_tot in_slow_mc in_no_route in_brd in_martian_dst in_martian_src  out_hit out_slow_tot out_slow_mc  gc_total gc_ignored gc_goal_miss gc_dst_overflow in_hlist_search out_hlist_search\n");
292 		return 0;
293 	}
294 
295 	seq_printf(seq,"%08x  %08x %08x %08x %08x %08x %08x %08x "
296 		   " %08x %08x %08x %08x %08x %08x %08x %08x %08x \n",
297 		   dst_entries_get_slow(&ipv4_dst_ops),
298 		   0, /* st->in_hit */
299 		   st->in_slow_tot,
300 		   st->in_slow_mc,
301 		   st->in_no_route,
302 		   st->in_brd,
303 		   st->in_martian_dst,
304 		   st->in_martian_src,
305 
306 		   0, /* st->out_hit */
307 		   st->out_slow_tot,
308 		   st->out_slow_mc,
309 
310 		   0, /* st->gc_total */
311 		   0, /* st->gc_ignored */
312 		   0, /* st->gc_goal_miss */
313 		   0, /* st->gc_dst_overflow */
314 		   0, /* st->in_hlist_search */
315 		   0  /* st->out_hlist_search */
316 		);
317 	return 0;
318 }
319 
320 static const struct seq_operations rt_cpu_seq_ops = {
321 	.start  = rt_cpu_seq_start,
322 	.next   = rt_cpu_seq_next,
323 	.stop   = rt_cpu_seq_stop,
324 	.show   = rt_cpu_seq_show,
325 };
326 
327 
328 static int rt_cpu_seq_open(struct inode *inode, struct file *file)
329 {
330 	return seq_open(file, &rt_cpu_seq_ops);
331 }
332 
333 static const struct file_operations rt_cpu_seq_fops = {
334 	.owner	 = THIS_MODULE,
335 	.open	 = rt_cpu_seq_open,
336 	.read	 = seq_read,
337 	.llseek	 = seq_lseek,
338 	.release = seq_release,
339 };
340 
341 #ifdef CONFIG_IP_ROUTE_CLASSID
342 static int rt_acct_proc_show(struct seq_file *m, void *v)
343 {
344 	struct ip_rt_acct *dst, *src;
345 	unsigned int i, j;
346 
347 	dst = kcalloc(256, sizeof(struct ip_rt_acct), GFP_KERNEL);
348 	if (!dst)
349 		return -ENOMEM;
350 
351 	for_each_possible_cpu(i) {
352 		src = (struct ip_rt_acct *)per_cpu_ptr(ip_rt_acct, i);
353 		for (j = 0; j < 256; j++) {
354 			dst[j].o_bytes   += src[j].o_bytes;
355 			dst[j].o_packets += src[j].o_packets;
356 			dst[j].i_bytes   += src[j].i_bytes;
357 			dst[j].i_packets += src[j].i_packets;
358 		}
359 	}
360 
361 	seq_write(m, dst, 256 * sizeof(struct ip_rt_acct));
362 	kfree(dst);
363 	return 0;
364 }
365 
366 static int rt_acct_proc_open(struct inode *inode, struct file *file)
367 {
368 	return single_open(file, rt_acct_proc_show, NULL);
369 }
370 
371 static const struct file_operations rt_acct_proc_fops = {
372 	.owner		= THIS_MODULE,
373 	.open		= rt_acct_proc_open,
374 	.read		= seq_read,
375 	.llseek		= seq_lseek,
376 	.release	= single_release,
377 };
378 #endif
379 
380 static int __net_init ip_rt_do_proc_init(struct net *net)
381 {
382 	struct proc_dir_entry *pde;
383 
384 	pde = proc_create("rt_cache", S_IRUGO, net->proc_net,
385 			  &rt_cache_seq_fops);
386 	if (!pde)
387 		goto err1;
388 
389 	pde = proc_create("rt_cache", S_IRUGO,
390 			  net->proc_net_stat, &rt_cpu_seq_fops);
391 	if (!pde)
392 		goto err2;
393 
394 #ifdef CONFIG_IP_ROUTE_CLASSID
395 	pde = proc_create("rt_acct", 0, net->proc_net, &rt_acct_proc_fops);
396 	if (!pde)
397 		goto err3;
398 #endif
399 	return 0;
400 
401 #ifdef CONFIG_IP_ROUTE_CLASSID
402 err3:
403 	remove_proc_entry("rt_cache", net->proc_net_stat);
404 #endif
405 err2:
406 	remove_proc_entry("rt_cache", net->proc_net);
407 err1:
408 	return -ENOMEM;
409 }
410 
411 static void __net_exit ip_rt_do_proc_exit(struct net *net)
412 {
413 	remove_proc_entry("rt_cache", net->proc_net_stat);
414 	remove_proc_entry("rt_cache", net->proc_net);
415 #ifdef CONFIG_IP_ROUTE_CLASSID
416 	remove_proc_entry("rt_acct", net->proc_net);
417 #endif
418 }
419 
420 static struct pernet_operations ip_rt_proc_ops __net_initdata =  {
421 	.init = ip_rt_do_proc_init,
422 	.exit = ip_rt_do_proc_exit,
423 };
424 
425 static int __init ip_rt_proc_init(void)
426 {
427 	return register_pernet_subsys(&ip_rt_proc_ops);
428 }
429 
430 #else
431 static inline int ip_rt_proc_init(void)
432 {
433 	return 0;
434 }
435 #endif /* CONFIG_PROC_FS */
436 
437 static inline bool rt_is_expired(const struct rtable *rth)
438 {
439 	return rth->rt_genid != rt_genid_ipv4(dev_net(rth->dst.dev));
440 }
441 
442 void rt_cache_flush(struct net *net)
443 {
444 	rt_genid_bump_ipv4(net);
445 }
446 
447 static struct neighbour *ipv4_neigh_lookup(const struct dst_entry *dst,
448 					   struct sk_buff *skb,
449 					   const void *daddr)
450 {
451 	struct net_device *dev = dst->dev;
452 	const __be32 *pkey = daddr;
453 	const struct rtable *rt;
454 	struct neighbour *n;
455 
456 	rt = (const struct rtable *) dst;
457 	if (rt->rt_gateway)
458 		pkey = (const __be32 *) &rt->rt_gateway;
459 	else if (skb)
460 		pkey = &ip_hdr(skb)->daddr;
461 
462 	n = __ipv4_neigh_lookup(dev, *(__force u32 *)pkey);
463 	if (n)
464 		return n;
465 	return neigh_create(&arp_tbl, pkey, dev);
466 }
467 
468 static void ipv4_confirm_neigh(const struct dst_entry *dst, const void *daddr)
469 {
470 	struct net_device *dev = dst->dev;
471 	const __be32 *pkey = daddr;
472 	const struct rtable *rt;
473 
474 	rt = (const struct rtable *)dst;
475 	if (rt->rt_gateway)
476 		pkey = (const __be32 *)&rt->rt_gateway;
477 	else if (!daddr ||
478 		 (rt->rt_flags &
479 		  (RTCF_MULTICAST | RTCF_BROADCAST | RTCF_LOCAL)))
480 		return;
481 
482 	__ipv4_confirm_neigh(dev, *(__force u32 *)pkey);
483 }
484 
485 #define IP_IDENTS_SZ 2048u
486 
487 static atomic_t *ip_idents __read_mostly;
488 static u32 *ip_tstamps __read_mostly;
489 
490 /* In order to protect privacy, we add a perturbation to identifiers
491  * if one generator is seldom used. This makes hard for an attacker
492  * to infer how many packets were sent between two points in time.
493  */
494 u32 ip_idents_reserve(u32 hash, int segs)
495 {
496 	u32 *p_tstamp = ip_tstamps + hash % IP_IDENTS_SZ;
497 	atomic_t *p_id = ip_idents + hash % IP_IDENTS_SZ;
498 	u32 old = READ_ONCE(*p_tstamp);
499 	u32 now = (u32)jiffies;
500 	u32 new, delta = 0;
501 
502 	if (old != now && cmpxchg(p_tstamp, old, now) == old)
503 		delta = prandom_u32_max(now - old);
504 
505 	/* Do not use atomic_add_return() as it makes UBSAN unhappy */
506 	do {
507 		old = (u32)atomic_read(p_id);
508 		new = old + delta + segs;
509 	} while (atomic_cmpxchg(p_id, old, new) != old);
510 
511 	return new - segs;
512 }
513 EXPORT_SYMBOL(ip_idents_reserve);
514 
515 void __ip_select_ident(struct net *net, struct iphdr *iph, int segs)
516 {
517 	static u32 ip_idents_hashrnd __read_mostly;
518 	u32 hash, id;
519 
520 	net_get_random_once(&ip_idents_hashrnd, sizeof(ip_idents_hashrnd));
521 
522 	hash = jhash_3words((__force u32)iph->daddr,
523 			    (__force u32)iph->saddr,
524 			    iph->protocol ^ net_hash_mix(net),
525 			    ip_idents_hashrnd);
526 	id = ip_idents_reserve(hash, segs);
527 	iph->id = htons(id);
528 }
529 EXPORT_SYMBOL(__ip_select_ident);
530 
531 static void __build_flow_key(const struct net *net, struct flowi4 *fl4,
532 			     const struct sock *sk,
533 			     const struct iphdr *iph,
534 			     int oif, u8 tos,
535 			     u8 prot, u32 mark, int flow_flags)
536 {
537 	if (sk) {
538 		const struct inet_sock *inet = inet_sk(sk);
539 
540 		oif = sk->sk_bound_dev_if;
541 		mark = sk->sk_mark;
542 		tos = RT_CONN_FLAGS(sk);
543 		prot = inet->hdrincl ? IPPROTO_RAW : sk->sk_protocol;
544 	}
545 	flowi4_init_output(fl4, oif, mark, tos,
546 			   RT_SCOPE_UNIVERSE, prot,
547 			   flow_flags,
548 			   iph->daddr, iph->saddr, 0, 0,
549 			   sock_net_uid(net, sk));
550 }
551 
552 static void build_skb_flow_key(struct flowi4 *fl4, const struct sk_buff *skb,
553 			       const struct sock *sk)
554 {
555 	const struct net *net = dev_net(skb->dev);
556 	const struct iphdr *iph = ip_hdr(skb);
557 	int oif = skb->dev->ifindex;
558 	u8 tos = RT_TOS(iph->tos);
559 	u8 prot = iph->protocol;
560 	u32 mark = skb->mark;
561 
562 	__build_flow_key(net, fl4, sk, iph, oif, tos, prot, mark, 0);
563 }
564 
565 static void build_sk_flow_key(struct flowi4 *fl4, const struct sock *sk)
566 {
567 	const struct inet_sock *inet = inet_sk(sk);
568 	const struct ip_options_rcu *inet_opt;
569 	__be32 daddr = inet->inet_daddr;
570 
571 	rcu_read_lock();
572 	inet_opt = rcu_dereference(inet->inet_opt);
573 	if (inet_opt && inet_opt->opt.srr)
574 		daddr = inet_opt->opt.faddr;
575 	flowi4_init_output(fl4, sk->sk_bound_dev_if, sk->sk_mark,
576 			   RT_CONN_FLAGS(sk), RT_SCOPE_UNIVERSE,
577 			   inet->hdrincl ? IPPROTO_RAW : sk->sk_protocol,
578 			   inet_sk_flowi_flags(sk),
579 			   daddr, inet->inet_saddr, 0, 0, sk->sk_uid);
580 	rcu_read_unlock();
581 }
582 
583 static void ip_rt_build_flow_key(struct flowi4 *fl4, const struct sock *sk,
584 				 const struct sk_buff *skb)
585 {
586 	if (skb)
587 		build_skb_flow_key(fl4, skb, sk);
588 	else
589 		build_sk_flow_key(fl4, sk);
590 }
591 
592 static DEFINE_SPINLOCK(fnhe_lock);
593 
594 static void fnhe_flush_routes(struct fib_nh_exception *fnhe)
595 {
596 	struct rtable *rt;
597 
598 	rt = rcu_dereference(fnhe->fnhe_rth_input);
599 	if (rt) {
600 		RCU_INIT_POINTER(fnhe->fnhe_rth_input, NULL);
601 		dst_dev_put(&rt->dst);
602 		dst_release(&rt->dst);
603 	}
604 	rt = rcu_dereference(fnhe->fnhe_rth_output);
605 	if (rt) {
606 		RCU_INIT_POINTER(fnhe->fnhe_rth_output, NULL);
607 		dst_dev_put(&rt->dst);
608 		dst_release(&rt->dst);
609 	}
610 }
611 
612 static struct fib_nh_exception *fnhe_oldest(struct fnhe_hash_bucket *hash)
613 {
614 	struct fib_nh_exception *fnhe, *oldest;
615 
616 	oldest = rcu_dereference(hash->chain);
617 	for (fnhe = rcu_dereference(oldest->fnhe_next); fnhe;
618 	     fnhe = rcu_dereference(fnhe->fnhe_next)) {
619 		if (time_before(fnhe->fnhe_stamp, oldest->fnhe_stamp))
620 			oldest = fnhe;
621 	}
622 	fnhe_flush_routes(oldest);
623 	return oldest;
624 }
625 
626 static inline u32 fnhe_hashfun(__be32 daddr)
627 {
628 	static u32 fnhe_hashrnd __read_mostly;
629 	u32 hval;
630 
631 	net_get_random_once(&fnhe_hashrnd, sizeof(fnhe_hashrnd));
632 	hval = jhash_1word((__force u32) daddr, fnhe_hashrnd);
633 	return hash_32(hval, FNHE_HASH_SHIFT);
634 }
635 
636 static void fill_route_from_fnhe(struct rtable *rt, struct fib_nh_exception *fnhe)
637 {
638 	rt->rt_pmtu = fnhe->fnhe_pmtu;
639 	rt->dst.expires = fnhe->fnhe_expires;
640 
641 	if (fnhe->fnhe_gw) {
642 		rt->rt_flags |= RTCF_REDIRECTED;
643 		rt->rt_gateway = fnhe->fnhe_gw;
644 		rt->rt_uses_gateway = 1;
645 	}
646 }
647 
648 static void update_or_create_fnhe(struct fib_nh *nh, __be32 daddr, __be32 gw,
649 				  u32 pmtu, unsigned long expires)
650 {
651 	struct fnhe_hash_bucket *hash;
652 	struct fib_nh_exception *fnhe;
653 	struct rtable *rt;
654 	unsigned int i;
655 	int depth;
656 	u32 hval = fnhe_hashfun(daddr);
657 
658 	spin_lock_bh(&fnhe_lock);
659 
660 	hash = rcu_dereference(nh->nh_exceptions);
661 	if (!hash) {
662 		hash = kzalloc(FNHE_HASH_SIZE * sizeof(*hash), GFP_ATOMIC);
663 		if (!hash)
664 			goto out_unlock;
665 		rcu_assign_pointer(nh->nh_exceptions, hash);
666 	}
667 
668 	hash += hval;
669 
670 	depth = 0;
671 	for (fnhe = rcu_dereference(hash->chain); fnhe;
672 	     fnhe = rcu_dereference(fnhe->fnhe_next)) {
673 		if (fnhe->fnhe_daddr == daddr)
674 			break;
675 		depth++;
676 	}
677 
678 	if (fnhe) {
679 		if (gw)
680 			fnhe->fnhe_gw = gw;
681 		if (pmtu) {
682 			fnhe->fnhe_pmtu = pmtu;
683 			fnhe->fnhe_expires = max(1UL, expires);
684 		}
685 		/* Update all cached dsts too */
686 		rt = rcu_dereference(fnhe->fnhe_rth_input);
687 		if (rt)
688 			fill_route_from_fnhe(rt, fnhe);
689 		rt = rcu_dereference(fnhe->fnhe_rth_output);
690 		if (rt)
691 			fill_route_from_fnhe(rt, fnhe);
692 	} else {
693 		if (depth > FNHE_RECLAIM_DEPTH)
694 			fnhe = fnhe_oldest(hash);
695 		else {
696 			fnhe = kzalloc(sizeof(*fnhe), GFP_ATOMIC);
697 			if (!fnhe)
698 				goto out_unlock;
699 
700 			fnhe->fnhe_next = hash->chain;
701 			rcu_assign_pointer(hash->chain, fnhe);
702 		}
703 		fnhe->fnhe_genid = fnhe_genid(dev_net(nh->nh_dev));
704 		fnhe->fnhe_daddr = daddr;
705 		fnhe->fnhe_gw = gw;
706 		fnhe->fnhe_pmtu = pmtu;
707 		fnhe->fnhe_expires = expires;
708 
709 		/* Exception created; mark the cached routes for the nexthop
710 		 * stale, so anyone caching it rechecks if this exception
711 		 * applies to them.
712 		 */
713 		rt = rcu_dereference(nh->nh_rth_input);
714 		if (rt)
715 			rt->dst.obsolete = DST_OBSOLETE_KILL;
716 
717 		for_each_possible_cpu(i) {
718 			struct rtable __rcu **prt;
719 			prt = per_cpu_ptr(nh->nh_pcpu_rth_output, i);
720 			rt = rcu_dereference(*prt);
721 			if (rt)
722 				rt->dst.obsolete = DST_OBSOLETE_KILL;
723 		}
724 	}
725 
726 	fnhe->fnhe_stamp = jiffies;
727 
728 out_unlock:
729 	spin_unlock_bh(&fnhe_lock);
730 }
731 
732 static void __ip_do_redirect(struct rtable *rt, struct sk_buff *skb, struct flowi4 *fl4,
733 			     bool kill_route)
734 {
735 	__be32 new_gw = icmp_hdr(skb)->un.gateway;
736 	__be32 old_gw = ip_hdr(skb)->saddr;
737 	struct net_device *dev = skb->dev;
738 	struct in_device *in_dev;
739 	struct fib_result res;
740 	struct neighbour *n;
741 	struct net *net;
742 
743 	switch (icmp_hdr(skb)->code & 7) {
744 	case ICMP_REDIR_NET:
745 	case ICMP_REDIR_NETTOS:
746 	case ICMP_REDIR_HOST:
747 	case ICMP_REDIR_HOSTTOS:
748 		break;
749 
750 	default:
751 		return;
752 	}
753 
754 	if (rt->rt_gateway != old_gw)
755 		return;
756 
757 	in_dev = __in_dev_get_rcu(dev);
758 	if (!in_dev)
759 		return;
760 
761 	net = dev_net(dev);
762 	if (new_gw == old_gw || !IN_DEV_RX_REDIRECTS(in_dev) ||
763 	    ipv4_is_multicast(new_gw) || ipv4_is_lbcast(new_gw) ||
764 	    ipv4_is_zeronet(new_gw))
765 		goto reject_redirect;
766 
767 	if (!IN_DEV_SHARED_MEDIA(in_dev)) {
768 		if (!inet_addr_onlink(in_dev, new_gw, old_gw))
769 			goto reject_redirect;
770 		if (IN_DEV_SEC_REDIRECTS(in_dev) && ip_fib_check_default(new_gw, dev))
771 			goto reject_redirect;
772 	} else {
773 		if (inet_addr_type(net, new_gw) != RTN_UNICAST)
774 			goto reject_redirect;
775 	}
776 
777 	n = __ipv4_neigh_lookup(rt->dst.dev, new_gw);
778 	if (!n)
779 		n = neigh_create(&arp_tbl, &new_gw, rt->dst.dev);
780 	if (!IS_ERR(n)) {
781 		if (!(n->nud_state & NUD_VALID)) {
782 			neigh_event_send(n, NULL);
783 		} else {
784 			if (fib_lookup(net, fl4, &res, 0) == 0) {
785 				struct fib_nh *nh = &FIB_RES_NH(res);
786 
787 				update_or_create_fnhe(nh, fl4->daddr, new_gw,
788 						0, jiffies + ip_rt_gc_timeout);
789 			}
790 			if (kill_route)
791 				rt->dst.obsolete = DST_OBSOLETE_KILL;
792 			call_netevent_notifiers(NETEVENT_NEIGH_UPDATE, n);
793 		}
794 		neigh_release(n);
795 	}
796 	return;
797 
798 reject_redirect:
799 #ifdef CONFIG_IP_ROUTE_VERBOSE
800 	if (IN_DEV_LOG_MARTIANS(in_dev)) {
801 		const struct iphdr *iph = (const struct iphdr *) skb->data;
802 		__be32 daddr = iph->daddr;
803 		__be32 saddr = iph->saddr;
804 
805 		net_info_ratelimited("Redirect from %pI4 on %s about %pI4 ignored\n"
806 				     "  Advised path = %pI4 -> %pI4\n",
807 				     &old_gw, dev->name, &new_gw,
808 				     &saddr, &daddr);
809 	}
810 #endif
811 	;
812 }
813 
814 static void ip_do_redirect(struct dst_entry *dst, struct sock *sk, struct sk_buff *skb)
815 {
816 	struct rtable *rt;
817 	struct flowi4 fl4;
818 	const struct iphdr *iph = (const struct iphdr *) skb->data;
819 	struct net *net = dev_net(skb->dev);
820 	int oif = skb->dev->ifindex;
821 	u8 tos = RT_TOS(iph->tos);
822 	u8 prot = iph->protocol;
823 	u32 mark = skb->mark;
824 
825 	rt = (struct rtable *) dst;
826 
827 	__build_flow_key(net, &fl4, sk, iph, oif, tos, prot, mark, 0);
828 	__ip_do_redirect(rt, skb, &fl4, true);
829 }
830 
831 static struct dst_entry *ipv4_negative_advice(struct dst_entry *dst)
832 {
833 	struct rtable *rt = (struct rtable *)dst;
834 	struct dst_entry *ret = dst;
835 
836 	if (rt) {
837 		if (dst->obsolete > 0) {
838 			ip_rt_put(rt);
839 			ret = NULL;
840 		} else if ((rt->rt_flags & RTCF_REDIRECTED) ||
841 			   rt->dst.expires) {
842 			ip_rt_put(rt);
843 			ret = NULL;
844 		}
845 	}
846 	return ret;
847 }
848 
849 /*
850  * Algorithm:
851  *	1. The first ip_rt_redirect_number redirects are sent
852  *	   with exponential backoff, then we stop sending them at all,
853  *	   assuming that the host ignores our redirects.
854  *	2. If we did not see packets requiring redirects
855  *	   during ip_rt_redirect_silence, we assume that the host
856  *	   forgot redirected route and start to send redirects again.
857  *
858  * This algorithm is much cheaper and more intelligent than dumb load limiting
859  * in icmp.c.
860  *
861  * NOTE. Do not forget to inhibit load limiting for redirects (redundant)
862  * and "frag. need" (breaks PMTU discovery) in icmp.c.
863  */
864 
865 void ip_rt_send_redirect(struct sk_buff *skb)
866 {
867 	struct rtable *rt = skb_rtable(skb);
868 	struct in_device *in_dev;
869 	struct inet_peer *peer;
870 	struct net *net;
871 	int log_martians;
872 	int vif;
873 
874 	rcu_read_lock();
875 	in_dev = __in_dev_get_rcu(rt->dst.dev);
876 	if (!in_dev || !IN_DEV_TX_REDIRECTS(in_dev)) {
877 		rcu_read_unlock();
878 		return;
879 	}
880 	log_martians = IN_DEV_LOG_MARTIANS(in_dev);
881 	vif = l3mdev_master_ifindex_rcu(rt->dst.dev);
882 	rcu_read_unlock();
883 
884 	net = dev_net(rt->dst.dev);
885 	peer = inet_getpeer_v4(net->ipv4.peers, ip_hdr(skb)->saddr, vif, 1);
886 	if (!peer) {
887 		icmp_send(skb, ICMP_REDIRECT, ICMP_REDIR_HOST,
888 			  rt_nexthop(rt, ip_hdr(skb)->daddr));
889 		return;
890 	}
891 
892 	/* No redirected packets during ip_rt_redirect_silence;
893 	 * reset the algorithm.
894 	 */
895 	if (time_after(jiffies, peer->rate_last + ip_rt_redirect_silence))
896 		peer->rate_tokens = 0;
897 
898 	/* Too many ignored redirects; do not send anything
899 	 * set dst.rate_last to the last seen redirected packet.
900 	 */
901 	if (peer->rate_tokens >= ip_rt_redirect_number) {
902 		peer->rate_last = jiffies;
903 		goto out_put_peer;
904 	}
905 
906 	/* Check for load limit; set rate_last to the latest sent
907 	 * redirect.
908 	 */
909 	if (peer->rate_tokens == 0 ||
910 	    time_after(jiffies,
911 		       (peer->rate_last +
912 			(ip_rt_redirect_load << peer->rate_tokens)))) {
913 		__be32 gw = rt_nexthop(rt, ip_hdr(skb)->daddr);
914 
915 		icmp_send(skb, ICMP_REDIRECT, ICMP_REDIR_HOST, gw);
916 		peer->rate_last = jiffies;
917 		++peer->rate_tokens;
918 #ifdef CONFIG_IP_ROUTE_VERBOSE
919 		if (log_martians &&
920 		    peer->rate_tokens == ip_rt_redirect_number)
921 			net_warn_ratelimited("host %pI4/if%d ignores redirects for %pI4 to %pI4\n",
922 					     &ip_hdr(skb)->saddr, inet_iif(skb),
923 					     &ip_hdr(skb)->daddr, &gw);
924 #endif
925 	}
926 out_put_peer:
927 	inet_putpeer(peer);
928 }
929 
930 static int ip_error(struct sk_buff *skb)
931 {
932 	struct in_device *in_dev = __in_dev_get_rcu(skb->dev);
933 	struct rtable *rt = skb_rtable(skb);
934 	struct inet_peer *peer;
935 	unsigned long now;
936 	struct net *net;
937 	bool send;
938 	int code;
939 
940 	/* IP on this device is disabled. */
941 	if (!in_dev)
942 		goto out;
943 
944 	net = dev_net(rt->dst.dev);
945 	if (!IN_DEV_FORWARD(in_dev)) {
946 		switch (rt->dst.error) {
947 		case EHOSTUNREACH:
948 			__IP_INC_STATS(net, IPSTATS_MIB_INADDRERRORS);
949 			break;
950 
951 		case ENETUNREACH:
952 			__IP_INC_STATS(net, IPSTATS_MIB_INNOROUTES);
953 			break;
954 		}
955 		goto out;
956 	}
957 
958 	switch (rt->dst.error) {
959 	case EINVAL:
960 	default:
961 		goto out;
962 	case EHOSTUNREACH:
963 		code = ICMP_HOST_UNREACH;
964 		break;
965 	case ENETUNREACH:
966 		code = ICMP_NET_UNREACH;
967 		__IP_INC_STATS(net, IPSTATS_MIB_INNOROUTES);
968 		break;
969 	case EACCES:
970 		code = ICMP_PKT_FILTERED;
971 		break;
972 	}
973 
974 	peer = inet_getpeer_v4(net->ipv4.peers, ip_hdr(skb)->saddr,
975 			       l3mdev_master_ifindex(skb->dev), 1);
976 
977 	send = true;
978 	if (peer) {
979 		now = jiffies;
980 		peer->rate_tokens += now - peer->rate_last;
981 		if (peer->rate_tokens > ip_rt_error_burst)
982 			peer->rate_tokens = ip_rt_error_burst;
983 		peer->rate_last = now;
984 		if (peer->rate_tokens >= ip_rt_error_cost)
985 			peer->rate_tokens -= ip_rt_error_cost;
986 		else
987 			send = false;
988 		inet_putpeer(peer);
989 	}
990 	if (send)
991 		icmp_send(skb, ICMP_DEST_UNREACH, code, 0);
992 
993 out:	kfree_skb(skb);
994 	return 0;
995 }
996 
997 static void __ip_rt_update_pmtu(struct rtable *rt, struct flowi4 *fl4, u32 mtu)
998 {
999 	struct dst_entry *dst = &rt->dst;
1000 	struct fib_result res;
1001 
1002 	if (dst_metric_locked(dst, RTAX_MTU))
1003 		return;
1004 
1005 	if (ipv4_mtu(dst) < mtu)
1006 		return;
1007 
1008 	if (mtu < ip_rt_min_pmtu)
1009 		mtu = ip_rt_min_pmtu;
1010 
1011 	if (rt->rt_pmtu == mtu &&
1012 	    time_before(jiffies, dst->expires - ip_rt_mtu_expires / 2))
1013 		return;
1014 
1015 	rcu_read_lock();
1016 	if (fib_lookup(dev_net(dst->dev), fl4, &res, 0) == 0) {
1017 		struct fib_nh *nh = &FIB_RES_NH(res);
1018 
1019 		update_or_create_fnhe(nh, fl4->daddr, 0, mtu,
1020 				      jiffies + ip_rt_mtu_expires);
1021 	}
1022 	rcu_read_unlock();
1023 }
1024 
1025 static void ip_rt_update_pmtu(struct dst_entry *dst, struct sock *sk,
1026 			      struct sk_buff *skb, u32 mtu)
1027 {
1028 	struct rtable *rt = (struct rtable *) dst;
1029 	struct flowi4 fl4;
1030 
1031 	ip_rt_build_flow_key(&fl4, sk, skb);
1032 	__ip_rt_update_pmtu(rt, &fl4, mtu);
1033 }
1034 
1035 void ipv4_update_pmtu(struct sk_buff *skb, struct net *net, u32 mtu,
1036 		      int oif, u32 mark, u8 protocol, int flow_flags)
1037 {
1038 	const struct iphdr *iph = (const struct iphdr *) skb->data;
1039 	struct flowi4 fl4;
1040 	struct rtable *rt;
1041 
1042 	if (!mark)
1043 		mark = IP4_REPLY_MARK(net, skb->mark);
1044 
1045 	__build_flow_key(net, &fl4, NULL, iph, oif,
1046 			 RT_TOS(iph->tos), protocol, mark, flow_flags);
1047 	rt = __ip_route_output_key(net, &fl4);
1048 	if (!IS_ERR(rt)) {
1049 		__ip_rt_update_pmtu(rt, &fl4, mtu);
1050 		ip_rt_put(rt);
1051 	}
1052 }
1053 EXPORT_SYMBOL_GPL(ipv4_update_pmtu);
1054 
1055 static void __ipv4_sk_update_pmtu(struct sk_buff *skb, struct sock *sk, u32 mtu)
1056 {
1057 	const struct iphdr *iph = (const struct iphdr *) skb->data;
1058 	struct flowi4 fl4;
1059 	struct rtable *rt;
1060 
1061 	__build_flow_key(sock_net(sk), &fl4, sk, iph, 0, 0, 0, 0, 0);
1062 
1063 	if (!fl4.flowi4_mark)
1064 		fl4.flowi4_mark = IP4_REPLY_MARK(sock_net(sk), skb->mark);
1065 
1066 	rt = __ip_route_output_key(sock_net(sk), &fl4);
1067 	if (!IS_ERR(rt)) {
1068 		__ip_rt_update_pmtu(rt, &fl4, mtu);
1069 		ip_rt_put(rt);
1070 	}
1071 }
1072 
1073 void ipv4_sk_update_pmtu(struct sk_buff *skb, struct sock *sk, u32 mtu)
1074 {
1075 	const struct iphdr *iph = (const struct iphdr *) skb->data;
1076 	struct flowi4 fl4;
1077 	struct rtable *rt;
1078 	struct dst_entry *odst = NULL;
1079 	bool new = false;
1080 	struct net *net = sock_net(sk);
1081 
1082 	bh_lock_sock(sk);
1083 
1084 	if (!ip_sk_accept_pmtu(sk))
1085 		goto out;
1086 
1087 	odst = sk_dst_get(sk);
1088 
1089 	if (sock_owned_by_user(sk) || !odst) {
1090 		__ipv4_sk_update_pmtu(skb, sk, mtu);
1091 		goto out;
1092 	}
1093 
1094 	__build_flow_key(net, &fl4, sk, iph, 0, 0, 0, 0, 0);
1095 
1096 	rt = (struct rtable *)odst;
1097 	if (odst->obsolete && !odst->ops->check(odst, 0)) {
1098 		rt = ip_route_output_flow(sock_net(sk), &fl4, sk);
1099 		if (IS_ERR(rt))
1100 			goto out;
1101 
1102 		new = true;
1103 	}
1104 
1105 	__ip_rt_update_pmtu((struct rtable *) rt->dst.path, &fl4, mtu);
1106 
1107 	if (!dst_check(&rt->dst, 0)) {
1108 		if (new)
1109 			dst_release(&rt->dst);
1110 
1111 		rt = ip_route_output_flow(sock_net(sk), &fl4, sk);
1112 		if (IS_ERR(rt))
1113 			goto out;
1114 
1115 		new = true;
1116 	}
1117 
1118 	if (new)
1119 		sk_dst_set(sk, &rt->dst);
1120 
1121 out:
1122 	bh_unlock_sock(sk);
1123 	dst_release(odst);
1124 }
1125 EXPORT_SYMBOL_GPL(ipv4_sk_update_pmtu);
1126 
1127 void ipv4_redirect(struct sk_buff *skb, struct net *net,
1128 		   int oif, u32 mark, u8 protocol, int flow_flags)
1129 {
1130 	const struct iphdr *iph = (const struct iphdr *) skb->data;
1131 	struct flowi4 fl4;
1132 	struct rtable *rt;
1133 
1134 	__build_flow_key(net, &fl4, NULL, iph, oif,
1135 			 RT_TOS(iph->tos), protocol, mark, flow_flags);
1136 	rt = __ip_route_output_key(net, &fl4);
1137 	if (!IS_ERR(rt)) {
1138 		__ip_do_redirect(rt, skb, &fl4, false);
1139 		ip_rt_put(rt);
1140 	}
1141 }
1142 EXPORT_SYMBOL_GPL(ipv4_redirect);
1143 
1144 void ipv4_sk_redirect(struct sk_buff *skb, struct sock *sk)
1145 {
1146 	const struct iphdr *iph = (const struct iphdr *) skb->data;
1147 	struct flowi4 fl4;
1148 	struct rtable *rt;
1149 	struct net *net = sock_net(sk);
1150 
1151 	__build_flow_key(net, &fl4, sk, iph, 0, 0, 0, 0, 0);
1152 	rt = __ip_route_output_key(net, &fl4);
1153 	if (!IS_ERR(rt)) {
1154 		__ip_do_redirect(rt, skb, &fl4, false);
1155 		ip_rt_put(rt);
1156 	}
1157 }
1158 EXPORT_SYMBOL_GPL(ipv4_sk_redirect);
1159 
1160 static struct dst_entry *ipv4_dst_check(struct dst_entry *dst, u32 cookie)
1161 {
1162 	struct rtable *rt = (struct rtable *) dst;
1163 
1164 	/* All IPV4 dsts are created with ->obsolete set to the value
1165 	 * DST_OBSOLETE_FORCE_CHK which forces validation calls down
1166 	 * into this function always.
1167 	 *
1168 	 * When a PMTU/redirect information update invalidates a route,
1169 	 * this is indicated by setting obsolete to DST_OBSOLETE_KILL or
1170 	 * DST_OBSOLETE_DEAD by dst_free().
1171 	 */
1172 	if (dst->obsolete != DST_OBSOLETE_FORCE_CHK || rt_is_expired(rt))
1173 		return NULL;
1174 	return dst;
1175 }
1176 
1177 static void ipv4_link_failure(struct sk_buff *skb)
1178 {
1179 	struct rtable *rt;
1180 
1181 	icmp_send(skb, ICMP_DEST_UNREACH, ICMP_HOST_UNREACH, 0);
1182 
1183 	rt = skb_rtable(skb);
1184 	if (rt)
1185 		dst_set_expires(&rt->dst, 0);
1186 }
1187 
1188 static int ip_rt_bug(struct net *net, struct sock *sk, struct sk_buff *skb)
1189 {
1190 	pr_debug("%s: %pI4 -> %pI4, %s\n",
1191 		 __func__, &ip_hdr(skb)->saddr, &ip_hdr(skb)->daddr,
1192 		 skb->dev ? skb->dev->name : "?");
1193 	kfree_skb(skb);
1194 	WARN_ON(1);
1195 	return 0;
1196 }
1197 
1198 /*
1199    We do not cache source address of outgoing interface,
1200    because it is used only by IP RR, TS and SRR options,
1201    so that it out of fast path.
1202 
1203    BTW remember: "addr" is allowed to be not aligned
1204    in IP options!
1205  */
1206 
1207 void ip_rt_get_source(u8 *addr, struct sk_buff *skb, struct rtable *rt)
1208 {
1209 	__be32 src;
1210 
1211 	if (rt_is_output_route(rt))
1212 		src = ip_hdr(skb)->saddr;
1213 	else {
1214 		struct fib_result res;
1215 		struct flowi4 fl4;
1216 		struct iphdr *iph;
1217 
1218 		iph = ip_hdr(skb);
1219 
1220 		memset(&fl4, 0, sizeof(fl4));
1221 		fl4.daddr = iph->daddr;
1222 		fl4.saddr = iph->saddr;
1223 		fl4.flowi4_tos = RT_TOS(iph->tos);
1224 		fl4.flowi4_oif = rt->dst.dev->ifindex;
1225 		fl4.flowi4_iif = skb->dev->ifindex;
1226 		fl4.flowi4_mark = skb->mark;
1227 
1228 		rcu_read_lock();
1229 		if (fib_lookup(dev_net(rt->dst.dev), &fl4, &res, 0) == 0)
1230 			src = FIB_RES_PREFSRC(dev_net(rt->dst.dev), res);
1231 		else
1232 			src = inet_select_addr(rt->dst.dev,
1233 					       rt_nexthop(rt, iph->daddr),
1234 					       RT_SCOPE_UNIVERSE);
1235 		rcu_read_unlock();
1236 	}
1237 	memcpy(addr, &src, 4);
1238 }
1239 
1240 #ifdef CONFIG_IP_ROUTE_CLASSID
1241 static void set_class_tag(struct rtable *rt, u32 tag)
1242 {
1243 	if (!(rt->dst.tclassid & 0xFFFF))
1244 		rt->dst.tclassid |= tag & 0xFFFF;
1245 	if (!(rt->dst.tclassid & 0xFFFF0000))
1246 		rt->dst.tclassid |= tag & 0xFFFF0000;
1247 }
1248 #endif
1249 
1250 static unsigned int ipv4_default_advmss(const struct dst_entry *dst)
1251 {
1252 	unsigned int header_size = sizeof(struct tcphdr) + sizeof(struct iphdr);
1253 	unsigned int advmss = max_t(unsigned int, dst->dev->mtu - header_size,
1254 				    ip_rt_min_advmss);
1255 
1256 	return min(advmss, IPV4_MAX_PMTU - header_size);
1257 }
1258 
1259 static unsigned int ipv4_mtu(const struct dst_entry *dst)
1260 {
1261 	const struct rtable *rt = (const struct rtable *) dst;
1262 	unsigned int mtu = rt->rt_pmtu;
1263 
1264 	if (!mtu || time_after_eq(jiffies, rt->dst.expires))
1265 		mtu = dst_metric_raw(dst, RTAX_MTU);
1266 
1267 	if (mtu)
1268 		return mtu;
1269 
1270 	mtu = READ_ONCE(dst->dev->mtu);
1271 
1272 	if (unlikely(dst_metric_locked(dst, RTAX_MTU))) {
1273 		if (rt->rt_uses_gateway && mtu > 576)
1274 			mtu = 576;
1275 	}
1276 
1277 	mtu = min_t(unsigned int, mtu, IP_MAX_MTU);
1278 
1279 	return mtu - lwtunnel_headroom(dst->lwtstate, mtu);
1280 }
1281 
1282 static struct fib_nh_exception *find_exception(struct fib_nh *nh, __be32 daddr)
1283 {
1284 	struct fnhe_hash_bucket *hash = rcu_dereference(nh->nh_exceptions);
1285 	struct fib_nh_exception *fnhe;
1286 	u32 hval;
1287 
1288 	if (!hash)
1289 		return NULL;
1290 
1291 	hval = fnhe_hashfun(daddr);
1292 
1293 	for (fnhe = rcu_dereference(hash[hval].chain); fnhe;
1294 	     fnhe = rcu_dereference(fnhe->fnhe_next)) {
1295 		if (fnhe->fnhe_daddr == daddr)
1296 			return fnhe;
1297 	}
1298 	return NULL;
1299 }
1300 
1301 static bool rt_bind_exception(struct rtable *rt, struct fib_nh_exception *fnhe,
1302 			      __be32 daddr, const bool do_cache)
1303 {
1304 	bool ret = false;
1305 
1306 	spin_lock_bh(&fnhe_lock);
1307 
1308 	if (daddr == fnhe->fnhe_daddr) {
1309 		struct rtable __rcu **porig;
1310 		struct rtable *orig;
1311 		int genid = fnhe_genid(dev_net(rt->dst.dev));
1312 
1313 		if (rt_is_input_route(rt))
1314 			porig = &fnhe->fnhe_rth_input;
1315 		else
1316 			porig = &fnhe->fnhe_rth_output;
1317 		orig = rcu_dereference(*porig);
1318 
1319 		if (fnhe->fnhe_genid != genid) {
1320 			fnhe->fnhe_genid = genid;
1321 			fnhe->fnhe_gw = 0;
1322 			fnhe->fnhe_pmtu = 0;
1323 			fnhe->fnhe_expires = 0;
1324 			fnhe_flush_routes(fnhe);
1325 			orig = NULL;
1326 		}
1327 		fill_route_from_fnhe(rt, fnhe);
1328 		if (!rt->rt_gateway)
1329 			rt->rt_gateway = daddr;
1330 
1331 		if (do_cache) {
1332 			dst_hold(&rt->dst);
1333 			rcu_assign_pointer(*porig, rt);
1334 			if (orig) {
1335 				dst_dev_put(&orig->dst);
1336 				dst_release(&orig->dst);
1337 			}
1338 			ret = true;
1339 		}
1340 
1341 		fnhe->fnhe_stamp = jiffies;
1342 	}
1343 	spin_unlock_bh(&fnhe_lock);
1344 
1345 	return ret;
1346 }
1347 
1348 static bool rt_cache_route(struct fib_nh *nh, struct rtable *rt)
1349 {
1350 	struct rtable *orig, *prev, **p;
1351 	bool ret = true;
1352 
1353 	if (rt_is_input_route(rt)) {
1354 		p = (struct rtable **)&nh->nh_rth_input;
1355 	} else {
1356 		p = (struct rtable **)raw_cpu_ptr(nh->nh_pcpu_rth_output);
1357 	}
1358 	orig = *p;
1359 
1360 	/* hold dst before doing cmpxchg() to avoid race condition
1361 	 * on this dst
1362 	 */
1363 	dst_hold(&rt->dst);
1364 	prev = cmpxchg(p, orig, rt);
1365 	if (prev == orig) {
1366 		if (orig) {
1367 			dst_dev_put(&orig->dst);
1368 			dst_release(&orig->dst);
1369 		}
1370 	} else {
1371 		dst_release(&rt->dst);
1372 		ret = false;
1373 	}
1374 
1375 	return ret;
1376 }
1377 
1378 struct uncached_list {
1379 	spinlock_t		lock;
1380 	struct list_head	head;
1381 };
1382 
1383 static DEFINE_PER_CPU_ALIGNED(struct uncached_list, rt_uncached_list);
1384 
1385 static void rt_add_uncached_list(struct rtable *rt)
1386 {
1387 	struct uncached_list *ul = raw_cpu_ptr(&rt_uncached_list);
1388 
1389 	rt->rt_uncached_list = ul;
1390 
1391 	spin_lock_bh(&ul->lock);
1392 	list_add_tail(&rt->rt_uncached, &ul->head);
1393 	spin_unlock_bh(&ul->lock);
1394 }
1395 
1396 static void ipv4_dst_destroy(struct dst_entry *dst)
1397 {
1398 	struct dst_metrics *p = (struct dst_metrics *)DST_METRICS_PTR(dst);
1399 	struct rtable *rt = (struct rtable *) dst;
1400 
1401 	if (p != &dst_default_metrics && refcount_dec_and_test(&p->refcnt))
1402 		kfree(p);
1403 
1404 	if (!list_empty(&rt->rt_uncached)) {
1405 		struct uncached_list *ul = rt->rt_uncached_list;
1406 
1407 		spin_lock_bh(&ul->lock);
1408 		list_del(&rt->rt_uncached);
1409 		spin_unlock_bh(&ul->lock);
1410 	}
1411 }
1412 
1413 void rt_flush_dev(struct net_device *dev)
1414 {
1415 	struct net *net = dev_net(dev);
1416 	struct rtable *rt;
1417 	int cpu;
1418 
1419 	for_each_possible_cpu(cpu) {
1420 		struct uncached_list *ul = &per_cpu(rt_uncached_list, cpu);
1421 
1422 		spin_lock_bh(&ul->lock);
1423 		list_for_each_entry(rt, &ul->head, rt_uncached) {
1424 			if (rt->dst.dev != dev)
1425 				continue;
1426 			rt->dst.dev = net->loopback_dev;
1427 			dev_hold(rt->dst.dev);
1428 			dev_put(dev);
1429 		}
1430 		spin_unlock_bh(&ul->lock);
1431 	}
1432 }
1433 
1434 static bool rt_cache_valid(const struct rtable *rt)
1435 {
1436 	return	rt &&
1437 		rt->dst.obsolete == DST_OBSOLETE_FORCE_CHK &&
1438 		!rt_is_expired(rt);
1439 }
1440 
1441 static void rt_set_nexthop(struct rtable *rt, __be32 daddr,
1442 			   const struct fib_result *res,
1443 			   struct fib_nh_exception *fnhe,
1444 			   struct fib_info *fi, u16 type, u32 itag,
1445 			   const bool do_cache)
1446 {
1447 	bool cached = false;
1448 
1449 	if (fi) {
1450 		struct fib_nh *nh = &FIB_RES_NH(*res);
1451 
1452 		if (nh->nh_gw && nh->nh_scope == RT_SCOPE_LINK) {
1453 			rt->rt_gateway = nh->nh_gw;
1454 			rt->rt_uses_gateway = 1;
1455 		}
1456 		dst_init_metrics(&rt->dst, fi->fib_metrics->metrics, true);
1457 		if (fi->fib_metrics != &dst_default_metrics) {
1458 			rt->dst._metrics |= DST_METRICS_REFCOUNTED;
1459 			refcount_inc(&fi->fib_metrics->refcnt);
1460 		}
1461 #ifdef CONFIG_IP_ROUTE_CLASSID
1462 		rt->dst.tclassid = nh->nh_tclassid;
1463 #endif
1464 		rt->dst.lwtstate = lwtstate_get(nh->nh_lwtstate);
1465 		if (unlikely(fnhe))
1466 			cached = rt_bind_exception(rt, fnhe, daddr, do_cache);
1467 		else if (do_cache)
1468 			cached = rt_cache_route(nh, rt);
1469 		if (unlikely(!cached)) {
1470 			/* Routes we intend to cache in nexthop exception or
1471 			 * FIB nexthop have the DST_NOCACHE bit clear.
1472 			 * However, if we are unsuccessful at storing this
1473 			 * route into the cache we really need to set it.
1474 			 */
1475 			if (!rt->rt_gateway)
1476 				rt->rt_gateway = daddr;
1477 			rt_add_uncached_list(rt);
1478 		}
1479 	} else
1480 		rt_add_uncached_list(rt);
1481 
1482 #ifdef CONFIG_IP_ROUTE_CLASSID
1483 #ifdef CONFIG_IP_MULTIPLE_TABLES
1484 	set_class_tag(rt, res->tclassid);
1485 #endif
1486 	set_class_tag(rt, itag);
1487 #endif
1488 }
1489 
1490 struct rtable *rt_dst_alloc(struct net_device *dev,
1491 			    unsigned int flags, u16 type,
1492 			    bool nopolicy, bool noxfrm, bool will_cache)
1493 {
1494 	struct rtable *rt;
1495 
1496 	rt = dst_alloc(&ipv4_dst_ops, dev, 1, DST_OBSOLETE_FORCE_CHK,
1497 		       (will_cache ? 0 : DST_HOST) |
1498 		       (nopolicy ? DST_NOPOLICY : 0) |
1499 		       (noxfrm ? DST_NOXFRM : 0));
1500 
1501 	if (rt) {
1502 		rt->rt_genid = rt_genid_ipv4(dev_net(dev));
1503 		rt->rt_flags = flags;
1504 		rt->rt_type = type;
1505 		rt->rt_is_input = 0;
1506 		rt->rt_iif = 0;
1507 		rt->rt_pmtu = 0;
1508 		rt->rt_gateway = 0;
1509 		rt->rt_uses_gateway = 0;
1510 		rt->rt_table_id = 0;
1511 		INIT_LIST_HEAD(&rt->rt_uncached);
1512 
1513 		rt->dst.output = ip_output;
1514 		if (flags & RTCF_LOCAL)
1515 			rt->dst.input = ip_local_deliver;
1516 	}
1517 
1518 	return rt;
1519 }
1520 EXPORT_SYMBOL(rt_dst_alloc);
1521 
1522 /* called in rcu_read_lock() section */
1523 int ip_mc_validate_source(struct sk_buff *skb, __be32 daddr, __be32 saddr,
1524 			  u8 tos, struct net_device *dev,
1525 			  struct in_device *in_dev, u32 *itag)
1526 {
1527 	int err;
1528 
1529 	/* Primary sanity checks. */
1530 	if (!in_dev)
1531 		return -EINVAL;
1532 
1533 	if (ipv4_is_multicast(saddr) || ipv4_is_lbcast(saddr) ||
1534 	    skb->protocol != htons(ETH_P_IP))
1535 		return -EINVAL;
1536 
1537 	if (ipv4_is_loopback(saddr) && !IN_DEV_ROUTE_LOCALNET(in_dev))
1538 		return -EINVAL;
1539 
1540 	if (ipv4_is_zeronet(saddr)) {
1541 		if (!ipv4_is_local_multicast(daddr))
1542 			return -EINVAL;
1543 	} else {
1544 		err = fib_validate_source(skb, saddr, 0, tos, 0, dev,
1545 					  in_dev, itag);
1546 		if (err < 0)
1547 			return err;
1548 	}
1549 	return 0;
1550 }
1551 
1552 /* called in rcu_read_lock() section */
1553 static int ip_route_input_mc(struct sk_buff *skb, __be32 daddr, __be32 saddr,
1554 			     u8 tos, struct net_device *dev, int our)
1555 {
1556 	struct in_device *in_dev = __in_dev_get_rcu(dev);
1557 	unsigned int flags = RTCF_MULTICAST;
1558 	struct rtable *rth;
1559 	u32 itag = 0;
1560 	int err;
1561 
1562 	err = ip_mc_validate_source(skb, daddr, saddr, tos, dev, in_dev, &itag);
1563 	if (err)
1564 		return err;
1565 
1566 	if (our)
1567 		flags |= RTCF_LOCAL;
1568 
1569 	rth = rt_dst_alloc(dev_net(dev)->loopback_dev, flags, RTN_MULTICAST,
1570 			   IN_DEV_CONF_GET(in_dev, NOPOLICY), false, false);
1571 	if (!rth)
1572 		return -ENOBUFS;
1573 
1574 #ifdef CONFIG_IP_ROUTE_CLASSID
1575 	rth->dst.tclassid = itag;
1576 #endif
1577 	rth->dst.output = ip_rt_bug;
1578 	rth->rt_is_input= 1;
1579 
1580 #ifdef CONFIG_IP_MROUTE
1581 	if (!ipv4_is_local_multicast(daddr) && IN_DEV_MFORWARD(in_dev))
1582 		rth->dst.input = ip_mr_input;
1583 #endif
1584 	RT_CACHE_STAT_INC(in_slow_mc);
1585 
1586 	skb_dst_set(skb, &rth->dst);
1587 	return 0;
1588 }
1589 
1590 
1591 static void ip_handle_martian_source(struct net_device *dev,
1592 				     struct in_device *in_dev,
1593 				     struct sk_buff *skb,
1594 				     __be32 daddr,
1595 				     __be32 saddr)
1596 {
1597 	RT_CACHE_STAT_INC(in_martian_src);
1598 #ifdef CONFIG_IP_ROUTE_VERBOSE
1599 	if (IN_DEV_LOG_MARTIANS(in_dev) && net_ratelimit()) {
1600 		/*
1601 		 *	RFC1812 recommendation, if source is martian,
1602 		 *	the only hint is MAC header.
1603 		 */
1604 		pr_warn("martian source %pI4 from %pI4, on dev %s\n",
1605 			&daddr, &saddr, dev->name);
1606 		if (dev->hard_header_len && skb_mac_header_was_set(skb)) {
1607 			print_hex_dump(KERN_WARNING, "ll header: ",
1608 				       DUMP_PREFIX_OFFSET, 16, 1,
1609 				       skb_mac_header(skb),
1610 				       dev->hard_header_len, true);
1611 		}
1612 	}
1613 #endif
1614 }
1615 
1616 static void ip_del_fnhe(struct fib_nh *nh, __be32 daddr)
1617 {
1618 	struct fnhe_hash_bucket *hash;
1619 	struct fib_nh_exception *fnhe, __rcu **fnhe_p;
1620 	u32 hval = fnhe_hashfun(daddr);
1621 
1622 	spin_lock_bh(&fnhe_lock);
1623 
1624 	hash = rcu_dereference_protected(nh->nh_exceptions,
1625 					 lockdep_is_held(&fnhe_lock));
1626 	hash += hval;
1627 
1628 	fnhe_p = &hash->chain;
1629 	fnhe = rcu_dereference_protected(*fnhe_p, lockdep_is_held(&fnhe_lock));
1630 	while (fnhe) {
1631 		if (fnhe->fnhe_daddr == daddr) {
1632 			rcu_assign_pointer(*fnhe_p, rcu_dereference_protected(
1633 				fnhe->fnhe_next, lockdep_is_held(&fnhe_lock)));
1634 			fnhe_flush_routes(fnhe);
1635 			kfree_rcu(fnhe, rcu);
1636 			break;
1637 		}
1638 		fnhe_p = &fnhe->fnhe_next;
1639 		fnhe = rcu_dereference_protected(fnhe->fnhe_next,
1640 						 lockdep_is_held(&fnhe_lock));
1641 	}
1642 
1643 	spin_unlock_bh(&fnhe_lock);
1644 }
1645 
1646 static void set_lwt_redirect(struct rtable *rth)
1647 {
1648 	if (lwtunnel_output_redirect(rth->dst.lwtstate)) {
1649 		rth->dst.lwtstate->orig_output = rth->dst.output;
1650 		rth->dst.output = lwtunnel_output;
1651 	}
1652 
1653 	if (lwtunnel_input_redirect(rth->dst.lwtstate)) {
1654 		rth->dst.lwtstate->orig_input = rth->dst.input;
1655 		rth->dst.input = lwtunnel_input;
1656 	}
1657 }
1658 
1659 /* called in rcu_read_lock() section */
1660 static int __mkroute_input(struct sk_buff *skb,
1661 			   const struct fib_result *res,
1662 			   struct in_device *in_dev,
1663 			   __be32 daddr, __be32 saddr, u32 tos)
1664 {
1665 	struct fib_nh_exception *fnhe;
1666 	struct rtable *rth;
1667 	int err;
1668 	struct in_device *out_dev;
1669 	bool do_cache;
1670 	u32 itag = 0;
1671 
1672 	/* get a working reference to the output device */
1673 	out_dev = __in_dev_get_rcu(FIB_RES_DEV(*res));
1674 	if (!out_dev) {
1675 		net_crit_ratelimited("Bug in ip_route_input_slow(). Please report.\n");
1676 		return -EINVAL;
1677 	}
1678 
1679 	err = fib_validate_source(skb, saddr, daddr, tos, FIB_RES_OIF(*res),
1680 				  in_dev->dev, in_dev, &itag);
1681 	if (err < 0) {
1682 		ip_handle_martian_source(in_dev->dev, in_dev, skb, daddr,
1683 					 saddr);
1684 
1685 		goto cleanup;
1686 	}
1687 
1688 	do_cache = res->fi && !itag;
1689 	if (out_dev == in_dev && err && IN_DEV_TX_REDIRECTS(out_dev) &&
1690 	    skb->protocol == htons(ETH_P_IP) &&
1691 	    (IN_DEV_SHARED_MEDIA(out_dev) ||
1692 	     inet_addr_onlink(out_dev, saddr, FIB_RES_GW(*res))))
1693 		IPCB(skb)->flags |= IPSKB_DOREDIRECT;
1694 
1695 	if (skb->protocol != htons(ETH_P_IP)) {
1696 		/* Not IP (i.e. ARP). Do not create route, if it is
1697 		 * invalid for proxy arp. DNAT routes are always valid.
1698 		 *
1699 		 * Proxy arp feature have been extended to allow, ARP
1700 		 * replies back to the same interface, to support
1701 		 * Private VLAN switch technologies. See arp.c.
1702 		 */
1703 		if (out_dev == in_dev &&
1704 		    IN_DEV_PROXY_ARP_PVLAN(in_dev) == 0) {
1705 			err = -EINVAL;
1706 			goto cleanup;
1707 		}
1708 	}
1709 
1710 	fnhe = find_exception(&FIB_RES_NH(*res), daddr);
1711 	if (do_cache) {
1712 		if (fnhe) {
1713 			rth = rcu_dereference(fnhe->fnhe_rth_input);
1714 			if (rth && rth->dst.expires &&
1715 			    time_after(jiffies, rth->dst.expires)) {
1716 				ip_del_fnhe(&FIB_RES_NH(*res), daddr);
1717 				fnhe = NULL;
1718 			} else {
1719 				goto rt_cache;
1720 			}
1721 		}
1722 
1723 		rth = rcu_dereference(FIB_RES_NH(*res).nh_rth_input);
1724 
1725 rt_cache:
1726 		if (rt_cache_valid(rth)) {
1727 			skb_dst_set_noref(skb, &rth->dst);
1728 			goto out;
1729 		}
1730 	}
1731 
1732 	rth = rt_dst_alloc(out_dev->dev, 0, res->type,
1733 			   IN_DEV_CONF_GET(in_dev, NOPOLICY),
1734 			   IN_DEV_CONF_GET(out_dev, NOXFRM), do_cache);
1735 	if (!rth) {
1736 		err = -ENOBUFS;
1737 		goto cleanup;
1738 	}
1739 
1740 	rth->rt_is_input = 1;
1741 	if (res->table)
1742 		rth->rt_table_id = res->table->tb_id;
1743 	RT_CACHE_STAT_INC(in_slow_tot);
1744 
1745 	rth->dst.input = ip_forward;
1746 
1747 	rt_set_nexthop(rth, daddr, res, fnhe, res->fi, res->type, itag,
1748 		       do_cache);
1749 	set_lwt_redirect(rth);
1750 	skb_dst_set(skb, &rth->dst);
1751 out:
1752 	err = 0;
1753  cleanup:
1754 	return err;
1755 }
1756 
1757 #ifdef CONFIG_IP_ROUTE_MULTIPATH
1758 /* To make ICMP packets follow the right flow, the multipath hash is
1759  * calculated from the inner IP addresses.
1760  */
1761 static void ip_multipath_l3_keys(const struct sk_buff *skb,
1762 				 struct flow_keys *hash_keys)
1763 {
1764 	const struct iphdr *outer_iph = ip_hdr(skb);
1765 	const struct iphdr *inner_iph;
1766 	const struct icmphdr *icmph;
1767 	struct iphdr _inner_iph;
1768 	struct icmphdr _icmph;
1769 
1770 	hash_keys->addrs.v4addrs.src = outer_iph->saddr;
1771 	hash_keys->addrs.v4addrs.dst = outer_iph->daddr;
1772 	if (likely(outer_iph->protocol != IPPROTO_ICMP))
1773 		return;
1774 
1775 	if (unlikely((outer_iph->frag_off & htons(IP_OFFSET)) != 0))
1776 		return;
1777 
1778 	icmph = skb_header_pointer(skb, outer_iph->ihl * 4, sizeof(_icmph),
1779 				   &_icmph);
1780 	if (!icmph)
1781 		return;
1782 
1783 	if (icmph->type != ICMP_DEST_UNREACH &&
1784 	    icmph->type != ICMP_REDIRECT &&
1785 	    icmph->type != ICMP_TIME_EXCEEDED &&
1786 	    icmph->type != ICMP_PARAMETERPROB)
1787 		return;
1788 
1789 	inner_iph = skb_header_pointer(skb,
1790 				       outer_iph->ihl * 4 + sizeof(_icmph),
1791 				       sizeof(_inner_iph), &_inner_iph);
1792 	if (!inner_iph)
1793 		return;
1794 	hash_keys->addrs.v4addrs.src = inner_iph->saddr;
1795 	hash_keys->addrs.v4addrs.dst = inner_iph->daddr;
1796 }
1797 
1798 /* if skb is set it will be used and fl4 can be NULL */
1799 int fib_multipath_hash(const struct fib_info *fi, const struct flowi4 *fl4,
1800 		       const struct sk_buff *skb)
1801 {
1802 	struct net *net = fi->fib_net;
1803 	struct flow_keys hash_keys;
1804 	u32 mhash;
1805 
1806 	switch (net->ipv4.sysctl_fib_multipath_hash_policy) {
1807 	case 0:
1808 		memset(&hash_keys, 0, sizeof(hash_keys));
1809 		hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS;
1810 		if (skb) {
1811 			ip_multipath_l3_keys(skb, &hash_keys);
1812 		} else {
1813 			hash_keys.addrs.v4addrs.src = fl4->saddr;
1814 			hash_keys.addrs.v4addrs.dst = fl4->daddr;
1815 		}
1816 		break;
1817 	case 1:
1818 		/* skb is currently provided only when forwarding */
1819 		if (skb) {
1820 			unsigned int flag = FLOW_DISSECTOR_F_STOP_AT_ENCAP;
1821 			struct flow_keys keys;
1822 
1823 			/* short-circuit if we already have L4 hash present */
1824 			if (skb->l4_hash)
1825 				return skb_get_hash_raw(skb) >> 1;
1826 			memset(&hash_keys, 0, sizeof(hash_keys));
1827 			skb_flow_dissect_flow_keys(skb, &keys, flag);
1828 			hash_keys.addrs.v4addrs.src = keys.addrs.v4addrs.src;
1829 			hash_keys.addrs.v4addrs.dst = keys.addrs.v4addrs.dst;
1830 			hash_keys.ports.src = keys.ports.src;
1831 			hash_keys.ports.dst = keys.ports.dst;
1832 			hash_keys.basic.ip_proto = keys.basic.ip_proto;
1833 		} else {
1834 			memset(&hash_keys, 0, sizeof(hash_keys));
1835 			hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS;
1836 			hash_keys.addrs.v4addrs.src = fl4->saddr;
1837 			hash_keys.addrs.v4addrs.dst = fl4->daddr;
1838 			hash_keys.ports.src = fl4->fl4_sport;
1839 			hash_keys.ports.dst = fl4->fl4_dport;
1840 			hash_keys.basic.ip_proto = fl4->flowi4_proto;
1841 		}
1842 		break;
1843 	}
1844 	mhash = flow_hash_from_keys(&hash_keys);
1845 
1846 	return mhash >> 1;
1847 }
1848 EXPORT_SYMBOL_GPL(fib_multipath_hash);
1849 #endif /* CONFIG_IP_ROUTE_MULTIPATH */
1850 
1851 static int ip_mkroute_input(struct sk_buff *skb,
1852 			    struct fib_result *res,
1853 			    struct in_device *in_dev,
1854 			    __be32 daddr, __be32 saddr, u32 tos)
1855 {
1856 #ifdef CONFIG_IP_ROUTE_MULTIPATH
1857 	if (res->fi && res->fi->fib_nhs > 1) {
1858 		int h = fib_multipath_hash(res->fi, NULL, skb);
1859 
1860 		fib_select_multipath(res, h);
1861 	}
1862 #endif
1863 
1864 	/* create a routing cache entry */
1865 	return __mkroute_input(skb, res, in_dev, daddr, saddr, tos);
1866 }
1867 
1868 /*
1869  *	NOTE. We drop all the packets that has local source
1870  *	addresses, because every properly looped back packet
1871  *	must have correct destination already attached by output routine.
1872  *
1873  *	Such approach solves two big problems:
1874  *	1. Not simplex devices are handled properly.
1875  *	2. IP spoofing attempts are filtered with 100% of guarantee.
1876  *	called with rcu_read_lock()
1877  */
1878 
1879 static int ip_route_input_slow(struct sk_buff *skb, __be32 daddr, __be32 saddr,
1880 			       u8 tos, struct net_device *dev,
1881 			       struct fib_result *res)
1882 {
1883 	struct in_device *in_dev = __in_dev_get_rcu(dev);
1884 	struct ip_tunnel_info *tun_info;
1885 	struct flowi4	fl4;
1886 	unsigned int	flags = 0;
1887 	u32		itag = 0;
1888 	struct rtable	*rth;
1889 	int		err = -EINVAL;
1890 	struct net    *net = dev_net(dev);
1891 	bool do_cache;
1892 
1893 	/* IP on this device is disabled. */
1894 
1895 	if (!in_dev)
1896 		goto out;
1897 
1898 	/* Check for the most weird martians, which can be not detected
1899 	   by fib_lookup.
1900 	 */
1901 
1902 	tun_info = skb_tunnel_info(skb);
1903 	if (tun_info && !(tun_info->mode & IP_TUNNEL_INFO_TX))
1904 		fl4.flowi4_tun_key.tun_id = tun_info->key.tun_id;
1905 	else
1906 		fl4.flowi4_tun_key.tun_id = 0;
1907 	skb_dst_drop(skb);
1908 
1909 	if (ipv4_is_multicast(saddr) || ipv4_is_lbcast(saddr))
1910 		goto martian_source;
1911 
1912 	res->fi = NULL;
1913 	res->table = NULL;
1914 	if (ipv4_is_lbcast(daddr) || (saddr == 0 && daddr == 0))
1915 		goto brd_input;
1916 
1917 	/* Accept zero addresses only to limited broadcast;
1918 	 * I even do not know to fix it or not. Waiting for complains :-)
1919 	 */
1920 	if (ipv4_is_zeronet(saddr))
1921 		goto martian_source;
1922 
1923 	if (ipv4_is_zeronet(daddr))
1924 		goto martian_destination;
1925 
1926 	/* Following code try to avoid calling IN_DEV_NET_ROUTE_LOCALNET(),
1927 	 * and call it once if daddr or/and saddr are loopback addresses
1928 	 */
1929 	if (ipv4_is_loopback(daddr)) {
1930 		if (!IN_DEV_NET_ROUTE_LOCALNET(in_dev, net))
1931 			goto martian_destination;
1932 	} else if (ipv4_is_loopback(saddr)) {
1933 		if (!IN_DEV_NET_ROUTE_LOCALNET(in_dev, net))
1934 			goto martian_source;
1935 	}
1936 
1937 	/*
1938 	 *	Now we are ready to route packet.
1939 	 */
1940 	fl4.flowi4_oif = 0;
1941 	fl4.flowi4_iif = dev->ifindex;
1942 	fl4.flowi4_mark = skb->mark;
1943 	fl4.flowi4_tos = tos;
1944 	fl4.flowi4_scope = RT_SCOPE_UNIVERSE;
1945 	fl4.flowi4_flags = 0;
1946 	fl4.daddr = daddr;
1947 	fl4.saddr = saddr;
1948 	fl4.flowi4_uid = sock_net_uid(net, NULL);
1949 	err = fib_lookup(net, &fl4, res, 0);
1950 	if (err != 0) {
1951 		if (!IN_DEV_FORWARD(in_dev))
1952 			err = -EHOSTUNREACH;
1953 		goto no_route;
1954 	}
1955 
1956 	if (res->type == RTN_BROADCAST)
1957 		goto brd_input;
1958 
1959 	if (res->type == RTN_LOCAL) {
1960 		err = fib_validate_source(skb, saddr, daddr, tos,
1961 					  0, dev, in_dev, &itag);
1962 		if (err < 0)
1963 			goto martian_source;
1964 		goto local_input;
1965 	}
1966 
1967 	if (!IN_DEV_FORWARD(in_dev)) {
1968 		err = -EHOSTUNREACH;
1969 		goto no_route;
1970 	}
1971 	if (res->type != RTN_UNICAST)
1972 		goto martian_destination;
1973 
1974 	err = ip_mkroute_input(skb, res, in_dev, daddr, saddr, tos);
1975 out:	return err;
1976 
1977 brd_input:
1978 	if (skb->protocol != htons(ETH_P_IP))
1979 		goto e_inval;
1980 
1981 	if (!ipv4_is_zeronet(saddr)) {
1982 		err = fib_validate_source(skb, saddr, 0, tos, 0, dev,
1983 					  in_dev, &itag);
1984 		if (err < 0)
1985 			goto martian_source;
1986 	}
1987 	flags |= RTCF_BROADCAST;
1988 	res->type = RTN_BROADCAST;
1989 	RT_CACHE_STAT_INC(in_brd);
1990 
1991 local_input:
1992 	do_cache = false;
1993 	if (res->fi) {
1994 		if (!itag) {
1995 			rth = rcu_dereference(FIB_RES_NH(*res).nh_rth_input);
1996 			if (rt_cache_valid(rth)) {
1997 				skb_dst_set_noref(skb, &rth->dst);
1998 				err = 0;
1999 				goto out;
2000 			}
2001 			do_cache = true;
2002 		}
2003 	}
2004 
2005 	rth = rt_dst_alloc(l3mdev_master_dev_rcu(dev) ? : net->loopback_dev,
2006 			   flags | RTCF_LOCAL, res->type,
2007 			   IN_DEV_CONF_GET(in_dev, NOPOLICY), false, do_cache);
2008 	if (!rth)
2009 		goto e_nobufs;
2010 
2011 	rth->dst.output= ip_rt_bug;
2012 #ifdef CONFIG_IP_ROUTE_CLASSID
2013 	rth->dst.tclassid = itag;
2014 #endif
2015 	rth->rt_is_input = 1;
2016 	if (res->table)
2017 		rth->rt_table_id = res->table->tb_id;
2018 
2019 	RT_CACHE_STAT_INC(in_slow_tot);
2020 	if (res->type == RTN_UNREACHABLE) {
2021 		rth->dst.input= ip_error;
2022 		rth->dst.error= -err;
2023 		rth->rt_flags 	&= ~RTCF_LOCAL;
2024 	}
2025 
2026 	if (do_cache) {
2027 		struct fib_nh *nh = &FIB_RES_NH(*res);
2028 
2029 		rth->dst.lwtstate = lwtstate_get(nh->nh_lwtstate);
2030 		if (lwtunnel_input_redirect(rth->dst.lwtstate)) {
2031 			WARN_ON(rth->dst.input == lwtunnel_input);
2032 			rth->dst.lwtstate->orig_input = rth->dst.input;
2033 			rth->dst.input = lwtunnel_input;
2034 		}
2035 
2036 		if (unlikely(!rt_cache_route(nh, rth)))
2037 			rt_add_uncached_list(rth);
2038 	}
2039 	skb_dst_set(skb, &rth->dst);
2040 	err = 0;
2041 	goto out;
2042 
2043 no_route:
2044 	RT_CACHE_STAT_INC(in_no_route);
2045 	res->type = RTN_UNREACHABLE;
2046 	res->fi = NULL;
2047 	res->table = NULL;
2048 	goto local_input;
2049 
2050 	/*
2051 	 *	Do not cache martian addresses: they should be logged (RFC1812)
2052 	 */
2053 martian_destination:
2054 	RT_CACHE_STAT_INC(in_martian_dst);
2055 #ifdef CONFIG_IP_ROUTE_VERBOSE
2056 	if (IN_DEV_LOG_MARTIANS(in_dev))
2057 		net_warn_ratelimited("martian destination %pI4 from %pI4, dev %s\n",
2058 				     &daddr, &saddr, dev->name);
2059 #endif
2060 
2061 e_inval:
2062 	err = -EINVAL;
2063 	goto out;
2064 
2065 e_nobufs:
2066 	err = -ENOBUFS;
2067 	goto out;
2068 
2069 martian_source:
2070 	ip_handle_martian_source(dev, in_dev, skb, daddr, saddr);
2071 	goto out;
2072 }
2073 
2074 int ip_route_input_noref(struct sk_buff *skb, __be32 daddr, __be32 saddr,
2075 			 u8 tos, struct net_device *dev)
2076 {
2077 	struct fib_result res;
2078 	int err;
2079 
2080 	tos &= IPTOS_RT_MASK;
2081 	rcu_read_lock();
2082 	err = ip_route_input_rcu(skb, daddr, saddr, tos, dev, &res);
2083 	rcu_read_unlock();
2084 
2085 	return err;
2086 }
2087 EXPORT_SYMBOL(ip_route_input_noref);
2088 
2089 /* called with rcu_read_lock held */
2090 int ip_route_input_rcu(struct sk_buff *skb, __be32 daddr, __be32 saddr,
2091 		       u8 tos, struct net_device *dev, struct fib_result *res)
2092 {
2093 	/* Multicast recognition logic is moved from route cache to here.
2094 	   The problem was that too many Ethernet cards have broken/missing
2095 	   hardware multicast filters :-( As result the host on multicasting
2096 	   network acquires a lot of useless route cache entries, sort of
2097 	   SDR messages from all the world. Now we try to get rid of them.
2098 	   Really, provided software IP multicast filter is organized
2099 	   reasonably (at least, hashed), it does not result in a slowdown
2100 	   comparing with route cache reject entries.
2101 	   Note, that multicast routers are not affected, because
2102 	   route cache entry is created eventually.
2103 	 */
2104 	if (ipv4_is_multicast(daddr)) {
2105 		struct in_device *in_dev = __in_dev_get_rcu(dev);
2106 		int our = 0;
2107 		int err = -EINVAL;
2108 
2109 		if (in_dev)
2110 			our = ip_check_mc_rcu(in_dev, daddr, saddr,
2111 					      ip_hdr(skb)->protocol);
2112 
2113 		/* check l3 master if no match yet */
2114 		if ((!in_dev || !our) && netif_is_l3_slave(dev)) {
2115 			struct in_device *l3_in_dev;
2116 
2117 			l3_in_dev = __in_dev_get_rcu(skb->dev);
2118 			if (l3_in_dev)
2119 				our = ip_check_mc_rcu(l3_in_dev, daddr, saddr,
2120 						      ip_hdr(skb)->protocol);
2121 		}
2122 
2123 		if (our
2124 #ifdef CONFIG_IP_MROUTE
2125 			||
2126 		    (!ipv4_is_local_multicast(daddr) &&
2127 		     IN_DEV_MFORWARD(in_dev))
2128 #endif
2129 		   ) {
2130 			err = ip_route_input_mc(skb, daddr, saddr,
2131 						tos, dev, our);
2132 		}
2133 		return err;
2134 	}
2135 
2136 	return ip_route_input_slow(skb, daddr, saddr, tos, dev, res);
2137 }
2138 
2139 /* called with rcu_read_lock() */
2140 static struct rtable *__mkroute_output(const struct fib_result *res,
2141 				       const struct flowi4 *fl4, int orig_oif,
2142 				       struct net_device *dev_out,
2143 				       unsigned int flags)
2144 {
2145 	struct fib_info *fi = res->fi;
2146 	struct fib_nh_exception *fnhe;
2147 	struct in_device *in_dev;
2148 	u16 type = res->type;
2149 	struct rtable *rth;
2150 	bool do_cache;
2151 
2152 	in_dev = __in_dev_get_rcu(dev_out);
2153 	if (!in_dev)
2154 		return ERR_PTR(-EINVAL);
2155 
2156 	if (likely(!IN_DEV_ROUTE_LOCALNET(in_dev)))
2157 		if (ipv4_is_loopback(fl4->saddr) &&
2158 		    !(dev_out->flags & IFF_LOOPBACK) &&
2159 		    !netif_is_l3_master(dev_out))
2160 			return ERR_PTR(-EINVAL);
2161 
2162 	if (ipv4_is_lbcast(fl4->daddr))
2163 		type = RTN_BROADCAST;
2164 	else if (ipv4_is_multicast(fl4->daddr))
2165 		type = RTN_MULTICAST;
2166 	else if (ipv4_is_zeronet(fl4->daddr))
2167 		return ERR_PTR(-EINVAL);
2168 
2169 	if (dev_out->flags & IFF_LOOPBACK)
2170 		flags |= RTCF_LOCAL;
2171 
2172 	do_cache = true;
2173 	if (type == RTN_BROADCAST) {
2174 		flags |= RTCF_BROADCAST | RTCF_LOCAL;
2175 		fi = NULL;
2176 	} else if (type == RTN_MULTICAST) {
2177 		flags |= RTCF_MULTICAST | RTCF_LOCAL;
2178 		if (!ip_check_mc_rcu(in_dev, fl4->daddr, fl4->saddr,
2179 				     fl4->flowi4_proto))
2180 			flags &= ~RTCF_LOCAL;
2181 		else
2182 			do_cache = false;
2183 		/* If multicast route do not exist use
2184 		 * default one, but do not gateway in this case.
2185 		 * Yes, it is hack.
2186 		 */
2187 		if (fi && res->prefixlen < 4)
2188 			fi = NULL;
2189 	} else if ((type == RTN_LOCAL) && (orig_oif != 0) &&
2190 		   (orig_oif != dev_out->ifindex)) {
2191 		/* For local routes that require a particular output interface
2192 		 * we do not want to cache the result.  Caching the result
2193 		 * causes incorrect behaviour when there are multiple source
2194 		 * addresses on the interface, the end result being that if the
2195 		 * intended recipient is waiting on that interface for the
2196 		 * packet he won't receive it because it will be delivered on
2197 		 * the loopback interface and the IP_PKTINFO ipi_ifindex will
2198 		 * be set to the loopback interface as well.
2199 		 */
2200 		fi = NULL;
2201 	}
2202 
2203 	fnhe = NULL;
2204 	do_cache &= fi != NULL;
2205 	if (do_cache) {
2206 		struct rtable __rcu **prth;
2207 		struct fib_nh *nh = &FIB_RES_NH(*res);
2208 
2209 		fnhe = find_exception(nh, fl4->daddr);
2210 		if (fnhe) {
2211 			prth = &fnhe->fnhe_rth_output;
2212 			rth = rcu_dereference(*prth);
2213 			if (rth && rth->dst.expires &&
2214 			    time_after(jiffies, rth->dst.expires)) {
2215 				ip_del_fnhe(nh, fl4->daddr);
2216 				fnhe = NULL;
2217 			} else {
2218 				goto rt_cache;
2219 			}
2220 		}
2221 
2222 		if (unlikely(fl4->flowi4_flags &
2223 			     FLOWI_FLAG_KNOWN_NH &&
2224 			     !(nh->nh_gw &&
2225 			       nh->nh_scope == RT_SCOPE_LINK))) {
2226 			do_cache = false;
2227 			goto add;
2228 		}
2229 		prth = raw_cpu_ptr(nh->nh_pcpu_rth_output);
2230 		rth = rcu_dereference(*prth);
2231 
2232 rt_cache:
2233 		if (rt_cache_valid(rth) && dst_hold_safe(&rth->dst))
2234 			return rth;
2235 	}
2236 
2237 add:
2238 	rth = rt_dst_alloc(dev_out, flags, type,
2239 			   IN_DEV_CONF_GET(in_dev, NOPOLICY),
2240 			   IN_DEV_CONF_GET(in_dev, NOXFRM),
2241 			   do_cache);
2242 	if (!rth)
2243 		return ERR_PTR(-ENOBUFS);
2244 
2245 	rth->rt_iif = orig_oif;
2246 	if (res->table)
2247 		rth->rt_table_id = res->table->tb_id;
2248 
2249 	RT_CACHE_STAT_INC(out_slow_tot);
2250 
2251 	if (flags & (RTCF_BROADCAST | RTCF_MULTICAST)) {
2252 		if (flags & RTCF_LOCAL &&
2253 		    !(dev_out->flags & IFF_LOOPBACK)) {
2254 			rth->dst.output = ip_mc_output;
2255 			RT_CACHE_STAT_INC(out_slow_mc);
2256 		}
2257 #ifdef CONFIG_IP_MROUTE
2258 		if (type == RTN_MULTICAST) {
2259 			if (IN_DEV_MFORWARD(in_dev) &&
2260 			    !ipv4_is_local_multicast(fl4->daddr)) {
2261 				rth->dst.input = ip_mr_input;
2262 				rth->dst.output = ip_mc_output;
2263 			}
2264 		}
2265 #endif
2266 	}
2267 
2268 	rt_set_nexthop(rth, fl4->daddr, res, fnhe, fi, type, 0, do_cache);
2269 	set_lwt_redirect(rth);
2270 
2271 	return rth;
2272 }
2273 
2274 /*
2275  * Major route resolver routine.
2276  */
2277 
2278 struct rtable *ip_route_output_key_hash(struct net *net, struct flowi4 *fl4,
2279 					const struct sk_buff *skb)
2280 {
2281 	__u8 tos = RT_FL_TOS(fl4);
2282 	struct fib_result res;
2283 	struct rtable *rth;
2284 
2285 	res.tclassid	= 0;
2286 	res.fi		= NULL;
2287 	res.table	= NULL;
2288 
2289 	fl4->flowi4_iif = LOOPBACK_IFINDEX;
2290 	fl4->flowi4_tos = tos & IPTOS_RT_MASK;
2291 	fl4->flowi4_scope = ((tos & RTO_ONLINK) ?
2292 			 RT_SCOPE_LINK : RT_SCOPE_UNIVERSE);
2293 
2294 	rcu_read_lock();
2295 	rth = ip_route_output_key_hash_rcu(net, fl4, &res, skb);
2296 	rcu_read_unlock();
2297 
2298 	return rth;
2299 }
2300 EXPORT_SYMBOL_GPL(ip_route_output_key_hash);
2301 
2302 struct rtable *ip_route_output_key_hash_rcu(struct net *net, struct flowi4 *fl4,
2303 					    struct fib_result *res,
2304 					    const struct sk_buff *skb)
2305 {
2306 	struct net_device *dev_out = NULL;
2307 	int orig_oif = fl4->flowi4_oif;
2308 	unsigned int flags = 0;
2309 	struct rtable *rth;
2310 	int err = -ENETUNREACH;
2311 
2312 	if (fl4->saddr) {
2313 		rth = ERR_PTR(-EINVAL);
2314 		if (ipv4_is_multicast(fl4->saddr) ||
2315 		    ipv4_is_lbcast(fl4->saddr) ||
2316 		    ipv4_is_zeronet(fl4->saddr))
2317 			goto out;
2318 
2319 		/* I removed check for oif == dev_out->oif here.
2320 		   It was wrong for two reasons:
2321 		   1. ip_dev_find(net, saddr) can return wrong iface, if saddr
2322 		      is assigned to multiple interfaces.
2323 		   2. Moreover, we are allowed to send packets with saddr
2324 		      of another iface. --ANK
2325 		 */
2326 
2327 		if (fl4->flowi4_oif == 0 &&
2328 		    (ipv4_is_multicast(fl4->daddr) ||
2329 		     ipv4_is_lbcast(fl4->daddr))) {
2330 			/* It is equivalent to inet_addr_type(saddr) == RTN_LOCAL */
2331 			dev_out = __ip_dev_find(net, fl4->saddr, false);
2332 			if (!dev_out)
2333 				goto out;
2334 
2335 			/* Special hack: user can direct multicasts
2336 			   and limited broadcast via necessary interface
2337 			   without fiddling with IP_MULTICAST_IF or IP_PKTINFO.
2338 			   This hack is not just for fun, it allows
2339 			   vic,vat and friends to work.
2340 			   They bind socket to loopback, set ttl to zero
2341 			   and expect that it will work.
2342 			   From the viewpoint of routing cache they are broken,
2343 			   because we are not allowed to build multicast path
2344 			   with loopback source addr (look, routing cache
2345 			   cannot know, that ttl is zero, so that packet
2346 			   will not leave this host and route is valid).
2347 			   Luckily, this hack is good workaround.
2348 			 */
2349 
2350 			fl4->flowi4_oif = dev_out->ifindex;
2351 			goto make_route;
2352 		}
2353 
2354 		if (!(fl4->flowi4_flags & FLOWI_FLAG_ANYSRC)) {
2355 			/* It is equivalent to inet_addr_type(saddr) == RTN_LOCAL */
2356 			if (!__ip_dev_find(net, fl4->saddr, false))
2357 				goto out;
2358 		}
2359 	}
2360 
2361 
2362 	if (fl4->flowi4_oif) {
2363 		dev_out = dev_get_by_index_rcu(net, fl4->flowi4_oif);
2364 		rth = ERR_PTR(-ENODEV);
2365 		if (!dev_out)
2366 			goto out;
2367 
2368 		/* RACE: Check return value of inet_select_addr instead. */
2369 		if (!(dev_out->flags & IFF_UP) || !__in_dev_get_rcu(dev_out)) {
2370 			rth = ERR_PTR(-ENETUNREACH);
2371 			goto out;
2372 		}
2373 		if (ipv4_is_local_multicast(fl4->daddr) ||
2374 		    ipv4_is_lbcast(fl4->daddr) ||
2375 		    fl4->flowi4_proto == IPPROTO_IGMP) {
2376 			if (!fl4->saddr)
2377 				fl4->saddr = inet_select_addr(dev_out, 0,
2378 							      RT_SCOPE_LINK);
2379 			goto make_route;
2380 		}
2381 		if (!fl4->saddr) {
2382 			if (ipv4_is_multicast(fl4->daddr))
2383 				fl4->saddr = inet_select_addr(dev_out, 0,
2384 							      fl4->flowi4_scope);
2385 			else if (!fl4->daddr)
2386 				fl4->saddr = inet_select_addr(dev_out, 0,
2387 							      RT_SCOPE_HOST);
2388 		}
2389 	}
2390 
2391 	if (!fl4->daddr) {
2392 		fl4->daddr = fl4->saddr;
2393 		if (!fl4->daddr)
2394 			fl4->daddr = fl4->saddr = htonl(INADDR_LOOPBACK);
2395 		dev_out = net->loopback_dev;
2396 		fl4->flowi4_oif = LOOPBACK_IFINDEX;
2397 		res->type = RTN_LOCAL;
2398 		flags |= RTCF_LOCAL;
2399 		goto make_route;
2400 	}
2401 
2402 	err = fib_lookup(net, fl4, res, 0);
2403 	if (err) {
2404 		res->fi = NULL;
2405 		res->table = NULL;
2406 		if (fl4->flowi4_oif &&
2407 		    (ipv4_is_multicast(fl4->daddr) ||
2408 		    !netif_index_is_l3_master(net, fl4->flowi4_oif))) {
2409 			/* Apparently, routing tables are wrong. Assume,
2410 			   that the destination is on link.
2411 
2412 			   WHY? DW.
2413 			   Because we are allowed to send to iface
2414 			   even if it has NO routes and NO assigned
2415 			   addresses. When oif is specified, routing
2416 			   tables are looked up with only one purpose:
2417 			   to catch if destination is gatewayed, rather than
2418 			   direct. Moreover, if MSG_DONTROUTE is set,
2419 			   we send packet, ignoring both routing tables
2420 			   and ifaddr state. --ANK
2421 
2422 
2423 			   We could make it even if oif is unknown,
2424 			   likely IPv6, but we do not.
2425 			 */
2426 
2427 			if (fl4->saddr == 0)
2428 				fl4->saddr = inet_select_addr(dev_out, 0,
2429 							      RT_SCOPE_LINK);
2430 			res->type = RTN_UNICAST;
2431 			goto make_route;
2432 		}
2433 		rth = ERR_PTR(err);
2434 		goto out;
2435 	}
2436 
2437 	if (res->type == RTN_LOCAL) {
2438 		if (!fl4->saddr) {
2439 			if (res->fi->fib_prefsrc)
2440 				fl4->saddr = res->fi->fib_prefsrc;
2441 			else
2442 				fl4->saddr = fl4->daddr;
2443 		}
2444 
2445 		/* L3 master device is the loopback for that domain */
2446 		dev_out = l3mdev_master_dev_rcu(FIB_RES_DEV(*res)) ? :
2447 			net->loopback_dev;
2448 
2449 		/* make sure orig_oif points to fib result device even
2450 		 * though packet rx/tx happens over loopback or l3mdev
2451 		 */
2452 		orig_oif = FIB_RES_OIF(*res);
2453 
2454 		fl4->flowi4_oif = dev_out->ifindex;
2455 		flags |= RTCF_LOCAL;
2456 		goto make_route;
2457 	}
2458 
2459 	fib_select_path(net, res, fl4, skb);
2460 
2461 	dev_out = FIB_RES_DEV(*res);
2462 	fl4->flowi4_oif = dev_out->ifindex;
2463 
2464 
2465 make_route:
2466 	rth = __mkroute_output(res, fl4, orig_oif, dev_out, flags);
2467 
2468 out:
2469 	return rth;
2470 }
2471 
2472 static struct dst_entry *ipv4_blackhole_dst_check(struct dst_entry *dst, u32 cookie)
2473 {
2474 	return NULL;
2475 }
2476 
2477 static unsigned int ipv4_blackhole_mtu(const struct dst_entry *dst)
2478 {
2479 	unsigned int mtu = dst_metric_raw(dst, RTAX_MTU);
2480 
2481 	return mtu ? : dst->dev->mtu;
2482 }
2483 
2484 static void ipv4_rt_blackhole_update_pmtu(struct dst_entry *dst, struct sock *sk,
2485 					  struct sk_buff *skb, u32 mtu)
2486 {
2487 }
2488 
2489 static void ipv4_rt_blackhole_redirect(struct dst_entry *dst, struct sock *sk,
2490 				       struct sk_buff *skb)
2491 {
2492 }
2493 
2494 static u32 *ipv4_rt_blackhole_cow_metrics(struct dst_entry *dst,
2495 					  unsigned long old)
2496 {
2497 	return NULL;
2498 }
2499 
2500 static struct dst_ops ipv4_dst_blackhole_ops = {
2501 	.family			=	AF_INET,
2502 	.check			=	ipv4_blackhole_dst_check,
2503 	.mtu			=	ipv4_blackhole_mtu,
2504 	.default_advmss		=	ipv4_default_advmss,
2505 	.update_pmtu		=	ipv4_rt_blackhole_update_pmtu,
2506 	.redirect		=	ipv4_rt_blackhole_redirect,
2507 	.cow_metrics		=	ipv4_rt_blackhole_cow_metrics,
2508 	.neigh_lookup		=	ipv4_neigh_lookup,
2509 };
2510 
2511 struct dst_entry *ipv4_blackhole_route(struct net *net, struct dst_entry *dst_orig)
2512 {
2513 	struct rtable *ort = (struct rtable *) dst_orig;
2514 	struct rtable *rt;
2515 
2516 	rt = dst_alloc(&ipv4_dst_blackhole_ops, NULL, 1, DST_OBSOLETE_DEAD, 0);
2517 	if (rt) {
2518 		struct dst_entry *new = &rt->dst;
2519 
2520 		new->__use = 1;
2521 		new->input = dst_discard;
2522 		new->output = dst_discard_out;
2523 
2524 		new->dev = net->loopback_dev;
2525 		if (new->dev)
2526 			dev_hold(new->dev);
2527 
2528 		rt->rt_is_input = ort->rt_is_input;
2529 		rt->rt_iif = ort->rt_iif;
2530 		rt->rt_pmtu = ort->rt_pmtu;
2531 
2532 		rt->rt_genid = rt_genid_ipv4(net);
2533 		rt->rt_flags = ort->rt_flags;
2534 		rt->rt_type = ort->rt_type;
2535 		rt->rt_gateway = ort->rt_gateway;
2536 		rt->rt_uses_gateway = ort->rt_uses_gateway;
2537 
2538 		INIT_LIST_HEAD(&rt->rt_uncached);
2539 	}
2540 
2541 	dst_release(dst_orig);
2542 
2543 	return rt ? &rt->dst : ERR_PTR(-ENOMEM);
2544 }
2545 
2546 struct rtable *ip_route_output_flow(struct net *net, struct flowi4 *flp4,
2547 				    const struct sock *sk)
2548 {
2549 	struct rtable *rt = __ip_route_output_key(net, flp4);
2550 
2551 	if (IS_ERR(rt))
2552 		return rt;
2553 
2554 	if (flp4->flowi4_proto)
2555 		rt = (struct rtable *)xfrm_lookup_route(net, &rt->dst,
2556 							flowi4_to_flowi(flp4),
2557 							sk, 0);
2558 
2559 	return rt;
2560 }
2561 EXPORT_SYMBOL_GPL(ip_route_output_flow);
2562 
2563 /* called with rcu_read_lock held */
2564 static int rt_fill_info(struct net *net,  __be32 dst, __be32 src, u32 table_id,
2565 			struct flowi4 *fl4, struct sk_buff *skb, u32 portid,
2566 			u32 seq)
2567 {
2568 	struct rtable *rt = skb_rtable(skb);
2569 	struct rtmsg *r;
2570 	struct nlmsghdr *nlh;
2571 	unsigned long expires = 0;
2572 	u32 error;
2573 	u32 metrics[RTAX_MAX];
2574 
2575 	nlh = nlmsg_put(skb, portid, seq, RTM_NEWROUTE, sizeof(*r), 0);
2576 	if (!nlh)
2577 		return -EMSGSIZE;
2578 
2579 	r = nlmsg_data(nlh);
2580 	r->rtm_family	 = AF_INET;
2581 	r->rtm_dst_len	= 32;
2582 	r->rtm_src_len	= 0;
2583 	r->rtm_tos	= fl4->flowi4_tos;
2584 	r->rtm_table	= table_id < 256 ? table_id : RT_TABLE_COMPAT;
2585 	if (nla_put_u32(skb, RTA_TABLE, table_id))
2586 		goto nla_put_failure;
2587 	r->rtm_type	= rt->rt_type;
2588 	r->rtm_scope	= RT_SCOPE_UNIVERSE;
2589 	r->rtm_protocol = RTPROT_UNSPEC;
2590 	r->rtm_flags	= (rt->rt_flags & ~0xFFFF) | RTM_F_CLONED;
2591 	if (rt->rt_flags & RTCF_NOTIFY)
2592 		r->rtm_flags |= RTM_F_NOTIFY;
2593 	if (IPCB(skb)->flags & IPSKB_DOREDIRECT)
2594 		r->rtm_flags |= RTCF_DOREDIRECT;
2595 
2596 	if (nla_put_in_addr(skb, RTA_DST, dst))
2597 		goto nla_put_failure;
2598 	if (src) {
2599 		r->rtm_src_len = 32;
2600 		if (nla_put_in_addr(skb, RTA_SRC, src))
2601 			goto nla_put_failure;
2602 	}
2603 	if (rt->dst.dev &&
2604 	    nla_put_u32(skb, RTA_OIF, rt->dst.dev->ifindex))
2605 		goto nla_put_failure;
2606 #ifdef CONFIG_IP_ROUTE_CLASSID
2607 	if (rt->dst.tclassid &&
2608 	    nla_put_u32(skb, RTA_FLOW, rt->dst.tclassid))
2609 		goto nla_put_failure;
2610 #endif
2611 	if (!rt_is_input_route(rt) &&
2612 	    fl4->saddr != src) {
2613 		if (nla_put_in_addr(skb, RTA_PREFSRC, fl4->saddr))
2614 			goto nla_put_failure;
2615 	}
2616 	if (rt->rt_uses_gateway &&
2617 	    nla_put_in_addr(skb, RTA_GATEWAY, rt->rt_gateway))
2618 		goto nla_put_failure;
2619 
2620 	expires = rt->dst.expires;
2621 	if (expires) {
2622 		unsigned long now = jiffies;
2623 
2624 		if (time_before(now, expires))
2625 			expires -= now;
2626 		else
2627 			expires = 0;
2628 	}
2629 
2630 	memcpy(metrics, dst_metrics_ptr(&rt->dst), sizeof(metrics));
2631 	if (rt->rt_pmtu && expires)
2632 		metrics[RTAX_MTU - 1] = rt->rt_pmtu;
2633 	if (rtnetlink_put_metrics(skb, metrics) < 0)
2634 		goto nla_put_failure;
2635 
2636 	if (fl4->flowi4_mark &&
2637 	    nla_put_u32(skb, RTA_MARK, fl4->flowi4_mark))
2638 		goto nla_put_failure;
2639 
2640 	if (!uid_eq(fl4->flowi4_uid, INVALID_UID) &&
2641 	    nla_put_u32(skb, RTA_UID,
2642 			from_kuid_munged(current_user_ns(), fl4->flowi4_uid)))
2643 		goto nla_put_failure;
2644 
2645 	error = rt->dst.error;
2646 
2647 	if (rt_is_input_route(rt)) {
2648 #ifdef CONFIG_IP_MROUTE
2649 		if (ipv4_is_multicast(dst) && !ipv4_is_local_multicast(dst) &&
2650 		    IPV4_DEVCONF_ALL(net, MC_FORWARDING)) {
2651 			int err = ipmr_get_route(net, skb,
2652 						 fl4->saddr, fl4->daddr,
2653 						 r, portid);
2654 
2655 			if (err <= 0) {
2656 				if (err == 0)
2657 					return 0;
2658 				goto nla_put_failure;
2659 			}
2660 		} else
2661 #endif
2662 			if (nla_put_u32(skb, RTA_IIF, skb->dev->ifindex))
2663 				goto nla_put_failure;
2664 	}
2665 
2666 	if (rtnl_put_cacheinfo(skb, &rt->dst, 0, expires, error) < 0)
2667 		goto nla_put_failure;
2668 
2669 	nlmsg_end(skb, nlh);
2670 	return 0;
2671 
2672 nla_put_failure:
2673 	nlmsg_cancel(skb, nlh);
2674 	return -EMSGSIZE;
2675 }
2676 
2677 static int inet_rtm_getroute(struct sk_buff *in_skb, struct nlmsghdr *nlh,
2678 			     struct netlink_ext_ack *extack)
2679 {
2680 	struct net *net = sock_net(in_skb->sk);
2681 	struct rtmsg *rtm;
2682 	struct nlattr *tb[RTA_MAX+1];
2683 	struct fib_result res = {};
2684 	struct rtable *rt = NULL;
2685 	struct flowi4 fl4;
2686 	__be32 dst = 0;
2687 	__be32 src = 0;
2688 	u32 iif;
2689 	int err;
2690 	int mark;
2691 	struct sk_buff *skb;
2692 	u32 table_id = RT_TABLE_MAIN;
2693 	kuid_t uid;
2694 
2695 	err = nlmsg_parse(nlh, sizeof(*rtm), tb, RTA_MAX, rtm_ipv4_policy,
2696 			  extack);
2697 	if (err < 0)
2698 		goto errout;
2699 
2700 	rtm = nlmsg_data(nlh);
2701 
2702 	skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
2703 	if (!skb) {
2704 		err = -ENOBUFS;
2705 		goto errout;
2706 	}
2707 
2708 	/* Reserve room for dummy headers, this skb can pass
2709 	   through good chunk of routing engine.
2710 	 */
2711 	skb_reset_mac_header(skb);
2712 	skb_reset_network_header(skb);
2713 
2714 	src = tb[RTA_SRC] ? nla_get_in_addr(tb[RTA_SRC]) : 0;
2715 	dst = tb[RTA_DST] ? nla_get_in_addr(tb[RTA_DST]) : 0;
2716 	iif = tb[RTA_IIF] ? nla_get_u32(tb[RTA_IIF]) : 0;
2717 	mark = tb[RTA_MARK] ? nla_get_u32(tb[RTA_MARK]) : 0;
2718 	if (tb[RTA_UID])
2719 		uid = make_kuid(current_user_ns(), nla_get_u32(tb[RTA_UID]));
2720 	else
2721 		uid = (iif ? INVALID_UID : current_uid());
2722 
2723 	/* Bugfix: need to give ip_route_input enough of an IP header to
2724 	 * not gag.
2725 	 */
2726 	ip_hdr(skb)->protocol = IPPROTO_UDP;
2727 	ip_hdr(skb)->saddr = src;
2728 	ip_hdr(skb)->daddr = dst;
2729 
2730 	skb_reserve(skb, MAX_HEADER + sizeof(struct iphdr));
2731 
2732 	memset(&fl4, 0, sizeof(fl4));
2733 	fl4.daddr = dst;
2734 	fl4.saddr = src;
2735 	fl4.flowi4_tos = rtm->rtm_tos;
2736 	fl4.flowi4_oif = tb[RTA_OIF] ? nla_get_u32(tb[RTA_OIF]) : 0;
2737 	fl4.flowi4_mark = mark;
2738 	fl4.flowi4_uid = uid;
2739 
2740 	rcu_read_lock();
2741 
2742 	if (iif) {
2743 		struct net_device *dev;
2744 
2745 		dev = dev_get_by_index_rcu(net, iif);
2746 		if (!dev) {
2747 			err = -ENODEV;
2748 			goto errout_free;
2749 		}
2750 
2751 		skb->protocol	= htons(ETH_P_IP);
2752 		skb->dev	= dev;
2753 		skb->mark	= mark;
2754 		err = ip_route_input_rcu(skb, dst, src, rtm->rtm_tos,
2755 					 dev, &res);
2756 
2757 		rt = skb_rtable(skb);
2758 		if (err == 0 && rt->dst.error)
2759 			err = -rt->dst.error;
2760 	} else {
2761 		rt = ip_route_output_key_hash_rcu(net, &fl4, &res, skb);
2762 		err = 0;
2763 		if (IS_ERR(rt))
2764 			err = PTR_ERR(rt);
2765 		else
2766 			skb_dst_set(skb, &rt->dst);
2767 	}
2768 
2769 	if (err)
2770 		goto errout_free;
2771 
2772 	if (rtm->rtm_flags & RTM_F_NOTIFY)
2773 		rt->rt_flags |= RTCF_NOTIFY;
2774 
2775 	if (rtm->rtm_flags & RTM_F_LOOKUP_TABLE)
2776 		table_id = rt->rt_table_id;
2777 
2778 	if (rtm->rtm_flags & RTM_F_FIB_MATCH) {
2779 		if (!res.fi) {
2780 			err = fib_props[res.type].error;
2781 			if (!err)
2782 				err = -EHOSTUNREACH;
2783 			goto errout_free;
2784 		}
2785 		err = fib_dump_info(skb, NETLINK_CB(in_skb).portid,
2786 				    nlh->nlmsg_seq, RTM_NEWROUTE, table_id,
2787 				    rt->rt_type, res.prefix, res.prefixlen,
2788 				    fl4.flowi4_tos, res.fi, 0);
2789 	} else {
2790 		err = rt_fill_info(net, dst, src, table_id, &fl4, skb,
2791 				   NETLINK_CB(in_skb).portid, nlh->nlmsg_seq);
2792 	}
2793 	if (err < 0)
2794 		goto errout_free;
2795 
2796 	rcu_read_unlock();
2797 
2798 	err = rtnl_unicast(skb, net, NETLINK_CB(in_skb).portid);
2799 errout:
2800 	return err;
2801 
2802 errout_free:
2803 	rcu_read_unlock();
2804 	kfree_skb(skb);
2805 	goto errout;
2806 }
2807 
2808 void ip_rt_multicast_event(struct in_device *in_dev)
2809 {
2810 	rt_cache_flush(dev_net(in_dev->dev));
2811 }
2812 
2813 #ifdef CONFIG_SYSCTL
2814 static int ip_rt_gc_interval __read_mostly  = 60 * HZ;
2815 static int ip_rt_gc_min_interval __read_mostly	= HZ / 2;
2816 static int ip_rt_gc_elasticity __read_mostly	= 8;
2817 
2818 static int ipv4_sysctl_rtcache_flush(struct ctl_table *__ctl, int write,
2819 					void __user *buffer,
2820 					size_t *lenp, loff_t *ppos)
2821 {
2822 	struct net *net = (struct net *)__ctl->extra1;
2823 
2824 	if (write) {
2825 		rt_cache_flush(net);
2826 		fnhe_genid_bump(net);
2827 		return 0;
2828 	}
2829 
2830 	return -EINVAL;
2831 }
2832 
2833 static struct ctl_table ipv4_route_table[] = {
2834 	{
2835 		.procname	= "gc_thresh",
2836 		.data		= &ipv4_dst_ops.gc_thresh,
2837 		.maxlen		= sizeof(int),
2838 		.mode		= 0644,
2839 		.proc_handler	= proc_dointvec,
2840 	},
2841 	{
2842 		.procname	= "max_size",
2843 		.data		= &ip_rt_max_size,
2844 		.maxlen		= sizeof(int),
2845 		.mode		= 0644,
2846 		.proc_handler	= proc_dointvec,
2847 	},
2848 	{
2849 		/*  Deprecated. Use gc_min_interval_ms */
2850 
2851 		.procname	= "gc_min_interval",
2852 		.data		= &ip_rt_gc_min_interval,
2853 		.maxlen		= sizeof(int),
2854 		.mode		= 0644,
2855 		.proc_handler	= proc_dointvec_jiffies,
2856 	},
2857 	{
2858 		.procname	= "gc_min_interval_ms",
2859 		.data		= &ip_rt_gc_min_interval,
2860 		.maxlen		= sizeof(int),
2861 		.mode		= 0644,
2862 		.proc_handler	= proc_dointvec_ms_jiffies,
2863 	},
2864 	{
2865 		.procname	= "gc_timeout",
2866 		.data		= &ip_rt_gc_timeout,
2867 		.maxlen		= sizeof(int),
2868 		.mode		= 0644,
2869 		.proc_handler	= proc_dointvec_jiffies,
2870 	},
2871 	{
2872 		.procname	= "gc_interval",
2873 		.data		= &ip_rt_gc_interval,
2874 		.maxlen		= sizeof(int),
2875 		.mode		= 0644,
2876 		.proc_handler	= proc_dointvec_jiffies,
2877 	},
2878 	{
2879 		.procname	= "redirect_load",
2880 		.data		= &ip_rt_redirect_load,
2881 		.maxlen		= sizeof(int),
2882 		.mode		= 0644,
2883 		.proc_handler	= proc_dointvec,
2884 	},
2885 	{
2886 		.procname	= "redirect_number",
2887 		.data		= &ip_rt_redirect_number,
2888 		.maxlen		= sizeof(int),
2889 		.mode		= 0644,
2890 		.proc_handler	= proc_dointvec,
2891 	},
2892 	{
2893 		.procname	= "redirect_silence",
2894 		.data		= &ip_rt_redirect_silence,
2895 		.maxlen		= sizeof(int),
2896 		.mode		= 0644,
2897 		.proc_handler	= proc_dointvec,
2898 	},
2899 	{
2900 		.procname	= "error_cost",
2901 		.data		= &ip_rt_error_cost,
2902 		.maxlen		= sizeof(int),
2903 		.mode		= 0644,
2904 		.proc_handler	= proc_dointvec,
2905 	},
2906 	{
2907 		.procname	= "error_burst",
2908 		.data		= &ip_rt_error_burst,
2909 		.maxlen		= sizeof(int),
2910 		.mode		= 0644,
2911 		.proc_handler	= proc_dointvec,
2912 	},
2913 	{
2914 		.procname	= "gc_elasticity",
2915 		.data		= &ip_rt_gc_elasticity,
2916 		.maxlen		= sizeof(int),
2917 		.mode		= 0644,
2918 		.proc_handler	= proc_dointvec,
2919 	},
2920 	{
2921 		.procname	= "mtu_expires",
2922 		.data		= &ip_rt_mtu_expires,
2923 		.maxlen		= sizeof(int),
2924 		.mode		= 0644,
2925 		.proc_handler	= proc_dointvec_jiffies,
2926 	},
2927 	{
2928 		.procname	= "min_pmtu",
2929 		.data		= &ip_rt_min_pmtu,
2930 		.maxlen		= sizeof(int),
2931 		.mode		= 0644,
2932 		.proc_handler	= proc_dointvec,
2933 	},
2934 	{
2935 		.procname	= "min_adv_mss",
2936 		.data		= &ip_rt_min_advmss,
2937 		.maxlen		= sizeof(int),
2938 		.mode		= 0644,
2939 		.proc_handler	= proc_dointvec,
2940 	},
2941 	{ }
2942 };
2943 
2944 static struct ctl_table ipv4_route_flush_table[] = {
2945 	{
2946 		.procname	= "flush",
2947 		.maxlen		= sizeof(int),
2948 		.mode		= 0200,
2949 		.proc_handler	= ipv4_sysctl_rtcache_flush,
2950 	},
2951 	{ },
2952 };
2953 
2954 static __net_init int sysctl_route_net_init(struct net *net)
2955 {
2956 	struct ctl_table *tbl;
2957 
2958 	tbl = ipv4_route_flush_table;
2959 	if (!net_eq(net, &init_net)) {
2960 		tbl = kmemdup(tbl, sizeof(ipv4_route_flush_table), GFP_KERNEL);
2961 		if (!tbl)
2962 			goto err_dup;
2963 
2964 		/* Don't export sysctls to unprivileged users */
2965 		if (net->user_ns != &init_user_ns)
2966 			tbl[0].procname = NULL;
2967 	}
2968 	tbl[0].extra1 = net;
2969 
2970 	net->ipv4.route_hdr = register_net_sysctl(net, "net/ipv4/route", tbl);
2971 	if (!net->ipv4.route_hdr)
2972 		goto err_reg;
2973 	return 0;
2974 
2975 err_reg:
2976 	if (tbl != ipv4_route_flush_table)
2977 		kfree(tbl);
2978 err_dup:
2979 	return -ENOMEM;
2980 }
2981 
2982 static __net_exit void sysctl_route_net_exit(struct net *net)
2983 {
2984 	struct ctl_table *tbl;
2985 
2986 	tbl = net->ipv4.route_hdr->ctl_table_arg;
2987 	unregister_net_sysctl_table(net->ipv4.route_hdr);
2988 	BUG_ON(tbl == ipv4_route_flush_table);
2989 	kfree(tbl);
2990 }
2991 
2992 static __net_initdata struct pernet_operations sysctl_route_ops = {
2993 	.init = sysctl_route_net_init,
2994 	.exit = sysctl_route_net_exit,
2995 };
2996 #endif
2997 
2998 static __net_init int rt_genid_init(struct net *net)
2999 {
3000 	atomic_set(&net->ipv4.rt_genid, 0);
3001 	atomic_set(&net->fnhe_genid, 0);
3002 	atomic_set(&net->ipv4.dev_addr_genid, get_random_int());
3003 	return 0;
3004 }
3005 
3006 static __net_initdata struct pernet_operations rt_genid_ops = {
3007 	.init = rt_genid_init,
3008 };
3009 
3010 static int __net_init ipv4_inetpeer_init(struct net *net)
3011 {
3012 	struct inet_peer_base *bp = kmalloc(sizeof(*bp), GFP_KERNEL);
3013 
3014 	if (!bp)
3015 		return -ENOMEM;
3016 	inet_peer_base_init(bp);
3017 	net->ipv4.peers = bp;
3018 	return 0;
3019 }
3020 
3021 static void __net_exit ipv4_inetpeer_exit(struct net *net)
3022 {
3023 	struct inet_peer_base *bp = net->ipv4.peers;
3024 
3025 	net->ipv4.peers = NULL;
3026 	inetpeer_invalidate_tree(bp);
3027 	kfree(bp);
3028 }
3029 
3030 static __net_initdata struct pernet_operations ipv4_inetpeer_ops = {
3031 	.init	=	ipv4_inetpeer_init,
3032 	.exit	=	ipv4_inetpeer_exit,
3033 };
3034 
3035 #ifdef CONFIG_IP_ROUTE_CLASSID
3036 struct ip_rt_acct __percpu *ip_rt_acct __read_mostly;
3037 #endif /* CONFIG_IP_ROUTE_CLASSID */
3038 
3039 int __init ip_rt_init(void)
3040 {
3041 	int rc = 0;
3042 	int cpu;
3043 
3044 	ip_idents = kmalloc(IP_IDENTS_SZ * sizeof(*ip_idents), GFP_KERNEL);
3045 	if (!ip_idents)
3046 		panic("IP: failed to allocate ip_idents\n");
3047 
3048 	prandom_bytes(ip_idents, IP_IDENTS_SZ * sizeof(*ip_idents));
3049 
3050 	ip_tstamps = kcalloc(IP_IDENTS_SZ, sizeof(*ip_tstamps), GFP_KERNEL);
3051 	if (!ip_tstamps)
3052 		panic("IP: failed to allocate ip_tstamps\n");
3053 
3054 	for_each_possible_cpu(cpu) {
3055 		struct uncached_list *ul = &per_cpu(rt_uncached_list, cpu);
3056 
3057 		INIT_LIST_HEAD(&ul->head);
3058 		spin_lock_init(&ul->lock);
3059 	}
3060 #ifdef CONFIG_IP_ROUTE_CLASSID
3061 	ip_rt_acct = __alloc_percpu(256 * sizeof(struct ip_rt_acct), __alignof__(struct ip_rt_acct));
3062 	if (!ip_rt_acct)
3063 		panic("IP: failed to allocate ip_rt_acct\n");
3064 #endif
3065 
3066 	ipv4_dst_ops.kmem_cachep =
3067 		kmem_cache_create("ip_dst_cache", sizeof(struct rtable), 0,
3068 				  SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL);
3069 
3070 	ipv4_dst_blackhole_ops.kmem_cachep = ipv4_dst_ops.kmem_cachep;
3071 
3072 	if (dst_entries_init(&ipv4_dst_ops) < 0)
3073 		panic("IP: failed to allocate ipv4_dst_ops counter\n");
3074 
3075 	if (dst_entries_init(&ipv4_dst_blackhole_ops) < 0)
3076 		panic("IP: failed to allocate ipv4_dst_blackhole_ops counter\n");
3077 
3078 	ipv4_dst_ops.gc_thresh = ~0;
3079 	ip_rt_max_size = INT_MAX;
3080 
3081 	devinet_init();
3082 	ip_fib_init();
3083 
3084 	if (ip_rt_proc_init())
3085 		pr_err("Unable to create route proc files\n");
3086 #ifdef CONFIG_XFRM
3087 	xfrm_init();
3088 	xfrm4_init();
3089 #endif
3090 	rtnl_register(PF_INET, RTM_GETROUTE, inet_rtm_getroute, NULL,
3091 		      RTNL_FLAG_DOIT_UNLOCKED);
3092 
3093 #ifdef CONFIG_SYSCTL
3094 	register_pernet_subsys(&sysctl_route_ops);
3095 #endif
3096 	register_pernet_subsys(&rt_genid_ops);
3097 	register_pernet_subsys(&ipv4_inetpeer_ops);
3098 	return rc;
3099 }
3100 
3101 #ifdef CONFIG_SYSCTL
3102 /*
3103  * We really need to sanitize the damn ipv4 init order, then all
3104  * this nonsense will go away.
3105  */
3106 void __init ip_static_sysctl_init(void)
3107 {
3108 	register_net_sysctl(&init_net, "net/ipv4/route", ipv4_route_table);
3109 }
3110 #endif
3111