1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3 * Linux INET6 implementation
4 * FIB front-end.
5 *
6 * Authors:
7 * Pedro Roque <roque@di.fc.ul.pt>
8 */
9
10 /* Changes:
11 *
12 * YOSHIFUJI Hideaki @USAGI
13 * reworked default router selection.
14 * - respect outgoing interface
15 * - select from (probably) reachable routers (i.e.
16 * routers in REACHABLE, STALE, DELAY or PROBE states).
17 * - always select the same router if it is (probably)
18 * reachable. otherwise, round-robin the list.
19 * Ville Nuorvala
20 * Fixed routing subtrees.
21 */
22
23 #define pr_fmt(fmt) "IPv6: " fmt
24
25 #include <linux/capability.h>
26 #include <linux/errno.h>
27 #include <linux/export.h>
28 #include <linux/types.h>
29 #include <linux/times.h>
30 #include <linux/socket.h>
31 #include <linux/sockios.h>
32 #include <linux/net.h>
33 #include <linux/route.h>
34 #include <linux/netdevice.h>
35 #include <linux/in6.h>
36 #include <linux/mroute6.h>
37 #include <linux/init.h>
38 #include <linux/if_arp.h>
39 #include <linux/proc_fs.h>
40 #include <linux/seq_file.h>
41 #include <linux/nsproxy.h>
42 #include <linux/slab.h>
43 #include <linux/jhash.h>
44 #include <linux/siphash.h>
45 #include <net/net_namespace.h>
46 #include <net/snmp.h>
47 #include <net/ipv6.h>
48 #include <net/ip6_fib.h>
49 #include <net/ip6_route.h>
50 #include <net/ndisc.h>
51 #include <net/addrconf.h>
52 #include <net/tcp.h>
53 #include <linux/rtnetlink.h>
54 #include <net/dst.h>
55 #include <net/dst_metadata.h>
56 #include <net/xfrm.h>
57 #include <net/netevent.h>
58 #include <net/netlink.h>
59 #include <net/rtnh.h>
60 #include <net/lwtunnel.h>
61 #include <net/ip_tunnels.h>
62 #include <net/l3mdev.h>
63 #include <net/ip.h>
64 #include <linux/uaccess.h>
65 #include <linux/btf_ids.h>
66
67 #ifdef CONFIG_SYSCTL
68 #include <linux/sysctl.h>
69 #endif
70
71 static int ip6_rt_type_to_error(u8 fib6_type);
72
73 #define CREATE_TRACE_POINTS
74 #include <trace/events/fib6.h>
75 EXPORT_TRACEPOINT_SYMBOL_GPL(fib6_table_lookup);
76 #undef CREATE_TRACE_POINTS
77
78 enum rt6_nud_state {
79 RT6_NUD_FAIL_HARD = -3,
80 RT6_NUD_FAIL_PROBE = -2,
81 RT6_NUD_FAIL_DO_RR = -1,
82 RT6_NUD_SUCCEED = 1
83 };
84
85 INDIRECT_CALLABLE_SCOPE
86 struct dst_entry *ip6_dst_check(struct dst_entry *dst, u32 cookie);
87 static unsigned int ip6_default_advmss(const struct dst_entry *dst);
88 INDIRECT_CALLABLE_SCOPE
89 unsigned int ip6_mtu(const struct dst_entry *dst);
90 static void ip6_negative_advice(struct sock *sk,
91 struct dst_entry *dst);
92 static void ip6_dst_destroy(struct dst_entry *);
93 static void ip6_dst_ifdown(struct dst_entry *,
94 struct net_device *dev);
95 static void ip6_dst_gc(struct dst_ops *ops);
96
97 static int ip6_pkt_discard(struct sk_buff *skb);
98 static int ip6_pkt_discard_out(struct net *net, struct sock *sk, struct sk_buff *skb);
99 static int ip6_pkt_prohibit(struct sk_buff *skb);
100 static int ip6_pkt_prohibit_out(struct net *net, struct sock *sk, struct sk_buff *skb);
101 static void ip6_link_failure(struct sk_buff *skb);
102 static void ip6_rt_update_pmtu(struct dst_entry *dst, struct sock *sk,
103 struct sk_buff *skb, u32 mtu,
104 bool confirm_neigh);
105 static void rt6_do_redirect(struct dst_entry *dst, struct sock *sk,
106 struct sk_buff *skb);
107 static int rt6_score_route(const struct fib6_nh *nh, u32 fib6_flags, int oif,
108 int strict);
109 static size_t rt6_nlmsg_size(struct fib6_info *f6i);
110 static int rt6_fill_node(struct net *net, struct sk_buff *skb,
111 struct fib6_info *rt, struct dst_entry *dst,
112 struct in6_addr *dest, struct in6_addr *src,
113 int iif, int type, u32 portid, u32 seq,
114 unsigned int flags);
115 static struct rt6_info *rt6_find_cached_rt(const struct fib6_result *res,
116 const struct in6_addr *daddr,
117 const struct in6_addr *saddr);
118
119 #ifdef CONFIG_IPV6_ROUTE_INFO
120 static struct fib6_info *rt6_add_route_info(struct net *net,
121 const struct in6_addr *prefix, int prefixlen,
122 const struct in6_addr *gwaddr,
123 struct net_device *dev,
124 unsigned int pref);
125 static struct fib6_info *rt6_get_route_info(struct net *net,
126 const struct in6_addr *prefix, int prefixlen,
127 const struct in6_addr *gwaddr,
128 struct net_device *dev);
129 #endif
130
131 struct uncached_list {
132 spinlock_t lock;
133 struct list_head head;
134 struct list_head quarantine;
135 };
136
137 static DEFINE_PER_CPU_ALIGNED(struct uncached_list, rt6_uncached_list);
138
rt6_uncached_list_add(struct rt6_info * rt)139 void rt6_uncached_list_add(struct rt6_info *rt)
140 {
141 struct uncached_list *ul = raw_cpu_ptr(&rt6_uncached_list);
142
143 rt->dst.rt_uncached_list = ul;
144
145 spin_lock_bh(&ul->lock);
146 list_add_tail(&rt->dst.rt_uncached, &ul->head);
147 spin_unlock_bh(&ul->lock);
148 }
149
rt6_uncached_list_del(struct rt6_info * rt)150 void rt6_uncached_list_del(struct rt6_info *rt)
151 {
152 if (!list_empty(&rt->dst.rt_uncached)) {
153 struct uncached_list *ul = rt->dst.rt_uncached_list;
154
155 spin_lock_bh(&ul->lock);
156 list_del_init(&rt->dst.rt_uncached);
157 spin_unlock_bh(&ul->lock);
158 }
159 }
160
rt6_uncached_list_flush_dev(struct net_device * dev)161 static void rt6_uncached_list_flush_dev(struct net_device *dev)
162 {
163 int cpu;
164
165 for_each_possible_cpu(cpu) {
166 struct uncached_list *ul = per_cpu_ptr(&rt6_uncached_list, cpu);
167 struct rt6_info *rt, *safe;
168
169 if (list_empty(&ul->head))
170 continue;
171
172 spin_lock_bh(&ul->lock);
173 list_for_each_entry_safe(rt, safe, &ul->head, dst.rt_uncached) {
174 struct inet6_dev *rt_idev = rt->rt6i_idev;
175 struct net_device *rt_dev = rt->dst.dev;
176 bool handled = false;
177
178 if (rt_idev && rt_idev->dev == dev) {
179 rt->rt6i_idev = in6_dev_get(blackhole_netdev);
180 in6_dev_put(rt_idev);
181 handled = true;
182 }
183
184 if (rt_dev == dev) {
185 rt->dst.dev = blackhole_netdev;
186 netdev_ref_replace(rt_dev, blackhole_netdev,
187 &rt->dst.dev_tracker,
188 GFP_ATOMIC);
189 handled = true;
190 }
191 if (handled)
192 list_move(&rt->dst.rt_uncached,
193 &ul->quarantine);
194 }
195 spin_unlock_bh(&ul->lock);
196 }
197 }
198
choose_neigh_daddr(const struct in6_addr * p,struct sk_buff * skb,const void * daddr)199 static inline const void *choose_neigh_daddr(const struct in6_addr *p,
200 struct sk_buff *skb,
201 const void *daddr)
202 {
203 if (!ipv6_addr_any(p))
204 return (const void *) p;
205 else if (skb)
206 return &ipv6_hdr(skb)->daddr;
207 return daddr;
208 }
209
ip6_neigh_lookup(const struct in6_addr * gw,struct net_device * dev,struct sk_buff * skb,const void * daddr)210 struct neighbour *ip6_neigh_lookup(const struct in6_addr *gw,
211 struct net_device *dev,
212 struct sk_buff *skb,
213 const void *daddr)
214 {
215 struct neighbour *n;
216
217 daddr = choose_neigh_daddr(gw, skb, daddr);
218 n = __ipv6_neigh_lookup(dev, daddr);
219 if (n)
220 return n;
221
222 n = neigh_create(&nd_tbl, daddr, dev);
223 return IS_ERR(n) ? NULL : n;
224 }
225
ip6_dst_neigh_lookup(const struct dst_entry * dst,struct sk_buff * skb,const void * daddr)226 static struct neighbour *ip6_dst_neigh_lookup(const struct dst_entry *dst,
227 struct sk_buff *skb,
228 const void *daddr)
229 {
230 const struct rt6_info *rt = dst_rt6_info(dst);
231
232 return ip6_neigh_lookup(rt6_nexthop(rt, &in6addr_any),
233 dst->dev, skb, daddr);
234 }
235
ip6_confirm_neigh(const struct dst_entry * dst,const void * daddr)236 static void ip6_confirm_neigh(const struct dst_entry *dst, const void *daddr)
237 {
238 const struct rt6_info *rt = dst_rt6_info(dst);
239 struct net_device *dev = dst->dev;
240
241 daddr = choose_neigh_daddr(rt6_nexthop(rt, &in6addr_any), NULL, daddr);
242 if (!daddr)
243 return;
244 if (dev->flags & (IFF_NOARP | IFF_LOOPBACK))
245 return;
246 if (ipv6_addr_is_multicast((const struct in6_addr *)daddr))
247 return;
248 __ipv6_confirm_neigh(dev, daddr);
249 }
250
251 static struct dst_ops ip6_dst_ops_template = {
252 .family = AF_INET6,
253 .gc = ip6_dst_gc,
254 .gc_thresh = 1024,
255 .check = ip6_dst_check,
256 .default_advmss = ip6_default_advmss,
257 .mtu = ip6_mtu,
258 .cow_metrics = dst_cow_metrics_generic,
259 .destroy = ip6_dst_destroy,
260 .ifdown = ip6_dst_ifdown,
261 .negative_advice = ip6_negative_advice,
262 .link_failure = ip6_link_failure,
263 .update_pmtu = ip6_rt_update_pmtu,
264 .redirect = rt6_do_redirect,
265 .local_out = __ip6_local_out,
266 .neigh_lookup = ip6_dst_neigh_lookup,
267 .confirm_neigh = ip6_confirm_neigh,
268 };
269
270 static struct dst_ops ip6_dst_blackhole_ops = {
271 .family = AF_INET6,
272 .default_advmss = ip6_default_advmss,
273 .neigh_lookup = ip6_dst_neigh_lookup,
274 .check = ip6_dst_check,
275 .destroy = ip6_dst_destroy,
276 .cow_metrics = dst_cow_metrics_generic,
277 .update_pmtu = dst_blackhole_update_pmtu,
278 .redirect = dst_blackhole_redirect,
279 .mtu = dst_blackhole_mtu,
280 };
281
282 static const u32 ip6_template_metrics[RTAX_MAX] = {
283 [RTAX_HOPLIMIT - 1] = 0,
284 };
285
286 static const struct fib6_info fib6_null_entry_template = {
287 .fib6_flags = (RTF_REJECT | RTF_NONEXTHOP),
288 .fib6_protocol = RTPROT_KERNEL,
289 .fib6_metric = ~(u32)0,
290 .fib6_ref = REFCOUNT_INIT(1),
291 .fib6_type = RTN_UNREACHABLE,
292 .fib6_metrics = (struct dst_metrics *)&dst_default_metrics,
293 };
294
295 static const struct rt6_info ip6_null_entry_template = {
296 .dst = {
297 .__rcuref = RCUREF_INIT(1),
298 .__use = 1,
299 .obsolete = DST_OBSOLETE_FORCE_CHK,
300 .error = -ENETUNREACH,
301 .input = ip6_pkt_discard,
302 .output = ip6_pkt_discard_out,
303 },
304 .rt6i_flags = (RTF_REJECT | RTF_NONEXTHOP),
305 };
306
307 #ifdef CONFIG_IPV6_MULTIPLE_TABLES
308
309 static const struct rt6_info ip6_prohibit_entry_template = {
310 .dst = {
311 .__rcuref = RCUREF_INIT(1),
312 .__use = 1,
313 .obsolete = DST_OBSOLETE_FORCE_CHK,
314 .error = -EACCES,
315 .input = ip6_pkt_prohibit,
316 .output = ip6_pkt_prohibit_out,
317 },
318 .rt6i_flags = (RTF_REJECT | RTF_NONEXTHOP),
319 };
320
321 static const struct rt6_info ip6_blk_hole_entry_template = {
322 .dst = {
323 .__rcuref = RCUREF_INIT(1),
324 .__use = 1,
325 .obsolete = DST_OBSOLETE_FORCE_CHK,
326 .error = -EINVAL,
327 .input = dst_discard,
328 .output = dst_discard_out,
329 },
330 .rt6i_flags = (RTF_REJECT | RTF_NONEXTHOP),
331 };
332
333 #endif
334
rt6_info_init(struct rt6_info * rt)335 static void rt6_info_init(struct rt6_info *rt)
336 {
337 memset_after(rt, 0, dst);
338 }
339
340 /* allocate dst with ip6_dst_ops */
ip6_dst_alloc(struct net * net,struct net_device * dev,int flags)341 struct rt6_info *ip6_dst_alloc(struct net *net, struct net_device *dev,
342 int flags)
343 {
344 struct rt6_info *rt = dst_alloc(&net->ipv6.ip6_dst_ops, dev,
345 1, DST_OBSOLETE_FORCE_CHK, flags);
346
347 if (rt) {
348 rt6_info_init(rt);
349 atomic_inc(&net->ipv6.rt6_stats->fib_rt_alloc);
350 }
351
352 return rt;
353 }
354 EXPORT_SYMBOL(ip6_dst_alloc);
355
ip6_dst_destroy(struct dst_entry * dst)356 static void ip6_dst_destroy(struct dst_entry *dst)
357 {
358 struct rt6_info *rt = dst_rt6_info(dst);
359 struct fib6_info *from;
360 struct inet6_dev *idev;
361
362 ip_dst_metrics_put(dst);
363 rt6_uncached_list_del(rt);
364
365 idev = rt->rt6i_idev;
366 if (idev) {
367 rt->rt6i_idev = NULL;
368 in6_dev_put(idev);
369 }
370
371 from = unrcu_pointer(xchg(&rt->from, NULL));
372 fib6_info_release(from);
373 }
374
ip6_dst_ifdown(struct dst_entry * dst,struct net_device * dev)375 static void ip6_dst_ifdown(struct dst_entry *dst, struct net_device *dev)
376 {
377 struct rt6_info *rt = dst_rt6_info(dst);
378 struct inet6_dev *idev = rt->rt6i_idev;
379 struct fib6_info *from;
380
381 if (idev && idev->dev != blackhole_netdev) {
382 struct inet6_dev *blackhole_idev = in6_dev_get(blackhole_netdev);
383
384 if (blackhole_idev) {
385 rt->rt6i_idev = blackhole_idev;
386 in6_dev_put(idev);
387 }
388 }
389 from = unrcu_pointer(xchg(&rt->from, NULL));
390 fib6_info_release(from);
391 }
392
__rt6_check_expired(const struct rt6_info * rt)393 static bool __rt6_check_expired(const struct rt6_info *rt)
394 {
395 if (rt->rt6i_flags & RTF_EXPIRES)
396 return time_after(jiffies, rt->dst.expires);
397 else
398 return false;
399 }
400
rt6_check_expired(const struct rt6_info * rt)401 static bool rt6_check_expired(const struct rt6_info *rt)
402 {
403 struct fib6_info *from;
404
405 from = rcu_dereference(rt->from);
406
407 if (rt->rt6i_flags & RTF_EXPIRES) {
408 if (time_after(jiffies, rt->dst.expires))
409 return true;
410 } else if (from) {
411 return rt->dst.obsolete != DST_OBSOLETE_FORCE_CHK ||
412 fib6_check_expired(from);
413 }
414 return false;
415 }
416
fib6_select_path(const struct net * net,struct fib6_result * res,struct flowi6 * fl6,int oif,bool have_oif_match,const struct sk_buff * skb,int strict)417 void fib6_select_path(const struct net *net, struct fib6_result *res,
418 struct flowi6 *fl6, int oif, bool have_oif_match,
419 const struct sk_buff *skb, int strict)
420 {
421 struct fib6_info *match = res->f6i;
422 struct fib6_info *sibling;
423
424 if (!match->nh && (!match->fib6_nsiblings || have_oif_match))
425 goto out;
426
427 if (match->nh && have_oif_match && res->nh)
428 return;
429
430 if (skb)
431 IP6CB(skb)->flags |= IP6SKB_MULTIPATH;
432
433 /* We might have already computed the hash for ICMPv6 errors. In such
434 * case it will always be non-zero. Otherwise now is the time to do it.
435 */
436 if (!fl6->mp_hash &&
437 (!match->nh || nexthop_is_multipath(match->nh)))
438 fl6->mp_hash = rt6_multipath_hash(net, fl6, skb, NULL);
439
440 if (unlikely(match->nh)) {
441 nexthop_path_fib6_result(res, fl6->mp_hash);
442 return;
443 }
444
445 if (fl6->mp_hash <= atomic_read(&match->fib6_nh->fib_nh_upper_bound))
446 goto out;
447
448 list_for_each_entry_rcu(sibling, &match->fib6_siblings,
449 fib6_siblings) {
450 const struct fib6_nh *nh = sibling->fib6_nh;
451 int nh_upper_bound;
452
453 nh_upper_bound = atomic_read(&nh->fib_nh_upper_bound);
454 if (fl6->mp_hash > nh_upper_bound)
455 continue;
456 if (rt6_score_route(nh, sibling->fib6_flags, oif, strict) < 0)
457 break;
458 match = sibling;
459 break;
460 }
461
462 out:
463 res->f6i = match;
464 res->nh = match->fib6_nh;
465 }
466
467 /*
468 * Route lookup. rcu_read_lock() should be held.
469 */
470
__rt6_device_match(struct net * net,const struct fib6_nh * nh,const struct in6_addr * saddr,int oif,int flags)471 static bool __rt6_device_match(struct net *net, const struct fib6_nh *nh,
472 const struct in6_addr *saddr, int oif, int flags)
473 {
474 const struct net_device *dev;
475
476 if (nh->fib_nh_flags & RTNH_F_DEAD)
477 return false;
478
479 dev = nh->fib_nh_dev;
480 if (oif) {
481 if (dev->ifindex == oif)
482 return true;
483 } else {
484 if (ipv6_chk_addr(net, saddr, dev,
485 flags & RT6_LOOKUP_F_IFACE))
486 return true;
487 }
488
489 return false;
490 }
491
492 struct fib6_nh_dm_arg {
493 struct net *net;
494 const struct in6_addr *saddr;
495 int oif;
496 int flags;
497 struct fib6_nh *nh;
498 };
499
__rt6_nh_dev_match(struct fib6_nh * nh,void * _arg)500 static int __rt6_nh_dev_match(struct fib6_nh *nh, void *_arg)
501 {
502 struct fib6_nh_dm_arg *arg = _arg;
503
504 arg->nh = nh;
505 return __rt6_device_match(arg->net, nh, arg->saddr, arg->oif,
506 arg->flags);
507 }
508
509 /* returns fib6_nh from nexthop or NULL */
rt6_nh_dev_match(struct net * net,struct nexthop * nh,struct fib6_result * res,const struct in6_addr * saddr,int oif,int flags)510 static struct fib6_nh *rt6_nh_dev_match(struct net *net, struct nexthop *nh,
511 struct fib6_result *res,
512 const struct in6_addr *saddr,
513 int oif, int flags)
514 {
515 struct fib6_nh_dm_arg arg = {
516 .net = net,
517 .saddr = saddr,
518 .oif = oif,
519 .flags = flags,
520 };
521
522 if (nexthop_is_blackhole(nh))
523 return NULL;
524
525 if (nexthop_for_each_fib6_nh(nh, __rt6_nh_dev_match, &arg))
526 return arg.nh;
527
528 return NULL;
529 }
530
rt6_device_match(struct net * net,struct fib6_result * res,const struct in6_addr * saddr,int oif,int flags)531 static void rt6_device_match(struct net *net, struct fib6_result *res,
532 const struct in6_addr *saddr, int oif, int flags)
533 {
534 struct fib6_info *f6i = res->f6i;
535 struct fib6_info *spf6i;
536 struct fib6_nh *nh;
537
538 if (!oif && ipv6_addr_any(saddr)) {
539 if (unlikely(f6i->nh)) {
540 nh = nexthop_fib6_nh(f6i->nh);
541 if (nexthop_is_blackhole(f6i->nh))
542 goto out_blackhole;
543 } else {
544 nh = f6i->fib6_nh;
545 }
546 if (!(nh->fib_nh_flags & RTNH_F_DEAD))
547 goto out;
548 }
549
550 for (spf6i = f6i; spf6i; spf6i = rcu_dereference(spf6i->fib6_next)) {
551 bool matched = false;
552
553 if (unlikely(spf6i->nh)) {
554 nh = rt6_nh_dev_match(net, spf6i->nh, res, saddr,
555 oif, flags);
556 if (nh)
557 matched = true;
558 } else {
559 nh = spf6i->fib6_nh;
560 if (__rt6_device_match(net, nh, saddr, oif, flags))
561 matched = true;
562 }
563 if (matched) {
564 res->f6i = spf6i;
565 goto out;
566 }
567 }
568
569 if (oif && flags & RT6_LOOKUP_F_IFACE) {
570 res->f6i = net->ipv6.fib6_null_entry;
571 nh = res->f6i->fib6_nh;
572 goto out;
573 }
574
575 if (unlikely(f6i->nh)) {
576 nh = nexthop_fib6_nh(f6i->nh);
577 if (nexthop_is_blackhole(f6i->nh))
578 goto out_blackhole;
579 } else {
580 nh = f6i->fib6_nh;
581 }
582
583 if (nh->fib_nh_flags & RTNH_F_DEAD) {
584 res->f6i = net->ipv6.fib6_null_entry;
585 nh = res->f6i->fib6_nh;
586 }
587 out:
588 res->nh = nh;
589 res->fib6_type = res->f6i->fib6_type;
590 res->fib6_flags = res->f6i->fib6_flags;
591 return;
592
593 out_blackhole:
594 res->fib6_flags |= RTF_REJECT;
595 res->fib6_type = RTN_BLACKHOLE;
596 res->nh = nh;
597 }
598
599 #ifdef CONFIG_IPV6_ROUTER_PREF
600 struct __rt6_probe_work {
601 struct work_struct work;
602 struct in6_addr target;
603 struct net_device *dev;
604 netdevice_tracker dev_tracker;
605 };
606
rt6_probe_deferred(struct work_struct * w)607 static void rt6_probe_deferred(struct work_struct *w)
608 {
609 struct in6_addr mcaddr;
610 struct __rt6_probe_work *work =
611 container_of(w, struct __rt6_probe_work, work);
612
613 addrconf_addr_solict_mult(&work->target, &mcaddr);
614 ndisc_send_ns(work->dev, &work->target, &mcaddr, NULL, 0);
615 netdev_put(work->dev, &work->dev_tracker);
616 kfree(work);
617 }
618
rt6_probe(struct fib6_nh * fib6_nh)619 static void rt6_probe(struct fib6_nh *fib6_nh)
620 {
621 struct __rt6_probe_work *work = NULL;
622 const struct in6_addr *nh_gw;
623 unsigned long last_probe;
624 struct neighbour *neigh;
625 struct net_device *dev;
626 struct inet6_dev *idev;
627
628 /*
629 * Okay, this does not seem to be appropriate
630 * for now, however, we need to check if it
631 * is really so; aka Router Reachability Probing.
632 *
633 * Router Reachability Probe MUST be rate-limited
634 * to no more than one per minute.
635 */
636 if (!fib6_nh->fib_nh_gw_family)
637 return;
638
639 nh_gw = &fib6_nh->fib_nh_gw6;
640 dev = fib6_nh->fib_nh_dev;
641 rcu_read_lock();
642 last_probe = READ_ONCE(fib6_nh->last_probe);
643 idev = __in6_dev_get(dev);
644 if (!idev)
645 goto out;
646 neigh = __ipv6_neigh_lookup_noref(dev, nh_gw);
647 if (neigh) {
648 if (READ_ONCE(neigh->nud_state) & NUD_VALID)
649 goto out;
650
651 write_lock_bh(&neigh->lock);
652 if (!(neigh->nud_state & NUD_VALID) &&
653 time_after(jiffies,
654 neigh->updated + idev->cnf.rtr_probe_interval)) {
655 work = kmalloc(sizeof(*work), GFP_ATOMIC);
656 if (work)
657 __neigh_set_probe_once(neigh);
658 }
659 write_unlock_bh(&neigh->lock);
660 } else if (time_after(jiffies, last_probe +
661 idev->cnf.rtr_probe_interval)) {
662 work = kmalloc(sizeof(*work), GFP_ATOMIC);
663 }
664
665 if (!work || cmpxchg(&fib6_nh->last_probe,
666 last_probe, jiffies) != last_probe) {
667 kfree(work);
668 } else {
669 INIT_WORK(&work->work, rt6_probe_deferred);
670 work->target = *nh_gw;
671 netdev_hold(dev, &work->dev_tracker, GFP_ATOMIC);
672 work->dev = dev;
673 schedule_work(&work->work);
674 }
675
676 out:
677 rcu_read_unlock();
678 }
679 #else
rt6_probe(struct fib6_nh * fib6_nh)680 static inline void rt6_probe(struct fib6_nh *fib6_nh)
681 {
682 }
683 #endif
684
685 /*
686 * Default Router Selection (RFC 2461 6.3.6)
687 */
rt6_check_neigh(const struct fib6_nh * fib6_nh)688 static enum rt6_nud_state rt6_check_neigh(const struct fib6_nh *fib6_nh)
689 {
690 enum rt6_nud_state ret = RT6_NUD_FAIL_HARD;
691 struct neighbour *neigh;
692
693 rcu_read_lock();
694 neigh = __ipv6_neigh_lookup_noref(fib6_nh->fib_nh_dev,
695 &fib6_nh->fib_nh_gw6);
696 if (neigh) {
697 u8 nud_state = READ_ONCE(neigh->nud_state);
698
699 if (nud_state & NUD_VALID)
700 ret = RT6_NUD_SUCCEED;
701 #ifdef CONFIG_IPV6_ROUTER_PREF
702 else if (!(nud_state & NUD_FAILED))
703 ret = RT6_NUD_SUCCEED;
704 else
705 ret = RT6_NUD_FAIL_PROBE;
706 #endif
707 } else {
708 ret = IS_ENABLED(CONFIG_IPV6_ROUTER_PREF) ?
709 RT6_NUD_SUCCEED : RT6_NUD_FAIL_DO_RR;
710 }
711 rcu_read_unlock();
712
713 return ret;
714 }
715
rt6_score_route(const struct fib6_nh * nh,u32 fib6_flags,int oif,int strict)716 static int rt6_score_route(const struct fib6_nh *nh, u32 fib6_flags, int oif,
717 int strict)
718 {
719 int m = 0;
720
721 if (!oif || nh->fib_nh_dev->ifindex == oif)
722 m = 2;
723
724 if (!m && (strict & RT6_LOOKUP_F_IFACE))
725 return RT6_NUD_FAIL_HARD;
726 #ifdef CONFIG_IPV6_ROUTER_PREF
727 m |= IPV6_DECODE_PREF(IPV6_EXTRACT_PREF(fib6_flags)) << 2;
728 #endif
729 if ((strict & RT6_LOOKUP_F_REACHABLE) &&
730 !(fib6_flags & RTF_NONEXTHOP) && nh->fib_nh_gw_family) {
731 int n = rt6_check_neigh(nh);
732 if (n < 0)
733 return n;
734 }
735 return m;
736 }
737
find_match(struct fib6_nh * nh,u32 fib6_flags,int oif,int strict,int * mpri,bool * do_rr)738 static bool find_match(struct fib6_nh *nh, u32 fib6_flags,
739 int oif, int strict, int *mpri, bool *do_rr)
740 {
741 bool match_do_rr = false;
742 bool rc = false;
743 int m;
744
745 if (nh->fib_nh_flags & RTNH_F_DEAD)
746 goto out;
747
748 if (ip6_ignore_linkdown(nh->fib_nh_dev) &&
749 nh->fib_nh_flags & RTNH_F_LINKDOWN &&
750 !(strict & RT6_LOOKUP_F_IGNORE_LINKSTATE))
751 goto out;
752
753 m = rt6_score_route(nh, fib6_flags, oif, strict);
754 if (m == RT6_NUD_FAIL_DO_RR) {
755 match_do_rr = true;
756 m = 0; /* lowest valid score */
757 } else if (m == RT6_NUD_FAIL_HARD) {
758 goto out;
759 }
760
761 if (strict & RT6_LOOKUP_F_REACHABLE)
762 rt6_probe(nh);
763
764 /* note that m can be RT6_NUD_FAIL_PROBE at this point */
765 if (m > *mpri) {
766 *do_rr = match_do_rr;
767 *mpri = m;
768 rc = true;
769 }
770 out:
771 return rc;
772 }
773
774 struct fib6_nh_frl_arg {
775 u32 flags;
776 int oif;
777 int strict;
778 int *mpri;
779 bool *do_rr;
780 struct fib6_nh *nh;
781 };
782
rt6_nh_find_match(struct fib6_nh * nh,void * _arg)783 static int rt6_nh_find_match(struct fib6_nh *nh, void *_arg)
784 {
785 struct fib6_nh_frl_arg *arg = _arg;
786
787 arg->nh = nh;
788 return find_match(nh, arg->flags, arg->oif, arg->strict,
789 arg->mpri, arg->do_rr);
790 }
791
__find_rr_leaf(struct fib6_info * f6i_start,struct fib6_info * nomatch,u32 metric,struct fib6_result * res,struct fib6_info ** cont,int oif,int strict,bool * do_rr,int * mpri)792 static void __find_rr_leaf(struct fib6_info *f6i_start,
793 struct fib6_info *nomatch, u32 metric,
794 struct fib6_result *res, struct fib6_info **cont,
795 int oif, int strict, bool *do_rr, int *mpri)
796 {
797 struct fib6_info *f6i;
798
799 for (f6i = f6i_start;
800 f6i && f6i != nomatch;
801 f6i = rcu_dereference(f6i->fib6_next)) {
802 bool matched = false;
803 struct fib6_nh *nh;
804
805 if (cont && f6i->fib6_metric != metric) {
806 *cont = f6i;
807 return;
808 }
809
810 if (fib6_check_expired(f6i))
811 continue;
812
813 if (unlikely(f6i->nh)) {
814 struct fib6_nh_frl_arg arg = {
815 .flags = f6i->fib6_flags,
816 .oif = oif,
817 .strict = strict,
818 .mpri = mpri,
819 .do_rr = do_rr
820 };
821
822 if (nexthop_is_blackhole(f6i->nh)) {
823 res->fib6_flags = RTF_REJECT;
824 res->fib6_type = RTN_BLACKHOLE;
825 res->f6i = f6i;
826 res->nh = nexthop_fib6_nh(f6i->nh);
827 return;
828 }
829 if (nexthop_for_each_fib6_nh(f6i->nh, rt6_nh_find_match,
830 &arg)) {
831 matched = true;
832 nh = arg.nh;
833 }
834 } else {
835 nh = f6i->fib6_nh;
836 if (find_match(nh, f6i->fib6_flags, oif, strict,
837 mpri, do_rr))
838 matched = true;
839 }
840 if (matched) {
841 res->f6i = f6i;
842 res->nh = nh;
843 res->fib6_flags = f6i->fib6_flags;
844 res->fib6_type = f6i->fib6_type;
845 }
846 }
847 }
848
find_rr_leaf(struct fib6_node * fn,struct fib6_info * leaf,struct fib6_info * rr_head,int oif,int strict,bool * do_rr,struct fib6_result * res)849 static void find_rr_leaf(struct fib6_node *fn, struct fib6_info *leaf,
850 struct fib6_info *rr_head, int oif, int strict,
851 bool *do_rr, struct fib6_result *res)
852 {
853 u32 metric = rr_head->fib6_metric;
854 struct fib6_info *cont = NULL;
855 int mpri = -1;
856
857 __find_rr_leaf(rr_head, NULL, metric, res, &cont,
858 oif, strict, do_rr, &mpri);
859
860 __find_rr_leaf(leaf, rr_head, metric, res, &cont,
861 oif, strict, do_rr, &mpri);
862
863 if (res->f6i || !cont)
864 return;
865
866 __find_rr_leaf(cont, NULL, metric, res, NULL,
867 oif, strict, do_rr, &mpri);
868 }
869
rt6_select(struct net * net,struct fib6_node * fn,int oif,struct fib6_result * res,int strict)870 static void rt6_select(struct net *net, struct fib6_node *fn, int oif,
871 struct fib6_result *res, int strict)
872 {
873 struct fib6_info *leaf = rcu_dereference(fn->leaf);
874 struct fib6_info *rt0;
875 bool do_rr = false;
876 int key_plen;
877
878 /* make sure this function or its helpers sets f6i */
879 res->f6i = NULL;
880
881 if (!leaf || leaf == net->ipv6.fib6_null_entry)
882 goto out;
883
884 rt0 = rcu_dereference(fn->rr_ptr);
885 if (!rt0)
886 rt0 = leaf;
887
888 /* Double check to make sure fn is not an intermediate node
889 * and fn->leaf does not points to its child's leaf
890 * (This might happen if all routes under fn are deleted from
891 * the tree and fib6_repair_tree() is called on the node.)
892 */
893 key_plen = rt0->fib6_dst.plen;
894 #ifdef CONFIG_IPV6_SUBTREES
895 if (rt0->fib6_src.plen)
896 key_plen = rt0->fib6_src.plen;
897 #endif
898 if (fn->fn_bit != key_plen)
899 goto out;
900
901 find_rr_leaf(fn, leaf, rt0, oif, strict, &do_rr, res);
902 if (do_rr) {
903 struct fib6_info *next = rcu_dereference(rt0->fib6_next);
904
905 /* no entries matched; do round-robin */
906 if (!next || next->fib6_metric != rt0->fib6_metric)
907 next = leaf;
908
909 if (next != rt0) {
910 spin_lock_bh(&leaf->fib6_table->tb6_lock);
911 /* make sure next is not being deleted from the tree */
912 if (next->fib6_node)
913 rcu_assign_pointer(fn->rr_ptr, next);
914 spin_unlock_bh(&leaf->fib6_table->tb6_lock);
915 }
916 }
917
918 out:
919 if (!res->f6i) {
920 res->f6i = net->ipv6.fib6_null_entry;
921 res->nh = res->f6i->fib6_nh;
922 res->fib6_flags = res->f6i->fib6_flags;
923 res->fib6_type = res->f6i->fib6_type;
924 }
925 }
926
rt6_is_gw_or_nonexthop(const struct fib6_result * res)927 static bool rt6_is_gw_or_nonexthop(const struct fib6_result *res)
928 {
929 return (res->f6i->fib6_flags & RTF_NONEXTHOP) ||
930 res->nh->fib_nh_gw_family;
931 }
932
933 #ifdef CONFIG_IPV6_ROUTE_INFO
rt6_route_rcv(struct net_device * dev,u8 * opt,int len,const struct in6_addr * gwaddr)934 int rt6_route_rcv(struct net_device *dev, u8 *opt, int len,
935 const struct in6_addr *gwaddr)
936 {
937 struct net *net = dev_net(dev);
938 struct route_info *rinfo = (struct route_info *) opt;
939 struct in6_addr prefix_buf, *prefix;
940 unsigned int pref;
941 unsigned long lifetime;
942 struct fib6_info *rt;
943
944 if (len < sizeof(struct route_info)) {
945 return -EINVAL;
946 }
947
948 /* Sanity check for prefix_len and length */
949 if (rinfo->length > 3) {
950 return -EINVAL;
951 } else if (rinfo->prefix_len > 128) {
952 return -EINVAL;
953 } else if (rinfo->prefix_len > 64) {
954 if (rinfo->length < 2) {
955 return -EINVAL;
956 }
957 } else if (rinfo->prefix_len > 0) {
958 if (rinfo->length < 1) {
959 return -EINVAL;
960 }
961 }
962
963 pref = rinfo->route_pref;
964 if (pref == ICMPV6_ROUTER_PREF_INVALID)
965 return -EINVAL;
966
967 lifetime = addrconf_timeout_fixup(ntohl(rinfo->lifetime), HZ);
968
969 if (rinfo->length == 3)
970 prefix = (struct in6_addr *)rinfo->prefix;
971 else {
972 /* this function is safe */
973 ipv6_addr_prefix(&prefix_buf,
974 (struct in6_addr *)rinfo->prefix,
975 rinfo->prefix_len);
976 prefix = &prefix_buf;
977 }
978
979 if (rinfo->prefix_len == 0)
980 rt = rt6_get_dflt_router(net, gwaddr, dev);
981 else
982 rt = rt6_get_route_info(net, prefix, rinfo->prefix_len,
983 gwaddr, dev);
984
985 if (rt && !lifetime) {
986 ip6_del_rt(net, rt, false);
987 rt = NULL;
988 }
989
990 if (!rt && lifetime)
991 rt = rt6_add_route_info(net, prefix, rinfo->prefix_len, gwaddr,
992 dev, pref);
993 else if (rt)
994 rt->fib6_flags = RTF_ROUTEINFO |
995 (rt->fib6_flags & ~RTF_PREF_MASK) | RTF_PREF(pref);
996
997 if (rt) {
998 if (!addrconf_finite_timeout(lifetime))
999 fib6_clean_expires(rt);
1000 else
1001 fib6_set_expires(rt, jiffies + HZ * lifetime);
1002
1003 fib6_info_release(rt);
1004 }
1005 return 0;
1006 }
1007 #endif
1008
1009 /*
1010 * Misc support functions
1011 */
1012
1013 /* called with rcu_lock held */
ip6_rt_get_dev_rcu(const struct fib6_result * res)1014 static struct net_device *ip6_rt_get_dev_rcu(const struct fib6_result *res)
1015 {
1016 struct net_device *dev = res->nh->fib_nh_dev;
1017
1018 if (res->fib6_flags & (RTF_LOCAL | RTF_ANYCAST)) {
1019 /* for copies of local routes, dst->dev needs to be the
1020 * device if it is a master device, the master device if
1021 * device is enslaved, and the loopback as the default
1022 */
1023 if (netif_is_l3_slave(dev) &&
1024 !rt6_need_strict(&res->f6i->fib6_dst.addr))
1025 dev = l3mdev_master_dev_rcu(dev);
1026 else if (!netif_is_l3_master(dev))
1027 dev = dev_net(dev)->loopback_dev;
1028 /* last case is netif_is_l3_master(dev) is true in which
1029 * case we want dev returned to be dev
1030 */
1031 }
1032
1033 return dev;
1034 }
1035
1036 static const int fib6_prop[RTN_MAX + 1] = {
1037 [RTN_UNSPEC] = 0,
1038 [RTN_UNICAST] = 0,
1039 [RTN_LOCAL] = 0,
1040 [RTN_BROADCAST] = 0,
1041 [RTN_ANYCAST] = 0,
1042 [RTN_MULTICAST] = 0,
1043 [RTN_BLACKHOLE] = -EINVAL,
1044 [RTN_UNREACHABLE] = -EHOSTUNREACH,
1045 [RTN_PROHIBIT] = -EACCES,
1046 [RTN_THROW] = -EAGAIN,
1047 [RTN_NAT] = -EINVAL,
1048 [RTN_XRESOLVE] = -EINVAL,
1049 };
1050
ip6_rt_type_to_error(u8 fib6_type)1051 static int ip6_rt_type_to_error(u8 fib6_type)
1052 {
1053 return fib6_prop[fib6_type];
1054 }
1055
fib6_info_dst_flags(struct fib6_info * rt)1056 static unsigned short fib6_info_dst_flags(struct fib6_info *rt)
1057 {
1058 unsigned short flags = 0;
1059
1060 if (rt->dst_nocount)
1061 flags |= DST_NOCOUNT;
1062 if (rt->dst_nopolicy)
1063 flags |= DST_NOPOLICY;
1064
1065 return flags;
1066 }
1067
ip6_rt_init_dst_reject(struct rt6_info * rt,u8 fib6_type)1068 static void ip6_rt_init_dst_reject(struct rt6_info *rt, u8 fib6_type)
1069 {
1070 rt->dst.error = ip6_rt_type_to_error(fib6_type);
1071
1072 switch (fib6_type) {
1073 case RTN_BLACKHOLE:
1074 rt->dst.output = dst_discard_out;
1075 rt->dst.input = dst_discard;
1076 break;
1077 case RTN_PROHIBIT:
1078 rt->dst.output = ip6_pkt_prohibit_out;
1079 rt->dst.input = ip6_pkt_prohibit;
1080 break;
1081 case RTN_THROW:
1082 case RTN_UNREACHABLE:
1083 default:
1084 rt->dst.output = ip6_pkt_discard_out;
1085 rt->dst.input = ip6_pkt_discard;
1086 break;
1087 }
1088 }
1089
ip6_rt_init_dst(struct rt6_info * rt,const struct fib6_result * res)1090 static void ip6_rt_init_dst(struct rt6_info *rt, const struct fib6_result *res)
1091 {
1092 struct fib6_info *f6i = res->f6i;
1093
1094 if (res->fib6_flags & RTF_REJECT) {
1095 ip6_rt_init_dst_reject(rt, res->fib6_type);
1096 return;
1097 }
1098
1099 rt->dst.error = 0;
1100 rt->dst.output = ip6_output;
1101
1102 if (res->fib6_type == RTN_LOCAL || res->fib6_type == RTN_ANYCAST) {
1103 rt->dst.input = ip6_input;
1104 } else if (ipv6_addr_type(&f6i->fib6_dst.addr) & IPV6_ADDR_MULTICAST) {
1105 rt->dst.input = ip6_mc_input;
1106 } else {
1107 rt->dst.input = ip6_forward;
1108 }
1109
1110 if (res->nh->fib_nh_lws) {
1111 rt->dst.lwtstate = lwtstate_get(res->nh->fib_nh_lws);
1112 lwtunnel_set_redirect(&rt->dst);
1113 }
1114
1115 rt->dst.lastuse = jiffies;
1116 }
1117
1118 /* Caller must already hold reference to @from */
rt6_set_from(struct rt6_info * rt,struct fib6_info * from)1119 static void rt6_set_from(struct rt6_info *rt, struct fib6_info *from)
1120 {
1121 rt->rt6i_flags &= ~RTF_EXPIRES;
1122 rcu_assign_pointer(rt->from, from);
1123 ip_dst_init_metrics(&rt->dst, from->fib6_metrics);
1124 }
1125
1126 /* Caller must already hold reference to f6i in result */
ip6_rt_copy_init(struct rt6_info * rt,const struct fib6_result * res)1127 static void ip6_rt_copy_init(struct rt6_info *rt, const struct fib6_result *res)
1128 {
1129 const struct fib6_nh *nh = res->nh;
1130 const struct net_device *dev = nh->fib_nh_dev;
1131 struct fib6_info *f6i = res->f6i;
1132
1133 ip6_rt_init_dst(rt, res);
1134
1135 rt->rt6i_dst = f6i->fib6_dst;
1136 rt->rt6i_idev = dev ? in6_dev_get(dev) : NULL;
1137 rt->rt6i_flags = res->fib6_flags;
1138 if (nh->fib_nh_gw_family) {
1139 rt->rt6i_gateway = nh->fib_nh_gw6;
1140 rt->rt6i_flags |= RTF_GATEWAY;
1141 }
1142 rt6_set_from(rt, f6i);
1143 #ifdef CONFIG_IPV6_SUBTREES
1144 rt->rt6i_src = f6i->fib6_src;
1145 #endif
1146 }
1147
fib6_backtrack(struct fib6_node * fn,struct in6_addr * saddr)1148 static struct fib6_node* fib6_backtrack(struct fib6_node *fn,
1149 struct in6_addr *saddr)
1150 {
1151 struct fib6_node *pn, *sn;
1152 while (1) {
1153 if (fn->fn_flags & RTN_TL_ROOT)
1154 return NULL;
1155 pn = rcu_dereference(fn->parent);
1156 sn = FIB6_SUBTREE(pn);
1157 if (sn && sn != fn)
1158 fn = fib6_node_lookup(sn, NULL, saddr);
1159 else
1160 fn = pn;
1161 if (fn->fn_flags & RTN_RTINFO)
1162 return fn;
1163 }
1164 }
1165
ip6_hold_safe(struct net * net,struct rt6_info ** prt)1166 static bool ip6_hold_safe(struct net *net, struct rt6_info **prt)
1167 {
1168 struct rt6_info *rt = *prt;
1169
1170 if (dst_hold_safe(&rt->dst))
1171 return true;
1172 if (net) {
1173 rt = net->ipv6.ip6_null_entry;
1174 dst_hold(&rt->dst);
1175 } else {
1176 rt = NULL;
1177 }
1178 *prt = rt;
1179 return false;
1180 }
1181
1182 /* called with rcu_lock held */
ip6_create_rt_rcu(const struct fib6_result * res)1183 static struct rt6_info *ip6_create_rt_rcu(const struct fib6_result *res)
1184 {
1185 struct net_device *dev = res->nh->fib_nh_dev;
1186 struct fib6_info *f6i = res->f6i;
1187 unsigned short flags;
1188 struct rt6_info *nrt;
1189
1190 if (!fib6_info_hold_safe(f6i))
1191 goto fallback;
1192
1193 flags = fib6_info_dst_flags(f6i);
1194 nrt = ip6_dst_alloc(dev_net(dev), dev, flags);
1195 if (!nrt) {
1196 fib6_info_release(f6i);
1197 goto fallback;
1198 }
1199
1200 ip6_rt_copy_init(nrt, res);
1201 return nrt;
1202
1203 fallback:
1204 nrt = dev_net(dev)->ipv6.ip6_null_entry;
1205 dst_hold(&nrt->dst);
1206 return nrt;
1207 }
1208
ip6_pol_route_lookup(struct net * net,struct fib6_table * table,struct flowi6 * fl6,const struct sk_buff * skb,int flags)1209 INDIRECT_CALLABLE_SCOPE struct rt6_info *ip6_pol_route_lookup(struct net *net,
1210 struct fib6_table *table,
1211 struct flowi6 *fl6,
1212 const struct sk_buff *skb,
1213 int flags)
1214 {
1215 struct fib6_result res = {};
1216 struct fib6_node *fn;
1217 struct rt6_info *rt;
1218
1219 rcu_read_lock();
1220 fn = fib6_node_lookup(&table->tb6_root, &fl6->daddr, &fl6->saddr);
1221 restart:
1222 res.f6i = rcu_dereference(fn->leaf);
1223 if (!res.f6i)
1224 res.f6i = net->ipv6.fib6_null_entry;
1225 else
1226 rt6_device_match(net, &res, &fl6->saddr, fl6->flowi6_oif,
1227 flags);
1228
1229 if (res.f6i == net->ipv6.fib6_null_entry) {
1230 fn = fib6_backtrack(fn, &fl6->saddr);
1231 if (fn)
1232 goto restart;
1233
1234 rt = net->ipv6.ip6_null_entry;
1235 dst_hold(&rt->dst);
1236 goto out;
1237 } else if (res.fib6_flags & RTF_REJECT) {
1238 goto do_create;
1239 }
1240
1241 fib6_select_path(net, &res, fl6, fl6->flowi6_oif,
1242 fl6->flowi6_oif != 0, skb, flags);
1243
1244 /* Search through exception table */
1245 rt = rt6_find_cached_rt(&res, &fl6->daddr, &fl6->saddr);
1246 if (rt) {
1247 if (ip6_hold_safe(net, &rt))
1248 dst_use_noref(&rt->dst, jiffies);
1249 } else {
1250 do_create:
1251 rt = ip6_create_rt_rcu(&res);
1252 }
1253
1254 out:
1255 trace_fib6_table_lookup(net, &res, table, fl6);
1256
1257 rcu_read_unlock();
1258
1259 return rt;
1260 }
1261
ip6_route_lookup(struct net * net,struct flowi6 * fl6,const struct sk_buff * skb,int flags)1262 struct dst_entry *ip6_route_lookup(struct net *net, struct flowi6 *fl6,
1263 const struct sk_buff *skb, int flags)
1264 {
1265 return fib6_rule_lookup(net, fl6, skb, flags, ip6_pol_route_lookup);
1266 }
1267 EXPORT_SYMBOL_GPL(ip6_route_lookup);
1268
rt6_lookup(struct net * net,const struct in6_addr * daddr,const struct in6_addr * saddr,int oif,const struct sk_buff * skb,int strict)1269 struct rt6_info *rt6_lookup(struct net *net, const struct in6_addr *daddr,
1270 const struct in6_addr *saddr, int oif,
1271 const struct sk_buff *skb, int strict)
1272 {
1273 struct flowi6 fl6 = {
1274 .flowi6_oif = oif,
1275 .daddr = *daddr,
1276 };
1277 struct dst_entry *dst;
1278 int flags = strict ? RT6_LOOKUP_F_IFACE : 0;
1279
1280 if (saddr) {
1281 memcpy(&fl6.saddr, saddr, sizeof(*saddr));
1282 flags |= RT6_LOOKUP_F_HAS_SADDR;
1283 }
1284
1285 dst = fib6_rule_lookup(net, &fl6, skb, flags, ip6_pol_route_lookup);
1286 if (dst->error == 0)
1287 return dst_rt6_info(dst);
1288
1289 dst_release(dst);
1290
1291 return NULL;
1292 }
1293 EXPORT_SYMBOL(rt6_lookup);
1294
1295 /* ip6_ins_rt is called with FREE table->tb6_lock.
1296 * It takes new route entry, the addition fails by any reason the
1297 * route is released.
1298 * Caller must hold dst before calling it.
1299 */
1300
__ip6_ins_rt(struct fib6_info * rt,struct nl_info * info,struct netlink_ext_ack * extack)1301 static int __ip6_ins_rt(struct fib6_info *rt, struct nl_info *info,
1302 struct netlink_ext_ack *extack)
1303 {
1304 int err;
1305 struct fib6_table *table;
1306
1307 table = rt->fib6_table;
1308 spin_lock_bh(&table->tb6_lock);
1309 err = fib6_add(&table->tb6_root, rt, info, extack);
1310 spin_unlock_bh(&table->tb6_lock);
1311
1312 return err;
1313 }
1314
ip6_ins_rt(struct net * net,struct fib6_info * rt)1315 int ip6_ins_rt(struct net *net, struct fib6_info *rt)
1316 {
1317 struct nl_info info = { .nl_net = net, };
1318
1319 return __ip6_ins_rt(rt, &info, NULL);
1320 }
1321
ip6_rt_cache_alloc(const struct fib6_result * res,const struct in6_addr * daddr,const struct in6_addr * saddr)1322 static struct rt6_info *ip6_rt_cache_alloc(const struct fib6_result *res,
1323 const struct in6_addr *daddr,
1324 const struct in6_addr *saddr)
1325 {
1326 struct fib6_info *f6i = res->f6i;
1327 struct net_device *dev;
1328 struct rt6_info *rt;
1329
1330 /*
1331 * Clone the route.
1332 */
1333
1334 if (!fib6_info_hold_safe(f6i))
1335 return NULL;
1336
1337 dev = ip6_rt_get_dev_rcu(res);
1338 rt = ip6_dst_alloc(dev_net(dev), dev, 0);
1339 if (!rt) {
1340 fib6_info_release(f6i);
1341 return NULL;
1342 }
1343
1344 ip6_rt_copy_init(rt, res);
1345 rt->rt6i_flags |= RTF_CACHE;
1346 rt->rt6i_dst.addr = *daddr;
1347 rt->rt6i_dst.plen = 128;
1348
1349 if (!rt6_is_gw_or_nonexthop(res)) {
1350 if (f6i->fib6_dst.plen != 128 &&
1351 ipv6_addr_equal(&f6i->fib6_dst.addr, daddr))
1352 rt->rt6i_flags |= RTF_ANYCAST;
1353 #ifdef CONFIG_IPV6_SUBTREES
1354 if (rt->rt6i_src.plen && saddr) {
1355 rt->rt6i_src.addr = *saddr;
1356 rt->rt6i_src.plen = 128;
1357 }
1358 #endif
1359 }
1360
1361 return rt;
1362 }
1363
ip6_rt_pcpu_alloc(const struct fib6_result * res)1364 static struct rt6_info *ip6_rt_pcpu_alloc(const struct fib6_result *res)
1365 {
1366 struct fib6_info *f6i = res->f6i;
1367 unsigned short flags = fib6_info_dst_flags(f6i);
1368 struct net_device *dev;
1369 struct rt6_info *pcpu_rt;
1370
1371 if (!fib6_info_hold_safe(f6i))
1372 return NULL;
1373
1374 rcu_read_lock();
1375 dev = ip6_rt_get_dev_rcu(res);
1376 pcpu_rt = ip6_dst_alloc(dev_net(dev), dev, flags | DST_NOCOUNT);
1377 rcu_read_unlock();
1378 if (!pcpu_rt) {
1379 fib6_info_release(f6i);
1380 return NULL;
1381 }
1382 ip6_rt_copy_init(pcpu_rt, res);
1383 pcpu_rt->rt6i_flags |= RTF_PCPU;
1384
1385 if (f6i->nh)
1386 pcpu_rt->sernum = rt_genid_ipv6(dev_net(dev));
1387
1388 return pcpu_rt;
1389 }
1390
rt6_is_valid(const struct rt6_info * rt6)1391 static bool rt6_is_valid(const struct rt6_info *rt6)
1392 {
1393 return rt6->sernum == rt_genid_ipv6(dev_net(rt6->dst.dev));
1394 }
1395
1396 /* It should be called with rcu_read_lock() acquired */
rt6_get_pcpu_route(const struct fib6_result * res)1397 static struct rt6_info *rt6_get_pcpu_route(const struct fib6_result *res)
1398 {
1399 struct rt6_info *pcpu_rt;
1400
1401 pcpu_rt = this_cpu_read(*res->nh->rt6i_pcpu);
1402
1403 if (pcpu_rt && pcpu_rt->sernum && !rt6_is_valid(pcpu_rt)) {
1404 struct rt6_info *prev, **p;
1405
1406 p = this_cpu_ptr(res->nh->rt6i_pcpu);
1407 /* Paired with READ_ONCE() in __fib6_drop_pcpu_from() */
1408 prev = xchg(p, NULL);
1409 if (prev) {
1410 dst_dev_put(&prev->dst);
1411 dst_release(&prev->dst);
1412 }
1413
1414 pcpu_rt = NULL;
1415 }
1416
1417 return pcpu_rt;
1418 }
1419
rt6_make_pcpu_route(struct net * net,const struct fib6_result * res)1420 static struct rt6_info *rt6_make_pcpu_route(struct net *net,
1421 const struct fib6_result *res)
1422 {
1423 struct rt6_info *pcpu_rt, *prev, **p;
1424
1425 pcpu_rt = ip6_rt_pcpu_alloc(res);
1426 if (!pcpu_rt)
1427 return NULL;
1428
1429 p = this_cpu_ptr(res->nh->rt6i_pcpu);
1430 prev = cmpxchg(p, NULL, pcpu_rt);
1431 BUG_ON(prev);
1432
1433 if (res->f6i->fib6_destroying) {
1434 struct fib6_info *from;
1435
1436 from = unrcu_pointer(xchg(&pcpu_rt->from, NULL));
1437 fib6_info_release(from);
1438 }
1439
1440 return pcpu_rt;
1441 }
1442
1443 /* exception hash table implementation
1444 */
1445 static DEFINE_SPINLOCK(rt6_exception_lock);
1446
1447 /* Remove rt6_ex from hash table and free the memory
1448 * Caller must hold rt6_exception_lock
1449 */
rt6_remove_exception(struct rt6_exception_bucket * bucket,struct rt6_exception * rt6_ex)1450 static void rt6_remove_exception(struct rt6_exception_bucket *bucket,
1451 struct rt6_exception *rt6_ex)
1452 {
1453 struct net *net;
1454
1455 if (!bucket || !rt6_ex)
1456 return;
1457
1458 net = dev_net(rt6_ex->rt6i->dst.dev);
1459 net->ipv6.rt6_stats->fib_rt_cache--;
1460
1461 /* purge completely the exception to allow releasing the held resources:
1462 * some [sk] cache may keep the dst around for unlimited time
1463 */
1464 dst_dev_put(&rt6_ex->rt6i->dst);
1465
1466 hlist_del_rcu(&rt6_ex->hlist);
1467 dst_release(&rt6_ex->rt6i->dst);
1468 kfree_rcu(rt6_ex, rcu);
1469 WARN_ON_ONCE(!bucket->depth);
1470 bucket->depth--;
1471 }
1472
1473 /* Remove oldest rt6_ex in bucket and free the memory
1474 * Caller must hold rt6_exception_lock
1475 */
rt6_exception_remove_oldest(struct rt6_exception_bucket * bucket)1476 static void rt6_exception_remove_oldest(struct rt6_exception_bucket *bucket)
1477 {
1478 struct rt6_exception *rt6_ex, *oldest = NULL;
1479
1480 if (!bucket)
1481 return;
1482
1483 hlist_for_each_entry(rt6_ex, &bucket->chain, hlist) {
1484 if (!oldest || time_before(rt6_ex->stamp, oldest->stamp))
1485 oldest = rt6_ex;
1486 }
1487 rt6_remove_exception(bucket, oldest);
1488 }
1489
rt6_exception_hash(const struct in6_addr * dst,const struct in6_addr * src)1490 static u32 rt6_exception_hash(const struct in6_addr *dst,
1491 const struct in6_addr *src)
1492 {
1493 static siphash_aligned_key_t rt6_exception_key;
1494 struct {
1495 struct in6_addr dst;
1496 struct in6_addr src;
1497 } __aligned(SIPHASH_ALIGNMENT) combined = {
1498 .dst = *dst,
1499 };
1500 u64 val;
1501
1502 net_get_random_once(&rt6_exception_key, sizeof(rt6_exception_key));
1503
1504 #ifdef CONFIG_IPV6_SUBTREES
1505 if (src)
1506 combined.src = *src;
1507 #endif
1508 val = siphash(&combined, sizeof(combined), &rt6_exception_key);
1509
1510 return hash_64(val, FIB6_EXCEPTION_BUCKET_SIZE_SHIFT);
1511 }
1512
1513 /* Helper function to find the cached rt in the hash table
1514 * and update bucket pointer to point to the bucket for this
1515 * (daddr, saddr) pair
1516 * Caller must hold rt6_exception_lock
1517 */
1518 static struct rt6_exception *
__rt6_find_exception_spinlock(struct rt6_exception_bucket ** bucket,const struct in6_addr * daddr,const struct in6_addr * saddr)1519 __rt6_find_exception_spinlock(struct rt6_exception_bucket **bucket,
1520 const struct in6_addr *daddr,
1521 const struct in6_addr *saddr)
1522 {
1523 struct rt6_exception *rt6_ex;
1524 u32 hval;
1525
1526 if (!(*bucket) || !daddr)
1527 return NULL;
1528
1529 hval = rt6_exception_hash(daddr, saddr);
1530 *bucket += hval;
1531
1532 hlist_for_each_entry(rt6_ex, &(*bucket)->chain, hlist) {
1533 struct rt6_info *rt6 = rt6_ex->rt6i;
1534 bool matched = ipv6_addr_equal(daddr, &rt6->rt6i_dst.addr);
1535
1536 #ifdef CONFIG_IPV6_SUBTREES
1537 if (matched && saddr)
1538 matched = ipv6_addr_equal(saddr, &rt6->rt6i_src.addr);
1539 #endif
1540 if (matched)
1541 return rt6_ex;
1542 }
1543 return NULL;
1544 }
1545
1546 /* Helper function to find the cached rt in the hash table
1547 * and update bucket pointer to point to the bucket for this
1548 * (daddr, saddr) pair
1549 * Caller must hold rcu_read_lock()
1550 */
1551 static struct rt6_exception *
__rt6_find_exception_rcu(struct rt6_exception_bucket ** bucket,const struct in6_addr * daddr,const struct in6_addr * saddr)1552 __rt6_find_exception_rcu(struct rt6_exception_bucket **bucket,
1553 const struct in6_addr *daddr,
1554 const struct in6_addr *saddr)
1555 {
1556 struct rt6_exception *rt6_ex;
1557 u32 hval;
1558
1559 WARN_ON_ONCE(!rcu_read_lock_held());
1560
1561 if (!(*bucket) || !daddr)
1562 return NULL;
1563
1564 hval = rt6_exception_hash(daddr, saddr);
1565 *bucket += hval;
1566
1567 hlist_for_each_entry_rcu(rt6_ex, &(*bucket)->chain, hlist) {
1568 struct rt6_info *rt6 = rt6_ex->rt6i;
1569 bool matched = ipv6_addr_equal(daddr, &rt6->rt6i_dst.addr);
1570
1571 #ifdef CONFIG_IPV6_SUBTREES
1572 if (matched && saddr)
1573 matched = ipv6_addr_equal(saddr, &rt6->rt6i_src.addr);
1574 #endif
1575 if (matched)
1576 return rt6_ex;
1577 }
1578 return NULL;
1579 }
1580
fib6_mtu(const struct fib6_result * res)1581 static unsigned int fib6_mtu(const struct fib6_result *res)
1582 {
1583 const struct fib6_nh *nh = res->nh;
1584 unsigned int mtu;
1585
1586 if (res->f6i->fib6_pmtu) {
1587 mtu = res->f6i->fib6_pmtu;
1588 } else {
1589 struct net_device *dev = nh->fib_nh_dev;
1590 struct inet6_dev *idev;
1591
1592 rcu_read_lock();
1593 idev = __in6_dev_get(dev);
1594 mtu = idev->cnf.mtu6;
1595 rcu_read_unlock();
1596 }
1597
1598 mtu = min_t(unsigned int, mtu, IP6_MAX_MTU);
1599
1600 return mtu - lwtunnel_headroom(nh->fib_nh_lws, mtu);
1601 }
1602
1603 #define FIB6_EXCEPTION_BUCKET_FLUSHED 0x1UL
1604
1605 /* used when the flushed bit is not relevant, only access to the bucket
1606 * (ie., all bucket users except rt6_insert_exception);
1607 *
1608 * called under rcu lock; sometimes called with rt6_exception_lock held
1609 */
1610 static
fib6_nh_get_excptn_bucket(const struct fib6_nh * nh,spinlock_t * lock)1611 struct rt6_exception_bucket *fib6_nh_get_excptn_bucket(const struct fib6_nh *nh,
1612 spinlock_t *lock)
1613 {
1614 struct rt6_exception_bucket *bucket;
1615
1616 if (lock)
1617 bucket = rcu_dereference_protected(nh->rt6i_exception_bucket,
1618 lockdep_is_held(lock));
1619 else
1620 bucket = rcu_dereference(nh->rt6i_exception_bucket);
1621
1622 /* remove bucket flushed bit if set */
1623 if (bucket) {
1624 unsigned long p = (unsigned long)bucket;
1625
1626 p &= ~FIB6_EXCEPTION_BUCKET_FLUSHED;
1627 bucket = (struct rt6_exception_bucket *)p;
1628 }
1629
1630 return bucket;
1631 }
1632
fib6_nh_excptn_bucket_flushed(struct rt6_exception_bucket * bucket)1633 static bool fib6_nh_excptn_bucket_flushed(struct rt6_exception_bucket *bucket)
1634 {
1635 unsigned long p = (unsigned long)bucket;
1636
1637 return !!(p & FIB6_EXCEPTION_BUCKET_FLUSHED);
1638 }
1639
1640 /* called with rt6_exception_lock held */
fib6_nh_excptn_bucket_set_flushed(struct fib6_nh * nh,spinlock_t * lock)1641 static void fib6_nh_excptn_bucket_set_flushed(struct fib6_nh *nh,
1642 spinlock_t *lock)
1643 {
1644 struct rt6_exception_bucket *bucket;
1645 unsigned long p;
1646
1647 bucket = rcu_dereference_protected(nh->rt6i_exception_bucket,
1648 lockdep_is_held(lock));
1649
1650 p = (unsigned long)bucket;
1651 p |= FIB6_EXCEPTION_BUCKET_FLUSHED;
1652 bucket = (struct rt6_exception_bucket *)p;
1653 rcu_assign_pointer(nh->rt6i_exception_bucket, bucket);
1654 }
1655
rt6_insert_exception(struct rt6_info * nrt,const struct fib6_result * res)1656 static int rt6_insert_exception(struct rt6_info *nrt,
1657 const struct fib6_result *res)
1658 {
1659 struct net *net = dev_net(nrt->dst.dev);
1660 struct rt6_exception_bucket *bucket;
1661 struct fib6_info *f6i = res->f6i;
1662 struct in6_addr *src_key = NULL;
1663 struct rt6_exception *rt6_ex;
1664 struct fib6_nh *nh = res->nh;
1665 int max_depth;
1666 int err = 0;
1667
1668 spin_lock_bh(&rt6_exception_lock);
1669
1670 bucket = rcu_dereference_protected(nh->rt6i_exception_bucket,
1671 lockdep_is_held(&rt6_exception_lock));
1672 if (!bucket) {
1673 bucket = kcalloc(FIB6_EXCEPTION_BUCKET_SIZE, sizeof(*bucket),
1674 GFP_ATOMIC);
1675 if (!bucket) {
1676 err = -ENOMEM;
1677 goto out;
1678 }
1679 rcu_assign_pointer(nh->rt6i_exception_bucket, bucket);
1680 } else if (fib6_nh_excptn_bucket_flushed(bucket)) {
1681 err = -EINVAL;
1682 goto out;
1683 }
1684
1685 #ifdef CONFIG_IPV6_SUBTREES
1686 /* fib6_src.plen != 0 indicates f6i is in subtree
1687 * and exception table is indexed by a hash of
1688 * both fib6_dst and fib6_src.
1689 * Otherwise, the exception table is indexed by
1690 * a hash of only fib6_dst.
1691 */
1692 if (f6i->fib6_src.plen)
1693 src_key = &nrt->rt6i_src.addr;
1694 #endif
1695 /* rt6_mtu_change() might lower mtu on f6i.
1696 * Only insert this exception route if its mtu
1697 * is less than f6i's mtu value.
1698 */
1699 if (dst_metric_raw(&nrt->dst, RTAX_MTU) >= fib6_mtu(res)) {
1700 err = -EINVAL;
1701 goto out;
1702 }
1703
1704 rt6_ex = __rt6_find_exception_spinlock(&bucket, &nrt->rt6i_dst.addr,
1705 src_key);
1706 if (rt6_ex)
1707 rt6_remove_exception(bucket, rt6_ex);
1708
1709 rt6_ex = kzalloc(sizeof(*rt6_ex), GFP_ATOMIC);
1710 if (!rt6_ex) {
1711 err = -ENOMEM;
1712 goto out;
1713 }
1714 rt6_ex->rt6i = nrt;
1715 rt6_ex->stamp = jiffies;
1716 hlist_add_head_rcu(&rt6_ex->hlist, &bucket->chain);
1717 bucket->depth++;
1718 net->ipv6.rt6_stats->fib_rt_cache++;
1719
1720 /* Randomize max depth to avoid some side channels attacks. */
1721 max_depth = FIB6_MAX_DEPTH + get_random_u32_below(FIB6_MAX_DEPTH);
1722 while (bucket->depth > max_depth)
1723 rt6_exception_remove_oldest(bucket);
1724
1725 out:
1726 spin_unlock_bh(&rt6_exception_lock);
1727
1728 /* Update fn->fn_sernum to invalidate all cached dst */
1729 if (!err) {
1730 spin_lock_bh(&f6i->fib6_table->tb6_lock);
1731 fib6_update_sernum(net, f6i);
1732 spin_unlock_bh(&f6i->fib6_table->tb6_lock);
1733 fib6_force_start_gc(net);
1734 }
1735
1736 return err;
1737 }
1738
fib6_nh_flush_exceptions(struct fib6_nh * nh,struct fib6_info * from)1739 static void fib6_nh_flush_exceptions(struct fib6_nh *nh, struct fib6_info *from)
1740 {
1741 struct rt6_exception_bucket *bucket;
1742 struct rt6_exception *rt6_ex;
1743 struct hlist_node *tmp;
1744 int i;
1745
1746 spin_lock_bh(&rt6_exception_lock);
1747
1748 bucket = fib6_nh_get_excptn_bucket(nh, &rt6_exception_lock);
1749 if (!bucket)
1750 goto out;
1751
1752 /* Prevent rt6_insert_exception() to recreate the bucket list */
1753 if (!from)
1754 fib6_nh_excptn_bucket_set_flushed(nh, &rt6_exception_lock);
1755
1756 for (i = 0; i < FIB6_EXCEPTION_BUCKET_SIZE; i++) {
1757 hlist_for_each_entry_safe(rt6_ex, tmp, &bucket->chain, hlist) {
1758 if (!from ||
1759 rcu_access_pointer(rt6_ex->rt6i->from) == from)
1760 rt6_remove_exception(bucket, rt6_ex);
1761 }
1762 WARN_ON_ONCE(!from && bucket->depth);
1763 bucket++;
1764 }
1765 out:
1766 spin_unlock_bh(&rt6_exception_lock);
1767 }
1768
rt6_nh_flush_exceptions(struct fib6_nh * nh,void * arg)1769 static int rt6_nh_flush_exceptions(struct fib6_nh *nh, void *arg)
1770 {
1771 struct fib6_info *f6i = arg;
1772
1773 fib6_nh_flush_exceptions(nh, f6i);
1774
1775 return 0;
1776 }
1777
rt6_flush_exceptions(struct fib6_info * f6i)1778 void rt6_flush_exceptions(struct fib6_info *f6i)
1779 {
1780 if (f6i->nh)
1781 nexthop_for_each_fib6_nh(f6i->nh, rt6_nh_flush_exceptions,
1782 f6i);
1783 else
1784 fib6_nh_flush_exceptions(f6i->fib6_nh, f6i);
1785 }
1786
1787 /* Find cached rt in the hash table inside passed in rt
1788 * Caller has to hold rcu_read_lock()
1789 */
rt6_find_cached_rt(const struct fib6_result * res,const struct in6_addr * daddr,const struct in6_addr * saddr)1790 static struct rt6_info *rt6_find_cached_rt(const struct fib6_result *res,
1791 const struct in6_addr *daddr,
1792 const struct in6_addr *saddr)
1793 {
1794 const struct in6_addr *src_key = NULL;
1795 struct rt6_exception_bucket *bucket;
1796 struct rt6_exception *rt6_ex;
1797 struct rt6_info *ret = NULL;
1798
1799 #ifdef CONFIG_IPV6_SUBTREES
1800 /* fib6i_src.plen != 0 indicates f6i is in subtree
1801 * and exception table is indexed by a hash of
1802 * both fib6_dst and fib6_src.
1803 * However, the src addr used to create the hash
1804 * might not be exactly the passed in saddr which
1805 * is a /128 addr from the flow.
1806 * So we need to use f6i->fib6_src to redo lookup
1807 * if the passed in saddr does not find anything.
1808 * (See the logic in ip6_rt_cache_alloc() on how
1809 * rt->rt6i_src is updated.)
1810 */
1811 if (res->f6i->fib6_src.plen)
1812 src_key = saddr;
1813 find_ex:
1814 #endif
1815 bucket = fib6_nh_get_excptn_bucket(res->nh, NULL);
1816 rt6_ex = __rt6_find_exception_rcu(&bucket, daddr, src_key);
1817
1818 if (rt6_ex && !rt6_check_expired(rt6_ex->rt6i))
1819 ret = rt6_ex->rt6i;
1820
1821 #ifdef CONFIG_IPV6_SUBTREES
1822 /* Use fib6_src as src_key and redo lookup */
1823 if (!ret && src_key && src_key != &res->f6i->fib6_src.addr) {
1824 src_key = &res->f6i->fib6_src.addr;
1825 goto find_ex;
1826 }
1827 #endif
1828
1829 return ret;
1830 }
1831
1832 /* Remove the passed in cached rt from the hash table that contains it */
fib6_nh_remove_exception(const struct fib6_nh * nh,int plen,const struct rt6_info * rt)1833 static int fib6_nh_remove_exception(const struct fib6_nh *nh, int plen,
1834 const struct rt6_info *rt)
1835 {
1836 const struct in6_addr *src_key = NULL;
1837 struct rt6_exception_bucket *bucket;
1838 struct rt6_exception *rt6_ex;
1839 int err;
1840
1841 if (!rcu_access_pointer(nh->rt6i_exception_bucket))
1842 return -ENOENT;
1843
1844 spin_lock_bh(&rt6_exception_lock);
1845 bucket = fib6_nh_get_excptn_bucket(nh, &rt6_exception_lock);
1846
1847 #ifdef CONFIG_IPV6_SUBTREES
1848 /* rt6i_src.plen != 0 indicates 'from' is in subtree
1849 * and exception table is indexed by a hash of
1850 * both rt6i_dst and rt6i_src.
1851 * Otherwise, the exception table is indexed by
1852 * a hash of only rt6i_dst.
1853 */
1854 if (plen)
1855 src_key = &rt->rt6i_src.addr;
1856 #endif
1857 rt6_ex = __rt6_find_exception_spinlock(&bucket,
1858 &rt->rt6i_dst.addr,
1859 src_key);
1860 if (rt6_ex) {
1861 rt6_remove_exception(bucket, rt6_ex);
1862 err = 0;
1863 } else {
1864 err = -ENOENT;
1865 }
1866
1867 spin_unlock_bh(&rt6_exception_lock);
1868 return err;
1869 }
1870
1871 struct fib6_nh_excptn_arg {
1872 struct rt6_info *rt;
1873 int plen;
1874 };
1875
rt6_nh_remove_exception_rt(struct fib6_nh * nh,void * _arg)1876 static int rt6_nh_remove_exception_rt(struct fib6_nh *nh, void *_arg)
1877 {
1878 struct fib6_nh_excptn_arg *arg = _arg;
1879 int err;
1880
1881 err = fib6_nh_remove_exception(nh, arg->plen, arg->rt);
1882 if (err == 0)
1883 return 1;
1884
1885 return 0;
1886 }
1887
rt6_remove_exception_rt(struct rt6_info * rt)1888 static int rt6_remove_exception_rt(struct rt6_info *rt)
1889 {
1890 struct fib6_info *from;
1891
1892 from = rcu_dereference(rt->from);
1893 if (!from || !(rt->rt6i_flags & RTF_CACHE))
1894 return -EINVAL;
1895
1896 if (from->nh) {
1897 struct fib6_nh_excptn_arg arg = {
1898 .rt = rt,
1899 .plen = from->fib6_src.plen
1900 };
1901 int rc;
1902
1903 /* rc = 1 means an entry was found */
1904 rc = nexthop_for_each_fib6_nh(from->nh,
1905 rt6_nh_remove_exception_rt,
1906 &arg);
1907 return rc ? 0 : -ENOENT;
1908 }
1909
1910 return fib6_nh_remove_exception(from->fib6_nh,
1911 from->fib6_src.plen, rt);
1912 }
1913
1914 /* Find rt6_ex which contains the passed in rt cache and
1915 * refresh its stamp
1916 */
fib6_nh_update_exception(const struct fib6_nh * nh,int plen,const struct rt6_info * rt)1917 static void fib6_nh_update_exception(const struct fib6_nh *nh, int plen,
1918 const struct rt6_info *rt)
1919 {
1920 const struct in6_addr *src_key = NULL;
1921 struct rt6_exception_bucket *bucket;
1922 struct rt6_exception *rt6_ex;
1923
1924 bucket = fib6_nh_get_excptn_bucket(nh, NULL);
1925 #ifdef CONFIG_IPV6_SUBTREES
1926 /* rt6i_src.plen != 0 indicates 'from' is in subtree
1927 * and exception table is indexed by a hash of
1928 * both rt6i_dst and rt6i_src.
1929 * Otherwise, the exception table is indexed by
1930 * a hash of only rt6i_dst.
1931 */
1932 if (plen)
1933 src_key = &rt->rt6i_src.addr;
1934 #endif
1935 rt6_ex = __rt6_find_exception_rcu(&bucket, &rt->rt6i_dst.addr, src_key);
1936 if (rt6_ex)
1937 rt6_ex->stamp = jiffies;
1938 }
1939
1940 struct fib6_nh_match_arg {
1941 const struct net_device *dev;
1942 const struct in6_addr *gw;
1943 struct fib6_nh *match;
1944 };
1945
1946 /* determine if fib6_nh has given device and gateway */
fib6_nh_find_match(struct fib6_nh * nh,void * _arg)1947 static int fib6_nh_find_match(struct fib6_nh *nh, void *_arg)
1948 {
1949 struct fib6_nh_match_arg *arg = _arg;
1950
1951 if (arg->dev != nh->fib_nh_dev ||
1952 (arg->gw && !nh->fib_nh_gw_family) ||
1953 (!arg->gw && nh->fib_nh_gw_family) ||
1954 (arg->gw && !ipv6_addr_equal(arg->gw, &nh->fib_nh_gw6)))
1955 return 0;
1956
1957 arg->match = nh;
1958
1959 /* found a match, break the loop */
1960 return 1;
1961 }
1962
rt6_update_exception_stamp_rt(struct rt6_info * rt)1963 static void rt6_update_exception_stamp_rt(struct rt6_info *rt)
1964 {
1965 struct fib6_info *from;
1966 struct fib6_nh *fib6_nh;
1967
1968 rcu_read_lock();
1969
1970 from = rcu_dereference(rt->from);
1971 if (!from || !(rt->rt6i_flags & RTF_CACHE))
1972 goto unlock;
1973
1974 if (from->nh) {
1975 struct fib6_nh_match_arg arg = {
1976 .dev = rt->dst.dev,
1977 .gw = &rt->rt6i_gateway,
1978 };
1979
1980 nexthop_for_each_fib6_nh(from->nh, fib6_nh_find_match, &arg);
1981
1982 if (!arg.match)
1983 goto unlock;
1984 fib6_nh = arg.match;
1985 } else {
1986 fib6_nh = from->fib6_nh;
1987 }
1988 fib6_nh_update_exception(fib6_nh, from->fib6_src.plen, rt);
1989 unlock:
1990 rcu_read_unlock();
1991 }
1992
rt6_mtu_change_route_allowed(struct inet6_dev * idev,struct rt6_info * rt,int mtu)1993 static bool rt6_mtu_change_route_allowed(struct inet6_dev *idev,
1994 struct rt6_info *rt, int mtu)
1995 {
1996 /* If the new MTU is lower than the route PMTU, this new MTU will be the
1997 * lowest MTU in the path: always allow updating the route PMTU to
1998 * reflect PMTU decreases.
1999 *
2000 * If the new MTU is higher, and the route PMTU is equal to the local
2001 * MTU, this means the old MTU is the lowest in the path, so allow
2002 * updating it: if other nodes now have lower MTUs, PMTU discovery will
2003 * handle this.
2004 */
2005
2006 if (dst_mtu(&rt->dst) >= mtu)
2007 return true;
2008
2009 if (dst_mtu(&rt->dst) == idev->cnf.mtu6)
2010 return true;
2011
2012 return false;
2013 }
2014
rt6_exceptions_update_pmtu(struct inet6_dev * idev,const struct fib6_nh * nh,int mtu)2015 static void rt6_exceptions_update_pmtu(struct inet6_dev *idev,
2016 const struct fib6_nh *nh, int mtu)
2017 {
2018 struct rt6_exception_bucket *bucket;
2019 struct rt6_exception *rt6_ex;
2020 int i;
2021
2022 bucket = fib6_nh_get_excptn_bucket(nh, &rt6_exception_lock);
2023 if (!bucket)
2024 return;
2025
2026 for (i = 0; i < FIB6_EXCEPTION_BUCKET_SIZE; i++) {
2027 hlist_for_each_entry(rt6_ex, &bucket->chain, hlist) {
2028 struct rt6_info *entry = rt6_ex->rt6i;
2029
2030 /* For RTF_CACHE with rt6i_pmtu == 0 (i.e. a redirected
2031 * route), the metrics of its rt->from have already
2032 * been updated.
2033 */
2034 if (dst_metric_raw(&entry->dst, RTAX_MTU) &&
2035 rt6_mtu_change_route_allowed(idev, entry, mtu))
2036 dst_metric_set(&entry->dst, RTAX_MTU, mtu);
2037 }
2038 bucket++;
2039 }
2040 }
2041
2042 #define RTF_CACHE_GATEWAY (RTF_GATEWAY | RTF_CACHE)
2043
fib6_nh_exceptions_clean_tohost(const struct fib6_nh * nh,const struct in6_addr * gateway)2044 static void fib6_nh_exceptions_clean_tohost(const struct fib6_nh *nh,
2045 const struct in6_addr *gateway)
2046 {
2047 struct rt6_exception_bucket *bucket;
2048 struct rt6_exception *rt6_ex;
2049 struct hlist_node *tmp;
2050 int i;
2051
2052 if (!rcu_access_pointer(nh->rt6i_exception_bucket))
2053 return;
2054
2055 spin_lock_bh(&rt6_exception_lock);
2056 bucket = fib6_nh_get_excptn_bucket(nh, &rt6_exception_lock);
2057 if (bucket) {
2058 for (i = 0; i < FIB6_EXCEPTION_BUCKET_SIZE; i++) {
2059 hlist_for_each_entry_safe(rt6_ex, tmp,
2060 &bucket->chain, hlist) {
2061 struct rt6_info *entry = rt6_ex->rt6i;
2062
2063 if ((entry->rt6i_flags & RTF_CACHE_GATEWAY) ==
2064 RTF_CACHE_GATEWAY &&
2065 ipv6_addr_equal(gateway,
2066 &entry->rt6i_gateway)) {
2067 rt6_remove_exception(bucket, rt6_ex);
2068 }
2069 }
2070 bucket++;
2071 }
2072 }
2073
2074 spin_unlock_bh(&rt6_exception_lock);
2075 }
2076
rt6_age_examine_exception(struct rt6_exception_bucket * bucket,struct rt6_exception * rt6_ex,struct fib6_gc_args * gc_args,unsigned long now)2077 static void rt6_age_examine_exception(struct rt6_exception_bucket *bucket,
2078 struct rt6_exception *rt6_ex,
2079 struct fib6_gc_args *gc_args,
2080 unsigned long now)
2081 {
2082 struct rt6_info *rt = rt6_ex->rt6i;
2083
2084 /* we are pruning and obsoleting aged-out and non gateway exceptions
2085 * even if others have still references to them, so that on next
2086 * dst_check() such references can be dropped.
2087 * EXPIRES exceptions - e.g. pmtu-generated ones are pruned when
2088 * expired, independently from their aging, as per RFC 8201 section 4
2089 */
2090 if (!(rt->rt6i_flags & RTF_EXPIRES)) {
2091 if (time_after_eq(now, rt->dst.lastuse + gc_args->timeout)) {
2092 RT6_TRACE("aging clone %p\n", rt);
2093 rt6_remove_exception(bucket, rt6_ex);
2094 return;
2095 }
2096 } else if (time_after(jiffies, rt->dst.expires)) {
2097 RT6_TRACE("purging expired route %p\n", rt);
2098 rt6_remove_exception(bucket, rt6_ex);
2099 return;
2100 }
2101
2102 if (rt->rt6i_flags & RTF_GATEWAY) {
2103 struct neighbour *neigh;
2104
2105 neigh = __ipv6_neigh_lookup_noref(rt->dst.dev, &rt->rt6i_gateway);
2106
2107 if (!(neigh && (neigh->flags & NTF_ROUTER))) {
2108 RT6_TRACE("purging route %p via non-router but gateway\n",
2109 rt);
2110 rt6_remove_exception(bucket, rt6_ex);
2111 return;
2112 }
2113 }
2114
2115 gc_args->more++;
2116 }
2117
fib6_nh_age_exceptions(const struct fib6_nh * nh,struct fib6_gc_args * gc_args,unsigned long now)2118 static void fib6_nh_age_exceptions(const struct fib6_nh *nh,
2119 struct fib6_gc_args *gc_args,
2120 unsigned long now)
2121 {
2122 struct rt6_exception_bucket *bucket;
2123 struct rt6_exception *rt6_ex;
2124 struct hlist_node *tmp;
2125 int i;
2126
2127 if (!rcu_access_pointer(nh->rt6i_exception_bucket))
2128 return;
2129
2130 rcu_read_lock_bh();
2131 spin_lock(&rt6_exception_lock);
2132 bucket = fib6_nh_get_excptn_bucket(nh, &rt6_exception_lock);
2133 if (bucket) {
2134 for (i = 0; i < FIB6_EXCEPTION_BUCKET_SIZE; i++) {
2135 hlist_for_each_entry_safe(rt6_ex, tmp,
2136 &bucket->chain, hlist) {
2137 rt6_age_examine_exception(bucket, rt6_ex,
2138 gc_args, now);
2139 }
2140 bucket++;
2141 }
2142 }
2143 spin_unlock(&rt6_exception_lock);
2144 rcu_read_unlock_bh();
2145 }
2146
2147 struct fib6_nh_age_excptn_arg {
2148 struct fib6_gc_args *gc_args;
2149 unsigned long now;
2150 };
2151
rt6_nh_age_exceptions(struct fib6_nh * nh,void * _arg)2152 static int rt6_nh_age_exceptions(struct fib6_nh *nh, void *_arg)
2153 {
2154 struct fib6_nh_age_excptn_arg *arg = _arg;
2155
2156 fib6_nh_age_exceptions(nh, arg->gc_args, arg->now);
2157 return 0;
2158 }
2159
rt6_age_exceptions(struct fib6_info * f6i,struct fib6_gc_args * gc_args,unsigned long now)2160 void rt6_age_exceptions(struct fib6_info *f6i,
2161 struct fib6_gc_args *gc_args,
2162 unsigned long now)
2163 {
2164 if (f6i->nh) {
2165 struct fib6_nh_age_excptn_arg arg = {
2166 .gc_args = gc_args,
2167 .now = now
2168 };
2169
2170 nexthop_for_each_fib6_nh(f6i->nh, rt6_nh_age_exceptions,
2171 &arg);
2172 } else {
2173 fib6_nh_age_exceptions(f6i->fib6_nh, gc_args, now);
2174 }
2175 }
2176
2177 /* must be called with rcu lock held */
fib6_table_lookup(struct net * net,struct fib6_table * table,int oif,struct flowi6 * fl6,struct fib6_result * res,int strict)2178 int fib6_table_lookup(struct net *net, struct fib6_table *table, int oif,
2179 struct flowi6 *fl6, struct fib6_result *res, int strict)
2180 {
2181 struct fib6_node *fn, *saved_fn;
2182
2183 fn = fib6_node_lookup(&table->tb6_root, &fl6->daddr, &fl6->saddr);
2184 saved_fn = fn;
2185
2186 redo_rt6_select:
2187 rt6_select(net, fn, oif, res, strict);
2188 if (res->f6i == net->ipv6.fib6_null_entry) {
2189 fn = fib6_backtrack(fn, &fl6->saddr);
2190 if (fn)
2191 goto redo_rt6_select;
2192 else if (strict & RT6_LOOKUP_F_REACHABLE) {
2193 /* also consider unreachable route */
2194 strict &= ~RT6_LOOKUP_F_REACHABLE;
2195 fn = saved_fn;
2196 goto redo_rt6_select;
2197 }
2198 }
2199
2200 trace_fib6_table_lookup(net, res, table, fl6);
2201
2202 return 0;
2203 }
2204
ip6_pol_route(struct net * net,struct fib6_table * table,int oif,struct flowi6 * fl6,const struct sk_buff * skb,int flags)2205 struct rt6_info *ip6_pol_route(struct net *net, struct fib6_table *table,
2206 int oif, struct flowi6 *fl6,
2207 const struct sk_buff *skb, int flags)
2208 {
2209 struct fib6_result res = {};
2210 struct rt6_info *rt = NULL;
2211 int strict = 0;
2212
2213 WARN_ON_ONCE((flags & RT6_LOOKUP_F_DST_NOREF) &&
2214 !rcu_read_lock_held());
2215
2216 strict |= flags & RT6_LOOKUP_F_IFACE;
2217 strict |= flags & RT6_LOOKUP_F_IGNORE_LINKSTATE;
2218 if (net->ipv6.devconf_all->forwarding == 0)
2219 strict |= RT6_LOOKUP_F_REACHABLE;
2220
2221 rcu_read_lock();
2222
2223 fib6_table_lookup(net, table, oif, fl6, &res, strict);
2224 if (res.f6i == net->ipv6.fib6_null_entry)
2225 goto out;
2226
2227 fib6_select_path(net, &res, fl6, oif, false, skb, strict);
2228
2229 /*Search through exception table */
2230 rt = rt6_find_cached_rt(&res, &fl6->daddr, &fl6->saddr);
2231 if (rt) {
2232 goto out;
2233 } else if (unlikely((fl6->flowi6_flags & FLOWI_FLAG_KNOWN_NH) &&
2234 !res.nh->fib_nh_gw_family)) {
2235 /* Create a RTF_CACHE clone which will not be
2236 * owned by the fib6 tree. It is for the special case where
2237 * the daddr in the skb during the neighbor look-up is different
2238 * from the fl6->daddr used to look-up route here.
2239 */
2240 rt = ip6_rt_cache_alloc(&res, &fl6->daddr, NULL);
2241
2242 if (rt) {
2243 /* 1 refcnt is taken during ip6_rt_cache_alloc().
2244 * As rt6_uncached_list_add() does not consume refcnt,
2245 * this refcnt is always returned to the caller even
2246 * if caller sets RT6_LOOKUP_F_DST_NOREF flag.
2247 */
2248 rt6_uncached_list_add(rt);
2249 rcu_read_unlock();
2250
2251 return rt;
2252 }
2253 } else {
2254 /* Get a percpu copy */
2255 local_bh_disable();
2256 rt = rt6_get_pcpu_route(&res);
2257
2258 if (!rt)
2259 rt = rt6_make_pcpu_route(net, &res);
2260
2261 local_bh_enable();
2262 }
2263 out:
2264 if (!rt)
2265 rt = net->ipv6.ip6_null_entry;
2266 if (!(flags & RT6_LOOKUP_F_DST_NOREF))
2267 ip6_hold_safe(net, &rt);
2268 rcu_read_unlock();
2269
2270 return rt;
2271 }
2272 EXPORT_SYMBOL_GPL(ip6_pol_route);
2273
ip6_pol_route_input(struct net * net,struct fib6_table * table,struct flowi6 * fl6,const struct sk_buff * skb,int flags)2274 INDIRECT_CALLABLE_SCOPE struct rt6_info *ip6_pol_route_input(struct net *net,
2275 struct fib6_table *table,
2276 struct flowi6 *fl6,
2277 const struct sk_buff *skb,
2278 int flags)
2279 {
2280 return ip6_pol_route(net, table, fl6->flowi6_iif, fl6, skb, flags);
2281 }
2282
ip6_route_input_lookup(struct net * net,struct net_device * dev,struct flowi6 * fl6,const struct sk_buff * skb,int flags)2283 struct dst_entry *ip6_route_input_lookup(struct net *net,
2284 struct net_device *dev,
2285 struct flowi6 *fl6,
2286 const struct sk_buff *skb,
2287 int flags)
2288 {
2289 if (rt6_need_strict(&fl6->daddr) && dev->type != ARPHRD_PIMREG)
2290 flags |= RT6_LOOKUP_F_IFACE;
2291
2292 return fib6_rule_lookup(net, fl6, skb, flags, ip6_pol_route_input);
2293 }
2294 EXPORT_SYMBOL_GPL(ip6_route_input_lookup);
2295
ip6_multipath_l3_keys(const struct sk_buff * skb,struct flow_keys * keys,struct flow_keys * flkeys)2296 static void ip6_multipath_l3_keys(const struct sk_buff *skb,
2297 struct flow_keys *keys,
2298 struct flow_keys *flkeys)
2299 {
2300 const struct ipv6hdr *outer_iph = ipv6_hdr(skb);
2301 const struct ipv6hdr *key_iph = outer_iph;
2302 struct flow_keys *_flkeys = flkeys;
2303 const struct ipv6hdr *inner_iph;
2304 const struct icmp6hdr *icmph;
2305 struct ipv6hdr _inner_iph;
2306 struct icmp6hdr _icmph;
2307
2308 if (likely(outer_iph->nexthdr != IPPROTO_ICMPV6))
2309 goto out;
2310
2311 icmph = skb_header_pointer(skb, skb_transport_offset(skb),
2312 sizeof(_icmph), &_icmph);
2313 if (!icmph)
2314 goto out;
2315
2316 if (!icmpv6_is_err(icmph->icmp6_type))
2317 goto out;
2318
2319 inner_iph = skb_header_pointer(skb,
2320 skb_transport_offset(skb) + sizeof(*icmph),
2321 sizeof(_inner_iph), &_inner_iph);
2322 if (!inner_iph)
2323 goto out;
2324
2325 key_iph = inner_iph;
2326 _flkeys = NULL;
2327 out:
2328 if (_flkeys) {
2329 keys->addrs.v6addrs.src = _flkeys->addrs.v6addrs.src;
2330 keys->addrs.v6addrs.dst = _flkeys->addrs.v6addrs.dst;
2331 keys->tags.flow_label = _flkeys->tags.flow_label;
2332 keys->basic.ip_proto = _flkeys->basic.ip_proto;
2333 } else {
2334 keys->addrs.v6addrs.src = key_iph->saddr;
2335 keys->addrs.v6addrs.dst = key_iph->daddr;
2336 keys->tags.flow_label = ip6_flowlabel(key_iph);
2337 keys->basic.ip_proto = key_iph->nexthdr;
2338 }
2339 }
2340
rt6_multipath_custom_hash_outer(const struct net * net,const struct sk_buff * skb,bool * p_has_inner)2341 static u32 rt6_multipath_custom_hash_outer(const struct net *net,
2342 const struct sk_buff *skb,
2343 bool *p_has_inner)
2344 {
2345 u32 hash_fields = ip6_multipath_hash_fields(net);
2346 struct flow_keys keys, hash_keys;
2347
2348 if (!(hash_fields & FIB_MULTIPATH_HASH_FIELD_OUTER_MASK))
2349 return 0;
2350
2351 memset(&hash_keys, 0, sizeof(hash_keys));
2352 skb_flow_dissect_flow_keys(skb, &keys, FLOW_DISSECTOR_F_STOP_AT_ENCAP);
2353
2354 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
2355 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_SRC_IP)
2356 hash_keys.addrs.v6addrs.src = keys.addrs.v6addrs.src;
2357 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_DST_IP)
2358 hash_keys.addrs.v6addrs.dst = keys.addrs.v6addrs.dst;
2359 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_IP_PROTO)
2360 hash_keys.basic.ip_proto = keys.basic.ip_proto;
2361 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_FLOWLABEL)
2362 hash_keys.tags.flow_label = keys.tags.flow_label;
2363 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_SRC_PORT)
2364 hash_keys.ports.src = keys.ports.src;
2365 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_DST_PORT)
2366 hash_keys.ports.dst = keys.ports.dst;
2367
2368 *p_has_inner = !!(keys.control.flags & FLOW_DIS_ENCAPSULATION);
2369 return flow_hash_from_keys(&hash_keys);
2370 }
2371
rt6_multipath_custom_hash_inner(const struct net * net,const struct sk_buff * skb,bool has_inner)2372 static u32 rt6_multipath_custom_hash_inner(const struct net *net,
2373 const struct sk_buff *skb,
2374 bool has_inner)
2375 {
2376 u32 hash_fields = ip6_multipath_hash_fields(net);
2377 struct flow_keys keys, hash_keys;
2378
2379 /* We assume the packet carries an encapsulation, but if none was
2380 * encountered during dissection of the outer flow, then there is no
2381 * point in calling the flow dissector again.
2382 */
2383 if (!has_inner)
2384 return 0;
2385
2386 if (!(hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_MASK))
2387 return 0;
2388
2389 memset(&hash_keys, 0, sizeof(hash_keys));
2390 skb_flow_dissect_flow_keys(skb, &keys, 0);
2391
2392 if (!(keys.control.flags & FLOW_DIS_ENCAPSULATION))
2393 return 0;
2394
2395 if (keys.control.addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) {
2396 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS;
2397 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_SRC_IP)
2398 hash_keys.addrs.v4addrs.src = keys.addrs.v4addrs.src;
2399 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_DST_IP)
2400 hash_keys.addrs.v4addrs.dst = keys.addrs.v4addrs.dst;
2401 } else if (keys.control.addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) {
2402 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
2403 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_SRC_IP)
2404 hash_keys.addrs.v6addrs.src = keys.addrs.v6addrs.src;
2405 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_DST_IP)
2406 hash_keys.addrs.v6addrs.dst = keys.addrs.v6addrs.dst;
2407 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_FLOWLABEL)
2408 hash_keys.tags.flow_label = keys.tags.flow_label;
2409 }
2410
2411 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_IP_PROTO)
2412 hash_keys.basic.ip_proto = keys.basic.ip_proto;
2413 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_SRC_PORT)
2414 hash_keys.ports.src = keys.ports.src;
2415 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_DST_PORT)
2416 hash_keys.ports.dst = keys.ports.dst;
2417
2418 return flow_hash_from_keys(&hash_keys);
2419 }
2420
rt6_multipath_custom_hash_skb(const struct net * net,const struct sk_buff * skb)2421 static u32 rt6_multipath_custom_hash_skb(const struct net *net,
2422 const struct sk_buff *skb)
2423 {
2424 u32 mhash, mhash_inner;
2425 bool has_inner = true;
2426
2427 mhash = rt6_multipath_custom_hash_outer(net, skb, &has_inner);
2428 mhash_inner = rt6_multipath_custom_hash_inner(net, skb, has_inner);
2429
2430 return jhash_2words(mhash, mhash_inner, 0);
2431 }
2432
rt6_multipath_custom_hash_fl6(const struct net * net,const struct flowi6 * fl6)2433 static u32 rt6_multipath_custom_hash_fl6(const struct net *net,
2434 const struct flowi6 *fl6)
2435 {
2436 u32 hash_fields = ip6_multipath_hash_fields(net);
2437 struct flow_keys hash_keys;
2438
2439 if (!(hash_fields & FIB_MULTIPATH_HASH_FIELD_OUTER_MASK))
2440 return 0;
2441
2442 memset(&hash_keys, 0, sizeof(hash_keys));
2443 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
2444 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_SRC_IP)
2445 hash_keys.addrs.v6addrs.src = fl6->saddr;
2446 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_DST_IP)
2447 hash_keys.addrs.v6addrs.dst = fl6->daddr;
2448 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_IP_PROTO)
2449 hash_keys.basic.ip_proto = fl6->flowi6_proto;
2450 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_FLOWLABEL)
2451 hash_keys.tags.flow_label = (__force u32)flowi6_get_flowlabel(fl6);
2452 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_SRC_PORT)
2453 hash_keys.ports.src = fl6->fl6_sport;
2454 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_DST_PORT)
2455 hash_keys.ports.dst = fl6->fl6_dport;
2456
2457 return flow_hash_from_keys(&hash_keys);
2458 }
2459
2460 /* if skb is set it will be used and fl6 can be NULL */
rt6_multipath_hash(const struct net * net,const struct flowi6 * fl6,const struct sk_buff * skb,struct flow_keys * flkeys)2461 u32 rt6_multipath_hash(const struct net *net, const struct flowi6 *fl6,
2462 const struct sk_buff *skb, struct flow_keys *flkeys)
2463 {
2464 struct flow_keys hash_keys;
2465 u32 mhash = 0;
2466
2467 switch (ip6_multipath_hash_policy(net)) {
2468 case 0:
2469 memset(&hash_keys, 0, sizeof(hash_keys));
2470 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
2471 if (skb) {
2472 ip6_multipath_l3_keys(skb, &hash_keys, flkeys);
2473 } else {
2474 hash_keys.addrs.v6addrs.src = fl6->saddr;
2475 hash_keys.addrs.v6addrs.dst = fl6->daddr;
2476 hash_keys.tags.flow_label = (__force u32)flowi6_get_flowlabel(fl6);
2477 hash_keys.basic.ip_proto = fl6->flowi6_proto;
2478 }
2479 mhash = flow_hash_from_keys(&hash_keys);
2480 break;
2481 case 1:
2482 if (skb) {
2483 unsigned int flag = FLOW_DISSECTOR_F_STOP_AT_ENCAP;
2484 struct flow_keys keys;
2485
2486 /* short-circuit if we already have L4 hash present */
2487 if (skb->l4_hash)
2488 return skb_get_hash_raw(skb) >> 1;
2489
2490 memset(&hash_keys, 0, sizeof(hash_keys));
2491
2492 if (!flkeys) {
2493 skb_flow_dissect_flow_keys(skb, &keys, flag);
2494 flkeys = &keys;
2495 }
2496 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
2497 hash_keys.addrs.v6addrs.src = flkeys->addrs.v6addrs.src;
2498 hash_keys.addrs.v6addrs.dst = flkeys->addrs.v6addrs.dst;
2499 hash_keys.ports.src = flkeys->ports.src;
2500 hash_keys.ports.dst = flkeys->ports.dst;
2501 hash_keys.basic.ip_proto = flkeys->basic.ip_proto;
2502 } else {
2503 memset(&hash_keys, 0, sizeof(hash_keys));
2504 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
2505 hash_keys.addrs.v6addrs.src = fl6->saddr;
2506 hash_keys.addrs.v6addrs.dst = fl6->daddr;
2507 hash_keys.ports.src = fl6->fl6_sport;
2508 hash_keys.ports.dst = fl6->fl6_dport;
2509 hash_keys.basic.ip_proto = fl6->flowi6_proto;
2510 }
2511 mhash = flow_hash_from_keys(&hash_keys);
2512 break;
2513 case 2:
2514 memset(&hash_keys, 0, sizeof(hash_keys));
2515 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
2516 if (skb) {
2517 struct flow_keys keys;
2518
2519 if (!flkeys) {
2520 skb_flow_dissect_flow_keys(skb, &keys, 0);
2521 flkeys = &keys;
2522 }
2523
2524 /* Inner can be v4 or v6 */
2525 if (flkeys->control.addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) {
2526 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS;
2527 hash_keys.addrs.v4addrs.src = flkeys->addrs.v4addrs.src;
2528 hash_keys.addrs.v4addrs.dst = flkeys->addrs.v4addrs.dst;
2529 } else if (flkeys->control.addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) {
2530 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
2531 hash_keys.addrs.v6addrs.src = flkeys->addrs.v6addrs.src;
2532 hash_keys.addrs.v6addrs.dst = flkeys->addrs.v6addrs.dst;
2533 hash_keys.tags.flow_label = flkeys->tags.flow_label;
2534 hash_keys.basic.ip_proto = flkeys->basic.ip_proto;
2535 } else {
2536 /* Same as case 0 */
2537 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
2538 ip6_multipath_l3_keys(skb, &hash_keys, flkeys);
2539 }
2540 } else {
2541 /* Same as case 0 */
2542 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
2543 hash_keys.addrs.v6addrs.src = fl6->saddr;
2544 hash_keys.addrs.v6addrs.dst = fl6->daddr;
2545 hash_keys.tags.flow_label = (__force u32)flowi6_get_flowlabel(fl6);
2546 hash_keys.basic.ip_proto = fl6->flowi6_proto;
2547 }
2548 mhash = flow_hash_from_keys(&hash_keys);
2549 break;
2550 case 3:
2551 if (skb)
2552 mhash = rt6_multipath_custom_hash_skb(net, skb);
2553 else
2554 mhash = rt6_multipath_custom_hash_fl6(net, fl6);
2555 break;
2556 }
2557
2558 return mhash >> 1;
2559 }
2560
2561 /* Called with rcu held */
ip6_route_input(struct sk_buff * skb)2562 void ip6_route_input(struct sk_buff *skb)
2563 {
2564 const struct ipv6hdr *iph = ipv6_hdr(skb);
2565 struct net *net = dev_net(skb->dev);
2566 int flags = RT6_LOOKUP_F_HAS_SADDR | RT6_LOOKUP_F_DST_NOREF;
2567 struct ip_tunnel_info *tun_info;
2568 struct flowi6 fl6 = {
2569 .flowi6_iif = skb->dev->ifindex,
2570 .daddr = iph->daddr,
2571 .saddr = iph->saddr,
2572 .flowlabel = ip6_flowinfo(iph),
2573 .flowi6_mark = skb->mark,
2574 .flowi6_proto = iph->nexthdr,
2575 };
2576 struct flow_keys *flkeys = NULL, _flkeys;
2577
2578 tun_info = skb_tunnel_info(skb);
2579 if (tun_info && !(tun_info->mode & IP_TUNNEL_INFO_TX))
2580 fl6.flowi6_tun_key.tun_id = tun_info->key.tun_id;
2581
2582 if (fib6_rules_early_flow_dissect(net, skb, &fl6, &_flkeys))
2583 flkeys = &_flkeys;
2584
2585 if (unlikely(fl6.flowi6_proto == IPPROTO_ICMPV6))
2586 fl6.mp_hash = rt6_multipath_hash(net, &fl6, skb, flkeys);
2587 skb_dst_drop(skb);
2588 skb_dst_set_noref(skb, ip6_route_input_lookup(net, skb->dev,
2589 &fl6, skb, flags));
2590 }
2591
ip6_pol_route_output(struct net * net,struct fib6_table * table,struct flowi6 * fl6,const struct sk_buff * skb,int flags)2592 INDIRECT_CALLABLE_SCOPE struct rt6_info *ip6_pol_route_output(struct net *net,
2593 struct fib6_table *table,
2594 struct flowi6 *fl6,
2595 const struct sk_buff *skb,
2596 int flags)
2597 {
2598 return ip6_pol_route(net, table, fl6->flowi6_oif, fl6, skb, flags);
2599 }
2600
ip6_route_output_flags_noref(struct net * net,const struct sock * sk,struct flowi6 * fl6,int flags)2601 static struct dst_entry *ip6_route_output_flags_noref(struct net *net,
2602 const struct sock *sk,
2603 struct flowi6 *fl6,
2604 int flags)
2605 {
2606 bool any_src;
2607
2608 if (ipv6_addr_type(&fl6->daddr) &
2609 (IPV6_ADDR_MULTICAST | IPV6_ADDR_LINKLOCAL)) {
2610 struct dst_entry *dst;
2611
2612 /* This function does not take refcnt on the dst */
2613 dst = l3mdev_link_scope_lookup(net, fl6);
2614 if (dst)
2615 return dst;
2616 }
2617
2618 fl6->flowi6_iif = LOOPBACK_IFINDEX;
2619
2620 flags |= RT6_LOOKUP_F_DST_NOREF;
2621 any_src = ipv6_addr_any(&fl6->saddr);
2622 if ((sk && sk->sk_bound_dev_if) || rt6_need_strict(&fl6->daddr) ||
2623 (fl6->flowi6_oif && any_src))
2624 flags |= RT6_LOOKUP_F_IFACE;
2625
2626 if (!any_src)
2627 flags |= RT6_LOOKUP_F_HAS_SADDR;
2628 else if (sk)
2629 flags |= rt6_srcprefs2flags(inet6_sk(sk)->srcprefs);
2630
2631 return fib6_rule_lookup(net, fl6, NULL, flags, ip6_pol_route_output);
2632 }
2633
ip6_route_output_flags(struct net * net,const struct sock * sk,struct flowi6 * fl6,int flags)2634 struct dst_entry *ip6_route_output_flags(struct net *net,
2635 const struct sock *sk,
2636 struct flowi6 *fl6,
2637 int flags)
2638 {
2639 struct dst_entry *dst;
2640 struct rt6_info *rt6;
2641
2642 rcu_read_lock();
2643 dst = ip6_route_output_flags_noref(net, sk, fl6, flags);
2644 rt6 = dst_rt6_info(dst);
2645 /* For dst cached in uncached_list, refcnt is already taken. */
2646 if (list_empty(&rt6->dst.rt_uncached) && !dst_hold_safe(dst)) {
2647 dst = &net->ipv6.ip6_null_entry->dst;
2648 dst_hold(dst);
2649 }
2650 rcu_read_unlock();
2651
2652 return dst;
2653 }
2654 EXPORT_SYMBOL_GPL(ip6_route_output_flags);
2655
ip6_blackhole_route(struct net * net,struct dst_entry * dst_orig)2656 struct dst_entry *ip6_blackhole_route(struct net *net, struct dst_entry *dst_orig)
2657 {
2658 struct rt6_info *rt, *ort = dst_rt6_info(dst_orig);
2659 struct net_device *loopback_dev = net->loopback_dev;
2660 struct dst_entry *new = NULL;
2661
2662 rt = dst_alloc(&ip6_dst_blackhole_ops, loopback_dev, 1,
2663 DST_OBSOLETE_DEAD, 0);
2664 if (rt) {
2665 rt6_info_init(rt);
2666 atomic_inc(&net->ipv6.rt6_stats->fib_rt_alloc);
2667
2668 new = &rt->dst;
2669 new->__use = 1;
2670 new->input = dst_discard;
2671 new->output = dst_discard_out;
2672
2673 dst_copy_metrics(new, &ort->dst);
2674
2675 rt->rt6i_idev = in6_dev_get(loopback_dev);
2676 rt->rt6i_gateway = ort->rt6i_gateway;
2677 rt->rt6i_flags = ort->rt6i_flags & ~RTF_PCPU;
2678
2679 memcpy(&rt->rt6i_dst, &ort->rt6i_dst, sizeof(struct rt6key));
2680 #ifdef CONFIG_IPV6_SUBTREES
2681 memcpy(&rt->rt6i_src, &ort->rt6i_src, sizeof(struct rt6key));
2682 #endif
2683 }
2684
2685 dst_release(dst_orig);
2686 return new ? new : ERR_PTR(-ENOMEM);
2687 }
2688
2689 /*
2690 * Destination cache support functions
2691 */
2692
fib6_check(struct fib6_info * f6i,u32 cookie)2693 static bool fib6_check(struct fib6_info *f6i, u32 cookie)
2694 {
2695 u32 rt_cookie = 0;
2696
2697 if (!fib6_get_cookie_safe(f6i, &rt_cookie) || rt_cookie != cookie)
2698 return false;
2699
2700 if (fib6_check_expired(f6i))
2701 return false;
2702
2703 return true;
2704 }
2705
rt6_check(struct rt6_info * rt,struct fib6_info * from,u32 cookie)2706 static struct dst_entry *rt6_check(struct rt6_info *rt,
2707 struct fib6_info *from,
2708 u32 cookie)
2709 {
2710 u32 rt_cookie = 0;
2711
2712 if (!from || !fib6_get_cookie_safe(from, &rt_cookie) ||
2713 rt_cookie != cookie)
2714 return NULL;
2715
2716 if (rt6_check_expired(rt))
2717 return NULL;
2718
2719 return &rt->dst;
2720 }
2721
rt6_dst_from_check(struct rt6_info * rt,struct fib6_info * from,u32 cookie)2722 static struct dst_entry *rt6_dst_from_check(struct rt6_info *rt,
2723 struct fib6_info *from,
2724 u32 cookie)
2725 {
2726 if (!__rt6_check_expired(rt) &&
2727 rt->dst.obsolete == DST_OBSOLETE_FORCE_CHK &&
2728 fib6_check(from, cookie))
2729 return &rt->dst;
2730 else
2731 return NULL;
2732 }
2733
ip6_dst_check(struct dst_entry * dst,u32 cookie)2734 INDIRECT_CALLABLE_SCOPE struct dst_entry *ip6_dst_check(struct dst_entry *dst,
2735 u32 cookie)
2736 {
2737 struct dst_entry *dst_ret;
2738 struct fib6_info *from;
2739 struct rt6_info *rt;
2740
2741 rt = dst_rt6_info(dst);
2742
2743 if (rt->sernum)
2744 return rt6_is_valid(rt) ? dst : NULL;
2745
2746 rcu_read_lock();
2747
2748 /* All IPV6 dsts are created with ->obsolete set to the value
2749 * DST_OBSOLETE_FORCE_CHK which forces validation calls down
2750 * into this function always.
2751 */
2752
2753 from = rcu_dereference(rt->from);
2754
2755 if (from && (rt->rt6i_flags & RTF_PCPU ||
2756 unlikely(!list_empty(&rt->dst.rt_uncached))))
2757 dst_ret = rt6_dst_from_check(rt, from, cookie);
2758 else
2759 dst_ret = rt6_check(rt, from, cookie);
2760
2761 rcu_read_unlock();
2762
2763 return dst_ret;
2764 }
2765 EXPORT_INDIRECT_CALLABLE(ip6_dst_check);
2766
ip6_negative_advice(struct sock * sk,struct dst_entry * dst)2767 static void ip6_negative_advice(struct sock *sk,
2768 struct dst_entry *dst)
2769 {
2770 struct rt6_info *rt = dst_rt6_info(dst);
2771
2772 if (rt->rt6i_flags & RTF_CACHE) {
2773 rcu_read_lock();
2774 if (rt6_check_expired(rt)) {
2775 /* rt/dst can not be destroyed yet,
2776 * because of rcu_read_lock()
2777 */
2778 sk_dst_reset(sk);
2779 rt6_remove_exception_rt(rt);
2780 }
2781 rcu_read_unlock();
2782 return;
2783 }
2784 sk_dst_reset(sk);
2785 }
2786
ip6_link_failure(struct sk_buff * skb)2787 static void ip6_link_failure(struct sk_buff *skb)
2788 {
2789 struct rt6_info *rt;
2790
2791 icmpv6_send(skb, ICMPV6_DEST_UNREACH, ICMPV6_ADDR_UNREACH, 0);
2792
2793 rt = dst_rt6_info(skb_dst(skb));
2794 if (rt) {
2795 rcu_read_lock();
2796 if (rt->rt6i_flags & RTF_CACHE) {
2797 rt6_remove_exception_rt(rt);
2798 } else {
2799 struct fib6_info *from;
2800 struct fib6_node *fn;
2801
2802 from = rcu_dereference(rt->from);
2803 if (from) {
2804 fn = rcu_dereference(from->fib6_node);
2805 if (fn && (rt->rt6i_flags & RTF_DEFAULT))
2806 WRITE_ONCE(fn->fn_sernum, -1);
2807 }
2808 }
2809 rcu_read_unlock();
2810 }
2811 }
2812
rt6_update_expires(struct rt6_info * rt0,int timeout)2813 static void rt6_update_expires(struct rt6_info *rt0, int timeout)
2814 {
2815 if (!(rt0->rt6i_flags & RTF_EXPIRES)) {
2816 struct fib6_info *from;
2817
2818 rcu_read_lock();
2819 from = rcu_dereference(rt0->from);
2820 if (from)
2821 rt0->dst.expires = from->expires;
2822 rcu_read_unlock();
2823 }
2824
2825 dst_set_expires(&rt0->dst, timeout);
2826 rt0->rt6i_flags |= RTF_EXPIRES;
2827 }
2828
rt6_do_update_pmtu(struct rt6_info * rt,u32 mtu)2829 static void rt6_do_update_pmtu(struct rt6_info *rt, u32 mtu)
2830 {
2831 struct net *net = dev_net(rt->dst.dev);
2832
2833 dst_metric_set(&rt->dst, RTAX_MTU, mtu);
2834 rt->rt6i_flags |= RTF_MODIFIED;
2835 rt6_update_expires(rt, net->ipv6.sysctl.ip6_rt_mtu_expires);
2836 }
2837
rt6_cache_allowed_for_pmtu(const struct rt6_info * rt)2838 static bool rt6_cache_allowed_for_pmtu(const struct rt6_info *rt)
2839 {
2840 return !(rt->rt6i_flags & RTF_CACHE) &&
2841 (rt->rt6i_flags & RTF_PCPU || rcu_access_pointer(rt->from));
2842 }
2843
__ip6_rt_update_pmtu(struct dst_entry * dst,const struct sock * sk,const struct ipv6hdr * iph,u32 mtu,bool confirm_neigh)2844 static void __ip6_rt_update_pmtu(struct dst_entry *dst, const struct sock *sk,
2845 const struct ipv6hdr *iph, u32 mtu,
2846 bool confirm_neigh)
2847 {
2848 const struct in6_addr *daddr, *saddr;
2849 struct rt6_info *rt6 = dst_rt6_info(dst);
2850
2851 /* Note: do *NOT* check dst_metric_locked(dst, RTAX_MTU)
2852 * IPv6 pmtu discovery isn't optional, so 'mtu lock' cannot disable it.
2853 * [see also comment in rt6_mtu_change_route()]
2854 */
2855
2856 if (iph) {
2857 daddr = &iph->daddr;
2858 saddr = &iph->saddr;
2859 } else if (sk) {
2860 daddr = &sk->sk_v6_daddr;
2861 saddr = &inet6_sk(sk)->saddr;
2862 } else {
2863 daddr = NULL;
2864 saddr = NULL;
2865 }
2866
2867 if (confirm_neigh)
2868 dst_confirm_neigh(dst, daddr);
2869
2870 if (mtu < IPV6_MIN_MTU)
2871 return;
2872 if (mtu >= dst_mtu(dst))
2873 return;
2874
2875 if (!rt6_cache_allowed_for_pmtu(rt6)) {
2876 rt6_do_update_pmtu(rt6, mtu);
2877 /* update rt6_ex->stamp for cache */
2878 if (rt6->rt6i_flags & RTF_CACHE)
2879 rt6_update_exception_stamp_rt(rt6);
2880 } else if (daddr) {
2881 struct fib6_result res = {};
2882 struct rt6_info *nrt6;
2883
2884 rcu_read_lock();
2885 res.f6i = rcu_dereference(rt6->from);
2886 if (!res.f6i)
2887 goto out_unlock;
2888
2889 res.fib6_flags = res.f6i->fib6_flags;
2890 res.fib6_type = res.f6i->fib6_type;
2891
2892 if (res.f6i->nh) {
2893 struct fib6_nh_match_arg arg = {
2894 .dev = dst->dev,
2895 .gw = &rt6->rt6i_gateway,
2896 };
2897
2898 nexthop_for_each_fib6_nh(res.f6i->nh,
2899 fib6_nh_find_match, &arg);
2900
2901 /* fib6_info uses a nexthop that does not have fib6_nh
2902 * using the dst->dev + gw. Should be impossible.
2903 */
2904 if (!arg.match)
2905 goto out_unlock;
2906
2907 res.nh = arg.match;
2908 } else {
2909 res.nh = res.f6i->fib6_nh;
2910 }
2911
2912 nrt6 = ip6_rt_cache_alloc(&res, daddr, saddr);
2913 if (nrt6) {
2914 rt6_do_update_pmtu(nrt6, mtu);
2915 if (rt6_insert_exception(nrt6, &res))
2916 dst_release_immediate(&nrt6->dst);
2917 }
2918 out_unlock:
2919 rcu_read_unlock();
2920 }
2921 }
2922
ip6_rt_update_pmtu(struct dst_entry * dst,struct sock * sk,struct sk_buff * skb,u32 mtu,bool confirm_neigh)2923 static void ip6_rt_update_pmtu(struct dst_entry *dst, struct sock *sk,
2924 struct sk_buff *skb, u32 mtu,
2925 bool confirm_neigh)
2926 {
2927 __ip6_rt_update_pmtu(dst, sk, skb ? ipv6_hdr(skb) : NULL, mtu,
2928 confirm_neigh);
2929 }
2930
ip6_update_pmtu(struct sk_buff * skb,struct net * net,__be32 mtu,int oif,u32 mark,kuid_t uid)2931 void ip6_update_pmtu(struct sk_buff *skb, struct net *net, __be32 mtu,
2932 int oif, u32 mark, kuid_t uid)
2933 {
2934 const struct ipv6hdr *iph = (struct ipv6hdr *) skb->data;
2935 struct dst_entry *dst;
2936 struct flowi6 fl6 = {
2937 .flowi6_oif = oif,
2938 .flowi6_mark = mark ? mark : IP6_REPLY_MARK(net, skb->mark),
2939 .daddr = iph->daddr,
2940 .saddr = iph->saddr,
2941 .flowlabel = ip6_flowinfo(iph),
2942 .flowi6_uid = uid,
2943 };
2944
2945 dst = ip6_route_output(net, NULL, &fl6);
2946 if (!dst->error)
2947 __ip6_rt_update_pmtu(dst, NULL, iph, ntohl(mtu), true);
2948 dst_release(dst);
2949 }
2950 EXPORT_SYMBOL_GPL(ip6_update_pmtu);
2951
ip6_sk_update_pmtu(struct sk_buff * skb,struct sock * sk,__be32 mtu)2952 void ip6_sk_update_pmtu(struct sk_buff *skb, struct sock *sk, __be32 mtu)
2953 {
2954 int oif = sk->sk_bound_dev_if;
2955 struct dst_entry *dst;
2956
2957 if (!oif && skb->dev)
2958 oif = l3mdev_master_ifindex(skb->dev);
2959
2960 ip6_update_pmtu(skb, sock_net(sk), mtu, oif, READ_ONCE(sk->sk_mark),
2961 sk->sk_uid);
2962
2963 dst = __sk_dst_get(sk);
2964 if (!dst || !dst->obsolete ||
2965 dst->ops->check(dst, inet6_sk(sk)->dst_cookie))
2966 return;
2967
2968 bh_lock_sock(sk);
2969 if (!sock_owned_by_user(sk) && !ipv6_addr_v4mapped(&sk->sk_v6_daddr))
2970 ip6_datagram_dst_update(sk, false);
2971 bh_unlock_sock(sk);
2972 }
2973 EXPORT_SYMBOL_GPL(ip6_sk_update_pmtu);
2974
ip6_sk_dst_store_flow(struct sock * sk,struct dst_entry * dst,const struct flowi6 * fl6)2975 void ip6_sk_dst_store_flow(struct sock *sk, struct dst_entry *dst,
2976 const struct flowi6 *fl6)
2977 {
2978 #ifdef CONFIG_IPV6_SUBTREES
2979 struct ipv6_pinfo *np = inet6_sk(sk);
2980 #endif
2981
2982 ip6_dst_store(sk, dst,
2983 ipv6_addr_equal(&fl6->daddr, &sk->sk_v6_daddr) ?
2984 &sk->sk_v6_daddr : NULL,
2985 #ifdef CONFIG_IPV6_SUBTREES
2986 ipv6_addr_equal(&fl6->saddr, &np->saddr) ?
2987 &np->saddr :
2988 #endif
2989 NULL);
2990 }
2991
ip6_redirect_nh_match(const struct fib6_result * res,struct flowi6 * fl6,const struct in6_addr * gw,struct rt6_info ** ret)2992 static bool ip6_redirect_nh_match(const struct fib6_result *res,
2993 struct flowi6 *fl6,
2994 const struct in6_addr *gw,
2995 struct rt6_info **ret)
2996 {
2997 const struct fib6_nh *nh = res->nh;
2998
2999 if (nh->fib_nh_flags & RTNH_F_DEAD || !nh->fib_nh_gw_family ||
3000 fl6->flowi6_oif != nh->fib_nh_dev->ifindex)
3001 return false;
3002
3003 /* rt_cache's gateway might be different from its 'parent'
3004 * in the case of an ip redirect.
3005 * So we keep searching in the exception table if the gateway
3006 * is different.
3007 */
3008 if (!ipv6_addr_equal(gw, &nh->fib_nh_gw6)) {
3009 struct rt6_info *rt_cache;
3010
3011 rt_cache = rt6_find_cached_rt(res, &fl6->daddr, &fl6->saddr);
3012 if (rt_cache &&
3013 ipv6_addr_equal(gw, &rt_cache->rt6i_gateway)) {
3014 *ret = rt_cache;
3015 return true;
3016 }
3017 return false;
3018 }
3019 return true;
3020 }
3021
3022 struct fib6_nh_rd_arg {
3023 struct fib6_result *res;
3024 struct flowi6 *fl6;
3025 const struct in6_addr *gw;
3026 struct rt6_info **ret;
3027 };
3028
fib6_nh_redirect_match(struct fib6_nh * nh,void * _arg)3029 static int fib6_nh_redirect_match(struct fib6_nh *nh, void *_arg)
3030 {
3031 struct fib6_nh_rd_arg *arg = _arg;
3032
3033 arg->res->nh = nh;
3034 return ip6_redirect_nh_match(arg->res, arg->fl6, arg->gw, arg->ret);
3035 }
3036
3037 /* Handle redirects */
3038 struct ip6rd_flowi {
3039 struct flowi6 fl6;
3040 struct in6_addr gateway;
3041 };
3042
__ip6_route_redirect(struct net * net,struct fib6_table * table,struct flowi6 * fl6,const struct sk_buff * skb,int flags)3043 INDIRECT_CALLABLE_SCOPE struct rt6_info *__ip6_route_redirect(struct net *net,
3044 struct fib6_table *table,
3045 struct flowi6 *fl6,
3046 const struct sk_buff *skb,
3047 int flags)
3048 {
3049 struct ip6rd_flowi *rdfl = (struct ip6rd_flowi *)fl6;
3050 struct rt6_info *ret = NULL;
3051 struct fib6_result res = {};
3052 struct fib6_nh_rd_arg arg = {
3053 .res = &res,
3054 .fl6 = fl6,
3055 .gw = &rdfl->gateway,
3056 .ret = &ret
3057 };
3058 struct fib6_info *rt;
3059 struct fib6_node *fn;
3060
3061 /* Get the "current" route for this destination and
3062 * check if the redirect has come from appropriate router.
3063 *
3064 * RFC 4861 specifies that redirects should only be
3065 * accepted if they come from the nexthop to the target.
3066 * Due to the way the routes are chosen, this notion
3067 * is a bit fuzzy and one might need to check all possible
3068 * routes.
3069 */
3070
3071 rcu_read_lock();
3072 fn = fib6_node_lookup(&table->tb6_root, &fl6->daddr, &fl6->saddr);
3073 restart:
3074 for_each_fib6_node_rt_rcu(fn) {
3075 res.f6i = rt;
3076 if (fib6_check_expired(rt))
3077 continue;
3078 if (rt->fib6_flags & RTF_REJECT)
3079 break;
3080 if (unlikely(rt->nh)) {
3081 if (nexthop_is_blackhole(rt->nh))
3082 continue;
3083 /* on match, res->nh is filled in and potentially ret */
3084 if (nexthop_for_each_fib6_nh(rt->nh,
3085 fib6_nh_redirect_match,
3086 &arg))
3087 goto out;
3088 } else {
3089 res.nh = rt->fib6_nh;
3090 if (ip6_redirect_nh_match(&res, fl6, &rdfl->gateway,
3091 &ret))
3092 goto out;
3093 }
3094 }
3095
3096 if (!rt)
3097 rt = net->ipv6.fib6_null_entry;
3098 else if (rt->fib6_flags & RTF_REJECT) {
3099 ret = net->ipv6.ip6_null_entry;
3100 goto out;
3101 }
3102
3103 if (rt == net->ipv6.fib6_null_entry) {
3104 fn = fib6_backtrack(fn, &fl6->saddr);
3105 if (fn)
3106 goto restart;
3107 }
3108
3109 res.f6i = rt;
3110 res.nh = rt->fib6_nh;
3111 out:
3112 if (ret) {
3113 ip6_hold_safe(net, &ret);
3114 } else {
3115 res.fib6_flags = res.f6i->fib6_flags;
3116 res.fib6_type = res.f6i->fib6_type;
3117 ret = ip6_create_rt_rcu(&res);
3118 }
3119
3120 rcu_read_unlock();
3121
3122 trace_fib6_table_lookup(net, &res, table, fl6);
3123 return ret;
3124 };
3125
ip6_route_redirect(struct net * net,const struct flowi6 * fl6,const struct sk_buff * skb,const struct in6_addr * gateway)3126 static struct dst_entry *ip6_route_redirect(struct net *net,
3127 const struct flowi6 *fl6,
3128 const struct sk_buff *skb,
3129 const struct in6_addr *gateway)
3130 {
3131 int flags = RT6_LOOKUP_F_HAS_SADDR;
3132 struct ip6rd_flowi rdfl;
3133
3134 rdfl.fl6 = *fl6;
3135 rdfl.gateway = *gateway;
3136
3137 return fib6_rule_lookup(net, &rdfl.fl6, skb,
3138 flags, __ip6_route_redirect);
3139 }
3140
ip6_redirect(struct sk_buff * skb,struct net * net,int oif,u32 mark,kuid_t uid)3141 void ip6_redirect(struct sk_buff *skb, struct net *net, int oif, u32 mark,
3142 kuid_t uid)
3143 {
3144 const struct ipv6hdr *iph = (struct ipv6hdr *) skb->data;
3145 struct dst_entry *dst;
3146 struct flowi6 fl6 = {
3147 .flowi6_iif = LOOPBACK_IFINDEX,
3148 .flowi6_oif = oif,
3149 .flowi6_mark = mark,
3150 .daddr = iph->daddr,
3151 .saddr = iph->saddr,
3152 .flowlabel = ip6_flowinfo(iph),
3153 .flowi6_uid = uid,
3154 };
3155
3156 dst = ip6_route_redirect(net, &fl6, skb, &ipv6_hdr(skb)->saddr);
3157 rt6_do_redirect(dst, NULL, skb);
3158 dst_release(dst);
3159 }
3160 EXPORT_SYMBOL_GPL(ip6_redirect);
3161
ip6_redirect_no_header(struct sk_buff * skb,struct net * net,int oif)3162 void ip6_redirect_no_header(struct sk_buff *skb, struct net *net, int oif)
3163 {
3164 const struct ipv6hdr *iph = ipv6_hdr(skb);
3165 const struct rd_msg *msg = (struct rd_msg *)icmp6_hdr(skb);
3166 struct dst_entry *dst;
3167 struct flowi6 fl6 = {
3168 .flowi6_iif = LOOPBACK_IFINDEX,
3169 .flowi6_oif = oif,
3170 .daddr = msg->dest,
3171 .saddr = iph->daddr,
3172 .flowi6_uid = sock_net_uid(net, NULL),
3173 };
3174
3175 dst = ip6_route_redirect(net, &fl6, skb, &iph->saddr);
3176 rt6_do_redirect(dst, NULL, skb);
3177 dst_release(dst);
3178 }
3179
ip6_sk_redirect(struct sk_buff * skb,struct sock * sk)3180 void ip6_sk_redirect(struct sk_buff *skb, struct sock *sk)
3181 {
3182 ip6_redirect(skb, sock_net(sk), sk->sk_bound_dev_if,
3183 READ_ONCE(sk->sk_mark), sk->sk_uid);
3184 }
3185 EXPORT_SYMBOL_GPL(ip6_sk_redirect);
3186
ip6_default_advmss(const struct dst_entry * dst)3187 static unsigned int ip6_default_advmss(const struct dst_entry *dst)
3188 {
3189 struct net_device *dev = dst->dev;
3190 unsigned int mtu = dst_mtu(dst);
3191 struct net *net;
3192
3193 mtu -= sizeof(struct ipv6hdr) + sizeof(struct tcphdr);
3194
3195 rcu_read_lock();
3196
3197 net = dev_net_rcu(dev);
3198 if (mtu < net->ipv6.sysctl.ip6_rt_min_advmss)
3199 mtu = net->ipv6.sysctl.ip6_rt_min_advmss;
3200
3201 rcu_read_unlock();
3202
3203 /*
3204 * Maximal non-jumbo IPv6 payload is IPV6_MAXPLEN and
3205 * corresponding MSS is IPV6_MAXPLEN - tcp_header_size.
3206 * IPV6_MAXPLEN is also valid and means: "any MSS,
3207 * rely only on pmtu discovery"
3208 */
3209 if (mtu > IPV6_MAXPLEN - sizeof(struct tcphdr))
3210 mtu = IPV6_MAXPLEN;
3211 return mtu;
3212 }
3213
ip6_mtu(const struct dst_entry * dst)3214 INDIRECT_CALLABLE_SCOPE unsigned int ip6_mtu(const struct dst_entry *dst)
3215 {
3216 return ip6_dst_mtu_maybe_forward(dst, false);
3217 }
3218 EXPORT_INDIRECT_CALLABLE(ip6_mtu);
3219
3220 /* MTU selection:
3221 * 1. mtu on route is locked - use it
3222 * 2. mtu from nexthop exception
3223 * 3. mtu from egress device
3224 *
3225 * based on ip6_dst_mtu_forward and exception logic of
3226 * rt6_find_cached_rt; called with rcu_read_lock
3227 */
ip6_mtu_from_fib6(const struct fib6_result * res,const struct in6_addr * daddr,const struct in6_addr * saddr)3228 u32 ip6_mtu_from_fib6(const struct fib6_result *res,
3229 const struct in6_addr *daddr,
3230 const struct in6_addr *saddr)
3231 {
3232 const struct fib6_nh *nh = res->nh;
3233 struct fib6_info *f6i = res->f6i;
3234 struct inet6_dev *idev;
3235 struct rt6_info *rt;
3236 u32 mtu = 0;
3237
3238 if (unlikely(fib6_metric_locked(f6i, RTAX_MTU))) {
3239 mtu = f6i->fib6_pmtu;
3240 if (mtu)
3241 goto out;
3242 }
3243
3244 rt = rt6_find_cached_rt(res, daddr, saddr);
3245 if (unlikely(rt)) {
3246 mtu = dst_metric_raw(&rt->dst, RTAX_MTU);
3247 } else {
3248 struct net_device *dev = nh->fib_nh_dev;
3249
3250 mtu = IPV6_MIN_MTU;
3251 idev = __in6_dev_get(dev);
3252 if (idev && idev->cnf.mtu6 > mtu)
3253 mtu = idev->cnf.mtu6;
3254 }
3255
3256 mtu = min_t(unsigned int, mtu, IP6_MAX_MTU);
3257 out:
3258 return mtu - lwtunnel_headroom(nh->fib_nh_lws, mtu);
3259 }
3260
icmp6_dst_alloc(struct net_device * dev,struct flowi6 * fl6)3261 struct dst_entry *icmp6_dst_alloc(struct net_device *dev,
3262 struct flowi6 *fl6)
3263 {
3264 struct dst_entry *dst;
3265 struct rt6_info *rt;
3266 struct inet6_dev *idev = in6_dev_get(dev);
3267 struct net *net = dev_net(dev);
3268
3269 if (unlikely(!idev))
3270 return ERR_PTR(-ENODEV);
3271
3272 rt = ip6_dst_alloc(net, dev, 0);
3273 if (unlikely(!rt)) {
3274 in6_dev_put(idev);
3275 dst = ERR_PTR(-ENOMEM);
3276 goto out;
3277 }
3278
3279 rt->dst.input = ip6_input;
3280 rt->dst.output = ip6_output;
3281 rt->rt6i_gateway = fl6->daddr;
3282 rt->rt6i_dst.addr = fl6->daddr;
3283 rt->rt6i_dst.plen = 128;
3284 rt->rt6i_idev = idev;
3285 dst_metric_set(&rt->dst, RTAX_HOPLIMIT, 0);
3286
3287 /* Add this dst into uncached_list so that rt6_disable_ip() can
3288 * do proper release of the net_device
3289 */
3290 rt6_uncached_list_add(rt);
3291
3292 dst = xfrm_lookup(net, &rt->dst, flowi6_to_flowi(fl6), NULL, 0);
3293
3294 out:
3295 return dst;
3296 }
3297
ip6_dst_gc(struct dst_ops * ops)3298 static void ip6_dst_gc(struct dst_ops *ops)
3299 {
3300 struct net *net = container_of(ops, struct net, ipv6.ip6_dst_ops);
3301 int rt_min_interval = net->ipv6.sysctl.ip6_rt_gc_min_interval;
3302 int rt_elasticity = net->ipv6.sysctl.ip6_rt_gc_elasticity;
3303 int rt_gc_timeout = net->ipv6.sysctl.ip6_rt_gc_timeout;
3304 unsigned long rt_last_gc = net->ipv6.ip6_rt_last_gc;
3305 unsigned int val;
3306 int entries;
3307
3308 if (time_after(rt_last_gc + rt_min_interval, jiffies))
3309 goto out;
3310
3311 fib6_run_gc(atomic_inc_return(&net->ipv6.ip6_rt_gc_expire), net, true);
3312 entries = dst_entries_get_slow(ops);
3313 if (entries < ops->gc_thresh)
3314 atomic_set(&net->ipv6.ip6_rt_gc_expire, rt_gc_timeout >> 1);
3315 out:
3316 val = atomic_read(&net->ipv6.ip6_rt_gc_expire);
3317 atomic_set(&net->ipv6.ip6_rt_gc_expire, val - (val >> rt_elasticity));
3318 }
3319
ip6_nh_lookup_table(struct net * net,struct fib6_config * cfg,const struct in6_addr * gw_addr,u32 tbid,int flags,struct fib6_result * res)3320 static int ip6_nh_lookup_table(struct net *net, struct fib6_config *cfg,
3321 const struct in6_addr *gw_addr, u32 tbid,
3322 int flags, struct fib6_result *res)
3323 {
3324 struct flowi6 fl6 = {
3325 .flowi6_oif = cfg->fc_ifindex,
3326 .daddr = *gw_addr,
3327 .saddr = cfg->fc_prefsrc,
3328 };
3329 struct fib6_table *table;
3330 int err;
3331
3332 table = fib6_get_table(net, tbid);
3333 if (!table)
3334 return -EINVAL;
3335
3336 if (!ipv6_addr_any(&cfg->fc_prefsrc))
3337 flags |= RT6_LOOKUP_F_HAS_SADDR;
3338
3339 flags |= RT6_LOOKUP_F_IGNORE_LINKSTATE;
3340
3341 err = fib6_table_lookup(net, table, cfg->fc_ifindex, &fl6, res, flags);
3342 if (!err && res->f6i != net->ipv6.fib6_null_entry)
3343 fib6_select_path(net, res, &fl6, cfg->fc_ifindex,
3344 cfg->fc_ifindex != 0, NULL, flags);
3345
3346 return err;
3347 }
3348
ip6_route_check_nh_onlink(struct net * net,struct fib6_config * cfg,const struct net_device * dev,struct netlink_ext_ack * extack)3349 static int ip6_route_check_nh_onlink(struct net *net,
3350 struct fib6_config *cfg,
3351 const struct net_device *dev,
3352 struct netlink_ext_ack *extack)
3353 {
3354 u32 tbid = l3mdev_fib_table_rcu(dev) ? : RT_TABLE_MAIN;
3355 const struct in6_addr *gw_addr = &cfg->fc_gateway;
3356 struct fib6_result res = {};
3357 int err;
3358
3359 err = ip6_nh_lookup_table(net, cfg, gw_addr, tbid, 0, &res);
3360 if (!err && !(res.fib6_flags & RTF_REJECT) &&
3361 /* ignore match if it is the default route */
3362 !ipv6_addr_any(&res.f6i->fib6_dst.addr) &&
3363 (res.fib6_type != RTN_UNICAST || dev != res.nh->fib_nh_dev)) {
3364 NL_SET_ERR_MSG(extack,
3365 "Nexthop has invalid gateway or device mismatch");
3366 err = -EINVAL;
3367 }
3368
3369 return err;
3370 }
3371
ip6_route_check_nh(struct net * net,struct fib6_config * cfg,struct net_device ** _dev,netdevice_tracker * dev_tracker,struct inet6_dev ** idev)3372 static int ip6_route_check_nh(struct net *net,
3373 struct fib6_config *cfg,
3374 struct net_device **_dev,
3375 netdevice_tracker *dev_tracker,
3376 struct inet6_dev **idev)
3377 {
3378 const struct in6_addr *gw_addr = &cfg->fc_gateway;
3379 struct net_device *dev = _dev ? *_dev : NULL;
3380 int flags = RT6_LOOKUP_F_IFACE;
3381 struct fib6_result res = {};
3382 int err = -EHOSTUNREACH;
3383
3384 if (cfg->fc_table) {
3385 err = ip6_nh_lookup_table(net, cfg, gw_addr,
3386 cfg->fc_table, flags, &res);
3387 /* gw_addr can not require a gateway or resolve to a reject
3388 * route. If a device is given, it must match the result.
3389 */
3390 if (err || res.fib6_flags & RTF_REJECT ||
3391 res.nh->fib_nh_gw_family ||
3392 (dev && dev != res.nh->fib_nh_dev))
3393 err = -EHOSTUNREACH;
3394 }
3395
3396 if (err < 0) {
3397 struct flowi6 fl6 = {
3398 .flowi6_oif = cfg->fc_ifindex,
3399 .daddr = *gw_addr,
3400 };
3401
3402 err = fib6_lookup(net, cfg->fc_ifindex, &fl6, &res, flags);
3403 if (err || res.fib6_flags & RTF_REJECT ||
3404 res.nh->fib_nh_gw_family)
3405 err = -EHOSTUNREACH;
3406
3407 if (err)
3408 return err;
3409
3410 fib6_select_path(net, &res, &fl6, cfg->fc_ifindex,
3411 cfg->fc_ifindex != 0, NULL, flags);
3412 }
3413
3414 err = 0;
3415 if (dev) {
3416 if (dev != res.nh->fib_nh_dev)
3417 err = -EHOSTUNREACH;
3418 } else {
3419 *_dev = dev = res.nh->fib_nh_dev;
3420 netdev_hold(dev, dev_tracker, GFP_ATOMIC);
3421 *idev = in6_dev_get(dev);
3422 }
3423
3424 return err;
3425 }
3426
ip6_validate_gw(struct net * net,struct fib6_config * cfg,struct net_device ** _dev,netdevice_tracker * dev_tracker,struct inet6_dev ** idev,struct netlink_ext_ack * extack)3427 static int ip6_validate_gw(struct net *net, struct fib6_config *cfg,
3428 struct net_device **_dev,
3429 netdevice_tracker *dev_tracker,
3430 struct inet6_dev **idev,
3431 struct netlink_ext_ack *extack)
3432 {
3433 const struct in6_addr *gw_addr = &cfg->fc_gateway;
3434 int gwa_type = ipv6_addr_type(gw_addr);
3435 bool skip_dev = gwa_type & IPV6_ADDR_LINKLOCAL ? false : true;
3436 const struct net_device *dev = *_dev;
3437 bool need_addr_check = !dev;
3438 int err = -EINVAL;
3439
3440 /* if gw_addr is local we will fail to detect this in case
3441 * address is still TENTATIVE (DAD in progress). rt6_lookup()
3442 * will return already-added prefix route via interface that
3443 * prefix route was assigned to, which might be non-loopback.
3444 */
3445 if (dev &&
3446 ipv6_chk_addr_and_flags(net, gw_addr, dev, skip_dev, 0, 0)) {
3447 NL_SET_ERR_MSG(extack, "Gateway can not be a local address");
3448 goto out;
3449 }
3450
3451 if (gwa_type != (IPV6_ADDR_LINKLOCAL | IPV6_ADDR_UNICAST)) {
3452 /* IPv6 strictly inhibits using not link-local
3453 * addresses as nexthop address.
3454 * Otherwise, router will not able to send redirects.
3455 * It is very good, but in some (rare!) circumstances
3456 * (SIT, PtP, NBMA NOARP links) it is handy to allow
3457 * some exceptions. --ANK
3458 * We allow IPv4-mapped nexthops to support RFC4798-type
3459 * addressing
3460 */
3461 if (!(gwa_type & (IPV6_ADDR_UNICAST | IPV6_ADDR_MAPPED))) {
3462 NL_SET_ERR_MSG(extack, "Invalid gateway address");
3463 goto out;
3464 }
3465
3466 rcu_read_lock();
3467
3468 if (cfg->fc_flags & RTNH_F_ONLINK)
3469 err = ip6_route_check_nh_onlink(net, cfg, dev, extack);
3470 else
3471 err = ip6_route_check_nh(net, cfg, _dev, dev_tracker,
3472 idev);
3473
3474 rcu_read_unlock();
3475
3476 if (err)
3477 goto out;
3478 }
3479
3480 /* reload in case device was changed */
3481 dev = *_dev;
3482
3483 err = -EINVAL;
3484 if (!dev) {
3485 NL_SET_ERR_MSG(extack, "Egress device not specified");
3486 goto out;
3487 } else if (dev->flags & IFF_LOOPBACK) {
3488 NL_SET_ERR_MSG(extack,
3489 "Egress device can not be loopback device for this route");
3490 goto out;
3491 }
3492
3493 /* if we did not check gw_addr above, do so now that the
3494 * egress device has been resolved.
3495 */
3496 if (need_addr_check &&
3497 ipv6_chk_addr_and_flags(net, gw_addr, dev, skip_dev, 0, 0)) {
3498 NL_SET_ERR_MSG(extack, "Gateway can not be a local address");
3499 goto out;
3500 }
3501
3502 err = 0;
3503 out:
3504 return err;
3505 }
3506
fib6_is_reject(u32 flags,struct net_device * dev,int addr_type)3507 static bool fib6_is_reject(u32 flags, struct net_device *dev, int addr_type)
3508 {
3509 if ((flags & RTF_REJECT) ||
3510 (dev && (dev->flags & IFF_LOOPBACK) &&
3511 !(addr_type & IPV6_ADDR_LOOPBACK) &&
3512 !(flags & (RTF_ANYCAST | RTF_LOCAL))))
3513 return true;
3514
3515 return false;
3516 }
3517
fib6_nh_init(struct net * net,struct fib6_nh * fib6_nh,struct fib6_config * cfg,gfp_t gfp_flags,struct netlink_ext_ack * extack)3518 int fib6_nh_init(struct net *net, struct fib6_nh *fib6_nh,
3519 struct fib6_config *cfg, gfp_t gfp_flags,
3520 struct netlink_ext_ack *extack)
3521 {
3522 netdevice_tracker *dev_tracker = &fib6_nh->fib_nh_dev_tracker;
3523 struct net_device *dev = NULL;
3524 struct inet6_dev *idev = NULL;
3525 int addr_type;
3526 int err;
3527
3528 fib6_nh->fib_nh_family = AF_INET6;
3529 #ifdef CONFIG_IPV6_ROUTER_PREF
3530 fib6_nh->last_probe = jiffies;
3531 #endif
3532 if (cfg->fc_is_fdb) {
3533 fib6_nh->fib_nh_gw6 = cfg->fc_gateway;
3534 fib6_nh->fib_nh_gw_family = AF_INET6;
3535 return 0;
3536 }
3537
3538 err = -ENODEV;
3539 if (cfg->fc_ifindex) {
3540 dev = netdev_get_by_index(net, cfg->fc_ifindex,
3541 dev_tracker, gfp_flags);
3542 if (!dev)
3543 goto out;
3544 idev = in6_dev_get(dev);
3545 if (!idev)
3546 goto out;
3547 }
3548
3549 if (cfg->fc_flags & RTNH_F_ONLINK) {
3550 if (!dev) {
3551 NL_SET_ERR_MSG(extack,
3552 "Nexthop device required for onlink");
3553 goto out;
3554 }
3555
3556 if (!(dev->flags & IFF_UP)) {
3557 NL_SET_ERR_MSG(extack, "Nexthop device is not up");
3558 err = -ENETDOWN;
3559 goto out;
3560 }
3561
3562 fib6_nh->fib_nh_flags |= RTNH_F_ONLINK;
3563 }
3564
3565 fib6_nh->fib_nh_weight = 1;
3566
3567 /* We cannot add true routes via loopback here,
3568 * they would result in kernel looping; promote them to reject routes
3569 */
3570 addr_type = ipv6_addr_type(&cfg->fc_dst);
3571 if (fib6_is_reject(cfg->fc_flags, dev, addr_type)) {
3572 /* hold loopback dev/idev if we haven't done so. */
3573 if (dev != net->loopback_dev) {
3574 if (dev) {
3575 netdev_put(dev, dev_tracker);
3576 in6_dev_put(idev);
3577 }
3578 dev = net->loopback_dev;
3579 netdev_hold(dev, dev_tracker, gfp_flags);
3580 idev = in6_dev_get(dev);
3581 if (!idev) {
3582 err = -ENODEV;
3583 goto out;
3584 }
3585 }
3586 goto pcpu_alloc;
3587 }
3588
3589 if (cfg->fc_flags & RTF_GATEWAY) {
3590 err = ip6_validate_gw(net, cfg, &dev, dev_tracker,
3591 &idev, extack);
3592 if (err)
3593 goto out;
3594
3595 fib6_nh->fib_nh_gw6 = cfg->fc_gateway;
3596 fib6_nh->fib_nh_gw_family = AF_INET6;
3597 }
3598
3599 err = -ENODEV;
3600 if (!dev)
3601 goto out;
3602
3603 if (!idev || idev->cnf.disable_ipv6) {
3604 NL_SET_ERR_MSG(extack, "IPv6 is disabled on nexthop device");
3605 err = -EACCES;
3606 goto out;
3607 }
3608
3609 if (!(dev->flags & IFF_UP) && !cfg->fc_ignore_dev_down) {
3610 NL_SET_ERR_MSG(extack, "Nexthop device is not up");
3611 err = -ENETDOWN;
3612 goto out;
3613 }
3614
3615 if (!(cfg->fc_flags & (RTF_LOCAL | RTF_ANYCAST)) &&
3616 !netif_carrier_ok(dev))
3617 fib6_nh->fib_nh_flags |= RTNH_F_LINKDOWN;
3618
3619 err = fib_nh_common_init(net, &fib6_nh->nh_common, cfg->fc_encap,
3620 cfg->fc_encap_type, cfg, gfp_flags, extack);
3621 if (err)
3622 goto out;
3623
3624 pcpu_alloc:
3625 fib6_nh->rt6i_pcpu = alloc_percpu_gfp(struct rt6_info *, gfp_flags);
3626 if (!fib6_nh->rt6i_pcpu) {
3627 err = -ENOMEM;
3628 goto out;
3629 }
3630
3631 fib6_nh->fib_nh_dev = dev;
3632 fib6_nh->fib_nh_oif = dev->ifindex;
3633 err = 0;
3634 out:
3635 if (idev)
3636 in6_dev_put(idev);
3637
3638 if (err) {
3639 lwtstate_put(fib6_nh->fib_nh_lws);
3640 fib6_nh->fib_nh_lws = NULL;
3641 netdev_put(dev, dev_tracker);
3642 }
3643
3644 return err;
3645 }
3646
fib6_nh_release(struct fib6_nh * fib6_nh)3647 void fib6_nh_release(struct fib6_nh *fib6_nh)
3648 {
3649 struct rt6_exception_bucket *bucket;
3650
3651 rcu_read_lock();
3652
3653 fib6_nh_flush_exceptions(fib6_nh, NULL);
3654 bucket = fib6_nh_get_excptn_bucket(fib6_nh, NULL);
3655 if (bucket) {
3656 rcu_assign_pointer(fib6_nh->rt6i_exception_bucket, NULL);
3657 kfree(bucket);
3658 }
3659
3660 rcu_read_unlock();
3661
3662 fib6_nh_release_dsts(fib6_nh);
3663 free_percpu(fib6_nh->rt6i_pcpu);
3664
3665 fib_nh_common_release(&fib6_nh->nh_common);
3666 }
3667
fib6_nh_release_dsts(struct fib6_nh * fib6_nh)3668 void fib6_nh_release_dsts(struct fib6_nh *fib6_nh)
3669 {
3670 int cpu;
3671
3672 if (!fib6_nh->rt6i_pcpu)
3673 return;
3674
3675 for_each_possible_cpu(cpu) {
3676 struct rt6_info *pcpu_rt, **ppcpu_rt;
3677
3678 ppcpu_rt = per_cpu_ptr(fib6_nh->rt6i_pcpu, cpu);
3679 pcpu_rt = xchg(ppcpu_rt, NULL);
3680 if (pcpu_rt) {
3681 dst_dev_put(&pcpu_rt->dst);
3682 dst_release(&pcpu_rt->dst);
3683 }
3684 }
3685 }
3686
ip6_route_info_create(struct fib6_config * cfg,gfp_t gfp_flags,struct netlink_ext_ack * extack)3687 static struct fib6_info *ip6_route_info_create(struct fib6_config *cfg,
3688 gfp_t gfp_flags,
3689 struct netlink_ext_ack *extack)
3690 {
3691 struct net *net = cfg->fc_nlinfo.nl_net;
3692 struct fib6_info *rt = NULL;
3693 struct nexthop *nh = NULL;
3694 struct fib6_table *table;
3695 struct fib6_nh *fib6_nh;
3696 int err = -EINVAL;
3697 int addr_type;
3698
3699 /* RTF_PCPU is an internal flag; can not be set by userspace */
3700 if (cfg->fc_flags & RTF_PCPU) {
3701 NL_SET_ERR_MSG(extack, "Userspace can not set RTF_PCPU");
3702 goto out;
3703 }
3704
3705 /* RTF_CACHE is an internal flag; can not be set by userspace */
3706 if (cfg->fc_flags & RTF_CACHE) {
3707 NL_SET_ERR_MSG(extack, "Userspace can not set RTF_CACHE");
3708 goto out;
3709 }
3710
3711 if (cfg->fc_type > RTN_MAX) {
3712 NL_SET_ERR_MSG(extack, "Invalid route type");
3713 goto out;
3714 }
3715
3716 if (cfg->fc_dst_len > 128) {
3717 NL_SET_ERR_MSG(extack, "Invalid prefix length");
3718 goto out;
3719 }
3720 if (cfg->fc_src_len > 128) {
3721 NL_SET_ERR_MSG(extack, "Invalid source address length");
3722 goto out;
3723 }
3724 #ifndef CONFIG_IPV6_SUBTREES
3725 if (cfg->fc_src_len) {
3726 NL_SET_ERR_MSG(extack,
3727 "Specifying source address requires IPV6_SUBTREES to be enabled");
3728 goto out;
3729 }
3730 #endif
3731 if (cfg->fc_nh_id) {
3732 nh = nexthop_find_by_id(net, cfg->fc_nh_id);
3733 if (!nh) {
3734 NL_SET_ERR_MSG(extack, "Nexthop id does not exist");
3735 goto out;
3736 }
3737 err = fib6_check_nexthop(nh, cfg, extack);
3738 if (err)
3739 goto out;
3740 }
3741
3742 err = -ENOBUFS;
3743 if (cfg->fc_nlinfo.nlh &&
3744 !(cfg->fc_nlinfo.nlh->nlmsg_flags & NLM_F_CREATE)) {
3745 table = fib6_get_table(net, cfg->fc_table);
3746 if (!table) {
3747 pr_warn("NLM_F_CREATE should be specified when creating new route\n");
3748 table = fib6_new_table(net, cfg->fc_table);
3749 }
3750 } else {
3751 table = fib6_new_table(net, cfg->fc_table);
3752 }
3753
3754 if (!table)
3755 goto out;
3756
3757 err = -ENOMEM;
3758 rt = fib6_info_alloc(gfp_flags, !nh);
3759 if (!rt)
3760 goto out;
3761
3762 rt->fib6_metrics = ip_fib_metrics_init(cfg->fc_mx, cfg->fc_mx_len,
3763 extack);
3764 if (IS_ERR(rt->fib6_metrics)) {
3765 err = PTR_ERR(rt->fib6_metrics);
3766 /* Do not leave garbage there. */
3767 rt->fib6_metrics = (struct dst_metrics *)&dst_default_metrics;
3768 goto out_free;
3769 }
3770
3771 if (cfg->fc_flags & RTF_ADDRCONF)
3772 rt->dst_nocount = true;
3773
3774 if (cfg->fc_flags & RTF_EXPIRES)
3775 fib6_set_expires(rt, jiffies +
3776 clock_t_to_jiffies(cfg->fc_expires));
3777 else
3778 fib6_clean_expires(rt);
3779
3780 if (cfg->fc_protocol == RTPROT_UNSPEC)
3781 cfg->fc_protocol = RTPROT_BOOT;
3782 rt->fib6_protocol = cfg->fc_protocol;
3783
3784 rt->fib6_table = table;
3785 rt->fib6_metric = cfg->fc_metric;
3786 rt->fib6_type = cfg->fc_type ? : RTN_UNICAST;
3787 rt->fib6_flags = cfg->fc_flags & ~RTF_GATEWAY;
3788
3789 ipv6_addr_prefix(&rt->fib6_dst.addr, &cfg->fc_dst, cfg->fc_dst_len);
3790 rt->fib6_dst.plen = cfg->fc_dst_len;
3791
3792 #ifdef CONFIG_IPV6_SUBTREES
3793 ipv6_addr_prefix(&rt->fib6_src.addr, &cfg->fc_src, cfg->fc_src_len);
3794 rt->fib6_src.plen = cfg->fc_src_len;
3795 #endif
3796 if (nh) {
3797 if (rt->fib6_src.plen) {
3798 NL_SET_ERR_MSG(extack, "Nexthops can not be used with source routing");
3799 goto out_free;
3800 }
3801 if (!nexthop_get(nh)) {
3802 NL_SET_ERR_MSG(extack, "Nexthop has been deleted");
3803 goto out_free;
3804 }
3805 rt->nh = nh;
3806 fib6_nh = nexthop_fib6_nh(rt->nh);
3807 } else {
3808 err = fib6_nh_init(net, rt->fib6_nh, cfg, gfp_flags, extack);
3809 if (err)
3810 goto out;
3811
3812 fib6_nh = rt->fib6_nh;
3813
3814 /* We cannot add true routes via loopback here, they would
3815 * result in kernel looping; promote them to reject routes
3816 */
3817 addr_type = ipv6_addr_type(&cfg->fc_dst);
3818 if (fib6_is_reject(cfg->fc_flags, rt->fib6_nh->fib_nh_dev,
3819 addr_type))
3820 rt->fib6_flags = RTF_REJECT | RTF_NONEXTHOP;
3821 }
3822
3823 if (!ipv6_addr_any(&cfg->fc_prefsrc)) {
3824 struct net_device *dev = fib6_nh->fib_nh_dev;
3825
3826 if (!ipv6_chk_addr(net, &cfg->fc_prefsrc, dev, 0)) {
3827 NL_SET_ERR_MSG(extack, "Invalid source address");
3828 err = -EINVAL;
3829 goto out;
3830 }
3831 rt->fib6_prefsrc.addr = cfg->fc_prefsrc;
3832 rt->fib6_prefsrc.plen = 128;
3833 } else
3834 rt->fib6_prefsrc.plen = 0;
3835
3836 return rt;
3837 out:
3838 fib6_info_release(rt);
3839 return ERR_PTR(err);
3840 out_free:
3841 ip_fib_metrics_put(rt->fib6_metrics);
3842 kfree(rt);
3843 return ERR_PTR(err);
3844 }
3845
ip6_route_add(struct fib6_config * cfg,gfp_t gfp_flags,struct netlink_ext_ack * extack)3846 int ip6_route_add(struct fib6_config *cfg, gfp_t gfp_flags,
3847 struct netlink_ext_ack *extack)
3848 {
3849 struct fib6_info *rt;
3850 int err;
3851
3852 rt = ip6_route_info_create(cfg, gfp_flags, extack);
3853 if (IS_ERR(rt))
3854 return PTR_ERR(rt);
3855
3856 err = __ip6_ins_rt(rt, &cfg->fc_nlinfo, extack);
3857 fib6_info_release(rt);
3858
3859 return err;
3860 }
3861
__ip6_del_rt(struct fib6_info * rt,struct nl_info * info)3862 static int __ip6_del_rt(struct fib6_info *rt, struct nl_info *info)
3863 {
3864 struct net *net = info->nl_net;
3865 struct fib6_table *table;
3866 int err;
3867
3868 if (rt == net->ipv6.fib6_null_entry) {
3869 err = -ENOENT;
3870 goto out;
3871 }
3872
3873 table = rt->fib6_table;
3874 spin_lock_bh(&table->tb6_lock);
3875 err = fib6_del(rt, info);
3876 spin_unlock_bh(&table->tb6_lock);
3877
3878 out:
3879 fib6_info_release(rt);
3880 return err;
3881 }
3882
ip6_del_rt(struct net * net,struct fib6_info * rt,bool skip_notify)3883 int ip6_del_rt(struct net *net, struct fib6_info *rt, bool skip_notify)
3884 {
3885 struct nl_info info = {
3886 .nl_net = net,
3887 .skip_notify = skip_notify
3888 };
3889
3890 return __ip6_del_rt(rt, &info);
3891 }
3892
__ip6_del_rt_siblings(struct fib6_info * rt,struct fib6_config * cfg)3893 static int __ip6_del_rt_siblings(struct fib6_info *rt, struct fib6_config *cfg)
3894 {
3895 struct nl_info *info = &cfg->fc_nlinfo;
3896 struct net *net = info->nl_net;
3897 struct sk_buff *skb = NULL;
3898 struct fib6_table *table;
3899 int err = -ENOENT;
3900
3901 if (rt == net->ipv6.fib6_null_entry)
3902 goto out_put;
3903 table = rt->fib6_table;
3904 spin_lock_bh(&table->tb6_lock);
3905
3906 if (rt->fib6_nsiblings && cfg->fc_delete_all_nh) {
3907 struct fib6_info *sibling, *next_sibling;
3908 struct fib6_node *fn;
3909
3910 /* prefer to send a single notification with all hops */
3911 skb = nlmsg_new(rt6_nlmsg_size(rt), gfp_any());
3912 if (skb) {
3913 u32 seq = info->nlh ? info->nlh->nlmsg_seq : 0;
3914
3915 if (rt6_fill_node(net, skb, rt, NULL,
3916 NULL, NULL, 0, RTM_DELROUTE,
3917 info->portid, seq, 0) < 0) {
3918 kfree_skb(skb);
3919 skb = NULL;
3920 } else
3921 info->skip_notify = 1;
3922 }
3923
3924 /* 'rt' points to the first sibling route. If it is not the
3925 * leaf, then we do not need to send a notification. Otherwise,
3926 * we need to check if the last sibling has a next route or not
3927 * and emit a replace or delete notification, respectively.
3928 */
3929 info->skip_notify_kernel = 1;
3930 fn = rcu_dereference_protected(rt->fib6_node,
3931 lockdep_is_held(&table->tb6_lock));
3932 if (rcu_access_pointer(fn->leaf) == rt) {
3933 struct fib6_info *last_sibling, *replace_rt;
3934
3935 last_sibling = list_last_entry(&rt->fib6_siblings,
3936 struct fib6_info,
3937 fib6_siblings);
3938 replace_rt = rcu_dereference_protected(
3939 last_sibling->fib6_next,
3940 lockdep_is_held(&table->tb6_lock));
3941 if (replace_rt)
3942 call_fib6_entry_notifiers_replace(net,
3943 replace_rt);
3944 else
3945 call_fib6_multipath_entry_notifiers(net,
3946 FIB_EVENT_ENTRY_DEL,
3947 rt, rt->fib6_nsiblings,
3948 NULL);
3949 }
3950 list_for_each_entry_safe(sibling, next_sibling,
3951 &rt->fib6_siblings,
3952 fib6_siblings) {
3953 err = fib6_del(sibling, info);
3954 if (err)
3955 goto out_unlock;
3956 }
3957 }
3958
3959 err = fib6_del(rt, info);
3960 out_unlock:
3961 spin_unlock_bh(&table->tb6_lock);
3962 out_put:
3963 fib6_info_release(rt);
3964
3965 if (skb) {
3966 rtnl_notify(skb, net, info->portid, RTNLGRP_IPV6_ROUTE,
3967 info->nlh, gfp_any());
3968 }
3969 return err;
3970 }
3971
__ip6_del_cached_rt(struct rt6_info * rt,struct fib6_config * cfg)3972 static int __ip6_del_cached_rt(struct rt6_info *rt, struct fib6_config *cfg)
3973 {
3974 int rc = -ESRCH;
3975
3976 if (cfg->fc_ifindex && rt->dst.dev->ifindex != cfg->fc_ifindex)
3977 goto out;
3978
3979 if (cfg->fc_flags & RTF_GATEWAY &&
3980 !ipv6_addr_equal(&cfg->fc_gateway, &rt->rt6i_gateway))
3981 goto out;
3982
3983 rc = rt6_remove_exception_rt(rt);
3984 out:
3985 return rc;
3986 }
3987
ip6_del_cached_rt(struct fib6_config * cfg,struct fib6_info * rt,struct fib6_nh * nh)3988 static int ip6_del_cached_rt(struct fib6_config *cfg, struct fib6_info *rt,
3989 struct fib6_nh *nh)
3990 {
3991 struct fib6_result res = {
3992 .f6i = rt,
3993 .nh = nh,
3994 };
3995 struct rt6_info *rt_cache;
3996
3997 rt_cache = rt6_find_cached_rt(&res, &cfg->fc_dst, &cfg->fc_src);
3998 if (rt_cache)
3999 return __ip6_del_cached_rt(rt_cache, cfg);
4000
4001 return 0;
4002 }
4003
4004 struct fib6_nh_del_cached_rt_arg {
4005 struct fib6_config *cfg;
4006 struct fib6_info *f6i;
4007 };
4008
fib6_nh_del_cached_rt(struct fib6_nh * nh,void * _arg)4009 static int fib6_nh_del_cached_rt(struct fib6_nh *nh, void *_arg)
4010 {
4011 struct fib6_nh_del_cached_rt_arg *arg = _arg;
4012 int rc;
4013
4014 rc = ip6_del_cached_rt(arg->cfg, arg->f6i, nh);
4015 return rc != -ESRCH ? rc : 0;
4016 }
4017
ip6_del_cached_rt_nh(struct fib6_config * cfg,struct fib6_info * f6i)4018 static int ip6_del_cached_rt_nh(struct fib6_config *cfg, struct fib6_info *f6i)
4019 {
4020 struct fib6_nh_del_cached_rt_arg arg = {
4021 .cfg = cfg,
4022 .f6i = f6i
4023 };
4024
4025 return nexthop_for_each_fib6_nh(f6i->nh, fib6_nh_del_cached_rt, &arg);
4026 }
4027
ip6_route_del(struct fib6_config * cfg,struct netlink_ext_ack * extack)4028 static int ip6_route_del(struct fib6_config *cfg,
4029 struct netlink_ext_ack *extack)
4030 {
4031 struct fib6_table *table;
4032 struct fib6_info *rt;
4033 struct fib6_node *fn;
4034 int err = -ESRCH;
4035
4036 table = fib6_get_table(cfg->fc_nlinfo.nl_net, cfg->fc_table);
4037 if (!table) {
4038 NL_SET_ERR_MSG(extack, "FIB table does not exist");
4039 return err;
4040 }
4041
4042 rcu_read_lock();
4043
4044 fn = fib6_locate(&table->tb6_root,
4045 &cfg->fc_dst, cfg->fc_dst_len,
4046 &cfg->fc_src, cfg->fc_src_len,
4047 !(cfg->fc_flags & RTF_CACHE));
4048
4049 if (fn) {
4050 for_each_fib6_node_rt_rcu(fn) {
4051 struct fib6_nh *nh;
4052
4053 if (rt->nh && cfg->fc_nh_id &&
4054 rt->nh->id != cfg->fc_nh_id)
4055 continue;
4056
4057 if (cfg->fc_flags & RTF_CACHE) {
4058 int rc = 0;
4059
4060 if (rt->nh) {
4061 rc = ip6_del_cached_rt_nh(cfg, rt);
4062 } else if (cfg->fc_nh_id) {
4063 continue;
4064 } else {
4065 nh = rt->fib6_nh;
4066 rc = ip6_del_cached_rt(cfg, rt, nh);
4067 }
4068 if (rc != -ESRCH) {
4069 rcu_read_unlock();
4070 return rc;
4071 }
4072 continue;
4073 }
4074
4075 if (cfg->fc_metric && cfg->fc_metric != rt->fib6_metric)
4076 continue;
4077 if (cfg->fc_protocol &&
4078 cfg->fc_protocol != rt->fib6_protocol)
4079 continue;
4080
4081 if (rt->nh) {
4082 if (!fib6_info_hold_safe(rt))
4083 continue;
4084 rcu_read_unlock();
4085
4086 return __ip6_del_rt(rt, &cfg->fc_nlinfo);
4087 }
4088 if (cfg->fc_nh_id)
4089 continue;
4090
4091 nh = rt->fib6_nh;
4092 if (cfg->fc_ifindex &&
4093 (!nh->fib_nh_dev ||
4094 nh->fib_nh_dev->ifindex != cfg->fc_ifindex))
4095 continue;
4096 if (cfg->fc_flags & RTF_GATEWAY &&
4097 !ipv6_addr_equal(&cfg->fc_gateway, &nh->fib_nh_gw6))
4098 continue;
4099 if (!fib6_info_hold_safe(rt))
4100 continue;
4101 rcu_read_unlock();
4102
4103 /* if gateway was specified only delete the one hop */
4104 if (cfg->fc_flags & RTF_GATEWAY)
4105 return __ip6_del_rt(rt, &cfg->fc_nlinfo);
4106
4107 return __ip6_del_rt_siblings(rt, cfg);
4108 }
4109 }
4110 rcu_read_unlock();
4111
4112 return err;
4113 }
4114
rt6_do_redirect(struct dst_entry * dst,struct sock * sk,struct sk_buff * skb)4115 static void rt6_do_redirect(struct dst_entry *dst, struct sock *sk, struct sk_buff *skb)
4116 {
4117 struct netevent_redirect netevent;
4118 struct rt6_info *rt, *nrt = NULL;
4119 struct fib6_result res = {};
4120 struct ndisc_options ndopts;
4121 struct inet6_dev *in6_dev;
4122 struct neighbour *neigh;
4123 struct rd_msg *msg;
4124 int optlen, on_link;
4125 u8 *lladdr;
4126
4127 optlen = skb_tail_pointer(skb) - skb_transport_header(skb);
4128 optlen -= sizeof(*msg);
4129
4130 if (optlen < 0) {
4131 net_dbg_ratelimited("rt6_do_redirect: packet too short\n");
4132 return;
4133 }
4134
4135 msg = (struct rd_msg *)icmp6_hdr(skb);
4136
4137 if (ipv6_addr_is_multicast(&msg->dest)) {
4138 net_dbg_ratelimited("rt6_do_redirect: destination address is multicast\n");
4139 return;
4140 }
4141
4142 on_link = 0;
4143 if (ipv6_addr_equal(&msg->dest, &msg->target)) {
4144 on_link = 1;
4145 } else if (ipv6_addr_type(&msg->target) !=
4146 (IPV6_ADDR_UNICAST|IPV6_ADDR_LINKLOCAL)) {
4147 net_dbg_ratelimited("rt6_do_redirect: target address is not link-local unicast\n");
4148 return;
4149 }
4150
4151 in6_dev = __in6_dev_get(skb->dev);
4152 if (!in6_dev)
4153 return;
4154 if (in6_dev->cnf.forwarding || !in6_dev->cnf.accept_redirects)
4155 return;
4156
4157 /* RFC2461 8.1:
4158 * The IP source address of the Redirect MUST be the same as the current
4159 * first-hop router for the specified ICMP Destination Address.
4160 */
4161
4162 if (!ndisc_parse_options(skb->dev, msg->opt, optlen, &ndopts)) {
4163 net_dbg_ratelimited("rt6_redirect: invalid ND options\n");
4164 return;
4165 }
4166
4167 lladdr = NULL;
4168 if (ndopts.nd_opts_tgt_lladdr) {
4169 lladdr = ndisc_opt_addr_data(ndopts.nd_opts_tgt_lladdr,
4170 skb->dev);
4171 if (!lladdr) {
4172 net_dbg_ratelimited("rt6_redirect: invalid link-layer address length\n");
4173 return;
4174 }
4175 }
4176
4177 rt = dst_rt6_info(dst);
4178 if (rt->rt6i_flags & RTF_REJECT) {
4179 net_dbg_ratelimited("rt6_redirect: source isn't a valid nexthop for redirect target\n");
4180 return;
4181 }
4182
4183 /* Redirect received -> path was valid.
4184 * Look, redirects are sent only in response to data packets,
4185 * so that this nexthop apparently is reachable. --ANK
4186 */
4187 dst_confirm_neigh(&rt->dst, &ipv6_hdr(skb)->saddr);
4188
4189 neigh = __neigh_lookup(&nd_tbl, &msg->target, skb->dev, 1);
4190 if (!neigh)
4191 return;
4192
4193 /*
4194 * We have finally decided to accept it.
4195 */
4196
4197 ndisc_update(skb->dev, neigh, lladdr, NUD_STALE,
4198 NEIGH_UPDATE_F_WEAK_OVERRIDE|
4199 NEIGH_UPDATE_F_OVERRIDE|
4200 (on_link ? 0 : (NEIGH_UPDATE_F_OVERRIDE_ISROUTER|
4201 NEIGH_UPDATE_F_ISROUTER)),
4202 NDISC_REDIRECT, &ndopts);
4203
4204 rcu_read_lock();
4205 res.f6i = rcu_dereference(rt->from);
4206 if (!res.f6i)
4207 goto out;
4208
4209 if (res.f6i->nh) {
4210 struct fib6_nh_match_arg arg = {
4211 .dev = dst->dev,
4212 .gw = &rt->rt6i_gateway,
4213 };
4214
4215 nexthop_for_each_fib6_nh(res.f6i->nh,
4216 fib6_nh_find_match, &arg);
4217
4218 /* fib6_info uses a nexthop that does not have fib6_nh
4219 * using the dst->dev. Should be impossible
4220 */
4221 if (!arg.match)
4222 goto out;
4223 res.nh = arg.match;
4224 } else {
4225 res.nh = res.f6i->fib6_nh;
4226 }
4227
4228 res.fib6_flags = res.f6i->fib6_flags;
4229 res.fib6_type = res.f6i->fib6_type;
4230 nrt = ip6_rt_cache_alloc(&res, &msg->dest, NULL);
4231 if (!nrt)
4232 goto out;
4233
4234 nrt->rt6i_flags = RTF_GATEWAY|RTF_UP|RTF_DYNAMIC|RTF_CACHE;
4235 if (on_link)
4236 nrt->rt6i_flags &= ~RTF_GATEWAY;
4237
4238 nrt->rt6i_gateway = *(struct in6_addr *)neigh->primary_key;
4239
4240 /* rt6_insert_exception() will take care of duplicated exceptions */
4241 if (rt6_insert_exception(nrt, &res)) {
4242 dst_release_immediate(&nrt->dst);
4243 goto out;
4244 }
4245
4246 netevent.old = &rt->dst;
4247 netevent.new = &nrt->dst;
4248 netevent.daddr = &msg->dest;
4249 netevent.neigh = neigh;
4250 call_netevent_notifiers(NETEVENT_REDIRECT, &netevent);
4251
4252 out:
4253 rcu_read_unlock();
4254 neigh_release(neigh);
4255 }
4256
4257 #ifdef CONFIG_IPV6_ROUTE_INFO
rt6_get_route_info(struct net * net,const struct in6_addr * prefix,int prefixlen,const struct in6_addr * gwaddr,struct net_device * dev)4258 static struct fib6_info *rt6_get_route_info(struct net *net,
4259 const struct in6_addr *prefix, int prefixlen,
4260 const struct in6_addr *gwaddr,
4261 struct net_device *dev)
4262 {
4263 u32 tb_id = l3mdev_fib_table(dev) ? : RT6_TABLE_INFO;
4264 int ifindex = dev->ifindex;
4265 struct fib6_node *fn;
4266 struct fib6_info *rt = NULL;
4267 struct fib6_table *table;
4268
4269 table = fib6_get_table(net, tb_id);
4270 if (!table)
4271 return NULL;
4272
4273 rcu_read_lock();
4274 fn = fib6_locate(&table->tb6_root, prefix, prefixlen, NULL, 0, true);
4275 if (!fn)
4276 goto out;
4277
4278 for_each_fib6_node_rt_rcu(fn) {
4279 /* these routes do not use nexthops */
4280 if (rt->nh)
4281 continue;
4282 if (rt->fib6_nh->fib_nh_dev->ifindex != ifindex)
4283 continue;
4284 if (!(rt->fib6_flags & RTF_ROUTEINFO) ||
4285 !rt->fib6_nh->fib_nh_gw_family)
4286 continue;
4287 if (!ipv6_addr_equal(&rt->fib6_nh->fib_nh_gw6, gwaddr))
4288 continue;
4289 if (!fib6_info_hold_safe(rt))
4290 continue;
4291 break;
4292 }
4293 out:
4294 rcu_read_unlock();
4295 return rt;
4296 }
4297
rt6_add_route_info(struct net * net,const struct in6_addr * prefix,int prefixlen,const struct in6_addr * gwaddr,struct net_device * dev,unsigned int pref)4298 static struct fib6_info *rt6_add_route_info(struct net *net,
4299 const struct in6_addr *prefix, int prefixlen,
4300 const struct in6_addr *gwaddr,
4301 struct net_device *dev,
4302 unsigned int pref)
4303 {
4304 struct fib6_config cfg = {
4305 .fc_metric = IP6_RT_PRIO_USER,
4306 .fc_ifindex = dev->ifindex,
4307 .fc_dst_len = prefixlen,
4308 .fc_flags = RTF_GATEWAY | RTF_ADDRCONF | RTF_ROUTEINFO |
4309 RTF_UP | RTF_PREF(pref),
4310 .fc_protocol = RTPROT_RA,
4311 .fc_type = RTN_UNICAST,
4312 .fc_nlinfo.portid = 0,
4313 .fc_nlinfo.nlh = NULL,
4314 .fc_nlinfo.nl_net = net,
4315 };
4316
4317 cfg.fc_table = l3mdev_fib_table(dev) ? : RT6_TABLE_INFO;
4318 cfg.fc_dst = *prefix;
4319 cfg.fc_gateway = *gwaddr;
4320
4321 /* We should treat it as a default route if prefix length is 0. */
4322 if (!prefixlen)
4323 cfg.fc_flags |= RTF_DEFAULT;
4324
4325 ip6_route_add(&cfg, GFP_ATOMIC, NULL);
4326
4327 return rt6_get_route_info(net, prefix, prefixlen, gwaddr, dev);
4328 }
4329 #endif
4330
rt6_get_dflt_router(struct net * net,const struct in6_addr * addr,struct net_device * dev)4331 struct fib6_info *rt6_get_dflt_router(struct net *net,
4332 const struct in6_addr *addr,
4333 struct net_device *dev)
4334 {
4335 u32 tb_id = l3mdev_fib_table(dev) ? : RT6_TABLE_DFLT;
4336 struct fib6_info *rt;
4337 struct fib6_table *table;
4338
4339 table = fib6_get_table(net, tb_id);
4340 if (!table)
4341 return NULL;
4342
4343 rcu_read_lock();
4344 for_each_fib6_node_rt_rcu(&table->tb6_root) {
4345 struct fib6_nh *nh;
4346
4347 /* RA routes do not use nexthops */
4348 if (rt->nh)
4349 continue;
4350
4351 nh = rt->fib6_nh;
4352 if (dev == nh->fib_nh_dev &&
4353 ((rt->fib6_flags & (RTF_ADDRCONF | RTF_DEFAULT)) == (RTF_ADDRCONF | RTF_DEFAULT)) &&
4354 ipv6_addr_equal(&nh->fib_nh_gw6, addr))
4355 break;
4356 }
4357 if (rt && !fib6_info_hold_safe(rt))
4358 rt = NULL;
4359 rcu_read_unlock();
4360 return rt;
4361 }
4362
rt6_add_dflt_router(struct net * net,const struct in6_addr * gwaddr,struct net_device * dev,unsigned int pref,u32 defrtr_usr_metric)4363 struct fib6_info *rt6_add_dflt_router(struct net *net,
4364 const struct in6_addr *gwaddr,
4365 struct net_device *dev,
4366 unsigned int pref,
4367 u32 defrtr_usr_metric)
4368 {
4369 struct fib6_config cfg = {
4370 .fc_table = l3mdev_fib_table(dev) ? : RT6_TABLE_DFLT,
4371 .fc_metric = defrtr_usr_metric,
4372 .fc_ifindex = dev->ifindex,
4373 .fc_flags = RTF_GATEWAY | RTF_ADDRCONF | RTF_DEFAULT |
4374 RTF_UP | RTF_EXPIRES | RTF_PREF(pref),
4375 .fc_protocol = RTPROT_RA,
4376 .fc_type = RTN_UNICAST,
4377 .fc_nlinfo.portid = 0,
4378 .fc_nlinfo.nlh = NULL,
4379 .fc_nlinfo.nl_net = net,
4380 };
4381
4382 cfg.fc_gateway = *gwaddr;
4383
4384 if (!ip6_route_add(&cfg, GFP_ATOMIC, NULL)) {
4385 struct fib6_table *table;
4386
4387 table = fib6_get_table(dev_net(dev), cfg.fc_table);
4388 if (table)
4389 table->flags |= RT6_TABLE_HAS_DFLT_ROUTER;
4390 }
4391
4392 return rt6_get_dflt_router(net, gwaddr, dev);
4393 }
4394
__rt6_purge_dflt_routers(struct net * net,struct fib6_table * table)4395 static void __rt6_purge_dflt_routers(struct net *net,
4396 struct fib6_table *table)
4397 {
4398 struct fib6_info *rt;
4399
4400 restart:
4401 rcu_read_lock();
4402 for_each_fib6_node_rt_rcu(&table->tb6_root) {
4403 struct net_device *dev = fib6_info_nh_dev(rt);
4404 struct inet6_dev *idev = dev ? __in6_dev_get(dev) : NULL;
4405
4406 if (rt->fib6_flags & (RTF_DEFAULT | RTF_ADDRCONF) &&
4407 (!idev || idev->cnf.accept_ra != 2) &&
4408 fib6_info_hold_safe(rt)) {
4409 rcu_read_unlock();
4410 ip6_del_rt(net, rt, false);
4411 goto restart;
4412 }
4413 }
4414 rcu_read_unlock();
4415
4416 table->flags &= ~RT6_TABLE_HAS_DFLT_ROUTER;
4417 }
4418
rt6_purge_dflt_routers(struct net * net)4419 void rt6_purge_dflt_routers(struct net *net)
4420 {
4421 struct fib6_table *table;
4422 struct hlist_head *head;
4423 unsigned int h;
4424
4425 rcu_read_lock();
4426
4427 for (h = 0; h < FIB6_TABLE_HASHSZ; h++) {
4428 head = &net->ipv6.fib_table_hash[h];
4429 hlist_for_each_entry_rcu(table, head, tb6_hlist) {
4430 if (table->flags & RT6_TABLE_HAS_DFLT_ROUTER)
4431 __rt6_purge_dflt_routers(net, table);
4432 }
4433 }
4434
4435 rcu_read_unlock();
4436 }
4437
rtmsg_to_fib6_config(struct net * net,struct in6_rtmsg * rtmsg,struct fib6_config * cfg)4438 static void rtmsg_to_fib6_config(struct net *net,
4439 struct in6_rtmsg *rtmsg,
4440 struct fib6_config *cfg)
4441 {
4442 *cfg = (struct fib6_config){
4443 .fc_table = l3mdev_fib_table_by_index(net, rtmsg->rtmsg_ifindex) ?
4444 : RT6_TABLE_MAIN,
4445 .fc_ifindex = rtmsg->rtmsg_ifindex,
4446 .fc_metric = rtmsg->rtmsg_metric,
4447 .fc_expires = rtmsg->rtmsg_info,
4448 .fc_dst_len = rtmsg->rtmsg_dst_len,
4449 .fc_src_len = rtmsg->rtmsg_src_len,
4450 .fc_flags = rtmsg->rtmsg_flags,
4451 .fc_type = rtmsg->rtmsg_type,
4452
4453 .fc_nlinfo.nl_net = net,
4454
4455 .fc_dst = rtmsg->rtmsg_dst,
4456 .fc_src = rtmsg->rtmsg_src,
4457 .fc_gateway = rtmsg->rtmsg_gateway,
4458 };
4459 }
4460
ipv6_route_ioctl(struct net * net,unsigned int cmd,struct in6_rtmsg * rtmsg)4461 int ipv6_route_ioctl(struct net *net, unsigned int cmd, struct in6_rtmsg *rtmsg)
4462 {
4463 struct fib6_config cfg;
4464 int err;
4465
4466 if (cmd != SIOCADDRT && cmd != SIOCDELRT)
4467 return -EINVAL;
4468 if (!ns_capable(net->user_ns, CAP_NET_ADMIN))
4469 return -EPERM;
4470
4471 rtmsg_to_fib6_config(net, rtmsg, &cfg);
4472
4473 rtnl_lock();
4474 switch (cmd) {
4475 case SIOCADDRT:
4476 /* Only do the default setting of fc_metric in route adding */
4477 if (cfg.fc_metric == 0)
4478 cfg.fc_metric = IP6_RT_PRIO_USER;
4479 err = ip6_route_add(&cfg, GFP_KERNEL, NULL);
4480 break;
4481 case SIOCDELRT:
4482 err = ip6_route_del(&cfg, NULL);
4483 break;
4484 }
4485 rtnl_unlock();
4486 return err;
4487 }
4488
4489 /*
4490 * Drop the packet on the floor
4491 */
4492
ip6_pkt_drop(struct sk_buff * skb,u8 code,int ipstats_mib_noroutes)4493 static int ip6_pkt_drop(struct sk_buff *skb, u8 code, int ipstats_mib_noroutes)
4494 {
4495 struct dst_entry *dst = skb_dst(skb);
4496 struct net *net = dev_net(dst->dev);
4497 struct inet6_dev *idev;
4498 SKB_DR(reason);
4499 int type;
4500
4501 if (netif_is_l3_master(skb->dev) ||
4502 dst->dev == net->loopback_dev)
4503 idev = __in6_dev_get_safely(dev_get_by_index_rcu(net, IP6CB(skb)->iif));
4504 else
4505 idev = ip6_dst_idev(dst);
4506
4507 switch (ipstats_mib_noroutes) {
4508 case IPSTATS_MIB_INNOROUTES:
4509 type = ipv6_addr_type(&ipv6_hdr(skb)->daddr);
4510 if (type == IPV6_ADDR_ANY) {
4511 SKB_DR_SET(reason, IP_INADDRERRORS);
4512 IP6_INC_STATS(net, idev, IPSTATS_MIB_INADDRERRORS);
4513 break;
4514 }
4515 SKB_DR_SET(reason, IP_INNOROUTES);
4516 fallthrough;
4517 case IPSTATS_MIB_OUTNOROUTES:
4518 SKB_DR_OR(reason, IP_OUTNOROUTES);
4519 IP6_INC_STATS(net, idev, ipstats_mib_noroutes);
4520 break;
4521 }
4522
4523 /* Start over by dropping the dst for l3mdev case */
4524 if (netif_is_l3_master(skb->dev))
4525 skb_dst_drop(skb);
4526
4527 icmpv6_send(skb, ICMPV6_DEST_UNREACH, code, 0);
4528 kfree_skb_reason(skb, reason);
4529 return 0;
4530 }
4531
ip6_pkt_discard(struct sk_buff * skb)4532 static int ip6_pkt_discard(struct sk_buff *skb)
4533 {
4534 return ip6_pkt_drop(skb, ICMPV6_NOROUTE, IPSTATS_MIB_INNOROUTES);
4535 }
4536
ip6_pkt_discard_out(struct net * net,struct sock * sk,struct sk_buff * skb)4537 static int ip6_pkt_discard_out(struct net *net, struct sock *sk, struct sk_buff *skb)
4538 {
4539 skb->dev = skb_dst(skb)->dev;
4540 return ip6_pkt_drop(skb, ICMPV6_NOROUTE, IPSTATS_MIB_OUTNOROUTES);
4541 }
4542
ip6_pkt_prohibit(struct sk_buff * skb)4543 static int ip6_pkt_prohibit(struct sk_buff *skb)
4544 {
4545 return ip6_pkt_drop(skb, ICMPV6_ADM_PROHIBITED, IPSTATS_MIB_INNOROUTES);
4546 }
4547
ip6_pkt_prohibit_out(struct net * net,struct sock * sk,struct sk_buff * skb)4548 static int ip6_pkt_prohibit_out(struct net *net, struct sock *sk, struct sk_buff *skb)
4549 {
4550 skb->dev = skb_dst(skb)->dev;
4551 return ip6_pkt_drop(skb, ICMPV6_ADM_PROHIBITED, IPSTATS_MIB_OUTNOROUTES);
4552 }
4553
4554 /*
4555 * Allocate a dst for local (unicast / anycast) address.
4556 */
4557
addrconf_f6i_alloc(struct net * net,struct inet6_dev * idev,const struct in6_addr * addr,bool anycast,gfp_t gfp_flags,struct netlink_ext_ack * extack)4558 struct fib6_info *addrconf_f6i_alloc(struct net *net,
4559 struct inet6_dev *idev,
4560 const struct in6_addr *addr,
4561 bool anycast, gfp_t gfp_flags,
4562 struct netlink_ext_ack *extack)
4563 {
4564 struct fib6_config cfg = {
4565 .fc_table = l3mdev_fib_table(idev->dev) ? : RT6_TABLE_LOCAL,
4566 .fc_ifindex = idev->dev->ifindex,
4567 .fc_flags = RTF_UP | RTF_NONEXTHOP,
4568 .fc_dst = *addr,
4569 .fc_dst_len = 128,
4570 .fc_protocol = RTPROT_KERNEL,
4571 .fc_nlinfo.nl_net = net,
4572 .fc_ignore_dev_down = true,
4573 };
4574 struct fib6_info *f6i;
4575
4576 if (anycast) {
4577 cfg.fc_type = RTN_ANYCAST;
4578 cfg.fc_flags |= RTF_ANYCAST;
4579 } else {
4580 cfg.fc_type = RTN_LOCAL;
4581 cfg.fc_flags |= RTF_LOCAL;
4582 }
4583
4584 f6i = ip6_route_info_create(&cfg, gfp_flags, extack);
4585 if (!IS_ERR(f6i)) {
4586 f6i->dst_nocount = true;
4587
4588 if (!anycast &&
4589 (net->ipv6.devconf_all->disable_policy ||
4590 idev->cnf.disable_policy))
4591 f6i->dst_nopolicy = true;
4592 }
4593
4594 return f6i;
4595 }
4596
4597 /* remove deleted ip from prefsrc entries */
4598 struct arg_dev_net_ip {
4599 struct net *net;
4600 struct in6_addr *addr;
4601 };
4602
fib6_remove_prefsrc(struct fib6_info * rt,void * arg)4603 static int fib6_remove_prefsrc(struct fib6_info *rt, void *arg)
4604 {
4605 struct net *net = ((struct arg_dev_net_ip *)arg)->net;
4606 struct in6_addr *addr = ((struct arg_dev_net_ip *)arg)->addr;
4607
4608 if (!rt->nh &&
4609 rt != net->ipv6.fib6_null_entry &&
4610 ipv6_addr_equal(addr, &rt->fib6_prefsrc.addr) &&
4611 !ipv6_chk_addr(net, addr, rt->fib6_nh->fib_nh_dev, 0)) {
4612 spin_lock_bh(&rt6_exception_lock);
4613 /* remove prefsrc entry */
4614 rt->fib6_prefsrc.plen = 0;
4615 spin_unlock_bh(&rt6_exception_lock);
4616 }
4617 return 0;
4618 }
4619
rt6_remove_prefsrc(struct inet6_ifaddr * ifp)4620 void rt6_remove_prefsrc(struct inet6_ifaddr *ifp)
4621 {
4622 struct net *net = dev_net(ifp->idev->dev);
4623 struct arg_dev_net_ip adni = {
4624 .net = net,
4625 .addr = &ifp->addr,
4626 };
4627 fib6_clean_all(net, fib6_remove_prefsrc, &adni);
4628 }
4629
4630 #define RTF_RA_ROUTER (RTF_ADDRCONF | RTF_DEFAULT)
4631
4632 /* Remove routers and update dst entries when gateway turn into host. */
fib6_clean_tohost(struct fib6_info * rt,void * arg)4633 static int fib6_clean_tohost(struct fib6_info *rt, void *arg)
4634 {
4635 struct in6_addr *gateway = (struct in6_addr *)arg;
4636 struct fib6_nh *nh;
4637
4638 /* RA routes do not use nexthops */
4639 if (rt->nh)
4640 return 0;
4641
4642 nh = rt->fib6_nh;
4643 if (((rt->fib6_flags & RTF_RA_ROUTER) == RTF_RA_ROUTER) &&
4644 nh->fib_nh_gw_family && ipv6_addr_equal(gateway, &nh->fib_nh_gw6))
4645 return -1;
4646
4647 /* Further clean up cached routes in exception table.
4648 * This is needed because cached route may have a different
4649 * gateway than its 'parent' in the case of an ip redirect.
4650 */
4651 fib6_nh_exceptions_clean_tohost(nh, gateway);
4652
4653 return 0;
4654 }
4655
rt6_clean_tohost(struct net * net,struct in6_addr * gateway)4656 void rt6_clean_tohost(struct net *net, struct in6_addr *gateway)
4657 {
4658 fib6_clean_all(net, fib6_clean_tohost, gateway);
4659 }
4660
4661 struct arg_netdev_event {
4662 const struct net_device *dev;
4663 union {
4664 unsigned char nh_flags;
4665 unsigned long event;
4666 };
4667 };
4668
rt6_multipath_first_sibling(const struct fib6_info * rt)4669 static struct fib6_info *rt6_multipath_first_sibling(const struct fib6_info *rt)
4670 {
4671 struct fib6_info *iter;
4672 struct fib6_node *fn;
4673
4674 fn = rcu_dereference_protected(rt->fib6_node,
4675 lockdep_is_held(&rt->fib6_table->tb6_lock));
4676 iter = rcu_dereference_protected(fn->leaf,
4677 lockdep_is_held(&rt->fib6_table->tb6_lock));
4678 while (iter) {
4679 if (iter->fib6_metric == rt->fib6_metric &&
4680 rt6_qualify_for_ecmp(iter))
4681 return iter;
4682 iter = rcu_dereference_protected(iter->fib6_next,
4683 lockdep_is_held(&rt->fib6_table->tb6_lock));
4684 }
4685
4686 return NULL;
4687 }
4688
4689 /* only called for fib entries with builtin fib6_nh */
rt6_is_dead(const struct fib6_info * rt)4690 static bool rt6_is_dead(const struct fib6_info *rt)
4691 {
4692 if (rt->fib6_nh->fib_nh_flags & RTNH_F_DEAD ||
4693 (rt->fib6_nh->fib_nh_flags & RTNH_F_LINKDOWN &&
4694 ip6_ignore_linkdown(rt->fib6_nh->fib_nh_dev)))
4695 return true;
4696
4697 return false;
4698 }
4699
rt6_multipath_total_weight(const struct fib6_info * rt)4700 static int rt6_multipath_total_weight(const struct fib6_info *rt)
4701 {
4702 struct fib6_info *iter;
4703 int total = 0;
4704
4705 if (!rt6_is_dead(rt))
4706 total += rt->fib6_nh->fib_nh_weight;
4707
4708 list_for_each_entry(iter, &rt->fib6_siblings, fib6_siblings) {
4709 if (!rt6_is_dead(iter))
4710 total += iter->fib6_nh->fib_nh_weight;
4711 }
4712
4713 return total;
4714 }
4715
rt6_upper_bound_set(struct fib6_info * rt,int * weight,int total)4716 static void rt6_upper_bound_set(struct fib6_info *rt, int *weight, int total)
4717 {
4718 int upper_bound = -1;
4719
4720 if (!rt6_is_dead(rt)) {
4721 *weight += rt->fib6_nh->fib_nh_weight;
4722 upper_bound = DIV_ROUND_CLOSEST_ULL((u64) (*weight) << 31,
4723 total) - 1;
4724 }
4725 atomic_set(&rt->fib6_nh->fib_nh_upper_bound, upper_bound);
4726 }
4727
rt6_multipath_upper_bound_set(struct fib6_info * rt,int total)4728 static void rt6_multipath_upper_bound_set(struct fib6_info *rt, int total)
4729 {
4730 struct fib6_info *iter;
4731 int weight = 0;
4732
4733 rt6_upper_bound_set(rt, &weight, total);
4734
4735 list_for_each_entry(iter, &rt->fib6_siblings, fib6_siblings)
4736 rt6_upper_bound_set(iter, &weight, total);
4737 }
4738
rt6_multipath_rebalance(struct fib6_info * rt)4739 void rt6_multipath_rebalance(struct fib6_info *rt)
4740 {
4741 struct fib6_info *first;
4742 int total;
4743
4744 /* In case the entire multipath route was marked for flushing,
4745 * then there is no need to rebalance upon the removal of every
4746 * sibling route.
4747 */
4748 if (!rt->fib6_nsiblings || rt->should_flush)
4749 return;
4750
4751 /* During lookup routes are evaluated in order, so we need to
4752 * make sure upper bounds are assigned from the first sibling
4753 * onwards.
4754 */
4755 first = rt6_multipath_first_sibling(rt);
4756 if (WARN_ON_ONCE(!first))
4757 return;
4758
4759 total = rt6_multipath_total_weight(first);
4760 rt6_multipath_upper_bound_set(first, total);
4761 }
4762
fib6_ifup(struct fib6_info * rt,void * p_arg)4763 static int fib6_ifup(struct fib6_info *rt, void *p_arg)
4764 {
4765 const struct arg_netdev_event *arg = p_arg;
4766 struct net *net = dev_net(arg->dev);
4767
4768 if (rt != net->ipv6.fib6_null_entry && !rt->nh &&
4769 rt->fib6_nh->fib_nh_dev == arg->dev) {
4770 rt->fib6_nh->fib_nh_flags &= ~arg->nh_flags;
4771 fib6_update_sernum_upto_root(net, rt);
4772 rt6_multipath_rebalance(rt);
4773 }
4774
4775 return 0;
4776 }
4777
rt6_sync_up(struct net_device * dev,unsigned char nh_flags)4778 void rt6_sync_up(struct net_device *dev, unsigned char nh_flags)
4779 {
4780 struct arg_netdev_event arg = {
4781 .dev = dev,
4782 {
4783 .nh_flags = nh_flags,
4784 },
4785 };
4786
4787 if (nh_flags & RTNH_F_DEAD && netif_carrier_ok(dev))
4788 arg.nh_flags |= RTNH_F_LINKDOWN;
4789
4790 fib6_clean_all(dev_net(dev), fib6_ifup, &arg);
4791 }
4792
4793 /* only called for fib entries with inline fib6_nh */
rt6_multipath_uses_dev(const struct fib6_info * rt,const struct net_device * dev)4794 static bool rt6_multipath_uses_dev(const struct fib6_info *rt,
4795 const struct net_device *dev)
4796 {
4797 struct fib6_info *iter;
4798
4799 if (rt->fib6_nh->fib_nh_dev == dev)
4800 return true;
4801 list_for_each_entry(iter, &rt->fib6_siblings, fib6_siblings)
4802 if (iter->fib6_nh->fib_nh_dev == dev)
4803 return true;
4804
4805 return false;
4806 }
4807
rt6_multipath_flush(struct fib6_info * rt)4808 static void rt6_multipath_flush(struct fib6_info *rt)
4809 {
4810 struct fib6_info *iter;
4811
4812 rt->should_flush = 1;
4813 list_for_each_entry(iter, &rt->fib6_siblings, fib6_siblings)
4814 iter->should_flush = 1;
4815 }
4816
rt6_multipath_dead_count(const struct fib6_info * rt,const struct net_device * down_dev)4817 static unsigned int rt6_multipath_dead_count(const struct fib6_info *rt,
4818 const struct net_device *down_dev)
4819 {
4820 struct fib6_info *iter;
4821 unsigned int dead = 0;
4822
4823 if (rt->fib6_nh->fib_nh_dev == down_dev ||
4824 rt->fib6_nh->fib_nh_flags & RTNH_F_DEAD)
4825 dead++;
4826 list_for_each_entry(iter, &rt->fib6_siblings, fib6_siblings)
4827 if (iter->fib6_nh->fib_nh_dev == down_dev ||
4828 iter->fib6_nh->fib_nh_flags & RTNH_F_DEAD)
4829 dead++;
4830
4831 return dead;
4832 }
4833
rt6_multipath_nh_flags_set(struct fib6_info * rt,const struct net_device * dev,unsigned char nh_flags)4834 static void rt6_multipath_nh_flags_set(struct fib6_info *rt,
4835 const struct net_device *dev,
4836 unsigned char nh_flags)
4837 {
4838 struct fib6_info *iter;
4839
4840 if (rt->fib6_nh->fib_nh_dev == dev)
4841 rt->fib6_nh->fib_nh_flags |= nh_flags;
4842 list_for_each_entry(iter, &rt->fib6_siblings, fib6_siblings)
4843 if (iter->fib6_nh->fib_nh_dev == dev)
4844 iter->fib6_nh->fib_nh_flags |= nh_flags;
4845 }
4846
4847 /* called with write lock held for table with rt */
fib6_ifdown(struct fib6_info * rt,void * p_arg)4848 static int fib6_ifdown(struct fib6_info *rt, void *p_arg)
4849 {
4850 const struct arg_netdev_event *arg = p_arg;
4851 const struct net_device *dev = arg->dev;
4852 struct net *net = dev_net(dev);
4853
4854 if (rt == net->ipv6.fib6_null_entry || rt->nh)
4855 return 0;
4856
4857 switch (arg->event) {
4858 case NETDEV_UNREGISTER:
4859 return rt->fib6_nh->fib_nh_dev == dev ? -1 : 0;
4860 case NETDEV_DOWN:
4861 if (rt->should_flush)
4862 return -1;
4863 if (!rt->fib6_nsiblings)
4864 return rt->fib6_nh->fib_nh_dev == dev ? -1 : 0;
4865 if (rt6_multipath_uses_dev(rt, dev)) {
4866 unsigned int count;
4867
4868 count = rt6_multipath_dead_count(rt, dev);
4869 if (rt->fib6_nsiblings + 1 == count) {
4870 rt6_multipath_flush(rt);
4871 return -1;
4872 }
4873 rt6_multipath_nh_flags_set(rt, dev, RTNH_F_DEAD |
4874 RTNH_F_LINKDOWN);
4875 fib6_update_sernum(net, rt);
4876 rt6_multipath_rebalance(rt);
4877 }
4878 return -2;
4879 case NETDEV_CHANGE:
4880 if (rt->fib6_nh->fib_nh_dev != dev ||
4881 rt->fib6_flags & (RTF_LOCAL | RTF_ANYCAST))
4882 break;
4883 rt->fib6_nh->fib_nh_flags |= RTNH_F_LINKDOWN;
4884 rt6_multipath_rebalance(rt);
4885 break;
4886 }
4887
4888 return 0;
4889 }
4890
rt6_sync_down_dev(struct net_device * dev,unsigned long event)4891 void rt6_sync_down_dev(struct net_device *dev, unsigned long event)
4892 {
4893 struct arg_netdev_event arg = {
4894 .dev = dev,
4895 {
4896 .event = event,
4897 },
4898 };
4899 struct net *net = dev_net(dev);
4900
4901 if (net->ipv6.sysctl.skip_notify_on_dev_down)
4902 fib6_clean_all_skip_notify(net, fib6_ifdown, &arg);
4903 else
4904 fib6_clean_all(net, fib6_ifdown, &arg);
4905 }
4906
rt6_disable_ip(struct net_device * dev,unsigned long event)4907 void rt6_disable_ip(struct net_device *dev, unsigned long event)
4908 {
4909 rt6_sync_down_dev(dev, event);
4910 rt6_uncached_list_flush_dev(dev);
4911 neigh_ifdown(&nd_tbl, dev);
4912 }
4913
4914 struct rt6_mtu_change_arg {
4915 struct net_device *dev;
4916 unsigned int mtu;
4917 struct fib6_info *f6i;
4918 };
4919
fib6_nh_mtu_change(struct fib6_nh * nh,void * _arg)4920 static int fib6_nh_mtu_change(struct fib6_nh *nh, void *_arg)
4921 {
4922 struct rt6_mtu_change_arg *arg = (struct rt6_mtu_change_arg *)_arg;
4923 struct fib6_info *f6i = arg->f6i;
4924
4925 /* For administrative MTU increase, there is no way to discover
4926 * IPv6 PMTU increase, so PMTU increase should be updated here.
4927 * Since RFC 1981 doesn't include administrative MTU increase
4928 * update PMTU increase is a MUST. (i.e. jumbo frame)
4929 */
4930 if (nh->fib_nh_dev == arg->dev) {
4931 struct inet6_dev *idev = __in6_dev_get(arg->dev);
4932 u32 mtu = f6i->fib6_pmtu;
4933
4934 if (mtu >= arg->mtu ||
4935 (mtu < arg->mtu && mtu == idev->cnf.mtu6))
4936 fib6_metric_set(f6i, RTAX_MTU, arg->mtu);
4937
4938 spin_lock_bh(&rt6_exception_lock);
4939 rt6_exceptions_update_pmtu(idev, nh, arg->mtu);
4940 spin_unlock_bh(&rt6_exception_lock);
4941 }
4942
4943 return 0;
4944 }
4945
rt6_mtu_change_route(struct fib6_info * f6i,void * p_arg)4946 static int rt6_mtu_change_route(struct fib6_info *f6i, void *p_arg)
4947 {
4948 struct rt6_mtu_change_arg *arg = (struct rt6_mtu_change_arg *) p_arg;
4949 struct inet6_dev *idev;
4950
4951 /* In IPv6 pmtu discovery is not optional,
4952 so that RTAX_MTU lock cannot disable it.
4953 We still use this lock to block changes
4954 caused by addrconf/ndisc.
4955 */
4956
4957 idev = __in6_dev_get(arg->dev);
4958 if (!idev)
4959 return 0;
4960
4961 if (fib6_metric_locked(f6i, RTAX_MTU))
4962 return 0;
4963
4964 arg->f6i = f6i;
4965 if (f6i->nh) {
4966 /* fib6_nh_mtu_change only returns 0, so this is safe */
4967 return nexthop_for_each_fib6_nh(f6i->nh, fib6_nh_mtu_change,
4968 arg);
4969 }
4970
4971 return fib6_nh_mtu_change(f6i->fib6_nh, arg);
4972 }
4973
rt6_mtu_change(struct net_device * dev,unsigned int mtu)4974 void rt6_mtu_change(struct net_device *dev, unsigned int mtu)
4975 {
4976 struct rt6_mtu_change_arg arg = {
4977 .dev = dev,
4978 .mtu = mtu,
4979 };
4980
4981 fib6_clean_all(dev_net(dev), rt6_mtu_change_route, &arg);
4982 }
4983
4984 static const struct nla_policy rtm_ipv6_policy[RTA_MAX+1] = {
4985 [RTA_UNSPEC] = { .strict_start_type = RTA_DPORT + 1 },
4986 [RTA_GATEWAY] = { .len = sizeof(struct in6_addr) },
4987 [RTA_PREFSRC] = { .len = sizeof(struct in6_addr) },
4988 [RTA_OIF] = { .type = NLA_U32 },
4989 [RTA_IIF] = { .type = NLA_U32 },
4990 [RTA_PRIORITY] = { .type = NLA_U32 },
4991 [RTA_METRICS] = { .type = NLA_NESTED },
4992 [RTA_MULTIPATH] = { .len = sizeof(struct rtnexthop) },
4993 [RTA_PREF] = { .type = NLA_U8 },
4994 [RTA_ENCAP_TYPE] = { .type = NLA_U16 },
4995 [RTA_ENCAP] = { .type = NLA_NESTED },
4996 [RTA_EXPIRES] = { .type = NLA_U32 },
4997 [RTA_UID] = { .type = NLA_U32 },
4998 [RTA_MARK] = { .type = NLA_U32 },
4999 [RTA_TABLE] = { .type = NLA_U32 },
5000 [RTA_IP_PROTO] = { .type = NLA_U8 },
5001 [RTA_SPORT] = { .type = NLA_U16 },
5002 [RTA_DPORT] = { .type = NLA_U16 },
5003 [RTA_NH_ID] = { .type = NLA_U32 },
5004 };
5005
rtm_to_fib6_config(struct sk_buff * skb,struct nlmsghdr * nlh,struct fib6_config * cfg,struct netlink_ext_ack * extack)5006 static int rtm_to_fib6_config(struct sk_buff *skb, struct nlmsghdr *nlh,
5007 struct fib6_config *cfg,
5008 struct netlink_ext_ack *extack)
5009 {
5010 struct rtmsg *rtm;
5011 struct nlattr *tb[RTA_MAX+1];
5012 unsigned int pref;
5013 int err;
5014
5015 err = nlmsg_parse_deprecated(nlh, sizeof(*rtm), tb, RTA_MAX,
5016 rtm_ipv6_policy, extack);
5017 if (err < 0)
5018 goto errout;
5019
5020 err = -EINVAL;
5021 rtm = nlmsg_data(nlh);
5022
5023 if (rtm->rtm_tos) {
5024 NL_SET_ERR_MSG(extack,
5025 "Invalid dsfield (tos): option not available for IPv6");
5026 goto errout;
5027 }
5028
5029 *cfg = (struct fib6_config){
5030 .fc_table = rtm->rtm_table,
5031 .fc_dst_len = rtm->rtm_dst_len,
5032 .fc_src_len = rtm->rtm_src_len,
5033 .fc_flags = RTF_UP,
5034 .fc_protocol = rtm->rtm_protocol,
5035 .fc_type = rtm->rtm_type,
5036
5037 .fc_nlinfo.portid = NETLINK_CB(skb).portid,
5038 .fc_nlinfo.nlh = nlh,
5039 .fc_nlinfo.nl_net = sock_net(skb->sk),
5040 };
5041
5042 if (rtm->rtm_type == RTN_UNREACHABLE ||
5043 rtm->rtm_type == RTN_BLACKHOLE ||
5044 rtm->rtm_type == RTN_PROHIBIT ||
5045 rtm->rtm_type == RTN_THROW)
5046 cfg->fc_flags |= RTF_REJECT;
5047
5048 if (rtm->rtm_type == RTN_LOCAL)
5049 cfg->fc_flags |= RTF_LOCAL;
5050
5051 if (rtm->rtm_flags & RTM_F_CLONED)
5052 cfg->fc_flags |= RTF_CACHE;
5053
5054 cfg->fc_flags |= (rtm->rtm_flags & RTNH_F_ONLINK);
5055
5056 if (tb[RTA_NH_ID]) {
5057 if (tb[RTA_GATEWAY] || tb[RTA_OIF] ||
5058 tb[RTA_MULTIPATH] || tb[RTA_ENCAP]) {
5059 NL_SET_ERR_MSG(extack,
5060 "Nexthop specification and nexthop id are mutually exclusive");
5061 goto errout;
5062 }
5063 cfg->fc_nh_id = nla_get_u32(tb[RTA_NH_ID]);
5064 }
5065
5066 if (tb[RTA_GATEWAY]) {
5067 cfg->fc_gateway = nla_get_in6_addr(tb[RTA_GATEWAY]);
5068 cfg->fc_flags |= RTF_GATEWAY;
5069 }
5070 if (tb[RTA_VIA]) {
5071 NL_SET_ERR_MSG(extack, "IPv6 does not support RTA_VIA attribute");
5072 goto errout;
5073 }
5074
5075 if (tb[RTA_DST]) {
5076 int plen = (rtm->rtm_dst_len + 7) >> 3;
5077
5078 if (nla_len(tb[RTA_DST]) < plen)
5079 goto errout;
5080
5081 nla_memcpy(&cfg->fc_dst, tb[RTA_DST], plen);
5082 }
5083
5084 if (tb[RTA_SRC]) {
5085 int plen = (rtm->rtm_src_len + 7) >> 3;
5086
5087 if (nla_len(tb[RTA_SRC]) < plen)
5088 goto errout;
5089
5090 nla_memcpy(&cfg->fc_src, tb[RTA_SRC], plen);
5091 }
5092
5093 if (tb[RTA_PREFSRC])
5094 cfg->fc_prefsrc = nla_get_in6_addr(tb[RTA_PREFSRC]);
5095
5096 if (tb[RTA_OIF])
5097 cfg->fc_ifindex = nla_get_u32(tb[RTA_OIF]);
5098
5099 if (tb[RTA_PRIORITY])
5100 cfg->fc_metric = nla_get_u32(tb[RTA_PRIORITY]);
5101
5102 if (tb[RTA_METRICS]) {
5103 cfg->fc_mx = nla_data(tb[RTA_METRICS]);
5104 cfg->fc_mx_len = nla_len(tb[RTA_METRICS]);
5105 }
5106
5107 if (tb[RTA_TABLE])
5108 cfg->fc_table = nla_get_u32(tb[RTA_TABLE]);
5109
5110 if (tb[RTA_MULTIPATH]) {
5111 cfg->fc_mp = nla_data(tb[RTA_MULTIPATH]);
5112 cfg->fc_mp_len = nla_len(tb[RTA_MULTIPATH]);
5113
5114 err = lwtunnel_valid_encap_type_attr(cfg->fc_mp,
5115 cfg->fc_mp_len, extack);
5116 if (err < 0)
5117 goto errout;
5118 }
5119
5120 if (tb[RTA_PREF]) {
5121 pref = nla_get_u8(tb[RTA_PREF]);
5122 if (pref != ICMPV6_ROUTER_PREF_LOW &&
5123 pref != ICMPV6_ROUTER_PREF_HIGH)
5124 pref = ICMPV6_ROUTER_PREF_MEDIUM;
5125 cfg->fc_flags |= RTF_PREF(pref);
5126 }
5127
5128 if (tb[RTA_ENCAP])
5129 cfg->fc_encap = tb[RTA_ENCAP];
5130
5131 if (tb[RTA_ENCAP_TYPE]) {
5132 cfg->fc_encap_type = nla_get_u16(tb[RTA_ENCAP_TYPE]);
5133
5134 err = lwtunnel_valid_encap_type(cfg->fc_encap_type, extack);
5135 if (err < 0)
5136 goto errout;
5137 }
5138
5139 if (tb[RTA_EXPIRES]) {
5140 unsigned long timeout = addrconf_timeout_fixup(nla_get_u32(tb[RTA_EXPIRES]), HZ);
5141
5142 if (addrconf_finite_timeout(timeout)) {
5143 cfg->fc_expires = jiffies_to_clock_t(timeout * HZ);
5144 cfg->fc_flags |= RTF_EXPIRES;
5145 }
5146 }
5147
5148 err = 0;
5149 errout:
5150 return err;
5151 }
5152
5153 struct rt6_nh {
5154 struct fib6_info *fib6_info;
5155 struct fib6_config r_cfg;
5156 struct list_head next;
5157 };
5158
ip6_route_info_append(struct net * net,struct list_head * rt6_nh_list,struct fib6_info * rt,struct fib6_config * r_cfg)5159 static int ip6_route_info_append(struct net *net,
5160 struct list_head *rt6_nh_list,
5161 struct fib6_info *rt,
5162 struct fib6_config *r_cfg)
5163 {
5164 struct rt6_nh *nh;
5165 int err = -EEXIST;
5166
5167 list_for_each_entry(nh, rt6_nh_list, next) {
5168 /* check if fib6_info already exists */
5169 if (rt6_duplicate_nexthop(nh->fib6_info, rt))
5170 return err;
5171 }
5172
5173 nh = kzalloc(sizeof(*nh), GFP_KERNEL);
5174 if (!nh)
5175 return -ENOMEM;
5176 nh->fib6_info = rt;
5177 memcpy(&nh->r_cfg, r_cfg, sizeof(*r_cfg));
5178 list_add_tail(&nh->next, rt6_nh_list);
5179
5180 return 0;
5181 }
5182
ip6_route_mpath_notify(struct fib6_info * rt,struct fib6_info * rt_last,struct nl_info * info,__u16 nlflags)5183 static void ip6_route_mpath_notify(struct fib6_info *rt,
5184 struct fib6_info *rt_last,
5185 struct nl_info *info,
5186 __u16 nlflags)
5187 {
5188 /* if this is an APPEND route, then rt points to the first route
5189 * inserted and rt_last points to last route inserted. Userspace
5190 * wants a consistent dump of the route which starts at the first
5191 * nexthop. Since sibling routes are always added at the end of
5192 * the list, find the first sibling of the last route appended
5193 */
5194 rcu_read_lock();
5195
5196 if ((nlflags & NLM_F_APPEND) && rt_last && rt_last->fib6_nsiblings) {
5197 rt = list_first_or_null_rcu(&rt_last->fib6_siblings,
5198 struct fib6_info,
5199 fib6_siblings);
5200 }
5201
5202 if (rt)
5203 inet6_rt_notify(RTM_NEWROUTE, rt, info, nlflags);
5204
5205 rcu_read_unlock();
5206 }
5207
ip6_route_mpath_should_notify(const struct fib6_info * rt)5208 static bool ip6_route_mpath_should_notify(const struct fib6_info *rt)
5209 {
5210 bool rt_can_ecmp = rt6_qualify_for_ecmp(rt);
5211 bool should_notify = false;
5212 struct fib6_info *leaf;
5213 struct fib6_node *fn;
5214
5215 rcu_read_lock();
5216 fn = rcu_dereference(rt->fib6_node);
5217 if (!fn)
5218 goto out;
5219
5220 leaf = rcu_dereference(fn->leaf);
5221 if (!leaf)
5222 goto out;
5223
5224 if (rt == leaf ||
5225 (rt_can_ecmp && rt->fib6_metric == leaf->fib6_metric &&
5226 rt6_qualify_for_ecmp(leaf)))
5227 should_notify = true;
5228 out:
5229 rcu_read_unlock();
5230
5231 return should_notify;
5232 }
5233
fib6_gw_from_attr(struct in6_addr * gw,struct nlattr * nla,struct netlink_ext_ack * extack)5234 static int fib6_gw_from_attr(struct in6_addr *gw, struct nlattr *nla,
5235 struct netlink_ext_ack *extack)
5236 {
5237 if (nla_len(nla) < sizeof(*gw)) {
5238 NL_SET_ERR_MSG(extack, "Invalid IPv6 address in RTA_GATEWAY");
5239 return -EINVAL;
5240 }
5241
5242 *gw = nla_get_in6_addr(nla);
5243
5244 return 0;
5245 }
5246
ip6_route_multipath_add(struct fib6_config * cfg,struct netlink_ext_ack * extack)5247 static int ip6_route_multipath_add(struct fib6_config *cfg,
5248 struct netlink_ext_ack *extack)
5249 {
5250 struct fib6_info *rt_notif = NULL, *rt_last = NULL;
5251 struct nl_info *info = &cfg->fc_nlinfo;
5252 struct fib6_config r_cfg;
5253 struct rtnexthop *rtnh;
5254 struct fib6_info *rt;
5255 struct rt6_nh *err_nh;
5256 struct rt6_nh *nh, *nh_safe;
5257 __u16 nlflags;
5258 int remaining;
5259 int attrlen;
5260 int err = 1;
5261 int nhn = 0;
5262 int replace = (cfg->fc_nlinfo.nlh &&
5263 (cfg->fc_nlinfo.nlh->nlmsg_flags & NLM_F_REPLACE));
5264 LIST_HEAD(rt6_nh_list);
5265
5266 nlflags = replace ? NLM_F_REPLACE : NLM_F_CREATE;
5267 if (info->nlh && info->nlh->nlmsg_flags & NLM_F_APPEND)
5268 nlflags |= NLM_F_APPEND;
5269
5270 remaining = cfg->fc_mp_len;
5271 rtnh = (struct rtnexthop *)cfg->fc_mp;
5272
5273 /* Parse a Multipath Entry and build a list (rt6_nh_list) of
5274 * fib6_info structs per nexthop
5275 */
5276 while (rtnh_ok(rtnh, remaining)) {
5277 memcpy(&r_cfg, cfg, sizeof(*cfg));
5278 if (rtnh->rtnh_ifindex)
5279 r_cfg.fc_ifindex = rtnh->rtnh_ifindex;
5280
5281 attrlen = rtnh_attrlen(rtnh);
5282 if (attrlen > 0) {
5283 struct nlattr *nla, *attrs = rtnh_attrs(rtnh);
5284
5285 nla = nla_find(attrs, attrlen, RTA_GATEWAY);
5286 if (nla) {
5287 err = fib6_gw_from_attr(&r_cfg.fc_gateway, nla,
5288 extack);
5289 if (err)
5290 goto cleanup;
5291
5292 r_cfg.fc_flags |= RTF_GATEWAY;
5293 }
5294 r_cfg.fc_encap = nla_find(attrs, attrlen, RTA_ENCAP);
5295
5296 /* RTA_ENCAP_TYPE length checked in
5297 * lwtunnel_valid_encap_type_attr
5298 */
5299 nla = nla_find(attrs, attrlen, RTA_ENCAP_TYPE);
5300 if (nla)
5301 r_cfg.fc_encap_type = nla_get_u16(nla);
5302 }
5303
5304 r_cfg.fc_flags |= (rtnh->rtnh_flags & RTNH_F_ONLINK);
5305 rt = ip6_route_info_create(&r_cfg, GFP_KERNEL, extack);
5306 if (IS_ERR(rt)) {
5307 err = PTR_ERR(rt);
5308 rt = NULL;
5309 goto cleanup;
5310 }
5311 if (!rt6_qualify_for_ecmp(rt)) {
5312 err = -EINVAL;
5313 NL_SET_ERR_MSG(extack,
5314 "Device only routes can not be added for IPv6 using the multipath API.");
5315 fib6_info_release(rt);
5316 goto cleanup;
5317 }
5318
5319 rt->fib6_nh->fib_nh_weight = rtnh->rtnh_hops + 1;
5320
5321 err = ip6_route_info_append(info->nl_net, &rt6_nh_list,
5322 rt, &r_cfg);
5323 if (err) {
5324 fib6_info_release(rt);
5325 goto cleanup;
5326 }
5327
5328 rtnh = rtnh_next(rtnh, &remaining);
5329 }
5330
5331 if (list_empty(&rt6_nh_list)) {
5332 NL_SET_ERR_MSG(extack,
5333 "Invalid nexthop configuration - no valid nexthops");
5334 return -EINVAL;
5335 }
5336
5337 /* for add and replace send one notification with all nexthops.
5338 * Skip the notification in fib6_add_rt2node and send one with
5339 * the full route when done
5340 */
5341 info->skip_notify = 1;
5342
5343 /* For add and replace, send one notification with all nexthops. For
5344 * append, send one notification with all appended nexthops.
5345 */
5346 info->skip_notify_kernel = 1;
5347
5348 err_nh = NULL;
5349 list_for_each_entry(nh, &rt6_nh_list, next) {
5350 err = __ip6_ins_rt(nh->fib6_info, info, extack);
5351
5352 if (err) {
5353 if (replace && nhn)
5354 NL_SET_ERR_MSG_MOD(extack,
5355 "multipath route replace failed (check consistency of installed routes)");
5356 err_nh = nh;
5357 goto add_errout;
5358 }
5359 /* save reference to last route successfully inserted */
5360 rt_last = nh->fib6_info;
5361
5362 /* save reference to first route for notification */
5363 if (!rt_notif)
5364 rt_notif = nh->fib6_info;
5365
5366 /* Because each route is added like a single route we remove
5367 * these flags after the first nexthop: if there is a collision,
5368 * we have already failed to add the first nexthop:
5369 * fib6_add_rt2node() has rejected it; when replacing, old
5370 * nexthops have been replaced by first new, the rest should
5371 * be added to it.
5372 */
5373 if (cfg->fc_nlinfo.nlh) {
5374 cfg->fc_nlinfo.nlh->nlmsg_flags &= ~(NLM_F_EXCL |
5375 NLM_F_REPLACE);
5376 cfg->fc_nlinfo.nlh->nlmsg_flags |= NLM_F_CREATE;
5377 }
5378 nhn++;
5379 }
5380
5381 /* An in-kernel notification should only be sent in case the new
5382 * multipath route is added as the first route in the node, or if
5383 * it was appended to it. We pass 'rt_notif' since it is the first
5384 * sibling and might allow us to skip some checks in the replace case.
5385 */
5386 if (ip6_route_mpath_should_notify(rt_notif)) {
5387 enum fib_event_type fib_event;
5388
5389 if (rt_notif->fib6_nsiblings != nhn - 1)
5390 fib_event = FIB_EVENT_ENTRY_APPEND;
5391 else
5392 fib_event = FIB_EVENT_ENTRY_REPLACE;
5393
5394 err = call_fib6_multipath_entry_notifiers(info->nl_net,
5395 fib_event, rt_notif,
5396 nhn - 1, extack);
5397 if (err) {
5398 /* Delete all the siblings that were just added */
5399 err_nh = NULL;
5400 goto add_errout;
5401 }
5402 }
5403
5404 /* success ... tell user about new route */
5405 ip6_route_mpath_notify(rt_notif, rt_last, info, nlflags);
5406 goto cleanup;
5407
5408 add_errout:
5409 /* send notification for routes that were added so that
5410 * the delete notifications sent by ip6_route_del are
5411 * coherent
5412 */
5413 if (rt_notif)
5414 ip6_route_mpath_notify(rt_notif, rt_last, info, nlflags);
5415
5416 /* Delete routes that were already added */
5417 list_for_each_entry(nh, &rt6_nh_list, next) {
5418 if (err_nh == nh)
5419 break;
5420 ip6_route_del(&nh->r_cfg, extack);
5421 }
5422
5423 cleanup:
5424 list_for_each_entry_safe(nh, nh_safe, &rt6_nh_list, next) {
5425 fib6_info_release(nh->fib6_info);
5426 list_del(&nh->next);
5427 kfree(nh);
5428 }
5429
5430 return err;
5431 }
5432
ip6_route_multipath_del(struct fib6_config * cfg,struct netlink_ext_ack * extack)5433 static int ip6_route_multipath_del(struct fib6_config *cfg,
5434 struct netlink_ext_ack *extack)
5435 {
5436 struct fib6_config r_cfg;
5437 struct rtnexthop *rtnh;
5438 int last_err = 0;
5439 int remaining;
5440 int attrlen;
5441 int err;
5442
5443 remaining = cfg->fc_mp_len;
5444 rtnh = (struct rtnexthop *)cfg->fc_mp;
5445
5446 /* Parse a Multipath Entry */
5447 while (rtnh_ok(rtnh, remaining)) {
5448 memcpy(&r_cfg, cfg, sizeof(*cfg));
5449 if (rtnh->rtnh_ifindex)
5450 r_cfg.fc_ifindex = rtnh->rtnh_ifindex;
5451
5452 attrlen = rtnh_attrlen(rtnh);
5453 if (attrlen > 0) {
5454 struct nlattr *nla, *attrs = rtnh_attrs(rtnh);
5455
5456 nla = nla_find(attrs, attrlen, RTA_GATEWAY);
5457 if (nla) {
5458 err = fib6_gw_from_attr(&r_cfg.fc_gateway, nla,
5459 extack);
5460 if (err) {
5461 last_err = err;
5462 goto next_rtnh;
5463 }
5464
5465 r_cfg.fc_flags |= RTF_GATEWAY;
5466 }
5467 }
5468 err = ip6_route_del(&r_cfg, extack);
5469 if (err)
5470 last_err = err;
5471
5472 next_rtnh:
5473 rtnh = rtnh_next(rtnh, &remaining);
5474 }
5475
5476 return last_err;
5477 }
5478
inet6_rtm_delroute(struct sk_buff * skb,struct nlmsghdr * nlh,struct netlink_ext_ack * extack)5479 static int inet6_rtm_delroute(struct sk_buff *skb, struct nlmsghdr *nlh,
5480 struct netlink_ext_ack *extack)
5481 {
5482 struct fib6_config cfg;
5483 int err;
5484
5485 err = rtm_to_fib6_config(skb, nlh, &cfg, extack);
5486 if (err < 0)
5487 return err;
5488
5489 if (cfg.fc_nh_id &&
5490 !nexthop_find_by_id(sock_net(skb->sk), cfg.fc_nh_id)) {
5491 NL_SET_ERR_MSG(extack, "Nexthop id does not exist");
5492 return -EINVAL;
5493 }
5494
5495 if (cfg.fc_mp)
5496 return ip6_route_multipath_del(&cfg, extack);
5497 else {
5498 cfg.fc_delete_all_nh = 1;
5499 return ip6_route_del(&cfg, extack);
5500 }
5501 }
5502
inet6_rtm_newroute(struct sk_buff * skb,struct nlmsghdr * nlh,struct netlink_ext_ack * extack)5503 static int inet6_rtm_newroute(struct sk_buff *skb, struct nlmsghdr *nlh,
5504 struct netlink_ext_ack *extack)
5505 {
5506 struct fib6_config cfg;
5507 int err;
5508
5509 err = rtm_to_fib6_config(skb, nlh, &cfg, extack);
5510 if (err < 0)
5511 return err;
5512
5513 if (cfg.fc_metric == 0)
5514 cfg.fc_metric = IP6_RT_PRIO_USER;
5515
5516 if (cfg.fc_mp)
5517 return ip6_route_multipath_add(&cfg, extack);
5518 else
5519 return ip6_route_add(&cfg, GFP_KERNEL, extack);
5520 }
5521
5522 /* add the overhead of this fib6_nh to nexthop_len */
rt6_nh_nlmsg_size(struct fib6_nh * nh,void * arg)5523 static int rt6_nh_nlmsg_size(struct fib6_nh *nh, void *arg)
5524 {
5525 int *nexthop_len = arg;
5526
5527 *nexthop_len += nla_total_size(0) /* RTA_MULTIPATH */
5528 + NLA_ALIGN(sizeof(struct rtnexthop))
5529 + nla_total_size(16); /* RTA_GATEWAY */
5530
5531 if (nh->fib_nh_lws) {
5532 /* RTA_ENCAP_TYPE */
5533 *nexthop_len += lwtunnel_get_encap_size(nh->fib_nh_lws);
5534 /* RTA_ENCAP */
5535 *nexthop_len += nla_total_size(2);
5536 }
5537
5538 return 0;
5539 }
5540
rt6_nlmsg_size(struct fib6_info * f6i)5541 static size_t rt6_nlmsg_size(struct fib6_info *f6i)
5542 {
5543 int nexthop_len;
5544
5545 if (f6i->nh) {
5546 nexthop_len = nla_total_size(4); /* RTA_NH_ID */
5547 nexthop_for_each_fib6_nh(f6i->nh, rt6_nh_nlmsg_size,
5548 &nexthop_len);
5549 } else {
5550 struct fib6_nh *nh = f6i->fib6_nh;
5551 struct fib6_info *sibling;
5552
5553 nexthop_len = 0;
5554 if (f6i->fib6_nsiblings) {
5555 rt6_nh_nlmsg_size(nh, &nexthop_len);
5556
5557 rcu_read_lock();
5558
5559 list_for_each_entry_rcu(sibling, &f6i->fib6_siblings,
5560 fib6_siblings) {
5561 rt6_nh_nlmsg_size(sibling->fib6_nh, &nexthop_len);
5562 }
5563
5564 rcu_read_unlock();
5565 }
5566 nexthop_len += lwtunnel_get_encap_size(nh->fib_nh_lws);
5567 }
5568
5569 return NLMSG_ALIGN(sizeof(struct rtmsg))
5570 + nla_total_size(16) /* RTA_SRC */
5571 + nla_total_size(16) /* RTA_DST */
5572 + nla_total_size(16) /* RTA_GATEWAY */
5573 + nla_total_size(16) /* RTA_PREFSRC */
5574 + nla_total_size(4) /* RTA_TABLE */
5575 + nla_total_size(4) /* RTA_IIF */
5576 + nla_total_size(4) /* RTA_OIF */
5577 + nla_total_size(4) /* RTA_PRIORITY */
5578 + RTAX_MAX * nla_total_size(4) /* RTA_METRICS */
5579 + nla_total_size(sizeof(struct rta_cacheinfo))
5580 + nla_total_size(TCP_CA_NAME_MAX) /* RTAX_CC_ALGO */
5581 + nla_total_size(1) /* RTA_PREF */
5582 + nexthop_len;
5583 }
5584
rt6_fill_node_nexthop(struct sk_buff * skb,struct nexthop * nh,unsigned char * flags)5585 static int rt6_fill_node_nexthop(struct sk_buff *skb, struct nexthop *nh,
5586 unsigned char *flags)
5587 {
5588 if (nexthop_is_multipath(nh)) {
5589 struct nlattr *mp;
5590
5591 mp = nla_nest_start_noflag(skb, RTA_MULTIPATH);
5592 if (!mp)
5593 goto nla_put_failure;
5594
5595 if (nexthop_mpath_fill_node(skb, nh, AF_INET6))
5596 goto nla_put_failure;
5597
5598 nla_nest_end(skb, mp);
5599 } else {
5600 struct fib6_nh *fib6_nh;
5601
5602 fib6_nh = nexthop_fib6_nh(nh);
5603 if (fib_nexthop_info(skb, &fib6_nh->nh_common, AF_INET6,
5604 flags, false) < 0)
5605 goto nla_put_failure;
5606 }
5607
5608 return 0;
5609
5610 nla_put_failure:
5611 return -EMSGSIZE;
5612 }
5613
rt6_fill_node(struct net * net,struct sk_buff * skb,struct fib6_info * rt,struct dst_entry * dst,struct in6_addr * dest,struct in6_addr * src,int iif,int type,u32 portid,u32 seq,unsigned int flags)5614 static int rt6_fill_node(struct net *net, struct sk_buff *skb,
5615 struct fib6_info *rt, struct dst_entry *dst,
5616 struct in6_addr *dest, struct in6_addr *src,
5617 int iif, int type, u32 portid, u32 seq,
5618 unsigned int flags)
5619 {
5620 struct rt6_info *rt6 = dst_rt6_info(dst);
5621 struct rt6key *rt6_dst, *rt6_src;
5622 u32 *pmetrics, table, rt6_flags;
5623 unsigned char nh_flags = 0;
5624 struct nlmsghdr *nlh;
5625 struct rtmsg *rtm;
5626 long expires = 0;
5627
5628 nlh = nlmsg_put(skb, portid, seq, type, sizeof(*rtm), flags);
5629 if (!nlh)
5630 return -EMSGSIZE;
5631
5632 if (rt6) {
5633 rt6_dst = &rt6->rt6i_dst;
5634 rt6_src = &rt6->rt6i_src;
5635 rt6_flags = rt6->rt6i_flags;
5636 } else {
5637 rt6_dst = &rt->fib6_dst;
5638 rt6_src = &rt->fib6_src;
5639 rt6_flags = rt->fib6_flags;
5640 }
5641
5642 rtm = nlmsg_data(nlh);
5643 rtm->rtm_family = AF_INET6;
5644 rtm->rtm_dst_len = rt6_dst->plen;
5645 rtm->rtm_src_len = rt6_src->plen;
5646 rtm->rtm_tos = 0;
5647 if (rt->fib6_table)
5648 table = rt->fib6_table->tb6_id;
5649 else
5650 table = RT6_TABLE_UNSPEC;
5651 rtm->rtm_table = table < 256 ? table : RT_TABLE_COMPAT;
5652 if (nla_put_u32(skb, RTA_TABLE, table))
5653 goto nla_put_failure;
5654
5655 rtm->rtm_type = rt->fib6_type;
5656 rtm->rtm_flags = 0;
5657 rtm->rtm_scope = RT_SCOPE_UNIVERSE;
5658 rtm->rtm_protocol = rt->fib6_protocol;
5659
5660 if (rt6_flags & RTF_CACHE)
5661 rtm->rtm_flags |= RTM_F_CLONED;
5662
5663 if (dest) {
5664 if (nla_put_in6_addr(skb, RTA_DST, dest))
5665 goto nla_put_failure;
5666 rtm->rtm_dst_len = 128;
5667 } else if (rtm->rtm_dst_len)
5668 if (nla_put_in6_addr(skb, RTA_DST, &rt6_dst->addr))
5669 goto nla_put_failure;
5670 #ifdef CONFIG_IPV6_SUBTREES
5671 if (src) {
5672 if (nla_put_in6_addr(skb, RTA_SRC, src))
5673 goto nla_put_failure;
5674 rtm->rtm_src_len = 128;
5675 } else if (rtm->rtm_src_len &&
5676 nla_put_in6_addr(skb, RTA_SRC, &rt6_src->addr))
5677 goto nla_put_failure;
5678 #endif
5679 if (iif) {
5680 #ifdef CONFIG_IPV6_MROUTE
5681 if (ipv6_addr_is_multicast(&rt6_dst->addr)) {
5682 int err = ip6mr_get_route(net, skb, rtm, portid);
5683
5684 if (err == 0)
5685 return 0;
5686 if (err < 0)
5687 goto nla_put_failure;
5688 } else
5689 #endif
5690 if (nla_put_u32(skb, RTA_IIF, iif))
5691 goto nla_put_failure;
5692 } else if (dest) {
5693 struct in6_addr saddr_buf;
5694 if (ip6_route_get_saddr(net, rt, dest, 0, 0, &saddr_buf) == 0 &&
5695 nla_put_in6_addr(skb, RTA_PREFSRC, &saddr_buf))
5696 goto nla_put_failure;
5697 }
5698
5699 if (rt->fib6_prefsrc.plen) {
5700 struct in6_addr saddr_buf;
5701 saddr_buf = rt->fib6_prefsrc.addr;
5702 if (nla_put_in6_addr(skb, RTA_PREFSRC, &saddr_buf))
5703 goto nla_put_failure;
5704 }
5705
5706 pmetrics = dst ? dst_metrics_ptr(dst) : rt->fib6_metrics->metrics;
5707 if (rtnetlink_put_metrics(skb, pmetrics) < 0)
5708 goto nla_put_failure;
5709
5710 if (nla_put_u32(skb, RTA_PRIORITY, rt->fib6_metric))
5711 goto nla_put_failure;
5712
5713 /* For multipath routes, walk the siblings list and add
5714 * each as a nexthop within RTA_MULTIPATH.
5715 */
5716 if (rt6) {
5717 if (rt6_flags & RTF_GATEWAY &&
5718 nla_put_in6_addr(skb, RTA_GATEWAY, &rt6->rt6i_gateway))
5719 goto nla_put_failure;
5720
5721 if (dst->dev && nla_put_u32(skb, RTA_OIF, dst->dev->ifindex))
5722 goto nla_put_failure;
5723
5724 if (dst->lwtstate &&
5725 lwtunnel_fill_encap(skb, dst->lwtstate, RTA_ENCAP, RTA_ENCAP_TYPE) < 0)
5726 goto nla_put_failure;
5727 } else if (rt->fib6_nsiblings) {
5728 struct fib6_info *sibling;
5729 struct nlattr *mp;
5730
5731 mp = nla_nest_start_noflag(skb, RTA_MULTIPATH);
5732 if (!mp)
5733 goto nla_put_failure;
5734
5735 if (fib_add_nexthop(skb, &rt->fib6_nh->nh_common,
5736 rt->fib6_nh->fib_nh_weight, AF_INET6,
5737 0) < 0)
5738 goto nla_put_failure;
5739
5740 rcu_read_lock();
5741
5742 list_for_each_entry_rcu(sibling, &rt->fib6_siblings,
5743 fib6_siblings) {
5744 if (fib_add_nexthop(skb, &sibling->fib6_nh->nh_common,
5745 sibling->fib6_nh->fib_nh_weight,
5746 AF_INET6, 0) < 0) {
5747 rcu_read_unlock();
5748
5749 goto nla_put_failure;
5750 }
5751 }
5752
5753 rcu_read_unlock();
5754
5755 nla_nest_end(skb, mp);
5756 } else if (rt->nh) {
5757 if (nla_put_u32(skb, RTA_NH_ID, rt->nh->id))
5758 goto nla_put_failure;
5759
5760 if (nexthop_is_blackhole(rt->nh))
5761 rtm->rtm_type = RTN_BLACKHOLE;
5762
5763 if (READ_ONCE(net->ipv4.sysctl_nexthop_compat_mode) &&
5764 rt6_fill_node_nexthop(skb, rt->nh, &nh_flags) < 0)
5765 goto nla_put_failure;
5766
5767 rtm->rtm_flags |= nh_flags;
5768 } else {
5769 if (fib_nexthop_info(skb, &rt->fib6_nh->nh_common, AF_INET6,
5770 &nh_flags, false) < 0)
5771 goto nla_put_failure;
5772
5773 rtm->rtm_flags |= nh_flags;
5774 }
5775
5776 if (rt6_flags & RTF_EXPIRES) {
5777 expires = dst ? dst->expires : rt->expires;
5778 expires -= jiffies;
5779 }
5780
5781 if (!dst) {
5782 if (READ_ONCE(rt->offload))
5783 rtm->rtm_flags |= RTM_F_OFFLOAD;
5784 if (READ_ONCE(rt->trap))
5785 rtm->rtm_flags |= RTM_F_TRAP;
5786 if (READ_ONCE(rt->offload_failed))
5787 rtm->rtm_flags |= RTM_F_OFFLOAD_FAILED;
5788 }
5789
5790 if (rtnl_put_cacheinfo(skb, dst, 0, expires, dst ? dst->error : 0) < 0)
5791 goto nla_put_failure;
5792
5793 if (nla_put_u8(skb, RTA_PREF, IPV6_EXTRACT_PREF(rt6_flags)))
5794 goto nla_put_failure;
5795
5796
5797 nlmsg_end(skb, nlh);
5798 return 0;
5799
5800 nla_put_failure:
5801 nlmsg_cancel(skb, nlh);
5802 return -EMSGSIZE;
5803 }
5804
fib6_info_nh_uses_dev(struct fib6_nh * nh,void * arg)5805 static int fib6_info_nh_uses_dev(struct fib6_nh *nh, void *arg)
5806 {
5807 const struct net_device *dev = arg;
5808
5809 if (nh->fib_nh_dev == dev)
5810 return 1;
5811
5812 return 0;
5813 }
5814
fib6_info_uses_dev(const struct fib6_info * f6i,const struct net_device * dev)5815 static bool fib6_info_uses_dev(const struct fib6_info *f6i,
5816 const struct net_device *dev)
5817 {
5818 if (f6i->nh) {
5819 struct net_device *_dev = (struct net_device *)dev;
5820
5821 return !!nexthop_for_each_fib6_nh(f6i->nh,
5822 fib6_info_nh_uses_dev,
5823 _dev);
5824 }
5825
5826 if (f6i->fib6_nh->fib_nh_dev == dev)
5827 return true;
5828
5829 if (f6i->fib6_nsiblings) {
5830 struct fib6_info *sibling, *next_sibling;
5831
5832 list_for_each_entry_safe(sibling, next_sibling,
5833 &f6i->fib6_siblings, fib6_siblings) {
5834 if (sibling->fib6_nh->fib_nh_dev == dev)
5835 return true;
5836 }
5837 }
5838
5839 return false;
5840 }
5841
5842 struct fib6_nh_exception_dump_walker {
5843 struct rt6_rtnl_dump_arg *dump;
5844 struct fib6_info *rt;
5845 unsigned int flags;
5846 unsigned int skip;
5847 unsigned int count;
5848 };
5849
rt6_nh_dump_exceptions(struct fib6_nh * nh,void * arg)5850 static int rt6_nh_dump_exceptions(struct fib6_nh *nh, void *arg)
5851 {
5852 struct fib6_nh_exception_dump_walker *w = arg;
5853 struct rt6_rtnl_dump_arg *dump = w->dump;
5854 struct rt6_exception_bucket *bucket;
5855 struct rt6_exception *rt6_ex;
5856 int i, err;
5857
5858 bucket = fib6_nh_get_excptn_bucket(nh, NULL);
5859 if (!bucket)
5860 return 0;
5861
5862 for (i = 0; i < FIB6_EXCEPTION_BUCKET_SIZE; i++) {
5863 hlist_for_each_entry(rt6_ex, &bucket->chain, hlist) {
5864 if (w->skip) {
5865 w->skip--;
5866 continue;
5867 }
5868
5869 /* Expiration of entries doesn't bump sernum, insertion
5870 * does. Removal is triggered by insertion, so we can
5871 * rely on the fact that if entries change between two
5872 * partial dumps, this node is scanned again completely,
5873 * see rt6_insert_exception() and fib6_dump_table().
5874 *
5875 * Count expired entries we go through as handled
5876 * entries that we'll skip next time, in case of partial
5877 * node dump. Otherwise, if entries expire meanwhile,
5878 * we'll skip the wrong amount.
5879 */
5880 if (rt6_check_expired(rt6_ex->rt6i)) {
5881 w->count++;
5882 continue;
5883 }
5884
5885 err = rt6_fill_node(dump->net, dump->skb, w->rt,
5886 &rt6_ex->rt6i->dst, NULL, NULL, 0,
5887 RTM_NEWROUTE,
5888 NETLINK_CB(dump->cb->skb).portid,
5889 dump->cb->nlh->nlmsg_seq, w->flags);
5890 if (err)
5891 return err;
5892
5893 w->count++;
5894 }
5895 bucket++;
5896 }
5897
5898 return 0;
5899 }
5900
5901 /* Return -1 if done with node, number of handled routes on partial dump */
rt6_dump_route(struct fib6_info * rt,void * p_arg,unsigned int skip)5902 int rt6_dump_route(struct fib6_info *rt, void *p_arg, unsigned int skip)
5903 {
5904 struct rt6_rtnl_dump_arg *arg = (struct rt6_rtnl_dump_arg *) p_arg;
5905 struct fib_dump_filter *filter = &arg->filter;
5906 unsigned int flags = NLM_F_MULTI;
5907 struct net *net = arg->net;
5908 int count = 0;
5909
5910 if (rt == net->ipv6.fib6_null_entry)
5911 return -1;
5912
5913 if ((filter->flags & RTM_F_PREFIX) &&
5914 !(rt->fib6_flags & RTF_PREFIX_RT)) {
5915 /* success since this is not a prefix route */
5916 return -1;
5917 }
5918 if (filter->filter_set &&
5919 ((filter->rt_type && rt->fib6_type != filter->rt_type) ||
5920 (filter->dev && !fib6_info_uses_dev(rt, filter->dev)) ||
5921 (filter->protocol && rt->fib6_protocol != filter->protocol))) {
5922 return -1;
5923 }
5924
5925 if (filter->filter_set ||
5926 !filter->dump_routes || !filter->dump_exceptions) {
5927 flags |= NLM_F_DUMP_FILTERED;
5928 }
5929
5930 if (filter->dump_routes) {
5931 if (skip) {
5932 skip--;
5933 } else {
5934 if (rt6_fill_node(net, arg->skb, rt, NULL, NULL, NULL,
5935 0, RTM_NEWROUTE,
5936 NETLINK_CB(arg->cb->skb).portid,
5937 arg->cb->nlh->nlmsg_seq, flags)) {
5938 return 0;
5939 }
5940 count++;
5941 }
5942 }
5943
5944 if (filter->dump_exceptions) {
5945 struct fib6_nh_exception_dump_walker w = { .dump = arg,
5946 .rt = rt,
5947 .flags = flags,
5948 .skip = skip,
5949 .count = 0 };
5950 int err;
5951
5952 rcu_read_lock();
5953 if (rt->nh) {
5954 err = nexthop_for_each_fib6_nh(rt->nh,
5955 rt6_nh_dump_exceptions,
5956 &w);
5957 } else {
5958 err = rt6_nh_dump_exceptions(rt->fib6_nh, &w);
5959 }
5960 rcu_read_unlock();
5961
5962 if (err)
5963 return count + w.count;
5964 }
5965
5966 return -1;
5967 }
5968
inet6_rtm_valid_getroute_req(struct sk_buff * skb,const struct nlmsghdr * nlh,struct nlattr ** tb,struct netlink_ext_ack * extack)5969 static int inet6_rtm_valid_getroute_req(struct sk_buff *skb,
5970 const struct nlmsghdr *nlh,
5971 struct nlattr **tb,
5972 struct netlink_ext_ack *extack)
5973 {
5974 struct rtmsg *rtm;
5975 int i, err;
5976
5977 if (nlh->nlmsg_len < nlmsg_msg_size(sizeof(*rtm))) {
5978 NL_SET_ERR_MSG_MOD(extack,
5979 "Invalid header for get route request");
5980 return -EINVAL;
5981 }
5982
5983 if (!netlink_strict_get_check(skb))
5984 return nlmsg_parse_deprecated(nlh, sizeof(*rtm), tb, RTA_MAX,
5985 rtm_ipv6_policy, extack);
5986
5987 rtm = nlmsg_data(nlh);
5988 if ((rtm->rtm_src_len && rtm->rtm_src_len != 128) ||
5989 (rtm->rtm_dst_len && rtm->rtm_dst_len != 128) ||
5990 rtm->rtm_table || rtm->rtm_protocol || rtm->rtm_scope ||
5991 rtm->rtm_type) {
5992 NL_SET_ERR_MSG_MOD(extack, "Invalid values in header for get route request");
5993 return -EINVAL;
5994 }
5995 if (rtm->rtm_flags & ~RTM_F_FIB_MATCH) {
5996 NL_SET_ERR_MSG_MOD(extack,
5997 "Invalid flags for get route request");
5998 return -EINVAL;
5999 }
6000
6001 err = nlmsg_parse_deprecated_strict(nlh, sizeof(*rtm), tb, RTA_MAX,
6002 rtm_ipv6_policy, extack);
6003 if (err)
6004 return err;
6005
6006 if ((tb[RTA_SRC] && !rtm->rtm_src_len) ||
6007 (tb[RTA_DST] && !rtm->rtm_dst_len)) {
6008 NL_SET_ERR_MSG_MOD(extack, "rtm_src_len and rtm_dst_len must be 128 for IPv6");
6009 return -EINVAL;
6010 }
6011
6012 for (i = 0; i <= RTA_MAX; i++) {
6013 if (!tb[i])
6014 continue;
6015
6016 switch (i) {
6017 case RTA_SRC:
6018 case RTA_DST:
6019 case RTA_IIF:
6020 case RTA_OIF:
6021 case RTA_MARK:
6022 case RTA_UID:
6023 case RTA_SPORT:
6024 case RTA_DPORT:
6025 case RTA_IP_PROTO:
6026 break;
6027 default:
6028 NL_SET_ERR_MSG_MOD(extack, "Unsupported attribute in get route request");
6029 return -EINVAL;
6030 }
6031 }
6032
6033 return 0;
6034 }
6035
inet6_rtm_getroute(struct sk_buff * in_skb,struct nlmsghdr * nlh,struct netlink_ext_ack * extack)6036 static int inet6_rtm_getroute(struct sk_buff *in_skb, struct nlmsghdr *nlh,
6037 struct netlink_ext_ack *extack)
6038 {
6039 struct net *net = sock_net(in_skb->sk);
6040 struct nlattr *tb[RTA_MAX+1];
6041 int err, iif = 0, oif = 0;
6042 struct fib6_info *from;
6043 struct dst_entry *dst;
6044 struct rt6_info *rt;
6045 struct sk_buff *skb;
6046 struct rtmsg *rtm;
6047 struct flowi6 fl6 = {};
6048 bool fibmatch;
6049
6050 err = inet6_rtm_valid_getroute_req(in_skb, nlh, tb, extack);
6051 if (err < 0)
6052 goto errout;
6053
6054 err = -EINVAL;
6055 rtm = nlmsg_data(nlh);
6056 fl6.flowlabel = ip6_make_flowinfo(rtm->rtm_tos, 0);
6057 fibmatch = !!(rtm->rtm_flags & RTM_F_FIB_MATCH);
6058
6059 if (tb[RTA_SRC]) {
6060 if (nla_len(tb[RTA_SRC]) < sizeof(struct in6_addr))
6061 goto errout;
6062
6063 fl6.saddr = *(struct in6_addr *)nla_data(tb[RTA_SRC]);
6064 }
6065
6066 if (tb[RTA_DST]) {
6067 if (nla_len(tb[RTA_DST]) < sizeof(struct in6_addr))
6068 goto errout;
6069
6070 fl6.daddr = *(struct in6_addr *)nla_data(tb[RTA_DST]);
6071 }
6072
6073 if (tb[RTA_IIF])
6074 iif = nla_get_u32(tb[RTA_IIF]);
6075
6076 if (tb[RTA_OIF])
6077 oif = nla_get_u32(tb[RTA_OIF]);
6078
6079 if (tb[RTA_MARK])
6080 fl6.flowi6_mark = nla_get_u32(tb[RTA_MARK]);
6081
6082 if (tb[RTA_UID])
6083 fl6.flowi6_uid = make_kuid(current_user_ns(),
6084 nla_get_u32(tb[RTA_UID]));
6085 else
6086 fl6.flowi6_uid = iif ? INVALID_UID : current_uid();
6087
6088 if (tb[RTA_SPORT])
6089 fl6.fl6_sport = nla_get_be16(tb[RTA_SPORT]);
6090
6091 if (tb[RTA_DPORT])
6092 fl6.fl6_dport = nla_get_be16(tb[RTA_DPORT]);
6093
6094 if (tb[RTA_IP_PROTO]) {
6095 err = rtm_getroute_parse_ip_proto(tb[RTA_IP_PROTO],
6096 &fl6.flowi6_proto, AF_INET6,
6097 extack);
6098 if (err)
6099 goto errout;
6100 }
6101
6102 if (iif) {
6103 struct net_device *dev;
6104 int flags = 0;
6105
6106 rcu_read_lock();
6107
6108 dev = dev_get_by_index_rcu(net, iif);
6109 if (!dev) {
6110 rcu_read_unlock();
6111 err = -ENODEV;
6112 goto errout;
6113 }
6114
6115 fl6.flowi6_iif = iif;
6116
6117 if (!ipv6_addr_any(&fl6.saddr))
6118 flags |= RT6_LOOKUP_F_HAS_SADDR;
6119
6120 dst = ip6_route_input_lookup(net, dev, &fl6, NULL, flags);
6121
6122 rcu_read_unlock();
6123 } else {
6124 fl6.flowi6_oif = oif;
6125
6126 dst = ip6_route_output(net, NULL, &fl6);
6127 }
6128
6129
6130 rt = dst_rt6_info(dst);
6131 if (rt->dst.error) {
6132 err = rt->dst.error;
6133 ip6_rt_put(rt);
6134 goto errout;
6135 }
6136
6137 if (rt == net->ipv6.ip6_null_entry) {
6138 err = rt->dst.error;
6139 ip6_rt_put(rt);
6140 goto errout;
6141 }
6142
6143 skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
6144 if (!skb) {
6145 ip6_rt_put(rt);
6146 err = -ENOBUFS;
6147 goto errout;
6148 }
6149
6150 skb_dst_set(skb, &rt->dst);
6151
6152 rcu_read_lock();
6153 from = rcu_dereference(rt->from);
6154 if (from) {
6155 if (fibmatch)
6156 err = rt6_fill_node(net, skb, from, NULL, NULL, NULL,
6157 iif, RTM_NEWROUTE,
6158 NETLINK_CB(in_skb).portid,
6159 nlh->nlmsg_seq, 0);
6160 else
6161 err = rt6_fill_node(net, skb, from, dst, &fl6.daddr,
6162 &fl6.saddr, iif, RTM_NEWROUTE,
6163 NETLINK_CB(in_skb).portid,
6164 nlh->nlmsg_seq, 0);
6165 } else {
6166 err = -ENETUNREACH;
6167 }
6168 rcu_read_unlock();
6169
6170 if (err < 0) {
6171 kfree_skb(skb);
6172 goto errout;
6173 }
6174
6175 err = rtnl_unicast(skb, net, NETLINK_CB(in_skb).portid);
6176 errout:
6177 return err;
6178 }
6179
inet6_rt_notify(int event,struct fib6_info * rt,struct nl_info * info,unsigned int nlm_flags)6180 void inet6_rt_notify(int event, struct fib6_info *rt, struct nl_info *info,
6181 unsigned int nlm_flags)
6182 {
6183 struct sk_buff *skb;
6184 struct net *net = info->nl_net;
6185 u32 seq;
6186 int err;
6187
6188 err = -ENOBUFS;
6189 seq = info->nlh ? info->nlh->nlmsg_seq : 0;
6190
6191 skb = nlmsg_new(rt6_nlmsg_size(rt), GFP_ATOMIC);
6192 if (!skb)
6193 goto errout;
6194
6195 err = rt6_fill_node(net, skb, rt, NULL, NULL, NULL, 0,
6196 event, info->portid, seq, nlm_flags);
6197 if (err < 0) {
6198 /* -EMSGSIZE implies BUG in rt6_nlmsg_size() */
6199 WARN_ON(err == -EMSGSIZE);
6200 kfree_skb(skb);
6201 goto errout;
6202 }
6203 rtnl_notify(skb, net, info->portid, RTNLGRP_IPV6_ROUTE,
6204 info->nlh, GFP_ATOMIC);
6205 return;
6206 errout:
6207 if (err < 0)
6208 rtnl_set_sk_err(net, RTNLGRP_IPV6_ROUTE, err);
6209 }
6210
fib6_rt_update(struct net * net,struct fib6_info * rt,struct nl_info * info)6211 void fib6_rt_update(struct net *net, struct fib6_info *rt,
6212 struct nl_info *info)
6213 {
6214 u32 seq = info->nlh ? info->nlh->nlmsg_seq : 0;
6215 struct sk_buff *skb;
6216 int err = -ENOBUFS;
6217
6218 skb = nlmsg_new(rt6_nlmsg_size(rt), gfp_any());
6219 if (!skb)
6220 goto errout;
6221
6222 err = rt6_fill_node(net, skb, rt, NULL, NULL, NULL, 0,
6223 RTM_NEWROUTE, info->portid, seq, NLM_F_REPLACE);
6224 if (err < 0) {
6225 /* -EMSGSIZE implies BUG in rt6_nlmsg_size() */
6226 WARN_ON(err == -EMSGSIZE);
6227 kfree_skb(skb);
6228 goto errout;
6229 }
6230 rtnl_notify(skb, net, info->portid, RTNLGRP_IPV6_ROUTE,
6231 info->nlh, gfp_any());
6232 return;
6233 errout:
6234 if (err < 0)
6235 rtnl_set_sk_err(net, RTNLGRP_IPV6_ROUTE, err);
6236 }
6237
fib6_info_hw_flags_set(struct net * net,struct fib6_info * f6i,bool offload,bool trap,bool offload_failed)6238 void fib6_info_hw_flags_set(struct net *net, struct fib6_info *f6i,
6239 bool offload, bool trap, bool offload_failed)
6240 {
6241 struct sk_buff *skb;
6242 int err;
6243
6244 if (READ_ONCE(f6i->offload) == offload &&
6245 READ_ONCE(f6i->trap) == trap &&
6246 READ_ONCE(f6i->offload_failed) == offload_failed)
6247 return;
6248
6249 WRITE_ONCE(f6i->offload, offload);
6250 WRITE_ONCE(f6i->trap, trap);
6251
6252 /* 2 means send notifications only if offload_failed was changed. */
6253 if (net->ipv6.sysctl.fib_notify_on_flag_change == 2 &&
6254 READ_ONCE(f6i->offload_failed) == offload_failed)
6255 return;
6256
6257 WRITE_ONCE(f6i->offload_failed, offload_failed);
6258
6259 if (!rcu_access_pointer(f6i->fib6_node))
6260 /* The route was removed from the tree, do not send
6261 * notification.
6262 */
6263 return;
6264
6265 if (!net->ipv6.sysctl.fib_notify_on_flag_change)
6266 return;
6267
6268 skb = nlmsg_new(rt6_nlmsg_size(f6i), GFP_KERNEL);
6269 if (!skb) {
6270 err = -ENOBUFS;
6271 goto errout;
6272 }
6273
6274 err = rt6_fill_node(net, skb, f6i, NULL, NULL, NULL, 0, RTM_NEWROUTE, 0,
6275 0, 0);
6276 if (err < 0) {
6277 /* -EMSGSIZE implies BUG in rt6_nlmsg_size() */
6278 WARN_ON(err == -EMSGSIZE);
6279 kfree_skb(skb);
6280 goto errout;
6281 }
6282
6283 rtnl_notify(skb, net, 0, RTNLGRP_IPV6_ROUTE, NULL, GFP_KERNEL);
6284 return;
6285
6286 errout:
6287 rtnl_set_sk_err(net, RTNLGRP_IPV6_ROUTE, err);
6288 }
6289 EXPORT_SYMBOL(fib6_info_hw_flags_set);
6290
ip6_route_dev_notify(struct notifier_block * this,unsigned long event,void * ptr)6291 static int ip6_route_dev_notify(struct notifier_block *this,
6292 unsigned long event, void *ptr)
6293 {
6294 struct net_device *dev = netdev_notifier_info_to_dev(ptr);
6295 struct net *net = dev_net(dev);
6296
6297 if (!(dev->flags & IFF_LOOPBACK))
6298 return NOTIFY_OK;
6299
6300 if (event == NETDEV_REGISTER) {
6301 net->ipv6.fib6_null_entry->fib6_nh->fib_nh_dev = dev;
6302 net->ipv6.ip6_null_entry->dst.dev = dev;
6303 net->ipv6.ip6_null_entry->rt6i_idev = in6_dev_get(dev);
6304 #ifdef CONFIG_IPV6_MULTIPLE_TABLES
6305 net->ipv6.ip6_prohibit_entry->dst.dev = dev;
6306 net->ipv6.ip6_prohibit_entry->rt6i_idev = in6_dev_get(dev);
6307 net->ipv6.ip6_blk_hole_entry->dst.dev = dev;
6308 net->ipv6.ip6_blk_hole_entry->rt6i_idev = in6_dev_get(dev);
6309 #endif
6310 } else if (event == NETDEV_UNREGISTER &&
6311 dev->reg_state != NETREG_UNREGISTERED) {
6312 /* NETDEV_UNREGISTER could be fired for multiple times by
6313 * netdev_wait_allrefs(). Make sure we only call this once.
6314 */
6315 in6_dev_put_clear(&net->ipv6.ip6_null_entry->rt6i_idev);
6316 #ifdef CONFIG_IPV6_MULTIPLE_TABLES
6317 in6_dev_put_clear(&net->ipv6.ip6_prohibit_entry->rt6i_idev);
6318 in6_dev_put_clear(&net->ipv6.ip6_blk_hole_entry->rt6i_idev);
6319 #endif
6320 }
6321
6322 return NOTIFY_OK;
6323 }
6324
6325 /*
6326 * /proc
6327 */
6328
6329 #ifdef CONFIG_PROC_FS
rt6_stats_seq_show(struct seq_file * seq,void * v)6330 static int rt6_stats_seq_show(struct seq_file *seq, void *v)
6331 {
6332 struct net *net = (struct net *)seq->private;
6333 seq_printf(seq, "%04x %04x %04x %04x %04x %04x %04x\n",
6334 net->ipv6.rt6_stats->fib_nodes,
6335 net->ipv6.rt6_stats->fib_route_nodes,
6336 atomic_read(&net->ipv6.rt6_stats->fib_rt_alloc),
6337 net->ipv6.rt6_stats->fib_rt_entries,
6338 net->ipv6.rt6_stats->fib_rt_cache,
6339 dst_entries_get_slow(&net->ipv6.ip6_dst_ops),
6340 net->ipv6.rt6_stats->fib_discarded_routes);
6341
6342 return 0;
6343 }
6344 #endif /* CONFIG_PROC_FS */
6345
6346 #ifdef CONFIG_SYSCTL
6347
ipv6_sysctl_rtcache_flush(struct ctl_table * ctl,int write,void * buffer,size_t * lenp,loff_t * ppos)6348 static int ipv6_sysctl_rtcache_flush(struct ctl_table *ctl, int write,
6349 void *buffer, size_t *lenp, loff_t *ppos)
6350 {
6351 struct net *net;
6352 int delay;
6353 int ret;
6354 if (!write)
6355 return -EINVAL;
6356
6357 ret = proc_dointvec(ctl, write, buffer, lenp, ppos);
6358 if (ret)
6359 return ret;
6360
6361 net = (struct net *)ctl->extra1;
6362 delay = net->ipv6.sysctl.flush_delay;
6363 fib6_run_gc(delay <= 0 ? 0 : (unsigned long)delay, net, delay > 0);
6364 return 0;
6365 }
6366
6367 static struct ctl_table ipv6_route_table_template[] = {
6368 {
6369 .procname = "max_size",
6370 .data = &init_net.ipv6.sysctl.ip6_rt_max_size,
6371 .maxlen = sizeof(int),
6372 .mode = 0644,
6373 .proc_handler = proc_dointvec,
6374 },
6375 {
6376 .procname = "gc_thresh",
6377 .data = &ip6_dst_ops_template.gc_thresh,
6378 .maxlen = sizeof(int),
6379 .mode = 0644,
6380 .proc_handler = proc_dointvec,
6381 },
6382 {
6383 .procname = "flush",
6384 .data = &init_net.ipv6.sysctl.flush_delay,
6385 .maxlen = sizeof(int),
6386 .mode = 0200,
6387 .proc_handler = ipv6_sysctl_rtcache_flush
6388 },
6389 {
6390 .procname = "gc_min_interval",
6391 .data = &init_net.ipv6.sysctl.ip6_rt_gc_min_interval,
6392 .maxlen = sizeof(int),
6393 .mode = 0644,
6394 .proc_handler = proc_dointvec_jiffies,
6395 },
6396 {
6397 .procname = "gc_timeout",
6398 .data = &init_net.ipv6.sysctl.ip6_rt_gc_timeout,
6399 .maxlen = sizeof(int),
6400 .mode = 0644,
6401 .proc_handler = proc_dointvec_jiffies,
6402 },
6403 {
6404 .procname = "gc_interval",
6405 .data = &init_net.ipv6.sysctl.ip6_rt_gc_interval,
6406 .maxlen = sizeof(int),
6407 .mode = 0644,
6408 .proc_handler = proc_dointvec_jiffies,
6409 },
6410 {
6411 .procname = "gc_elasticity",
6412 .data = &init_net.ipv6.sysctl.ip6_rt_gc_elasticity,
6413 .maxlen = sizeof(int),
6414 .mode = 0644,
6415 .proc_handler = proc_dointvec,
6416 },
6417 {
6418 .procname = "mtu_expires",
6419 .data = &init_net.ipv6.sysctl.ip6_rt_mtu_expires,
6420 .maxlen = sizeof(int),
6421 .mode = 0644,
6422 .proc_handler = proc_dointvec_jiffies,
6423 },
6424 {
6425 .procname = "min_adv_mss",
6426 .data = &init_net.ipv6.sysctl.ip6_rt_min_advmss,
6427 .maxlen = sizeof(int),
6428 .mode = 0644,
6429 .proc_handler = proc_dointvec,
6430 },
6431 {
6432 .procname = "gc_min_interval_ms",
6433 .data = &init_net.ipv6.sysctl.ip6_rt_gc_min_interval,
6434 .maxlen = sizeof(int),
6435 .mode = 0644,
6436 .proc_handler = proc_dointvec_ms_jiffies,
6437 },
6438 {
6439 .procname = "skip_notify_on_dev_down",
6440 .data = &init_net.ipv6.sysctl.skip_notify_on_dev_down,
6441 .maxlen = sizeof(u8),
6442 .mode = 0644,
6443 .proc_handler = proc_dou8vec_minmax,
6444 .extra1 = SYSCTL_ZERO,
6445 .extra2 = SYSCTL_ONE,
6446 },
6447 { }
6448 };
6449
ipv6_route_sysctl_init(struct net * net)6450 struct ctl_table * __net_init ipv6_route_sysctl_init(struct net *net)
6451 {
6452 struct ctl_table *table;
6453
6454 table = kmemdup(ipv6_route_table_template,
6455 sizeof(ipv6_route_table_template),
6456 GFP_KERNEL);
6457
6458 if (table) {
6459 table[0].data = &net->ipv6.sysctl.ip6_rt_max_size;
6460 table[1].data = &net->ipv6.ip6_dst_ops.gc_thresh;
6461 table[2].data = &net->ipv6.sysctl.flush_delay;
6462 table[2].extra1 = net;
6463 table[3].data = &net->ipv6.sysctl.ip6_rt_gc_min_interval;
6464 table[4].data = &net->ipv6.sysctl.ip6_rt_gc_timeout;
6465 table[5].data = &net->ipv6.sysctl.ip6_rt_gc_interval;
6466 table[6].data = &net->ipv6.sysctl.ip6_rt_gc_elasticity;
6467 table[7].data = &net->ipv6.sysctl.ip6_rt_mtu_expires;
6468 table[8].data = &net->ipv6.sysctl.ip6_rt_min_advmss;
6469 table[9].data = &net->ipv6.sysctl.ip6_rt_gc_min_interval;
6470 table[10].data = &net->ipv6.sysctl.skip_notify_on_dev_down;
6471
6472 /* Don't export sysctls to unprivileged users */
6473 if (net->user_ns != &init_user_ns)
6474 table[1].procname = NULL;
6475 }
6476
6477 return table;
6478 }
6479
ipv6_route_sysctl_table_size(struct net * net)6480 size_t ipv6_route_sysctl_table_size(struct net *net)
6481 {
6482 /* Don't export sysctls to unprivileged users */
6483 if (net->user_ns != &init_user_ns)
6484 return 1;
6485
6486 return ARRAY_SIZE(ipv6_route_table_template);
6487 }
6488 #endif
6489
ip6_route_net_init(struct net * net)6490 static int __net_init ip6_route_net_init(struct net *net)
6491 {
6492 int ret = -ENOMEM;
6493
6494 memcpy(&net->ipv6.ip6_dst_ops, &ip6_dst_ops_template,
6495 sizeof(net->ipv6.ip6_dst_ops));
6496
6497 if (dst_entries_init(&net->ipv6.ip6_dst_ops) < 0)
6498 goto out_ip6_dst_ops;
6499
6500 net->ipv6.fib6_null_entry = fib6_info_alloc(GFP_KERNEL, true);
6501 if (!net->ipv6.fib6_null_entry)
6502 goto out_ip6_dst_entries;
6503 memcpy(net->ipv6.fib6_null_entry, &fib6_null_entry_template,
6504 sizeof(*net->ipv6.fib6_null_entry));
6505
6506 net->ipv6.ip6_null_entry = kmemdup(&ip6_null_entry_template,
6507 sizeof(*net->ipv6.ip6_null_entry),
6508 GFP_KERNEL);
6509 if (!net->ipv6.ip6_null_entry)
6510 goto out_fib6_null_entry;
6511 net->ipv6.ip6_null_entry->dst.ops = &net->ipv6.ip6_dst_ops;
6512 dst_init_metrics(&net->ipv6.ip6_null_entry->dst,
6513 ip6_template_metrics, true);
6514 INIT_LIST_HEAD(&net->ipv6.ip6_null_entry->dst.rt_uncached);
6515
6516 #ifdef CONFIG_IPV6_MULTIPLE_TABLES
6517 net->ipv6.fib6_has_custom_rules = false;
6518 net->ipv6.ip6_prohibit_entry = kmemdup(&ip6_prohibit_entry_template,
6519 sizeof(*net->ipv6.ip6_prohibit_entry),
6520 GFP_KERNEL);
6521 if (!net->ipv6.ip6_prohibit_entry)
6522 goto out_ip6_null_entry;
6523 net->ipv6.ip6_prohibit_entry->dst.ops = &net->ipv6.ip6_dst_ops;
6524 dst_init_metrics(&net->ipv6.ip6_prohibit_entry->dst,
6525 ip6_template_metrics, true);
6526 INIT_LIST_HEAD(&net->ipv6.ip6_prohibit_entry->dst.rt_uncached);
6527
6528 net->ipv6.ip6_blk_hole_entry = kmemdup(&ip6_blk_hole_entry_template,
6529 sizeof(*net->ipv6.ip6_blk_hole_entry),
6530 GFP_KERNEL);
6531 if (!net->ipv6.ip6_blk_hole_entry)
6532 goto out_ip6_prohibit_entry;
6533 net->ipv6.ip6_blk_hole_entry->dst.ops = &net->ipv6.ip6_dst_ops;
6534 dst_init_metrics(&net->ipv6.ip6_blk_hole_entry->dst,
6535 ip6_template_metrics, true);
6536 INIT_LIST_HEAD(&net->ipv6.ip6_blk_hole_entry->dst.rt_uncached);
6537 #ifdef CONFIG_IPV6_SUBTREES
6538 net->ipv6.fib6_routes_require_src = 0;
6539 #endif
6540 #endif
6541
6542 net->ipv6.sysctl.flush_delay = 0;
6543 net->ipv6.sysctl.ip6_rt_max_size = INT_MAX;
6544 net->ipv6.sysctl.ip6_rt_gc_min_interval = HZ / 2;
6545 net->ipv6.sysctl.ip6_rt_gc_timeout = 60*HZ;
6546 net->ipv6.sysctl.ip6_rt_gc_interval = 30*HZ;
6547 net->ipv6.sysctl.ip6_rt_gc_elasticity = 9;
6548 net->ipv6.sysctl.ip6_rt_mtu_expires = 10*60*HZ;
6549 net->ipv6.sysctl.ip6_rt_min_advmss = IPV6_MIN_MTU - 20 - 40;
6550 net->ipv6.sysctl.skip_notify_on_dev_down = 0;
6551
6552 atomic_set(&net->ipv6.ip6_rt_gc_expire, 30*HZ);
6553
6554 ret = 0;
6555 out:
6556 return ret;
6557
6558 #ifdef CONFIG_IPV6_MULTIPLE_TABLES
6559 out_ip6_prohibit_entry:
6560 kfree(net->ipv6.ip6_prohibit_entry);
6561 out_ip6_null_entry:
6562 kfree(net->ipv6.ip6_null_entry);
6563 #endif
6564 out_fib6_null_entry:
6565 kfree(net->ipv6.fib6_null_entry);
6566 out_ip6_dst_entries:
6567 dst_entries_destroy(&net->ipv6.ip6_dst_ops);
6568 out_ip6_dst_ops:
6569 goto out;
6570 }
6571
ip6_route_net_exit(struct net * net)6572 static void __net_exit ip6_route_net_exit(struct net *net)
6573 {
6574 kfree(net->ipv6.fib6_null_entry);
6575 kfree(net->ipv6.ip6_null_entry);
6576 #ifdef CONFIG_IPV6_MULTIPLE_TABLES
6577 kfree(net->ipv6.ip6_prohibit_entry);
6578 kfree(net->ipv6.ip6_blk_hole_entry);
6579 #endif
6580 dst_entries_destroy(&net->ipv6.ip6_dst_ops);
6581 }
6582
ip6_route_net_init_late(struct net * net)6583 static int __net_init ip6_route_net_init_late(struct net *net)
6584 {
6585 #ifdef CONFIG_PROC_FS
6586 if (!proc_create_net("ipv6_route", 0, net->proc_net,
6587 &ipv6_route_seq_ops,
6588 sizeof(struct ipv6_route_iter)))
6589 return -ENOMEM;
6590
6591 if (!proc_create_net_single("rt6_stats", 0444, net->proc_net,
6592 rt6_stats_seq_show, NULL)) {
6593 remove_proc_entry("ipv6_route", net->proc_net);
6594 return -ENOMEM;
6595 }
6596 #endif
6597 return 0;
6598 }
6599
ip6_route_net_exit_late(struct net * net)6600 static void __net_exit ip6_route_net_exit_late(struct net *net)
6601 {
6602 #ifdef CONFIG_PROC_FS
6603 remove_proc_entry("ipv6_route", net->proc_net);
6604 remove_proc_entry("rt6_stats", net->proc_net);
6605 #endif
6606 }
6607
6608 static struct pernet_operations ip6_route_net_ops = {
6609 .init = ip6_route_net_init,
6610 .exit = ip6_route_net_exit,
6611 };
6612
ipv6_inetpeer_init(struct net * net)6613 static int __net_init ipv6_inetpeer_init(struct net *net)
6614 {
6615 struct inet_peer_base *bp = kmalloc(sizeof(*bp), GFP_KERNEL);
6616
6617 if (!bp)
6618 return -ENOMEM;
6619 inet_peer_base_init(bp);
6620 net->ipv6.peers = bp;
6621 return 0;
6622 }
6623
ipv6_inetpeer_exit(struct net * net)6624 static void __net_exit ipv6_inetpeer_exit(struct net *net)
6625 {
6626 struct inet_peer_base *bp = net->ipv6.peers;
6627
6628 net->ipv6.peers = NULL;
6629 inetpeer_invalidate_tree(bp);
6630 kfree(bp);
6631 }
6632
6633 static struct pernet_operations ipv6_inetpeer_ops = {
6634 .init = ipv6_inetpeer_init,
6635 .exit = ipv6_inetpeer_exit,
6636 };
6637
6638 static struct pernet_operations ip6_route_net_late_ops = {
6639 .init = ip6_route_net_init_late,
6640 .exit = ip6_route_net_exit_late,
6641 };
6642
6643 static struct notifier_block ip6_route_dev_notifier = {
6644 .notifier_call = ip6_route_dev_notify,
6645 .priority = ADDRCONF_NOTIFY_PRIORITY - 10,
6646 };
6647
ip6_route_init_special_entries(void)6648 void __init ip6_route_init_special_entries(void)
6649 {
6650 /* Registering of the loopback is done before this portion of code,
6651 * the loopback reference in rt6_info will not be taken, do it
6652 * manually for init_net */
6653 init_net.ipv6.fib6_null_entry->fib6_nh->fib_nh_dev = init_net.loopback_dev;
6654 init_net.ipv6.ip6_null_entry->dst.dev = init_net.loopback_dev;
6655 init_net.ipv6.ip6_null_entry->rt6i_idev = in6_dev_get(init_net.loopback_dev);
6656 #ifdef CONFIG_IPV6_MULTIPLE_TABLES
6657 init_net.ipv6.ip6_prohibit_entry->dst.dev = init_net.loopback_dev;
6658 init_net.ipv6.ip6_prohibit_entry->rt6i_idev = in6_dev_get(init_net.loopback_dev);
6659 init_net.ipv6.ip6_blk_hole_entry->dst.dev = init_net.loopback_dev;
6660 init_net.ipv6.ip6_blk_hole_entry->rt6i_idev = in6_dev_get(init_net.loopback_dev);
6661 #endif
6662 }
6663
6664 #if IS_BUILTIN(CONFIG_IPV6)
6665 #if defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_PROC_FS)
6666 DEFINE_BPF_ITER_FUNC(ipv6_route, struct bpf_iter_meta *meta, struct fib6_info *rt)
6667
6668 BTF_ID_LIST(btf_fib6_info_id)
6669 BTF_ID(struct, fib6_info)
6670
6671 static const struct bpf_iter_seq_info ipv6_route_seq_info = {
6672 .seq_ops = &ipv6_route_seq_ops,
6673 .init_seq_private = bpf_iter_init_seq_net,
6674 .fini_seq_private = bpf_iter_fini_seq_net,
6675 .seq_priv_size = sizeof(struct ipv6_route_iter),
6676 };
6677
6678 static struct bpf_iter_reg ipv6_route_reg_info = {
6679 .target = "ipv6_route",
6680 .ctx_arg_info_size = 1,
6681 .ctx_arg_info = {
6682 { offsetof(struct bpf_iter__ipv6_route, rt),
6683 PTR_TO_BTF_ID_OR_NULL },
6684 },
6685 .seq_info = &ipv6_route_seq_info,
6686 };
6687
bpf_iter_register(void)6688 static int __init bpf_iter_register(void)
6689 {
6690 ipv6_route_reg_info.ctx_arg_info[0].btf_id = *btf_fib6_info_id;
6691 return bpf_iter_reg_target(&ipv6_route_reg_info);
6692 }
6693
bpf_iter_unregister(void)6694 static void bpf_iter_unregister(void)
6695 {
6696 bpf_iter_unreg_target(&ipv6_route_reg_info);
6697 }
6698 #endif
6699 #endif
6700
ip6_route_init(void)6701 int __init ip6_route_init(void)
6702 {
6703 int ret;
6704 int cpu;
6705
6706 ret = -ENOMEM;
6707 ip6_dst_ops_template.kmem_cachep =
6708 kmem_cache_create("ip6_dst_cache", sizeof(struct rt6_info), 0,
6709 SLAB_HWCACHE_ALIGN | SLAB_ACCOUNT, NULL);
6710 if (!ip6_dst_ops_template.kmem_cachep)
6711 goto out;
6712
6713 ret = dst_entries_init(&ip6_dst_blackhole_ops);
6714 if (ret)
6715 goto out_kmem_cache;
6716
6717 ret = register_pernet_subsys(&ipv6_inetpeer_ops);
6718 if (ret)
6719 goto out_dst_entries;
6720
6721 ret = register_pernet_subsys(&ip6_route_net_ops);
6722 if (ret)
6723 goto out_register_inetpeer;
6724
6725 ip6_dst_blackhole_ops.kmem_cachep = ip6_dst_ops_template.kmem_cachep;
6726
6727 ret = fib6_init();
6728 if (ret)
6729 goto out_register_subsys;
6730
6731 ret = xfrm6_init();
6732 if (ret)
6733 goto out_fib6_init;
6734
6735 ret = fib6_rules_init();
6736 if (ret)
6737 goto xfrm6_init;
6738
6739 ret = register_pernet_subsys(&ip6_route_net_late_ops);
6740 if (ret)
6741 goto fib6_rules_init;
6742
6743 ret = rtnl_register_module(THIS_MODULE, PF_INET6, RTM_NEWROUTE,
6744 inet6_rtm_newroute, NULL, 0);
6745 if (ret < 0)
6746 goto out_register_late_subsys;
6747
6748 ret = rtnl_register_module(THIS_MODULE, PF_INET6, RTM_DELROUTE,
6749 inet6_rtm_delroute, NULL, 0);
6750 if (ret < 0)
6751 goto out_register_late_subsys;
6752
6753 ret = rtnl_register_module(THIS_MODULE, PF_INET6, RTM_GETROUTE,
6754 inet6_rtm_getroute, NULL,
6755 RTNL_FLAG_DOIT_UNLOCKED);
6756 if (ret < 0)
6757 goto out_register_late_subsys;
6758
6759 ret = register_netdevice_notifier(&ip6_route_dev_notifier);
6760 if (ret)
6761 goto out_register_late_subsys;
6762
6763 #if IS_BUILTIN(CONFIG_IPV6)
6764 #if defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_PROC_FS)
6765 ret = bpf_iter_register();
6766 if (ret)
6767 goto out_register_late_subsys;
6768 #endif
6769 #endif
6770
6771 for_each_possible_cpu(cpu) {
6772 struct uncached_list *ul = per_cpu_ptr(&rt6_uncached_list, cpu);
6773
6774 INIT_LIST_HEAD(&ul->head);
6775 INIT_LIST_HEAD(&ul->quarantine);
6776 spin_lock_init(&ul->lock);
6777 }
6778
6779 out:
6780 return ret;
6781
6782 out_register_late_subsys:
6783 rtnl_unregister_all(PF_INET6);
6784 unregister_pernet_subsys(&ip6_route_net_late_ops);
6785 fib6_rules_init:
6786 fib6_rules_cleanup();
6787 xfrm6_init:
6788 xfrm6_fini();
6789 out_fib6_init:
6790 fib6_gc_cleanup();
6791 out_register_subsys:
6792 unregister_pernet_subsys(&ip6_route_net_ops);
6793 out_register_inetpeer:
6794 unregister_pernet_subsys(&ipv6_inetpeer_ops);
6795 out_dst_entries:
6796 dst_entries_destroy(&ip6_dst_blackhole_ops);
6797 out_kmem_cache:
6798 kmem_cache_destroy(ip6_dst_ops_template.kmem_cachep);
6799 goto out;
6800 }
6801
ip6_route_cleanup(void)6802 void ip6_route_cleanup(void)
6803 {
6804 #if IS_BUILTIN(CONFIG_IPV6)
6805 #if defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_PROC_FS)
6806 bpf_iter_unregister();
6807 #endif
6808 #endif
6809 unregister_netdevice_notifier(&ip6_route_dev_notifier);
6810 unregister_pernet_subsys(&ip6_route_net_late_ops);
6811 fib6_rules_cleanup();
6812 xfrm6_fini();
6813 fib6_gc_cleanup();
6814 unregister_pernet_subsys(&ipv6_inetpeer_ops);
6815 unregister_pernet_subsys(&ip6_route_net_ops);
6816 dst_entries_destroy(&ip6_dst_blackhole_ops);
6817 kmem_cache_destroy(ip6_dst_ops_template.kmem_cachep);
6818 }
6819