1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Linux INET6 implementation 4 * FIB front-end. 5 * 6 * Authors: 7 * Pedro Roque <roque@di.fc.ul.pt> 8 */ 9 10 /* Changes: 11 * 12 * YOSHIFUJI Hideaki @USAGI 13 * reworked default router selection. 14 * - respect outgoing interface 15 * - select from (probably) reachable routers (i.e. 16 * routers in REACHABLE, STALE, DELAY or PROBE states). 17 * - always select the same router if it is (probably) 18 * reachable. otherwise, round-robin the list. 19 * Ville Nuorvala 20 * Fixed routing subtrees. 21 */ 22 23 #define pr_fmt(fmt) "IPv6: " fmt 24 25 #include <linux/capability.h> 26 #include <linux/errno.h> 27 #include <linux/export.h> 28 #include <linux/types.h> 29 #include <linux/times.h> 30 #include <linux/socket.h> 31 #include <linux/sockios.h> 32 #include <linux/net.h> 33 #include <linux/route.h> 34 #include <linux/netdevice.h> 35 #include <linux/in6.h> 36 #include <linux/mroute6.h> 37 #include <linux/init.h> 38 #include <linux/if_arp.h> 39 #include <linux/proc_fs.h> 40 #include <linux/seq_file.h> 41 #include <linux/nsproxy.h> 42 #include <linux/slab.h> 43 #include <linux/jhash.h> 44 #include <net/net_namespace.h> 45 #include <net/snmp.h> 46 #include <net/ipv6.h> 47 #include <net/ip6_fib.h> 48 #include <net/ip6_route.h> 49 #include <net/ndisc.h> 50 #include <net/addrconf.h> 51 #include <net/tcp.h> 52 #include <linux/rtnetlink.h> 53 #include <net/dst.h> 54 #include <net/dst_metadata.h> 55 #include <net/xfrm.h> 56 #include <net/netevent.h> 57 #include <net/netlink.h> 58 #include <net/rtnh.h> 59 #include <net/lwtunnel.h> 60 #include <net/ip_tunnels.h> 61 #include <net/l3mdev.h> 62 #include <net/ip.h> 63 #include <linux/uaccess.h> 64 #include <linux/btf_ids.h> 65 66 #ifdef CONFIG_SYSCTL 67 #include <linux/sysctl.h> 68 #endif 69 70 static int ip6_rt_type_to_error(u8 fib6_type); 71 72 #define CREATE_TRACE_POINTS 73 #include <trace/events/fib6.h> 74 EXPORT_TRACEPOINT_SYMBOL_GPL(fib6_table_lookup); 75 #undef CREATE_TRACE_POINTS 76 77 enum rt6_nud_state { 78 RT6_NUD_FAIL_HARD = -3, 79 RT6_NUD_FAIL_PROBE = -2, 80 RT6_NUD_FAIL_DO_RR = -1, 81 RT6_NUD_SUCCEED = 1 82 }; 83 84 INDIRECT_CALLABLE_SCOPE 85 struct dst_entry *ip6_dst_check(struct dst_entry *dst, u32 cookie); 86 static unsigned int ip6_default_advmss(const struct dst_entry *dst); 87 INDIRECT_CALLABLE_SCOPE 88 unsigned int ip6_mtu(const struct dst_entry *dst); 89 static struct dst_entry *ip6_negative_advice(struct dst_entry *); 90 static void ip6_dst_destroy(struct dst_entry *); 91 static void ip6_dst_ifdown(struct dst_entry *, 92 struct net_device *dev, int how); 93 static int ip6_dst_gc(struct dst_ops *ops); 94 95 static int ip6_pkt_discard(struct sk_buff *skb); 96 static int ip6_pkt_discard_out(struct net *net, struct sock *sk, struct sk_buff *skb); 97 static int ip6_pkt_prohibit(struct sk_buff *skb); 98 static int ip6_pkt_prohibit_out(struct net *net, struct sock *sk, struct sk_buff *skb); 99 static void ip6_link_failure(struct sk_buff *skb); 100 static void ip6_rt_update_pmtu(struct dst_entry *dst, struct sock *sk, 101 struct sk_buff *skb, u32 mtu, 102 bool confirm_neigh); 103 static void rt6_do_redirect(struct dst_entry *dst, struct sock *sk, 104 struct sk_buff *skb); 105 static int rt6_score_route(const struct fib6_nh *nh, u32 fib6_flags, int oif, 106 int strict); 107 static size_t rt6_nlmsg_size(struct fib6_info *f6i); 108 static int rt6_fill_node(struct net *net, struct sk_buff *skb, 109 struct fib6_info *rt, struct dst_entry *dst, 110 struct in6_addr *dest, struct in6_addr *src, 111 int iif, int type, u32 portid, u32 seq, 112 unsigned int flags); 113 static struct rt6_info *rt6_find_cached_rt(const struct fib6_result *res, 114 const struct in6_addr *daddr, 115 const struct in6_addr *saddr); 116 117 #ifdef CONFIG_IPV6_ROUTE_INFO 118 static struct fib6_info *rt6_add_route_info(struct net *net, 119 const struct in6_addr *prefix, int prefixlen, 120 const struct in6_addr *gwaddr, 121 struct net_device *dev, 122 unsigned int pref); 123 static struct fib6_info *rt6_get_route_info(struct net *net, 124 const struct in6_addr *prefix, int prefixlen, 125 const struct in6_addr *gwaddr, 126 struct net_device *dev); 127 #endif 128 129 struct uncached_list { 130 spinlock_t lock; 131 struct list_head head; 132 }; 133 134 static DEFINE_PER_CPU_ALIGNED(struct uncached_list, rt6_uncached_list); 135 136 void rt6_uncached_list_add(struct rt6_info *rt) 137 { 138 struct uncached_list *ul = raw_cpu_ptr(&rt6_uncached_list); 139 140 rt->rt6i_uncached_list = ul; 141 142 spin_lock_bh(&ul->lock); 143 list_add_tail(&rt->rt6i_uncached, &ul->head); 144 spin_unlock_bh(&ul->lock); 145 } 146 147 void rt6_uncached_list_del(struct rt6_info *rt) 148 { 149 if (!list_empty(&rt->rt6i_uncached)) { 150 struct uncached_list *ul = rt->rt6i_uncached_list; 151 struct net *net = dev_net(rt->dst.dev); 152 153 spin_lock_bh(&ul->lock); 154 list_del(&rt->rt6i_uncached); 155 atomic_dec(&net->ipv6.rt6_stats->fib_rt_uncache); 156 spin_unlock_bh(&ul->lock); 157 } 158 } 159 160 static void rt6_uncached_list_flush_dev(struct net *net, struct net_device *dev) 161 { 162 struct net_device *loopback_dev = net->loopback_dev; 163 int cpu; 164 165 if (dev == loopback_dev) 166 return; 167 168 for_each_possible_cpu(cpu) { 169 struct uncached_list *ul = per_cpu_ptr(&rt6_uncached_list, cpu); 170 struct rt6_info *rt; 171 172 spin_lock_bh(&ul->lock); 173 list_for_each_entry(rt, &ul->head, rt6i_uncached) { 174 struct inet6_dev *rt_idev = rt->rt6i_idev; 175 struct net_device *rt_dev = rt->dst.dev; 176 177 if (rt_idev->dev == dev) { 178 rt->rt6i_idev = in6_dev_get(loopback_dev); 179 in6_dev_put(rt_idev); 180 } 181 182 if (rt_dev == dev) { 183 rt->dst.dev = blackhole_netdev; 184 dev_hold(rt->dst.dev); 185 dev_put(rt_dev); 186 } 187 } 188 spin_unlock_bh(&ul->lock); 189 } 190 } 191 192 static inline const void *choose_neigh_daddr(const struct in6_addr *p, 193 struct sk_buff *skb, 194 const void *daddr) 195 { 196 if (!ipv6_addr_any(p)) 197 return (const void *) p; 198 else if (skb) 199 return &ipv6_hdr(skb)->daddr; 200 return daddr; 201 } 202 203 struct neighbour *ip6_neigh_lookup(const struct in6_addr *gw, 204 struct net_device *dev, 205 struct sk_buff *skb, 206 const void *daddr) 207 { 208 struct neighbour *n; 209 210 daddr = choose_neigh_daddr(gw, skb, daddr); 211 n = __ipv6_neigh_lookup(dev, daddr); 212 if (n) 213 return n; 214 215 n = neigh_create(&nd_tbl, daddr, dev); 216 return IS_ERR(n) ? NULL : n; 217 } 218 219 static struct neighbour *ip6_dst_neigh_lookup(const struct dst_entry *dst, 220 struct sk_buff *skb, 221 const void *daddr) 222 { 223 const struct rt6_info *rt = container_of(dst, struct rt6_info, dst); 224 225 return ip6_neigh_lookup(rt6_nexthop(rt, &in6addr_any), 226 dst->dev, skb, daddr); 227 } 228 229 static void ip6_confirm_neigh(const struct dst_entry *dst, const void *daddr) 230 { 231 struct net_device *dev = dst->dev; 232 struct rt6_info *rt = (struct rt6_info *)dst; 233 234 daddr = choose_neigh_daddr(rt6_nexthop(rt, &in6addr_any), NULL, daddr); 235 if (!daddr) 236 return; 237 if (dev->flags & (IFF_NOARP | IFF_LOOPBACK)) 238 return; 239 if (ipv6_addr_is_multicast((const struct in6_addr *)daddr)) 240 return; 241 __ipv6_confirm_neigh(dev, daddr); 242 } 243 244 static struct dst_ops ip6_dst_ops_template = { 245 .family = AF_INET6, 246 .gc = ip6_dst_gc, 247 .gc_thresh = 1024, 248 .check = ip6_dst_check, 249 .default_advmss = ip6_default_advmss, 250 .mtu = ip6_mtu, 251 .cow_metrics = dst_cow_metrics_generic, 252 .destroy = ip6_dst_destroy, 253 .ifdown = ip6_dst_ifdown, 254 .negative_advice = ip6_negative_advice, 255 .link_failure = ip6_link_failure, 256 .update_pmtu = ip6_rt_update_pmtu, 257 .redirect = rt6_do_redirect, 258 .local_out = __ip6_local_out, 259 .neigh_lookup = ip6_dst_neigh_lookup, 260 .confirm_neigh = ip6_confirm_neigh, 261 }; 262 263 static unsigned int ip6_blackhole_mtu(const struct dst_entry *dst) 264 { 265 unsigned int mtu = dst_metric_raw(dst, RTAX_MTU); 266 267 return mtu ? : dst->dev->mtu; 268 } 269 270 static void ip6_rt_blackhole_update_pmtu(struct dst_entry *dst, struct sock *sk, 271 struct sk_buff *skb, u32 mtu, 272 bool confirm_neigh) 273 { 274 } 275 276 static void ip6_rt_blackhole_redirect(struct dst_entry *dst, struct sock *sk, 277 struct sk_buff *skb) 278 { 279 } 280 281 static struct dst_ops ip6_dst_blackhole_ops = { 282 .family = AF_INET6, 283 .destroy = ip6_dst_destroy, 284 .check = ip6_dst_check, 285 .mtu = ip6_blackhole_mtu, 286 .default_advmss = ip6_default_advmss, 287 .update_pmtu = ip6_rt_blackhole_update_pmtu, 288 .redirect = ip6_rt_blackhole_redirect, 289 .cow_metrics = dst_cow_metrics_generic, 290 .neigh_lookup = ip6_dst_neigh_lookup, 291 }; 292 293 static const u32 ip6_template_metrics[RTAX_MAX] = { 294 [RTAX_HOPLIMIT - 1] = 0, 295 }; 296 297 static const struct fib6_info fib6_null_entry_template = { 298 .fib6_flags = (RTF_REJECT | RTF_NONEXTHOP), 299 .fib6_protocol = RTPROT_KERNEL, 300 .fib6_metric = ~(u32)0, 301 .fib6_ref = REFCOUNT_INIT(1), 302 .fib6_type = RTN_UNREACHABLE, 303 .fib6_metrics = (struct dst_metrics *)&dst_default_metrics, 304 }; 305 306 static const struct rt6_info ip6_null_entry_template = { 307 .dst = { 308 .__refcnt = ATOMIC_INIT(1), 309 .__use = 1, 310 .obsolete = DST_OBSOLETE_FORCE_CHK, 311 .error = -ENETUNREACH, 312 .input = ip6_pkt_discard, 313 .output = ip6_pkt_discard_out, 314 }, 315 .rt6i_flags = (RTF_REJECT | RTF_NONEXTHOP), 316 }; 317 318 #ifdef CONFIG_IPV6_MULTIPLE_TABLES 319 320 static const struct rt6_info ip6_prohibit_entry_template = { 321 .dst = { 322 .__refcnt = ATOMIC_INIT(1), 323 .__use = 1, 324 .obsolete = DST_OBSOLETE_FORCE_CHK, 325 .error = -EACCES, 326 .input = ip6_pkt_prohibit, 327 .output = ip6_pkt_prohibit_out, 328 }, 329 .rt6i_flags = (RTF_REJECT | RTF_NONEXTHOP), 330 }; 331 332 static const struct rt6_info ip6_blk_hole_entry_template = { 333 .dst = { 334 .__refcnt = ATOMIC_INIT(1), 335 .__use = 1, 336 .obsolete = DST_OBSOLETE_FORCE_CHK, 337 .error = -EINVAL, 338 .input = dst_discard, 339 .output = dst_discard_out, 340 }, 341 .rt6i_flags = (RTF_REJECT | RTF_NONEXTHOP), 342 }; 343 344 #endif 345 346 static void rt6_info_init(struct rt6_info *rt) 347 { 348 struct dst_entry *dst = &rt->dst; 349 350 memset(dst + 1, 0, sizeof(*rt) - sizeof(*dst)); 351 INIT_LIST_HEAD(&rt->rt6i_uncached); 352 } 353 354 /* allocate dst with ip6_dst_ops */ 355 struct rt6_info *ip6_dst_alloc(struct net *net, struct net_device *dev, 356 int flags) 357 { 358 struct rt6_info *rt = dst_alloc(&net->ipv6.ip6_dst_ops, dev, 359 1, DST_OBSOLETE_FORCE_CHK, flags); 360 361 if (rt) { 362 rt6_info_init(rt); 363 atomic_inc(&net->ipv6.rt6_stats->fib_rt_alloc); 364 } 365 366 return rt; 367 } 368 EXPORT_SYMBOL(ip6_dst_alloc); 369 370 static void ip6_dst_destroy(struct dst_entry *dst) 371 { 372 struct rt6_info *rt = (struct rt6_info *)dst; 373 struct fib6_info *from; 374 struct inet6_dev *idev; 375 376 ip_dst_metrics_put(dst); 377 rt6_uncached_list_del(rt); 378 379 idev = rt->rt6i_idev; 380 if (idev) { 381 rt->rt6i_idev = NULL; 382 in6_dev_put(idev); 383 } 384 385 from = xchg((__force struct fib6_info **)&rt->from, NULL); 386 fib6_info_release(from); 387 } 388 389 static void ip6_dst_ifdown(struct dst_entry *dst, struct net_device *dev, 390 int how) 391 { 392 struct rt6_info *rt = (struct rt6_info *)dst; 393 struct inet6_dev *idev = rt->rt6i_idev; 394 struct net_device *loopback_dev = 395 dev_net(dev)->loopback_dev; 396 397 if (idev && idev->dev != loopback_dev) { 398 struct inet6_dev *loopback_idev = in6_dev_get(loopback_dev); 399 if (loopback_idev) { 400 rt->rt6i_idev = loopback_idev; 401 in6_dev_put(idev); 402 } 403 } 404 } 405 406 static bool __rt6_check_expired(const struct rt6_info *rt) 407 { 408 if (rt->rt6i_flags & RTF_EXPIRES) 409 return time_after(jiffies, rt->dst.expires); 410 else 411 return false; 412 } 413 414 static bool rt6_check_expired(const struct rt6_info *rt) 415 { 416 struct fib6_info *from; 417 418 from = rcu_dereference(rt->from); 419 420 if (rt->rt6i_flags & RTF_EXPIRES) { 421 if (time_after(jiffies, rt->dst.expires)) 422 return true; 423 } else if (from) { 424 return rt->dst.obsolete != DST_OBSOLETE_FORCE_CHK || 425 fib6_check_expired(from); 426 } 427 return false; 428 } 429 430 void fib6_select_path(const struct net *net, struct fib6_result *res, 431 struct flowi6 *fl6, int oif, bool have_oif_match, 432 const struct sk_buff *skb, int strict) 433 { 434 struct fib6_info *sibling, *next_sibling; 435 struct fib6_info *match = res->f6i; 436 437 if (!match->nh && (!match->fib6_nsiblings || have_oif_match)) 438 goto out; 439 440 if (match->nh && have_oif_match && res->nh) 441 return; 442 443 /* We might have already computed the hash for ICMPv6 errors. In such 444 * case it will always be non-zero. Otherwise now is the time to do it. 445 */ 446 if (!fl6->mp_hash && 447 (!match->nh || nexthop_is_multipath(match->nh))) 448 fl6->mp_hash = rt6_multipath_hash(net, fl6, skb, NULL); 449 450 if (unlikely(match->nh)) { 451 nexthop_path_fib6_result(res, fl6->mp_hash); 452 return; 453 } 454 455 if (fl6->mp_hash <= atomic_read(&match->fib6_nh->fib_nh_upper_bound)) 456 goto out; 457 458 list_for_each_entry_safe(sibling, next_sibling, &match->fib6_siblings, 459 fib6_siblings) { 460 const struct fib6_nh *nh = sibling->fib6_nh; 461 int nh_upper_bound; 462 463 nh_upper_bound = atomic_read(&nh->fib_nh_upper_bound); 464 if (fl6->mp_hash > nh_upper_bound) 465 continue; 466 if (rt6_score_route(nh, sibling->fib6_flags, oif, strict) < 0) 467 break; 468 match = sibling; 469 break; 470 } 471 472 out: 473 res->f6i = match; 474 res->nh = match->fib6_nh; 475 } 476 477 /* 478 * Route lookup. rcu_read_lock() should be held. 479 */ 480 481 static bool __rt6_device_match(struct net *net, const struct fib6_nh *nh, 482 const struct in6_addr *saddr, int oif, int flags) 483 { 484 const struct net_device *dev; 485 486 if (nh->fib_nh_flags & RTNH_F_DEAD) 487 return false; 488 489 dev = nh->fib_nh_dev; 490 if (oif) { 491 if (dev->ifindex == oif) 492 return true; 493 } else { 494 if (ipv6_chk_addr(net, saddr, dev, 495 flags & RT6_LOOKUP_F_IFACE)) 496 return true; 497 } 498 499 return false; 500 } 501 502 struct fib6_nh_dm_arg { 503 struct net *net; 504 const struct in6_addr *saddr; 505 int oif; 506 int flags; 507 struct fib6_nh *nh; 508 }; 509 510 static int __rt6_nh_dev_match(struct fib6_nh *nh, void *_arg) 511 { 512 struct fib6_nh_dm_arg *arg = _arg; 513 514 arg->nh = nh; 515 return __rt6_device_match(arg->net, nh, arg->saddr, arg->oif, 516 arg->flags); 517 } 518 519 /* returns fib6_nh from nexthop or NULL */ 520 static struct fib6_nh *rt6_nh_dev_match(struct net *net, struct nexthop *nh, 521 struct fib6_result *res, 522 const struct in6_addr *saddr, 523 int oif, int flags) 524 { 525 struct fib6_nh_dm_arg arg = { 526 .net = net, 527 .saddr = saddr, 528 .oif = oif, 529 .flags = flags, 530 }; 531 532 if (nexthop_is_blackhole(nh)) 533 return NULL; 534 535 if (nexthop_for_each_fib6_nh(nh, __rt6_nh_dev_match, &arg)) 536 return arg.nh; 537 538 return NULL; 539 } 540 541 static void rt6_device_match(struct net *net, struct fib6_result *res, 542 const struct in6_addr *saddr, int oif, int flags) 543 { 544 struct fib6_info *f6i = res->f6i; 545 struct fib6_info *spf6i; 546 struct fib6_nh *nh; 547 548 if (!oif && ipv6_addr_any(saddr)) { 549 if (unlikely(f6i->nh)) { 550 nh = nexthop_fib6_nh(f6i->nh); 551 if (nexthop_is_blackhole(f6i->nh)) 552 goto out_blackhole; 553 } else { 554 nh = f6i->fib6_nh; 555 } 556 if (!(nh->fib_nh_flags & RTNH_F_DEAD)) 557 goto out; 558 } 559 560 for (spf6i = f6i; spf6i; spf6i = rcu_dereference(spf6i->fib6_next)) { 561 bool matched = false; 562 563 if (unlikely(spf6i->nh)) { 564 nh = rt6_nh_dev_match(net, spf6i->nh, res, saddr, 565 oif, flags); 566 if (nh) 567 matched = true; 568 } else { 569 nh = spf6i->fib6_nh; 570 if (__rt6_device_match(net, nh, saddr, oif, flags)) 571 matched = true; 572 } 573 if (matched) { 574 res->f6i = spf6i; 575 goto out; 576 } 577 } 578 579 if (oif && flags & RT6_LOOKUP_F_IFACE) { 580 res->f6i = net->ipv6.fib6_null_entry; 581 nh = res->f6i->fib6_nh; 582 goto out; 583 } 584 585 if (unlikely(f6i->nh)) { 586 nh = nexthop_fib6_nh(f6i->nh); 587 if (nexthop_is_blackhole(f6i->nh)) 588 goto out_blackhole; 589 } else { 590 nh = f6i->fib6_nh; 591 } 592 593 if (nh->fib_nh_flags & RTNH_F_DEAD) { 594 res->f6i = net->ipv6.fib6_null_entry; 595 nh = res->f6i->fib6_nh; 596 } 597 out: 598 res->nh = nh; 599 res->fib6_type = res->f6i->fib6_type; 600 res->fib6_flags = res->f6i->fib6_flags; 601 return; 602 603 out_blackhole: 604 res->fib6_flags |= RTF_REJECT; 605 res->fib6_type = RTN_BLACKHOLE; 606 res->nh = nh; 607 } 608 609 #ifdef CONFIG_IPV6_ROUTER_PREF 610 struct __rt6_probe_work { 611 struct work_struct work; 612 struct in6_addr target; 613 struct net_device *dev; 614 }; 615 616 static void rt6_probe_deferred(struct work_struct *w) 617 { 618 struct in6_addr mcaddr; 619 struct __rt6_probe_work *work = 620 container_of(w, struct __rt6_probe_work, work); 621 622 addrconf_addr_solict_mult(&work->target, &mcaddr); 623 ndisc_send_ns(work->dev, &work->target, &mcaddr, NULL, 0); 624 dev_put(work->dev); 625 kfree(work); 626 } 627 628 static void rt6_probe(struct fib6_nh *fib6_nh) 629 { 630 struct __rt6_probe_work *work = NULL; 631 const struct in6_addr *nh_gw; 632 unsigned long last_probe; 633 struct neighbour *neigh; 634 struct net_device *dev; 635 struct inet6_dev *idev; 636 637 /* 638 * Okay, this does not seem to be appropriate 639 * for now, however, we need to check if it 640 * is really so; aka Router Reachability Probing. 641 * 642 * Router Reachability Probe MUST be rate-limited 643 * to no more than one per minute. 644 */ 645 if (!fib6_nh->fib_nh_gw_family) 646 return; 647 648 nh_gw = &fib6_nh->fib_nh_gw6; 649 dev = fib6_nh->fib_nh_dev; 650 rcu_read_lock_bh(); 651 last_probe = READ_ONCE(fib6_nh->last_probe); 652 idev = __in6_dev_get(dev); 653 neigh = __ipv6_neigh_lookup_noref(dev, nh_gw); 654 if (neigh) { 655 if (neigh->nud_state & NUD_VALID) 656 goto out; 657 658 write_lock(&neigh->lock); 659 if (!(neigh->nud_state & NUD_VALID) && 660 time_after(jiffies, 661 neigh->updated + idev->cnf.rtr_probe_interval)) { 662 work = kmalloc(sizeof(*work), GFP_ATOMIC); 663 if (work) 664 __neigh_set_probe_once(neigh); 665 } 666 write_unlock(&neigh->lock); 667 } else if (time_after(jiffies, last_probe + 668 idev->cnf.rtr_probe_interval)) { 669 work = kmalloc(sizeof(*work), GFP_ATOMIC); 670 } 671 672 if (!work || cmpxchg(&fib6_nh->last_probe, 673 last_probe, jiffies) != last_probe) { 674 kfree(work); 675 } else { 676 INIT_WORK(&work->work, rt6_probe_deferred); 677 work->target = *nh_gw; 678 dev_hold(dev); 679 work->dev = dev; 680 schedule_work(&work->work); 681 } 682 683 out: 684 rcu_read_unlock_bh(); 685 } 686 #else 687 static inline void rt6_probe(struct fib6_nh *fib6_nh) 688 { 689 } 690 #endif 691 692 /* 693 * Default Router Selection (RFC 2461 6.3.6) 694 */ 695 static enum rt6_nud_state rt6_check_neigh(const struct fib6_nh *fib6_nh) 696 { 697 enum rt6_nud_state ret = RT6_NUD_FAIL_HARD; 698 struct neighbour *neigh; 699 700 rcu_read_lock_bh(); 701 neigh = __ipv6_neigh_lookup_noref(fib6_nh->fib_nh_dev, 702 &fib6_nh->fib_nh_gw6); 703 if (neigh) { 704 read_lock(&neigh->lock); 705 if (neigh->nud_state & NUD_VALID) 706 ret = RT6_NUD_SUCCEED; 707 #ifdef CONFIG_IPV6_ROUTER_PREF 708 else if (!(neigh->nud_state & NUD_FAILED)) 709 ret = RT6_NUD_SUCCEED; 710 else 711 ret = RT6_NUD_FAIL_PROBE; 712 #endif 713 read_unlock(&neigh->lock); 714 } else { 715 ret = IS_ENABLED(CONFIG_IPV6_ROUTER_PREF) ? 716 RT6_NUD_SUCCEED : RT6_NUD_FAIL_DO_RR; 717 } 718 rcu_read_unlock_bh(); 719 720 return ret; 721 } 722 723 static int rt6_score_route(const struct fib6_nh *nh, u32 fib6_flags, int oif, 724 int strict) 725 { 726 int m = 0; 727 728 if (!oif || nh->fib_nh_dev->ifindex == oif) 729 m = 2; 730 731 if (!m && (strict & RT6_LOOKUP_F_IFACE)) 732 return RT6_NUD_FAIL_HARD; 733 #ifdef CONFIG_IPV6_ROUTER_PREF 734 m |= IPV6_DECODE_PREF(IPV6_EXTRACT_PREF(fib6_flags)) << 2; 735 #endif 736 if ((strict & RT6_LOOKUP_F_REACHABLE) && 737 !(fib6_flags & RTF_NONEXTHOP) && nh->fib_nh_gw_family) { 738 int n = rt6_check_neigh(nh); 739 if (n < 0) 740 return n; 741 } 742 return m; 743 } 744 745 static bool find_match(struct fib6_nh *nh, u32 fib6_flags, 746 int oif, int strict, int *mpri, bool *do_rr) 747 { 748 bool match_do_rr = false; 749 bool rc = false; 750 int m; 751 752 if (nh->fib_nh_flags & RTNH_F_DEAD) 753 goto out; 754 755 if (ip6_ignore_linkdown(nh->fib_nh_dev) && 756 nh->fib_nh_flags & RTNH_F_LINKDOWN && 757 !(strict & RT6_LOOKUP_F_IGNORE_LINKSTATE)) 758 goto out; 759 760 m = rt6_score_route(nh, fib6_flags, oif, strict); 761 if (m == RT6_NUD_FAIL_DO_RR) { 762 match_do_rr = true; 763 m = 0; /* lowest valid score */ 764 } else if (m == RT6_NUD_FAIL_HARD) { 765 goto out; 766 } 767 768 if (strict & RT6_LOOKUP_F_REACHABLE) 769 rt6_probe(nh); 770 771 /* note that m can be RT6_NUD_FAIL_PROBE at this point */ 772 if (m > *mpri) { 773 *do_rr = match_do_rr; 774 *mpri = m; 775 rc = true; 776 } 777 out: 778 return rc; 779 } 780 781 struct fib6_nh_frl_arg { 782 u32 flags; 783 int oif; 784 int strict; 785 int *mpri; 786 bool *do_rr; 787 struct fib6_nh *nh; 788 }; 789 790 static int rt6_nh_find_match(struct fib6_nh *nh, void *_arg) 791 { 792 struct fib6_nh_frl_arg *arg = _arg; 793 794 arg->nh = nh; 795 return find_match(nh, arg->flags, arg->oif, arg->strict, 796 arg->mpri, arg->do_rr); 797 } 798 799 static void __find_rr_leaf(struct fib6_info *f6i_start, 800 struct fib6_info *nomatch, u32 metric, 801 struct fib6_result *res, struct fib6_info **cont, 802 int oif, int strict, bool *do_rr, int *mpri) 803 { 804 struct fib6_info *f6i; 805 806 for (f6i = f6i_start; 807 f6i && f6i != nomatch; 808 f6i = rcu_dereference(f6i->fib6_next)) { 809 bool matched = false; 810 struct fib6_nh *nh; 811 812 if (cont && f6i->fib6_metric != metric) { 813 *cont = f6i; 814 return; 815 } 816 817 if (fib6_check_expired(f6i)) 818 continue; 819 820 if (unlikely(f6i->nh)) { 821 struct fib6_nh_frl_arg arg = { 822 .flags = f6i->fib6_flags, 823 .oif = oif, 824 .strict = strict, 825 .mpri = mpri, 826 .do_rr = do_rr 827 }; 828 829 if (nexthop_is_blackhole(f6i->nh)) { 830 res->fib6_flags = RTF_REJECT; 831 res->fib6_type = RTN_BLACKHOLE; 832 res->f6i = f6i; 833 res->nh = nexthop_fib6_nh(f6i->nh); 834 return; 835 } 836 if (nexthop_for_each_fib6_nh(f6i->nh, rt6_nh_find_match, 837 &arg)) { 838 matched = true; 839 nh = arg.nh; 840 } 841 } else { 842 nh = f6i->fib6_nh; 843 if (find_match(nh, f6i->fib6_flags, oif, strict, 844 mpri, do_rr)) 845 matched = true; 846 } 847 if (matched) { 848 res->f6i = f6i; 849 res->nh = nh; 850 res->fib6_flags = f6i->fib6_flags; 851 res->fib6_type = f6i->fib6_type; 852 } 853 } 854 } 855 856 static void find_rr_leaf(struct fib6_node *fn, struct fib6_info *leaf, 857 struct fib6_info *rr_head, int oif, int strict, 858 bool *do_rr, struct fib6_result *res) 859 { 860 u32 metric = rr_head->fib6_metric; 861 struct fib6_info *cont = NULL; 862 int mpri = -1; 863 864 __find_rr_leaf(rr_head, NULL, metric, res, &cont, 865 oif, strict, do_rr, &mpri); 866 867 __find_rr_leaf(leaf, rr_head, metric, res, &cont, 868 oif, strict, do_rr, &mpri); 869 870 if (res->f6i || !cont) 871 return; 872 873 __find_rr_leaf(cont, NULL, metric, res, NULL, 874 oif, strict, do_rr, &mpri); 875 } 876 877 static void rt6_select(struct net *net, struct fib6_node *fn, int oif, 878 struct fib6_result *res, int strict) 879 { 880 struct fib6_info *leaf = rcu_dereference(fn->leaf); 881 struct fib6_info *rt0; 882 bool do_rr = false; 883 int key_plen; 884 885 /* make sure this function or its helpers sets f6i */ 886 res->f6i = NULL; 887 888 if (!leaf || leaf == net->ipv6.fib6_null_entry) 889 goto out; 890 891 rt0 = rcu_dereference(fn->rr_ptr); 892 if (!rt0) 893 rt0 = leaf; 894 895 /* Double check to make sure fn is not an intermediate node 896 * and fn->leaf does not points to its child's leaf 897 * (This might happen if all routes under fn are deleted from 898 * the tree and fib6_repair_tree() is called on the node.) 899 */ 900 key_plen = rt0->fib6_dst.plen; 901 #ifdef CONFIG_IPV6_SUBTREES 902 if (rt0->fib6_src.plen) 903 key_plen = rt0->fib6_src.plen; 904 #endif 905 if (fn->fn_bit != key_plen) 906 goto out; 907 908 find_rr_leaf(fn, leaf, rt0, oif, strict, &do_rr, res); 909 if (do_rr) { 910 struct fib6_info *next = rcu_dereference(rt0->fib6_next); 911 912 /* no entries matched; do round-robin */ 913 if (!next || next->fib6_metric != rt0->fib6_metric) 914 next = leaf; 915 916 if (next != rt0) { 917 spin_lock_bh(&leaf->fib6_table->tb6_lock); 918 /* make sure next is not being deleted from the tree */ 919 if (next->fib6_node) 920 rcu_assign_pointer(fn->rr_ptr, next); 921 spin_unlock_bh(&leaf->fib6_table->tb6_lock); 922 } 923 } 924 925 out: 926 if (!res->f6i) { 927 res->f6i = net->ipv6.fib6_null_entry; 928 res->nh = res->f6i->fib6_nh; 929 res->fib6_flags = res->f6i->fib6_flags; 930 res->fib6_type = res->f6i->fib6_type; 931 } 932 } 933 934 static bool rt6_is_gw_or_nonexthop(const struct fib6_result *res) 935 { 936 return (res->f6i->fib6_flags & RTF_NONEXTHOP) || 937 res->nh->fib_nh_gw_family; 938 } 939 940 #ifdef CONFIG_IPV6_ROUTE_INFO 941 int rt6_route_rcv(struct net_device *dev, u8 *opt, int len, 942 const struct in6_addr *gwaddr) 943 { 944 struct net *net = dev_net(dev); 945 struct route_info *rinfo = (struct route_info *) opt; 946 struct in6_addr prefix_buf, *prefix; 947 unsigned int pref; 948 unsigned long lifetime; 949 struct fib6_info *rt; 950 951 if (len < sizeof(struct route_info)) { 952 return -EINVAL; 953 } 954 955 /* Sanity check for prefix_len and length */ 956 if (rinfo->length > 3) { 957 return -EINVAL; 958 } else if (rinfo->prefix_len > 128) { 959 return -EINVAL; 960 } else if (rinfo->prefix_len > 64) { 961 if (rinfo->length < 2) { 962 return -EINVAL; 963 } 964 } else if (rinfo->prefix_len > 0) { 965 if (rinfo->length < 1) { 966 return -EINVAL; 967 } 968 } 969 970 pref = rinfo->route_pref; 971 if (pref == ICMPV6_ROUTER_PREF_INVALID) 972 return -EINVAL; 973 974 lifetime = addrconf_timeout_fixup(ntohl(rinfo->lifetime), HZ); 975 976 if (rinfo->length == 3) 977 prefix = (struct in6_addr *)rinfo->prefix; 978 else { 979 /* this function is safe */ 980 ipv6_addr_prefix(&prefix_buf, 981 (struct in6_addr *)rinfo->prefix, 982 rinfo->prefix_len); 983 prefix = &prefix_buf; 984 } 985 986 if (rinfo->prefix_len == 0) 987 rt = rt6_get_dflt_router(net, gwaddr, dev); 988 else 989 rt = rt6_get_route_info(net, prefix, rinfo->prefix_len, 990 gwaddr, dev); 991 992 if (rt && !lifetime) { 993 ip6_del_rt(net, rt, false); 994 rt = NULL; 995 } 996 997 if (!rt && lifetime) 998 rt = rt6_add_route_info(net, prefix, rinfo->prefix_len, gwaddr, 999 dev, pref); 1000 else if (rt) 1001 rt->fib6_flags = RTF_ROUTEINFO | 1002 (rt->fib6_flags & ~RTF_PREF_MASK) | RTF_PREF(pref); 1003 1004 if (rt) { 1005 if (!addrconf_finite_timeout(lifetime)) 1006 fib6_clean_expires(rt); 1007 else 1008 fib6_set_expires(rt, jiffies + HZ * lifetime); 1009 1010 fib6_info_release(rt); 1011 } 1012 return 0; 1013 } 1014 #endif 1015 1016 /* 1017 * Misc support functions 1018 */ 1019 1020 /* called with rcu_lock held */ 1021 static struct net_device *ip6_rt_get_dev_rcu(const struct fib6_result *res) 1022 { 1023 struct net_device *dev = res->nh->fib_nh_dev; 1024 1025 if (res->fib6_flags & (RTF_LOCAL | RTF_ANYCAST)) { 1026 /* for copies of local routes, dst->dev needs to be the 1027 * device if it is a master device, the master device if 1028 * device is enslaved, and the loopback as the default 1029 */ 1030 if (netif_is_l3_slave(dev) && 1031 !rt6_need_strict(&res->f6i->fib6_dst.addr)) 1032 dev = l3mdev_master_dev_rcu(dev); 1033 else if (!netif_is_l3_master(dev)) 1034 dev = dev_net(dev)->loopback_dev; 1035 /* last case is netif_is_l3_master(dev) is true in which 1036 * case we want dev returned to be dev 1037 */ 1038 } 1039 1040 return dev; 1041 } 1042 1043 static const int fib6_prop[RTN_MAX + 1] = { 1044 [RTN_UNSPEC] = 0, 1045 [RTN_UNICAST] = 0, 1046 [RTN_LOCAL] = 0, 1047 [RTN_BROADCAST] = 0, 1048 [RTN_ANYCAST] = 0, 1049 [RTN_MULTICAST] = 0, 1050 [RTN_BLACKHOLE] = -EINVAL, 1051 [RTN_UNREACHABLE] = -EHOSTUNREACH, 1052 [RTN_PROHIBIT] = -EACCES, 1053 [RTN_THROW] = -EAGAIN, 1054 [RTN_NAT] = -EINVAL, 1055 [RTN_XRESOLVE] = -EINVAL, 1056 }; 1057 1058 static int ip6_rt_type_to_error(u8 fib6_type) 1059 { 1060 return fib6_prop[fib6_type]; 1061 } 1062 1063 static unsigned short fib6_info_dst_flags(struct fib6_info *rt) 1064 { 1065 unsigned short flags = 0; 1066 1067 if (rt->dst_nocount) 1068 flags |= DST_NOCOUNT; 1069 if (rt->dst_nopolicy) 1070 flags |= DST_NOPOLICY; 1071 1072 return flags; 1073 } 1074 1075 static void ip6_rt_init_dst_reject(struct rt6_info *rt, u8 fib6_type) 1076 { 1077 rt->dst.error = ip6_rt_type_to_error(fib6_type); 1078 1079 switch (fib6_type) { 1080 case RTN_BLACKHOLE: 1081 rt->dst.output = dst_discard_out; 1082 rt->dst.input = dst_discard; 1083 break; 1084 case RTN_PROHIBIT: 1085 rt->dst.output = ip6_pkt_prohibit_out; 1086 rt->dst.input = ip6_pkt_prohibit; 1087 break; 1088 case RTN_THROW: 1089 case RTN_UNREACHABLE: 1090 default: 1091 rt->dst.output = ip6_pkt_discard_out; 1092 rt->dst.input = ip6_pkt_discard; 1093 break; 1094 } 1095 } 1096 1097 static void ip6_rt_init_dst(struct rt6_info *rt, const struct fib6_result *res) 1098 { 1099 struct fib6_info *f6i = res->f6i; 1100 1101 if (res->fib6_flags & RTF_REJECT) { 1102 ip6_rt_init_dst_reject(rt, res->fib6_type); 1103 return; 1104 } 1105 1106 rt->dst.error = 0; 1107 rt->dst.output = ip6_output; 1108 1109 if (res->fib6_type == RTN_LOCAL || res->fib6_type == RTN_ANYCAST) { 1110 rt->dst.input = ip6_input; 1111 } else if (ipv6_addr_type(&f6i->fib6_dst.addr) & IPV6_ADDR_MULTICAST) { 1112 rt->dst.input = ip6_mc_input; 1113 } else { 1114 rt->dst.input = ip6_forward; 1115 } 1116 1117 if (res->nh->fib_nh_lws) { 1118 rt->dst.lwtstate = lwtstate_get(res->nh->fib_nh_lws); 1119 lwtunnel_set_redirect(&rt->dst); 1120 } 1121 1122 rt->dst.lastuse = jiffies; 1123 } 1124 1125 /* Caller must already hold reference to @from */ 1126 static void rt6_set_from(struct rt6_info *rt, struct fib6_info *from) 1127 { 1128 rt->rt6i_flags &= ~RTF_EXPIRES; 1129 rcu_assign_pointer(rt->from, from); 1130 ip_dst_init_metrics(&rt->dst, from->fib6_metrics); 1131 } 1132 1133 /* Caller must already hold reference to f6i in result */ 1134 static void ip6_rt_copy_init(struct rt6_info *rt, const struct fib6_result *res) 1135 { 1136 const struct fib6_nh *nh = res->nh; 1137 const struct net_device *dev = nh->fib_nh_dev; 1138 struct fib6_info *f6i = res->f6i; 1139 1140 ip6_rt_init_dst(rt, res); 1141 1142 rt->rt6i_dst = f6i->fib6_dst; 1143 rt->rt6i_idev = dev ? in6_dev_get(dev) : NULL; 1144 rt->rt6i_flags = res->fib6_flags; 1145 if (nh->fib_nh_gw_family) { 1146 rt->rt6i_gateway = nh->fib_nh_gw6; 1147 rt->rt6i_flags |= RTF_GATEWAY; 1148 } 1149 rt6_set_from(rt, f6i); 1150 #ifdef CONFIG_IPV6_SUBTREES 1151 rt->rt6i_src = f6i->fib6_src; 1152 #endif 1153 } 1154 1155 static struct fib6_node* fib6_backtrack(struct fib6_node *fn, 1156 struct in6_addr *saddr) 1157 { 1158 struct fib6_node *pn, *sn; 1159 while (1) { 1160 if (fn->fn_flags & RTN_TL_ROOT) 1161 return NULL; 1162 pn = rcu_dereference(fn->parent); 1163 sn = FIB6_SUBTREE(pn); 1164 if (sn && sn != fn) 1165 fn = fib6_node_lookup(sn, NULL, saddr); 1166 else 1167 fn = pn; 1168 if (fn->fn_flags & RTN_RTINFO) 1169 return fn; 1170 } 1171 } 1172 1173 static bool ip6_hold_safe(struct net *net, struct rt6_info **prt) 1174 { 1175 struct rt6_info *rt = *prt; 1176 1177 if (dst_hold_safe(&rt->dst)) 1178 return true; 1179 if (net) { 1180 rt = net->ipv6.ip6_null_entry; 1181 dst_hold(&rt->dst); 1182 } else { 1183 rt = NULL; 1184 } 1185 *prt = rt; 1186 return false; 1187 } 1188 1189 /* called with rcu_lock held */ 1190 static struct rt6_info *ip6_create_rt_rcu(const struct fib6_result *res) 1191 { 1192 struct net_device *dev = res->nh->fib_nh_dev; 1193 struct fib6_info *f6i = res->f6i; 1194 unsigned short flags; 1195 struct rt6_info *nrt; 1196 1197 if (!fib6_info_hold_safe(f6i)) 1198 goto fallback; 1199 1200 flags = fib6_info_dst_flags(f6i); 1201 nrt = ip6_dst_alloc(dev_net(dev), dev, flags); 1202 if (!nrt) { 1203 fib6_info_release(f6i); 1204 goto fallback; 1205 } 1206 1207 ip6_rt_copy_init(nrt, res); 1208 return nrt; 1209 1210 fallback: 1211 nrt = dev_net(dev)->ipv6.ip6_null_entry; 1212 dst_hold(&nrt->dst); 1213 return nrt; 1214 } 1215 1216 INDIRECT_CALLABLE_SCOPE struct rt6_info *ip6_pol_route_lookup(struct net *net, 1217 struct fib6_table *table, 1218 struct flowi6 *fl6, 1219 const struct sk_buff *skb, 1220 int flags) 1221 { 1222 struct fib6_result res = {}; 1223 struct fib6_node *fn; 1224 struct rt6_info *rt; 1225 1226 if (fl6->flowi6_flags & FLOWI_FLAG_SKIP_NH_OIF) 1227 flags &= ~RT6_LOOKUP_F_IFACE; 1228 1229 rcu_read_lock(); 1230 fn = fib6_node_lookup(&table->tb6_root, &fl6->daddr, &fl6->saddr); 1231 restart: 1232 res.f6i = rcu_dereference(fn->leaf); 1233 if (!res.f6i) 1234 res.f6i = net->ipv6.fib6_null_entry; 1235 else 1236 rt6_device_match(net, &res, &fl6->saddr, fl6->flowi6_oif, 1237 flags); 1238 1239 if (res.f6i == net->ipv6.fib6_null_entry) { 1240 fn = fib6_backtrack(fn, &fl6->saddr); 1241 if (fn) 1242 goto restart; 1243 1244 rt = net->ipv6.ip6_null_entry; 1245 dst_hold(&rt->dst); 1246 goto out; 1247 } else if (res.fib6_flags & RTF_REJECT) { 1248 goto do_create; 1249 } 1250 1251 fib6_select_path(net, &res, fl6, fl6->flowi6_oif, 1252 fl6->flowi6_oif != 0, skb, flags); 1253 1254 /* Search through exception table */ 1255 rt = rt6_find_cached_rt(&res, &fl6->daddr, &fl6->saddr); 1256 if (rt) { 1257 if (ip6_hold_safe(net, &rt)) 1258 dst_use_noref(&rt->dst, jiffies); 1259 } else { 1260 do_create: 1261 rt = ip6_create_rt_rcu(&res); 1262 } 1263 1264 out: 1265 trace_fib6_table_lookup(net, &res, table, fl6); 1266 1267 rcu_read_unlock(); 1268 1269 return rt; 1270 } 1271 1272 struct dst_entry *ip6_route_lookup(struct net *net, struct flowi6 *fl6, 1273 const struct sk_buff *skb, int flags) 1274 { 1275 return fib6_rule_lookup(net, fl6, skb, flags, ip6_pol_route_lookup); 1276 } 1277 EXPORT_SYMBOL_GPL(ip6_route_lookup); 1278 1279 struct rt6_info *rt6_lookup(struct net *net, const struct in6_addr *daddr, 1280 const struct in6_addr *saddr, int oif, 1281 const struct sk_buff *skb, int strict) 1282 { 1283 struct flowi6 fl6 = { 1284 .flowi6_oif = oif, 1285 .daddr = *daddr, 1286 }; 1287 struct dst_entry *dst; 1288 int flags = strict ? RT6_LOOKUP_F_IFACE : 0; 1289 1290 if (saddr) { 1291 memcpy(&fl6.saddr, saddr, sizeof(*saddr)); 1292 flags |= RT6_LOOKUP_F_HAS_SADDR; 1293 } 1294 1295 dst = fib6_rule_lookup(net, &fl6, skb, flags, ip6_pol_route_lookup); 1296 if (dst->error == 0) 1297 return (struct rt6_info *) dst; 1298 1299 dst_release(dst); 1300 1301 return NULL; 1302 } 1303 EXPORT_SYMBOL(rt6_lookup); 1304 1305 /* ip6_ins_rt is called with FREE table->tb6_lock. 1306 * It takes new route entry, the addition fails by any reason the 1307 * route is released. 1308 * Caller must hold dst before calling it. 1309 */ 1310 1311 static int __ip6_ins_rt(struct fib6_info *rt, struct nl_info *info, 1312 struct netlink_ext_ack *extack) 1313 { 1314 int err; 1315 struct fib6_table *table; 1316 1317 table = rt->fib6_table; 1318 spin_lock_bh(&table->tb6_lock); 1319 err = fib6_add(&table->tb6_root, rt, info, extack); 1320 spin_unlock_bh(&table->tb6_lock); 1321 1322 return err; 1323 } 1324 1325 int ip6_ins_rt(struct net *net, struct fib6_info *rt) 1326 { 1327 struct nl_info info = { .nl_net = net, }; 1328 1329 return __ip6_ins_rt(rt, &info, NULL); 1330 } 1331 1332 static struct rt6_info *ip6_rt_cache_alloc(const struct fib6_result *res, 1333 const struct in6_addr *daddr, 1334 const struct in6_addr *saddr) 1335 { 1336 struct fib6_info *f6i = res->f6i; 1337 struct net_device *dev; 1338 struct rt6_info *rt; 1339 1340 /* 1341 * Clone the route. 1342 */ 1343 1344 if (!fib6_info_hold_safe(f6i)) 1345 return NULL; 1346 1347 dev = ip6_rt_get_dev_rcu(res); 1348 rt = ip6_dst_alloc(dev_net(dev), dev, 0); 1349 if (!rt) { 1350 fib6_info_release(f6i); 1351 return NULL; 1352 } 1353 1354 ip6_rt_copy_init(rt, res); 1355 rt->rt6i_flags |= RTF_CACHE; 1356 rt->rt6i_dst.addr = *daddr; 1357 rt->rt6i_dst.plen = 128; 1358 1359 if (!rt6_is_gw_or_nonexthop(res)) { 1360 if (f6i->fib6_dst.plen != 128 && 1361 ipv6_addr_equal(&f6i->fib6_dst.addr, daddr)) 1362 rt->rt6i_flags |= RTF_ANYCAST; 1363 #ifdef CONFIG_IPV6_SUBTREES 1364 if (rt->rt6i_src.plen && saddr) { 1365 rt->rt6i_src.addr = *saddr; 1366 rt->rt6i_src.plen = 128; 1367 } 1368 #endif 1369 } 1370 1371 return rt; 1372 } 1373 1374 static struct rt6_info *ip6_rt_pcpu_alloc(const struct fib6_result *res) 1375 { 1376 struct fib6_info *f6i = res->f6i; 1377 unsigned short flags = fib6_info_dst_flags(f6i); 1378 struct net_device *dev; 1379 struct rt6_info *pcpu_rt; 1380 1381 if (!fib6_info_hold_safe(f6i)) 1382 return NULL; 1383 1384 rcu_read_lock(); 1385 dev = ip6_rt_get_dev_rcu(res); 1386 pcpu_rt = ip6_dst_alloc(dev_net(dev), dev, flags | DST_NOCOUNT); 1387 rcu_read_unlock(); 1388 if (!pcpu_rt) { 1389 fib6_info_release(f6i); 1390 return NULL; 1391 } 1392 ip6_rt_copy_init(pcpu_rt, res); 1393 pcpu_rt->rt6i_flags |= RTF_PCPU; 1394 1395 if (f6i->nh) 1396 pcpu_rt->sernum = rt_genid_ipv6(dev_net(dev)); 1397 1398 return pcpu_rt; 1399 } 1400 1401 static bool rt6_is_valid(const struct rt6_info *rt6) 1402 { 1403 return rt6->sernum == rt_genid_ipv6(dev_net(rt6->dst.dev)); 1404 } 1405 1406 /* It should be called with rcu_read_lock() acquired */ 1407 static struct rt6_info *rt6_get_pcpu_route(const struct fib6_result *res) 1408 { 1409 struct rt6_info *pcpu_rt; 1410 1411 pcpu_rt = this_cpu_read(*res->nh->rt6i_pcpu); 1412 1413 if (pcpu_rt && pcpu_rt->sernum && !rt6_is_valid(pcpu_rt)) { 1414 struct rt6_info *prev, **p; 1415 1416 p = this_cpu_ptr(res->nh->rt6i_pcpu); 1417 prev = xchg(p, NULL); 1418 if (prev) { 1419 dst_dev_put(&prev->dst); 1420 dst_release(&prev->dst); 1421 } 1422 1423 pcpu_rt = NULL; 1424 } 1425 1426 return pcpu_rt; 1427 } 1428 1429 static struct rt6_info *rt6_make_pcpu_route(struct net *net, 1430 const struct fib6_result *res) 1431 { 1432 struct rt6_info *pcpu_rt, *prev, **p; 1433 1434 pcpu_rt = ip6_rt_pcpu_alloc(res); 1435 if (!pcpu_rt) 1436 return NULL; 1437 1438 p = this_cpu_ptr(res->nh->rt6i_pcpu); 1439 prev = cmpxchg(p, NULL, pcpu_rt); 1440 BUG_ON(prev); 1441 1442 if (res->f6i->fib6_destroying) { 1443 struct fib6_info *from; 1444 1445 from = xchg((__force struct fib6_info **)&pcpu_rt->from, NULL); 1446 fib6_info_release(from); 1447 } 1448 1449 return pcpu_rt; 1450 } 1451 1452 /* exception hash table implementation 1453 */ 1454 static DEFINE_SPINLOCK(rt6_exception_lock); 1455 1456 /* Remove rt6_ex from hash table and free the memory 1457 * Caller must hold rt6_exception_lock 1458 */ 1459 static void rt6_remove_exception(struct rt6_exception_bucket *bucket, 1460 struct rt6_exception *rt6_ex) 1461 { 1462 struct fib6_info *from; 1463 struct net *net; 1464 1465 if (!bucket || !rt6_ex) 1466 return; 1467 1468 net = dev_net(rt6_ex->rt6i->dst.dev); 1469 net->ipv6.rt6_stats->fib_rt_cache--; 1470 1471 /* purge completely the exception to allow releasing the held resources: 1472 * some [sk] cache may keep the dst around for unlimited time 1473 */ 1474 from = xchg((__force struct fib6_info **)&rt6_ex->rt6i->from, NULL); 1475 fib6_info_release(from); 1476 dst_dev_put(&rt6_ex->rt6i->dst); 1477 1478 hlist_del_rcu(&rt6_ex->hlist); 1479 dst_release(&rt6_ex->rt6i->dst); 1480 kfree_rcu(rt6_ex, rcu); 1481 WARN_ON_ONCE(!bucket->depth); 1482 bucket->depth--; 1483 } 1484 1485 /* Remove oldest rt6_ex in bucket and free the memory 1486 * Caller must hold rt6_exception_lock 1487 */ 1488 static void rt6_exception_remove_oldest(struct rt6_exception_bucket *bucket) 1489 { 1490 struct rt6_exception *rt6_ex, *oldest = NULL; 1491 1492 if (!bucket) 1493 return; 1494 1495 hlist_for_each_entry(rt6_ex, &bucket->chain, hlist) { 1496 if (!oldest || time_before(rt6_ex->stamp, oldest->stamp)) 1497 oldest = rt6_ex; 1498 } 1499 rt6_remove_exception(bucket, oldest); 1500 } 1501 1502 static u32 rt6_exception_hash(const struct in6_addr *dst, 1503 const struct in6_addr *src) 1504 { 1505 static u32 seed __read_mostly; 1506 u32 val; 1507 1508 net_get_random_once(&seed, sizeof(seed)); 1509 val = jhash2((const u32 *)dst, sizeof(*dst)/sizeof(u32), seed); 1510 1511 #ifdef CONFIG_IPV6_SUBTREES 1512 if (src) 1513 val = jhash2((const u32 *)src, sizeof(*src)/sizeof(u32), val); 1514 #endif 1515 return hash_32(val, FIB6_EXCEPTION_BUCKET_SIZE_SHIFT); 1516 } 1517 1518 /* Helper function to find the cached rt in the hash table 1519 * and update bucket pointer to point to the bucket for this 1520 * (daddr, saddr) pair 1521 * Caller must hold rt6_exception_lock 1522 */ 1523 static struct rt6_exception * 1524 __rt6_find_exception_spinlock(struct rt6_exception_bucket **bucket, 1525 const struct in6_addr *daddr, 1526 const struct in6_addr *saddr) 1527 { 1528 struct rt6_exception *rt6_ex; 1529 u32 hval; 1530 1531 if (!(*bucket) || !daddr) 1532 return NULL; 1533 1534 hval = rt6_exception_hash(daddr, saddr); 1535 *bucket += hval; 1536 1537 hlist_for_each_entry(rt6_ex, &(*bucket)->chain, hlist) { 1538 struct rt6_info *rt6 = rt6_ex->rt6i; 1539 bool matched = ipv6_addr_equal(daddr, &rt6->rt6i_dst.addr); 1540 1541 #ifdef CONFIG_IPV6_SUBTREES 1542 if (matched && saddr) 1543 matched = ipv6_addr_equal(saddr, &rt6->rt6i_src.addr); 1544 #endif 1545 if (matched) 1546 return rt6_ex; 1547 } 1548 return NULL; 1549 } 1550 1551 /* Helper function to find the cached rt in the hash table 1552 * and update bucket pointer to point to the bucket for this 1553 * (daddr, saddr) pair 1554 * Caller must hold rcu_read_lock() 1555 */ 1556 static struct rt6_exception * 1557 __rt6_find_exception_rcu(struct rt6_exception_bucket **bucket, 1558 const struct in6_addr *daddr, 1559 const struct in6_addr *saddr) 1560 { 1561 struct rt6_exception *rt6_ex; 1562 u32 hval; 1563 1564 WARN_ON_ONCE(!rcu_read_lock_held()); 1565 1566 if (!(*bucket) || !daddr) 1567 return NULL; 1568 1569 hval = rt6_exception_hash(daddr, saddr); 1570 *bucket += hval; 1571 1572 hlist_for_each_entry_rcu(rt6_ex, &(*bucket)->chain, hlist) { 1573 struct rt6_info *rt6 = rt6_ex->rt6i; 1574 bool matched = ipv6_addr_equal(daddr, &rt6->rt6i_dst.addr); 1575 1576 #ifdef CONFIG_IPV6_SUBTREES 1577 if (matched && saddr) 1578 matched = ipv6_addr_equal(saddr, &rt6->rt6i_src.addr); 1579 #endif 1580 if (matched) 1581 return rt6_ex; 1582 } 1583 return NULL; 1584 } 1585 1586 static unsigned int fib6_mtu(const struct fib6_result *res) 1587 { 1588 const struct fib6_nh *nh = res->nh; 1589 unsigned int mtu; 1590 1591 if (res->f6i->fib6_pmtu) { 1592 mtu = res->f6i->fib6_pmtu; 1593 } else { 1594 struct net_device *dev = nh->fib_nh_dev; 1595 struct inet6_dev *idev; 1596 1597 rcu_read_lock(); 1598 idev = __in6_dev_get(dev); 1599 mtu = idev->cnf.mtu6; 1600 rcu_read_unlock(); 1601 } 1602 1603 mtu = min_t(unsigned int, mtu, IP6_MAX_MTU); 1604 1605 return mtu - lwtunnel_headroom(nh->fib_nh_lws, mtu); 1606 } 1607 1608 #define FIB6_EXCEPTION_BUCKET_FLUSHED 0x1UL 1609 1610 /* used when the flushed bit is not relevant, only access to the bucket 1611 * (ie., all bucket users except rt6_insert_exception); 1612 * 1613 * called under rcu lock; sometimes called with rt6_exception_lock held 1614 */ 1615 static 1616 struct rt6_exception_bucket *fib6_nh_get_excptn_bucket(const struct fib6_nh *nh, 1617 spinlock_t *lock) 1618 { 1619 struct rt6_exception_bucket *bucket; 1620 1621 if (lock) 1622 bucket = rcu_dereference_protected(nh->rt6i_exception_bucket, 1623 lockdep_is_held(lock)); 1624 else 1625 bucket = rcu_dereference(nh->rt6i_exception_bucket); 1626 1627 /* remove bucket flushed bit if set */ 1628 if (bucket) { 1629 unsigned long p = (unsigned long)bucket; 1630 1631 p &= ~FIB6_EXCEPTION_BUCKET_FLUSHED; 1632 bucket = (struct rt6_exception_bucket *)p; 1633 } 1634 1635 return bucket; 1636 } 1637 1638 static bool fib6_nh_excptn_bucket_flushed(struct rt6_exception_bucket *bucket) 1639 { 1640 unsigned long p = (unsigned long)bucket; 1641 1642 return !!(p & FIB6_EXCEPTION_BUCKET_FLUSHED); 1643 } 1644 1645 /* called with rt6_exception_lock held */ 1646 static void fib6_nh_excptn_bucket_set_flushed(struct fib6_nh *nh, 1647 spinlock_t *lock) 1648 { 1649 struct rt6_exception_bucket *bucket; 1650 unsigned long p; 1651 1652 bucket = rcu_dereference_protected(nh->rt6i_exception_bucket, 1653 lockdep_is_held(lock)); 1654 1655 p = (unsigned long)bucket; 1656 p |= FIB6_EXCEPTION_BUCKET_FLUSHED; 1657 bucket = (struct rt6_exception_bucket *)p; 1658 rcu_assign_pointer(nh->rt6i_exception_bucket, bucket); 1659 } 1660 1661 static int rt6_insert_exception(struct rt6_info *nrt, 1662 const struct fib6_result *res) 1663 { 1664 struct net *net = dev_net(nrt->dst.dev); 1665 struct rt6_exception_bucket *bucket; 1666 struct fib6_info *f6i = res->f6i; 1667 struct in6_addr *src_key = NULL; 1668 struct rt6_exception *rt6_ex; 1669 struct fib6_nh *nh = res->nh; 1670 int err = 0; 1671 1672 spin_lock_bh(&rt6_exception_lock); 1673 1674 bucket = rcu_dereference_protected(nh->rt6i_exception_bucket, 1675 lockdep_is_held(&rt6_exception_lock)); 1676 if (!bucket) { 1677 bucket = kcalloc(FIB6_EXCEPTION_BUCKET_SIZE, sizeof(*bucket), 1678 GFP_ATOMIC); 1679 if (!bucket) { 1680 err = -ENOMEM; 1681 goto out; 1682 } 1683 rcu_assign_pointer(nh->rt6i_exception_bucket, bucket); 1684 } else if (fib6_nh_excptn_bucket_flushed(bucket)) { 1685 err = -EINVAL; 1686 goto out; 1687 } 1688 1689 #ifdef CONFIG_IPV6_SUBTREES 1690 /* fib6_src.plen != 0 indicates f6i is in subtree 1691 * and exception table is indexed by a hash of 1692 * both fib6_dst and fib6_src. 1693 * Otherwise, the exception table is indexed by 1694 * a hash of only fib6_dst. 1695 */ 1696 if (f6i->fib6_src.plen) 1697 src_key = &nrt->rt6i_src.addr; 1698 #endif 1699 /* rt6_mtu_change() might lower mtu on f6i. 1700 * Only insert this exception route if its mtu 1701 * is less than f6i's mtu value. 1702 */ 1703 if (dst_metric_raw(&nrt->dst, RTAX_MTU) >= fib6_mtu(res)) { 1704 err = -EINVAL; 1705 goto out; 1706 } 1707 1708 rt6_ex = __rt6_find_exception_spinlock(&bucket, &nrt->rt6i_dst.addr, 1709 src_key); 1710 if (rt6_ex) 1711 rt6_remove_exception(bucket, rt6_ex); 1712 1713 rt6_ex = kzalloc(sizeof(*rt6_ex), GFP_ATOMIC); 1714 if (!rt6_ex) { 1715 err = -ENOMEM; 1716 goto out; 1717 } 1718 rt6_ex->rt6i = nrt; 1719 rt6_ex->stamp = jiffies; 1720 hlist_add_head_rcu(&rt6_ex->hlist, &bucket->chain); 1721 bucket->depth++; 1722 net->ipv6.rt6_stats->fib_rt_cache++; 1723 1724 if (bucket->depth > FIB6_MAX_DEPTH) 1725 rt6_exception_remove_oldest(bucket); 1726 1727 out: 1728 spin_unlock_bh(&rt6_exception_lock); 1729 1730 /* Update fn->fn_sernum to invalidate all cached dst */ 1731 if (!err) { 1732 spin_lock_bh(&f6i->fib6_table->tb6_lock); 1733 fib6_update_sernum(net, f6i); 1734 spin_unlock_bh(&f6i->fib6_table->tb6_lock); 1735 fib6_force_start_gc(net); 1736 } 1737 1738 return err; 1739 } 1740 1741 static void fib6_nh_flush_exceptions(struct fib6_nh *nh, struct fib6_info *from) 1742 { 1743 struct rt6_exception_bucket *bucket; 1744 struct rt6_exception *rt6_ex; 1745 struct hlist_node *tmp; 1746 int i; 1747 1748 spin_lock_bh(&rt6_exception_lock); 1749 1750 bucket = fib6_nh_get_excptn_bucket(nh, &rt6_exception_lock); 1751 if (!bucket) 1752 goto out; 1753 1754 /* Prevent rt6_insert_exception() to recreate the bucket list */ 1755 if (!from) 1756 fib6_nh_excptn_bucket_set_flushed(nh, &rt6_exception_lock); 1757 1758 for (i = 0; i < FIB6_EXCEPTION_BUCKET_SIZE; i++) { 1759 hlist_for_each_entry_safe(rt6_ex, tmp, &bucket->chain, hlist) { 1760 if (!from || 1761 rcu_access_pointer(rt6_ex->rt6i->from) == from) 1762 rt6_remove_exception(bucket, rt6_ex); 1763 } 1764 WARN_ON_ONCE(!from && bucket->depth); 1765 bucket++; 1766 } 1767 out: 1768 spin_unlock_bh(&rt6_exception_lock); 1769 } 1770 1771 static int rt6_nh_flush_exceptions(struct fib6_nh *nh, void *arg) 1772 { 1773 struct fib6_info *f6i = arg; 1774 1775 fib6_nh_flush_exceptions(nh, f6i); 1776 1777 return 0; 1778 } 1779 1780 void rt6_flush_exceptions(struct fib6_info *f6i) 1781 { 1782 if (f6i->nh) 1783 nexthop_for_each_fib6_nh(f6i->nh, rt6_nh_flush_exceptions, 1784 f6i); 1785 else 1786 fib6_nh_flush_exceptions(f6i->fib6_nh, f6i); 1787 } 1788 1789 /* Find cached rt in the hash table inside passed in rt 1790 * Caller has to hold rcu_read_lock() 1791 */ 1792 static struct rt6_info *rt6_find_cached_rt(const struct fib6_result *res, 1793 const struct in6_addr *daddr, 1794 const struct in6_addr *saddr) 1795 { 1796 const struct in6_addr *src_key = NULL; 1797 struct rt6_exception_bucket *bucket; 1798 struct rt6_exception *rt6_ex; 1799 struct rt6_info *ret = NULL; 1800 1801 #ifdef CONFIG_IPV6_SUBTREES 1802 /* fib6i_src.plen != 0 indicates f6i is in subtree 1803 * and exception table is indexed by a hash of 1804 * both fib6_dst and fib6_src. 1805 * However, the src addr used to create the hash 1806 * might not be exactly the passed in saddr which 1807 * is a /128 addr from the flow. 1808 * So we need to use f6i->fib6_src to redo lookup 1809 * if the passed in saddr does not find anything. 1810 * (See the logic in ip6_rt_cache_alloc() on how 1811 * rt->rt6i_src is updated.) 1812 */ 1813 if (res->f6i->fib6_src.plen) 1814 src_key = saddr; 1815 find_ex: 1816 #endif 1817 bucket = fib6_nh_get_excptn_bucket(res->nh, NULL); 1818 rt6_ex = __rt6_find_exception_rcu(&bucket, daddr, src_key); 1819 1820 if (rt6_ex && !rt6_check_expired(rt6_ex->rt6i)) 1821 ret = rt6_ex->rt6i; 1822 1823 #ifdef CONFIG_IPV6_SUBTREES 1824 /* Use fib6_src as src_key and redo lookup */ 1825 if (!ret && src_key && src_key != &res->f6i->fib6_src.addr) { 1826 src_key = &res->f6i->fib6_src.addr; 1827 goto find_ex; 1828 } 1829 #endif 1830 1831 return ret; 1832 } 1833 1834 /* Remove the passed in cached rt from the hash table that contains it */ 1835 static int fib6_nh_remove_exception(const struct fib6_nh *nh, int plen, 1836 const struct rt6_info *rt) 1837 { 1838 const struct in6_addr *src_key = NULL; 1839 struct rt6_exception_bucket *bucket; 1840 struct rt6_exception *rt6_ex; 1841 int err; 1842 1843 if (!rcu_access_pointer(nh->rt6i_exception_bucket)) 1844 return -ENOENT; 1845 1846 spin_lock_bh(&rt6_exception_lock); 1847 bucket = fib6_nh_get_excptn_bucket(nh, &rt6_exception_lock); 1848 1849 #ifdef CONFIG_IPV6_SUBTREES 1850 /* rt6i_src.plen != 0 indicates 'from' is in subtree 1851 * and exception table is indexed by a hash of 1852 * both rt6i_dst and rt6i_src. 1853 * Otherwise, the exception table is indexed by 1854 * a hash of only rt6i_dst. 1855 */ 1856 if (plen) 1857 src_key = &rt->rt6i_src.addr; 1858 #endif 1859 rt6_ex = __rt6_find_exception_spinlock(&bucket, 1860 &rt->rt6i_dst.addr, 1861 src_key); 1862 if (rt6_ex) { 1863 rt6_remove_exception(bucket, rt6_ex); 1864 err = 0; 1865 } else { 1866 err = -ENOENT; 1867 } 1868 1869 spin_unlock_bh(&rt6_exception_lock); 1870 return err; 1871 } 1872 1873 struct fib6_nh_excptn_arg { 1874 struct rt6_info *rt; 1875 int plen; 1876 }; 1877 1878 static int rt6_nh_remove_exception_rt(struct fib6_nh *nh, void *_arg) 1879 { 1880 struct fib6_nh_excptn_arg *arg = _arg; 1881 int err; 1882 1883 err = fib6_nh_remove_exception(nh, arg->plen, arg->rt); 1884 if (err == 0) 1885 return 1; 1886 1887 return 0; 1888 } 1889 1890 static int rt6_remove_exception_rt(struct rt6_info *rt) 1891 { 1892 struct fib6_info *from; 1893 1894 from = rcu_dereference(rt->from); 1895 if (!from || !(rt->rt6i_flags & RTF_CACHE)) 1896 return -EINVAL; 1897 1898 if (from->nh) { 1899 struct fib6_nh_excptn_arg arg = { 1900 .rt = rt, 1901 .plen = from->fib6_src.plen 1902 }; 1903 int rc; 1904 1905 /* rc = 1 means an entry was found */ 1906 rc = nexthop_for_each_fib6_nh(from->nh, 1907 rt6_nh_remove_exception_rt, 1908 &arg); 1909 return rc ? 0 : -ENOENT; 1910 } 1911 1912 return fib6_nh_remove_exception(from->fib6_nh, 1913 from->fib6_src.plen, rt); 1914 } 1915 1916 /* Find rt6_ex which contains the passed in rt cache and 1917 * refresh its stamp 1918 */ 1919 static void fib6_nh_update_exception(const struct fib6_nh *nh, int plen, 1920 const struct rt6_info *rt) 1921 { 1922 const struct in6_addr *src_key = NULL; 1923 struct rt6_exception_bucket *bucket; 1924 struct rt6_exception *rt6_ex; 1925 1926 bucket = fib6_nh_get_excptn_bucket(nh, NULL); 1927 #ifdef CONFIG_IPV6_SUBTREES 1928 /* rt6i_src.plen != 0 indicates 'from' is in subtree 1929 * and exception table is indexed by a hash of 1930 * both rt6i_dst and rt6i_src. 1931 * Otherwise, the exception table is indexed by 1932 * a hash of only rt6i_dst. 1933 */ 1934 if (plen) 1935 src_key = &rt->rt6i_src.addr; 1936 #endif 1937 rt6_ex = __rt6_find_exception_rcu(&bucket, &rt->rt6i_dst.addr, src_key); 1938 if (rt6_ex) 1939 rt6_ex->stamp = jiffies; 1940 } 1941 1942 struct fib6_nh_match_arg { 1943 const struct net_device *dev; 1944 const struct in6_addr *gw; 1945 struct fib6_nh *match; 1946 }; 1947 1948 /* determine if fib6_nh has given device and gateway */ 1949 static int fib6_nh_find_match(struct fib6_nh *nh, void *_arg) 1950 { 1951 struct fib6_nh_match_arg *arg = _arg; 1952 1953 if (arg->dev != nh->fib_nh_dev || 1954 (arg->gw && !nh->fib_nh_gw_family) || 1955 (!arg->gw && nh->fib_nh_gw_family) || 1956 (arg->gw && !ipv6_addr_equal(arg->gw, &nh->fib_nh_gw6))) 1957 return 0; 1958 1959 arg->match = nh; 1960 1961 /* found a match, break the loop */ 1962 return 1; 1963 } 1964 1965 static void rt6_update_exception_stamp_rt(struct rt6_info *rt) 1966 { 1967 struct fib6_info *from; 1968 struct fib6_nh *fib6_nh; 1969 1970 rcu_read_lock(); 1971 1972 from = rcu_dereference(rt->from); 1973 if (!from || !(rt->rt6i_flags & RTF_CACHE)) 1974 goto unlock; 1975 1976 if (from->nh) { 1977 struct fib6_nh_match_arg arg = { 1978 .dev = rt->dst.dev, 1979 .gw = &rt->rt6i_gateway, 1980 }; 1981 1982 nexthop_for_each_fib6_nh(from->nh, fib6_nh_find_match, &arg); 1983 1984 if (!arg.match) 1985 goto unlock; 1986 fib6_nh = arg.match; 1987 } else { 1988 fib6_nh = from->fib6_nh; 1989 } 1990 fib6_nh_update_exception(fib6_nh, from->fib6_src.plen, rt); 1991 unlock: 1992 rcu_read_unlock(); 1993 } 1994 1995 static bool rt6_mtu_change_route_allowed(struct inet6_dev *idev, 1996 struct rt6_info *rt, int mtu) 1997 { 1998 /* If the new MTU is lower than the route PMTU, this new MTU will be the 1999 * lowest MTU in the path: always allow updating the route PMTU to 2000 * reflect PMTU decreases. 2001 * 2002 * If the new MTU is higher, and the route PMTU is equal to the local 2003 * MTU, this means the old MTU is the lowest in the path, so allow 2004 * updating it: if other nodes now have lower MTUs, PMTU discovery will 2005 * handle this. 2006 */ 2007 2008 if (dst_mtu(&rt->dst) >= mtu) 2009 return true; 2010 2011 if (dst_mtu(&rt->dst) == idev->cnf.mtu6) 2012 return true; 2013 2014 return false; 2015 } 2016 2017 static void rt6_exceptions_update_pmtu(struct inet6_dev *idev, 2018 const struct fib6_nh *nh, int mtu) 2019 { 2020 struct rt6_exception_bucket *bucket; 2021 struct rt6_exception *rt6_ex; 2022 int i; 2023 2024 bucket = fib6_nh_get_excptn_bucket(nh, &rt6_exception_lock); 2025 if (!bucket) 2026 return; 2027 2028 for (i = 0; i < FIB6_EXCEPTION_BUCKET_SIZE; i++) { 2029 hlist_for_each_entry(rt6_ex, &bucket->chain, hlist) { 2030 struct rt6_info *entry = rt6_ex->rt6i; 2031 2032 /* For RTF_CACHE with rt6i_pmtu == 0 (i.e. a redirected 2033 * route), the metrics of its rt->from have already 2034 * been updated. 2035 */ 2036 if (dst_metric_raw(&entry->dst, RTAX_MTU) && 2037 rt6_mtu_change_route_allowed(idev, entry, mtu)) 2038 dst_metric_set(&entry->dst, RTAX_MTU, mtu); 2039 } 2040 bucket++; 2041 } 2042 } 2043 2044 #define RTF_CACHE_GATEWAY (RTF_GATEWAY | RTF_CACHE) 2045 2046 static void fib6_nh_exceptions_clean_tohost(const struct fib6_nh *nh, 2047 const struct in6_addr *gateway) 2048 { 2049 struct rt6_exception_bucket *bucket; 2050 struct rt6_exception *rt6_ex; 2051 struct hlist_node *tmp; 2052 int i; 2053 2054 if (!rcu_access_pointer(nh->rt6i_exception_bucket)) 2055 return; 2056 2057 spin_lock_bh(&rt6_exception_lock); 2058 bucket = fib6_nh_get_excptn_bucket(nh, &rt6_exception_lock); 2059 if (bucket) { 2060 for (i = 0; i < FIB6_EXCEPTION_BUCKET_SIZE; i++) { 2061 hlist_for_each_entry_safe(rt6_ex, tmp, 2062 &bucket->chain, hlist) { 2063 struct rt6_info *entry = rt6_ex->rt6i; 2064 2065 if ((entry->rt6i_flags & RTF_CACHE_GATEWAY) == 2066 RTF_CACHE_GATEWAY && 2067 ipv6_addr_equal(gateway, 2068 &entry->rt6i_gateway)) { 2069 rt6_remove_exception(bucket, rt6_ex); 2070 } 2071 } 2072 bucket++; 2073 } 2074 } 2075 2076 spin_unlock_bh(&rt6_exception_lock); 2077 } 2078 2079 static void rt6_age_examine_exception(struct rt6_exception_bucket *bucket, 2080 struct rt6_exception *rt6_ex, 2081 struct fib6_gc_args *gc_args, 2082 unsigned long now) 2083 { 2084 struct rt6_info *rt = rt6_ex->rt6i; 2085 2086 /* we are pruning and obsoleting aged-out and non gateway exceptions 2087 * even if others have still references to them, so that on next 2088 * dst_check() such references can be dropped. 2089 * EXPIRES exceptions - e.g. pmtu-generated ones are pruned when 2090 * expired, independently from their aging, as per RFC 8201 section 4 2091 */ 2092 if (!(rt->rt6i_flags & RTF_EXPIRES)) { 2093 if (time_after_eq(now, rt->dst.lastuse + gc_args->timeout)) { 2094 RT6_TRACE("aging clone %p\n", rt); 2095 rt6_remove_exception(bucket, rt6_ex); 2096 return; 2097 } 2098 } else if (time_after(jiffies, rt->dst.expires)) { 2099 RT6_TRACE("purging expired route %p\n", rt); 2100 rt6_remove_exception(bucket, rt6_ex); 2101 return; 2102 } 2103 2104 if (rt->rt6i_flags & RTF_GATEWAY) { 2105 struct neighbour *neigh; 2106 __u8 neigh_flags = 0; 2107 2108 neigh = __ipv6_neigh_lookup_noref(rt->dst.dev, &rt->rt6i_gateway); 2109 if (neigh) 2110 neigh_flags = neigh->flags; 2111 2112 if (!(neigh_flags & NTF_ROUTER)) { 2113 RT6_TRACE("purging route %p via non-router but gateway\n", 2114 rt); 2115 rt6_remove_exception(bucket, rt6_ex); 2116 return; 2117 } 2118 } 2119 2120 gc_args->more++; 2121 } 2122 2123 static void fib6_nh_age_exceptions(const struct fib6_nh *nh, 2124 struct fib6_gc_args *gc_args, 2125 unsigned long now) 2126 { 2127 struct rt6_exception_bucket *bucket; 2128 struct rt6_exception *rt6_ex; 2129 struct hlist_node *tmp; 2130 int i; 2131 2132 if (!rcu_access_pointer(nh->rt6i_exception_bucket)) 2133 return; 2134 2135 rcu_read_lock_bh(); 2136 spin_lock(&rt6_exception_lock); 2137 bucket = fib6_nh_get_excptn_bucket(nh, &rt6_exception_lock); 2138 if (bucket) { 2139 for (i = 0; i < FIB6_EXCEPTION_BUCKET_SIZE; i++) { 2140 hlist_for_each_entry_safe(rt6_ex, tmp, 2141 &bucket->chain, hlist) { 2142 rt6_age_examine_exception(bucket, rt6_ex, 2143 gc_args, now); 2144 } 2145 bucket++; 2146 } 2147 } 2148 spin_unlock(&rt6_exception_lock); 2149 rcu_read_unlock_bh(); 2150 } 2151 2152 struct fib6_nh_age_excptn_arg { 2153 struct fib6_gc_args *gc_args; 2154 unsigned long now; 2155 }; 2156 2157 static int rt6_nh_age_exceptions(struct fib6_nh *nh, void *_arg) 2158 { 2159 struct fib6_nh_age_excptn_arg *arg = _arg; 2160 2161 fib6_nh_age_exceptions(nh, arg->gc_args, arg->now); 2162 return 0; 2163 } 2164 2165 void rt6_age_exceptions(struct fib6_info *f6i, 2166 struct fib6_gc_args *gc_args, 2167 unsigned long now) 2168 { 2169 if (f6i->nh) { 2170 struct fib6_nh_age_excptn_arg arg = { 2171 .gc_args = gc_args, 2172 .now = now 2173 }; 2174 2175 nexthop_for_each_fib6_nh(f6i->nh, rt6_nh_age_exceptions, 2176 &arg); 2177 } else { 2178 fib6_nh_age_exceptions(f6i->fib6_nh, gc_args, now); 2179 } 2180 } 2181 2182 /* must be called with rcu lock held */ 2183 int fib6_table_lookup(struct net *net, struct fib6_table *table, int oif, 2184 struct flowi6 *fl6, struct fib6_result *res, int strict) 2185 { 2186 struct fib6_node *fn, *saved_fn; 2187 2188 fn = fib6_node_lookup(&table->tb6_root, &fl6->daddr, &fl6->saddr); 2189 saved_fn = fn; 2190 2191 if (fl6->flowi6_flags & FLOWI_FLAG_SKIP_NH_OIF) 2192 oif = 0; 2193 2194 redo_rt6_select: 2195 rt6_select(net, fn, oif, res, strict); 2196 if (res->f6i == net->ipv6.fib6_null_entry) { 2197 fn = fib6_backtrack(fn, &fl6->saddr); 2198 if (fn) 2199 goto redo_rt6_select; 2200 else if (strict & RT6_LOOKUP_F_REACHABLE) { 2201 /* also consider unreachable route */ 2202 strict &= ~RT6_LOOKUP_F_REACHABLE; 2203 fn = saved_fn; 2204 goto redo_rt6_select; 2205 } 2206 } 2207 2208 trace_fib6_table_lookup(net, res, table, fl6); 2209 2210 return 0; 2211 } 2212 2213 struct rt6_info *ip6_pol_route(struct net *net, struct fib6_table *table, 2214 int oif, struct flowi6 *fl6, 2215 const struct sk_buff *skb, int flags) 2216 { 2217 struct fib6_result res = {}; 2218 struct rt6_info *rt = NULL; 2219 int strict = 0; 2220 2221 WARN_ON_ONCE((flags & RT6_LOOKUP_F_DST_NOREF) && 2222 !rcu_read_lock_held()); 2223 2224 strict |= flags & RT6_LOOKUP_F_IFACE; 2225 strict |= flags & RT6_LOOKUP_F_IGNORE_LINKSTATE; 2226 if (net->ipv6.devconf_all->forwarding == 0) 2227 strict |= RT6_LOOKUP_F_REACHABLE; 2228 2229 rcu_read_lock(); 2230 2231 fib6_table_lookup(net, table, oif, fl6, &res, strict); 2232 if (res.f6i == net->ipv6.fib6_null_entry) 2233 goto out; 2234 2235 fib6_select_path(net, &res, fl6, oif, false, skb, strict); 2236 2237 /*Search through exception table */ 2238 rt = rt6_find_cached_rt(&res, &fl6->daddr, &fl6->saddr); 2239 if (rt) { 2240 goto out; 2241 } else if (unlikely((fl6->flowi6_flags & FLOWI_FLAG_KNOWN_NH) && 2242 !res.nh->fib_nh_gw_family)) { 2243 /* Create a RTF_CACHE clone which will not be 2244 * owned by the fib6 tree. It is for the special case where 2245 * the daddr in the skb during the neighbor look-up is different 2246 * from the fl6->daddr used to look-up route here. 2247 */ 2248 rt = ip6_rt_cache_alloc(&res, &fl6->daddr, NULL); 2249 2250 if (rt) { 2251 /* 1 refcnt is taken during ip6_rt_cache_alloc(). 2252 * As rt6_uncached_list_add() does not consume refcnt, 2253 * this refcnt is always returned to the caller even 2254 * if caller sets RT6_LOOKUP_F_DST_NOREF flag. 2255 */ 2256 rt6_uncached_list_add(rt); 2257 atomic_inc(&net->ipv6.rt6_stats->fib_rt_uncache); 2258 rcu_read_unlock(); 2259 2260 return rt; 2261 } 2262 } else { 2263 /* Get a percpu copy */ 2264 local_bh_disable(); 2265 rt = rt6_get_pcpu_route(&res); 2266 2267 if (!rt) 2268 rt = rt6_make_pcpu_route(net, &res); 2269 2270 local_bh_enable(); 2271 } 2272 out: 2273 if (!rt) 2274 rt = net->ipv6.ip6_null_entry; 2275 if (!(flags & RT6_LOOKUP_F_DST_NOREF)) 2276 ip6_hold_safe(net, &rt); 2277 rcu_read_unlock(); 2278 2279 return rt; 2280 } 2281 EXPORT_SYMBOL_GPL(ip6_pol_route); 2282 2283 INDIRECT_CALLABLE_SCOPE struct rt6_info *ip6_pol_route_input(struct net *net, 2284 struct fib6_table *table, 2285 struct flowi6 *fl6, 2286 const struct sk_buff *skb, 2287 int flags) 2288 { 2289 return ip6_pol_route(net, table, fl6->flowi6_iif, fl6, skb, flags); 2290 } 2291 2292 struct dst_entry *ip6_route_input_lookup(struct net *net, 2293 struct net_device *dev, 2294 struct flowi6 *fl6, 2295 const struct sk_buff *skb, 2296 int flags) 2297 { 2298 if (rt6_need_strict(&fl6->daddr) && dev->type != ARPHRD_PIMREG) 2299 flags |= RT6_LOOKUP_F_IFACE; 2300 2301 return fib6_rule_lookup(net, fl6, skb, flags, ip6_pol_route_input); 2302 } 2303 EXPORT_SYMBOL_GPL(ip6_route_input_lookup); 2304 2305 static void ip6_multipath_l3_keys(const struct sk_buff *skb, 2306 struct flow_keys *keys, 2307 struct flow_keys *flkeys) 2308 { 2309 const struct ipv6hdr *outer_iph = ipv6_hdr(skb); 2310 const struct ipv6hdr *key_iph = outer_iph; 2311 struct flow_keys *_flkeys = flkeys; 2312 const struct ipv6hdr *inner_iph; 2313 const struct icmp6hdr *icmph; 2314 struct ipv6hdr _inner_iph; 2315 struct icmp6hdr _icmph; 2316 2317 if (likely(outer_iph->nexthdr != IPPROTO_ICMPV6)) 2318 goto out; 2319 2320 icmph = skb_header_pointer(skb, skb_transport_offset(skb), 2321 sizeof(_icmph), &_icmph); 2322 if (!icmph) 2323 goto out; 2324 2325 if (!icmpv6_is_err(icmph->icmp6_type)) 2326 goto out; 2327 2328 inner_iph = skb_header_pointer(skb, 2329 skb_transport_offset(skb) + sizeof(*icmph), 2330 sizeof(_inner_iph), &_inner_iph); 2331 if (!inner_iph) 2332 goto out; 2333 2334 key_iph = inner_iph; 2335 _flkeys = NULL; 2336 out: 2337 if (_flkeys) { 2338 keys->addrs.v6addrs.src = _flkeys->addrs.v6addrs.src; 2339 keys->addrs.v6addrs.dst = _flkeys->addrs.v6addrs.dst; 2340 keys->tags.flow_label = _flkeys->tags.flow_label; 2341 keys->basic.ip_proto = _flkeys->basic.ip_proto; 2342 } else { 2343 keys->addrs.v6addrs.src = key_iph->saddr; 2344 keys->addrs.v6addrs.dst = key_iph->daddr; 2345 keys->tags.flow_label = ip6_flowlabel(key_iph); 2346 keys->basic.ip_proto = key_iph->nexthdr; 2347 } 2348 } 2349 2350 /* if skb is set it will be used and fl6 can be NULL */ 2351 u32 rt6_multipath_hash(const struct net *net, const struct flowi6 *fl6, 2352 const struct sk_buff *skb, struct flow_keys *flkeys) 2353 { 2354 struct flow_keys hash_keys; 2355 u32 mhash; 2356 2357 switch (ip6_multipath_hash_policy(net)) { 2358 case 0: 2359 memset(&hash_keys, 0, sizeof(hash_keys)); 2360 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS; 2361 if (skb) { 2362 ip6_multipath_l3_keys(skb, &hash_keys, flkeys); 2363 } else { 2364 hash_keys.addrs.v6addrs.src = fl6->saddr; 2365 hash_keys.addrs.v6addrs.dst = fl6->daddr; 2366 hash_keys.tags.flow_label = (__force u32)flowi6_get_flowlabel(fl6); 2367 hash_keys.basic.ip_proto = fl6->flowi6_proto; 2368 } 2369 break; 2370 case 1: 2371 if (skb) { 2372 unsigned int flag = FLOW_DISSECTOR_F_STOP_AT_ENCAP; 2373 struct flow_keys keys; 2374 2375 /* short-circuit if we already have L4 hash present */ 2376 if (skb->l4_hash) 2377 return skb_get_hash_raw(skb) >> 1; 2378 2379 memset(&hash_keys, 0, sizeof(hash_keys)); 2380 2381 if (!flkeys) { 2382 skb_flow_dissect_flow_keys(skb, &keys, flag); 2383 flkeys = &keys; 2384 } 2385 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS; 2386 hash_keys.addrs.v6addrs.src = flkeys->addrs.v6addrs.src; 2387 hash_keys.addrs.v6addrs.dst = flkeys->addrs.v6addrs.dst; 2388 hash_keys.ports.src = flkeys->ports.src; 2389 hash_keys.ports.dst = flkeys->ports.dst; 2390 hash_keys.basic.ip_proto = flkeys->basic.ip_proto; 2391 } else { 2392 memset(&hash_keys, 0, sizeof(hash_keys)); 2393 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS; 2394 hash_keys.addrs.v6addrs.src = fl6->saddr; 2395 hash_keys.addrs.v6addrs.dst = fl6->daddr; 2396 hash_keys.ports.src = fl6->fl6_sport; 2397 hash_keys.ports.dst = fl6->fl6_dport; 2398 hash_keys.basic.ip_proto = fl6->flowi6_proto; 2399 } 2400 break; 2401 case 2: 2402 memset(&hash_keys, 0, sizeof(hash_keys)); 2403 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS; 2404 if (skb) { 2405 struct flow_keys keys; 2406 2407 if (!flkeys) { 2408 skb_flow_dissect_flow_keys(skb, &keys, 0); 2409 flkeys = &keys; 2410 } 2411 2412 /* Inner can be v4 or v6 */ 2413 if (flkeys->control.addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) { 2414 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS; 2415 hash_keys.addrs.v4addrs.src = flkeys->addrs.v4addrs.src; 2416 hash_keys.addrs.v4addrs.dst = flkeys->addrs.v4addrs.dst; 2417 } else if (flkeys->control.addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) { 2418 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS; 2419 hash_keys.addrs.v6addrs.src = flkeys->addrs.v6addrs.src; 2420 hash_keys.addrs.v6addrs.dst = flkeys->addrs.v6addrs.dst; 2421 hash_keys.tags.flow_label = flkeys->tags.flow_label; 2422 hash_keys.basic.ip_proto = flkeys->basic.ip_proto; 2423 } else { 2424 /* Same as case 0 */ 2425 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS; 2426 ip6_multipath_l3_keys(skb, &hash_keys, flkeys); 2427 } 2428 } else { 2429 /* Same as case 0 */ 2430 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS; 2431 hash_keys.addrs.v6addrs.src = fl6->saddr; 2432 hash_keys.addrs.v6addrs.dst = fl6->daddr; 2433 hash_keys.tags.flow_label = (__force u32)flowi6_get_flowlabel(fl6); 2434 hash_keys.basic.ip_proto = fl6->flowi6_proto; 2435 } 2436 break; 2437 } 2438 mhash = flow_hash_from_keys(&hash_keys); 2439 2440 return mhash >> 1; 2441 } 2442 2443 /* Called with rcu held */ 2444 void ip6_route_input(struct sk_buff *skb) 2445 { 2446 const struct ipv6hdr *iph = ipv6_hdr(skb); 2447 struct net *net = dev_net(skb->dev); 2448 int flags = RT6_LOOKUP_F_HAS_SADDR | RT6_LOOKUP_F_DST_NOREF; 2449 struct ip_tunnel_info *tun_info; 2450 struct flowi6 fl6 = { 2451 .flowi6_iif = skb->dev->ifindex, 2452 .daddr = iph->daddr, 2453 .saddr = iph->saddr, 2454 .flowlabel = ip6_flowinfo(iph), 2455 .flowi6_mark = skb->mark, 2456 .flowi6_proto = iph->nexthdr, 2457 }; 2458 struct flow_keys *flkeys = NULL, _flkeys; 2459 2460 tun_info = skb_tunnel_info(skb); 2461 if (tun_info && !(tun_info->mode & IP_TUNNEL_INFO_TX)) 2462 fl6.flowi6_tun_key.tun_id = tun_info->key.tun_id; 2463 2464 if (fib6_rules_early_flow_dissect(net, skb, &fl6, &_flkeys)) 2465 flkeys = &_flkeys; 2466 2467 if (unlikely(fl6.flowi6_proto == IPPROTO_ICMPV6)) 2468 fl6.mp_hash = rt6_multipath_hash(net, &fl6, skb, flkeys); 2469 skb_dst_drop(skb); 2470 skb_dst_set_noref(skb, ip6_route_input_lookup(net, skb->dev, 2471 &fl6, skb, flags)); 2472 } 2473 2474 INDIRECT_CALLABLE_SCOPE struct rt6_info *ip6_pol_route_output(struct net *net, 2475 struct fib6_table *table, 2476 struct flowi6 *fl6, 2477 const struct sk_buff *skb, 2478 int flags) 2479 { 2480 return ip6_pol_route(net, table, fl6->flowi6_oif, fl6, skb, flags); 2481 } 2482 2483 struct dst_entry *ip6_route_output_flags_noref(struct net *net, 2484 const struct sock *sk, 2485 struct flowi6 *fl6, int flags) 2486 { 2487 bool any_src; 2488 2489 if (ipv6_addr_type(&fl6->daddr) & 2490 (IPV6_ADDR_MULTICAST | IPV6_ADDR_LINKLOCAL)) { 2491 struct dst_entry *dst; 2492 2493 /* This function does not take refcnt on the dst */ 2494 dst = l3mdev_link_scope_lookup(net, fl6); 2495 if (dst) 2496 return dst; 2497 } 2498 2499 fl6->flowi6_iif = LOOPBACK_IFINDEX; 2500 2501 flags |= RT6_LOOKUP_F_DST_NOREF; 2502 any_src = ipv6_addr_any(&fl6->saddr); 2503 if ((sk && sk->sk_bound_dev_if) || rt6_need_strict(&fl6->daddr) || 2504 (fl6->flowi6_oif && any_src)) 2505 flags |= RT6_LOOKUP_F_IFACE; 2506 2507 if (!any_src) 2508 flags |= RT6_LOOKUP_F_HAS_SADDR; 2509 else if (sk) 2510 flags |= rt6_srcprefs2flags(inet6_sk(sk)->srcprefs); 2511 2512 return fib6_rule_lookup(net, fl6, NULL, flags, ip6_pol_route_output); 2513 } 2514 EXPORT_SYMBOL_GPL(ip6_route_output_flags_noref); 2515 2516 struct dst_entry *ip6_route_output_flags(struct net *net, 2517 const struct sock *sk, 2518 struct flowi6 *fl6, 2519 int flags) 2520 { 2521 struct dst_entry *dst; 2522 struct rt6_info *rt6; 2523 2524 rcu_read_lock(); 2525 dst = ip6_route_output_flags_noref(net, sk, fl6, flags); 2526 rt6 = (struct rt6_info *)dst; 2527 /* For dst cached in uncached_list, refcnt is already taken. */ 2528 if (list_empty(&rt6->rt6i_uncached) && !dst_hold_safe(dst)) { 2529 dst = &net->ipv6.ip6_null_entry->dst; 2530 dst_hold(dst); 2531 } 2532 rcu_read_unlock(); 2533 2534 return dst; 2535 } 2536 EXPORT_SYMBOL_GPL(ip6_route_output_flags); 2537 2538 struct dst_entry *ip6_blackhole_route(struct net *net, struct dst_entry *dst_orig) 2539 { 2540 struct rt6_info *rt, *ort = (struct rt6_info *) dst_orig; 2541 struct net_device *loopback_dev = net->loopback_dev; 2542 struct dst_entry *new = NULL; 2543 2544 rt = dst_alloc(&ip6_dst_blackhole_ops, loopback_dev, 1, 2545 DST_OBSOLETE_DEAD, 0); 2546 if (rt) { 2547 rt6_info_init(rt); 2548 atomic_inc(&net->ipv6.rt6_stats->fib_rt_alloc); 2549 2550 new = &rt->dst; 2551 new->__use = 1; 2552 new->input = dst_discard; 2553 new->output = dst_discard_out; 2554 2555 dst_copy_metrics(new, &ort->dst); 2556 2557 rt->rt6i_idev = in6_dev_get(loopback_dev); 2558 rt->rt6i_gateway = ort->rt6i_gateway; 2559 rt->rt6i_flags = ort->rt6i_flags & ~RTF_PCPU; 2560 2561 memcpy(&rt->rt6i_dst, &ort->rt6i_dst, sizeof(struct rt6key)); 2562 #ifdef CONFIG_IPV6_SUBTREES 2563 memcpy(&rt->rt6i_src, &ort->rt6i_src, sizeof(struct rt6key)); 2564 #endif 2565 } 2566 2567 dst_release(dst_orig); 2568 return new ? new : ERR_PTR(-ENOMEM); 2569 } 2570 2571 /* 2572 * Destination cache support functions 2573 */ 2574 2575 static bool fib6_check(struct fib6_info *f6i, u32 cookie) 2576 { 2577 u32 rt_cookie = 0; 2578 2579 if (!fib6_get_cookie_safe(f6i, &rt_cookie) || rt_cookie != cookie) 2580 return false; 2581 2582 if (fib6_check_expired(f6i)) 2583 return false; 2584 2585 return true; 2586 } 2587 2588 static struct dst_entry *rt6_check(struct rt6_info *rt, 2589 struct fib6_info *from, 2590 u32 cookie) 2591 { 2592 u32 rt_cookie = 0; 2593 2594 if (!from || !fib6_get_cookie_safe(from, &rt_cookie) || 2595 rt_cookie != cookie) 2596 return NULL; 2597 2598 if (rt6_check_expired(rt)) 2599 return NULL; 2600 2601 return &rt->dst; 2602 } 2603 2604 static struct dst_entry *rt6_dst_from_check(struct rt6_info *rt, 2605 struct fib6_info *from, 2606 u32 cookie) 2607 { 2608 if (!__rt6_check_expired(rt) && 2609 rt->dst.obsolete == DST_OBSOLETE_FORCE_CHK && 2610 fib6_check(from, cookie)) 2611 return &rt->dst; 2612 else 2613 return NULL; 2614 } 2615 2616 INDIRECT_CALLABLE_SCOPE struct dst_entry *ip6_dst_check(struct dst_entry *dst, 2617 u32 cookie) 2618 { 2619 struct dst_entry *dst_ret; 2620 struct fib6_info *from; 2621 struct rt6_info *rt; 2622 2623 rt = container_of(dst, struct rt6_info, dst); 2624 2625 if (rt->sernum) 2626 return rt6_is_valid(rt) ? dst : NULL; 2627 2628 rcu_read_lock(); 2629 2630 /* All IPV6 dsts are created with ->obsolete set to the value 2631 * DST_OBSOLETE_FORCE_CHK which forces validation calls down 2632 * into this function always. 2633 */ 2634 2635 from = rcu_dereference(rt->from); 2636 2637 if (from && (rt->rt6i_flags & RTF_PCPU || 2638 unlikely(!list_empty(&rt->rt6i_uncached)))) 2639 dst_ret = rt6_dst_from_check(rt, from, cookie); 2640 else 2641 dst_ret = rt6_check(rt, from, cookie); 2642 2643 rcu_read_unlock(); 2644 2645 return dst_ret; 2646 } 2647 EXPORT_INDIRECT_CALLABLE(ip6_dst_check); 2648 2649 static struct dst_entry *ip6_negative_advice(struct dst_entry *dst) 2650 { 2651 struct rt6_info *rt = (struct rt6_info *) dst; 2652 2653 if (rt) { 2654 if (rt->rt6i_flags & RTF_CACHE) { 2655 rcu_read_lock(); 2656 if (rt6_check_expired(rt)) { 2657 rt6_remove_exception_rt(rt); 2658 dst = NULL; 2659 } 2660 rcu_read_unlock(); 2661 } else { 2662 dst_release(dst); 2663 dst = NULL; 2664 } 2665 } 2666 return dst; 2667 } 2668 2669 static void ip6_link_failure(struct sk_buff *skb) 2670 { 2671 struct rt6_info *rt; 2672 2673 icmpv6_send(skb, ICMPV6_DEST_UNREACH, ICMPV6_ADDR_UNREACH, 0); 2674 2675 rt = (struct rt6_info *) skb_dst(skb); 2676 if (rt) { 2677 rcu_read_lock(); 2678 if (rt->rt6i_flags & RTF_CACHE) { 2679 rt6_remove_exception_rt(rt); 2680 } else { 2681 struct fib6_info *from; 2682 struct fib6_node *fn; 2683 2684 from = rcu_dereference(rt->from); 2685 if (from) { 2686 fn = rcu_dereference(from->fib6_node); 2687 if (fn && (rt->rt6i_flags & RTF_DEFAULT)) 2688 fn->fn_sernum = -1; 2689 } 2690 } 2691 rcu_read_unlock(); 2692 } 2693 } 2694 2695 static void rt6_update_expires(struct rt6_info *rt0, int timeout) 2696 { 2697 if (!(rt0->rt6i_flags & RTF_EXPIRES)) { 2698 struct fib6_info *from; 2699 2700 rcu_read_lock(); 2701 from = rcu_dereference(rt0->from); 2702 if (from) 2703 rt0->dst.expires = from->expires; 2704 rcu_read_unlock(); 2705 } 2706 2707 dst_set_expires(&rt0->dst, timeout); 2708 rt0->rt6i_flags |= RTF_EXPIRES; 2709 } 2710 2711 static void rt6_do_update_pmtu(struct rt6_info *rt, u32 mtu) 2712 { 2713 struct net *net = dev_net(rt->dst.dev); 2714 2715 dst_metric_set(&rt->dst, RTAX_MTU, mtu); 2716 rt->rt6i_flags |= RTF_MODIFIED; 2717 rt6_update_expires(rt, net->ipv6.sysctl.ip6_rt_mtu_expires); 2718 } 2719 2720 static bool rt6_cache_allowed_for_pmtu(const struct rt6_info *rt) 2721 { 2722 return !(rt->rt6i_flags & RTF_CACHE) && 2723 (rt->rt6i_flags & RTF_PCPU || rcu_access_pointer(rt->from)); 2724 } 2725 2726 static void __ip6_rt_update_pmtu(struct dst_entry *dst, const struct sock *sk, 2727 const struct ipv6hdr *iph, u32 mtu, 2728 bool confirm_neigh) 2729 { 2730 const struct in6_addr *daddr, *saddr; 2731 struct rt6_info *rt6 = (struct rt6_info *)dst; 2732 2733 /* Note: do *NOT* check dst_metric_locked(dst, RTAX_MTU) 2734 * IPv6 pmtu discovery isn't optional, so 'mtu lock' cannot disable it. 2735 * [see also comment in rt6_mtu_change_route()] 2736 */ 2737 2738 if (iph) { 2739 daddr = &iph->daddr; 2740 saddr = &iph->saddr; 2741 } else if (sk) { 2742 daddr = &sk->sk_v6_daddr; 2743 saddr = &inet6_sk(sk)->saddr; 2744 } else { 2745 daddr = NULL; 2746 saddr = NULL; 2747 } 2748 2749 if (confirm_neigh) 2750 dst_confirm_neigh(dst, daddr); 2751 2752 if (mtu < IPV6_MIN_MTU) 2753 return; 2754 if (mtu >= dst_mtu(dst)) 2755 return; 2756 2757 if (!rt6_cache_allowed_for_pmtu(rt6)) { 2758 rt6_do_update_pmtu(rt6, mtu); 2759 /* update rt6_ex->stamp for cache */ 2760 if (rt6->rt6i_flags & RTF_CACHE) 2761 rt6_update_exception_stamp_rt(rt6); 2762 } else if (daddr) { 2763 struct fib6_result res = {}; 2764 struct rt6_info *nrt6; 2765 2766 rcu_read_lock(); 2767 res.f6i = rcu_dereference(rt6->from); 2768 if (!res.f6i) 2769 goto out_unlock; 2770 2771 res.fib6_flags = res.f6i->fib6_flags; 2772 res.fib6_type = res.f6i->fib6_type; 2773 2774 if (res.f6i->nh) { 2775 struct fib6_nh_match_arg arg = { 2776 .dev = dst->dev, 2777 .gw = &rt6->rt6i_gateway, 2778 }; 2779 2780 nexthop_for_each_fib6_nh(res.f6i->nh, 2781 fib6_nh_find_match, &arg); 2782 2783 /* fib6_info uses a nexthop that does not have fib6_nh 2784 * using the dst->dev + gw. Should be impossible. 2785 */ 2786 if (!arg.match) 2787 goto out_unlock; 2788 2789 res.nh = arg.match; 2790 } else { 2791 res.nh = res.f6i->fib6_nh; 2792 } 2793 2794 nrt6 = ip6_rt_cache_alloc(&res, daddr, saddr); 2795 if (nrt6) { 2796 rt6_do_update_pmtu(nrt6, mtu); 2797 if (rt6_insert_exception(nrt6, &res)) 2798 dst_release_immediate(&nrt6->dst); 2799 } 2800 out_unlock: 2801 rcu_read_unlock(); 2802 } 2803 } 2804 2805 static void ip6_rt_update_pmtu(struct dst_entry *dst, struct sock *sk, 2806 struct sk_buff *skb, u32 mtu, 2807 bool confirm_neigh) 2808 { 2809 __ip6_rt_update_pmtu(dst, sk, skb ? ipv6_hdr(skb) : NULL, mtu, 2810 confirm_neigh); 2811 } 2812 2813 void ip6_update_pmtu(struct sk_buff *skb, struct net *net, __be32 mtu, 2814 int oif, u32 mark, kuid_t uid) 2815 { 2816 const struct ipv6hdr *iph = (struct ipv6hdr *) skb->data; 2817 struct dst_entry *dst; 2818 struct flowi6 fl6 = { 2819 .flowi6_oif = oif, 2820 .flowi6_mark = mark ? mark : IP6_REPLY_MARK(net, skb->mark), 2821 .daddr = iph->daddr, 2822 .saddr = iph->saddr, 2823 .flowlabel = ip6_flowinfo(iph), 2824 .flowi6_uid = uid, 2825 }; 2826 2827 dst = ip6_route_output(net, NULL, &fl6); 2828 if (!dst->error) 2829 __ip6_rt_update_pmtu(dst, NULL, iph, ntohl(mtu), true); 2830 dst_release(dst); 2831 } 2832 EXPORT_SYMBOL_GPL(ip6_update_pmtu); 2833 2834 void ip6_sk_update_pmtu(struct sk_buff *skb, struct sock *sk, __be32 mtu) 2835 { 2836 int oif = sk->sk_bound_dev_if; 2837 struct dst_entry *dst; 2838 2839 if (!oif && skb->dev) 2840 oif = l3mdev_master_ifindex(skb->dev); 2841 2842 ip6_update_pmtu(skb, sock_net(sk), mtu, oif, sk->sk_mark, sk->sk_uid); 2843 2844 dst = __sk_dst_get(sk); 2845 if (!dst || !dst->obsolete || 2846 dst->ops->check(dst, inet6_sk(sk)->dst_cookie)) 2847 return; 2848 2849 bh_lock_sock(sk); 2850 if (!sock_owned_by_user(sk) && !ipv6_addr_v4mapped(&sk->sk_v6_daddr)) 2851 ip6_datagram_dst_update(sk, false); 2852 bh_unlock_sock(sk); 2853 } 2854 EXPORT_SYMBOL_GPL(ip6_sk_update_pmtu); 2855 2856 void ip6_sk_dst_store_flow(struct sock *sk, struct dst_entry *dst, 2857 const struct flowi6 *fl6) 2858 { 2859 #ifdef CONFIG_IPV6_SUBTREES 2860 struct ipv6_pinfo *np = inet6_sk(sk); 2861 #endif 2862 2863 ip6_dst_store(sk, dst, 2864 ipv6_addr_equal(&fl6->daddr, &sk->sk_v6_daddr) ? 2865 &sk->sk_v6_daddr : NULL, 2866 #ifdef CONFIG_IPV6_SUBTREES 2867 ipv6_addr_equal(&fl6->saddr, &np->saddr) ? 2868 &np->saddr : 2869 #endif 2870 NULL); 2871 } 2872 2873 static bool ip6_redirect_nh_match(const struct fib6_result *res, 2874 struct flowi6 *fl6, 2875 const struct in6_addr *gw, 2876 struct rt6_info **ret) 2877 { 2878 const struct fib6_nh *nh = res->nh; 2879 2880 if (nh->fib_nh_flags & RTNH_F_DEAD || !nh->fib_nh_gw_family || 2881 fl6->flowi6_oif != nh->fib_nh_dev->ifindex) 2882 return false; 2883 2884 /* rt_cache's gateway might be different from its 'parent' 2885 * in the case of an ip redirect. 2886 * So we keep searching in the exception table if the gateway 2887 * is different. 2888 */ 2889 if (!ipv6_addr_equal(gw, &nh->fib_nh_gw6)) { 2890 struct rt6_info *rt_cache; 2891 2892 rt_cache = rt6_find_cached_rt(res, &fl6->daddr, &fl6->saddr); 2893 if (rt_cache && 2894 ipv6_addr_equal(gw, &rt_cache->rt6i_gateway)) { 2895 *ret = rt_cache; 2896 return true; 2897 } 2898 return false; 2899 } 2900 return true; 2901 } 2902 2903 struct fib6_nh_rd_arg { 2904 struct fib6_result *res; 2905 struct flowi6 *fl6; 2906 const struct in6_addr *gw; 2907 struct rt6_info **ret; 2908 }; 2909 2910 static int fib6_nh_redirect_match(struct fib6_nh *nh, void *_arg) 2911 { 2912 struct fib6_nh_rd_arg *arg = _arg; 2913 2914 arg->res->nh = nh; 2915 return ip6_redirect_nh_match(arg->res, arg->fl6, arg->gw, arg->ret); 2916 } 2917 2918 /* Handle redirects */ 2919 struct ip6rd_flowi { 2920 struct flowi6 fl6; 2921 struct in6_addr gateway; 2922 }; 2923 2924 INDIRECT_CALLABLE_SCOPE struct rt6_info *__ip6_route_redirect(struct net *net, 2925 struct fib6_table *table, 2926 struct flowi6 *fl6, 2927 const struct sk_buff *skb, 2928 int flags) 2929 { 2930 struct ip6rd_flowi *rdfl = (struct ip6rd_flowi *)fl6; 2931 struct rt6_info *ret = NULL; 2932 struct fib6_result res = {}; 2933 struct fib6_nh_rd_arg arg = { 2934 .res = &res, 2935 .fl6 = fl6, 2936 .gw = &rdfl->gateway, 2937 .ret = &ret 2938 }; 2939 struct fib6_info *rt; 2940 struct fib6_node *fn; 2941 2942 /* l3mdev_update_flow overrides oif if the device is enslaved; in 2943 * this case we must match on the real ingress device, so reset it 2944 */ 2945 if (fl6->flowi6_flags & FLOWI_FLAG_SKIP_NH_OIF) 2946 fl6->flowi6_oif = skb->dev->ifindex; 2947 2948 /* Get the "current" route for this destination and 2949 * check if the redirect has come from appropriate router. 2950 * 2951 * RFC 4861 specifies that redirects should only be 2952 * accepted if they come from the nexthop to the target. 2953 * Due to the way the routes are chosen, this notion 2954 * is a bit fuzzy and one might need to check all possible 2955 * routes. 2956 */ 2957 2958 rcu_read_lock(); 2959 fn = fib6_node_lookup(&table->tb6_root, &fl6->daddr, &fl6->saddr); 2960 restart: 2961 for_each_fib6_node_rt_rcu(fn) { 2962 res.f6i = rt; 2963 if (fib6_check_expired(rt)) 2964 continue; 2965 if (rt->fib6_flags & RTF_REJECT) 2966 break; 2967 if (unlikely(rt->nh)) { 2968 if (nexthop_is_blackhole(rt->nh)) 2969 continue; 2970 /* on match, res->nh is filled in and potentially ret */ 2971 if (nexthop_for_each_fib6_nh(rt->nh, 2972 fib6_nh_redirect_match, 2973 &arg)) 2974 goto out; 2975 } else { 2976 res.nh = rt->fib6_nh; 2977 if (ip6_redirect_nh_match(&res, fl6, &rdfl->gateway, 2978 &ret)) 2979 goto out; 2980 } 2981 } 2982 2983 if (!rt) 2984 rt = net->ipv6.fib6_null_entry; 2985 else if (rt->fib6_flags & RTF_REJECT) { 2986 ret = net->ipv6.ip6_null_entry; 2987 goto out; 2988 } 2989 2990 if (rt == net->ipv6.fib6_null_entry) { 2991 fn = fib6_backtrack(fn, &fl6->saddr); 2992 if (fn) 2993 goto restart; 2994 } 2995 2996 res.f6i = rt; 2997 res.nh = rt->fib6_nh; 2998 out: 2999 if (ret) { 3000 ip6_hold_safe(net, &ret); 3001 } else { 3002 res.fib6_flags = res.f6i->fib6_flags; 3003 res.fib6_type = res.f6i->fib6_type; 3004 ret = ip6_create_rt_rcu(&res); 3005 } 3006 3007 rcu_read_unlock(); 3008 3009 trace_fib6_table_lookup(net, &res, table, fl6); 3010 return ret; 3011 }; 3012 3013 static struct dst_entry *ip6_route_redirect(struct net *net, 3014 const struct flowi6 *fl6, 3015 const struct sk_buff *skb, 3016 const struct in6_addr *gateway) 3017 { 3018 int flags = RT6_LOOKUP_F_HAS_SADDR; 3019 struct ip6rd_flowi rdfl; 3020 3021 rdfl.fl6 = *fl6; 3022 rdfl.gateway = *gateway; 3023 3024 return fib6_rule_lookup(net, &rdfl.fl6, skb, 3025 flags, __ip6_route_redirect); 3026 } 3027 3028 void ip6_redirect(struct sk_buff *skb, struct net *net, int oif, u32 mark, 3029 kuid_t uid) 3030 { 3031 const struct ipv6hdr *iph = (struct ipv6hdr *) skb->data; 3032 struct dst_entry *dst; 3033 struct flowi6 fl6 = { 3034 .flowi6_iif = LOOPBACK_IFINDEX, 3035 .flowi6_oif = oif, 3036 .flowi6_mark = mark, 3037 .daddr = iph->daddr, 3038 .saddr = iph->saddr, 3039 .flowlabel = ip6_flowinfo(iph), 3040 .flowi6_uid = uid, 3041 }; 3042 3043 dst = ip6_route_redirect(net, &fl6, skb, &ipv6_hdr(skb)->saddr); 3044 rt6_do_redirect(dst, NULL, skb); 3045 dst_release(dst); 3046 } 3047 EXPORT_SYMBOL_GPL(ip6_redirect); 3048 3049 void ip6_redirect_no_header(struct sk_buff *skb, struct net *net, int oif) 3050 { 3051 const struct ipv6hdr *iph = ipv6_hdr(skb); 3052 const struct rd_msg *msg = (struct rd_msg *)icmp6_hdr(skb); 3053 struct dst_entry *dst; 3054 struct flowi6 fl6 = { 3055 .flowi6_iif = LOOPBACK_IFINDEX, 3056 .flowi6_oif = oif, 3057 .daddr = msg->dest, 3058 .saddr = iph->daddr, 3059 .flowi6_uid = sock_net_uid(net, NULL), 3060 }; 3061 3062 dst = ip6_route_redirect(net, &fl6, skb, &iph->saddr); 3063 rt6_do_redirect(dst, NULL, skb); 3064 dst_release(dst); 3065 } 3066 3067 void ip6_sk_redirect(struct sk_buff *skb, struct sock *sk) 3068 { 3069 ip6_redirect(skb, sock_net(sk), sk->sk_bound_dev_if, sk->sk_mark, 3070 sk->sk_uid); 3071 } 3072 EXPORT_SYMBOL_GPL(ip6_sk_redirect); 3073 3074 static unsigned int ip6_default_advmss(const struct dst_entry *dst) 3075 { 3076 struct net_device *dev = dst->dev; 3077 unsigned int mtu = dst_mtu(dst); 3078 struct net *net = dev_net(dev); 3079 3080 mtu -= sizeof(struct ipv6hdr) + sizeof(struct tcphdr); 3081 3082 if (mtu < net->ipv6.sysctl.ip6_rt_min_advmss) 3083 mtu = net->ipv6.sysctl.ip6_rt_min_advmss; 3084 3085 /* 3086 * Maximal non-jumbo IPv6 payload is IPV6_MAXPLEN and 3087 * corresponding MSS is IPV6_MAXPLEN - tcp_header_size. 3088 * IPV6_MAXPLEN is also valid and means: "any MSS, 3089 * rely only on pmtu discovery" 3090 */ 3091 if (mtu > IPV6_MAXPLEN - sizeof(struct tcphdr)) 3092 mtu = IPV6_MAXPLEN; 3093 return mtu; 3094 } 3095 3096 INDIRECT_CALLABLE_SCOPE unsigned int ip6_mtu(const struct dst_entry *dst) 3097 { 3098 struct inet6_dev *idev; 3099 unsigned int mtu; 3100 3101 mtu = dst_metric_raw(dst, RTAX_MTU); 3102 if (mtu) 3103 goto out; 3104 3105 mtu = IPV6_MIN_MTU; 3106 3107 rcu_read_lock(); 3108 idev = __in6_dev_get(dst->dev); 3109 if (idev) 3110 mtu = idev->cnf.mtu6; 3111 rcu_read_unlock(); 3112 3113 out: 3114 mtu = min_t(unsigned int, mtu, IP6_MAX_MTU); 3115 3116 return mtu - lwtunnel_headroom(dst->lwtstate, mtu); 3117 } 3118 EXPORT_INDIRECT_CALLABLE(ip6_mtu); 3119 3120 /* MTU selection: 3121 * 1. mtu on route is locked - use it 3122 * 2. mtu from nexthop exception 3123 * 3. mtu from egress device 3124 * 3125 * based on ip6_dst_mtu_forward and exception logic of 3126 * rt6_find_cached_rt; called with rcu_read_lock 3127 */ 3128 u32 ip6_mtu_from_fib6(const struct fib6_result *res, 3129 const struct in6_addr *daddr, 3130 const struct in6_addr *saddr) 3131 { 3132 const struct fib6_nh *nh = res->nh; 3133 struct fib6_info *f6i = res->f6i; 3134 struct inet6_dev *idev; 3135 struct rt6_info *rt; 3136 u32 mtu = 0; 3137 3138 if (unlikely(fib6_metric_locked(f6i, RTAX_MTU))) { 3139 mtu = f6i->fib6_pmtu; 3140 if (mtu) 3141 goto out; 3142 } 3143 3144 rt = rt6_find_cached_rt(res, daddr, saddr); 3145 if (unlikely(rt)) { 3146 mtu = dst_metric_raw(&rt->dst, RTAX_MTU); 3147 } else { 3148 struct net_device *dev = nh->fib_nh_dev; 3149 3150 mtu = IPV6_MIN_MTU; 3151 idev = __in6_dev_get(dev); 3152 if (idev && idev->cnf.mtu6 > mtu) 3153 mtu = idev->cnf.mtu6; 3154 } 3155 3156 mtu = min_t(unsigned int, mtu, IP6_MAX_MTU); 3157 out: 3158 return mtu - lwtunnel_headroom(nh->fib_nh_lws, mtu); 3159 } 3160 3161 struct dst_entry *icmp6_dst_alloc(struct net_device *dev, 3162 struct flowi6 *fl6) 3163 { 3164 struct dst_entry *dst; 3165 struct rt6_info *rt; 3166 struct inet6_dev *idev = in6_dev_get(dev); 3167 struct net *net = dev_net(dev); 3168 3169 if (unlikely(!idev)) 3170 return ERR_PTR(-ENODEV); 3171 3172 rt = ip6_dst_alloc(net, dev, 0); 3173 if (unlikely(!rt)) { 3174 in6_dev_put(idev); 3175 dst = ERR_PTR(-ENOMEM); 3176 goto out; 3177 } 3178 3179 rt->dst.input = ip6_input; 3180 rt->dst.output = ip6_output; 3181 rt->rt6i_gateway = fl6->daddr; 3182 rt->rt6i_dst.addr = fl6->daddr; 3183 rt->rt6i_dst.plen = 128; 3184 rt->rt6i_idev = idev; 3185 dst_metric_set(&rt->dst, RTAX_HOPLIMIT, 0); 3186 3187 /* Add this dst into uncached_list so that rt6_disable_ip() can 3188 * do proper release of the net_device 3189 */ 3190 rt6_uncached_list_add(rt); 3191 atomic_inc(&net->ipv6.rt6_stats->fib_rt_uncache); 3192 3193 dst = xfrm_lookup(net, &rt->dst, flowi6_to_flowi(fl6), NULL, 0); 3194 3195 out: 3196 return dst; 3197 } 3198 3199 static int ip6_dst_gc(struct dst_ops *ops) 3200 { 3201 struct net *net = container_of(ops, struct net, ipv6.ip6_dst_ops); 3202 int rt_min_interval = net->ipv6.sysctl.ip6_rt_gc_min_interval; 3203 int rt_max_size = net->ipv6.sysctl.ip6_rt_max_size; 3204 int rt_elasticity = net->ipv6.sysctl.ip6_rt_gc_elasticity; 3205 int rt_gc_timeout = net->ipv6.sysctl.ip6_rt_gc_timeout; 3206 unsigned long rt_last_gc = net->ipv6.ip6_rt_last_gc; 3207 int entries; 3208 3209 entries = dst_entries_get_fast(ops); 3210 if (entries > rt_max_size) 3211 entries = dst_entries_get_slow(ops); 3212 3213 if (time_after(rt_last_gc + rt_min_interval, jiffies) && 3214 entries <= rt_max_size) 3215 goto out; 3216 3217 net->ipv6.ip6_rt_gc_expire++; 3218 fib6_run_gc(net->ipv6.ip6_rt_gc_expire, net, true); 3219 entries = dst_entries_get_slow(ops); 3220 if (entries < ops->gc_thresh) 3221 net->ipv6.ip6_rt_gc_expire = rt_gc_timeout>>1; 3222 out: 3223 net->ipv6.ip6_rt_gc_expire -= net->ipv6.ip6_rt_gc_expire>>rt_elasticity; 3224 return entries > rt_max_size; 3225 } 3226 3227 static int ip6_nh_lookup_table(struct net *net, struct fib6_config *cfg, 3228 const struct in6_addr *gw_addr, u32 tbid, 3229 int flags, struct fib6_result *res) 3230 { 3231 struct flowi6 fl6 = { 3232 .flowi6_oif = cfg->fc_ifindex, 3233 .daddr = *gw_addr, 3234 .saddr = cfg->fc_prefsrc, 3235 }; 3236 struct fib6_table *table; 3237 int err; 3238 3239 table = fib6_get_table(net, tbid); 3240 if (!table) 3241 return -EINVAL; 3242 3243 if (!ipv6_addr_any(&cfg->fc_prefsrc)) 3244 flags |= RT6_LOOKUP_F_HAS_SADDR; 3245 3246 flags |= RT6_LOOKUP_F_IGNORE_LINKSTATE; 3247 3248 err = fib6_table_lookup(net, table, cfg->fc_ifindex, &fl6, res, flags); 3249 if (!err && res->f6i != net->ipv6.fib6_null_entry) 3250 fib6_select_path(net, res, &fl6, cfg->fc_ifindex, 3251 cfg->fc_ifindex != 0, NULL, flags); 3252 3253 return err; 3254 } 3255 3256 static int ip6_route_check_nh_onlink(struct net *net, 3257 struct fib6_config *cfg, 3258 const struct net_device *dev, 3259 struct netlink_ext_ack *extack) 3260 { 3261 u32 tbid = l3mdev_fib_table_rcu(dev) ? : RT_TABLE_MAIN; 3262 const struct in6_addr *gw_addr = &cfg->fc_gateway; 3263 struct fib6_result res = {}; 3264 int err; 3265 3266 err = ip6_nh_lookup_table(net, cfg, gw_addr, tbid, 0, &res); 3267 if (!err && !(res.fib6_flags & RTF_REJECT) && 3268 /* ignore match if it is the default route */ 3269 !ipv6_addr_any(&res.f6i->fib6_dst.addr) && 3270 (res.fib6_type != RTN_UNICAST || dev != res.nh->fib_nh_dev)) { 3271 NL_SET_ERR_MSG(extack, 3272 "Nexthop has invalid gateway or device mismatch"); 3273 err = -EINVAL; 3274 } 3275 3276 return err; 3277 } 3278 3279 static int ip6_route_check_nh(struct net *net, 3280 struct fib6_config *cfg, 3281 struct net_device **_dev, 3282 struct inet6_dev **idev) 3283 { 3284 const struct in6_addr *gw_addr = &cfg->fc_gateway; 3285 struct net_device *dev = _dev ? *_dev : NULL; 3286 int flags = RT6_LOOKUP_F_IFACE; 3287 struct fib6_result res = {}; 3288 int err = -EHOSTUNREACH; 3289 3290 if (cfg->fc_table) { 3291 err = ip6_nh_lookup_table(net, cfg, gw_addr, 3292 cfg->fc_table, flags, &res); 3293 /* gw_addr can not require a gateway or resolve to a reject 3294 * route. If a device is given, it must match the result. 3295 */ 3296 if (err || res.fib6_flags & RTF_REJECT || 3297 res.nh->fib_nh_gw_family || 3298 (dev && dev != res.nh->fib_nh_dev)) 3299 err = -EHOSTUNREACH; 3300 } 3301 3302 if (err < 0) { 3303 struct flowi6 fl6 = { 3304 .flowi6_oif = cfg->fc_ifindex, 3305 .daddr = *gw_addr, 3306 }; 3307 3308 err = fib6_lookup(net, cfg->fc_ifindex, &fl6, &res, flags); 3309 if (err || res.fib6_flags & RTF_REJECT || 3310 res.nh->fib_nh_gw_family) 3311 err = -EHOSTUNREACH; 3312 3313 if (err) 3314 return err; 3315 3316 fib6_select_path(net, &res, &fl6, cfg->fc_ifindex, 3317 cfg->fc_ifindex != 0, NULL, flags); 3318 } 3319 3320 err = 0; 3321 if (dev) { 3322 if (dev != res.nh->fib_nh_dev) 3323 err = -EHOSTUNREACH; 3324 } else { 3325 *_dev = dev = res.nh->fib_nh_dev; 3326 dev_hold(dev); 3327 *idev = in6_dev_get(dev); 3328 } 3329 3330 return err; 3331 } 3332 3333 static int ip6_validate_gw(struct net *net, struct fib6_config *cfg, 3334 struct net_device **_dev, struct inet6_dev **idev, 3335 struct netlink_ext_ack *extack) 3336 { 3337 const struct in6_addr *gw_addr = &cfg->fc_gateway; 3338 int gwa_type = ipv6_addr_type(gw_addr); 3339 bool skip_dev = gwa_type & IPV6_ADDR_LINKLOCAL ? false : true; 3340 const struct net_device *dev = *_dev; 3341 bool need_addr_check = !dev; 3342 int err = -EINVAL; 3343 3344 /* if gw_addr is local we will fail to detect this in case 3345 * address is still TENTATIVE (DAD in progress). rt6_lookup() 3346 * will return already-added prefix route via interface that 3347 * prefix route was assigned to, which might be non-loopback. 3348 */ 3349 if (dev && 3350 ipv6_chk_addr_and_flags(net, gw_addr, dev, skip_dev, 0, 0)) { 3351 NL_SET_ERR_MSG(extack, "Gateway can not be a local address"); 3352 goto out; 3353 } 3354 3355 if (gwa_type != (IPV6_ADDR_LINKLOCAL | IPV6_ADDR_UNICAST)) { 3356 /* IPv6 strictly inhibits using not link-local 3357 * addresses as nexthop address. 3358 * Otherwise, router will not able to send redirects. 3359 * It is very good, but in some (rare!) circumstances 3360 * (SIT, PtP, NBMA NOARP links) it is handy to allow 3361 * some exceptions. --ANK 3362 * We allow IPv4-mapped nexthops to support RFC4798-type 3363 * addressing 3364 */ 3365 if (!(gwa_type & (IPV6_ADDR_UNICAST | IPV6_ADDR_MAPPED))) { 3366 NL_SET_ERR_MSG(extack, "Invalid gateway address"); 3367 goto out; 3368 } 3369 3370 rcu_read_lock(); 3371 3372 if (cfg->fc_flags & RTNH_F_ONLINK) 3373 err = ip6_route_check_nh_onlink(net, cfg, dev, extack); 3374 else 3375 err = ip6_route_check_nh(net, cfg, _dev, idev); 3376 3377 rcu_read_unlock(); 3378 3379 if (err) 3380 goto out; 3381 } 3382 3383 /* reload in case device was changed */ 3384 dev = *_dev; 3385 3386 err = -EINVAL; 3387 if (!dev) { 3388 NL_SET_ERR_MSG(extack, "Egress device not specified"); 3389 goto out; 3390 } else if (dev->flags & IFF_LOOPBACK) { 3391 NL_SET_ERR_MSG(extack, 3392 "Egress device can not be loopback device for this route"); 3393 goto out; 3394 } 3395 3396 /* if we did not check gw_addr above, do so now that the 3397 * egress device has been resolved. 3398 */ 3399 if (need_addr_check && 3400 ipv6_chk_addr_and_flags(net, gw_addr, dev, skip_dev, 0, 0)) { 3401 NL_SET_ERR_MSG(extack, "Gateway can not be a local address"); 3402 goto out; 3403 } 3404 3405 err = 0; 3406 out: 3407 return err; 3408 } 3409 3410 static bool fib6_is_reject(u32 flags, struct net_device *dev, int addr_type) 3411 { 3412 if ((flags & RTF_REJECT) || 3413 (dev && (dev->flags & IFF_LOOPBACK) && 3414 !(addr_type & IPV6_ADDR_LOOPBACK) && 3415 !(flags & (RTF_ANYCAST | RTF_LOCAL)))) 3416 return true; 3417 3418 return false; 3419 } 3420 3421 int fib6_nh_init(struct net *net, struct fib6_nh *fib6_nh, 3422 struct fib6_config *cfg, gfp_t gfp_flags, 3423 struct netlink_ext_ack *extack) 3424 { 3425 struct net_device *dev = NULL; 3426 struct inet6_dev *idev = NULL; 3427 int addr_type; 3428 int err; 3429 3430 fib6_nh->fib_nh_family = AF_INET6; 3431 #ifdef CONFIG_IPV6_ROUTER_PREF 3432 fib6_nh->last_probe = jiffies; 3433 #endif 3434 if (cfg->fc_is_fdb) { 3435 fib6_nh->fib_nh_gw6 = cfg->fc_gateway; 3436 fib6_nh->fib_nh_gw_family = AF_INET6; 3437 return 0; 3438 } 3439 3440 err = -ENODEV; 3441 if (cfg->fc_ifindex) { 3442 dev = dev_get_by_index(net, cfg->fc_ifindex); 3443 if (!dev) 3444 goto out; 3445 idev = in6_dev_get(dev); 3446 if (!idev) 3447 goto out; 3448 } 3449 3450 if (cfg->fc_flags & RTNH_F_ONLINK) { 3451 if (!dev) { 3452 NL_SET_ERR_MSG(extack, 3453 "Nexthop device required for onlink"); 3454 goto out; 3455 } 3456 3457 if (!(dev->flags & IFF_UP)) { 3458 NL_SET_ERR_MSG(extack, "Nexthop device is not up"); 3459 err = -ENETDOWN; 3460 goto out; 3461 } 3462 3463 fib6_nh->fib_nh_flags |= RTNH_F_ONLINK; 3464 } 3465 3466 fib6_nh->fib_nh_weight = 1; 3467 3468 /* We cannot add true routes via loopback here, 3469 * they would result in kernel looping; promote them to reject routes 3470 */ 3471 addr_type = ipv6_addr_type(&cfg->fc_dst); 3472 if (fib6_is_reject(cfg->fc_flags, dev, addr_type)) { 3473 /* hold loopback dev/idev if we haven't done so. */ 3474 if (dev != net->loopback_dev) { 3475 if (dev) { 3476 dev_put(dev); 3477 in6_dev_put(idev); 3478 } 3479 dev = net->loopback_dev; 3480 dev_hold(dev); 3481 idev = in6_dev_get(dev); 3482 if (!idev) { 3483 err = -ENODEV; 3484 goto out; 3485 } 3486 } 3487 goto pcpu_alloc; 3488 } 3489 3490 if (cfg->fc_flags & RTF_GATEWAY) { 3491 err = ip6_validate_gw(net, cfg, &dev, &idev, extack); 3492 if (err) 3493 goto out; 3494 3495 fib6_nh->fib_nh_gw6 = cfg->fc_gateway; 3496 fib6_nh->fib_nh_gw_family = AF_INET6; 3497 } 3498 3499 err = -ENODEV; 3500 if (!dev) 3501 goto out; 3502 3503 if (idev->cnf.disable_ipv6) { 3504 NL_SET_ERR_MSG(extack, "IPv6 is disabled on nexthop device"); 3505 err = -EACCES; 3506 goto out; 3507 } 3508 3509 if (!(dev->flags & IFF_UP) && !cfg->fc_ignore_dev_down) { 3510 NL_SET_ERR_MSG(extack, "Nexthop device is not up"); 3511 err = -ENETDOWN; 3512 goto out; 3513 } 3514 3515 if (!(cfg->fc_flags & (RTF_LOCAL | RTF_ANYCAST)) && 3516 !netif_carrier_ok(dev)) 3517 fib6_nh->fib_nh_flags |= RTNH_F_LINKDOWN; 3518 3519 err = fib_nh_common_init(net, &fib6_nh->nh_common, cfg->fc_encap, 3520 cfg->fc_encap_type, cfg, gfp_flags, extack); 3521 if (err) 3522 goto out; 3523 3524 pcpu_alloc: 3525 fib6_nh->rt6i_pcpu = alloc_percpu_gfp(struct rt6_info *, gfp_flags); 3526 if (!fib6_nh->rt6i_pcpu) { 3527 err = -ENOMEM; 3528 goto out; 3529 } 3530 3531 fib6_nh->fib_nh_dev = dev; 3532 fib6_nh->fib_nh_oif = dev->ifindex; 3533 err = 0; 3534 out: 3535 if (idev) 3536 in6_dev_put(idev); 3537 3538 if (err) { 3539 lwtstate_put(fib6_nh->fib_nh_lws); 3540 fib6_nh->fib_nh_lws = NULL; 3541 if (dev) 3542 dev_put(dev); 3543 } 3544 3545 return err; 3546 } 3547 3548 void fib6_nh_release(struct fib6_nh *fib6_nh) 3549 { 3550 struct rt6_exception_bucket *bucket; 3551 3552 rcu_read_lock(); 3553 3554 fib6_nh_flush_exceptions(fib6_nh, NULL); 3555 bucket = fib6_nh_get_excptn_bucket(fib6_nh, NULL); 3556 if (bucket) { 3557 rcu_assign_pointer(fib6_nh->rt6i_exception_bucket, NULL); 3558 kfree(bucket); 3559 } 3560 3561 rcu_read_unlock(); 3562 3563 if (fib6_nh->rt6i_pcpu) { 3564 int cpu; 3565 3566 for_each_possible_cpu(cpu) { 3567 struct rt6_info **ppcpu_rt; 3568 struct rt6_info *pcpu_rt; 3569 3570 ppcpu_rt = per_cpu_ptr(fib6_nh->rt6i_pcpu, cpu); 3571 pcpu_rt = *ppcpu_rt; 3572 if (pcpu_rt) { 3573 dst_dev_put(&pcpu_rt->dst); 3574 dst_release(&pcpu_rt->dst); 3575 *ppcpu_rt = NULL; 3576 } 3577 } 3578 3579 free_percpu(fib6_nh->rt6i_pcpu); 3580 } 3581 3582 fib_nh_common_release(&fib6_nh->nh_common); 3583 } 3584 3585 static struct fib6_info *ip6_route_info_create(struct fib6_config *cfg, 3586 gfp_t gfp_flags, 3587 struct netlink_ext_ack *extack) 3588 { 3589 struct net *net = cfg->fc_nlinfo.nl_net; 3590 struct fib6_info *rt = NULL; 3591 struct nexthop *nh = NULL; 3592 struct fib6_table *table; 3593 struct fib6_nh *fib6_nh; 3594 int err = -EINVAL; 3595 int addr_type; 3596 3597 /* RTF_PCPU is an internal flag; can not be set by userspace */ 3598 if (cfg->fc_flags & RTF_PCPU) { 3599 NL_SET_ERR_MSG(extack, "Userspace can not set RTF_PCPU"); 3600 goto out; 3601 } 3602 3603 /* RTF_CACHE is an internal flag; can not be set by userspace */ 3604 if (cfg->fc_flags & RTF_CACHE) { 3605 NL_SET_ERR_MSG(extack, "Userspace can not set RTF_CACHE"); 3606 goto out; 3607 } 3608 3609 if (cfg->fc_type > RTN_MAX) { 3610 NL_SET_ERR_MSG(extack, "Invalid route type"); 3611 goto out; 3612 } 3613 3614 if (cfg->fc_dst_len > 128) { 3615 NL_SET_ERR_MSG(extack, "Invalid prefix length"); 3616 goto out; 3617 } 3618 if (cfg->fc_src_len > 128) { 3619 NL_SET_ERR_MSG(extack, "Invalid source address length"); 3620 goto out; 3621 } 3622 #ifndef CONFIG_IPV6_SUBTREES 3623 if (cfg->fc_src_len) { 3624 NL_SET_ERR_MSG(extack, 3625 "Specifying source address requires IPV6_SUBTREES to be enabled"); 3626 goto out; 3627 } 3628 #endif 3629 if (cfg->fc_nh_id) { 3630 nh = nexthop_find_by_id(net, cfg->fc_nh_id); 3631 if (!nh) { 3632 NL_SET_ERR_MSG(extack, "Nexthop id does not exist"); 3633 goto out; 3634 } 3635 err = fib6_check_nexthop(nh, cfg, extack); 3636 if (err) 3637 goto out; 3638 } 3639 3640 err = -ENOBUFS; 3641 if (cfg->fc_nlinfo.nlh && 3642 !(cfg->fc_nlinfo.nlh->nlmsg_flags & NLM_F_CREATE)) { 3643 table = fib6_get_table(net, cfg->fc_table); 3644 if (!table) { 3645 pr_warn("NLM_F_CREATE should be specified when creating new route\n"); 3646 table = fib6_new_table(net, cfg->fc_table); 3647 } 3648 } else { 3649 table = fib6_new_table(net, cfg->fc_table); 3650 } 3651 3652 if (!table) 3653 goto out; 3654 3655 err = -ENOMEM; 3656 rt = fib6_info_alloc(gfp_flags, !nh); 3657 if (!rt) 3658 goto out; 3659 3660 rt->fib6_metrics = ip_fib_metrics_init(net, cfg->fc_mx, cfg->fc_mx_len, 3661 extack); 3662 if (IS_ERR(rt->fib6_metrics)) { 3663 err = PTR_ERR(rt->fib6_metrics); 3664 /* Do not leave garbage there. */ 3665 rt->fib6_metrics = (struct dst_metrics *)&dst_default_metrics; 3666 goto out; 3667 } 3668 3669 if (cfg->fc_flags & RTF_ADDRCONF) 3670 rt->dst_nocount = true; 3671 3672 if (cfg->fc_flags & RTF_EXPIRES) 3673 fib6_set_expires(rt, jiffies + 3674 clock_t_to_jiffies(cfg->fc_expires)); 3675 else 3676 fib6_clean_expires(rt); 3677 3678 if (cfg->fc_protocol == RTPROT_UNSPEC) 3679 cfg->fc_protocol = RTPROT_BOOT; 3680 rt->fib6_protocol = cfg->fc_protocol; 3681 3682 rt->fib6_table = table; 3683 rt->fib6_metric = cfg->fc_metric; 3684 rt->fib6_type = cfg->fc_type ? : RTN_UNICAST; 3685 rt->fib6_flags = cfg->fc_flags & ~RTF_GATEWAY; 3686 3687 ipv6_addr_prefix(&rt->fib6_dst.addr, &cfg->fc_dst, cfg->fc_dst_len); 3688 rt->fib6_dst.plen = cfg->fc_dst_len; 3689 3690 #ifdef CONFIG_IPV6_SUBTREES 3691 ipv6_addr_prefix(&rt->fib6_src.addr, &cfg->fc_src, cfg->fc_src_len); 3692 rt->fib6_src.plen = cfg->fc_src_len; 3693 #endif 3694 if (nh) { 3695 if (rt->fib6_src.plen) { 3696 NL_SET_ERR_MSG(extack, "Nexthops can not be used with source routing"); 3697 goto out; 3698 } 3699 if (!nexthop_get(nh)) { 3700 NL_SET_ERR_MSG(extack, "Nexthop has been deleted"); 3701 goto out; 3702 } 3703 rt->nh = nh; 3704 fib6_nh = nexthop_fib6_nh(rt->nh); 3705 } else { 3706 err = fib6_nh_init(net, rt->fib6_nh, cfg, gfp_flags, extack); 3707 if (err) 3708 goto out; 3709 3710 fib6_nh = rt->fib6_nh; 3711 3712 /* We cannot add true routes via loopback here, they would 3713 * result in kernel looping; promote them to reject routes 3714 */ 3715 addr_type = ipv6_addr_type(&cfg->fc_dst); 3716 if (fib6_is_reject(cfg->fc_flags, rt->fib6_nh->fib_nh_dev, 3717 addr_type)) 3718 rt->fib6_flags = RTF_REJECT | RTF_NONEXTHOP; 3719 } 3720 3721 if (!ipv6_addr_any(&cfg->fc_prefsrc)) { 3722 struct net_device *dev = fib6_nh->fib_nh_dev; 3723 3724 if (!ipv6_chk_addr(net, &cfg->fc_prefsrc, dev, 0)) { 3725 NL_SET_ERR_MSG(extack, "Invalid source address"); 3726 err = -EINVAL; 3727 goto out; 3728 } 3729 rt->fib6_prefsrc.addr = cfg->fc_prefsrc; 3730 rt->fib6_prefsrc.plen = 128; 3731 } else 3732 rt->fib6_prefsrc.plen = 0; 3733 3734 return rt; 3735 out: 3736 fib6_info_release(rt); 3737 return ERR_PTR(err); 3738 } 3739 3740 int ip6_route_add(struct fib6_config *cfg, gfp_t gfp_flags, 3741 struct netlink_ext_ack *extack) 3742 { 3743 struct fib6_info *rt; 3744 int err; 3745 3746 rt = ip6_route_info_create(cfg, gfp_flags, extack); 3747 if (IS_ERR(rt)) 3748 return PTR_ERR(rt); 3749 3750 err = __ip6_ins_rt(rt, &cfg->fc_nlinfo, extack); 3751 fib6_info_release(rt); 3752 3753 return err; 3754 } 3755 3756 static int __ip6_del_rt(struct fib6_info *rt, struct nl_info *info) 3757 { 3758 struct net *net = info->nl_net; 3759 struct fib6_table *table; 3760 int err; 3761 3762 if (rt == net->ipv6.fib6_null_entry) { 3763 err = -ENOENT; 3764 goto out; 3765 } 3766 3767 table = rt->fib6_table; 3768 spin_lock_bh(&table->tb6_lock); 3769 err = fib6_del(rt, info); 3770 spin_unlock_bh(&table->tb6_lock); 3771 3772 out: 3773 fib6_info_release(rt); 3774 return err; 3775 } 3776 3777 int ip6_del_rt(struct net *net, struct fib6_info *rt, bool skip_notify) 3778 { 3779 struct nl_info info = { 3780 .nl_net = net, 3781 .skip_notify = skip_notify 3782 }; 3783 3784 return __ip6_del_rt(rt, &info); 3785 } 3786 3787 static int __ip6_del_rt_siblings(struct fib6_info *rt, struct fib6_config *cfg) 3788 { 3789 struct nl_info *info = &cfg->fc_nlinfo; 3790 struct net *net = info->nl_net; 3791 struct sk_buff *skb = NULL; 3792 struct fib6_table *table; 3793 int err = -ENOENT; 3794 3795 if (rt == net->ipv6.fib6_null_entry) 3796 goto out_put; 3797 table = rt->fib6_table; 3798 spin_lock_bh(&table->tb6_lock); 3799 3800 if (rt->fib6_nsiblings && cfg->fc_delete_all_nh) { 3801 struct fib6_info *sibling, *next_sibling; 3802 struct fib6_node *fn; 3803 3804 /* prefer to send a single notification with all hops */ 3805 skb = nlmsg_new(rt6_nlmsg_size(rt), gfp_any()); 3806 if (skb) { 3807 u32 seq = info->nlh ? info->nlh->nlmsg_seq : 0; 3808 3809 if (rt6_fill_node(net, skb, rt, NULL, 3810 NULL, NULL, 0, RTM_DELROUTE, 3811 info->portid, seq, 0) < 0) { 3812 kfree_skb(skb); 3813 skb = NULL; 3814 } else 3815 info->skip_notify = 1; 3816 } 3817 3818 /* 'rt' points to the first sibling route. If it is not the 3819 * leaf, then we do not need to send a notification. Otherwise, 3820 * we need to check if the last sibling has a next route or not 3821 * and emit a replace or delete notification, respectively. 3822 */ 3823 info->skip_notify_kernel = 1; 3824 fn = rcu_dereference_protected(rt->fib6_node, 3825 lockdep_is_held(&table->tb6_lock)); 3826 if (rcu_access_pointer(fn->leaf) == rt) { 3827 struct fib6_info *last_sibling, *replace_rt; 3828 3829 last_sibling = list_last_entry(&rt->fib6_siblings, 3830 struct fib6_info, 3831 fib6_siblings); 3832 replace_rt = rcu_dereference_protected( 3833 last_sibling->fib6_next, 3834 lockdep_is_held(&table->tb6_lock)); 3835 if (replace_rt) 3836 call_fib6_entry_notifiers_replace(net, 3837 replace_rt); 3838 else 3839 call_fib6_multipath_entry_notifiers(net, 3840 FIB_EVENT_ENTRY_DEL, 3841 rt, rt->fib6_nsiblings, 3842 NULL); 3843 } 3844 list_for_each_entry_safe(sibling, next_sibling, 3845 &rt->fib6_siblings, 3846 fib6_siblings) { 3847 err = fib6_del(sibling, info); 3848 if (err) 3849 goto out_unlock; 3850 } 3851 } 3852 3853 err = fib6_del(rt, info); 3854 out_unlock: 3855 spin_unlock_bh(&table->tb6_lock); 3856 out_put: 3857 fib6_info_release(rt); 3858 3859 if (skb) { 3860 rtnl_notify(skb, net, info->portid, RTNLGRP_IPV6_ROUTE, 3861 info->nlh, gfp_any()); 3862 } 3863 return err; 3864 } 3865 3866 static int __ip6_del_cached_rt(struct rt6_info *rt, struct fib6_config *cfg) 3867 { 3868 int rc = -ESRCH; 3869 3870 if (cfg->fc_ifindex && rt->dst.dev->ifindex != cfg->fc_ifindex) 3871 goto out; 3872 3873 if (cfg->fc_flags & RTF_GATEWAY && 3874 !ipv6_addr_equal(&cfg->fc_gateway, &rt->rt6i_gateway)) 3875 goto out; 3876 3877 rc = rt6_remove_exception_rt(rt); 3878 out: 3879 return rc; 3880 } 3881 3882 static int ip6_del_cached_rt(struct fib6_config *cfg, struct fib6_info *rt, 3883 struct fib6_nh *nh) 3884 { 3885 struct fib6_result res = { 3886 .f6i = rt, 3887 .nh = nh, 3888 }; 3889 struct rt6_info *rt_cache; 3890 3891 rt_cache = rt6_find_cached_rt(&res, &cfg->fc_dst, &cfg->fc_src); 3892 if (rt_cache) 3893 return __ip6_del_cached_rt(rt_cache, cfg); 3894 3895 return 0; 3896 } 3897 3898 struct fib6_nh_del_cached_rt_arg { 3899 struct fib6_config *cfg; 3900 struct fib6_info *f6i; 3901 }; 3902 3903 static int fib6_nh_del_cached_rt(struct fib6_nh *nh, void *_arg) 3904 { 3905 struct fib6_nh_del_cached_rt_arg *arg = _arg; 3906 int rc; 3907 3908 rc = ip6_del_cached_rt(arg->cfg, arg->f6i, nh); 3909 return rc != -ESRCH ? rc : 0; 3910 } 3911 3912 static int ip6_del_cached_rt_nh(struct fib6_config *cfg, struct fib6_info *f6i) 3913 { 3914 struct fib6_nh_del_cached_rt_arg arg = { 3915 .cfg = cfg, 3916 .f6i = f6i 3917 }; 3918 3919 return nexthop_for_each_fib6_nh(f6i->nh, fib6_nh_del_cached_rt, &arg); 3920 } 3921 3922 static int ip6_route_del(struct fib6_config *cfg, 3923 struct netlink_ext_ack *extack) 3924 { 3925 struct fib6_table *table; 3926 struct fib6_info *rt; 3927 struct fib6_node *fn; 3928 int err = -ESRCH; 3929 3930 table = fib6_get_table(cfg->fc_nlinfo.nl_net, cfg->fc_table); 3931 if (!table) { 3932 NL_SET_ERR_MSG(extack, "FIB table does not exist"); 3933 return err; 3934 } 3935 3936 rcu_read_lock(); 3937 3938 fn = fib6_locate(&table->tb6_root, 3939 &cfg->fc_dst, cfg->fc_dst_len, 3940 &cfg->fc_src, cfg->fc_src_len, 3941 !(cfg->fc_flags & RTF_CACHE)); 3942 3943 if (fn) { 3944 for_each_fib6_node_rt_rcu(fn) { 3945 struct fib6_nh *nh; 3946 3947 if (rt->nh && cfg->fc_nh_id && 3948 rt->nh->id != cfg->fc_nh_id) 3949 continue; 3950 3951 if (cfg->fc_flags & RTF_CACHE) { 3952 int rc = 0; 3953 3954 if (rt->nh) { 3955 rc = ip6_del_cached_rt_nh(cfg, rt); 3956 } else if (cfg->fc_nh_id) { 3957 continue; 3958 } else { 3959 nh = rt->fib6_nh; 3960 rc = ip6_del_cached_rt(cfg, rt, nh); 3961 } 3962 if (rc != -ESRCH) { 3963 rcu_read_unlock(); 3964 return rc; 3965 } 3966 continue; 3967 } 3968 3969 if (cfg->fc_metric && cfg->fc_metric != rt->fib6_metric) 3970 continue; 3971 if (cfg->fc_protocol && 3972 cfg->fc_protocol != rt->fib6_protocol) 3973 continue; 3974 3975 if (rt->nh) { 3976 if (!fib6_info_hold_safe(rt)) 3977 continue; 3978 rcu_read_unlock(); 3979 3980 return __ip6_del_rt(rt, &cfg->fc_nlinfo); 3981 } 3982 if (cfg->fc_nh_id) 3983 continue; 3984 3985 nh = rt->fib6_nh; 3986 if (cfg->fc_ifindex && 3987 (!nh->fib_nh_dev || 3988 nh->fib_nh_dev->ifindex != cfg->fc_ifindex)) 3989 continue; 3990 if (cfg->fc_flags & RTF_GATEWAY && 3991 !ipv6_addr_equal(&cfg->fc_gateway, &nh->fib_nh_gw6)) 3992 continue; 3993 if (!fib6_info_hold_safe(rt)) 3994 continue; 3995 rcu_read_unlock(); 3996 3997 /* if gateway was specified only delete the one hop */ 3998 if (cfg->fc_flags & RTF_GATEWAY) 3999 return __ip6_del_rt(rt, &cfg->fc_nlinfo); 4000 4001 return __ip6_del_rt_siblings(rt, cfg); 4002 } 4003 } 4004 rcu_read_unlock(); 4005 4006 return err; 4007 } 4008 4009 static void rt6_do_redirect(struct dst_entry *dst, struct sock *sk, struct sk_buff *skb) 4010 { 4011 struct netevent_redirect netevent; 4012 struct rt6_info *rt, *nrt = NULL; 4013 struct fib6_result res = {}; 4014 struct ndisc_options ndopts; 4015 struct inet6_dev *in6_dev; 4016 struct neighbour *neigh; 4017 struct rd_msg *msg; 4018 int optlen, on_link; 4019 u8 *lladdr; 4020 4021 optlen = skb_tail_pointer(skb) - skb_transport_header(skb); 4022 optlen -= sizeof(*msg); 4023 4024 if (optlen < 0) { 4025 net_dbg_ratelimited("rt6_do_redirect: packet too short\n"); 4026 return; 4027 } 4028 4029 msg = (struct rd_msg *)icmp6_hdr(skb); 4030 4031 if (ipv6_addr_is_multicast(&msg->dest)) { 4032 net_dbg_ratelimited("rt6_do_redirect: destination address is multicast\n"); 4033 return; 4034 } 4035 4036 on_link = 0; 4037 if (ipv6_addr_equal(&msg->dest, &msg->target)) { 4038 on_link = 1; 4039 } else if (ipv6_addr_type(&msg->target) != 4040 (IPV6_ADDR_UNICAST|IPV6_ADDR_LINKLOCAL)) { 4041 net_dbg_ratelimited("rt6_do_redirect: target address is not link-local unicast\n"); 4042 return; 4043 } 4044 4045 in6_dev = __in6_dev_get(skb->dev); 4046 if (!in6_dev) 4047 return; 4048 if (in6_dev->cnf.forwarding || !in6_dev->cnf.accept_redirects) 4049 return; 4050 4051 /* RFC2461 8.1: 4052 * The IP source address of the Redirect MUST be the same as the current 4053 * first-hop router for the specified ICMP Destination Address. 4054 */ 4055 4056 if (!ndisc_parse_options(skb->dev, msg->opt, optlen, &ndopts)) { 4057 net_dbg_ratelimited("rt6_redirect: invalid ND options\n"); 4058 return; 4059 } 4060 4061 lladdr = NULL; 4062 if (ndopts.nd_opts_tgt_lladdr) { 4063 lladdr = ndisc_opt_addr_data(ndopts.nd_opts_tgt_lladdr, 4064 skb->dev); 4065 if (!lladdr) { 4066 net_dbg_ratelimited("rt6_redirect: invalid link-layer address length\n"); 4067 return; 4068 } 4069 } 4070 4071 rt = (struct rt6_info *) dst; 4072 if (rt->rt6i_flags & RTF_REJECT) { 4073 net_dbg_ratelimited("rt6_redirect: source isn't a valid nexthop for redirect target\n"); 4074 return; 4075 } 4076 4077 /* Redirect received -> path was valid. 4078 * Look, redirects are sent only in response to data packets, 4079 * so that this nexthop apparently is reachable. --ANK 4080 */ 4081 dst_confirm_neigh(&rt->dst, &ipv6_hdr(skb)->saddr); 4082 4083 neigh = __neigh_lookup(&nd_tbl, &msg->target, skb->dev, 1); 4084 if (!neigh) 4085 return; 4086 4087 /* 4088 * We have finally decided to accept it. 4089 */ 4090 4091 ndisc_update(skb->dev, neigh, lladdr, NUD_STALE, 4092 NEIGH_UPDATE_F_WEAK_OVERRIDE| 4093 NEIGH_UPDATE_F_OVERRIDE| 4094 (on_link ? 0 : (NEIGH_UPDATE_F_OVERRIDE_ISROUTER| 4095 NEIGH_UPDATE_F_ISROUTER)), 4096 NDISC_REDIRECT, &ndopts); 4097 4098 rcu_read_lock(); 4099 res.f6i = rcu_dereference(rt->from); 4100 if (!res.f6i) 4101 goto out; 4102 4103 if (res.f6i->nh) { 4104 struct fib6_nh_match_arg arg = { 4105 .dev = dst->dev, 4106 .gw = &rt->rt6i_gateway, 4107 }; 4108 4109 nexthop_for_each_fib6_nh(res.f6i->nh, 4110 fib6_nh_find_match, &arg); 4111 4112 /* fib6_info uses a nexthop that does not have fib6_nh 4113 * using the dst->dev. Should be impossible 4114 */ 4115 if (!arg.match) 4116 goto out; 4117 res.nh = arg.match; 4118 } else { 4119 res.nh = res.f6i->fib6_nh; 4120 } 4121 4122 res.fib6_flags = res.f6i->fib6_flags; 4123 res.fib6_type = res.f6i->fib6_type; 4124 nrt = ip6_rt_cache_alloc(&res, &msg->dest, NULL); 4125 if (!nrt) 4126 goto out; 4127 4128 nrt->rt6i_flags = RTF_GATEWAY|RTF_UP|RTF_DYNAMIC|RTF_CACHE; 4129 if (on_link) 4130 nrt->rt6i_flags &= ~RTF_GATEWAY; 4131 4132 nrt->rt6i_gateway = *(struct in6_addr *)neigh->primary_key; 4133 4134 /* rt6_insert_exception() will take care of duplicated exceptions */ 4135 if (rt6_insert_exception(nrt, &res)) { 4136 dst_release_immediate(&nrt->dst); 4137 goto out; 4138 } 4139 4140 netevent.old = &rt->dst; 4141 netevent.new = &nrt->dst; 4142 netevent.daddr = &msg->dest; 4143 netevent.neigh = neigh; 4144 call_netevent_notifiers(NETEVENT_REDIRECT, &netevent); 4145 4146 out: 4147 rcu_read_unlock(); 4148 neigh_release(neigh); 4149 } 4150 4151 #ifdef CONFIG_IPV6_ROUTE_INFO 4152 static struct fib6_info *rt6_get_route_info(struct net *net, 4153 const struct in6_addr *prefix, int prefixlen, 4154 const struct in6_addr *gwaddr, 4155 struct net_device *dev) 4156 { 4157 u32 tb_id = l3mdev_fib_table(dev) ? : RT6_TABLE_INFO; 4158 int ifindex = dev->ifindex; 4159 struct fib6_node *fn; 4160 struct fib6_info *rt = NULL; 4161 struct fib6_table *table; 4162 4163 table = fib6_get_table(net, tb_id); 4164 if (!table) 4165 return NULL; 4166 4167 rcu_read_lock(); 4168 fn = fib6_locate(&table->tb6_root, prefix, prefixlen, NULL, 0, true); 4169 if (!fn) 4170 goto out; 4171 4172 for_each_fib6_node_rt_rcu(fn) { 4173 /* these routes do not use nexthops */ 4174 if (rt->nh) 4175 continue; 4176 if (rt->fib6_nh->fib_nh_dev->ifindex != ifindex) 4177 continue; 4178 if (!(rt->fib6_flags & RTF_ROUTEINFO) || 4179 !rt->fib6_nh->fib_nh_gw_family) 4180 continue; 4181 if (!ipv6_addr_equal(&rt->fib6_nh->fib_nh_gw6, gwaddr)) 4182 continue; 4183 if (!fib6_info_hold_safe(rt)) 4184 continue; 4185 break; 4186 } 4187 out: 4188 rcu_read_unlock(); 4189 return rt; 4190 } 4191 4192 static struct fib6_info *rt6_add_route_info(struct net *net, 4193 const struct in6_addr *prefix, int prefixlen, 4194 const struct in6_addr *gwaddr, 4195 struct net_device *dev, 4196 unsigned int pref) 4197 { 4198 struct fib6_config cfg = { 4199 .fc_metric = IP6_RT_PRIO_USER, 4200 .fc_ifindex = dev->ifindex, 4201 .fc_dst_len = prefixlen, 4202 .fc_flags = RTF_GATEWAY | RTF_ADDRCONF | RTF_ROUTEINFO | 4203 RTF_UP | RTF_PREF(pref), 4204 .fc_protocol = RTPROT_RA, 4205 .fc_type = RTN_UNICAST, 4206 .fc_nlinfo.portid = 0, 4207 .fc_nlinfo.nlh = NULL, 4208 .fc_nlinfo.nl_net = net, 4209 }; 4210 4211 cfg.fc_table = l3mdev_fib_table(dev) ? : RT6_TABLE_INFO; 4212 cfg.fc_dst = *prefix; 4213 cfg.fc_gateway = *gwaddr; 4214 4215 /* We should treat it as a default route if prefix length is 0. */ 4216 if (!prefixlen) 4217 cfg.fc_flags |= RTF_DEFAULT; 4218 4219 ip6_route_add(&cfg, GFP_ATOMIC, NULL); 4220 4221 return rt6_get_route_info(net, prefix, prefixlen, gwaddr, dev); 4222 } 4223 #endif 4224 4225 struct fib6_info *rt6_get_dflt_router(struct net *net, 4226 const struct in6_addr *addr, 4227 struct net_device *dev) 4228 { 4229 u32 tb_id = l3mdev_fib_table(dev) ? : RT6_TABLE_DFLT; 4230 struct fib6_info *rt; 4231 struct fib6_table *table; 4232 4233 table = fib6_get_table(net, tb_id); 4234 if (!table) 4235 return NULL; 4236 4237 rcu_read_lock(); 4238 for_each_fib6_node_rt_rcu(&table->tb6_root) { 4239 struct fib6_nh *nh; 4240 4241 /* RA routes do not use nexthops */ 4242 if (rt->nh) 4243 continue; 4244 4245 nh = rt->fib6_nh; 4246 if (dev == nh->fib_nh_dev && 4247 ((rt->fib6_flags & (RTF_ADDRCONF | RTF_DEFAULT)) == (RTF_ADDRCONF | RTF_DEFAULT)) && 4248 ipv6_addr_equal(&nh->fib_nh_gw6, addr)) 4249 break; 4250 } 4251 if (rt && !fib6_info_hold_safe(rt)) 4252 rt = NULL; 4253 rcu_read_unlock(); 4254 return rt; 4255 } 4256 4257 struct fib6_info *rt6_add_dflt_router(struct net *net, 4258 const struct in6_addr *gwaddr, 4259 struct net_device *dev, 4260 unsigned int pref, 4261 u32 defrtr_usr_metric) 4262 { 4263 struct fib6_config cfg = { 4264 .fc_table = l3mdev_fib_table(dev) ? : RT6_TABLE_DFLT, 4265 .fc_metric = defrtr_usr_metric, 4266 .fc_ifindex = dev->ifindex, 4267 .fc_flags = RTF_GATEWAY | RTF_ADDRCONF | RTF_DEFAULT | 4268 RTF_UP | RTF_EXPIRES | RTF_PREF(pref), 4269 .fc_protocol = RTPROT_RA, 4270 .fc_type = RTN_UNICAST, 4271 .fc_nlinfo.portid = 0, 4272 .fc_nlinfo.nlh = NULL, 4273 .fc_nlinfo.nl_net = net, 4274 }; 4275 4276 cfg.fc_gateway = *gwaddr; 4277 4278 if (!ip6_route_add(&cfg, GFP_ATOMIC, NULL)) { 4279 struct fib6_table *table; 4280 4281 table = fib6_get_table(dev_net(dev), cfg.fc_table); 4282 if (table) 4283 table->flags |= RT6_TABLE_HAS_DFLT_ROUTER; 4284 } 4285 4286 return rt6_get_dflt_router(net, gwaddr, dev); 4287 } 4288 4289 static void __rt6_purge_dflt_routers(struct net *net, 4290 struct fib6_table *table) 4291 { 4292 struct fib6_info *rt; 4293 4294 restart: 4295 rcu_read_lock(); 4296 for_each_fib6_node_rt_rcu(&table->tb6_root) { 4297 struct net_device *dev = fib6_info_nh_dev(rt); 4298 struct inet6_dev *idev = dev ? __in6_dev_get(dev) : NULL; 4299 4300 if (rt->fib6_flags & (RTF_DEFAULT | RTF_ADDRCONF) && 4301 (!idev || idev->cnf.accept_ra != 2) && 4302 fib6_info_hold_safe(rt)) { 4303 rcu_read_unlock(); 4304 ip6_del_rt(net, rt, false); 4305 goto restart; 4306 } 4307 } 4308 rcu_read_unlock(); 4309 4310 table->flags &= ~RT6_TABLE_HAS_DFLT_ROUTER; 4311 } 4312 4313 void rt6_purge_dflt_routers(struct net *net) 4314 { 4315 struct fib6_table *table; 4316 struct hlist_head *head; 4317 unsigned int h; 4318 4319 rcu_read_lock(); 4320 4321 for (h = 0; h < FIB6_TABLE_HASHSZ; h++) { 4322 head = &net->ipv6.fib_table_hash[h]; 4323 hlist_for_each_entry_rcu(table, head, tb6_hlist) { 4324 if (table->flags & RT6_TABLE_HAS_DFLT_ROUTER) 4325 __rt6_purge_dflt_routers(net, table); 4326 } 4327 } 4328 4329 rcu_read_unlock(); 4330 } 4331 4332 static void rtmsg_to_fib6_config(struct net *net, 4333 struct in6_rtmsg *rtmsg, 4334 struct fib6_config *cfg) 4335 { 4336 *cfg = (struct fib6_config){ 4337 .fc_table = l3mdev_fib_table_by_index(net, rtmsg->rtmsg_ifindex) ? 4338 : RT6_TABLE_MAIN, 4339 .fc_ifindex = rtmsg->rtmsg_ifindex, 4340 .fc_metric = rtmsg->rtmsg_metric ? : IP6_RT_PRIO_USER, 4341 .fc_expires = rtmsg->rtmsg_info, 4342 .fc_dst_len = rtmsg->rtmsg_dst_len, 4343 .fc_src_len = rtmsg->rtmsg_src_len, 4344 .fc_flags = rtmsg->rtmsg_flags, 4345 .fc_type = rtmsg->rtmsg_type, 4346 4347 .fc_nlinfo.nl_net = net, 4348 4349 .fc_dst = rtmsg->rtmsg_dst, 4350 .fc_src = rtmsg->rtmsg_src, 4351 .fc_gateway = rtmsg->rtmsg_gateway, 4352 }; 4353 } 4354 4355 int ipv6_route_ioctl(struct net *net, unsigned int cmd, struct in6_rtmsg *rtmsg) 4356 { 4357 struct fib6_config cfg; 4358 int err; 4359 4360 if (cmd != SIOCADDRT && cmd != SIOCDELRT) 4361 return -EINVAL; 4362 if (!ns_capable(net->user_ns, CAP_NET_ADMIN)) 4363 return -EPERM; 4364 4365 rtmsg_to_fib6_config(net, rtmsg, &cfg); 4366 4367 rtnl_lock(); 4368 switch (cmd) { 4369 case SIOCADDRT: 4370 err = ip6_route_add(&cfg, GFP_KERNEL, NULL); 4371 break; 4372 case SIOCDELRT: 4373 err = ip6_route_del(&cfg, NULL); 4374 break; 4375 } 4376 rtnl_unlock(); 4377 return err; 4378 } 4379 4380 /* 4381 * Drop the packet on the floor 4382 */ 4383 4384 static int ip6_pkt_drop(struct sk_buff *skb, u8 code, int ipstats_mib_noroutes) 4385 { 4386 struct dst_entry *dst = skb_dst(skb); 4387 struct net *net = dev_net(dst->dev); 4388 struct inet6_dev *idev; 4389 int type; 4390 4391 if (netif_is_l3_master(skb->dev) && 4392 dst->dev == net->loopback_dev) 4393 idev = __in6_dev_get_safely(dev_get_by_index_rcu(net, IP6CB(skb)->iif)); 4394 else 4395 idev = ip6_dst_idev(dst); 4396 4397 switch (ipstats_mib_noroutes) { 4398 case IPSTATS_MIB_INNOROUTES: 4399 type = ipv6_addr_type(&ipv6_hdr(skb)->daddr); 4400 if (type == IPV6_ADDR_ANY) { 4401 IP6_INC_STATS(net, idev, IPSTATS_MIB_INADDRERRORS); 4402 break; 4403 } 4404 fallthrough; 4405 case IPSTATS_MIB_OUTNOROUTES: 4406 IP6_INC_STATS(net, idev, ipstats_mib_noroutes); 4407 break; 4408 } 4409 4410 /* Start over by dropping the dst for l3mdev case */ 4411 if (netif_is_l3_master(skb->dev)) 4412 skb_dst_drop(skb); 4413 4414 icmpv6_send(skb, ICMPV6_DEST_UNREACH, code, 0); 4415 kfree_skb(skb); 4416 return 0; 4417 } 4418 4419 static int ip6_pkt_discard(struct sk_buff *skb) 4420 { 4421 return ip6_pkt_drop(skb, ICMPV6_NOROUTE, IPSTATS_MIB_INNOROUTES); 4422 } 4423 4424 static int ip6_pkt_discard_out(struct net *net, struct sock *sk, struct sk_buff *skb) 4425 { 4426 skb->dev = skb_dst(skb)->dev; 4427 return ip6_pkt_drop(skb, ICMPV6_NOROUTE, IPSTATS_MIB_OUTNOROUTES); 4428 } 4429 4430 static int ip6_pkt_prohibit(struct sk_buff *skb) 4431 { 4432 return ip6_pkt_drop(skb, ICMPV6_ADM_PROHIBITED, IPSTATS_MIB_INNOROUTES); 4433 } 4434 4435 static int ip6_pkt_prohibit_out(struct net *net, struct sock *sk, struct sk_buff *skb) 4436 { 4437 skb->dev = skb_dst(skb)->dev; 4438 return ip6_pkt_drop(skb, ICMPV6_ADM_PROHIBITED, IPSTATS_MIB_OUTNOROUTES); 4439 } 4440 4441 /* 4442 * Allocate a dst for local (unicast / anycast) address. 4443 */ 4444 4445 struct fib6_info *addrconf_f6i_alloc(struct net *net, 4446 struct inet6_dev *idev, 4447 const struct in6_addr *addr, 4448 bool anycast, gfp_t gfp_flags) 4449 { 4450 struct fib6_config cfg = { 4451 .fc_table = l3mdev_fib_table(idev->dev) ? : RT6_TABLE_LOCAL, 4452 .fc_ifindex = idev->dev->ifindex, 4453 .fc_flags = RTF_UP | RTF_NONEXTHOP, 4454 .fc_dst = *addr, 4455 .fc_dst_len = 128, 4456 .fc_protocol = RTPROT_KERNEL, 4457 .fc_nlinfo.nl_net = net, 4458 .fc_ignore_dev_down = true, 4459 }; 4460 struct fib6_info *f6i; 4461 4462 if (anycast) { 4463 cfg.fc_type = RTN_ANYCAST; 4464 cfg.fc_flags |= RTF_ANYCAST; 4465 } else { 4466 cfg.fc_type = RTN_LOCAL; 4467 cfg.fc_flags |= RTF_LOCAL; 4468 } 4469 4470 f6i = ip6_route_info_create(&cfg, gfp_flags, NULL); 4471 if (!IS_ERR(f6i)) 4472 f6i->dst_nocount = true; 4473 return f6i; 4474 } 4475 4476 /* remove deleted ip from prefsrc entries */ 4477 struct arg_dev_net_ip { 4478 struct net_device *dev; 4479 struct net *net; 4480 struct in6_addr *addr; 4481 }; 4482 4483 static int fib6_remove_prefsrc(struct fib6_info *rt, void *arg) 4484 { 4485 struct net_device *dev = ((struct arg_dev_net_ip *)arg)->dev; 4486 struct net *net = ((struct arg_dev_net_ip *)arg)->net; 4487 struct in6_addr *addr = ((struct arg_dev_net_ip *)arg)->addr; 4488 4489 if (!rt->nh && 4490 ((void *)rt->fib6_nh->fib_nh_dev == dev || !dev) && 4491 rt != net->ipv6.fib6_null_entry && 4492 ipv6_addr_equal(addr, &rt->fib6_prefsrc.addr)) { 4493 spin_lock_bh(&rt6_exception_lock); 4494 /* remove prefsrc entry */ 4495 rt->fib6_prefsrc.plen = 0; 4496 spin_unlock_bh(&rt6_exception_lock); 4497 } 4498 return 0; 4499 } 4500 4501 void rt6_remove_prefsrc(struct inet6_ifaddr *ifp) 4502 { 4503 struct net *net = dev_net(ifp->idev->dev); 4504 struct arg_dev_net_ip adni = { 4505 .dev = ifp->idev->dev, 4506 .net = net, 4507 .addr = &ifp->addr, 4508 }; 4509 fib6_clean_all(net, fib6_remove_prefsrc, &adni); 4510 } 4511 4512 #define RTF_RA_ROUTER (RTF_ADDRCONF | RTF_DEFAULT) 4513 4514 /* Remove routers and update dst entries when gateway turn into host. */ 4515 static int fib6_clean_tohost(struct fib6_info *rt, void *arg) 4516 { 4517 struct in6_addr *gateway = (struct in6_addr *)arg; 4518 struct fib6_nh *nh; 4519 4520 /* RA routes do not use nexthops */ 4521 if (rt->nh) 4522 return 0; 4523 4524 nh = rt->fib6_nh; 4525 if (((rt->fib6_flags & RTF_RA_ROUTER) == RTF_RA_ROUTER) && 4526 nh->fib_nh_gw_family && ipv6_addr_equal(gateway, &nh->fib_nh_gw6)) 4527 return -1; 4528 4529 /* Further clean up cached routes in exception table. 4530 * This is needed because cached route may have a different 4531 * gateway than its 'parent' in the case of an ip redirect. 4532 */ 4533 fib6_nh_exceptions_clean_tohost(nh, gateway); 4534 4535 return 0; 4536 } 4537 4538 void rt6_clean_tohost(struct net *net, struct in6_addr *gateway) 4539 { 4540 fib6_clean_all(net, fib6_clean_tohost, gateway); 4541 } 4542 4543 struct arg_netdev_event { 4544 const struct net_device *dev; 4545 union { 4546 unsigned char nh_flags; 4547 unsigned long event; 4548 }; 4549 }; 4550 4551 static struct fib6_info *rt6_multipath_first_sibling(const struct fib6_info *rt) 4552 { 4553 struct fib6_info *iter; 4554 struct fib6_node *fn; 4555 4556 fn = rcu_dereference_protected(rt->fib6_node, 4557 lockdep_is_held(&rt->fib6_table->tb6_lock)); 4558 iter = rcu_dereference_protected(fn->leaf, 4559 lockdep_is_held(&rt->fib6_table->tb6_lock)); 4560 while (iter) { 4561 if (iter->fib6_metric == rt->fib6_metric && 4562 rt6_qualify_for_ecmp(iter)) 4563 return iter; 4564 iter = rcu_dereference_protected(iter->fib6_next, 4565 lockdep_is_held(&rt->fib6_table->tb6_lock)); 4566 } 4567 4568 return NULL; 4569 } 4570 4571 /* only called for fib entries with builtin fib6_nh */ 4572 static bool rt6_is_dead(const struct fib6_info *rt) 4573 { 4574 if (rt->fib6_nh->fib_nh_flags & RTNH_F_DEAD || 4575 (rt->fib6_nh->fib_nh_flags & RTNH_F_LINKDOWN && 4576 ip6_ignore_linkdown(rt->fib6_nh->fib_nh_dev))) 4577 return true; 4578 4579 return false; 4580 } 4581 4582 static int rt6_multipath_total_weight(const struct fib6_info *rt) 4583 { 4584 struct fib6_info *iter; 4585 int total = 0; 4586 4587 if (!rt6_is_dead(rt)) 4588 total += rt->fib6_nh->fib_nh_weight; 4589 4590 list_for_each_entry(iter, &rt->fib6_siblings, fib6_siblings) { 4591 if (!rt6_is_dead(iter)) 4592 total += iter->fib6_nh->fib_nh_weight; 4593 } 4594 4595 return total; 4596 } 4597 4598 static void rt6_upper_bound_set(struct fib6_info *rt, int *weight, int total) 4599 { 4600 int upper_bound = -1; 4601 4602 if (!rt6_is_dead(rt)) { 4603 *weight += rt->fib6_nh->fib_nh_weight; 4604 upper_bound = DIV_ROUND_CLOSEST_ULL((u64) (*weight) << 31, 4605 total) - 1; 4606 } 4607 atomic_set(&rt->fib6_nh->fib_nh_upper_bound, upper_bound); 4608 } 4609 4610 static void rt6_multipath_upper_bound_set(struct fib6_info *rt, int total) 4611 { 4612 struct fib6_info *iter; 4613 int weight = 0; 4614 4615 rt6_upper_bound_set(rt, &weight, total); 4616 4617 list_for_each_entry(iter, &rt->fib6_siblings, fib6_siblings) 4618 rt6_upper_bound_set(iter, &weight, total); 4619 } 4620 4621 void rt6_multipath_rebalance(struct fib6_info *rt) 4622 { 4623 struct fib6_info *first; 4624 int total; 4625 4626 /* In case the entire multipath route was marked for flushing, 4627 * then there is no need to rebalance upon the removal of every 4628 * sibling route. 4629 */ 4630 if (!rt->fib6_nsiblings || rt->should_flush) 4631 return; 4632 4633 /* During lookup routes are evaluated in order, so we need to 4634 * make sure upper bounds are assigned from the first sibling 4635 * onwards. 4636 */ 4637 first = rt6_multipath_first_sibling(rt); 4638 if (WARN_ON_ONCE(!first)) 4639 return; 4640 4641 total = rt6_multipath_total_weight(first); 4642 rt6_multipath_upper_bound_set(first, total); 4643 } 4644 4645 static int fib6_ifup(struct fib6_info *rt, void *p_arg) 4646 { 4647 const struct arg_netdev_event *arg = p_arg; 4648 struct net *net = dev_net(arg->dev); 4649 4650 if (rt != net->ipv6.fib6_null_entry && !rt->nh && 4651 rt->fib6_nh->fib_nh_dev == arg->dev) { 4652 rt->fib6_nh->fib_nh_flags &= ~arg->nh_flags; 4653 fib6_update_sernum_upto_root(net, rt); 4654 rt6_multipath_rebalance(rt); 4655 } 4656 4657 return 0; 4658 } 4659 4660 void rt6_sync_up(struct net_device *dev, unsigned char nh_flags) 4661 { 4662 struct arg_netdev_event arg = { 4663 .dev = dev, 4664 { 4665 .nh_flags = nh_flags, 4666 }, 4667 }; 4668 4669 if (nh_flags & RTNH_F_DEAD && netif_carrier_ok(dev)) 4670 arg.nh_flags |= RTNH_F_LINKDOWN; 4671 4672 fib6_clean_all(dev_net(dev), fib6_ifup, &arg); 4673 } 4674 4675 /* only called for fib entries with inline fib6_nh */ 4676 static bool rt6_multipath_uses_dev(const struct fib6_info *rt, 4677 const struct net_device *dev) 4678 { 4679 struct fib6_info *iter; 4680 4681 if (rt->fib6_nh->fib_nh_dev == dev) 4682 return true; 4683 list_for_each_entry(iter, &rt->fib6_siblings, fib6_siblings) 4684 if (iter->fib6_nh->fib_nh_dev == dev) 4685 return true; 4686 4687 return false; 4688 } 4689 4690 static void rt6_multipath_flush(struct fib6_info *rt) 4691 { 4692 struct fib6_info *iter; 4693 4694 rt->should_flush = 1; 4695 list_for_each_entry(iter, &rt->fib6_siblings, fib6_siblings) 4696 iter->should_flush = 1; 4697 } 4698 4699 static unsigned int rt6_multipath_dead_count(const struct fib6_info *rt, 4700 const struct net_device *down_dev) 4701 { 4702 struct fib6_info *iter; 4703 unsigned int dead = 0; 4704 4705 if (rt->fib6_nh->fib_nh_dev == down_dev || 4706 rt->fib6_nh->fib_nh_flags & RTNH_F_DEAD) 4707 dead++; 4708 list_for_each_entry(iter, &rt->fib6_siblings, fib6_siblings) 4709 if (iter->fib6_nh->fib_nh_dev == down_dev || 4710 iter->fib6_nh->fib_nh_flags & RTNH_F_DEAD) 4711 dead++; 4712 4713 return dead; 4714 } 4715 4716 static void rt6_multipath_nh_flags_set(struct fib6_info *rt, 4717 const struct net_device *dev, 4718 unsigned char nh_flags) 4719 { 4720 struct fib6_info *iter; 4721 4722 if (rt->fib6_nh->fib_nh_dev == dev) 4723 rt->fib6_nh->fib_nh_flags |= nh_flags; 4724 list_for_each_entry(iter, &rt->fib6_siblings, fib6_siblings) 4725 if (iter->fib6_nh->fib_nh_dev == dev) 4726 iter->fib6_nh->fib_nh_flags |= nh_flags; 4727 } 4728 4729 /* called with write lock held for table with rt */ 4730 static int fib6_ifdown(struct fib6_info *rt, void *p_arg) 4731 { 4732 const struct arg_netdev_event *arg = p_arg; 4733 const struct net_device *dev = arg->dev; 4734 struct net *net = dev_net(dev); 4735 4736 if (rt == net->ipv6.fib6_null_entry || rt->nh) 4737 return 0; 4738 4739 switch (arg->event) { 4740 case NETDEV_UNREGISTER: 4741 return rt->fib6_nh->fib_nh_dev == dev ? -1 : 0; 4742 case NETDEV_DOWN: 4743 if (rt->should_flush) 4744 return -1; 4745 if (!rt->fib6_nsiblings) 4746 return rt->fib6_nh->fib_nh_dev == dev ? -1 : 0; 4747 if (rt6_multipath_uses_dev(rt, dev)) { 4748 unsigned int count; 4749 4750 count = rt6_multipath_dead_count(rt, dev); 4751 if (rt->fib6_nsiblings + 1 == count) { 4752 rt6_multipath_flush(rt); 4753 return -1; 4754 } 4755 rt6_multipath_nh_flags_set(rt, dev, RTNH_F_DEAD | 4756 RTNH_F_LINKDOWN); 4757 fib6_update_sernum(net, rt); 4758 rt6_multipath_rebalance(rt); 4759 } 4760 return -2; 4761 case NETDEV_CHANGE: 4762 if (rt->fib6_nh->fib_nh_dev != dev || 4763 rt->fib6_flags & (RTF_LOCAL | RTF_ANYCAST)) 4764 break; 4765 rt->fib6_nh->fib_nh_flags |= RTNH_F_LINKDOWN; 4766 rt6_multipath_rebalance(rt); 4767 break; 4768 } 4769 4770 return 0; 4771 } 4772 4773 void rt6_sync_down_dev(struct net_device *dev, unsigned long event) 4774 { 4775 struct arg_netdev_event arg = { 4776 .dev = dev, 4777 { 4778 .event = event, 4779 }, 4780 }; 4781 struct net *net = dev_net(dev); 4782 4783 if (net->ipv6.sysctl.skip_notify_on_dev_down) 4784 fib6_clean_all_skip_notify(net, fib6_ifdown, &arg); 4785 else 4786 fib6_clean_all(net, fib6_ifdown, &arg); 4787 } 4788 4789 void rt6_disable_ip(struct net_device *dev, unsigned long event) 4790 { 4791 rt6_sync_down_dev(dev, event); 4792 rt6_uncached_list_flush_dev(dev_net(dev), dev); 4793 neigh_ifdown(&nd_tbl, dev); 4794 } 4795 4796 struct rt6_mtu_change_arg { 4797 struct net_device *dev; 4798 unsigned int mtu; 4799 struct fib6_info *f6i; 4800 }; 4801 4802 static int fib6_nh_mtu_change(struct fib6_nh *nh, void *_arg) 4803 { 4804 struct rt6_mtu_change_arg *arg = (struct rt6_mtu_change_arg *)_arg; 4805 struct fib6_info *f6i = arg->f6i; 4806 4807 /* For administrative MTU increase, there is no way to discover 4808 * IPv6 PMTU increase, so PMTU increase should be updated here. 4809 * Since RFC 1981 doesn't include administrative MTU increase 4810 * update PMTU increase is a MUST. (i.e. jumbo frame) 4811 */ 4812 if (nh->fib_nh_dev == arg->dev) { 4813 struct inet6_dev *idev = __in6_dev_get(arg->dev); 4814 u32 mtu = f6i->fib6_pmtu; 4815 4816 if (mtu >= arg->mtu || 4817 (mtu < arg->mtu && mtu == idev->cnf.mtu6)) 4818 fib6_metric_set(f6i, RTAX_MTU, arg->mtu); 4819 4820 spin_lock_bh(&rt6_exception_lock); 4821 rt6_exceptions_update_pmtu(idev, nh, arg->mtu); 4822 spin_unlock_bh(&rt6_exception_lock); 4823 } 4824 4825 return 0; 4826 } 4827 4828 static int rt6_mtu_change_route(struct fib6_info *f6i, void *p_arg) 4829 { 4830 struct rt6_mtu_change_arg *arg = (struct rt6_mtu_change_arg *) p_arg; 4831 struct inet6_dev *idev; 4832 4833 /* In IPv6 pmtu discovery is not optional, 4834 so that RTAX_MTU lock cannot disable it. 4835 We still use this lock to block changes 4836 caused by addrconf/ndisc. 4837 */ 4838 4839 idev = __in6_dev_get(arg->dev); 4840 if (!idev) 4841 return 0; 4842 4843 if (fib6_metric_locked(f6i, RTAX_MTU)) 4844 return 0; 4845 4846 arg->f6i = f6i; 4847 if (f6i->nh) { 4848 /* fib6_nh_mtu_change only returns 0, so this is safe */ 4849 return nexthop_for_each_fib6_nh(f6i->nh, fib6_nh_mtu_change, 4850 arg); 4851 } 4852 4853 return fib6_nh_mtu_change(f6i->fib6_nh, arg); 4854 } 4855 4856 void rt6_mtu_change(struct net_device *dev, unsigned int mtu) 4857 { 4858 struct rt6_mtu_change_arg arg = { 4859 .dev = dev, 4860 .mtu = mtu, 4861 }; 4862 4863 fib6_clean_all(dev_net(dev), rt6_mtu_change_route, &arg); 4864 } 4865 4866 static const struct nla_policy rtm_ipv6_policy[RTA_MAX+1] = { 4867 [RTA_UNSPEC] = { .strict_start_type = RTA_DPORT + 1 }, 4868 [RTA_GATEWAY] = { .len = sizeof(struct in6_addr) }, 4869 [RTA_PREFSRC] = { .len = sizeof(struct in6_addr) }, 4870 [RTA_OIF] = { .type = NLA_U32 }, 4871 [RTA_IIF] = { .type = NLA_U32 }, 4872 [RTA_PRIORITY] = { .type = NLA_U32 }, 4873 [RTA_METRICS] = { .type = NLA_NESTED }, 4874 [RTA_MULTIPATH] = { .len = sizeof(struct rtnexthop) }, 4875 [RTA_PREF] = { .type = NLA_U8 }, 4876 [RTA_ENCAP_TYPE] = { .type = NLA_U16 }, 4877 [RTA_ENCAP] = { .type = NLA_NESTED }, 4878 [RTA_EXPIRES] = { .type = NLA_U32 }, 4879 [RTA_UID] = { .type = NLA_U32 }, 4880 [RTA_MARK] = { .type = NLA_U32 }, 4881 [RTA_TABLE] = { .type = NLA_U32 }, 4882 [RTA_IP_PROTO] = { .type = NLA_U8 }, 4883 [RTA_SPORT] = { .type = NLA_U16 }, 4884 [RTA_DPORT] = { .type = NLA_U16 }, 4885 [RTA_NH_ID] = { .type = NLA_U32 }, 4886 }; 4887 4888 static int rtm_to_fib6_config(struct sk_buff *skb, struct nlmsghdr *nlh, 4889 struct fib6_config *cfg, 4890 struct netlink_ext_ack *extack) 4891 { 4892 struct rtmsg *rtm; 4893 struct nlattr *tb[RTA_MAX+1]; 4894 unsigned int pref; 4895 int err; 4896 4897 err = nlmsg_parse_deprecated(nlh, sizeof(*rtm), tb, RTA_MAX, 4898 rtm_ipv6_policy, extack); 4899 if (err < 0) 4900 goto errout; 4901 4902 err = -EINVAL; 4903 rtm = nlmsg_data(nlh); 4904 4905 *cfg = (struct fib6_config){ 4906 .fc_table = rtm->rtm_table, 4907 .fc_dst_len = rtm->rtm_dst_len, 4908 .fc_src_len = rtm->rtm_src_len, 4909 .fc_flags = RTF_UP, 4910 .fc_protocol = rtm->rtm_protocol, 4911 .fc_type = rtm->rtm_type, 4912 4913 .fc_nlinfo.portid = NETLINK_CB(skb).portid, 4914 .fc_nlinfo.nlh = nlh, 4915 .fc_nlinfo.nl_net = sock_net(skb->sk), 4916 }; 4917 4918 if (rtm->rtm_type == RTN_UNREACHABLE || 4919 rtm->rtm_type == RTN_BLACKHOLE || 4920 rtm->rtm_type == RTN_PROHIBIT || 4921 rtm->rtm_type == RTN_THROW) 4922 cfg->fc_flags |= RTF_REJECT; 4923 4924 if (rtm->rtm_type == RTN_LOCAL) 4925 cfg->fc_flags |= RTF_LOCAL; 4926 4927 if (rtm->rtm_flags & RTM_F_CLONED) 4928 cfg->fc_flags |= RTF_CACHE; 4929 4930 cfg->fc_flags |= (rtm->rtm_flags & RTNH_F_ONLINK); 4931 4932 if (tb[RTA_NH_ID]) { 4933 if (tb[RTA_GATEWAY] || tb[RTA_OIF] || 4934 tb[RTA_MULTIPATH] || tb[RTA_ENCAP]) { 4935 NL_SET_ERR_MSG(extack, 4936 "Nexthop specification and nexthop id are mutually exclusive"); 4937 goto errout; 4938 } 4939 cfg->fc_nh_id = nla_get_u32(tb[RTA_NH_ID]); 4940 } 4941 4942 if (tb[RTA_GATEWAY]) { 4943 cfg->fc_gateway = nla_get_in6_addr(tb[RTA_GATEWAY]); 4944 cfg->fc_flags |= RTF_GATEWAY; 4945 } 4946 if (tb[RTA_VIA]) { 4947 NL_SET_ERR_MSG(extack, "IPv6 does not support RTA_VIA attribute"); 4948 goto errout; 4949 } 4950 4951 if (tb[RTA_DST]) { 4952 int plen = (rtm->rtm_dst_len + 7) >> 3; 4953 4954 if (nla_len(tb[RTA_DST]) < plen) 4955 goto errout; 4956 4957 nla_memcpy(&cfg->fc_dst, tb[RTA_DST], plen); 4958 } 4959 4960 if (tb[RTA_SRC]) { 4961 int plen = (rtm->rtm_src_len + 7) >> 3; 4962 4963 if (nla_len(tb[RTA_SRC]) < plen) 4964 goto errout; 4965 4966 nla_memcpy(&cfg->fc_src, tb[RTA_SRC], plen); 4967 } 4968 4969 if (tb[RTA_PREFSRC]) 4970 cfg->fc_prefsrc = nla_get_in6_addr(tb[RTA_PREFSRC]); 4971 4972 if (tb[RTA_OIF]) 4973 cfg->fc_ifindex = nla_get_u32(tb[RTA_OIF]); 4974 4975 if (tb[RTA_PRIORITY]) 4976 cfg->fc_metric = nla_get_u32(tb[RTA_PRIORITY]); 4977 4978 if (tb[RTA_METRICS]) { 4979 cfg->fc_mx = nla_data(tb[RTA_METRICS]); 4980 cfg->fc_mx_len = nla_len(tb[RTA_METRICS]); 4981 } 4982 4983 if (tb[RTA_TABLE]) 4984 cfg->fc_table = nla_get_u32(tb[RTA_TABLE]); 4985 4986 if (tb[RTA_MULTIPATH]) { 4987 cfg->fc_mp = nla_data(tb[RTA_MULTIPATH]); 4988 cfg->fc_mp_len = nla_len(tb[RTA_MULTIPATH]); 4989 4990 err = lwtunnel_valid_encap_type_attr(cfg->fc_mp, 4991 cfg->fc_mp_len, extack); 4992 if (err < 0) 4993 goto errout; 4994 } 4995 4996 if (tb[RTA_PREF]) { 4997 pref = nla_get_u8(tb[RTA_PREF]); 4998 if (pref != ICMPV6_ROUTER_PREF_LOW && 4999 pref != ICMPV6_ROUTER_PREF_HIGH) 5000 pref = ICMPV6_ROUTER_PREF_MEDIUM; 5001 cfg->fc_flags |= RTF_PREF(pref); 5002 } 5003 5004 if (tb[RTA_ENCAP]) 5005 cfg->fc_encap = tb[RTA_ENCAP]; 5006 5007 if (tb[RTA_ENCAP_TYPE]) { 5008 cfg->fc_encap_type = nla_get_u16(tb[RTA_ENCAP_TYPE]); 5009 5010 err = lwtunnel_valid_encap_type(cfg->fc_encap_type, extack); 5011 if (err < 0) 5012 goto errout; 5013 } 5014 5015 if (tb[RTA_EXPIRES]) { 5016 unsigned long timeout = addrconf_timeout_fixup(nla_get_u32(tb[RTA_EXPIRES]), HZ); 5017 5018 if (addrconf_finite_timeout(timeout)) { 5019 cfg->fc_expires = jiffies_to_clock_t(timeout * HZ); 5020 cfg->fc_flags |= RTF_EXPIRES; 5021 } 5022 } 5023 5024 err = 0; 5025 errout: 5026 return err; 5027 } 5028 5029 struct rt6_nh { 5030 struct fib6_info *fib6_info; 5031 struct fib6_config r_cfg; 5032 struct list_head next; 5033 }; 5034 5035 static int ip6_route_info_append(struct net *net, 5036 struct list_head *rt6_nh_list, 5037 struct fib6_info *rt, 5038 struct fib6_config *r_cfg) 5039 { 5040 struct rt6_nh *nh; 5041 int err = -EEXIST; 5042 5043 list_for_each_entry(nh, rt6_nh_list, next) { 5044 /* check if fib6_info already exists */ 5045 if (rt6_duplicate_nexthop(nh->fib6_info, rt)) 5046 return err; 5047 } 5048 5049 nh = kzalloc(sizeof(*nh), GFP_KERNEL); 5050 if (!nh) 5051 return -ENOMEM; 5052 nh->fib6_info = rt; 5053 memcpy(&nh->r_cfg, r_cfg, sizeof(*r_cfg)); 5054 list_add_tail(&nh->next, rt6_nh_list); 5055 5056 return 0; 5057 } 5058 5059 static void ip6_route_mpath_notify(struct fib6_info *rt, 5060 struct fib6_info *rt_last, 5061 struct nl_info *info, 5062 __u16 nlflags) 5063 { 5064 /* if this is an APPEND route, then rt points to the first route 5065 * inserted and rt_last points to last route inserted. Userspace 5066 * wants a consistent dump of the route which starts at the first 5067 * nexthop. Since sibling routes are always added at the end of 5068 * the list, find the first sibling of the last route appended 5069 */ 5070 if ((nlflags & NLM_F_APPEND) && rt_last && rt_last->fib6_nsiblings) { 5071 rt = list_first_entry(&rt_last->fib6_siblings, 5072 struct fib6_info, 5073 fib6_siblings); 5074 } 5075 5076 if (rt) 5077 inet6_rt_notify(RTM_NEWROUTE, rt, info, nlflags); 5078 } 5079 5080 static bool ip6_route_mpath_should_notify(const struct fib6_info *rt) 5081 { 5082 bool rt_can_ecmp = rt6_qualify_for_ecmp(rt); 5083 bool should_notify = false; 5084 struct fib6_info *leaf; 5085 struct fib6_node *fn; 5086 5087 rcu_read_lock(); 5088 fn = rcu_dereference(rt->fib6_node); 5089 if (!fn) 5090 goto out; 5091 5092 leaf = rcu_dereference(fn->leaf); 5093 if (!leaf) 5094 goto out; 5095 5096 if (rt == leaf || 5097 (rt_can_ecmp && rt->fib6_metric == leaf->fib6_metric && 5098 rt6_qualify_for_ecmp(leaf))) 5099 should_notify = true; 5100 out: 5101 rcu_read_unlock(); 5102 5103 return should_notify; 5104 } 5105 5106 static int ip6_route_multipath_add(struct fib6_config *cfg, 5107 struct netlink_ext_ack *extack) 5108 { 5109 struct fib6_info *rt_notif = NULL, *rt_last = NULL; 5110 struct nl_info *info = &cfg->fc_nlinfo; 5111 struct fib6_config r_cfg; 5112 struct rtnexthop *rtnh; 5113 struct fib6_info *rt; 5114 struct rt6_nh *err_nh; 5115 struct rt6_nh *nh, *nh_safe; 5116 __u16 nlflags; 5117 int remaining; 5118 int attrlen; 5119 int err = 1; 5120 int nhn = 0; 5121 int replace = (cfg->fc_nlinfo.nlh && 5122 (cfg->fc_nlinfo.nlh->nlmsg_flags & NLM_F_REPLACE)); 5123 LIST_HEAD(rt6_nh_list); 5124 5125 nlflags = replace ? NLM_F_REPLACE : NLM_F_CREATE; 5126 if (info->nlh && info->nlh->nlmsg_flags & NLM_F_APPEND) 5127 nlflags |= NLM_F_APPEND; 5128 5129 remaining = cfg->fc_mp_len; 5130 rtnh = (struct rtnexthop *)cfg->fc_mp; 5131 5132 /* Parse a Multipath Entry and build a list (rt6_nh_list) of 5133 * fib6_info structs per nexthop 5134 */ 5135 while (rtnh_ok(rtnh, remaining)) { 5136 memcpy(&r_cfg, cfg, sizeof(*cfg)); 5137 if (rtnh->rtnh_ifindex) 5138 r_cfg.fc_ifindex = rtnh->rtnh_ifindex; 5139 5140 attrlen = rtnh_attrlen(rtnh); 5141 if (attrlen > 0) { 5142 struct nlattr *nla, *attrs = rtnh_attrs(rtnh); 5143 5144 nla = nla_find(attrs, attrlen, RTA_GATEWAY); 5145 if (nla) { 5146 r_cfg.fc_gateway = nla_get_in6_addr(nla); 5147 r_cfg.fc_flags |= RTF_GATEWAY; 5148 } 5149 r_cfg.fc_encap = nla_find(attrs, attrlen, RTA_ENCAP); 5150 nla = nla_find(attrs, attrlen, RTA_ENCAP_TYPE); 5151 if (nla) 5152 r_cfg.fc_encap_type = nla_get_u16(nla); 5153 } 5154 5155 r_cfg.fc_flags |= (rtnh->rtnh_flags & RTNH_F_ONLINK); 5156 rt = ip6_route_info_create(&r_cfg, GFP_KERNEL, extack); 5157 if (IS_ERR(rt)) { 5158 err = PTR_ERR(rt); 5159 rt = NULL; 5160 goto cleanup; 5161 } 5162 if (!rt6_qualify_for_ecmp(rt)) { 5163 err = -EINVAL; 5164 NL_SET_ERR_MSG(extack, 5165 "Device only routes can not be added for IPv6 using the multipath API."); 5166 fib6_info_release(rt); 5167 goto cleanup; 5168 } 5169 5170 rt->fib6_nh->fib_nh_weight = rtnh->rtnh_hops + 1; 5171 5172 err = ip6_route_info_append(info->nl_net, &rt6_nh_list, 5173 rt, &r_cfg); 5174 if (err) { 5175 fib6_info_release(rt); 5176 goto cleanup; 5177 } 5178 5179 rtnh = rtnh_next(rtnh, &remaining); 5180 } 5181 5182 if (list_empty(&rt6_nh_list)) { 5183 NL_SET_ERR_MSG(extack, 5184 "Invalid nexthop configuration - no valid nexthops"); 5185 return -EINVAL; 5186 } 5187 5188 /* for add and replace send one notification with all nexthops. 5189 * Skip the notification in fib6_add_rt2node and send one with 5190 * the full route when done 5191 */ 5192 info->skip_notify = 1; 5193 5194 /* For add and replace, send one notification with all nexthops. For 5195 * append, send one notification with all appended nexthops. 5196 */ 5197 info->skip_notify_kernel = 1; 5198 5199 err_nh = NULL; 5200 list_for_each_entry(nh, &rt6_nh_list, next) { 5201 err = __ip6_ins_rt(nh->fib6_info, info, extack); 5202 fib6_info_release(nh->fib6_info); 5203 5204 if (!err) { 5205 /* save reference to last route successfully inserted */ 5206 rt_last = nh->fib6_info; 5207 5208 /* save reference to first route for notification */ 5209 if (!rt_notif) 5210 rt_notif = nh->fib6_info; 5211 } 5212 5213 /* nh->fib6_info is used or freed at this point, reset to NULL*/ 5214 nh->fib6_info = NULL; 5215 if (err) { 5216 if (replace && nhn) 5217 NL_SET_ERR_MSG_MOD(extack, 5218 "multipath route replace failed (check consistency of installed routes)"); 5219 err_nh = nh; 5220 goto add_errout; 5221 } 5222 5223 /* Because each route is added like a single route we remove 5224 * these flags after the first nexthop: if there is a collision, 5225 * we have already failed to add the first nexthop: 5226 * fib6_add_rt2node() has rejected it; when replacing, old 5227 * nexthops have been replaced by first new, the rest should 5228 * be added to it. 5229 */ 5230 cfg->fc_nlinfo.nlh->nlmsg_flags &= ~(NLM_F_EXCL | 5231 NLM_F_REPLACE); 5232 cfg->fc_nlinfo.nlh->nlmsg_flags |= NLM_F_CREATE; 5233 nhn++; 5234 } 5235 5236 /* An in-kernel notification should only be sent in case the new 5237 * multipath route is added as the first route in the node, or if 5238 * it was appended to it. We pass 'rt_notif' since it is the first 5239 * sibling and might allow us to skip some checks in the replace case. 5240 */ 5241 if (ip6_route_mpath_should_notify(rt_notif)) { 5242 enum fib_event_type fib_event; 5243 5244 if (rt_notif->fib6_nsiblings != nhn - 1) 5245 fib_event = FIB_EVENT_ENTRY_APPEND; 5246 else 5247 fib_event = FIB_EVENT_ENTRY_REPLACE; 5248 5249 err = call_fib6_multipath_entry_notifiers(info->nl_net, 5250 fib_event, rt_notif, 5251 nhn - 1, extack); 5252 if (err) { 5253 /* Delete all the siblings that were just added */ 5254 err_nh = NULL; 5255 goto add_errout; 5256 } 5257 } 5258 5259 /* success ... tell user about new route */ 5260 ip6_route_mpath_notify(rt_notif, rt_last, info, nlflags); 5261 goto cleanup; 5262 5263 add_errout: 5264 /* send notification for routes that were added so that 5265 * the delete notifications sent by ip6_route_del are 5266 * coherent 5267 */ 5268 if (rt_notif) 5269 ip6_route_mpath_notify(rt_notif, rt_last, info, nlflags); 5270 5271 /* Delete routes that were already added */ 5272 list_for_each_entry(nh, &rt6_nh_list, next) { 5273 if (err_nh == nh) 5274 break; 5275 ip6_route_del(&nh->r_cfg, extack); 5276 } 5277 5278 cleanup: 5279 list_for_each_entry_safe(nh, nh_safe, &rt6_nh_list, next) { 5280 if (nh->fib6_info) 5281 fib6_info_release(nh->fib6_info); 5282 list_del(&nh->next); 5283 kfree(nh); 5284 } 5285 5286 return err; 5287 } 5288 5289 static int ip6_route_multipath_del(struct fib6_config *cfg, 5290 struct netlink_ext_ack *extack) 5291 { 5292 struct fib6_config r_cfg; 5293 struct rtnexthop *rtnh; 5294 int last_err = 0; 5295 int remaining; 5296 int attrlen; 5297 int err; 5298 5299 remaining = cfg->fc_mp_len; 5300 rtnh = (struct rtnexthop *)cfg->fc_mp; 5301 5302 /* Parse a Multipath Entry */ 5303 while (rtnh_ok(rtnh, remaining)) { 5304 memcpy(&r_cfg, cfg, sizeof(*cfg)); 5305 if (rtnh->rtnh_ifindex) 5306 r_cfg.fc_ifindex = rtnh->rtnh_ifindex; 5307 5308 attrlen = rtnh_attrlen(rtnh); 5309 if (attrlen > 0) { 5310 struct nlattr *nla, *attrs = rtnh_attrs(rtnh); 5311 5312 nla = nla_find(attrs, attrlen, RTA_GATEWAY); 5313 if (nla) { 5314 nla_memcpy(&r_cfg.fc_gateway, nla, 16); 5315 r_cfg.fc_flags |= RTF_GATEWAY; 5316 } 5317 } 5318 err = ip6_route_del(&r_cfg, extack); 5319 if (err) 5320 last_err = err; 5321 5322 rtnh = rtnh_next(rtnh, &remaining); 5323 } 5324 5325 return last_err; 5326 } 5327 5328 static int inet6_rtm_delroute(struct sk_buff *skb, struct nlmsghdr *nlh, 5329 struct netlink_ext_ack *extack) 5330 { 5331 struct fib6_config cfg; 5332 int err; 5333 5334 err = rtm_to_fib6_config(skb, nlh, &cfg, extack); 5335 if (err < 0) 5336 return err; 5337 5338 if (cfg.fc_nh_id && 5339 !nexthop_find_by_id(sock_net(skb->sk), cfg.fc_nh_id)) { 5340 NL_SET_ERR_MSG(extack, "Nexthop id does not exist"); 5341 return -EINVAL; 5342 } 5343 5344 if (cfg.fc_mp) 5345 return ip6_route_multipath_del(&cfg, extack); 5346 else { 5347 cfg.fc_delete_all_nh = 1; 5348 return ip6_route_del(&cfg, extack); 5349 } 5350 } 5351 5352 static int inet6_rtm_newroute(struct sk_buff *skb, struct nlmsghdr *nlh, 5353 struct netlink_ext_ack *extack) 5354 { 5355 struct fib6_config cfg; 5356 int err; 5357 5358 err = rtm_to_fib6_config(skb, nlh, &cfg, extack); 5359 if (err < 0) 5360 return err; 5361 5362 if (cfg.fc_metric == 0) 5363 cfg.fc_metric = IP6_RT_PRIO_USER; 5364 5365 if (cfg.fc_mp) 5366 return ip6_route_multipath_add(&cfg, extack); 5367 else 5368 return ip6_route_add(&cfg, GFP_KERNEL, extack); 5369 } 5370 5371 /* add the overhead of this fib6_nh to nexthop_len */ 5372 static int rt6_nh_nlmsg_size(struct fib6_nh *nh, void *arg) 5373 { 5374 int *nexthop_len = arg; 5375 5376 *nexthop_len += nla_total_size(0) /* RTA_MULTIPATH */ 5377 + NLA_ALIGN(sizeof(struct rtnexthop)) 5378 + nla_total_size(16); /* RTA_GATEWAY */ 5379 5380 if (nh->fib_nh_lws) { 5381 /* RTA_ENCAP_TYPE */ 5382 *nexthop_len += lwtunnel_get_encap_size(nh->fib_nh_lws); 5383 /* RTA_ENCAP */ 5384 *nexthop_len += nla_total_size(2); 5385 } 5386 5387 return 0; 5388 } 5389 5390 static size_t rt6_nlmsg_size(struct fib6_info *f6i) 5391 { 5392 int nexthop_len; 5393 5394 if (f6i->nh) { 5395 nexthop_len = nla_total_size(4); /* RTA_NH_ID */ 5396 nexthop_for_each_fib6_nh(f6i->nh, rt6_nh_nlmsg_size, 5397 &nexthop_len); 5398 } else { 5399 struct fib6_nh *nh = f6i->fib6_nh; 5400 5401 nexthop_len = 0; 5402 if (f6i->fib6_nsiblings) { 5403 nexthop_len = nla_total_size(0) /* RTA_MULTIPATH */ 5404 + NLA_ALIGN(sizeof(struct rtnexthop)) 5405 + nla_total_size(16) /* RTA_GATEWAY */ 5406 + lwtunnel_get_encap_size(nh->fib_nh_lws); 5407 5408 nexthop_len *= f6i->fib6_nsiblings; 5409 } 5410 nexthop_len += lwtunnel_get_encap_size(nh->fib_nh_lws); 5411 } 5412 5413 return NLMSG_ALIGN(sizeof(struct rtmsg)) 5414 + nla_total_size(16) /* RTA_SRC */ 5415 + nla_total_size(16) /* RTA_DST */ 5416 + nla_total_size(16) /* RTA_GATEWAY */ 5417 + nla_total_size(16) /* RTA_PREFSRC */ 5418 + nla_total_size(4) /* RTA_TABLE */ 5419 + nla_total_size(4) /* RTA_IIF */ 5420 + nla_total_size(4) /* RTA_OIF */ 5421 + nla_total_size(4) /* RTA_PRIORITY */ 5422 + RTAX_MAX * nla_total_size(4) /* RTA_METRICS */ 5423 + nla_total_size(sizeof(struct rta_cacheinfo)) 5424 + nla_total_size(TCP_CA_NAME_MAX) /* RTAX_CC_ALGO */ 5425 + nla_total_size(1) /* RTA_PREF */ 5426 + nexthop_len; 5427 } 5428 5429 static int rt6_fill_node_nexthop(struct sk_buff *skb, struct nexthop *nh, 5430 unsigned char *flags) 5431 { 5432 if (nexthop_is_multipath(nh)) { 5433 struct nlattr *mp; 5434 5435 mp = nla_nest_start_noflag(skb, RTA_MULTIPATH); 5436 if (!mp) 5437 goto nla_put_failure; 5438 5439 if (nexthop_mpath_fill_node(skb, nh, AF_INET6)) 5440 goto nla_put_failure; 5441 5442 nla_nest_end(skb, mp); 5443 } else { 5444 struct fib6_nh *fib6_nh; 5445 5446 fib6_nh = nexthop_fib6_nh(nh); 5447 if (fib_nexthop_info(skb, &fib6_nh->nh_common, AF_INET6, 5448 flags, false) < 0) 5449 goto nla_put_failure; 5450 } 5451 5452 return 0; 5453 5454 nla_put_failure: 5455 return -EMSGSIZE; 5456 } 5457 5458 static int rt6_fill_node(struct net *net, struct sk_buff *skb, 5459 struct fib6_info *rt, struct dst_entry *dst, 5460 struct in6_addr *dest, struct in6_addr *src, 5461 int iif, int type, u32 portid, u32 seq, 5462 unsigned int flags) 5463 { 5464 struct rt6_info *rt6 = (struct rt6_info *)dst; 5465 struct rt6key *rt6_dst, *rt6_src; 5466 u32 *pmetrics, table, rt6_flags; 5467 unsigned char nh_flags = 0; 5468 struct nlmsghdr *nlh; 5469 struct rtmsg *rtm; 5470 long expires = 0; 5471 5472 nlh = nlmsg_put(skb, portid, seq, type, sizeof(*rtm), flags); 5473 if (!nlh) 5474 return -EMSGSIZE; 5475 5476 if (rt6) { 5477 rt6_dst = &rt6->rt6i_dst; 5478 rt6_src = &rt6->rt6i_src; 5479 rt6_flags = rt6->rt6i_flags; 5480 } else { 5481 rt6_dst = &rt->fib6_dst; 5482 rt6_src = &rt->fib6_src; 5483 rt6_flags = rt->fib6_flags; 5484 } 5485 5486 rtm = nlmsg_data(nlh); 5487 rtm->rtm_family = AF_INET6; 5488 rtm->rtm_dst_len = rt6_dst->plen; 5489 rtm->rtm_src_len = rt6_src->plen; 5490 rtm->rtm_tos = 0; 5491 if (rt->fib6_table) 5492 table = rt->fib6_table->tb6_id; 5493 else 5494 table = RT6_TABLE_UNSPEC; 5495 rtm->rtm_table = table < 256 ? table : RT_TABLE_COMPAT; 5496 if (nla_put_u32(skb, RTA_TABLE, table)) 5497 goto nla_put_failure; 5498 5499 rtm->rtm_type = rt->fib6_type; 5500 rtm->rtm_flags = 0; 5501 rtm->rtm_scope = RT_SCOPE_UNIVERSE; 5502 rtm->rtm_protocol = rt->fib6_protocol; 5503 5504 if (rt6_flags & RTF_CACHE) 5505 rtm->rtm_flags |= RTM_F_CLONED; 5506 5507 if (dest) { 5508 if (nla_put_in6_addr(skb, RTA_DST, dest)) 5509 goto nla_put_failure; 5510 rtm->rtm_dst_len = 128; 5511 } else if (rtm->rtm_dst_len) 5512 if (nla_put_in6_addr(skb, RTA_DST, &rt6_dst->addr)) 5513 goto nla_put_failure; 5514 #ifdef CONFIG_IPV6_SUBTREES 5515 if (src) { 5516 if (nla_put_in6_addr(skb, RTA_SRC, src)) 5517 goto nla_put_failure; 5518 rtm->rtm_src_len = 128; 5519 } else if (rtm->rtm_src_len && 5520 nla_put_in6_addr(skb, RTA_SRC, &rt6_src->addr)) 5521 goto nla_put_failure; 5522 #endif 5523 if (iif) { 5524 #ifdef CONFIG_IPV6_MROUTE 5525 if (ipv6_addr_is_multicast(&rt6_dst->addr)) { 5526 int err = ip6mr_get_route(net, skb, rtm, portid); 5527 5528 if (err == 0) 5529 return 0; 5530 if (err < 0) 5531 goto nla_put_failure; 5532 } else 5533 #endif 5534 if (nla_put_u32(skb, RTA_IIF, iif)) 5535 goto nla_put_failure; 5536 } else if (dest) { 5537 struct in6_addr saddr_buf; 5538 if (ip6_route_get_saddr(net, rt, dest, 0, &saddr_buf) == 0 && 5539 nla_put_in6_addr(skb, RTA_PREFSRC, &saddr_buf)) 5540 goto nla_put_failure; 5541 } 5542 5543 if (rt->fib6_prefsrc.plen) { 5544 struct in6_addr saddr_buf; 5545 saddr_buf = rt->fib6_prefsrc.addr; 5546 if (nla_put_in6_addr(skb, RTA_PREFSRC, &saddr_buf)) 5547 goto nla_put_failure; 5548 } 5549 5550 pmetrics = dst ? dst_metrics_ptr(dst) : rt->fib6_metrics->metrics; 5551 if (rtnetlink_put_metrics(skb, pmetrics) < 0) 5552 goto nla_put_failure; 5553 5554 if (nla_put_u32(skb, RTA_PRIORITY, rt->fib6_metric)) 5555 goto nla_put_failure; 5556 5557 /* For multipath routes, walk the siblings list and add 5558 * each as a nexthop within RTA_MULTIPATH. 5559 */ 5560 if (rt6) { 5561 if (rt6_flags & RTF_GATEWAY && 5562 nla_put_in6_addr(skb, RTA_GATEWAY, &rt6->rt6i_gateway)) 5563 goto nla_put_failure; 5564 5565 if (dst->dev && nla_put_u32(skb, RTA_OIF, dst->dev->ifindex)) 5566 goto nla_put_failure; 5567 5568 if (dst->lwtstate && 5569 lwtunnel_fill_encap(skb, dst->lwtstate, RTA_ENCAP, RTA_ENCAP_TYPE) < 0) 5570 goto nla_put_failure; 5571 } else if (rt->fib6_nsiblings) { 5572 struct fib6_info *sibling, *next_sibling; 5573 struct nlattr *mp; 5574 5575 mp = nla_nest_start_noflag(skb, RTA_MULTIPATH); 5576 if (!mp) 5577 goto nla_put_failure; 5578 5579 if (fib_add_nexthop(skb, &rt->fib6_nh->nh_common, 5580 rt->fib6_nh->fib_nh_weight, AF_INET6) < 0) 5581 goto nla_put_failure; 5582 5583 list_for_each_entry_safe(sibling, next_sibling, 5584 &rt->fib6_siblings, fib6_siblings) { 5585 if (fib_add_nexthop(skb, &sibling->fib6_nh->nh_common, 5586 sibling->fib6_nh->fib_nh_weight, 5587 AF_INET6) < 0) 5588 goto nla_put_failure; 5589 } 5590 5591 nla_nest_end(skb, mp); 5592 } else if (rt->nh) { 5593 if (nla_put_u32(skb, RTA_NH_ID, rt->nh->id)) 5594 goto nla_put_failure; 5595 5596 if (nexthop_is_blackhole(rt->nh)) 5597 rtm->rtm_type = RTN_BLACKHOLE; 5598 5599 if (net->ipv4.sysctl_nexthop_compat_mode && 5600 rt6_fill_node_nexthop(skb, rt->nh, &nh_flags) < 0) 5601 goto nla_put_failure; 5602 5603 rtm->rtm_flags |= nh_flags; 5604 } else { 5605 if (fib_nexthop_info(skb, &rt->fib6_nh->nh_common, AF_INET6, 5606 &nh_flags, false) < 0) 5607 goto nla_put_failure; 5608 5609 rtm->rtm_flags |= nh_flags; 5610 } 5611 5612 if (rt6_flags & RTF_EXPIRES) { 5613 expires = dst ? dst->expires : rt->expires; 5614 expires -= jiffies; 5615 } 5616 5617 if (!dst) { 5618 if (rt->offload) 5619 rtm->rtm_flags |= RTM_F_OFFLOAD; 5620 if (rt->trap) 5621 rtm->rtm_flags |= RTM_F_TRAP; 5622 if (rt->offload_failed) 5623 rtm->rtm_flags |= RTM_F_OFFLOAD_FAILED; 5624 } 5625 5626 if (rtnl_put_cacheinfo(skb, dst, 0, expires, dst ? dst->error : 0) < 0) 5627 goto nla_put_failure; 5628 5629 if (nla_put_u8(skb, RTA_PREF, IPV6_EXTRACT_PREF(rt6_flags))) 5630 goto nla_put_failure; 5631 5632 5633 nlmsg_end(skb, nlh); 5634 return 0; 5635 5636 nla_put_failure: 5637 nlmsg_cancel(skb, nlh); 5638 return -EMSGSIZE; 5639 } 5640 5641 static int fib6_info_nh_uses_dev(struct fib6_nh *nh, void *arg) 5642 { 5643 const struct net_device *dev = arg; 5644 5645 if (nh->fib_nh_dev == dev) 5646 return 1; 5647 5648 return 0; 5649 } 5650 5651 static bool fib6_info_uses_dev(const struct fib6_info *f6i, 5652 const struct net_device *dev) 5653 { 5654 if (f6i->nh) { 5655 struct net_device *_dev = (struct net_device *)dev; 5656 5657 return !!nexthop_for_each_fib6_nh(f6i->nh, 5658 fib6_info_nh_uses_dev, 5659 _dev); 5660 } 5661 5662 if (f6i->fib6_nh->fib_nh_dev == dev) 5663 return true; 5664 5665 if (f6i->fib6_nsiblings) { 5666 struct fib6_info *sibling, *next_sibling; 5667 5668 list_for_each_entry_safe(sibling, next_sibling, 5669 &f6i->fib6_siblings, fib6_siblings) { 5670 if (sibling->fib6_nh->fib_nh_dev == dev) 5671 return true; 5672 } 5673 } 5674 5675 return false; 5676 } 5677 5678 struct fib6_nh_exception_dump_walker { 5679 struct rt6_rtnl_dump_arg *dump; 5680 struct fib6_info *rt; 5681 unsigned int flags; 5682 unsigned int skip; 5683 unsigned int count; 5684 }; 5685 5686 static int rt6_nh_dump_exceptions(struct fib6_nh *nh, void *arg) 5687 { 5688 struct fib6_nh_exception_dump_walker *w = arg; 5689 struct rt6_rtnl_dump_arg *dump = w->dump; 5690 struct rt6_exception_bucket *bucket; 5691 struct rt6_exception *rt6_ex; 5692 int i, err; 5693 5694 bucket = fib6_nh_get_excptn_bucket(nh, NULL); 5695 if (!bucket) 5696 return 0; 5697 5698 for (i = 0; i < FIB6_EXCEPTION_BUCKET_SIZE; i++) { 5699 hlist_for_each_entry(rt6_ex, &bucket->chain, hlist) { 5700 if (w->skip) { 5701 w->skip--; 5702 continue; 5703 } 5704 5705 /* Expiration of entries doesn't bump sernum, insertion 5706 * does. Removal is triggered by insertion, so we can 5707 * rely on the fact that if entries change between two 5708 * partial dumps, this node is scanned again completely, 5709 * see rt6_insert_exception() and fib6_dump_table(). 5710 * 5711 * Count expired entries we go through as handled 5712 * entries that we'll skip next time, in case of partial 5713 * node dump. Otherwise, if entries expire meanwhile, 5714 * we'll skip the wrong amount. 5715 */ 5716 if (rt6_check_expired(rt6_ex->rt6i)) { 5717 w->count++; 5718 continue; 5719 } 5720 5721 err = rt6_fill_node(dump->net, dump->skb, w->rt, 5722 &rt6_ex->rt6i->dst, NULL, NULL, 0, 5723 RTM_NEWROUTE, 5724 NETLINK_CB(dump->cb->skb).portid, 5725 dump->cb->nlh->nlmsg_seq, w->flags); 5726 if (err) 5727 return err; 5728 5729 w->count++; 5730 } 5731 bucket++; 5732 } 5733 5734 return 0; 5735 } 5736 5737 /* Return -1 if done with node, number of handled routes on partial dump */ 5738 int rt6_dump_route(struct fib6_info *rt, void *p_arg, unsigned int skip) 5739 { 5740 struct rt6_rtnl_dump_arg *arg = (struct rt6_rtnl_dump_arg *) p_arg; 5741 struct fib_dump_filter *filter = &arg->filter; 5742 unsigned int flags = NLM_F_MULTI; 5743 struct net *net = arg->net; 5744 int count = 0; 5745 5746 if (rt == net->ipv6.fib6_null_entry) 5747 return -1; 5748 5749 if ((filter->flags & RTM_F_PREFIX) && 5750 !(rt->fib6_flags & RTF_PREFIX_RT)) { 5751 /* success since this is not a prefix route */ 5752 return -1; 5753 } 5754 if (filter->filter_set && 5755 ((filter->rt_type && rt->fib6_type != filter->rt_type) || 5756 (filter->dev && !fib6_info_uses_dev(rt, filter->dev)) || 5757 (filter->protocol && rt->fib6_protocol != filter->protocol))) { 5758 return -1; 5759 } 5760 5761 if (filter->filter_set || 5762 !filter->dump_routes || !filter->dump_exceptions) { 5763 flags |= NLM_F_DUMP_FILTERED; 5764 } 5765 5766 if (filter->dump_routes) { 5767 if (skip) { 5768 skip--; 5769 } else { 5770 if (rt6_fill_node(net, arg->skb, rt, NULL, NULL, NULL, 5771 0, RTM_NEWROUTE, 5772 NETLINK_CB(arg->cb->skb).portid, 5773 arg->cb->nlh->nlmsg_seq, flags)) { 5774 return 0; 5775 } 5776 count++; 5777 } 5778 } 5779 5780 if (filter->dump_exceptions) { 5781 struct fib6_nh_exception_dump_walker w = { .dump = arg, 5782 .rt = rt, 5783 .flags = flags, 5784 .skip = skip, 5785 .count = 0 }; 5786 int err; 5787 5788 rcu_read_lock(); 5789 if (rt->nh) { 5790 err = nexthop_for_each_fib6_nh(rt->nh, 5791 rt6_nh_dump_exceptions, 5792 &w); 5793 } else { 5794 err = rt6_nh_dump_exceptions(rt->fib6_nh, &w); 5795 } 5796 rcu_read_unlock(); 5797 5798 if (err) 5799 return count += w.count; 5800 } 5801 5802 return -1; 5803 } 5804 5805 static int inet6_rtm_valid_getroute_req(struct sk_buff *skb, 5806 const struct nlmsghdr *nlh, 5807 struct nlattr **tb, 5808 struct netlink_ext_ack *extack) 5809 { 5810 struct rtmsg *rtm; 5811 int i, err; 5812 5813 if (nlh->nlmsg_len < nlmsg_msg_size(sizeof(*rtm))) { 5814 NL_SET_ERR_MSG_MOD(extack, 5815 "Invalid header for get route request"); 5816 return -EINVAL; 5817 } 5818 5819 if (!netlink_strict_get_check(skb)) 5820 return nlmsg_parse_deprecated(nlh, sizeof(*rtm), tb, RTA_MAX, 5821 rtm_ipv6_policy, extack); 5822 5823 rtm = nlmsg_data(nlh); 5824 if ((rtm->rtm_src_len && rtm->rtm_src_len != 128) || 5825 (rtm->rtm_dst_len && rtm->rtm_dst_len != 128) || 5826 rtm->rtm_table || rtm->rtm_protocol || rtm->rtm_scope || 5827 rtm->rtm_type) { 5828 NL_SET_ERR_MSG_MOD(extack, "Invalid values in header for get route request"); 5829 return -EINVAL; 5830 } 5831 if (rtm->rtm_flags & ~RTM_F_FIB_MATCH) { 5832 NL_SET_ERR_MSG_MOD(extack, 5833 "Invalid flags for get route request"); 5834 return -EINVAL; 5835 } 5836 5837 err = nlmsg_parse_deprecated_strict(nlh, sizeof(*rtm), tb, RTA_MAX, 5838 rtm_ipv6_policy, extack); 5839 if (err) 5840 return err; 5841 5842 if ((tb[RTA_SRC] && !rtm->rtm_src_len) || 5843 (tb[RTA_DST] && !rtm->rtm_dst_len)) { 5844 NL_SET_ERR_MSG_MOD(extack, "rtm_src_len and rtm_dst_len must be 128 for IPv6"); 5845 return -EINVAL; 5846 } 5847 5848 for (i = 0; i <= RTA_MAX; i++) { 5849 if (!tb[i]) 5850 continue; 5851 5852 switch (i) { 5853 case RTA_SRC: 5854 case RTA_DST: 5855 case RTA_IIF: 5856 case RTA_OIF: 5857 case RTA_MARK: 5858 case RTA_UID: 5859 case RTA_SPORT: 5860 case RTA_DPORT: 5861 case RTA_IP_PROTO: 5862 break; 5863 default: 5864 NL_SET_ERR_MSG_MOD(extack, "Unsupported attribute in get route request"); 5865 return -EINVAL; 5866 } 5867 } 5868 5869 return 0; 5870 } 5871 5872 static int inet6_rtm_getroute(struct sk_buff *in_skb, struct nlmsghdr *nlh, 5873 struct netlink_ext_ack *extack) 5874 { 5875 struct net *net = sock_net(in_skb->sk); 5876 struct nlattr *tb[RTA_MAX+1]; 5877 int err, iif = 0, oif = 0; 5878 struct fib6_info *from; 5879 struct dst_entry *dst; 5880 struct rt6_info *rt; 5881 struct sk_buff *skb; 5882 struct rtmsg *rtm; 5883 struct flowi6 fl6 = {}; 5884 bool fibmatch; 5885 5886 err = inet6_rtm_valid_getroute_req(in_skb, nlh, tb, extack); 5887 if (err < 0) 5888 goto errout; 5889 5890 err = -EINVAL; 5891 rtm = nlmsg_data(nlh); 5892 fl6.flowlabel = ip6_make_flowinfo(rtm->rtm_tos, 0); 5893 fibmatch = !!(rtm->rtm_flags & RTM_F_FIB_MATCH); 5894 5895 if (tb[RTA_SRC]) { 5896 if (nla_len(tb[RTA_SRC]) < sizeof(struct in6_addr)) 5897 goto errout; 5898 5899 fl6.saddr = *(struct in6_addr *)nla_data(tb[RTA_SRC]); 5900 } 5901 5902 if (tb[RTA_DST]) { 5903 if (nla_len(tb[RTA_DST]) < sizeof(struct in6_addr)) 5904 goto errout; 5905 5906 fl6.daddr = *(struct in6_addr *)nla_data(tb[RTA_DST]); 5907 } 5908 5909 if (tb[RTA_IIF]) 5910 iif = nla_get_u32(tb[RTA_IIF]); 5911 5912 if (tb[RTA_OIF]) 5913 oif = nla_get_u32(tb[RTA_OIF]); 5914 5915 if (tb[RTA_MARK]) 5916 fl6.flowi6_mark = nla_get_u32(tb[RTA_MARK]); 5917 5918 if (tb[RTA_UID]) 5919 fl6.flowi6_uid = make_kuid(current_user_ns(), 5920 nla_get_u32(tb[RTA_UID])); 5921 else 5922 fl6.flowi6_uid = iif ? INVALID_UID : current_uid(); 5923 5924 if (tb[RTA_SPORT]) 5925 fl6.fl6_sport = nla_get_be16(tb[RTA_SPORT]); 5926 5927 if (tb[RTA_DPORT]) 5928 fl6.fl6_dport = nla_get_be16(tb[RTA_DPORT]); 5929 5930 if (tb[RTA_IP_PROTO]) { 5931 err = rtm_getroute_parse_ip_proto(tb[RTA_IP_PROTO], 5932 &fl6.flowi6_proto, AF_INET6, 5933 extack); 5934 if (err) 5935 goto errout; 5936 } 5937 5938 if (iif) { 5939 struct net_device *dev; 5940 int flags = 0; 5941 5942 rcu_read_lock(); 5943 5944 dev = dev_get_by_index_rcu(net, iif); 5945 if (!dev) { 5946 rcu_read_unlock(); 5947 err = -ENODEV; 5948 goto errout; 5949 } 5950 5951 fl6.flowi6_iif = iif; 5952 5953 if (!ipv6_addr_any(&fl6.saddr)) 5954 flags |= RT6_LOOKUP_F_HAS_SADDR; 5955 5956 dst = ip6_route_input_lookup(net, dev, &fl6, NULL, flags); 5957 5958 rcu_read_unlock(); 5959 } else { 5960 fl6.flowi6_oif = oif; 5961 5962 dst = ip6_route_output(net, NULL, &fl6); 5963 } 5964 5965 5966 rt = container_of(dst, struct rt6_info, dst); 5967 if (rt->dst.error) { 5968 err = rt->dst.error; 5969 ip6_rt_put(rt); 5970 goto errout; 5971 } 5972 5973 if (rt == net->ipv6.ip6_null_entry) { 5974 err = rt->dst.error; 5975 ip6_rt_put(rt); 5976 goto errout; 5977 } 5978 5979 skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL); 5980 if (!skb) { 5981 ip6_rt_put(rt); 5982 err = -ENOBUFS; 5983 goto errout; 5984 } 5985 5986 skb_dst_set(skb, &rt->dst); 5987 5988 rcu_read_lock(); 5989 from = rcu_dereference(rt->from); 5990 if (from) { 5991 if (fibmatch) 5992 err = rt6_fill_node(net, skb, from, NULL, NULL, NULL, 5993 iif, RTM_NEWROUTE, 5994 NETLINK_CB(in_skb).portid, 5995 nlh->nlmsg_seq, 0); 5996 else 5997 err = rt6_fill_node(net, skb, from, dst, &fl6.daddr, 5998 &fl6.saddr, iif, RTM_NEWROUTE, 5999 NETLINK_CB(in_skb).portid, 6000 nlh->nlmsg_seq, 0); 6001 } else { 6002 err = -ENETUNREACH; 6003 } 6004 rcu_read_unlock(); 6005 6006 if (err < 0) { 6007 kfree_skb(skb); 6008 goto errout; 6009 } 6010 6011 err = rtnl_unicast(skb, net, NETLINK_CB(in_skb).portid); 6012 errout: 6013 return err; 6014 } 6015 6016 void inet6_rt_notify(int event, struct fib6_info *rt, struct nl_info *info, 6017 unsigned int nlm_flags) 6018 { 6019 struct sk_buff *skb; 6020 struct net *net = info->nl_net; 6021 u32 seq; 6022 int err; 6023 6024 err = -ENOBUFS; 6025 seq = info->nlh ? info->nlh->nlmsg_seq : 0; 6026 6027 skb = nlmsg_new(rt6_nlmsg_size(rt), gfp_any()); 6028 if (!skb) 6029 goto errout; 6030 6031 err = rt6_fill_node(net, skb, rt, NULL, NULL, NULL, 0, 6032 event, info->portid, seq, nlm_flags); 6033 if (err < 0) { 6034 /* -EMSGSIZE implies BUG in rt6_nlmsg_size() */ 6035 WARN_ON(err == -EMSGSIZE); 6036 kfree_skb(skb); 6037 goto errout; 6038 } 6039 rtnl_notify(skb, net, info->portid, RTNLGRP_IPV6_ROUTE, 6040 info->nlh, gfp_any()); 6041 return; 6042 errout: 6043 if (err < 0) 6044 rtnl_set_sk_err(net, RTNLGRP_IPV6_ROUTE, err); 6045 } 6046 6047 void fib6_rt_update(struct net *net, struct fib6_info *rt, 6048 struct nl_info *info) 6049 { 6050 u32 seq = info->nlh ? info->nlh->nlmsg_seq : 0; 6051 struct sk_buff *skb; 6052 int err = -ENOBUFS; 6053 6054 skb = nlmsg_new(rt6_nlmsg_size(rt), gfp_any()); 6055 if (!skb) 6056 goto errout; 6057 6058 err = rt6_fill_node(net, skb, rt, NULL, NULL, NULL, 0, 6059 RTM_NEWROUTE, info->portid, seq, NLM_F_REPLACE); 6060 if (err < 0) { 6061 /* -EMSGSIZE implies BUG in rt6_nlmsg_size() */ 6062 WARN_ON(err == -EMSGSIZE); 6063 kfree_skb(skb); 6064 goto errout; 6065 } 6066 rtnl_notify(skb, net, info->portid, RTNLGRP_IPV6_ROUTE, 6067 info->nlh, gfp_any()); 6068 return; 6069 errout: 6070 if (err < 0) 6071 rtnl_set_sk_err(net, RTNLGRP_IPV6_ROUTE, err); 6072 } 6073 6074 void fib6_info_hw_flags_set(struct net *net, struct fib6_info *f6i, 6075 bool offload, bool trap, bool offload_failed) 6076 { 6077 struct sk_buff *skb; 6078 int err; 6079 6080 if (f6i->offload == offload && f6i->trap == trap && 6081 f6i->offload_failed == offload_failed) 6082 return; 6083 6084 f6i->offload = offload; 6085 f6i->trap = trap; 6086 6087 /* 2 means send notifications only if offload_failed was changed. */ 6088 if (net->ipv6.sysctl.fib_notify_on_flag_change == 2 && 6089 f6i->offload_failed == offload_failed) 6090 return; 6091 6092 f6i->offload_failed = offload_failed; 6093 6094 if (!rcu_access_pointer(f6i->fib6_node)) 6095 /* The route was removed from the tree, do not send 6096 * notfication. 6097 */ 6098 return; 6099 6100 if (!net->ipv6.sysctl.fib_notify_on_flag_change) 6101 return; 6102 6103 skb = nlmsg_new(rt6_nlmsg_size(f6i), GFP_KERNEL); 6104 if (!skb) { 6105 err = -ENOBUFS; 6106 goto errout; 6107 } 6108 6109 err = rt6_fill_node(net, skb, f6i, NULL, NULL, NULL, 0, RTM_NEWROUTE, 0, 6110 0, 0); 6111 if (err < 0) { 6112 /* -EMSGSIZE implies BUG in rt6_nlmsg_size() */ 6113 WARN_ON(err == -EMSGSIZE); 6114 kfree_skb(skb); 6115 goto errout; 6116 } 6117 6118 rtnl_notify(skb, net, 0, RTNLGRP_IPV6_ROUTE, NULL, GFP_KERNEL); 6119 return; 6120 6121 errout: 6122 rtnl_set_sk_err(net, RTNLGRP_IPV6_ROUTE, err); 6123 } 6124 EXPORT_SYMBOL(fib6_info_hw_flags_set); 6125 6126 static int ip6_route_dev_notify(struct notifier_block *this, 6127 unsigned long event, void *ptr) 6128 { 6129 struct net_device *dev = netdev_notifier_info_to_dev(ptr); 6130 struct net *net = dev_net(dev); 6131 6132 if (!(dev->flags & IFF_LOOPBACK)) 6133 return NOTIFY_OK; 6134 6135 if (event == NETDEV_REGISTER) { 6136 net->ipv6.fib6_null_entry->fib6_nh->fib_nh_dev = dev; 6137 net->ipv6.ip6_null_entry->dst.dev = dev; 6138 net->ipv6.ip6_null_entry->rt6i_idev = in6_dev_get(dev); 6139 #ifdef CONFIG_IPV6_MULTIPLE_TABLES 6140 net->ipv6.ip6_prohibit_entry->dst.dev = dev; 6141 net->ipv6.ip6_prohibit_entry->rt6i_idev = in6_dev_get(dev); 6142 net->ipv6.ip6_blk_hole_entry->dst.dev = dev; 6143 net->ipv6.ip6_blk_hole_entry->rt6i_idev = in6_dev_get(dev); 6144 #endif 6145 } else if (event == NETDEV_UNREGISTER && 6146 dev->reg_state != NETREG_UNREGISTERED) { 6147 /* NETDEV_UNREGISTER could be fired for multiple times by 6148 * netdev_wait_allrefs(). Make sure we only call this once. 6149 */ 6150 in6_dev_put_clear(&net->ipv6.ip6_null_entry->rt6i_idev); 6151 #ifdef CONFIG_IPV6_MULTIPLE_TABLES 6152 in6_dev_put_clear(&net->ipv6.ip6_prohibit_entry->rt6i_idev); 6153 in6_dev_put_clear(&net->ipv6.ip6_blk_hole_entry->rt6i_idev); 6154 #endif 6155 } 6156 6157 return NOTIFY_OK; 6158 } 6159 6160 /* 6161 * /proc 6162 */ 6163 6164 #ifdef CONFIG_PROC_FS 6165 static int rt6_stats_seq_show(struct seq_file *seq, void *v) 6166 { 6167 struct net *net = (struct net *)seq->private; 6168 seq_printf(seq, "%04x %04x %04x %04x %04x %04x %04x\n", 6169 net->ipv6.rt6_stats->fib_nodes, 6170 net->ipv6.rt6_stats->fib_route_nodes, 6171 atomic_read(&net->ipv6.rt6_stats->fib_rt_alloc), 6172 net->ipv6.rt6_stats->fib_rt_entries, 6173 net->ipv6.rt6_stats->fib_rt_cache, 6174 dst_entries_get_slow(&net->ipv6.ip6_dst_ops), 6175 net->ipv6.rt6_stats->fib_discarded_routes); 6176 6177 return 0; 6178 } 6179 #endif /* CONFIG_PROC_FS */ 6180 6181 #ifdef CONFIG_SYSCTL 6182 6183 static int ipv6_sysctl_rtcache_flush(struct ctl_table *ctl, int write, 6184 void *buffer, size_t *lenp, loff_t *ppos) 6185 { 6186 struct net *net; 6187 int delay; 6188 int ret; 6189 if (!write) 6190 return -EINVAL; 6191 6192 net = (struct net *)ctl->extra1; 6193 delay = net->ipv6.sysctl.flush_delay; 6194 ret = proc_dointvec(ctl, write, buffer, lenp, ppos); 6195 if (ret) 6196 return ret; 6197 6198 fib6_run_gc(delay <= 0 ? 0 : (unsigned long)delay, net, delay > 0); 6199 return 0; 6200 } 6201 6202 static struct ctl_table ipv6_route_table_template[] = { 6203 { 6204 .procname = "flush", 6205 .data = &init_net.ipv6.sysctl.flush_delay, 6206 .maxlen = sizeof(int), 6207 .mode = 0200, 6208 .proc_handler = ipv6_sysctl_rtcache_flush 6209 }, 6210 { 6211 .procname = "gc_thresh", 6212 .data = &ip6_dst_ops_template.gc_thresh, 6213 .maxlen = sizeof(int), 6214 .mode = 0644, 6215 .proc_handler = proc_dointvec, 6216 }, 6217 { 6218 .procname = "max_size", 6219 .data = &init_net.ipv6.sysctl.ip6_rt_max_size, 6220 .maxlen = sizeof(int), 6221 .mode = 0644, 6222 .proc_handler = proc_dointvec, 6223 }, 6224 { 6225 .procname = "gc_min_interval", 6226 .data = &init_net.ipv6.sysctl.ip6_rt_gc_min_interval, 6227 .maxlen = sizeof(int), 6228 .mode = 0644, 6229 .proc_handler = proc_dointvec_jiffies, 6230 }, 6231 { 6232 .procname = "gc_timeout", 6233 .data = &init_net.ipv6.sysctl.ip6_rt_gc_timeout, 6234 .maxlen = sizeof(int), 6235 .mode = 0644, 6236 .proc_handler = proc_dointvec_jiffies, 6237 }, 6238 { 6239 .procname = "gc_interval", 6240 .data = &init_net.ipv6.sysctl.ip6_rt_gc_interval, 6241 .maxlen = sizeof(int), 6242 .mode = 0644, 6243 .proc_handler = proc_dointvec_jiffies, 6244 }, 6245 { 6246 .procname = "gc_elasticity", 6247 .data = &init_net.ipv6.sysctl.ip6_rt_gc_elasticity, 6248 .maxlen = sizeof(int), 6249 .mode = 0644, 6250 .proc_handler = proc_dointvec, 6251 }, 6252 { 6253 .procname = "mtu_expires", 6254 .data = &init_net.ipv6.sysctl.ip6_rt_mtu_expires, 6255 .maxlen = sizeof(int), 6256 .mode = 0644, 6257 .proc_handler = proc_dointvec_jiffies, 6258 }, 6259 { 6260 .procname = "min_adv_mss", 6261 .data = &init_net.ipv6.sysctl.ip6_rt_min_advmss, 6262 .maxlen = sizeof(int), 6263 .mode = 0644, 6264 .proc_handler = proc_dointvec, 6265 }, 6266 { 6267 .procname = "gc_min_interval_ms", 6268 .data = &init_net.ipv6.sysctl.ip6_rt_gc_min_interval, 6269 .maxlen = sizeof(int), 6270 .mode = 0644, 6271 .proc_handler = proc_dointvec_ms_jiffies, 6272 }, 6273 { 6274 .procname = "skip_notify_on_dev_down", 6275 .data = &init_net.ipv6.sysctl.skip_notify_on_dev_down, 6276 .maxlen = sizeof(int), 6277 .mode = 0644, 6278 .proc_handler = proc_dointvec_minmax, 6279 .extra1 = SYSCTL_ZERO, 6280 .extra2 = SYSCTL_ONE, 6281 }, 6282 { } 6283 }; 6284 6285 struct ctl_table * __net_init ipv6_route_sysctl_init(struct net *net) 6286 { 6287 struct ctl_table *table; 6288 6289 table = kmemdup(ipv6_route_table_template, 6290 sizeof(ipv6_route_table_template), 6291 GFP_KERNEL); 6292 6293 if (table) { 6294 table[0].data = &net->ipv6.sysctl.flush_delay; 6295 table[0].extra1 = net; 6296 table[1].data = &net->ipv6.ip6_dst_ops.gc_thresh; 6297 table[2].data = &net->ipv6.sysctl.ip6_rt_max_size; 6298 table[3].data = &net->ipv6.sysctl.ip6_rt_gc_min_interval; 6299 table[4].data = &net->ipv6.sysctl.ip6_rt_gc_timeout; 6300 table[5].data = &net->ipv6.sysctl.ip6_rt_gc_interval; 6301 table[6].data = &net->ipv6.sysctl.ip6_rt_gc_elasticity; 6302 table[7].data = &net->ipv6.sysctl.ip6_rt_mtu_expires; 6303 table[8].data = &net->ipv6.sysctl.ip6_rt_min_advmss; 6304 table[9].data = &net->ipv6.sysctl.ip6_rt_gc_min_interval; 6305 table[10].data = &net->ipv6.sysctl.skip_notify_on_dev_down; 6306 6307 /* Don't export sysctls to unprivileged users */ 6308 if (net->user_ns != &init_user_ns) 6309 table[0].procname = NULL; 6310 } 6311 6312 return table; 6313 } 6314 #endif 6315 6316 static int __net_init ip6_route_net_init(struct net *net) 6317 { 6318 int ret = -ENOMEM; 6319 6320 memcpy(&net->ipv6.ip6_dst_ops, &ip6_dst_ops_template, 6321 sizeof(net->ipv6.ip6_dst_ops)); 6322 6323 if (dst_entries_init(&net->ipv6.ip6_dst_ops) < 0) 6324 goto out_ip6_dst_ops; 6325 6326 net->ipv6.fib6_null_entry = fib6_info_alloc(GFP_KERNEL, true); 6327 if (!net->ipv6.fib6_null_entry) 6328 goto out_ip6_dst_entries; 6329 memcpy(net->ipv6.fib6_null_entry, &fib6_null_entry_template, 6330 sizeof(*net->ipv6.fib6_null_entry)); 6331 6332 net->ipv6.ip6_null_entry = kmemdup(&ip6_null_entry_template, 6333 sizeof(*net->ipv6.ip6_null_entry), 6334 GFP_KERNEL); 6335 if (!net->ipv6.ip6_null_entry) 6336 goto out_fib6_null_entry; 6337 net->ipv6.ip6_null_entry->dst.ops = &net->ipv6.ip6_dst_ops; 6338 dst_init_metrics(&net->ipv6.ip6_null_entry->dst, 6339 ip6_template_metrics, true); 6340 INIT_LIST_HEAD(&net->ipv6.ip6_null_entry->rt6i_uncached); 6341 6342 #ifdef CONFIG_IPV6_MULTIPLE_TABLES 6343 net->ipv6.fib6_has_custom_rules = false; 6344 net->ipv6.ip6_prohibit_entry = kmemdup(&ip6_prohibit_entry_template, 6345 sizeof(*net->ipv6.ip6_prohibit_entry), 6346 GFP_KERNEL); 6347 if (!net->ipv6.ip6_prohibit_entry) 6348 goto out_ip6_null_entry; 6349 net->ipv6.ip6_prohibit_entry->dst.ops = &net->ipv6.ip6_dst_ops; 6350 dst_init_metrics(&net->ipv6.ip6_prohibit_entry->dst, 6351 ip6_template_metrics, true); 6352 INIT_LIST_HEAD(&net->ipv6.ip6_prohibit_entry->rt6i_uncached); 6353 6354 net->ipv6.ip6_blk_hole_entry = kmemdup(&ip6_blk_hole_entry_template, 6355 sizeof(*net->ipv6.ip6_blk_hole_entry), 6356 GFP_KERNEL); 6357 if (!net->ipv6.ip6_blk_hole_entry) 6358 goto out_ip6_prohibit_entry; 6359 net->ipv6.ip6_blk_hole_entry->dst.ops = &net->ipv6.ip6_dst_ops; 6360 dst_init_metrics(&net->ipv6.ip6_blk_hole_entry->dst, 6361 ip6_template_metrics, true); 6362 INIT_LIST_HEAD(&net->ipv6.ip6_blk_hole_entry->rt6i_uncached); 6363 #ifdef CONFIG_IPV6_SUBTREES 6364 net->ipv6.fib6_routes_require_src = 0; 6365 #endif 6366 #endif 6367 6368 net->ipv6.sysctl.flush_delay = 0; 6369 net->ipv6.sysctl.ip6_rt_max_size = 4096; 6370 net->ipv6.sysctl.ip6_rt_gc_min_interval = HZ / 2; 6371 net->ipv6.sysctl.ip6_rt_gc_timeout = 60*HZ; 6372 net->ipv6.sysctl.ip6_rt_gc_interval = 30*HZ; 6373 net->ipv6.sysctl.ip6_rt_gc_elasticity = 9; 6374 net->ipv6.sysctl.ip6_rt_mtu_expires = 10*60*HZ; 6375 net->ipv6.sysctl.ip6_rt_min_advmss = IPV6_MIN_MTU - 20 - 40; 6376 net->ipv6.sysctl.skip_notify_on_dev_down = 0; 6377 6378 net->ipv6.ip6_rt_gc_expire = 30*HZ; 6379 6380 ret = 0; 6381 out: 6382 return ret; 6383 6384 #ifdef CONFIG_IPV6_MULTIPLE_TABLES 6385 out_ip6_prohibit_entry: 6386 kfree(net->ipv6.ip6_prohibit_entry); 6387 out_ip6_null_entry: 6388 kfree(net->ipv6.ip6_null_entry); 6389 #endif 6390 out_fib6_null_entry: 6391 kfree(net->ipv6.fib6_null_entry); 6392 out_ip6_dst_entries: 6393 dst_entries_destroy(&net->ipv6.ip6_dst_ops); 6394 out_ip6_dst_ops: 6395 goto out; 6396 } 6397 6398 static void __net_exit ip6_route_net_exit(struct net *net) 6399 { 6400 kfree(net->ipv6.fib6_null_entry); 6401 kfree(net->ipv6.ip6_null_entry); 6402 #ifdef CONFIG_IPV6_MULTIPLE_TABLES 6403 kfree(net->ipv6.ip6_prohibit_entry); 6404 kfree(net->ipv6.ip6_blk_hole_entry); 6405 #endif 6406 dst_entries_destroy(&net->ipv6.ip6_dst_ops); 6407 } 6408 6409 static int __net_init ip6_route_net_init_late(struct net *net) 6410 { 6411 #ifdef CONFIG_PROC_FS 6412 proc_create_net("ipv6_route", 0, net->proc_net, &ipv6_route_seq_ops, 6413 sizeof(struct ipv6_route_iter)); 6414 proc_create_net_single("rt6_stats", 0444, net->proc_net, 6415 rt6_stats_seq_show, NULL); 6416 #endif 6417 return 0; 6418 } 6419 6420 static void __net_exit ip6_route_net_exit_late(struct net *net) 6421 { 6422 #ifdef CONFIG_PROC_FS 6423 remove_proc_entry("ipv6_route", net->proc_net); 6424 remove_proc_entry("rt6_stats", net->proc_net); 6425 #endif 6426 } 6427 6428 static struct pernet_operations ip6_route_net_ops = { 6429 .init = ip6_route_net_init, 6430 .exit = ip6_route_net_exit, 6431 }; 6432 6433 static int __net_init ipv6_inetpeer_init(struct net *net) 6434 { 6435 struct inet_peer_base *bp = kmalloc(sizeof(*bp), GFP_KERNEL); 6436 6437 if (!bp) 6438 return -ENOMEM; 6439 inet_peer_base_init(bp); 6440 net->ipv6.peers = bp; 6441 return 0; 6442 } 6443 6444 static void __net_exit ipv6_inetpeer_exit(struct net *net) 6445 { 6446 struct inet_peer_base *bp = net->ipv6.peers; 6447 6448 net->ipv6.peers = NULL; 6449 inetpeer_invalidate_tree(bp); 6450 kfree(bp); 6451 } 6452 6453 static struct pernet_operations ipv6_inetpeer_ops = { 6454 .init = ipv6_inetpeer_init, 6455 .exit = ipv6_inetpeer_exit, 6456 }; 6457 6458 static struct pernet_operations ip6_route_net_late_ops = { 6459 .init = ip6_route_net_init_late, 6460 .exit = ip6_route_net_exit_late, 6461 }; 6462 6463 static struct notifier_block ip6_route_dev_notifier = { 6464 .notifier_call = ip6_route_dev_notify, 6465 .priority = ADDRCONF_NOTIFY_PRIORITY - 10, 6466 }; 6467 6468 void __init ip6_route_init_special_entries(void) 6469 { 6470 /* Registering of the loopback is done before this portion of code, 6471 * the loopback reference in rt6_info will not be taken, do it 6472 * manually for init_net */ 6473 init_net.ipv6.fib6_null_entry->fib6_nh->fib_nh_dev = init_net.loopback_dev; 6474 init_net.ipv6.ip6_null_entry->dst.dev = init_net.loopback_dev; 6475 init_net.ipv6.ip6_null_entry->rt6i_idev = in6_dev_get(init_net.loopback_dev); 6476 #ifdef CONFIG_IPV6_MULTIPLE_TABLES 6477 init_net.ipv6.ip6_prohibit_entry->dst.dev = init_net.loopback_dev; 6478 init_net.ipv6.ip6_prohibit_entry->rt6i_idev = in6_dev_get(init_net.loopback_dev); 6479 init_net.ipv6.ip6_blk_hole_entry->dst.dev = init_net.loopback_dev; 6480 init_net.ipv6.ip6_blk_hole_entry->rt6i_idev = in6_dev_get(init_net.loopback_dev); 6481 #endif 6482 } 6483 6484 #if IS_BUILTIN(CONFIG_IPV6) 6485 #if defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_PROC_FS) 6486 DEFINE_BPF_ITER_FUNC(ipv6_route, struct bpf_iter_meta *meta, struct fib6_info *rt) 6487 6488 BTF_ID_LIST(btf_fib6_info_id) 6489 BTF_ID(struct, fib6_info) 6490 6491 static const struct bpf_iter_seq_info ipv6_route_seq_info = { 6492 .seq_ops = &ipv6_route_seq_ops, 6493 .init_seq_private = bpf_iter_init_seq_net, 6494 .fini_seq_private = bpf_iter_fini_seq_net, 6495 .seq_priv_size = sizeof(struct ipv6_route_iter), 6496 }; 6497 6498 static struct bpf_iter_reg ipv6_route_reg_info = { 6499 .target = "ipv6_route", 6500 .ctx_arg_info_size = 1, 6501 .ctx_arg_info = { 6502 { offsetof(struct bpf_iter__ipv6_route, rt), 6503 PTR_TO_BTF_ID_OR_NULL }, 6504 }, 6505 .seq_info = &ipv6_route_seq_info, 6506 }; 6507 6508 static int __init bpf_iter_register(void) 6509 { 6510 ipv6_route_reg_info.ctx_arg_info[0].btf_id = *btf_fib6_info_id; 6511 return bpf_iter_reg_target(&ipv6_route_reg_info); 6512 } 6513 6514 static void bpf_iter_unregister(void) 6515 { 6516 bpf_iter_unreg_target(&ipv6_route_reg_info); 6517 } 6518 #endif 6519 #endif 6520 6521 int __init ip6_route_init(void) 6522 { 6523 int ret; 6524 int cpu; 6525 6526 ret = -ENOMEM; 6527 ip6_dst_ops_template.kmem_cachep = 6528 kmem_cache_create("ip6_dst_cache", sizeof(struct rt6_info), 0, 6529 SLAB_HWCACHE_ALIGN, NULL); 6530 if (!ip6_dst_ops_template.kmem_cachep) 6531 goto out; 6532 6533 ret = dst_entries_init(&ip6_dst_blackhole_ops); 6534 if (ret) 6535 goto out_kmem_cache; 6536 6537 ret = register_pernet_subsys(&ipv6_inetpeer_ops); 6538 if (ret) 6539 goto out_dst_entries; 6540 6541 ret = register_pernet_subsys(&ip6_route_net_ops); 6542 if (ret) 6543 goto out_register_inetpeer; 6544 6545 ip6_dst_blackhole_ops.kmem_cachep = ip6_dst_ops_template.kmem_cachep; 6546 6547 ret = fib6_init(); 6548 if (ret) 6549 goto out_register_subsys; 6550 6551 ret = xfrm6_init(); 6552 if (ret) 6553 goto out_fib6_init; 6554 6555 ret = fib6_rules_init(); 6556 if (ret) 6557 goto xfrm6_init; 6558 6559 ret = register_pernet_subsys(&ip6_route_net_late_ops); 6560 if (ret) 6561 goto fib6_rules_init; 6562 6563 ret = rtnl_register_module(THIS_MODULE, PF_INET6, RTM_NEWROUTE, 6564 inet6_rtm_newroute, NULL, 0); 6565 if (ret < 0) 6566 goto out_register_late_subsys; 6567 6568 ret = rtnl_register_module(THIS_MODULE, PF_INET6, RTM_DELROUTE, 6569 inet6_rtm_delroute, NULL, 0); 6570 if (ret < 0) 6571 goto out_register_late_subsys; 6572 6573 ret = rtnl_register_module(THIS_MODULE, PF_INET6, RTM_GETROUTE, 6574 inet6_rtm_getroute, NULL, 6575 RTNL_FLAG_DOIT_UNLOCKED); 6576 if (ret < 0) 6577 goto out_register_late_subsys; 6578 6579 ret = register_netdevice_notifier(&ip6_route_dev_notifier); 6580 if (ret) 6581 goto out_register_late_subsys; 6582 6583 #if IS_BUILTIN(CONFIG_IPV6) 6584 #if defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_PROC_FS) 6585 ret = bpf_iter_register(); 6586 if (ret) 6587 goto out_register_late_subsys; 6588 #endif 6589 #endif 6590 6591 for_each_possible_cpu(cpu) { 6592 struct uncached_list *ul = per_cpu_ptr(&rt6_uncached_list, cpu); 6593 6594 INIT_LIST_HEAD(&ul->head); 6595 spin_lock_init(&ul->lock); 6596 } 6597 6598 out: 6599 return ret; 6600 6601 out_register_late_subsys: 6602 rtnl_unregister_all(PF_INET6); 6603 unregister_pernet_subsys(&ip6_route_net_late_ops); 6604 fib6_rules_init: 6605 fib6_rules_cleanup(); 6606 xfrm6_init: 6607 xfrm6_fini(); 6608 out_fib6_init: 6609 fib6_gc_cleanup(); 6610 out_register_subsys: 6611 unregister_pernet_subsys(&ip6_route_net_ops); 6612 out_register_inetpeer: 6613 unregister_pernet_subsys(&ipv6_inetpeer_ops); 6614 out_dst_entries: 6615 dst_entries_destroy(&ip6_dst_blackhole_ops); 6616 out_kmem_cache: 6617 kmem_cache_destroy(ip6_dst_ops_template.kmem_cachep); 6618 goto out; 6619 } 6620 6621 void ip6_route_cleanup(void) 6622 { 6623 #if IS_BUILTIN(CONFIG_IPV6) 6624 #if defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_PROC_FS) 6625 bpf_iter_unregister(); 6626 #endif 6627 #endif 6628 unregister_netdevice_notifier(&ip6_route_dev_notifier); 6629 unregister_pernet_subsys(&ip6_route_net_late_ops); 6630 fib6_rules_cleanup(); 6631 xfrm6_fini(); 6632 fib6_gc_cleanup(); 6633 unregister_pernet_subsys(&ipv6_inetpeer_ops); 6634 unregister_pernet_subsys(&ip6_route_net_ops); 6635 dst_entries_destroy(&ip6_dst_blackhole_ops); 6636 kmem_cache_destroy(ip6_dst_ops_template.kmem_cachep); 6637 } 6638