1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Linux INET6 implementation 4 * FIB front-end. 5 * 6 * Authors: 7 * Pedro Roque <roque@di.fc.ul.pt> 8 */ 9 10 /* Changes: 11 * 12 * YOSHIFUJI Hideaki @USAGI 13 * reworked default router selection. 14 * - respect outgoing interface 15 * - select from (probably) reachable routers (i.e. 16 * routers in REACHABLE, STALE, DELAY or PROBE states). 17 * - always select the same router if it is (probably) 18 * reachable. otherwise, round-robin the list. 19 * Ville Nuorvala 20 * Fixed routing subtrees. 21 */ 22 23 #define pr_fmt(fmt) "IPv6: " fmt 24 25 #include <linux/capability.h> 26 #include <linux/errno.h> 27 #include <linux/export.h> 28 #include <linux/types.h> 29 #include <linux/times.h> 30 #include <linux/socket.h> 31 #include <linux/sockios.h> 32 #include <linux/net.h> 33 #include <linux/route.h> 34 #include <linux/netdevice.h> 35 #include <linux/in6.h> 36 #include <linux/mroute6.h> 37 #include <linux/init.h> 38 #include <linux/if_arp.h> 39 #include <linux/proc_fs.h> 40 #include <linux/seq_file.h> 41 #include <linux/nsproxy.h> 42 #include <linux/slab.h> 43 #include <linux/jhash.h> 44 #include <linux/siphash.h> 45 #include <net/net_namespace.h> 46 #include <net/snmp.h> 47 #include <net/ipv6.h> 48 #include <net/ip6_fib.h> 49 #include <net/ip6_route.h> 50 #include <net/ndisc.h> 51 #include <net/addrconf.h> 52 #include <net/tcp.h> 53 #include <linux/rtnetlink.h> 54 #include <net/dst.h> 55 #include <net/dst_metadata.h> 56 #include <net/xfrm.h> 57 #include <net/netevent.h> 58 #include <net/netlink.h> 59 #include <net/rtnh.h> 60 #include <net/lwtunnel.h> 61 #include <net/ip_tunnels.h> 62 #include <net/l3mdev.h> 63 #include <net/ip.h> 64 #include <linux/uaccess.h> 65 #include <linux/btf_ids.h> 66 67 #ifdef CONFIG_SYSCTL 68 #include <linux/sysctl.h> 69 #endif 70 71 static int ip6_rt_type_to_error(u8 fib6_type); 72 73 #define CREATE_TRACE_POINTS 74 #include <trace/events/fib6.h> 75 EXPORT_TRACEPOINT_SYMBOL_GPL(fib6_table_lookup); 76 #undef CREATE_TRACE_POINTS 77 78 enum rt6_nud_state { 79 RT6_NUD_FAIL_HARD = -3, 80 RT6_NUD_FAIL_PROBE = -2, 81 RT6_NUD_FAIL_DO_RR = -1, 82 RT6_NUD_SUCCEED = 1 83 }; 84 85 INDIRECT_CALLABLE_SCOPE 86 struct dst_entry *ip6_dst_check(struct dst_entry *dst, u32 cookie); 87 static unsigned int ip6_default_advmss(const struct dst_entry *dst); 88 INDIRECT_CALLABLE_SCOPE 89 unsigned int ip6_mtu(const struct dst_entry *dst); 90 static void ip6_negative_advice(struct sock *sk, 91 struct dst_entry *dst); 92 static void ip6_dst_destroy(struct dst_entry *); 93 static void ip6_dst_ifdown(struct dst_entry *, 94 struct net_device *dev); 95 static void ip6_dst_gc(struct dst_ops *ops); 96 97 static int ip6_pkt_discard(struct sk_buff *skb); 98 static int ip6_pkt_discard_out(struct net *net, struct sock *sk, struct sk_buff *skb); 99 static int ip6_pkt_prohibit(struct sk_buff *skb); 100 static int ip6_pkt_prohibit_out(struct net *net, struct sock *sk, struct sk_buff *skb); 101 static void ip6_link_failure(struct sk_buff *skb); 102 static void ip6_rt_update_pmtu(struct dst_entry *dst, struct sock *sk, 103 struct sk_buff *skb, u32 mtu, 104 bool confirm_neigh); 105 static void rt6_do_redirect(struct dst_entry *dst, struct sock *sk, 106 struct sk_buff *skb); 107 static int rt6_score_route(const struct fib6_nh *nh, u32 fib6_flags, int oif, 108 int strict); 109 static size_t rt6_nlmsg_size(struct fib6_info *f6i); 110 static int rt6_fill_node(struct net *net, struct sk_buff *skb, 111 struct fib6_info *rt, struct dst_entry *dst, 112 struct in6_addr *dest, struct in6_addr *src, 113 int iif, int type, u32 portid, u32 seq, 114 unsigned int flags); 115 static struct rt6_info *rt6_find_cached_rt(const struct fib6_result *res, 116 const struct in6_addr *daddr, 117 const struct in6_addr *saddr); 118 119 #ifdef CONFIG_IPV6_ROUTE_INFO 120 static struct fib6_info *rt6_add_route_info(struct net *net, 121 const struct in6_addr *prefix, int prefixlen, 122 const struct in6_addr *gwaddr, 123 struct net_device *dev, 124 unsigned int pref); 125 static struct fib6_info *rt6_get_route_info(struct net *net, 126 const struct in6_addr *prefix, int prefixlen, 127 const struct in6_addr *gwaddr, 128 struct net_device *dev); 129 #endif 130 131 struct uncached_list { 132 spinlock_t lock; 133 struct list_head head; 134 struct list_head quarantine; 135 }; 136 137 static DEFINE_PER_CPU_ALIGNED(struct uncached_list, rt6_uncached_list); 138 139 void rt6_uncached_list_add(struct rt6_info *rt) 140 { 141 struct uncached_list *ul = raw_cpu_ptr(&rt6_uncached_list); 142 143 rt->dst.rt_uncached_list = ul; 144 145 spin_lock_bh(&ul->lock); 146 list_add_tail(&rt->dst.rt_uncached, &ul->head); 147 spin_unlock_bh(&ul->lock); 148 } 149 150 void rt6_uncached_list_del(struct rt6_info *rt) 151 { 152 if (!list_empty(&rt->dst.rt_uncached)) { 153 struct uncached_list *ul = rt->dst.rt_uncached_list; 154 155 spin_lock_bh(&ul->lock); 156 list_del_init(&rt->dst.rt_uncached); 157 spin_unlock_bh(&ul->lock); 158 } 159 } 160 161 static void rt6_uncached_list_flush_dev(struct net_device *dev) 162 { 163 int cpu; 164 165 for_each_possible_cpu(cpu) { 166 struct uncached_list *ul = per_cpu_ptr(&rt6_uncached_list, cpu); 167 struct rt6_info *rt, *safe; 168 169 if (list_empty(&ul->head)) 170 continue; 171 172 spin_lock_bh(&ul->lock); 173 list_for_each_entry_safe(rt, safe, &ul->head, dst.rt_uncached) { 174 struct inet6_dev *rt_idev = rt->rt6i_idev; 175 struct net_device *rt_dev = rt->dst.dev; 176 bool handled = false; 177 178 if (rt_idev->dev == dev) { 179 rt->rt6i_idev = in6_dev_get(blackhole_netdev); 180 in6_dev_put(rt_idev); 181 handled = true; 182 } 183 184 if (rt_dev == dev) { 185 rt->dst.dev = blackhole_netdev; 186 netdev_ref_replace(rt_dev, blackhole_netdev, 187 &rt->dst.dev_tracker, 188 GFP_ATOMIC); 189 handled = true; 190 } 191 if (handled) 192 list_move(&rt->dst.rt_uncached, 193 &ul->quarantine); 194 } 195 spin_unlock_bh(&ul->lock); 196 } 197 } 198 199 static inline const void *choose_neigh_daddr(const struct in6_addr *p, 200 struct sk_buff *skb, 201 const void *daddr) 202 { 203 if (!ipv6_addr_any(p)) 204 return (const void *) p; 205 else if (skb) 206 return &ipv6_hdr(skb)->daddr; 207 return daddr; 208 } 209 210 struct neighbour *ip6_neigh_lookup(const struct in6_addr *gw, 211 struct net_device *dev, 212 struct sk_buff *skb, 213 const void *daddr) 214 { 215 struct neighbour *n; 216 217 daddr = choose_neigh_daddr(gw, skb, daddr); 218 n = __ipv6_neigh_lookup(dev, daddr); 219 if (n) 220 return n; 221 222 n = neigh_create(&nd_tbl, daddr, dev); 223 return IS_ERR(n) ? NULL : n; 224 } 225 226 static struct neighbour *ip6_dst_neigh_lookup(const struct dst_entry *dst, 227 struct sk_buff *skb, 228 const void *daddr) 229 { 230 const struct rt6_info *rt = container_of(dst, struct rt6_info, dst); 231 232 return ip6_neigh_lookup(rt6_nexthop(rt, &in6addr_any), 233 dst->dev, skb, daddr); 234 } 235 236 static void ip6_confirm_neigh(const struct dst_entry *dst, const void *daddr) 237 { 238 struct net_device *dev = dst->dev; 239 struct rt6_info *rt = (struct rt6_info *)dst; 240 241 daddr = choose_neigh_daddr(rt6_nexthop(rt, &in6addr_any), NULL, daddr); 242 if (!daddr) 243 return; 244 if (dev->flags & (IFF_NOARP | IFF_LOOPBACK)) 245 return; 246 if (ipv6_addr_is_multicast((const struct in6_addr *)daddr)) 247 return; 248 __ipv6_confirm_neigh(dev, daddr); 249 } 250 251 static struct dst_ops ip6_dst_ops_template = { 252 .family = AF_INET6, 253 .gc = ip6_dst_gc, 254 .gc_thresh = 1024, 255 .check = ip6_dst_check, 256 .default_advmss = ip6_default_advmss, 257 .mtu = ip6_mtu, 258 .cow_metrics = dst_cow_metrics_generic, 259 .destroy = ip6_dst_destroy, 260 .ifdown = ip6_dst_ifdown, 261 .negative_advice = ip6_negative_advice, 262 .link_failure = ip6_link_failure, 263 .update_pmtu = ip6_rt_update_pmtu, 264 .redirect = rt6_do_redirect, 265 .local_out = __ip6_local_out, 266 .neigh_lookup = ip6_dst_neigh_lookup, 267 .confirm_neigh = ip6_confirm_neigh, 268 }; 269 270 static struct dst_ops ip6_dst_blackhole_ops = { 271 .family = AF_INET6, 272 .default_advmss = ip6_default_advmss, 273 .neigh_lookup = ip6_dst_neigh_lookup, 274 .check = ip6_dst_check, 275 .destroy = ip6_dst_destroy, 276 .cow_metrics = dst_cow_metrics_generic, 277 .update_pmtu = dst_blackhole_update_pmtu, 278 .redirect = dst_blackhole_redirect, 279 .mtu = dst_blackhole_mtu, 280 }; 281 282 static const u32 ip6_template_metrics[RTAX_MAX] = { 283 [RTAX_HOPLIMIT - 1] = 0, 284 }; 285 286 static const struct fib6_info fib6_null_entry_template = { 287 .fib6_flags = (RTF_REJECT | RTF_NONEXTHOP), 288 .fib6_protocol = RTPROT_KERNEL, 289 .fib6_metric = ~(u32)0, 290 .fib6_ref = REFCOUNT_INIT(1), 291 .fib6_type = RTN_UNREACHABLE, 292 .fib6_metrics = (struct dst_metrics *)&dst_default_metrics, 293 }; 294 295 static const struct rt6_info ip6_null_entry_template = { 296 .dst = { 297 .__rcuref = RCUREF_INIT(1), 298 .__use = 1, 299 .obsolete = DST_OBSOLETE_FORCE_CHK, 300 .error = -ENETUNREACH, 301 .input = ip6_pkt_discard, 302 .output = ip6_pkt_discard_out, 303 }, 304 .rt6i_flags = (RTF_REJECT | RTF_NONEXTHOP), 305 }; 306 307 #ifdef CONFIG_IPV6_MULTIPLE_TABLES 308 309 static const struct rt6_info ip6_prohibit_entry_template = { 310 .dst = { 311 .__rcuref = RCUREF_INIT(1), 312 .__use = 1, 313 .obsolete = DST_OBSOLETE_FORCE_CHK, 314 .error = -EACCES, 315 .input = ip6_pkt_prohibit, 316 .output = ip6_pkt_prohibit_out, 317 }, 318 .rt6i_flags = (RTF_REJECT | RTF_NONEXTHOP), 319 }; 320 321 static const struct rt6_info ip6_blk_hole_entry_template = { 322 .dst = { 323 .__rcuref = RCUREF_INIT(1), 324 .__use = 1, 325 .obsolete = DST_OBSOLETE_FORCE_CHK, 326 .error = -EINVAL, 327 .input = dst_discard, 328 .output = dst_discard_out, 329 }, 330 .rt6i_flags = (RTF_REJECT | RTF_NONEXTHOP), 331 }; 332 333 #endif 334 335 static void rt6_info_init(struct rt6_info *rt) 336 { 337 memset_after(rt, 0, dst); 338 } 339 340 /* allocate dst with ip6_dst_ops */ 341 struct rt6_info *ip6_dst_alloc(struct net *net, struct net_device *dev, 342 int flags) 343 { 344 struct rt6_info *rt = dst_alloc(&net->ipv6.ip6_dst_ops, dev, 345 1, DST_OBSOLETE_FORCE_CHK, flags); 346 347 if (rt) { 348 rt6_info_init(rt); 349 atomic_inc(&net->ipv6.rt6_stats->fib_rt_alloc); 350 } 351 352 return rt; 353 } 354 EXPORT_SYMBOL(ip6_dst_alloc); 355 356 static void ip6_dst_destroy(struct dst_entry *dst) 357 { 358 struct rt6_info *rt = (struct rt6_info *)dst; 359 struct fib6_info *from; 360 struct inet6_dev *idev; 361 362 ip_dst_metrics_put(dst); 363 rt6_uncached_list_del(rt); 364 365 idev = rt->rt6i_idev; 366 if (idev) { 367 rt->rt6i_idev = NULL; 368 in6_dev_put(idev); 369 } 370 371 from = xchg((__force struct fib6_info **)&rt->from, NULL); 372 fib6_info_release(from); 373 } 374 375 static void ip6_dst_ifdown(struct dst_entry *dst, struct net_device *dev) 376 { 377 struct rt6_info *rt = (struct rt6_info *)dst; 378 struct inet6_dev *idev = rt->rt6i_idev; 379 380 if (idev && idev->dev != blackhole_netdev) { 381 struct inet6_dev *blackhole_idev = in6_dev_get(blackhole_netdev); 382 383 if (blackhole_idev) { 384 rt->rt6i_idev = blackhole_idev; 385 in6_dev_put(idev); 386 } 387 } 388 } 389 390 static bool __rt6_check_expired(const struct rt6_info *rt) 391 { 392 if (rt->rt6i_flags & RTF_EXPIRES) 393 return time_after(jiffies, rt->dst.expires); 394 else 395 return false; 396 } 397 398 static bool rt6_check_expired(const struct rt6_info *rt) 399 { 400 struct fib6_info *from; 401 402 from = rcu_dereference(rt->from); 403 404 if (rt->rt6i_flags & RTF_EXPIRES) { 405 if (time_after(jiffies, rt->dst.expires)) 406 return true; 407 } else if (from) { 408 return rt->dst.obsolete != DST_OBSOLETE_FORCE_CHK || 409 fib6_check_expired(from); 410 } 411 return false; 412 } 413 414 void fib6_select_path(const struct net *net, struct fib6_result *res, 415 struct flowi6 *fl6, int oif, bool have_oif_match, 416 const struct sk_buff *skb, int strict) 417 { 418 struct fib6_info *sibling, *next_sibling; 419 struct fib6_info *match = res->f6i; 420 421 if (!match->nh && (!match->fib6_nsiblings || have_oif_match)) 422 goto out; 423 424 if (match->nh && have_oif_match && res->nh) 425 return; 426 427 if (skb) 428 IP6CB(skb)->flags |= IP6SKB_MULTIPATH; 429 430 /* We might have already computed the hash for ICMPv6 errors. In such 431 * case it will always be non-zero. Otherwise now is the time to do it. 432 */ 433 if (!fl6->mp_hash && 434 (!match->nh || nexthop_is_multipath(match->nh))) 435 fl6->mp_hash = rt6_multipath_hash(net, fl6, skb, NULL); 436 437 if (unlikely(match->nh)) { 438 nexthop_path_fib6_result(res, fl6->mp_hash); 439 return; 440 } 441 442 if (fl6->mp_hash <= atomic_read(&match->fib6_nh->fib_nh_upper_bound)) 443 goto out; 444 445 list_for_each_entry_safe(sibling, next_sibling, &match->fib6_siblings, 446 fib6_siblings) { 447 const struct fib6_nh *nh = sibling->fib6_nh; 448 int nh_upper_bound; 449 450 nh_upper_bound = atomic_read(&nh->fib_nh_upper_bound); 451 if (fl6->mp_hash > nh_upper_bound) 452 continue; 453 if (rt6_score_route(nh, sibling->fib6_flags, oif, strict) < 0) 454 break; 455 match = sibling; 456 break; 457 } 458 459 out: 460 res->f6i = match; 461 res->nh = match->fib6_nh; 462 } 463 464 /* 465 * Route lookup. rcu_read_lock() should be held. 466 */ 467 468 static bool __rt6_device_match(struct net *net, const struct fib6_nh *nh, 469 const struct in6_addr *saddr, int oif, int flags) 470 { 471 const struct net_device *dev; 472 473 if (nh->fib_nh_flags & RTNH_F_DEAD) 474 return false; 475 476 dev = nh->fib_nh_dev; 477 if (oif) { 478 if (dev->ifindex == oif) 479 return true; 480 } else { 481 if (ipv6_chk_addr(net, saddr, dev, 482 flags & RT6_LOOKUP_F_IFACE)) 483 return true; 484 } 485 486 return false; 487 } 488 489 struct fib6_nh_dm_arg { 490 struct net *net; 491 const struct in6_addr *saddr; 492 int oif; 493 int flags; 494 struct fib6_nh *nh; 495 }; 496 497 static int __rt6_nh_dev_match(struct fib6_nh *nh, void *_arg) 498 { 499 struct fib6_nh_dm_arg *arg = _arg; 500 501 arg->nh = nh; 502 return __rt6_device_match(arg->net, nh, arg->saddr, arg->oif, 503 arg->flags); 504 } 505 506 /* returns fib6_nh from nexthop or NULL */ 507 static struct fib6_nh *rt6_nh_dev_match(struct net *net, struct nexthop *nh, 508 struct fib6_result *res, 509 const struct in6_addr *saddr, 510 int oif, int flags) 511 { 512 struct fib6_nh_dm_arg arg = { 513 .net = net, 514 .saddr = saddr, 515 .oif = oif, 516 .flags = flags, 517 }; 518 519 if (nexthop_is_blackhole(nh)) 520 return NULL; 521 522 if (nexthop_for_each_fib6_nh(nh, __rt6_nh_dev_match, &arg)) 523 return arg.nh; 524 525 return NULL; 526 } 527 528 static void rt6_device_match(struct net *net, struct fib6_result *res, 529 const struct in6_addr *saddr, int oif, int flags) 530 { 531 struct fib6_info *f6i = res->f6i; 532 struct fib6_info *spf6i; 533 struct fib6_nh *nh; 534 535 if (!oif && ipv6_addr_any(saddr)) { 536 if (unlikely(f6i->nh)) { 537 nh = nexthop_fib6_nh(f6i->nh); 538 if (nexthop_is_blackhole(f6i->nh)) 539 goto out_blackhole; 540 } else { 541 nh = f6i->fib6_nh; 542 } 543 if (!(nh->fib_nh_flags & RTNH_F_DEAD)) 544 goto out; 545 } 546 547 for (spf6i = f6i; spf6i; spf6i = rcu_dereference(spf6i->fib6_next)) { 548 bool matched = false; 549 550 if (unlikely(spf6i->nh)) { 551 nh = rt6_nh_dev_match(net, spf6i->nh, res, saddr, 552 oif, flags); 553 if (nh) 554 matched = true; 555 } else { 556 nh = spf6i->fib6_nh; 557 if (__rt6_device_match(net, nh, saddr, oif, flags)) 558 matched = true; 559 } 560 if (matched) { 561 res->f6i = spf6i; 562 goto out; 563 } 564 } 565 566 if (oif && flags & RT6_LOOKUP_F_IFACE) { 567 res->f6i = net->ipv6.fib6_null_entry; 568 nh = res->f6i->fib6_nh; 569 goto out; 570 } 571 572 if (unlikely(f6i->nh)) { 573 nh = nexthop_fib6_nh(f6i->nh); 574 if (nexthop_is_blackhole(f6i->nh)) 575 goto out_blackhole; 576 } else { 577 nh = f6i->fib6_nh; 578 } 579 580 if (nh->fib_nh_flags & RTNH_F_DEAD) { 581 res->f6i = net->ipv6.fib6_null_entry; 582 nh = res->f6i->fib6_nh; 583 } 584 out: 585 res->nh = nh; 586 res->fib6_type = res->f6i->fib6_type; 587 res->fib6_flags = res->f6i->fib6_flags; 588 return; 589 590 out_blackhole: 591 res->fib6_flags |= RTF_REJECT; 592 res->fib6_type = RTN_BLACKHOLE; 593 res->nh = nh; 594 } 595 596 #ifdef CONFIG_IPV6_ROUTER_PREF 597 struct __rt6_probe_work { 598 struct work_struct work; 599 struct in6_addr target; 600 struct net_device *dev; 601 netdevice_tracker dev_tracker; 602 }; 603 604 static void rt6_probe_deferred(struct work_struct *w) 605 { 606 struct in6_addr mcaddr; 607 struct __rt6_probe_work *work = 608 container_of(w, struct __rt6_probe_work, work); 609 610 addrconf_addr_solict_mult(&work->target, &mcaddr); 611 ndisc_send_ns(work->dev, &work->target, &mcaddr, NULL, 0); 612 netdev_put(work->dev, &work->dev_tracker); 613 kfree(work); 614 } 615 616 static void rt6_probe(struct fib6_nh *fib6_nh) 617 { 618 struct __rt6_probe_work *work = NULL; 619 const struct in6_addr *nh_gw; 620 unsigned long last_probe; 621 struct neighbour *neigh; 622 struct net_device *dev; 623 struct inet6_dev *idev; 624 625 /* 626 * Okay, this does not seem to be appropriate 627 * for now, however, we need to check if it 628 * is really so; aka Router Reachability Probing. 629 * 630 * Router Reachability Probe MUST be rate-limited 631 * to no more than one per minute. 632 */ 633 if (!fib6_nh->fib_nh_gw_family) 634 return; 635 636 nh_gw = &fib6_nh->fib_nh_gw6; 637 dev = fib6_nh->fib_nh_dev; 638 rcu_read_lock(); 639 last_probe = READ_ONCE(fib6_nh->last_probe); 640 idev = __in6_dev_get(dev); 641 neigh = __ipv6_neigh_lookup_noref(dev, nh_gw); 642 if (neigh) { 643 if (READ_ONCE(neigh->nud_state) & NUD_VALID) 644 goto out; 645 646 write_lock_bh(&neigh->lock); 647 if (!(neigh->nud_state & NUD_VALID) && 648 time_after(jiffies, 649 neigh->updated + idev->cnf.rtr_probe_interval)) { 650 work = kmalloc(sizeof(*work), GFP_ATOMIC); 651 if (work) 652 __neigh_set_probe_once(neigh); 653 } 654 write_unlock_bh(&neigh->lock); 655 } else if (time_after(jiffies, last_probe + 656 idev->cnf.rtr_probe_interval)) { 657 work = kmalloc(sizeof(*work), GFP_ATOMIC); 658 } 659 660 if (!work || cmpxchg(&fib6_nh->last_probe, 661 last_probe, jiffies) != last_probe) { 662 kfree(work); 663 } else { 664 INIT_WORK(&work->work, rt6_probe_deferred); 665 work->target = *nh_gw; 666 netdev_hold(dev, &work->dev_tracker, GFP_ATOMIC); 667 work->dev = dev; 668 schedule_work(&work->work); 669 } 670 671 out: 672 rcu_read_unlock(); 673 } 674 #else 675 static inline void rt6_probe(struct fib6_nh *fib6_nh) 676 { 677 } 678 #endif 679 680 /* 681 * Default Router Selection (RFC 2461 6.3.6) 682 */ 683 static enum rt6_nud_state rt6_check_neigh(const struct fib6_nh *fib6_nh) 684 { 685 enum rt6_nud_state ret = RT6_NUD_FAIL_HARD; 686 struct neighbour *neigh; 687 688 rcu_read_lock(); 689 neigh = __ipv6_neigh_lookup_noref(fib6_nh->fib_nh_dev, 690 &fib6_nh->fib_nh_gw6); 691 if (neigh) { 692 u8 nud_state = READ_ONCE(neigh->nud_state); 693 694 if (nud_state & NUD_VALID) 695 ret = RT6_NUD_SUCCEED; 696 #ifdef CONFIG_IPV6_ROUTER_PREF 697 else if (!(nud_state & NUD_FAILED)) 698 ret = RT6_NUD_SUCCEED; 699 else 700 ret = RT6_NUD_FAIL_PROBE; 701 #endif 702 } else { 703 ret = IS_ENABLED(CONFIG_IPV6_ROUTER_PREF) ? 704 RT6_NUD_SUCCEED : RT6_NUD_FAIL_DO_RR; 705 } 706 rcu_read_unlock(); 707 708 return ret; 709 } 710 711 static int rt6_score_route(const struct fib6_nh *nh, u32 fib6_flags, int oif, 712 int strict) 713 { 714 int m = 0; 715 716 if (!oif || nh->fib_nh_dev->ifindex == oif) 717 m = 2; 718 719 if (!m && (strict & RT6_LOOKUP_F_IFACE)) 720 return RT6_NUD_FAIL_HARD; 721 #ifdef CONFIG_IPV6_ROUTER_PREF 722 m |= IPV6_DECODE_PREF(IPV6_EXTRACT_PREF(fib6_flags)) << 2; 723 #endif 724 if ((strict & RT6_LOOKUP_F_REACHABLE) && 725 !(fib6_flags & RTF_NONEXTHOP) && nh->fib_nh_gw_family) { 726 int n = rt6_check_neigh(nh); 727 if (n < 0) 728 return n; 729 } 730 return m; 731 } 732 733 static bool find_match(struct fib6_nh *nh, u32 fib6_flags, 734 int oif, int strict, int *mpri, bool *do_rr) 735 { 736 bool match_do_rr = false; 737 bool rc = false; 738 int m; 739 740 if (nh->fib_nh_flags & RTNH_F_DEAD) 741 goto out; 742 743 if (ip6_ignore_linkdown(nh->fib_nh_dev) && 744 nh->fib_nh_flags & RTNH_F_LINKDOWN && 745 !(strict & RT6_LOOKUP_F_IGNORE_LINKSTATE)) 746 goto out; 747 748 m = rt6_score_route(nh, fib6_flags, oif, strict); 749 if (m == RT6_NUD_FAIL_DO_RR) { 750 match_do_rr = true; 751 m = 0; /* lowest valid score */ 752 } else if (m == RT6_NUD_FAIL_HARD) { 753 goto out; 754 } 755 756 if (strict & RT6_LOOKUP_F_REACHABLE) 757 rt6_probe(nh); 758 759 /* note that m can be RT6_NUD_FAIL_PROBE at this point */ 760 if (m > *mpri) { 761 *do_rr = match_do_rr; 762 *mpri = m; 763 rc = true; 764 } 765 out: 766 return rc; 767 } 768 769 struct fib6_nh_frl_arg { 770 u32 flags; 771 int oif; 772 int strict; 773 int *mpri; 774 bool *do_rr; 775 struct fib6_nh *nh; 776 }; 777 778 static int rt6_nh_find_match(struct fib6_nh *nh, void *_arg) 779 { 780 struct fib6_nh_frl_arg *arg = _arg; 781 782 arg->nh = nh; 783 return find_match(nh, arg->flags, arg->oif, arg->strict, 784 arg->mpri, arg->do_rr); 785 } 786 787 static void __find_rr_leaf(struct fib6_info *f6i_start, 788 struct fib6_info *nomatch, u32 metric, 789 struct fib6_result *res, struct fib6_info **cont, 790 int oif, int strict, bool *do_rr, int *mpri) 791 { 792 struct fib6_info *f6i; 793 794 for (f6i = f6i_start; 795 f6i && f6i != nomatch; 796 f6i = rcu_dereference(f6i->fib6_next)) { 797 bool matched = false; 798 struct fib6_nh *nh; 799 800 if (cont && f6i->fib6_metric != metric) { 801 *cont = f6i; 802 return; 803 } 804 805 if (fib6_check_expired(f6i)) 806 continue; 807 808 if (unlikely(f6i->nh)) { 809 struct fib6_nh_frl_arg arg = { 810 .flags = f6i->fib6_flags, 811 .oif = oif, 812 .strict = strict, 813 .mpri = mpri, 814 .do_rr = do_rr 815 }; 816 817 if (nexthop_is_blackhole(f6i->nh)) { 818 res->fib6_flags = RTF_REJECT; 819 res->fib6_type = RTN_BLACKHOLE; 820 res->f6i = f6i; 821 res->nh = nexthop_fib6_nh(f6i->nh); 822 return; 823 } 824 if (nexthop_for_each_fib6_nh(f6i->nh, rt6_nh_find_match, 825 &arg)) { 826 matched = true; 827 nh = arg.nh; 828 } 829 } else { 830 nh = f6i->fib6_nh; 831 if (find_match(nh, f6i->fib6_flags, oif, strict, 832 mpri, do_rr)) 833 matched = true; 834 } 835 if (matched) { 836 res->f6i = f6i; 837 res->nh = nh; 838 res->fib6_flags = f6i->fib6_flags; 839 res->fib6_type = f6i->fib6_type; 840 } 841 } 842 } 843 844 static void find_rr_leaf(struct fib6_node *fn, struct fib6_info *leaf, 845 struct fib6_info *rr_head, int oif, int strict, 846 bool *do_rr, struct fib6_result *res) 847 { 848 u32 metric = rr_head->fib6_metric; 849 struct fib6_info *cont = NULL; 850 int mpri = -1; 851 852 __find_rr_leaf(rr_head, NULL, metric, res, &cont, 853 oif, strict, do_rr, &mpri); 854 855 __find_rr_leaf(leaf, rr_head, metric, res, &cont, 856 oif, strict, do_rr, &mpri); 857 858 if (res->f6i || !cont) 859 return; 860 861 __find_rr_leaf(cont, NULL, metric, res, NULL, 862 oif, strict, do_rr, &mpri); 863 } 864 865 static void rt6_select(struct net *net, struct fib6_node *fn, int oif, 866 struct fib6_result *res, int strict) 867 { 868 struct fib6_info *leaf = rcu_dereference(fn->leaf); 869 struct fib6_info *rt0; 870 bool do_rr = false; 871 int key_plen; 872 873 /* make sure this function or its helpers sets f6i */ 874 res->f6i = NULL; 875 876 if (!leaf || leaf == net->ipv6.fib6_null_entry) 877 goto out; 878 879 rt0 = rcu_dereference(fn->rr_ptr); 880 if (!rt0) 881 rt0 = leaf; 882 883 /* Double check to make sure fn is not an intermediate node 884 * and fn->leaf does not points to its child's leaf 885 * (This might happen if all routes under fn are deleted from 886 * the tree and fib6_repair_tree() is called on the node.) 887 */ 888 key_plen = rt0->fib6_dst.plen; 889 #ifdef CONFIG_IPV6_SUBTREES 890 if (rt0->fib6_src.plen) 891 key_plen = rt0->fib6_src.plen; 892 #endif 893 if (fn->fn_bit != key_plen) 894 goto out; 895 896 find_rr_leaf(fn, leaf, rt0, oif, strict, &do_rr, res); 897 if (do_rr) { 898 struct fib6_info *next = rcu_dereference(rt0->fib6_next); 899 900 /* no entries matched; do round-robin */ 901 if (!next || next->fib6_metric != rt0->fib6_metric) 902 next = leaf; 903 904 if (next != rt0) { 905 spin_lock_bh(&leaf->fib6_table->tb6_lock); 906 /* make sure next is not being deleted from the tree */ 907 if (next->fib6_node) 908 rcu_assign_pointer(fn->rr_ptr, next); 909 spin_unlock_bh(&leaf->fib6_table->tb6_lock); 910 } 911 } 912 913 out: 914 if (!res->f6i) { 915 res->f6i = net->ipv6.fib6_null_entry; 916 res->nh = res->f6i->fib6_nh; 917 res->fib6_flags = res->f6i->fib6_flags; 918 res->fib6_type = res->f6i->fib6_type; 919 } 920 } 921 922 static bool rt6_is_gw_or_nonexthop(const struct fib6_result *res) 923 { 924 return (res->f6i->fib6_flags & RTF_NONEXTHOP) || 925 res->nh->fib_nh_gw_family; 926 } 927 928 #ifdef CONFIG_IPV6_ROUTE_INFO 929 int rt6_route_rcv(struct net_device *dev, u8 *opt, int len, 930 const struct in6_addr *gwaddr) 931 { 932 struct net *net = dev_net(dev); 933 struct route_info *rinfo = (struct route_info *) opt; 934 struct in6_addr prefix_buf, *prefix; 935 unsigned int pref; 936 unsigned long lifetime; 937 struct fib6_info *rt; 938 939 if (len < sizeof(struct route_info)) { 940 return -EINVAL; 941 } 942 943 /* Sanity check for prefix_len and length */ 944 if (rinfo->length > 3) { 945 return -EINVAL; 946 } else if (rinfo->prefix_len > 128) { 947 return -EINVAL; 948 } else if (rinfo->prefix_len > 64) { 949 if (rinfo->length < 2) { 950 return -EINVAL; 951 } 952 } else if (rinfo->prefix_len > 0) { 953 if (rinfo->length < 1) { 954 return -EINVAL; 955 } 956 } 957 958 pref = rinfo->route_pref; 959 if (pref == ICMPV6_ROUTER_PREF_INVALID) 960 return -EINVAL; 961 962 lifetime = addrconf_timeout_fixup(ntohl(rinfo->lifetime), HZ); 963 964 if (rinfo->length == 3) 965 prefix = (struct in6_addr *)rinfo->prefix; 966 else { 967 /* this function is safe */ 968 ipv6_addr_prefix(&prefix_buf, 969 (struct in6_addr *)rinfo->prefix, 970 rinfo->prefix_len); 971 prefix = &prefix_buf; 972 } 973 974 if (rinfo->prefix_len == 0) 975 rt = rt6_get_dflt_router(net, gwaddr, dev); 976 else 977 rt = rt6_get_route_info(net, prefix, rinfo->prefix_len, 978 gwaddr, dev); 979 980 if (rt && !lifetime) { 981 ip6_del_rt(net, rt, false); 982 rt = NULL; 983 } 984 985 if (!rt && lifetime) 986 rt = rt6_add_route_info(net, prefix, rinfo->prefix_len, gwaddr, 987 dev, pref); 988 else if (rt) 989 rt->fib6_flags = RTF_ROUTEINFO | 990 (rt->fib6_flags & ~RTF_PREF_MASK) | RTF_PREF(pref); 991 992 if (rt) { 993 if (!addrconf_finite_timeout(lifetime)) 994 fib6_clean_expires(rt); 995 else 996 fib6_set_expires(rt, jiffies + HZ * lifetime); 997 998 fib6_info_release(rt); 999 } 1000 return 0; 1001 } 1002 #endif 1003 1004 /* 1005 * Misc support functions 1006 */ 1007 1008 /* called with rcu_lock held */ 1009 static struct net_device *ip6_rt_get_dev_rcu(const struct fib6_result *res) 1010 { 1011 struct net_device *dev = res->nh->fib_nh_dev; 1012 1013 if (res->fib6_flags & (RTF_LOCAL | RTF_ANYCAST)) { 1014 /* for copies of local routes, dst->dev needs to be the 1015 * device if it is a master device, the master device if 1016 * device is enslaved, and the loopback as the default 1017 */ 1018 if (netif_is_l3_slave(dev) && 1019 !rt6_need_strict(&res->f6i->fib6_dst.addr)) 1020 dev = l3mdev_master_dev_rcu(dev); 1021 else if (!netif_is_l3_master(dev)) 1022 dev = dev_net(dev)->loopback_dev; 1023 /* last case is netif_is_l3_master(dev) is true in which 1024 * case we want dev returned to be dev 1025 */ 1026 } 1027 1028 return dev; 1029 } 1030 1031 static const int fib6_prop[RTN_MAX + 1] = { 1032 [RTN_UNSPEC] = 0, 1033 [RTN_UNICAST] = 0, 1034 [RTN_LOCAL] = 0, 1035 [RTN_BROADCAST] = 0, 1036 [RTN_ANYCAST] = 0, 1037 [RTN_MULTICAST] = 0, 1038 [RTN_BLACKHOLE] = -EINVAL, 1039 [RTN_UNREACHABLE] = -EHOSTUNREACH, 1040 [RTN_PROHIBIT] = -EACCES, 1041 [RTN_THROW] = -EAGAIN, 1042 [RTN_NAT] = -EINVAL, 1043 [RTN_XRESOLVE] = -EINVAL, 1044 }; 1045 1046 static int ip6_rt_type_to_error(u8 fib6_type) 1047 { 1048 return fib6_prop[fib6_type]; 1049 } 1050 1051 static unsigned short fib6_info_dst_flags(struct fib6_info *rt) 1052 { 1053 unsigned short flags = 0; 1054 1055 if (rt->dst_nocount) 1056 flags |= DST_NOCOUNT; 1057 if (rt->dst_nopolicy) 1058 flags |= DST_NOPOLICY; 1059 1060 return flags; 1061 } 1062 1063 static void ip6_rt_init_dst_reject(struct rt6_info *rt, u8 fib6_type) 1064 { 1065 rt->dst.error = ip6_rt_type_to_error(fib6_type); 1066 1067 switch (fib6_type) { 1068 case RTN_BLACKHOLE: 1069 rt->dst.output = dst_discard_out; 1070 rt->dst.input = dst_discard; 1071 break; 1072 case RTN_PROHIBIT: 1073 rt->dst.output = ip6_pkt_prohibit_out; 1074 rt->dst.input = ip6_pkt_prohibit; 1075 break; 1076 case RTN_THROW: 1077 case RTN_UNREACHABLE: 1078 default: 1079 rt->dst.output = ip6_pkt_discard_out; 1080 rt->dst.input = ip6_pkt_discard; 1081 break; 1082 } 1083 } 1084 1085 static void ip6_rt_init_dst(struct rt6_info *rt, const struct fib6_result *res) 1086 { 1087 struct fib6_info *f6i = res->f6i; 1088 1089 if (res->fib6_flags & RTF_REJECT) { 1090 ip6_rt_init_dst_reject(rt, res->fib6_type); 1091 return; 1092 } 1093 1094 rt->dst.error = 0; 1095 rt->dst.output = ip6_output; 1096 1097 if (res->fib6_type == RTN_LOCAL || res->fib6_type == RTN_ANYCAST) { 1098 rt->dst.input = ip6_input; 1099 } else if (ipv6_addr_type(&f6i->fib6_dst.addr) & IPV6_ADDR_MULTICAST) { 1100 rt->dst.input = ip6_mc_input; 1101 } else { 1102 rt->dst.input = ip6_forward; 1103 } 1104 1105 if (res->nh->fib_nh_lws) { 1106 rt->dst.lwtstate = lwtstate_get(res->nh->fib_nh_lws); 1107 lwtunnel_set_redirect(&rt->dst); 1108 } 1109 1110 rt->dst.lastuse = jiffies; 1111 } 1112 1113 /* Caller must already hold reference to @from */ 1114 static void rt6_set_from(struct rt6_info *rt, struct fib6_info *from) 1115 { 1116 rt->rt6i_flags &= ~RTF_EXPIRES; 1117 rcu_assign_pointer(rt->from, from); 1118 ip_dst_init_metrics(&rt->dst, from->fib6_metrics); 1119 } 1120 1121 /* Caller must already hold reference to f6i in result */ 1122 static void ip6_rt_copy_init(struct rt6_info *rt, const struct fib6_result *res) 1123 { 1124 const struct fib6_nh *nh = res->nh; 1125 const struct net_device *dev = nh->fib_nh_dev; 1126 struct fib6_info *f6i = res->f6i; 1127 1128 ip6_rt_init_dst(rt, res); 1129 1130 rt->rt6i_dst = f6i->fib6_dst; 1131 rt->rt6i_idev = dev ? in6_dev_get(dev) : NULL; 1132 rt->rt6i_flags = res->fib6_flags; 1133 if (nh->fib_nh_gw_family) { 1134 rt->rt6i_gateway = nh->fib_nh_gw6; 1135 rt->rt6i_flags |= RTF_GATEWAY; 1136 } 1137 rt6_set_from(rt, f6i); 1138 #ifdef CONFIG_IPV6_SUBTREES 1139 rt->rt6i_src = f6i->fib6_src; 1140 #endif 1141 } 1142 1143 static struct fib6_node* fib6_backtrack(struct fib6_node *fn, 1144 struct in6_addr *saddr) 1145 { 1146 struct fib6_node *pn, *sn; 1147 while (1) { 1148 if (fn->fn_flags & RTN_TL_ROOT) 1149 return NULL; 1150 pn = rcu_dereference(fn->parent); 1151 sn = FIB6_SUBTREE(pn); 1152 if (sn && sn != fn) 1153 fn = fib6_node_lookup(sn, NULL, saddr); 1154 else 1155 fn = pn; 1156 if (fn->fn_flags & RTN_RTINFO) 1157 return fn; 1158 } 1159 } 1160 1161 static bool ip6_hold_safe(struct net *net, struct rt6_info **prt) 1162 { 1163 struct rt6_info *rt = *prt; 1164 1165 if (dst_hold_safe(&rt->dst)) 1166 return true; 1167 if (net) { 1168 rt = net->ipv6.ip6_null_entry; 1169 dst_hold(&rt->dst); 1170 } else { 1171 rt = NULL; 1172 } 1173 *prt = rt; 1174 return false; 1175 } 1176 1177 /* called with rcu_lock held */ 1178 static struct rt6_info *ip6_create_rt_rcu(const struct fib6_result *res) 1179 { 1180 struct net_device *dev = res->nh->fib_nh_dev; 1181 struct fib6_info *f6i = res->f6i; 1182 unsigned short flags; 1183 struct rt6_info *nrt; 1184 1185 if (!fib6_info_hold_safe(f6i)) 1186 goto fallback; 1187 1188 flags = fib6_info_dst_flags(f6i); 1189 nrt = ip6_dst_alloc(dev_net(dev), dev, flags); 1190 if (!nrt) { 1191 fib6_info_release(f6i); 1192 goto fallback; 1193 } 1194 1195 ip6_rt_copy_init(nrt, res); 1196 return nrt; 1197 1198 fallback: 1199 nrt = dev_net(dev)->ipv6.ip6_null_entry; 1200 dst_hold(&nrt->dst); 1201 return nrt; 1202 } 1203 1204 INDIRECT_CALLABLE_SCOPE struct rt6_info *ip6_pol_route_lookup(struct net *net, 1205 struct fib6_table *table, 1206 struct flowi6 *fl6, 1207 const struct sk_buff *skb, 1208 int flags) 1209 { 1210 struct fib6_result res = {}; 1211 struct fib6_node *fn; 1212 struct rt6_info *rt; 1213 1214 rcu_read_lock(); 1215 fn = fib6_node_lookup(&table->tb6_root, &fl6->daddr, &fl6->saddr); 1216 restart: 1217 res.f6i = rcu_dereference(fn->leaf); 1218 if (!res.f6i) 1219 res.f6i = net->ipv6.fib6_null_entry; 1220 else 1221 rt6_device_match(net, &res, &fl6->saddr, fl6->flowi6_oif, 1222 flags); 1223 1224 if (res.f6i == net->ipv6.fib6_null_entry) { 1225 fn = fib6_backtrack(fn, &fl6->saddr); 1226 if (fn) 1227 goto restart; 1228 1229 rt = net->ipv6.ip6_null_entry; 1230 dst_hold(&rt->dst); 1231 goto out; 1232 } else if (res.fib6_flags & RTF_REJECT) { 1233 goto do_create; 1234 } 1235 1236 fib6_select_path(net, &res, fl6, fl6->flowi6_oif, 1237 fl6->flowi6_oif != 0, skb, flags); 1238 1239 /* Search through exception table */ 1240 rt = rt6_find_cached_rt(&res, &fl6->daddr, &fl6->saddr); 1241 if (rt) { 1242 if (ip6_hold_safe(net, &rt)) 1243 dst_use_noref(&rt->dst, jiffies); 1244 } else { 1245 do_create: 1246 rt = ip6_create_rt_rcu(&res); 1247 } 1248 1249 out: 1250 trace_fib6_table_lookup(net, &res, table, fl6); 1251 1252 rcu_read_unlock(); 1253 1254 return rt; 1255 } 1256 1257 struct dst_entry *ip6_route_lookup(struct net *net, struct flowi6 *fl6, 1258 const struct sk_buff *skb, int flags) 1259 { 1260 return fib6_rule_lookup(net, fl6, skb, flags, ip6_pol_route_lookup); 1261 } 1262 EXPORT_SYMBOL_GPL(ip6_route_lookup); 1263 1264 struct rt6_info *rt6_lookup(struct net *net, const struct in6_addr *daddr, 1265 const struct in6_addr *saddr, int oif, 1266 const struct sk_buff *skb, int strict) 1267 { 1268 struct flowi6 fl6 = { 1269 .flowi6_oif = oif, 1270 .daddr = *daddr, 1271 }; 1272 struct dst_entry *dst; 1273 int flags = strict ? RT6_LOOKUP_F_IFACE : 0; 1274 1275 if (saddr) { 1276 memcpy(&fl6.saddr, saddr, sizeof(*saddr)); 1277 flags |= RT6_LOOKUP_F_HAS_SADDR; 1278 } 1279 1280 dst = fib6_rule_lookup(net, &fl6, skb, flags, ip6_pol_route_lookup); 1281 if (dst->error == 0) 1282 return (struct rt6_info *) dst; 1283 1284 dst_release(dst); 1285 1286 return NULL; 1287 } 1288 EXPORT_SYMBOL(rt6_lookup); 1289 1290 /* ip6_ins_rt is called with FREE table->tb6_lock. 1291 * It takes new route entry, the addition fails by any reason the 1292 * route is released. 1293 * Caller must hold dst before calling it. 1294 */ 1295 1296 static int __ip6_ins_rt(struct fib6_info *rt, struct nl_info *info, 1297 struct netlink_ext_ack *extack) 1298 { 1299 int err; 1300 struct fib6_table *table; 1301 1302 table = rt->fib6_table; 1303 spin_lock_bh(&table->tb6_lock); 1304 err = fib6_add(&table->tb6_root, rt, info, extack); 1305 spin_unlock_bh(&table->tb6_lock); 1306 1307 return err; 1308 } 1309 1310 int ip6_ins_rt(struct net *net, struct fib6_info *rt) 1311 { 1312 struct nl_info info = { .nl_net = net, }; 1313 1314 return __ip6_ins_rt(rt, &info, NULL); 1315 } 1316 1317 static struct rt6_info *ip6_rt_cache_alloc(const struct fib6_result *res, 1318 const struct in6_addr *daddr, 1319 const struct in6_addr *saddr) 1320 { 1321 struct fib6_info *f6i = res->f6i; 1322 struct net_device *dev; 1323 struct rt6_info *rt; 1324 1325 /* 1326 * Clone the route. 1327 */ 1328 1329 if (!fib6_info_hold_safe(f6i)) 1330 return NULL; 1331 1332 dev = ip6_rt_get_dev_rcu(res); 1333 rt = ip6_dst_alloc(dev_net(dev), dev, 0); 1334 if (!rt) { 1335 fib6_info_release(f6i); 1336 return NULL; 1337 } 1338 1339 ip6_rt_copy_init(rt, res); 1340 rt->rt6i_flags |= RTF_CACHE; 1341 rt->rt6i_dst.addr = *daddr; 1342 rt->rt6i_dst.plen = 128; 1343 1344 if (!rt6_is_gw_or_nonexthop(res)) { 1345 if (f6i->fib6_dst.plen != 128 && 1346 ipv6_addr_equal(&f6i->fib6_dst.addr, daddr)) 1347 rt->rt6i_flags |= RTF_ANYCAST; 1348 #ifdef CONFIG_IPV6_SUBTREES 1349 if (rt->rt6i_src.plen && saddr) { 1350 rt->rt6i_src.addr = *saddr; 1351 rt->rt6i_src.plen = 128; 1352 } 1353 #endif 1354 } 1355 1356 return rt; 1357 } 1358 1359 static struct rt6_info *ip6_rt_pcpu_alloc(const struct fib6_result *res) 1360 { 1361 struct fib6_info *f6i = res->f6i; 1362 unsigned short flags = fib6_info_dst_flags(f6i); 1363 struct net_device *dev; 1364 struct rt6_info *pcpu_rt; 1365 1366 if (!fib6_info_hold_safe(f6i)) 1367 return NULL; 1368 1369 rcu_read_lock(); 1370 dev = ip6_rt_get_dev_rcu(res); 1371 pcpu_rt = ip6_dst_alloc(dev_net(dev), dev, flags | DST_NOCOUNT); 1372 rcu_read_unlock(); 1373 if (!pcpu_rt) { 1374 fib6_info_release(f6i); 1375 return NULL; 1376 } 1377 ip6_rt_copy_init(pcpu_rt, res); 1378 pcpu_rt->rt6i_flags |= RTF_PCPU; 1379 1380 if (f6i->nh) 1381 pcpu_rt->sernum = rt_genid_ipv6(dev_net(dev)); 1382 1383 return pcpu_rt; 1384 } 1385 1386 static bool rt6_is_valid(const struct rt6_info *rt6) 1387 { 1388 return rt6->sernum == rt_genid_ipv6(dev_net(rt6->dst.dev)); 1389 } 1390 1391 /* It should be called with rcu_read_lock() acquired */ 1392 static struct rt6_info *rt6_get_pcpu_route(const struct fib6_result *res) 1393 { 1394 struct rt6_info *pcpu_rt; 1395 1396 pcpu_rt = this_cpu_read(*res->nh->rt6i_pcpu); 1397 1398 if (pcpu_rt && pcpu_rt->sernum && !rt6_is_valid(pcpu_rt)) { 1399 struct rt6_info *prev, **p; 1400 1401 p = this_cpu_ptr(res->nh->rt6i_pcpu); 1402 prev = xchg(p, NULL); 1403 if (prev) { 1404 dst_dev_put(&prev->dst); 1405 dst_release(&prev->dst); 1406 } 1407 1408 pcpu_rt = NULL; 1409 } 1410 1411 return pcpu_rt; 1412 } 1413 1414 static struct rt6_info *rt6_make_pcpu_route(struct net *net, 1415 const struct fib6_result *res) 1416 { 1417 struct rt6_info *pcpu_rt, *prev, **p; 1418 1419 pcpu_rt = ip6_rt_pcpu_alloc(res); 1420 if (!pcpu_rt) 1421 return NULL; 1422 1423 p = this_cpu_ptr(res->nh->rt6i_pcpu); 1424 prev = cmpxchg(p, NULL, pcpu_rt); 1425 BUG_ON(prev); 1426 1427 if (res->f6i->fib6_destroying) { 1428 struct fib6_info *from; 1429 1430 from = xchg((__force struct fib6_info **)&pcpu_rt->from, NULL); 1431 fib6_info_release(from); 1432 } 1433 1434 return pcpu_rt; 1435 } 1436 1437 /* exception hash table implementation 1438 */ 1439 static DEFINE_SPINLOCK(rt6_exception_lock); 1440 1441 /* Remove rt6_ex from hash table and free the memory 1442 * Caller must hold rt6_exception_lock 1443 */ 1444 static void rt6_remove_exception(struct rt6_exception_bucket *bucket, 1445 struct rt6_exception *rt6_ex) 1446 { 1447 struct fib6_info *from; 1448 struct net *net; 1449 1450 if (!bucket || !rt6_ex) 1451 return; 1452 1453 net = dev_net(rt6_ex->rt6i->dst.dev); 1454 net->ipv6.rt6_stats->fib_rt_cache--; 1455 1456 /* purge completely the exception to allow releasing the held resources: 1457 * some [sk] cache may keep the dst around for unlimited time 1458 */ 1459 from = xchg((__force struct fib6_info **)&rt6_ex->rt6i->from, NULL); 1460 fib6_info_release(from); 1461 dst_dev_put(&rt6_ex->rt6i->dst); 1462 1463 hlist_del_rcu(&rt6_ex->hlist); 1464 dst_release(&rt6_ex->rt6i->dst); 1465 kfree_rcu(rt6_ex, rcu); 1466 WARN_ON_ONCE(!bucket->depth); 1467 bucket->depth--; 1468 } 1469 1470 /* Remove oldest rt6_ex in bucket and free the memory 1471 * Caller must hold rt6_exception_lock 1472 */ 1473 static void rt6_exception_remove_oldest(struct rt6_exception_bucket *bucket) 1474 { 1475 struct rt6_exception *rt6_ex, *oldest = NULL; 1476 1477 if (!bucket) 1478 return; 1479 1480 hlist_for_each_entry(rt6_ex, &bucket->chain, hlist) { 1481 if (!oldest || time_before(rt6_ex->stamp, oldest->stamp)) 1482 oldest = rt6_ex; 1483 } 1484 rt6_remove_exception(bucket, oldest); 1485 } 1486 1487 static u32 rt6_exception_hash(const struct in6_addr *dst, 1488 const struct in6_addr *src) 1489 { 1490 static siphash_aligned_key_t rt6_exception_key; 1491 struct { 1492 struct in6_addr dst; 1493 struct in6_addr src; 1494 } __aligned(SIPHASH_ALIGNMENT) combined = { 1495 .dst = *dst, 1496 }; 1497 u64 val; 1498 1499 net_get_random_once(&rt6_exception_key, sizeof(rt6_exception_key)); 1500 1501 #ifdef CONFIG_IPV6_SUBTREES 1502 if (src) 1503 combined.src = *src; 1504 #endif 1505 val = siphash(&combined, sizeof(combined), &rt6_exception_key); 1506 1507 return hash_64(val, FIB6_EXCEPTION_BUCKET_SIZE_SHIFT); 1508 } 1509 1510 /* Helper function to find the cached rt in the hash table 1511 * and update bucket pointer to point to the bucket for this 1512 * (daddr, saddr) pair 1513 * Caller must hold rt6_exception_lock 1514 */ 1515 static struct rt6_exception * 1516 __rt6_find_exception_spinlock(struct rt6_exception_bucket **bucket, 1517 const struct in6_addr *daddr, 1518 const struct in6_addr *saddr) 1519 { 1520 struct rt6_exception *rt6_ex; 1521 u32 hval; 1522 1523 if (!(*bucket) || !daddr) 1524 return NULL; 1525 1526 hval = rt6_exception_hash(daddr, saddr); 1527 *bucket += hval; 1528 1529 hlist_for_each_entry(rt6_ex, &(*bucket)->chain, hlist) { 1530 struct rt6_info *rt6 = rt6_ex->rt6i; 1531 bool matched = ipv6_addr_equal(daddr, &rt6->rt6i_dst.addr); 1532 1533 #ifdef CONFIG_IPV6_SUBTREES 1534 if (matched && saddr) 1535 matched = ipv6_addr_equal(saddr, &rt6->rt6i_src.addr); 1536 #endif 1537 if (matched) 1538 return rt6_ex; 1539 } 1540 return NULL; 1541 } 1542 1543 /* Helper function to find the cached rt in the hash table 1544 * and update bucket pointer to point to the bucket for this 1545 * (daddr, saddr) pair 1546 * Caller must hold rcu_read_lock() 1547 */ 1548 static struct rt6_exception * 1549 __rt6_find_exception_rcu(struct rt6_exception_bucket **bucket, 1550 const struct in6_addr *daddr, 1551 const struct in6_addr *saddr) 1552 { 1553 struct rt6_exception *rt6_ex; 1554 u32 hval; 1555 1556 WARN_ON_ONCE(!rcu_read_lock_held()); 1557 1558 if (!(*bucket) || !daddr) 1559 return NULL; 1560 1561 hval = rt6_exception_hash(daddr, saddr); 1562 *bucket += hval; 1563 1564 hlist_for_each_entry_rcu(rt6_ex, &(*bucket)->chain, hlist) { 1565 struct rt6_info *rt6 = rt6_ex->rt6i; 1566 bool matched = ipv6_addr_equal(daddr, &rt6->rt6i_dst.addr); 1567 1568 #ifdef CONFIG_IPV6_SUBTREES 1569 if (matched && saddr) 1570 matched = ipv6_addr_equal(saddr, &rt6->rt6i_src.addr); 1571 #endif 1572 if (matched) 1573 return rt6_ex; 1574 } 1575 return NULL; 1576 } 1577 1578 static unsigned int fib6_mtu(const struct fib6_result *res) 1579 { 1580 const struct fib6_nh *nh = res->nh; 1581 unsigned int mtu; 1582 1583 if (res->f6i->fib6_pmtu) { 1584 mtu = res->f6i->fib6_pmtu; 1585 } else { 1586 struct net_device *dev = nh->fib_nh_dev; 1587 struct inet6_dev *idev; 1588 1589 rcu_read_lock(); 1590 idev = __in6_dev_get(dev); 1591 mtu = idev->cnf.mtu6; 1592 rcu_read_unlock(); 1593 } 1594 1595 mtu = min_t(unsigned int, mtu, IP6_MAX_MTU); 1596 1597 return mtu - lwtunnel_headroom(nh->fib_nh_lws, mtu); 1598 } 1599 1600 #define FIB6_EXCEPTION_BUCKET_FLUSHED 0x1UL 1601 1602 /* used when the flushed bit is not relevant, only access to the bucket 1603 * (ie., all bucket users except rt6_insert_exception); 1604 * 1605 * called under rcu lock; sometimes called with rt6_exception_lock held 1606 */ 1607 static 1608 struct rt6_exception_bucket *fib6_nh_get_excptn_bucket(const struct fib6_nh *nh, 1609 spinlock_t *lock) 1610 { 1611 struct rt6_exception_bucket *bucket; 1612 1613 if (lock) 1614 bucket = rcu_dereference_protected(nh->rt6i_exception_bucket, 1615 lockdep_is_held(lock)); 1616 else 1617 bucket = rcu_dereference(nh->rt6i_exception_bucket); 1618 1619 /* remove bucket flushed bit if set */ 1620 if (bucket) { 1621 unsigned long p = (unsigned long)bucket; 1622 1623 p &= ~FIB6_EXCEPTION_BUCKET_FLUSHED; 1624 bucket = (struct rt6_exception_bucket *)p; 1625 } 1626 1627 return bucket; 1628 } 1629 1630 static bool fib6_nh_excptn_bucket_flushed(struct rt6_exception_bucket *bucket) 1631 { 1632 unsigned long p = (unsigned long)bucket; 1633 1634 return !!(p & FIB6_EXCEPTION_BUCKET_FLUSHED); 1635 } 1636 1637 /* called with rt6_exception_lock held */ 1638 static void fib6_nh_excptn_bucket_set_flushed(struct fib6_nh *nh, 1639 spinlock_t *lock) 1640 { 1641 struct rt6_exception_bucket *bucket; 1642 unsigned long p; 1643 1644 bucket = rcu_dereference_protected(nh->rt6i_exception_bucket, 1645 lockdep_is_held(lock)); 1646 1647 p = (unsigned long)bucket; 1648 p |= FIB6_EXCEPTION_BUCKET_FLUSHED; 1649 bucket = (struct rt6_exception_bucket *)p; 1650 rcu_assign_pointer(nh->rt6i_exception_bucket, bucket); 1651 } 1652 1653 static int rt6_insert_exception(struct rt6_info *nrt, 1654 const struct fib6_result *res) 1655 { 1656 struct net *net = dev_net(nrt->dst.dev); 1657 struct rt6_exception_bucket *bucket; 1658 struct fib6_info *f6i = res->f6i; 1659 struct in6_addr *src_key = NULL; 1660 struct rt6_exception *rt6_ex; 1661 struct fib6_nh *nh = res->nh; 1662 int max_depth; 1663 int err = 0; 1664 1665 spin_lock_bh(&rt6_exception_lock); 1666 1667 bucket = rcu_dereference_protected(nh->rt6i_exception_bucket, 1668 lockdep_is_held(&rt6_exception_lock)); 1669 if (!bucket) { 1670 bucket = kcalloc(FIB6_EXCEPTION_BUCKET_SIZE, sizeof(*bucket), 1671 GFP_ATOMIC); 1672 if (!bucket) { 1673 err = -ENOMEM; 1674 goto out; 1675 } 1676 rcu_assign_pointer(nh->rt6i_exception_bucket, bucket); 1677 } else if (fib6_nh_excptn_bucket_flushed(bucket)) { 1678 err = -EINVAL; 1679 goto out; 1680 } 1681 1682 #ifdef CONFIG_IPV6_SUBTREES 1683 /* fib6_src.plen != 0 indicates f6i is in subtree 1684 * and exception table is indexed by a hash of 1685 * both fib6_dst and fib6_src. 1686 * Otherwise, the exception table is indexed by 1687 * a hash of only fib6_dst. 1688 */ 1689 if (f6i->fib6_src.plen) 1690 src_key = &nrt->rt6i_src.addr; 1691 #endif 1692 /* rt6_mtu_change() might lower mtu on f6i. 1693 * Only insert this exception route if its mtu 1694 * is less than f6i's mtu value. 1695 */ 1696 if (dst_metric_raw(&nrt->dst, RTAX_MTU) >= fib6_mtu(res)) { 1697 err = -EINVAL; 1698 goto out; 1699 } 1700 1701 rt6_ex = __rt6_find_exception_spinlock(&bucket, &nrt->rt6i_dst.addr, 1702 src_key); 1703 if (rt6_ex) 1704 rt6_remove_exception(bucket, rt6_ex); 1705 1706 rt6_ex = kzalloc(sizeof(*rt6_ex), GFP_ATOMIC); 1707 if (!rt6_ex) { 1708 err = -ENOMEM; 1709 goto out; 1710 } 1711 rt6_ex->rt6i = nrt; 1712 rt6_ex->stamp = jiffies; 1713 hlist_add_head_rcu(&rt6_ex->hlist, &bucket->chain); 1714 bucket->depth++; 1715 net->ipv6.rt6_stats->fib_rt_cache++; 1716 1717 /* Randomize max depth to avoid some side channels attacks. */ 1718 max_depth = FIB6_MAX_DEPTH + get_random_u32_below(FIB6_MAX_DEPTH); 1719 while (bucket->depth > max_depth) 1720 rt6_exception_remove_oldest(bucket); 1721 1722 out: 1723 spin_unlock_bh(&rt6_exception_lock); 1724 1725 /* Update fn->fn_sernum to invalidate all cached dst */ 1726 if (!err) { 1727 spin_lock_bh(&f6i->fib6_table->tb6_lock); 1728 fib6_update_sernum(net, f6i); 1729 spin_unlock_bh(&f6i->fib6_table->tb6_lock); 1730 fib6_force_start_gc(net); 1731 } 1732 1733 return err; 1734 } 1735 1736 static void fib6_nh_flush_exceptions(struct fib6_nh *nh, struct fib6_info *from) 1737 { 1738 struct rt6_exception_bucket *bucket; 1739 struct rt6_exception *rt6_ex; 1740 struct hlist_node *tmp; 1741 int i; 1742 1743 spin_lock_bh(&rt6_exception_lock); 1744 1745 bucket = fib6_nh_get_excptn_bucket(nh, &rt6_exception_lock); 1746 if (!bucket) 1747 goto out; 1748 1749 /* Prevent rt6_insert_exception() to recreate the bucket list */ 1750 if (!from) 1751 fib6_nh_excptn_bucket_set_flushed(nh, &rt6_exception_lock); 1752 1753 for (i = 0; i < FIB6_EXCEPTION_BUCKET_SIZE; i++) { 1754 hlist_for_each_entry_safe(rt6_ex, tmp, &bucket->chain, hlist) { 1755 if (!from || 1756 rcu_access_pointer(rt6_ex->rt6i->from) == from) 1757 rt6_remove_exception(bucket, rt6_ex); 1758 } 1759 WARN_ON_ONCE(!from && bucket->depth); 1760 bucket++; 1761 } 1762 out: 1763 spin_unlock_bh(&rt6_exception_lock); 1764 } 1765 1766 static int rt6_nh_flush_exceptions(struct fib6_nh *nh, void *arg) 1767 { 1768 struct fib6_info *f6i = arg; 1769 1770 fib6_nh_flush_exceptions(nh, f6i); 1771 1772 return 0; 1773 } 1774 1775 void rt6_flush_exceptions(struct fib6_info *f6i) 1776 { 1777 if (f6i->nh) 1778 nexthop_for_each_fib6_nh(f6i->nh, rt6_nh_flush_exceptions, 1779 f6i); 1780 else 1781 fib6_nh_flush_exceptions(f6i->fib6_nh, f6i); 1782 } 1783 1784 /* Find cached rt in the hash table inside passed in rt 1785 * Caller has to hold rcu_read_lock() 1786 */ 1787 static struct rt6_info *rt6_find_cached_rt(const struct fib6_result *res, 1788 const struct in6_addr *daddr, 1789 const struct in6_addr *saddr) 1790 { 1791 const struct in6_addr *src_key = NULL; 1792 struct rt6_exception_bucket *bucket; 1793 struct rt6_exception *rt6_ex; 1794 struct rt6_info *ret = NULL; 1795 1796 #ifdef CONFIG_IPV6_SUBTREES 1797 /* fib6i_src.plen != 0 indicates f6i is in subtree 1798 * and exception table is indexed by a hash of 1799 * both fib6_dst and fib6_src. 1800 * However, the src addr used to create the hash 1801 * might not be exactly the passed in saddr which 1802 * is a /128 addr from the flow. 1803 * So we need to use f6i->fib6_src to redo lookup 1804 * if the passed in saddr does not find anything. 1805 * (See the logic in ip6_rt_cache_alloc() on how 1806 * rt->rt6i_src is updated.) 1807 */ 1808 if (res->f6i->fib6_src.plen) 1809 src_key = saddr; 1810 find_ex: 1811 #endif 1812 bucket = fib6_nh_get_excptn_bucket(res->nh, NULL); 1813 rt6_ex = __rt6_find_exception_rcu(&bucket, daddr, src_key); 1814 1815 if (rt6_ex && !rt6_check_expired(rt6_ex->rt6i)) 1816 ret = rt6_ex->rt6i; 1817 1818 #ifdef CONFIG_IPV6_SUBTREES 1819 /* Use fib6_src as src_key and redo lookup */ 1820 if (!ret && src_key && src_key != &res->f6i->fib6_src.addr) { 1821 src_key = &res->f6i->fib6_src.addr; 1822 goto find_ex; 1823 } 1824 #endif 1825 1826 return ret; 1827 } 1828 1829 /* Remove the passed in cached rt from the hash table that contains it */ 1830 static int fib6_nh_remove_exception(const struct fib6_nh *nh, int plen, 1831 const struct rt6_info *rt) 1832 { 1833 const struct in6_addr *src_key = NULL; 1834 struct rt6_exception_bucket *bucket; 1835 struct rt6_exception *rt6_ex; 1836 int err; 1837 1838 if (!rcu_access_pointer(nh->rt6i_exception_bucket)) 1839 return -ENOENT; 1840 1841 spin_lock_bh(&rt6_exception_lock); 1842 bucket = fib6_nh_get_excptn_bucket(nh, &rt6_exception_lock); 1843 1844 #ifdef CONFIG_IPV6_SUBTREES 1845 /* rt6i_src.plen != 0 indicates 'from' is in subtree 1846 * and exception table is indexed by a hash of 1847 * both rt6i_dst and rt6i_src. 1848 * Otherwise, the exception table is indexed by 1849 * a hash of only rt6i_dst. 1850 */ 1851 if (plen) 1852 src_key = &rt->rt6i_src.addr; 1853 #endif 1854 rt6_ex = __rt6_find_exception_spinlock(&bucket, 1855 &rt->rt6i_dst.addr, 1856 src_key); 1857 if (rt6_ex) { 1858 rt6_remove_exception(bucket, rt6_ex); 1859 err = 0; 1860 } else { 1861 err = -ENOENT; 1862 } 1863 1864 spin_unlock_bh(&rt6_exception_lock); 1865 return err; 1866 } 1867 1868 struct fib6_nh_excptn_arg { 1869 struct rt6_info *rt; 1870 int plen; 1871 }; 1872 1873 static int rt6_nh_remove_exception_rt(struct fib6_nh *nh, void *_arg) 1874 { 1875 struct fib6_nh_excptn_arg *arg = _arg; 1876 int err; 1877 1878 err = fib6_nh_remove_exception(nh, arg->plen, arg->rt); 1879 if (err == 0) 1880 return 1; 1881 1882 return 0; 1883 } 1884 1885 static int rt6_remove_exception_rt(struct rt6_info *rt) 1886 { 1887 struct fib6_info *from; 1888 1889 from = rcu_dereference(rt->from); 1890 if (!from || !(rt->rt6i_flags & RTF_CACHE)) 1891 return -EINVAL; 1892 1893 if (from->nh) { 1894 struct fib6_nh_excptn_arg arg = { 1895 .rt = rt, 1896 .plen = from->fib6_src.plen 1897 }; 1898 int rc; 1899 1900 /* rc = 1 means an entry was found */ 1901 rc = nexthop_for_each_fib6_nh(from->nh, 1902 rt6_nh_remove_exception_rt, 1903 &arg); 1904 return rc ? 0 : -ENOENT; 1905 } 1906 1907 return fib6_nh_remove_exception(from->fib6_nh, 1908 from->fib6_src.plen, rt); 1909 } 1910 1911 /* Find rt6_ex which contains the passed in rt cache and 1912 * refresh its stamp 1913 */ 1914 static void fib6_nh_update_exception(const struct fib6_nh *nh, int plen, 1915 const struct rt6_info *rt) 1916 { 1917 const struct in6_addr *src_key = NULL; 1918 struct rt6_exception_bucket *bucket; 1919 struct rt6_exception *rt6_ex; 1920 1921 bucket = fib6_nh_get_excptn_bucket(nh, NULL); 1922 #ifdef CONFIG_IPV6_SUBTREES 1923 /* rt6i_src.plen != 0 indicates 'from' is in subtree 1924 * and exception table is indexed by a hash of 1925 * both rt6i_dst and rt6i_src. 1926 * Otherwise, the exception table is indexed by 1927 * a hash of only rt6i_dst. 1928 */ 1929 if (plen) 1930 src_key = &rt->rt6i_src.addr; 1931 #endif 1932 rt6_ex = __rt6_find_exception_rcu(&bucket, &rt->rt6i_dst.addr, src_key); 1933 if (rt6_ex) 1934 rt6_ex->stamp = jiffies; 1935 } 1936 1937 struct fib6_nh_match_arg { 1938 const struct net_device *dev; 1939 const struct in6_addr *gw; 1940 struct fib6_nh *match; 1941 }; 1942 1943 /* determine if fib6_nh has given device and gateway */ 1944 static int fib6_nh_find_match(struct fib6_nh *nh, void *_arg) 1945 { 1946 struct fib6_nh_match_arg *arg = _arg; 1947 1948 if (arg->dev != nh->fib_nh_dev || 1949 (arg->gw && !nh->fib_nh_gw_family) || 1950 (!arg->gw && nh->fib_nh_gw_family) || 1951 (arg->gw && !ipv6_addr_equal(arg->gw, &nh->fib_nh_gw6))) 1952 return 0; 1953 1954 arg->match = nh; 1955 1956 /* found a match, break the loop */ 1957 return 1; 1958 } 1959 1960 static void rt6_update_exception_stamp_rt(struct rt6_info *rt) 1961 { 1962 struct fib6_info *from; 1963 struct fib6_nh *fib6_nh; 1964 1965 rcu_read_lock(); 1966 1967 from = rcu_dereference(rt->from); 1968 if (!from || !(rt->rt6i_flags & RTF_CACHE)) 1969 goto unlock; 1970 1971 if (from->nh) { 1972 struct fib6_nh_match_arg arg = { 1973 .dev = rt->dst.dev, 1974 .gw = &rt->rt6i_gateway, 1975 }; 1976 1977 nexthop_for_each_fib6_nh(from->nh, fib6_nh_find_match, &arg); 1978 1979 if (!arg.match) 1980 goto unlock; 1981 fib6_nh = arg.match; 1982 } else { 1983 fib6_nh = from->fib6_nh; 1984 } 1985 fib6_nh_update_exception(fib6_nh, from->fib6_src.plen, rt); 1986 unlock: 1987 rcu_read_unlock(); 1988 } 1989 1990 static bool rt6_mtu_change_route_allowed(struct inet6_dev *idev, 1991 struct rt6_info *rt, int mtu) 1992 { 1993 /* If the new MTU is lower than the route PMTU, this new MTU will be the 1994 * lowest MTU in the path: always allow updating the route PMTU to 1995 * reflect PMTU decreases. 1996 * 1997 * If the new MTU is higher, and the route PMTU is equal to the local 1998 * MTU, this means the old MTU is the lowest in the path, so allow 1999 * updating it: if other nodes now have lower MTUs, PMTU discovery will 2000 * handle this. 2001 */ 2002 2003 if (dst_mtu(&rt->dst) >= mtu) 2004 return true; 2005 2006 if (dst_mtu(&rt->dst) == idev->cnf.mtu6) 2007 return true; 2008 2009 return false; 2010 } 2011 2012 static void rt6_exceptions_update_pmtu(struct inet6_dev *idev, 2013 const struct fib6_nh *nh, int mtu) 2014 { 2015 struct rt6_exception_bucket *bucket; 2016 struct rt6_exception *rt6_ex; 2017 int i; 2018 2019 bucket = fib6_nh_get_excptn_bucket(nh, &rt6_exception_lock); 2020 if (!bucket) 2021 return; 2022 2023 for (i = 0; i < FIB6_EXCEPTION_BUCKET_SIZE; i++) { 2024 hlist_for_each_entry(rt6_ex, &bucket->chain, hlist) { 2025 struct rt6_info *entry = rt6_ex->rt6i; 2026 2027 /* For RTF_CACHE with rt6i_pmtu == 0 (i.e. a redirected 2028 * route), the metrics of its rt->from have already 2029 * been updated. 2030 */ 2031 if (dst_metric_raw(&entry->dst, RTAX_MTU) && 2032 rt6_mtu_change_route_allowed(idev, entry, mtu)) 2033 dst_metric_set(&entry->dst, RTAX_MTU, mtu); 2034 } 2035 bucket++; 2036 } 2037 } 2038 2039 #define RTF_CACHE_GATEWAY (RTF_GATEWAY | RTF_CACHE) 2040 2041 static void fib6_nh_exceptions_clean_tohost(const struct fib6_nh *nh, 2042 const struct in6_addr *gateway) 2043 { 2044 struct rt6_exception_bucket *bucket; 2045 struct rt6_exception *rt6_ex; 2046 struct hlist_node *tmp; 2047 int i; 2048 2049 if (!rcu_access_pointer(nh->rt6i_exception_bucket)) 2050 return; 2051 2052 spin_lock_bh(&rt6_exception_lock); 2053 bucket = fib6_nh_get_excptn_bucket(nh, &rt6_exception_lock); 2054 if (bucket) { 2055 for (i = 0; i < FIB6_EXCEPTION_BUCKET_SIZE; i++) { 2056 hlist_for_each_entry_safe(rt6_ex, tmp, 2057 &bucket->chain, hlist) { 2058 struct rt6_info *entry = rt6_ex->rt6i; 2059 2060 if ((entry->rt6i_flags & RTF_CACHE_GATEWAY) == 2061 RTF_CACHE_GATEWAY && 2062 ipv6_addr_equal(gateway, 2063 &entry->rt6i_gateway)) { 2064 rt6_remove_exception(bucket, rt6_ex); 2065 } 2066 } 2067 bucket++; 2068 } 2069 } 2070 2071 spin_unlock_bh(&rt6_exception_lock); 2072 } 2073 2074 static void rt6_age_examine_exception(struct rt6_exception_bucket *bucket, 2075 struct rt6_exception *rt6_ex, 2076 struct fib6_gc_args *gc_args, 2077 unsigned long now) 2078 { 2079 struct rt6_info *rt = rt6_ex->rt6i; 2080 2081 /* we are pruning and obsoleting aged-out and non gateway exceptions 2082 * even if others have still references to them, so that on next 2083 * dst_check() such references can be dropped. 2084 * EXPIRES exceptions - e.g. pmtu-generated ones are pruned when 2085 * expired, independently from their aging, as per RFC 8201 section 4 2086 */ 2087 if (!(rt->rt6i_flags & RTF_EXPIRES)) { 2088 if (time_after_eq(now, rt->dst.lastuse + gc_args->timeout)) { 2089 RT6_TRACE("aging clone %p\n", rt); 2090 rt6_remove_exception(bucket, rt6_ex); 2091 return; 2092 } 2093 } else if (time_after(jiffies, rt->dst.expires)) { 2094 RT6_TRACE("purging expired route %p\n", rt); 2095 rt6_remove_exception(bucket, rt6_ex); 2096 return; 2097 } 2098 2099 if (rt->rt6i_flags & RTF_GATEWAY) { 2100 struct neighbour *neigh; 2101 2102 neigh = __ipv6_neigh_lookup_noref(rt->dst.dev, &rt->rt6i_gateway); 2103 2104 if (!(neigh && (neigh->flags & NTF_ROUTER))) { 2105 RT6_TRACE("purging route %p via non-router but gateway\n", 2106 rt); 2107 rt6_remove_exception(bucket, rt6_ex); 2108 return; 2109 } 2110 } 2111 2112 gc_args->more++; 2113 } 2114 2115 static void fib6_nh_age_exceptions(const struct fib6_nh *nh, 2116 struct fib6_gc_args *gc_args, 2117 unsigned long now) 2118 { 2119 struct rt6_exception_bucket *bucket; 2120 struct rt6_exception *rt6_ex; 2121 struct hlist_node *tmp; 2122 int i; 2123 2124 if (!rcu_access_pointer(nh->rt6i_exception_bucket)) 2125 return; 2126 2127 rcu_read_lock_bh(); 2128 spin_lock(&rt6_exception_lock); 2129 bucket = fib6_nh_get_excptn_bucket(nh, &rt6_exception_lock); 2130 if (bucket) { 2131 for (i = 0; i < FIB6_EXCEPTION_BUCKET_SIZE; i++) { 2132 hlist_for_each_entry_safe(rt6_ex, tmp, 2133 &bucket->chain, hlist) { 2134 rt6_age_examine_exception(bucket, rt6_ex, 2135 gc_args, now); 2136 } 2137 bucket++; 2138 } 2139 } 2140 spin_unlock(&rt6_exception_lock); 2141 rcu_read_unlock_bh(); 2142 } 2143 2144 struct fib6_nh_age_excptn_arg { 2145 struct fib6_gc_args *gc_args; 2146 unsigned long now; 2147 }; 2148 2149 static int rt6_nh_age_exceptions(struct fib6_nh *nh, void *_arg) 2150 { 2151 struct fib6_nh_age_excptn_arg *arg = _arg; 2152 2153 fib6_nh_age_exceptions(nh, arg->gc_args, arg->now); 2154 return 0; 2155 } 2156 2157 void rt6_age_exceptions(struct fib6_info *f6i, 2158 struct fib6_gc_args *gc_args, 2159 unsigned long now) 2160 { 2161 if (f6i->nh) { 2162 struct fib6_nh_age_excptn_arg arg = { 2163 .gc_args = gc_args, 2164 .now = now 2165 }; 2166 2167 nexthop_for_each_fib6_nh(f6i->nh, rt6_nh_age_exceptions, 2168 &arg); 2169 } else { 2170 fib6_nh_age_exceptions(f6i->fib6_nh, gc_args, now); 2171 } 2172 } 2173 2174 /* must be called with rcu lock held */ 2175 int fib6_table_lookup(struct net *net, struct fib6_table *table, int oif, 2176 struct flowi6 *fl6, struct fib6_result *res, int strict) 2177 { 2178 struct fib6_node *fn, *saved_fn; 2179 2180 fn = fib6_node_lookup(&table->tb6_root, &fl6->daddr, &fl6->saddr); 2181 saved_fn = fn; 2182 2183 redo_rt6_select: 2184 rt6_select(net, fn, oif, res, strict); 2185 if (res->f6i == net->ipv6.fib6_null_entry) { 2186 fn = fib6_backtrack(fn, &fl6->saddr); 2187 if (fn) 2188 goto redo_rt6_select; 2189 else if (strict & RT6_LOOKUP_F_REACHABLE) { 2190 /* also consider unreachable route */ 2191 strict &= ~RT6_LOOKUP_F_REACHABLE; 2192 fn = saved_fn; 2193 goto redo_rt6_select; 2194 } 2195 } 2196 2197 trace_fib6_table_lookup(net, res, table, fl6); 2198 2199 return 0; 2200 } 2201 2202 struct rt6_info *ip6_pol_route(struct net *net, struct fib6_table *table, 2203 int oif, struct flowi6 *fl6, 2204 const struct sk_buff *skb, int flags) 2205 { 2206 struct fib6_result res = {}; 2207 struct rt6_info *rt = NULL; 2208 int strict = 0; 2209 2210 WARN_ON_ONCE((flags & RT6_LOOKUP_F_DST_NOREF) && 2211 !rcu_read_lock_held()); 2212 2213 strict |= flags & RT6_LOOKUP_F_IFACE; 2214 strict |= flags & RT6_LOOKUP_F_IGNORE_LINKSTATE; 2215 if (net->ipv6.devconf_all->forwarding == 0) 2216 strict |= RT6_LOOKUP_F_REACHABLE; 2217 2218 rcu_read_lock(); 2219 2220 fib6_table_lookup(net, table, oif, fl6, &res, strict); 2221 if (res.f6i == net->ipv6.fib6_null_entry) 2222 goto out; 2223 2224 fib6_select_path(net, &res, fl6, oif, false, skb, strict); 2225 2226 /*Search through exception table */ 2227 rt = rt6_find_cached_rt(&res, &fl6->daddr, &fl6->saddr); 2228 if (rt) { 2229 goto out; 2230 } else if (unlikely((fl6->flowi6_flags & FLOWI_FLAG_KNOWN_NH) && 2231 !res.nh->fib_nh_gw_family)) { 2232 /* Create a RTF_CACHE clone which will not be 2233 * owned by the fib6 tree. It is for the special case where 2234 * the daddr in the skb during the neighbor look-up is different 2235 * from the fl6->daddr used to look-up route here. 2236 */ 2237 rt = ip6_rt_cache_alloc(&res, &fl6->daddr, NULL); 2238 2239 if (rt) { 2240 /* 1 refcnt is taken during ip6_rt_cache_alloc(). 2241 * As rt6_uncached_list_add() does not consume refcnt, 2242 * this refcnt is always returned to the caller even 2243 * if caller sets RT6_LOOKUP_F_DST_NOREF flag. 2244 */ 2245 rt6_uncached_list_add(rt); 2246 rcu_read_unlock(); 2247 2248 return rt; 2249 } 2250 } else { 2251 /* Get a percpu copy */ 2252 local_bh_disable(); 2253 rt = rt6_get_pcpu_route(&res); 2254 2255 if (!rt) 2256 rt = rt6_make_pcpu_route(net, &res); 2257 2258 local_bh_enable(); 2259 } 2260 out: 2261 if (!rt) 2262 rt = net->ipv6.ip6_null_entry; 2263 if (!(flags & RT6_LOOKUP_F_DST_NOREF)) 2264 ip6_hold_safe(net, &rt); 2265 rcu_read_unlock(); 2266 2267 return rt; 2268 } 2269 EXPORT_SYMBOL_GPL(ip6_pol_route); 2270 2271 INDIRECT_CALLABLE_SCOPE struct rt6_info *ip6_pol_route_input(struct net *net, 2272 struct fib6_table *table, 2273 struct flowi6 *fl6, 2274 const struct sk_buff *skb, 2275 int flags) 2276 { 2277 return ip6_pol_route(net, table, fl6->flowi6_iif, fl6, skb, flags); 2278 } 2279 2280 struct dst_entry *ip6_route_input_lookup(struct net *net, 2281 struct net_device *dev, 2282 struct flowi6 *fl6, 2283 const struct sk_buff *skb, 2284 int flags) 2285 { 2286 if (rt6_need_strict(&fl6->daddr) && dev->type != ARPHRD_PIMREG) 2287 flags |= RT6_LOOKUP_F_IFACE; 2288 2289 return fib6_rule_lookup(net, fl6, skb, flags, ip6_pol_route_input); 2290 } 2291 EXPORT_SYMBOL_GPL(ip6_route_input_lookup); 2292 2293 static void ip6_multipath_l3_keys(const struct sk_buff *skb, 2294 struct flow_keys *keys, 2295 struct flow_keys *flkeys) 2296 { 2297 const struct ipv6hdr *outer_iph = ipv6_hdr(skb); 2298 const struct ipv6hdr *key_iph = outer_iph; 2299 struct flow_keys *_flkeys = flkeys; 2300 const struct ipv6hdr *inner_iph; 2301 const struct icmp6hdr *icmph; 2302 struct ipv6hdr _inner_iph; 2303 struct icmp6hdr _icmph; 2304 2305 if (likely(outer_iph->nexthdr != IPPROTO_ICMPV6)) 2306 goto out; 2307 2308 icmph = skb_header_pointer(skb, skb_transport_offset(skb), 2309 sizeof(_icmph), &_icmph); 2310 if (!icmph) 2311 goto out; 2312 2313 if (!icmpv6_is_err(icmph->icmp6_type)) 2314 goto out; 2315 2316 inner_iph = skb_header_pointer(skb, 2317 skb_transport_offset(skb) + sizeof(*icmph), 2318 sizeof(_inner_iph), &_inner_iph); 2319 if (!inner_iph) 2320 goto out; 2321 2322 key_iph = inner_iph; 2323 _flkeys = NULL; 2324 out: 2325 if (_flkeys) { 2326 keys->addrs.v6addrs.src = _flkeys->addrs.v6addrs.src; 2327 keys->addrs.v6addrs.dst = _flkeys->addrs.v6addrs.dst; 2328 keys->tags.flow_label = _flkeys->tags.flow_label; 2329 keys->basic.ip_proto = _flkeys->basic.ip_proto; 2330 } else { 2331 keys->addrs.v6addrs.src = key_iph->saddr; 2332 keys->addrs.v6addrs.dst = key_iph->daddr; 2333 keys->tags.flow_label = ip6_flowlabel(key_iph); 2334 keys->basic.ip_proto = key_iph->nexthdr; 2335 } 2336 } 2337 2338 static u32 rt6_multipath_custom_hash_outer(const struct net *net, 2339 const struct sk_buff *skb, 2340 bool *p_has_inner) 2341 { 2342 u32 hash_fields = ip6_multipath_hash_fields(net); 2343 struct flow_keys keys, hash_keys; 2344 2345 if (!(hash_fields & FIB_MULTIPATH_HASH_FIELD_OUTER_MASK)) 2346 return 0; 2347 2348 memset(&hash_keys, 0, sizeof(hash_keys)); 2349 skb_flow_dissect_flow_keys(skb, &keys, FLOW_DISSECTOR_F_STOP_AT_ENCAP); 2350 2351 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS; 2352 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_SRC_IP) 2353 hash_keys.addrs.v6addrs.src = keys.addrs.v6addrs.src; 2354 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_DST_IP) 2355 hash_keys.addrs.v6addrs.dst = keys.addrs.v6addrs.dst; 2356 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_IP_PROTO) 2357 hash_keys.basic.ip_proto = keys.basic.ip_proto; 2358 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_FLOWLABEL) 2359 hash_keys.tags.flow_label = keys.tags.flow_label; 2360 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_SRC_PORT) 2361 hash_keys.ports.src = keys.ports.src; 2362 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_DST_PORT) 2363 hash_keys.ports.dst = keys.ports.dst; 2364 2365 *p_has_inner = !!(keys.control.flags & FLOW_DIS_ENCAPSULATION); 2366 return flow_hash_from_keys(&hash_keys); 2367 } 2368 2369 static u32 rt6_multipath_custom_hash_inner(const struct net *net, 2370 const struct sk_buff *skb, 2371 bool has_inner) 2372 { 2373 u32 hash_fields = ip6_multipath_hash_fields(net); 2374 struct flow_keys keys, hash_keys; 2375 2376 /* We assume the packet carries an encapsulation, but if none was 2377 * encountered during dissection of the outer flow, then there is no 2378 * point in calling the flow dissector again. 2379 */ 2380 if (!has_inner) 2381 return 0; 2382 2383 if (!(hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_MASK)) 2384 return 0; 2385 2386 memset(&hash_keys, 0, sizeof(hash_keys)); 2387 skb_flow_dissect_flow_keys(skb, &keys, 0); 2388 2389 if (!(keys.control.flags & FLOW_DIS_ENCAPSULATION)) 2390 return 0; 2391 2392 if (keys.control.addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) { 2393 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS; 2394 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_SRC_IP) 2395 hash_keys.addrs.v4addrs.src = keys.addrs.v4addrs.src; 2396 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_DST_IP) 2397 hash_keys.addrs.v4addrs.dst = keys.addrs.v4addrs.dst; 2398 } else if (keys.control.addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) { 2399 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS; 2400 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_SRC_IP) 2401 hash_keys.addrs.v6addrs.src = keys.addrs.v6addrs.src; 2402 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_DST_IP) 2403 hash_keys.addrs.v6addrs.dst = keys.addrs.v6addrs.dst; 2404 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_FLOWLABEL) 2405 hash_keys.tags.flow_label = keys.tags.flow_label; 2406 } 2407 2408 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_IP_PROTO) 2409 hash_keys.basic.ip_proto = keys.basic.ip_proto; 2410 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_SRC_PORT) 2411 hash_keys.ports.src = keys.ports.src; 2412 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_DST_PORT) 2413 hash_keys.ports.dst = keys.ports.dst; 2414 2415 return flow_hash_from_keys(&hash_keys); 2416 } 2417 2418 static u32 rt6_multipath_custom_hash_skb(const struct net *net, 2419 const struct sk_buff *skb) 2420 { 2421 u32 mhash, mhash_inner; 2422 bool has_inner = true; 2423 2424 mhash = rt6_multipath_custom_hash_outer(net, skb, &has_inner); 2425 mhash_inner = rt6_multipath_custom_hash_inner(net, skb, has_inner); 2426 2427 return jhash_2words(mhash, mhash_inner, 0); 2428 } 2429 2430 static u32 rt6_multipath_custom_hash_fl6(const struct net *net, 2431 const struct flowi6 *fl6) 2432 { 2433 u32 hash_fields = ip6_multipath_hash_fields(net); 2434 struct flow_keys hash_keys; 2435 2436 if (!(hash_fields & FIB_MULTIPATH_HASH_FIELD_OUTER_MASK)) 2437 return 0; 2438 2439 memset(&hash_keys, 0, sizeof(hash_keys)); 2440 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS; 2441 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_SRC_IP) 2442 hash_keys.addrs.v6addrs.src = fl6->saddr; 2443 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_DST_IP) 2444 hash_keys.addrs.v6addrs.dst = fl6->daddr; 2445 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_IP_PROTO) 2446 hash_keys.basic.ip_proto = fl6->flowi6_proto; 2447 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_FLOWLABEL) 2448 hash_keys.tags.flow_label = (__force u32)flowi6_get_flowlabel(fl6); 2449 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_SRC_PORT) 2450 hash_keys.ports.src = fl6->fl6_sport; 2451 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_DST_PORT) 2452 hash_keys.ports.dst = fl6->fl6_dport; 2453 2454 return flow_hash_from_keys(&hash_keys); 2455 } 2456 2457 /* if skb is set it will be used and fl6 can be NULL */ 2458 u32 rt6_multipath_hash(const struct net *net, const struct flowi6 *fl6, 2459 const struct sk_buff *skb, struct flow_keys *flkeys) 2460 { 2461 struct flow_keys hash_keys; 2462 u32 mhash = 0; 2463 2464 switch (ip6_multipath_hash_policy(net)) { 2465 case 0: 2466 memset(&hash_keys, 0, sizeof(hash_keys)); 2467 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS; 2468 if (skb) { 2469 ip6_multipath_l3_keys(skb, &hash_keys, flkeys); 2470 } else { 2471 hash_keys.addrs.v6addrs.src = fl6->saddr; 2472 hash_keys.addrs.v6addrs.dst = fl6->daddr; 2473 hash_keys.tags.flow_label = (__force u32)flowi6_get_flowlabel(fl6); 2474 hash_keys.basic.ip_proto = fl6->flowi6_proto; 2475 } 2476 mhash = flow_hash_from_keys(&hash_keys); 2477 break; 2478 case 1: 2479 if (skb) { 2480 unsigned int flag = FLOW_DISSECTOR_F_STOP_AT_ENCAP; 2481 struct flow_keys keys; 2482 2483 /* short-circuit if we already have L4 hash present */ 2484 if (skb->l4_hash) 2485 return skb_get_hash_raw(skb) >> 1; 2486 2487 memset(&hash_keys, 0, sizeof(hash_keys)); 2488 2489 if (!flkeys) { 2490 skb_flow_dissect_flow_keys(skb, &keys, flag); 2491 flkeys = &keys; 2492 } 2493 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS; 2494 hash_keys.addrs.v6addrs.src = flkeys->addrs.v6addrs.src; 2495 hash_keys.addrs.v6addrs.dst = flkeys->addrs.v6addrs.dst; 2496 hash_keys.ports.src = flkeys->ports.src; 2497 hash_keys.ports.dst = flkeys->ports.dst; 2498 hash_keys.basic.ip_proto = flkeys->basic.ip_proto; 2499 } else { 2500 memset(&hash_keys, 0, sizeof(hash_keys)); 2501 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS; 2502 hash_keys.addrs.v6addrs.src = fl6->saddr; 2503 hash_keys.addrs.v6addrs.dst = fl6->daddr; 2504 hash_keys.ports.src = fl6->fl6_sport; 2505 hash_keys.ports.dst = fl6->fl6_dport; 2506 hash_keys.basic.ip_proto = fl6->flowi6_proto; 2507 } 2508 mhash = flow_hash_from_keys(&hash_keys); 2509 break; 2510 case 2: 2511 memset(&hash_keys, 0, sizeof(hash_keys)); 2512 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS; 2513 if (skb) { 2514 struct flow_keys keys; 2515 2516 if (!flkeys) { 2517 skb_flow_dissect_flow_keys(skb, &keys, 0); 2518 flkeys = &keys; 2519 } 2520 2521 /* Inner can be v4 or v6 */ 2522 if (flkeys->control.addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) { 2523 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS; 2524 hash_keys.addrs.v4addrs.src = flkeys->addrs.v4addrs.src; 2525 hash_keys.addrs.v4addrs.dst = flkeys->addrs.v4addrs.dst; 2526 } else if (flkeys->control.addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) { 2527 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS; 2528 hash_keys.addrs.v6addrs.src = flkeys->addrs.v6addrs.src; 2529 hash_keys.addrs.v6addrs.dst = flkeys->addrs.v6addrs.dst; 2530 hash_keys.tags.flow_label = flkeys->tags.flow_label; 2531 hash_keys.basic.ip_proto = flkeys->basic.ip_proto; 2532 } else { 2533 /* Same as case 0 */ 2534 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS; 2535 ip6_multipath_l3_keys(skb, &hash_keys, flkeys); 2536 } 2537 } else { 2538 /* Same as case 0 */ 2539 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS; 2540 hash_keys.addrs.v6addrs.src = fl6->saddr; 2541 hash_keys.addrs.v6addrs.dst = fl6->daddr; 2542 hash_keys.tags.flow_label = (__force u32)flowi6_get_flowlabel(fl6); 2543 hash_keys.basic.ip_proto = fl6->flowi6_proto; 2544 } 2545 mhash = flow_hash_from_keys(&hash_keys); 2546 break; 2547 case 3: 2548 if (skb) 2549 mhash = rt6_multipath_custom_hash_skb(net, skb); 2550 else 2551 mhash = rt6_multipath_custom_hash_fl6(net, fl6); 2552 break; 2553 } 2554 2555 return mhash >> 1; 2556 } 2557 2558 /* Called with rcu held */ 2559 void ip6_route_input(struct sk_buff *skb) 2560 { 2561 const struct ipv6hdr *iph = ipv6_hdr(skb); 2562 struct net *net = dev_net(skb->dev); 2563 int flags = RT6_LOOKUP_F_HAS_SADDR | RT6_LOOKUP_F_DST_NOREF; 2564 struct ip_tunnel_info *tun_info; 2565 struct flowi6 fl6 = { 2566 .flowi6_iif = skb->dev->ifindex, 2567 .daddr = iph->daddr, 2568 .saddr = iph->saddr, 2569 .flowlabel = ip6_flowinfo(iph), 2570 .flowi6_mark = skb->mark, 2571 .flowi6_proto = iph->nexthdr, 2572 }; 2573 struct flow_keys *flkeys = NULL, _flkeys; 2574 2575 tun_info = skb_tunnel_info(skb); 2576 if (tun_info && !(tun_info->mode & IP_TUNNEL_INFO_TX)) 2577 fl6.flowi6_tun_key.tun_id = tun_info->key.tun_id; 2578 2579 if (fib6_rules_early_flow_dissect(net, skb, &fl6, &_flkeys)) 2580 flkeys = &_flkeys; 2581 2582 if (unlikely(fl6.flowi6_proto == IPPROTO_ICMPV6)) 2583 fl6.mp_hash = rt6_multipath_hash(net, &fl6, skb, flkeys); 2584 skb_dst_drop(skb); 2585 skb_dst_set_noref(skb, ip6_route_input_lookup(net, skb->dev, 2586 &fl6, skb, flags)); 2587 } 2588 2589 INDIRECT_CALLABLE_SCOPE struct rt6_info *ip6_pol_route_output(struct net *net, 2590 struct fib6_table *table, 2591 struct flowi6 *fl6, 2592 const struct sk_buff *skb, 2593 int flags) 2594 { 2595 return ip6_pol_route(net, table, fl6->flowi6_oif, fl6, skb, flags); 2596 } 2597 2598 static struct dst_entry *ip6_route_output_flags_noref(struct net *net, 2599 const struct sock *sk, 2600 struct flowi6 *fl6, 2601 int flags) 2602 { 2603 bool any_src; 2604 2605 if (ipv6_addr_type(&fl6->daddr) & 2606 (IPV6_ADDR_MULTICAST | IPV6_ADDR_LINKLOCAL)) { 2607 struct dst_entry *dst; 2608 2609 /* This function does not take refcnt on the dst */ 2610 dst = l3mdev_link_scope_lookup(net, fl6); 2611 if (dst) 2612 return dst; 2613 } 2614 2615 fl6->flowi6_iif = LOOPBACK_IFINDEX; 2616 2617 flags |= RT6_LOOKUP_F_DST_NOREF; 2618 any_src = ipv6_addr_any(&fl6->saddr); 2619 if ((sk && sk->sk_bound_dev_if) || rt6_need_strict(&fl6->daddr) || 2620 (fl6->flowi6_oif && any_src)) 2621 flags |= RT6_LOOKUP_F_IFACE; 2622 2623 if (!any_src) 2624 flags |= RT6_LOOKUP_F_HAS_SADDR; 2625 else if (sk) 2626 flags |= rt6_srcprefs2flags(inet6_sk(sk)->srcprefs); 2627 2628 return fib6_rule_lookup(net, fl6, NULL, flags, ip6_pol_route_output); 2629 } 2630 2631 struct dst_entry *ip6_route_output_flags(struct net *net, 2632 const struct sock *sk, 2633 struct flowi6 *fl6, 2634 int flags) 2635 { 2636 struct dst_entry *dst; 2637 struct rt6_info *rt6; 2638 2639 rcu_read_lock(); 2640 dst = ip6_route_output_flags_noref(net, sk, fl6, flags); 2641 rt6 = (struct rt6_info *)dst; 2642 /* For dst cached in uncached_list, refcnt is already taken. */ 2643 if (list_empty(&rt6->dst.rt_uncached) && !dst_hold_safe(dst)) { 2644 dst = &net->ipv6.ip6_null_entry->dst; 2645 dst_hold(dst); 2646 } 2647 rcu_read_unlock(); 2648 2649 return dst; 2650 } 2651 EXPORT_SYMBOL_GPL(ip6_route_output_flags); 2652 2653 struct dst_entry *ip6_blackhole_route(struct net *net, struct dst_entry *dst_orig) 2654 { 2655 struct rt6_info *rt, *ort = (struct rt6_info *) dst_orig; 2656 struct net_device *loopback_dev = net->loopback_dev; 2657 struct dst_entry *new = NULL; 2658 2659 rt = dst_alloc(&ip6_dst_blackhole_ops, loopback_dev, 1, 2660 DST_OBSOLETE_DEAD, 0); 2661 if (rt) { 2662 rt6_info_init(rt); 2663 atomic_inc(&net->ipv6.rt6_stats->fib_rt_alloc); 2664 2665 new = &rt->dst; 2666 new->__use = 1; 2667 new->input = dst_discard; 2668 new->output = dst_discard_out; 2669 2670 dst_copy_metrics(new, &ort->dst); 2671 2672 rt->rt6i_idev = in6_dev_get(loopback_dev); 2673 rt->rt6i_gateway = ort->rt6i_gateway; 2674 rt->rt6i_flags = ort->rt6i_flags & ~RTF_PCPU; 2675 2676 memcpy(&rt->rt6i_dst, &ort->rt6i_dst, sizeof(struct rt6key)); 2677 #ifdef CONFIG_IPV6_SUBTREES 2678 memcpy(&rt->rt6i_src, &ort->rt6i_src, sizeof(struct rt6key)); 2679 #endif 2680 } 2681 2682 dst_release(dst_orig); 2683 return new ? new : ERR_PTR(-ENOMEM); 2684 } 2685 2686 /* 2687 * Destination cache support functions 2688 */ 2689 2690 static bool fib6_check(struct fib6_info *f6i, u32 cookie) 2691 { 2692 u32 rt_cookie = 0; 2693 2694 if (!fib6_get_cookie_safe(f6i, &rt_cookie) || rt_cookie != cookie) 2695 return false; 2696 2697 if (fib6_check_expired(f6i)) 2698 return false; 2699 2700 return true; 2701 } 2702 2703 static struct dst_entry *rt6_check(struct rt6_info *rt, 2704 struct fib6_info *from, 2705 u32 cookie) 2706 { 2707 u32 rt_cookie = 0; 2708 2709 if (!from || !fib6_get_cookie_safe(from, &rt_cookie) || 2710 rt_cookie != cookie) 2711 return NULL; 2712 2713 if (rt6_check_expired(rt)) 2714 return NULL; 2715 2716 return &rt->dst; 2717 } 2718 2719 static struct dst_entry *rt6_dst_from_check(struct rt6_info *rt, 2720 struct fib6_info *from, 2721 u32 cookie) 2722 { 2723 if (!__rt6_check_expired(rt) && 2724 rt->dst.obsolete == DST_OBSOLETE_FORCE_CHK && 2725 fib6_check(from, cookie)) 2726 return &rt->dst; 2727 else 2728 return NULL; 2729 } 2730 2731 INDIRECT_CALLABLE_SCOPE struct dst_entry *ip6_dst_check(struct dst_entry *dst, 2732 u32 cookie) 2733 { 2734 struct dst_entry *dst_ret; 2735 struct fib6_info *from; 2736 struct rt6_info *rt; 2737 2738 rt = container_of(dst, struct rt6_info, dst); 2739 2740 if (rt->sernum) 2741 return rt6_is_valid(rt) ? dst : NULL; 2742 2743 rcu_read_lock(); 2744 2745 /* All IPV6 dsts are created with ->obsolete set to the value 2746 * DST_OBSOLETE_FORCE_CHK which forces validation calls down 2747 * into this function always. 2748 */ 2749 2750 from = rcu_dereference(rt->from); 2751 2752 if (from && (rt->rt6i_flags & RTF_PCPU || 2753 unlikely(!list_empty(&rt->dst.rt_uncached)))) 2754 dst_ret = rt6_dst_from_check(rt, from, cookie); 2755 else 2756 dst_ret = rt6_check(rt, from, cookie); 2757 2758 rcu_read_unlock(); 2759 2760 return dst_ret; 2761 } 2762 EXPORT_INDIRECT_CALLABLE(ip6_dst_check); 2763 2764 static void ip6_negative_advice(struct sock *sk, 2765 struct dst_entry *dst) 2766 { 2767 struct rt6_info *rt = (struct rt6_info *) dst; 2768 2769 if (rt->rt6i_flags & RTF_CACHE) { 2770 rcu_read_lock(); 2771 if (rt6_check_expired(rt)) { 2772 /* counteract the dst_release() in sk_dst_reset() */ 2773 dst_hold(dst); 2774 sk_dst_reset(sk); 2775 2776 rt6_remove_exception_rt(rt); 2777 } 2778 rcu_read_unlock(); 2779 return; 2780 } 2781 sk_dst_reset(sk); 2782 } 2783 2784 static void ip6_link_failure(struct sk_buff *skb) 2785 { 2786 struct rt6_info *rt; 2787 2788 icmpv6_send(skb, ICMPV6_DEST_UNREACH, ICMPV6_ADDR_UNREACH, 0); 2789 2790 rt = (struct rt6_info *) skb_dst(skb); 2791 if (rt) { 2792 rcu_read_lock(); 2793 if (rt->rt6i_flags & RTF_CACHE) { 2794 rt6_remove_exception_rt(rt); 2795 } else { 2796 struct fib6_info *from; 2797 struct fib6_node *fn; 2798 2799 from = rcu_dereference(rt->from); 2800 if (from) { 2801 fn = rcu_dereference(from->fib6_node); 2802 if (fn && (rt->rt6i_flags & RTF_DEFAULT)) 2803 WRITE_ONCE(fn->fn_sernum, -1); 2804 } 2805 } 2806 rcu_read_unlock(); 2807 } 2808 } 2809 2810 static void rt6_update_expires(struct rt6_info *rt0, int timeout) 2811 { 2812 if (!(rt0->rt6i_flags & RTF_EXPIRES)) { 2813 struct fib6_info *from; 2814 2815 rcu_read_lock(); 2816 from = rcu_dereference(rt0->from); 2817 if (from) 2818 rt0->dst.expires = from->expires; 2819 rcu_read_unlock(); 2820 } 2821 2822 dst_set_expires(&rt0->dst, timeout); 2823 rt0->rt6i_flags |= RTF_EXPIRES; 2824 } 2825 2826 static void rt6_do_update_pmtu(struct rt6_info *rt, u32 mtu) 2827 { 2828 struct net *net = dev_net(rt->dst.dev); 2829 2830 dst_metric_set(&rt->dst, RTAX_MTU, mtu); 2831 rt->rt6i_flags |= RTF_MODIFIED; 2832 rt6_update_expires(rt, net->ipv6.sysctl.ip6_rt_mtu_expires); 2833 } 2834 2835 static bool rt6_cache_allowed_for_pmtu(const struct rt6_info *rt) 2836 { 2837 return !(rt->rt6i_flags & RTF_CACHE) && 2838 (rt->rt6i_flags & RTF_PCPU || rcu_access_pointer(rt->from)); 2839 } 2840 2841 static void __ip6_rt_update_pmtu(struct dst_entry *dst, const struct sock *sk, 2842 const struct ipv6hdr *iph, u32 mtu, 2843 bool confirm_neigh) 2844 { 2845 const struct in6_addr *daddr, *saddr; 2846 struct rt6_info *rt6 = (struct rt6_info *)dst; 2847 2848 /* Note: do *NOT* check dst_metric_locked(dst, RTAX_MTU) 2849 * IPv6 pmtu discovery isn't optional, so 'mtu lock' cannot disable it. 2850 * [see also comment in rt6_mtu_change_route()] 2851 */ 2852 2853 if (iph) { 2854 daddr = &iph->daddr; 2855 saddr = &iph->saddr; 2856 } else if (sk) { 2857 daddr = &sk->sk_v6_daddr; 2858 saddr = &inet6_sk(sk)->saddr; 2859 } else { 2860 daddr = NULL; 2861 saddr = NULL; 2862 } 2863 2864 if (confirm_neigh) 2865 dst_confirm_neigh(dst, daddr); 2866 2867 if (mtu < IPV6_MIN_MTU) 2868 return; 2869 if (mtu >= dst_mtu(dst)) 2870 return; 2871 2872 if (!rt6_cache_allowed_for_pmtu(rt6)) { 2873 rt6_do_update_pmtu(rt6, mtu); 2874 /* update rt6_ex->stamp for cache */ 2875 if (rt6->rt6i_flags & RTF_CACHE) 2876 rt6_update_exception_stamp_rt(rt6); 2877 } else if (daddr) { 2878 struct fib6_result res = {}; 2879 struct rt6_info *nrt6; 2880 2881 rcu_read_lock(); 2882 res.f6i = rcu_dereference(rt6->from); 2883 if (!res.f6i) 2884 goto out_unlock; 2885 2886 res.fib6_flags = res.f6i->fib6_flags; 2887 res.fib6_type = res.f6i->fib6_type; 2888 2889 if (res.f6i->nh) { 2890 struct fib6_nh_match_arg arg = { 2891 .dev = dst->dev, 2892 .gw = &rt6->rt6i_gateway, 2893 }; 2894 2895 nexthop_for_each_fib6_nh(res.f6i->nh, 2896 fib6_nh_find_match, &arg); 2897 2898 /* fib6_info uses a nexthop that does not have fib6_nh 2899 * using the dst->dev + gw. Should be impossible. 2900 */ 2901 if (!arg.match) 2902 goto out_unlock; 2903 2904 res.nh = arg.match; 2905 } else { 2906 res.nh = res.f6i->fib6_nh; 2907 } 2908 2909 nrt6 = ip6_rt_cache_alloc(&res, daddr, saddr); 2910 if (nrt6) { 2911 rt6_do_update_pmtu(nrt6, mtu); 2912 if (rt6_insert_exception(nrt6, &res)) 2913 dst_release_immediate(&nrt6->dst); 2914 } 2915 out_unlock: 2916 rcu_read_unlock(); 2917 } 2918 } 2919 2920 static void ip6_rt_update_pmtu(struct dst_entry *dst, struct sock *sk, 2921 struct sk_buff *skb, u32 mtu, 2922 bool confirm_neigh) 2923 { 2924 __ip6_rt_update_pmtu(dst, sk, skb ? ipv6_hdr(skb) : NULL, mtu, 2925 confirm_neigh); 2926 } 2927 2928 void ip6_update_pmtu(struct sk_buff *skb, struct net *net, __be32 mtu, 2929 int oif, u32 mark, kuid_t uid) 2930 { 2931 const struct ipv6hdr *iph = (struct ipv6hdr *) skb->data; 2932 struct dst_entry *dst; 2933 struct flowi6 fl6 = { 2934 .flowi6_oif = oif, 2935 .flowi6_mark = mark ? mark : IP6_REPLY_MARK(net, skb->mark), 2936 .daddr = iph->daddr, 2937 .saddr = iph->saddr, 2938 .flowlabel = ip6_flowinfo(iph), 2939 .flowi6_uid = uid, 2940 }; 2941 2942 dst = ip6_route_output(net, NULL, &fl6); 2943 if (!dst->error) 2944 __ip6_rt_update_pmtu(dst, NULL, iph, ntohl(mtu), true); 2945 dst_release(dst); 2946 } 2947 EXPORT_SYMBOL_GPL(ip6_update_pmtu); 2948 2949 void ip6_sk_update_pmtu(struct sk_buff *skb, struct sock *sk, __be32 mtu) 2950 { 2951 int oif = sk->sk_bound_dev_if; 2952 struct dst_entry *dst; 2953 2954 if (!oif && skb->dev) 2955 oif = l3mdev_master_ifindex(skb->dev); 2956 2957 ip6_update_pmtu(skb, sock_net(sk), mtu, oif, READ_ONCE(sk->sk_mark), 2958 sk->sk_uid); 2959 2960 dst = __sk_dst_get(sk); 2961 if (!dst || !dst->obsolete || 2962 dst->ops->check(dst, inet6_sk(sk)->dst_cookie)) 2963 return; 2964 2965 bh_lock_sock(sk); 2966 if (!sock_owned_by_user(sk) && !ipv6_addr_v4mapped(&sk->sk_v6_daddr)) 2967 ip6_datagram_dst_update(sk, false); 2968 bh_unlock_sock(sk); 2969 } 2970 EXPORT_SYMBOL_GPL(ip6_sk_update_pmtu); 2971 2972 void ip6_sk_dst_store_flow(struct sock *sk, struct dst_entry *dst, 2973 const struct flowi6 *fl6) 2974 { 2975 #ifdef CONFIG_IPV6_SUBTREES 2976 struct ipv6_pinfo *np = inet6_sk(sk); 2977 #endif 2978 2979 ip6_dst_store(sk, dst, 2980 ipv6_addr_equal(&fl6->daddr, &sk->sk_v6_daddr) ? 2981 &sk->sk_v6_daddr : NULL, 2982 #ifdef CONFIG_IPV6_SUBTREES 2983 ipv6_addr_equal(&fl6->saddr, &np->saddr) ? 2984 &np->saddr : 2985 #endif 2986 NULL); 2987 } 2988 2989 static bool ip6_redirect_nh_match(const struct fib6_result *res, 2990 struct flowi6 *fl6, 2991 const struct in6_addr *gw, 2992 struct rt6_info **ret) 2993 { 2994 const struct fib6_nh *nh = res->nh; 2995 2996 if (nh->fib_nh_flags & RTNH_F_DEAD || !nh->fib_nh_gw_family || 2997 fl6->flowi6_oif != nh->fib_nh_dev->ifindex) 2998 return false; 2999 3000 /* rt_cache's gateway might be different from its 'parent' 3001 * in the case of an ip redirect. 3002 * So we keep searching in the exception table if the gateway 3003 * is different. 3004 */ 3005 if (!ipv6_addr_equal(gw, &nh->fib_nh_gw6)) { 3006 struct rt6_info *rt_cache; 3007 3008 rt_cache = rt6_find_cached_rt(res, &fl6->daddr, &fl6->saddr); 3009 if (rt_cache && 3010 ipv6_addr_equal(gw, &rt_cache->rt6i_gateway)) { 3011 *ret = rt_cache; 3012 return true; 3013 } 3014 return false; 3015 } 3016 return true; 3017 } 3018 3019 struct fib6_nh_rd_arg { 3020 struct fib6_result *res; 3021 struct flowi6 *fl6; 3022 const struct in6_addr *gw; 3023 struct rt6_info **ret; 3024 }; 3025 3026 static int fib6_nh_redirect_match(struct fib6_nh *nh, void *_arg) 3027 { 3028 struct fib6_nh_rd_arg *arg = _arg; 3029 3030 arg->res->nh = nh; 3031 return ip6_redirect_nh_match(arg->res, arg->fl6, arg->gw, arg->ret); 3032 } 3033 3034 /* Handle redirects */ 3035 struct ip6rd_flowi { 3036 struct flowi6 fl6; 3037 struct in6_addr gateway; 3038 }; 3039 3040 INDIRECT_CALLABLE_SCOPE struct rt6_info *__ip6_route_redirect(struct net *net, 3041 struct fib6_table *table, 3042 struct flowi6 *fl6, 3043 const struct sk_buff *skb, 3044 int flags) 3045 { 3046 struct ip6rd_flowi *rdfl = (struct ip6rd_flowi *)fl6; 3047 struct rt6_info *ret = NULL; 3048 struct fib6_result res = {}; 3049 struct fib6_nh_rd_arg arg = { 3050 .res = &res, 3051 .fl6 = fl6, 3052 .gw = &rdfl->gateway, 3053 .ret = &ret 3054 }; 3055 struct fib6_info *rt; 3056 struct fib6_node *fn; 3057 3058 /* Get the "current" route for this destination and 3059 * check if the redirect has come from appropriate router. 3060 * 3061 * RFC 4861 specifies that redirects should only be 3062 * accepted if they come from the nexthop to the target. 3063 * Due to the way the routes are chosen, this notion 3064 * is a bit fuzzy and one might need to check all possible 3065 * routes. 3066 */ 3067 3068 rcu_read_lock(); 3069 fn = fib6_node_lookup(&table->tb6_root, &fl6->daddr, &fl6->saddr); 3070 restart: 3071 for_each_fib6_node_rt_rcu(fn) { 3072 res.f6i = rt; 3073 if (fib6_check_expired(rt)) 3074 continue; 3075 if (rt->fib6_flags & RTF_REJECT) 3076 break; 3077 if (unlikely(rt->nh)) { 3078 if (nexthop_is_blackhole(rt->nh)) 3079 continue; 3080 /* on match, res->nh is filled in and potentially ret */ 3081 if (nexthop_for_each_fib6_nh(rt->nh, 3082 fib6_nh_redirect_match, 3083 &arg)) 3084 goto out; 3085 } else { 3086 res.nh = rt->fib6_nh; 3087 if (ip6_redirect_nh_match(&res, fl6, &rdfl->gateway, 3088 &ret)) 3089 goto out; 3090 } 3091 } 3092 3093 if (!rt) 3094 rt = net->ipv6.fib6_null_entry; 3095 else if (rt->fib6_flags & RTF_REJECT) { 3096 ret = net->ipv6.ip6_null_entry; 3097 goto out; 3098 } 3099 3100 if (rt == net->ipv6.fib6_null_entry) { 3101 fn = fib6_backtrack(fn, &fl6->saddr); 3102 if (fn) 3103 goto restart; 3104 } 3105 3106 res.f6i = rt; 3107 res.nh = rt->fib6_nh; 3108 out: 3109 if (ret) { 3110 ip6_hold_safe(net, &ret); 3111 } else { 3112 res.fib6_flags = res.f6i->fib6_flags; 3113 res.fib6_type = res.f6i->fib6_type; 3114 ret = ip6_create_rt_rcu(&res); 3115 } 3116 3117 rcu_read_unlock(); 3118 3119 trace_fib6_table_lookup(net, &res, table, fl6); 3120 return ret; 3121 }; 3122 3123 static struct dst_entry *ip6_route_redirect(struct net *net, 3124 const struct flowi6 *fl6, 3125 const struct sk_buff *skb, 3126 const struct in6_addr *gateway) 3127 { 3128 int flags = RT6_LOOKUP_F_HAS_SADDR; 3129 struct ip6rd_flowi rdfl; 3130 3131 rdfl.fl6 = *fl6; 3132 rdfl.gateway = *gateway; 3133 3134 return fib6_rule_lookup(net, &rdfl.fl6, skb, 3135 flags, __ip6_route_redirect); 3136 } 3137 3138 void ip6_redirect(struct sk_buff *skb, struct net *net, int oif, u32 mark, 3139 kuid_t uid) 3140 { 3141 const struct ipv6hdr *iph = (struct ipv6hdr *) skb->data; 3142 struct dst_entry *dst; 3143 struct flowi6 fl6 = { 3144 .flowi6_iif = LOOPBACK_IFINDEX, 3145 .flowi6_oif = oif, 3146 .flowi6_mark = mark, 3147 .daddr = iph->daddr, 3148 .saddr = iph->saddr, 3149 .flowlabel = ip6_flowinfo(iph), 3150 .flowi6_uid = uid, 3151 }; 3152 3153 dst = ip6_route_redirect(net, &fl6, skb, &ipv6_hdr(skb)->saddr); 3154 rt6_do_redirect(dst, NULL, skb); 3155 dst_release(dst); 3156 } 3157 EXPORT_SYMBOL_GPL(ip6_redirect); 3158 3159 void ip6_redirect_no_header(struct sk_buff *skb, struct net *net, int oif) 3160 { 3161 const struct ipv6hdr *iph = ipv6_hdr(skb); 3162 const struct rd_msg *msg = (struct rd_msg *)icmp6_hdr(skb); 3163 struct dst_entry *dst; 3164 struct flowi6 fl6 = { 3165 .flowi6_iif = LOOPBACK_IFINDEX, 3166 .flowi6_oif = oif, 3167 .daddr = msg->dest, 3168 .saddr = iph->daddr, 3169 .flowi6_uid = sock_net_uid(net, NULL), 3170 }; 3171 3172 dst = ip6_route_redirect(net, &fl6, skb, &iph->saddr); 3173 rt6_do_redirect(dst, NULL, skb); 3174 dst_release(dst); 3175 } 3176 3177 void ip6_sk_redirect(struct sk_buff *skb, struct sock *sk) 3178 { 3179 ip6_redirect(skb, sock_net(sk), sk->sk_bound_dev_if, 3180 READ_ONCE(sk->sk_mark), sk->sk_uid); 3181 } 3182 EXPORT_SYMBOL_GPL(ip6_sk_redirect); 3183 3184 static unsigned int ip6_default_advmss(const struct dst_entry *dst) 3185 { 3186 struct net_device *dev = dst->dev; 3187 unsigned int mtu = dst_mtu(dst); 3188 struct net *net = dev_net(dev); 3189 3190 mtu -= sizeof(struct ipv6hdr) + sizeof(struct tcphdr); 3191 3192 if (mtu < net->ipv6.sysctl.ip6_rt_min_advmss) 3193 mtu = net->ipv6.sysctl.ip6_rt_min_advmss; 3194 3195 /* 3196 * Maximal non-jumbo IPv6 payload is IPV6_MAXPLEN and 3197 * corresponding MSS is IPV6_MAXPLEN - tcp_header_size. 3198 * IPV6_MAXPLEN is also valid and means: "any MSS, 3199 * rely only on pmtu discovery" 3200 */ 3201 if (mtu > IPV6_MAXPLEN - sizeof(struct tcphdr)) 3202 mtu = IPV6_MAXPLEN; 3203 return mtu; 3204 } 3205 3206 INDIRECT_CALLABLE_SCOPE unsigned int ip6_mtu(const struct dst_entry *dst) 3207 { 3208 return ip6_dst_mtu_maybe_forward(dst, false); 3209 } 3210 EXPORT_INDIRECT_CALLABLE(ip6_mtu); 3211 3212 /* MTU selection: 3213 * 1. mtu on route is locked - use it 3214 * 2. mtu from nexthop exception 3215 * 3. mtu from egress device 3216 * 3217 * based on ip6_dst_mtu_forward and exception logic of 3218 * rt6_find_cached_rt; called with rcu_read_lock 3219 */ 3220 u32 ip6_mtu_from_fib6(const struct fib6_result *res, 3221 const struct in6_addr *daddr, 3222 const struct in6_addr *saddr) 3223 { 3224 const struct fib6_nh *nh = res->nh; 3225 struct fib6_info *f6i = res->f6i; 3226 struct inet6_dev *idev; 3227 struct rt6_info *rt; 3228 u32 mtu = 0; 3229 3230 if (unlikely(fib6_metric_locked(f6i, RTAX_MTU))) { 3231 mtu = f6i->fib6_pmtu; 3232 if (mtu) 3233 goto out; 3234 } 3235 3236 rt = rt6_find_cached_rt(res, daddr, saddr); 3237 if (unlikely(rt)) { 3238 mtu = dst_metric_raw(&rt->dst, RTAX_MTU); 3239 } else { 3240 struct net_device *dev = nh->fib_nh_dev; 3241 3242 mtu = IPV6_MIN_MTU; 3243 idev = __in6_dev_get(dev); 3244 if (idev && idev->cnf.mtu6 > mtu) 3245 mtu = idev->cnf.mtu6; 3246 } 3247 3248 mtu = min_t(unsigned int, mtu, IP6_MAX_MTU); 3249 out: 3250 return mtu - lwtunnel_headroom(nh->fib_nh_lws, mtu); 3251 } 3252 3253 struct dst_entry *icmp6_dst_alloc(struct net_device *dev, 3254 struct flowi6 *fl6) 3255 { 3256 struct dst_entry *dst; 3257 struct rt6_info *rt; 3258 struct inet6_dev *idev = in6_dev_get(dev); 3259 struct net *net = dev_net(dev); 3260 3261 if (unlikely(!idev)) 3262 return ERR_PTR(-ENODEV); 3263 3264 rt = ip6_dst_alloc(net, dev, 0); 3265 if (unlikely(!rt)) { 3266 in6_dev_put(idev); 3267 dst = ERR_PTR(-ENOMEM); 3268 goto out; 3269 } 3270 3271 rt->dst.input = ip6_input; 3272 rt->dst.output = ip6_output; 3273 rt->rt6i_gateway = fl6->daddr; 3274 rt->rt6i_dst.addr = fl6->daddr; 3275 rt->rt6i_dst.plen = 128; 3276 rt->rt6i_idev = idev; 3277 dst_metric_set(&rt->dst, RTAX_HOPLIMIT, 0); 3278 3279 /* Add this dst into uncached_list so that rt6_disable_ip() can 3280 * do proper release of the net_device 3281 */ 3282 rt6_uncached_list_add(rt); 3283 3284 dst = xfrm_lookup(net, &rt->dst, flowi6_to_flowi(fl6), NULL, 0); 3285 3286 out: 3287 return dst; 3288 } 3289 3290 static void ip6_dst_gc(struct dst_ops *ops) 3291 { 3292 struct net *net = container_of(ops, struct net, ipv6.ip6_dst_ops); 3293 int rt_min_interval = net->ipv6.sysctl.ip6_rt_gc_min_interval; 3294 int rt_elasticity = net->ipv6.sysctl.ip6_rt_gc_elasticity; 3295 int rt_gc_timeout = net->ipv6.sysctl.ip6_rt_gc_timeout; 3296 unsigned long rt_last_gc = net->ipv6.ip6_rt_last_gc; 3297 unsigned int val; 3298 int entries; 3299 3300 if (time_after(rt_last_gc + rt_min_interval, jiffies)) 3301 goto out; 3302 3303 fib6_run_gc(atomic_inc_return(&net->ipv6.ip6_rt_gc_expire), net, true); 3304 entries = dst_entries_get_slow(ops); 3305 if (entries < ops->gc_thresh) 3306 atomic_set(&net->ipv6.ip6_rt_gc_expire, rt_gc_timeout >> 1); 3307 out: 3308 val = atomic_read(&net->ipv6.ip6_rt_gc_expire); 3309 atomic_set(&net->ipv6.ip6_rt_gc_expire, val - (val >> rt_elasticity)); 3310 } 3311 3312 static int ip6_nh_lookup_table(struct net *net, struct fib6_config *cfg, 3313 const struct in6_addr *gw_addr, u32 tbid, 3314 int flags, struct fib6_result *res) 3315 { 3316 struct flowi6 fl6 = { 3317 .flowi6_oif = cfg->fc_ifindex, 3318 .daddr = *gw_addr, 3319 .saddr = cfg->fc_prefsrc, 3320 }; 3321 struct fib6_table *table; 3322 int err; 3323 3324 table = fib6_get_table(net, tbid); 3325 if (!table) 3326 return -EINVAL; 3327 3328 if (!ipv6_addr_any(&cfg->fc_prefsrc)) 3329 flags |= RT6_LOOKUP_F_HAS_SADDR; 3330 3331 flags |= RT6_LOOKUP_F_IGNORE_LINKSTATE; 3332 3333 err = fib6_table_lookup(net, table, cfg->fc_ifindex, &fl6, res, flags); 3334 if (!err && res->f6i != net->ipv6.fib6_null_entry) 3335 fib6_select_path(net, res, &fl6, cfg->fc_ifindex, 3336 cfg->fc_ifindex != 0, NULL, flags); 3337 3338 return err; 3339 } 3340 3341 static int ip6_route_check_nh_onlink(struct net *net, 3342 struct fib6_config *cfg, 3343 const struct net_device *dev, 3344 struct netlink_ext_ack *extack) 3345 { 3346 u32 tbid = l3mdev_fib_table_rcu(dev) ? : RT_TABLE_MAIN; 3347 const struct in6_addr *gw_addr = &cfg->fc_gateway; 3348 struct fib6_result res = {}; 3349 int err; 3350 3351 err = ip6_nh_lookup_table(net, cfg, gw_addr, tbid, 0, &res); 3352 if (!err && !(res.fib6_flags & RTF_REJECT) && 3353 /* ignore match if it is the default route */ 3354 !ipv6_addr_any(&res.f6i->fib6_dst.addr) && 3355 (res.fib6_type != RTN_UNICAST || dev != res.nh->fib_nh_dev)) { 3356 NL_SET_ERR_MSG(extack, 3357 "Nexthop has invalid gateway or device mismatch"); 3358 err = -EINVAL; 3359 } 3360 3361 return err; 3362 } 3363 3364 static int ip6_route_check_nh(struct net *net, 3365 struct fib6_config *cfg, 3366 struct net_device **_dev, 3367 netdevice_tracker *dev_tracker, 3368 struct inet6_dev **idev) 3369 { 3370 const struct in6_addr *gw_addr = &cfg->fc_gateway; 3371 struct net_device *dev = _dev ? *_dev : NULL; 3372 int flags = RT6_LOOKUP_F_IFACE; 3373 struct fib6_result res = {}; 3374 int err = -EHOSTUNREACH; 3375 3376 if (cfg->fc_table) { 3377 err = ip6_nh_lookup_table(net, cfg, gw_addr, 3378 cfg->fc_table, flags, &res); 3379 /* gw_addr can not require a gateway or resolve to a reject 3380 * route. If a device is given, it must match the result. 3381 */ 3382 if (err || res.fib6_flags & RTF_REJECT || 3383 res.nh->fib_nh_gw_family || 3384 (dev && dev != res.nh->fib_nh_dev)) 3385 err = -EHOSTUNREACH; 3386 } 3387 3388 if (err < 0) { 3389 struct flowi6 fl6 = { 3390 .flowi6_oif = cfg->fc_ifindex, 3391 .daddr = *gw_addr, 3392 }; 3393 3394 err = fib6_lookup(net, cfg->fc_ifindex, &fl6, &res, flags); 3395 if (err || res.fib6_flags & RTF_REJECT || 3396 res.nh->fib_nh_gw_family) 3397 err = -EHOSTUNREACH; 3398 3399 if (err) 3400 return err; 3401 3402 fib6_select_path(net, &res, &fl6, cfg->fc_ifindex, 3403 cfg->fc_ifindex != 0, NULL, flags); 3404 } 3405 3406 err = 0; 3407 if (dev) { 3408 if (dev != res.nh->fib_nh_dev) 3409 err = -EHOSTUNREACH; 3410 } else { 3411 *_dev = dev = res.nh->fib_nh_dev; 3412 netdev_hold(dev, dev_tracker, GFP_ATOMIC); 3413 *idev = in6_dev_get(dev); 3414 } 3415 3416 return err; 3417 } 3418 3419 static int ip6_validate_gw(struct net *net, struct fib6_config *cfg, 3420 struct net_device **_dev, 3421 netdevice_tracker *dev_tracker, 3422 struct inet6_dev **idev, 3423 struct netlink_ext_ack *extack) 3424 { 3425 const struct in6_addr *gw_addr = &cfg->fc_gateway; 3426 int gwa_type = ipv6_addr_type(gw_addr); 3427 bool skip_dev = gwa_type & IPV6_ADDR_LINKLOCAL ? false : true; 3428 const struct net_device *dev = *_dev; 3429 bool need_addr_check = !dev; 3430 int err = -EINVAL; 3431 3432 /* if gw_addr is local we will fail to detect this in case 3433 * address is still TENTATIVE (DAD in progress). rt6_lookup() 3434 * will return already-added prefix route via interface that 3435 * prefix route was assigned to, which might be non-loopback. 3436 */ 3437 if (dev && 3438 ipv6_chk_addr_and_flags(net, gw_addr, dev, skip_dev, 0, 0)) { 3439 NL_SET_ERR_MSG(extack, "Gateway can not be a local address"); 3440 goto out; 3441 } 3442 3443 if (gwa_type != (IPV6_ADDR_LINKLOCAL | IPV6_ADDR_UNICAST)) { 3444 /* IPv6 strictly inhibits using not link-local 3445 * addresses as nexthop address. 3446 * Otherwise, router will not able to send redirects. 3447 * It is very good, but in some (rare!) circumstances 3448 * (SIT, PtP, NBMA NOARP links) it is handy to allow 3449 * some exceptions. --ANK 3450 * We allow IPv4-mapped nexthops to support RFC4798-type 3451 * addressing 3452 */ 3453 if (!(gwa_type & (IPV6_ADDR_UNICAST | IPV6_ADDR_MAPPED))) { 3454 NL_SET_ERR_MSG(extack, "Invalid gateway address"); 3455 goto out; 3456 } 3457 3458 rcu_read_lock(); 3459 3460 if (cfg->fc_flags & RTNH_F_ONLINK) 3461 err = ip6_route_check_nh_onlink(net, cfg, dev, extack); 3462 else 3463 err = ip6_route_check_nh(net, cfg, _dev, dev_tracker, 3464 idev); 3465 3466 rcu_read_unlock(); 3467 3468 if (err) 3469 goto out; 3470 } 3471 3472 /* reload in case device was changed */ 3473 dev = *_dev; 3474 3475 err = -EINVAL; 3476 if (!dev) { 3477 NL_SET_ERR_MSG(extack, "Egress device not specified"); 3478 goto out; 3479 } else if (dev->flags & IFF_LOOPBACK) { 3480 NL_SET_ERR_MSG(extack, 3481 "Egress device can not be loopback device for this route"); 3482 goto out; 3483 } 3484 3485 /* if we did not check gw_addr above, do so now that the 3486 * egress device has been resolved. 3487 */ 3488 if (need_addr_check && 3489 ipv6_chk_addr_and_flags(net, gw_addr, dev, skip_dev, 0, 0)) { 3490 NL_SET_ERR_MSG(extack, "Gateway can not be a local address"); 3491 goto out; 3492 } 3493 3494 err = 0; 3495 out: 3496 return err; 3497 } 3498 3499 static bool fib6_is_reject(u32 flags, struct net_device *dev, int addr_type) 3500 { 3501 if ((flags & RTF_REJECT) || 3502 (dev && (dev->flags & IFF_LOOPBACK) && 3503 !(addr_type & IPV6_ADDR_LOOPBACK) && 3504 !(flags & (RTF_ANYCAST | RTF_LOCAL)))) 3505 return true; 3506 3507 return false; 3508 } 3509 3510 int fib6_nh_init(struct net *net, struct fib6_nh *fib6_nh, 3511 struct fib6_config *cfg, gfp_t gfp_flags, 3512 struct netlink_ext_ack *extack) 3513 { 3514 netdevice_tracker *dev_tracker = &fib6_nh->fib_nh_dev_tracker; 3515 struct net_device *dev = NULL; 3516 struct inet6_dev *idev = NULL; 3517 int addr_type; 3518 int err; 3519 3520 fib6_nh->fib_nh_family = AF_INET6; 3521 #ifdef CONFIG_IPV6_ROUTER_PREF 3522 fib6_nh->last_probe = jiffies; 3523 #endif 3524 if (cfg->fc_is_fdb) { 3525 fib6_nh->fib_nh_gw6 = cfg->fc_gateway; 3526 fib6_nh->fib_nh_gw_family = AF_INET6; 3527 return 0; 3528 } 3529 3530 err = -ENODEV; 3531 if (cfg->fc_ifindex) { 3532 dev = netdev_get_by_index(net, cfg->fc_ifindex, 3533 dev_tracker, gfp_flags); 3534 if (!dev) 3535 goto out; 3536 idev = in6_dev_get(dev); 3537 if (!idev) 3538 goto out; 3539 } 3540 3541 if (cfg->fc_flags & RTNH_F_ONLINK) { 3542 if (!dev) { 3543 NL_SET_ERR_MSG(extack, 3544 "Nexthop device required for onlink"); 3545 goto out; 3546 } 3547 3548 if (!(dev->flags & IFF_UP)) { 3549 NL_SET_ERR_MSG(extack, "Nexthop device is not up"); 3550 err = -ENETDOWN; 3551 goto out; 3552 } 3553 3554 fib6_nh->fib_nh_flags |= RTNH_F_ONLINK; 3555 } 3556 3557 fib6_nh->fib_nh_weight = 1; 3558 3559 /* We cannot add true routes via loopback here, 3560 * they would result in kernel looping; promote them to reject routes 3561 */ 3562 addr_type = ipv6_addr_type(&cfg->fc_dst); 3563 if (fib6_is_reject(cfg->fc_flags, dev, addr_type)) { 3564 /* hold loopback dev/idev if we haven't done so. */ 3565 if (dev != net->loopback_dev) { 3566 if (dev) { 3567 netdev_put(dev, dev_tracker); 3568 in6_dev_put(idev); 3569 } 3570 dev = net->loopback_dev; 3571 netdev_hold(dev, dev_tracker, gfp_flags); 3572 idev = in6_dev_get(dev); 3573 if (!idev) { 3574 err = -ENODEV; 3575 goto out; 3576 } 3577 } 3578 goto pcpu_alloc; 3579 } 3580 3581 if (cfg->fc_flags & RTF_GATEWAY) { 3582 err = ip6_validate_gw(net, cfg, &dev, dev_tracker, 3583 &idev, extack); 3584 if (err) 3585 goto out; 3586 3587 fib6_nh->fib_nh_gw6 = cfg->fc_gateway; 3588 fib6_nh->fib_nh_gw_family = AF_INET6; 3589 } 3590 3591 err = -ENODEV; 3592 if (!dev) 3593 goto out; 3594 3595 if (idev->cnf.disable_ipv6) { 3596 NL_SET_ERR_MSG(extack, "IPv6 is disabled on nexthop device"); 3597 err = -EACCES; 3598 goto out; 3599 } 3600 3601 if (!(dev->flags & IFF_UP) && !cfg->fc_ignore_dev_down) { 3602 NL_SET_ERR_MSG(extack, "Nexthop device is not up"); 3603 err = -ENETDOWN; 3604 goto out; 3605 } 3606 3607 if (!(cfg->fc_flags & (RTF_LOCAL | RTF_ANYCAST)) && 3608 !netif_carrier_ok(dev)) 3609 fib6_nh->fib_nh_flags |= RTNH_F_LINKDOWN; 3610 3611 err = fib_nh_common_init(net, &fib6_nh->nh_common, cfg->fc_encap, 3612 cfg->fc_encap_type, cfg, gfp_flags, extack); 3613 if (err) 3614 goto out; 3615 3616 pcpu_alloc: 3617 fib6_nh->rt6i_pcpu = alloc_percpu_gfp(struct rt6_info *, gfp_flags); 3618 if (!fib6_nh->rt6i_pcpu) { 3619 err = -ENOMEM; 3620 goto out; 3621 } 3622 3623 fib6_nh->fib_nh_dev = dev; 3624 fib6_nh->fib_nh_oif = dev->ifindex; 3625 err = 0; 3626 out: 3627 if (idev) 3628 in6_dev_put(idev); 3629 3630 if (err) { 3631 lwtstate_put(fib6_nh->fib_nh_lws); 3632 fib6_nh->fib_nh_lws = NULL; 3633 netdev_put(dev, dev_tracker); 3634 } 3635 3636 return err; 3637 } 3638 3639 void fib6_nh_release(struct fib6_nh *fib6_nh) 3640 { 3641 struct rt6_exception_bucket *bucket; 3642 3643 rcu_read_lock(); 3644 3645 fib6_nh_flush_exceptions(fib6_nh, NULL); 3646 bucket = fib6_nh_get_excptn_bucket(fib6_nh, NULL); 3647 if (bucket) { 3648 rcu_assign_pointer(fib6_nh->rt6i_exception_bucket, NULL); 3649 kfree(bucket); 3650 } 3651 3652 rcu_read_unlock(); 3653 3654 fib6_nh_release_dsts(fib6_nh); 3655 free_percpu(fib6_nh->rt6i_pcpu); 3656 3657 fib_nh_common_release(&fib6_nh->nh_common); 3658 } 3659 3660 void fib6_nh_release_dsts(struct fib6_nh *fib6_nh) 3661 { 3662 int cpu; 3663 3664 if (!fib6_nh->rt6i_pcpu) 3665 return; 3666 3667 for_each_possible_cpu(cpu) { 3668 struct rt6_info *pcpu_rt, **ppcpu_rt; 3669 3670 ppcpu_rt = per_cpu_ptr(fib6_nh->rt6i_pcpu, cpu); 3671 pcpu_rt = xchg(ppcpu_rt, NULL); 3672 if (pcpu_rt) { 3673 dst_dev_put(&pcpu_rt->dst); 3674 dst_release(&pcpu_rt->dst); 3675 } 3676 } 3677 } 3678 3679 static struct fib6_info *ip6_route_info_create(struct fib6_config *cfg, 3680 gfp_t gfp_flags, 3681 struct netlink_ext_ack *extack) 3682 { 3683 struct net *net = cfg->fc_nlinfo.nl_net; 3684 struct fib6_info *rt = NULL; 3685 struct nexthop *nh = NULL; 3686 struct fib6_table *table; 3687 struct fib6_nh *fib6_nh; 3688 int err = -EINVAL; 3689 int addr_type; 3690 3691 /* RTF_PCPU is an internal flag; can not be set by userspace */ 3692 if (cfg->fc_flags & RTF_PCPU) { 3693 NL_SET_ERR_MSG(extack, "Userspace can not set RTF_PCPU"); 3694 goto out; 3695 } 3696 3697 /* RTF_CACHE is an internal flag; can not be set by userspace */ 3698 if (cfg->fc_flags & RTF_CACHE) { 3699 NL_SET_ERR_MSG(extack, "Userspace can not set RTF_CACHE"); 3700 goto out; 3701 } 3702 3703 if (cfg->fc_type > RTN_MAX) { 3704 NL_SET_ERR_MSG(extack, "Invalid route type"); 3705 goto out; 3706 } 3707 3708 if (cfg->fc_dst_len > 128) { 3709 NL_SET_ERR_MSG(extack, "Invalid prefix length"); 3710 goto out; 3711 } 3712 if (cfg->fc_src_len > 128) { 3713 NL_SET_ERR_MSG(extack, "Invalid source address length"); 3714 goto out; 3715 } 3716 #ifndef CONFIG_IPV6_SUBTREES 3717 if (cfg->fc_src_len) { 3718 NL_SET_ERR_MSG(extack, 3719 "Specifying source address requires IPV6_SUBTREES to be enabled"); 3720 goto out; 3721 } 3722 #endif 3723 if (cfg->fc_nh_id) { 3724 nh = nexthop_find_by_id(net, cfg->fc_nh_id); 3725 if (!nh) { 3726 NL_SET_ERR_MSG(extack, "Nexthop id does not exist"); 3727 goto out; 3728 } 3729 err = fib6_check_nexthop(nh, cfg, extack); 3730 if (err) 3731 goto out; 3732 } 3733 3734 err = -ENOBUFS; 3735 if (cfg->fc_nlinfo.nlh && 3736 !(cfg->fc_nlinfo.nlh->nlmsg_flags & NLM_F_CREATE)) { 3737 table = fib6_get_table(net, cfg->fc_table); 3738 if (!table) { 3739 pr_warn("NLM_F_CREATE should be specified when creating new route\n"); 3740 table = fib6_new_table(net, cfg->fc_table); 3741 } 3742 } else { 3743 table = fib6_new_table(net, cfg->fc_table); 3744 } 3745 3746 if (!table) 3747 goto out; 3748 3749 err = -ENOMEM; 3750 rt = fib6_info_alloc(gfp_flags, !nh); 3751 if (!rt) 3752 goto out; 3753 3754 rt->fib6_metrics = ip_fib_metrics_init(net, cfg->fc_mx, cfg->fc_mx_len, 3755 extack); 3756 if (IS_ERR(rt->fib6_metrics)) { 3757 err = PTR_ERR(rt->fib6_metrics); 3758 /* Do not leave garbage there. */ 3759 rt->fib6_metrics = (struct dst_metrics *)&dst_default_metrics; 3760 goto out_free; 3761 } 3762 3763 if (cfg->fc_flags & RTF_ADDRCONF) 3764 rt->dst_nocount = true; 3765 3766 if (cfg->fc_flags & RTF_EXPIRES) 3767 fib6_set_expires(rt, jiffies + 3768 clock_t_to_jiffies(cfg->fc_expires)); 3769 else 3770 fib6_clean_expires(rt); 3771 3772 if (cfg->fc_protocol == RTPROT_UNSPEC) 3773 cfg->fc_protocol = RTPROT_BOOT; 3774 rt->fib6_protocol = cfg->fc_protocol; 3775 3776 rt->fib6_table = table; 3777 rt->fib6_metric = cfg->fc_metric; 3778 rt->fib6_type = cfg->fc_type ? : RTN_UNICAST; 3779 rt->fib6_flags = cfg->fc_flags & ~RTF_GATEWAY; 3780 3781 ipv6_addr_prefix(&rt->fib6_dst.addr, &cfg->fc_dst, cfg->fc_dst_len); 3782 rt->fib6_dst.plen = cfg->fc_dst_len; 3783 3784 #ifdef CONFIG_IPV6_SUBTREES 3785 ipv6_addr_prefix(&rt->fib6_src.addr, &cfg->fc_src, cfg->fc_src_len); 3786 rt->fib6_src.plen = cfg->fc_src_len; 3787 #endif 3788 if (nh) { 3789 if (rt->fib6_src.plen) { 3790 NL_SET_ERR_MSG(extack, "Nexthops can not be used with source routing"); 3791 goto out_free; 3792 } 3793 if (!nexthop_get(nh)) { 3794 NL_SET_ERR_MSG(extack, "Nexthop has been deleted"); 3795 goto out_free; 3796 } 3797 rt->nh = nh; 3798 fib6_nh = nexthop_fib6_nh(rt->nh); 3799 } else { 3800 err = fib6_nh_init(net, rt->fib6_nh, cfg, gfp_flags, extack); 3801 if (err) 3802 goto out; 3803 3804 fib6_nh = rt->fib6_nh; 3805 3806 /* We cannot add true routes via loopback here, they would 3807 * result in kernel looping; promote them to reject routes 3808 */ 3809 addr_type = ipv6_addr_type(&cfg->fc_dst); 3810 if (fib6_is_reject(cfg->fc_flags, rt->fib6_nh->fib_nh_dev, 3811 addr_type)) 3812 rt->fib6_flags = RTF_REJECT | RTF_NONEXTHOP; 3813 } 3814 3815 if (!ipv6_addr_any(&cfg->fc_prefsrc)) { 3816 struct net_device *dev = fib6_nh->fib_nh_dev; 3817 3818 if (!ipv6_chk_addr(net, &cfg->fc_prefsrc, dev, 0)) { 3819 NL_SET_ERR_MSG(extack, "Invalid source address"); 3820 err = -EINVAL; 3821 goto out; 3822 } 3823 rt->fib6_prefsrc.addr = cfg->fc_prefsrc; 3824 rt->fib6_prefsrc.plen = 128; 3825 } else 3826 rt->fib6_prefsrc.plen = 0; 3827 3828 return rt; 3829 out: 3830 fib6_info_release(rt); 3831 return ERR_PTR(err); 3832 out_free: 3833 ip_fib_metrics_put(rt->fib6_metrics); 3834 kfree(rt); 3835 return ERR_PTR(err); 3836 } 3837 3838 int ip6_route_add(struct fib6_config *cfg, gfp_t gfp_flags, 3839 struct netlink_ext_ack *extack) 3840 { 3841 struct fib6_info *rt; 3842 int err; 3843 3844 rt = ip6_route_info_create(cfg, gfp_flags, extack); 3845 if (IS_ERR(rt)) 3846 return PTR_ERR(rt); 3847 3848 err = __ip6_ins_rt(rt, &cfg->fc_nlinfo, extack); 3849 fib6_info_release(rt); 3850 3851 return err; 3852 } 3853 3854 static int __ip6_del_rt(struct fib6_info *rt, struct nl_info *info) 3855 { 3856 struct net *net = info->nl_net; 3857 struct fib6_table *table; 3858 int err; 3859 3860 if (rt == net->ipv6.fib6_null_entry) { 3861 err = -ENOENT; 3862 goto out; 3863 } 3864 3865 table = rt->fib6_table; 3866 spin_lock_bh(&table->tb6_lock); 3867 err = fib6_del(rt, info); 3868 spin_unlock_bh(&table->tb6_lock); 3869 3870 out: 3871 fib6_info_release(rt); 3872 return err; 3873 } 3874 3875 int ip6_del_rt(struct net *net, struct fib6_info *rt, bool skip_notify) 3876 { 3877 struct nl_info info = { 3878 .nl_net = net, 3879 .skip_notify = skip_notify 3880 }; 3881 3882 return __ip6_del_rt(rt, &info); 3883 } 3884 3885 static int __ip6_del_rt_siblings(struct fib6_info *rt, struct fib6_config *cfg) 3886 { 3887 struct nl_info *info = &cfg->fc_nlinfo; 3888 struct net *net = info->nl_net; 3889 struct sk_buff *skb = NULL; 3890 struct fib6_table *table; 3891 int err = -ENOENT; 3892 3893 if (rt == net->ipv6.fib6_null_entry) 3894 goto out_put; 3895 table = rt->fib6_table; 3896 spin_lock_bh(&table->tb6_lock); 3897 3898 if (rt->fib6_nsiblings && cfg->fc_delete_all_nh) { 3899 struct fib6_info *sibling, *next_sibling; 3900 struct fib6_node *fn; 3901 3902 /* prefer to send a single notification with all hops */ 3903 skb = nlmsg_new(rt6_nlmsg_size(rt), gfp_any()); 3904 if (skb) { 3905 u32 seq = info->nlh ? info->nlh->nlmsg_seq : 0; 3906 3907 if (rt6_fill_node(net, skb, rt, NULL, 3908 NULL, NULL, 0, RTM_DELROUTE, 3909 info->portid, seq, 0) < 0) { 3910 kfree_skb(skb); 3911 skb = NULL; 3912 } else 3913 info->skip_notify = 1; 3914 } 3915 3916 /* 'rt' points to the first sibling route. If it is not the 3917 * leaf, then we do not need to send a notification. Otherwise, 3918 * we need to check if the last sibling has a next route or not 3919 * and emit a replace or delete notification, respectively. 3920 */ 3921 info->skip_notify_kernel = 1; 3922 fn = rcu_dereference_protected(rt->fib6_node, 3923 lockdep_is_held(&table->tb6_lock)); 3924 if (rcu_access_pointer(fn->leaf) == rt) { 3925 struct fib6_info *last_sibling, *replace_rt; 3926 3927 last_sibling = list_last_entry(&rt->fib6_siblings, 3928 struct fib6_info, 3929 fib6_siblings); 3930 replace_rt = rcu_dereference_protected( 3931 last_sibling->fib6_next, 3932 lockdep_is_held(&table->tb6_lock)); 3933 if (replace_rt) 3934 call_fib6_entry_notifiers_replace(net, 3935 replace_rt); 3936 else 3937 call_fib6_multipath_entry_notifiers(net, 3938 FIB_EVENT_ENTRY_DEL, 3939 rt, rt->fib6_nsiblings, 3940 NULL); 3941 } 3942 list_for_each_entry_safe(sibling, next_sibling, 3943 &rt->fib6_siblings, 3944 fib6_siblings) { 3945 err = fib6_del(sibling, info); 3946 if (err) 3947 goto out_unlock; 3948 } 3949 } 3950 3951 err = fib6_del(rt, info); 3952 out_unlock: 3953 spin_unlock_bh(&table->tb6_lock); 3954 out_put: 3955 fib6_info_release(rt); 3956 3957 if (skb) { 3958 rtnl_notify(skb, net, info->portid, RTNLGRP_IPV6_ROUTE, 3959 info->nlh, gfp_any()); 3960 } 3961 return err; 3962 } 3963 3964 static int __ip6_del_cached_rt(struct rt6_info *rt, struct fib6_config *cfg) 3965 { 3966 int rc = -ESRCH; 3967 3968 if (cfg->fc_ifindex && rt->dst.dev->ifindex != cfg->fc_ifindex) 3969 goto out; 3970 3971 if (cfg->fc_flags & RTF_GATEWAY && 3972 !ipv6_addr_equal(&cfg->fc_gateway, &rt->rt6i_gateway)) 3973 goto out; 3974 3975 rc = rt6_remove_exception_rt(rt); 3976 out: 3977 return rc; 3978 } 3979 3980 static int ip6_del_cached_rt(struct fib6_config *cfg, struct fib6_info *rt, 3981 struct fib6_nh *nh) 3982 { 3983 struct fib6_result res = { 3984 .f6i = rt, 3985 .nh = nh, 3986 }; 3987 struct rt6_info *rt_cache; 3988 3989 rt_cache = rt6_find_cached_rt(&res, &cfg->fc_dst, &cfg->fc_src); 3990 if (rt_cache) 3991 return __ip6_del_cached_rt(rt_cache, cfg); 3992 3993 return 0; 3994 } 3995 3996 struct fib6_nh_del_cached_rt_arg { 3997 struct fib6_config *cfg; 3998 struct fib6_info *f6i; 3999 }; 4000 4001 static int fib6_nh_del_cached_rt(struct fib6_nh *nh, void *_arg) 4002 { 4003 struct fib6_nh_del_cached_rt_arg *arg = _arg; 4004 int rc; 4005 4006 rc = ip6_del_cached_rt(arg->cfg, arg->f6i, nh); 4007 return rc != -ESRCH ? rc : 0; 4008 } 4009 4010 static int ip6_del_cached_rt_nh(struct fib6_config *cfg, struct fib6_info *f6i) 4011 { 4012 struct fib6_nh_del_cached_rt_arg arg = { 4013 .cfg = cfg, 4014 .f6i = f6i 4015 }; 4016 4017 return nexthop_for_each_fib6_nh(f6i->nh, fib6_nh_del_cached_rt, &arg); 4018 } 4019 4020 static int ip6_route_del(struct fib6_config *cfg, 4021 struct netlink_ext_ack *extack) 4022 { 4023 struct fib6_table *table; 4024 struct fib6_info *rt; 4025 struct fib6_node *fn; 4026 int err = -ESRCH; 4027 4028 table = fib6_get_table(cfg->fc_nlinfo.nl_net, cfg->fc_table); 4029 if (!table) { 4030 NL_SET_ERR_MSG(extack, "FIB table does not exist"); 4031 return err; 4032 } 4033 4034 rcu_read_lock(); 4035 4036 fn = fib6_locate(&table->tb6_root, 4037 &cfg->fc_dst, cfg->fc_dst_len, 4038 &cfg->fc_src, cfg->fc_src_len, 4039 !(cfg->fc_flags & RTF_CACHE)); 4040 4041 if (fn) { 4042 for_each_fib6_node_rt_rcu(fn) { 4043 struct fib6_nh *nh; 4044 4045 if (rt->nh && cfg->fc_nh_id && 4046 rt->nh->id != cfg->fc_nh_id) 4047 continue; 4048 4049 if (cfg->fc_flags & RTF_CACHE) { 4050 int rc = 0; 4051 4052 if (rt->nh) { 4053 rc = ip6_del_cached_rt_nh(cfg, rt); 4054 } else if (cfg->fc_nh_id) { 4055 continue; 4056 } else { 4057 nh = rt->fib6_nh; 4058 rc = ip6_del_cached_rt(cfg, rt, nh); 4059 } 4060 if (rc != -ESRCH) { 4061 rcu_read_unlock(); 4062 return rc; 4063 } 4064 continue; 4065 } 4066 4067 if (cfg->fc_metric && cfg->fc_metric != rt->fib6_metric) 4068 continue; 4069 if (cfg->fc_protocol && 4070 cfg->fc_protocol != rt->fib6_protocol) 4071 continue; 4072 4073 if (rt->nh) { 4074 if (!fib6_info_hold_safe(rt)) 4075 continue; 4076 rcu_read_unlock(); 4077 4078 return __ip6_del_rt(rt, &cfg->fc_nlinfo); 4079 } 4080 if (cfg->fc_nh_id) 4081 continue; 4082 4083 nh = rt->fib6_nh; 4084 if (cfg->fc_ifindex && 4085 (!nh->fib_nh_dev || 4086 nh->fib_nh_dev->ifindex != cfg->fc_ifindex)) 4087 continue; 4088 if (cfg->fc_flags & RTF_GATEWAY && 4089 !ipv6_addr_equal(&cfg->fc_gateway, &nh->fib_nh_gw6)) 4090 continue; 4091 if (!fib6_info_hold_safe(rt)) 4092 continue; 4093 rcu_read_unlock(); 4094 4095 /* if gateway was specified only delete the one hop */ 4096 if (cfg->fc_flags & RTF_GATEWAY) 4097 return __ip6_del_rt(rt, &cfg->fc_nlinfo); 4098 4099 return __ip6_del_rt_siblings(rt, cfg); 4100 } 4101 } 4102 rcu_read_unlock(); 4103 4104 return err; 4105 } 4106 4107 static void rt6_do_redirect(struct dst_entry *dst, struct sock *sk, struct sk_buff *skb) 4108 { 4109 struct netevent_redirect netevent; 4110 struct rt6_info *rt, *nrt = NULL; 4111 struct fib6_result res = {}; 4112 struct ndisc_options ndopts; 4113 struct inet6_dev *in6_dev; 4114 struct neighbour *neigh; 4115 struct rd_msg *msg; 4116 int optlen, on_link; 4117 u8 *lladdr; 4118 4119 optlen = skb_tail_pointer(skb) - skb_transport_header(skb); 4120 optlen -= sizeof(*msg); 4121 4122 if (optlen < 0) { 4123 net_dbg_ratelimited("rt6_do_redirect: packet too short\n"); 4124 return; 4125 } 4126 4127 msg = (struct rd_msg *)icmp6_hdr(skb); 4128 4129 if (ipv6_addr_is_multicast(&msg->dest)) { 4130 net_dbg_ratelimited("rt6_do_redirect: destination address is multicast\n"); 4131 return; 4132 } 4133 4134 on_link = 0; 4135 if (ipv6_addr_equal(&msg->dest, &msg->target)) { 4136 on_link = 1; 4137 } else if (ipv6_addr_type(&msg->target) != 4138 (IPV6_ADDR_UNICAST|IPV6_ADDR_LINKLOCAL)) { 4139 net_dbg_ratelimited("rt6_do_redirect: target address is not link-local unicast\n"); 4140 return; 4141 } 4142 4143 in6_dev = __in6_dev_get(skb->dev); 4144 if (!in6_dev) 4145 return; 4146 if (in6_dev->cnf.forwarding || !in6_dev->cnf.accept_redirects) 4147 return; 4148 4149 /* RFC2461 8.1: 4150 * The IP source address of the Redirect MUST be the same as the current 4151 * first-hop router for the specified ICMP Destination Address. 4152 */ 4153 4154 if (!ndisc_parse_options(skb->dev, msg->opt, optlen, &ndopts)) { 4155 net_dbg_ratelimited("rt6_redirect: invalid ND options\n"); 4156 return; 4157 } 4158 4159 lladdr = NULL; 4160 if (ndopts.nd_opts_tgt_lladdr) { 4161 lladdr = ndisc_opt_addr_data(ndopts.nd_opts_tgt_lladdr, 4162 skb->dev); 4163 if (!lladdr) { 4164 net_dbg_ratelimited("rt6_redirect: invalid link-layer address length\n"); 4165 return; 4166 } 4167 } 4168 4169 rt = (struct rt6_info *) dst; 4170 if (rt->rt6i_flags & RTF_REJECT) { 4171 net_dbg_ratelimited("rt6_redirect: source isn't a valid nexthop for redirect target\n"); 4172 return; 4173 } 4174 4175 /* Redirect received -> path was valid. 4176 * Look, redirects are sent only in response to data packets, 4177 * so that this nexthop apparently is reachable. --ANK 4178 */ 4179 dst_confirm_neigh(&rt->dst, &ipv6_hdr(skb)->saddr); 4180 4181 neigh = __neigh_lookup(&nd_tbl, &msg->target, skb->dev, 1); 4182 if (!neigh) 4183 return; 4184 4185 /* 4186 * We have finally decided to accept it. 4187 */ 4188 4189 ndisc_update(skb->dev, neigh, lladdr, NUD_STALE, 4190 NEIGH_UPDATE_F_WEAK_OVERRIDE| 4191 NEIGH_UPDATE_F_OVERRIDE| 4192 (on_link ? 0 : (NEIGH_UPDATE_F_OVERRIDE_ISROUTER| 4193 NEIGH_UPDATE_F_ISROUTER)), 4194 NDISC_REDIRECT, &ndopts); 4195 4196 rcu_read_lock(); 4197 res.f6i = rcu_dereference(rt->from); 4198 if (!res.f6i) 4199 goto out; 4200 4201 if (res.f6i->nh) { 4202 struct fib6_nh_match_arg arg = { 4203 .dev = dst->dev, 4204 .gw = &rt->rt6i_gateway, 4205 }; 4206 4207 nexthop_for_each_fib6_nh(res.f6i->nh, 4208 fib6_nh_find_match, &arg); 4209 4210 /* fib6_info uses a nexthop that does not have fib6_nh 4211 * using the dst->dev. Should be impossible 4212 */ 4213 if (!arg.match) 4214 goto out; 4215 res.nh = arg.match; 4216 } else { 4217 res.nh = res.f6i->fib6_nh; 4218 } 4219 4220 res.fib6_flags = res.f6i->fib6_flags; 4221 res.fib6_type = res.f6i->fib6_type; 4222 nrt = ip6_rt_cache_alloc(&res, &msg->dest, NULL); 4223 if (!nrt) 4224 goto out; 4225 4226 nrt->rt6i_flags = RTF_GATEWAY|RTF_UP|RTF_DYNAMIC|RTF_CACHE; 4227 if (on_link) 4228 nrt->rt6i_flags &= ~RTF_GATEWAY; 4229 4230 nrt->rt6i_gateway = *(struct in6_addr *)neigh->primary_key; 4231 4232 /* rt6_insert_exception() will take care of duplicated exceptions */ 4233 if (rt6_insert_exception(nrt, &res)) { 4234 dst_release_immediate(&nrt->dst); 4235 goto out; 4236 } 4237 4238 netevent.old = &rt->dst; 4239 netevent.new = &nrt->dst; 4240 netevent.daddr = &msg->dest; 4241 netevent.neigh = neigh; 4242 call_netevent_notifiers(NETEVENT_REDIRECT, &netevent); 4243 4244 out: 4245 rcu_read_unlock(); 4246 neigh_release(neigh); 4247 } 4248 4249 #ifdef CONFIG_IPV6_ROUTE_INFO 4250 static struct fib6_info *rt6_get_route_info(struct net *net, 4251 const struct in6_addr *prefix, int prefixlen, 4252 const struct in6_addr *gwaddr, 4253 struct net_device *dev) 4254 { 4255 u32 tb_id = l3mdev_fib_table(dev) ? : RT6_TABLE_INFO; 4256 int ifindex = dev->ifindex; 4257 struct fib6_node *fn; 4258 struct fib6_info *rt = NULL; 4259 struct fib6_table *table; 4260 4261 table = fib6_get_table(net, tb_id); 4262 if (!table) 4263 return NULL; 4264 4265 rcu_read_lock(); 4266 fn = fib6_locate(&table->tb6_root, prefix, prefixlen, NULL, 0, true); 4267 if (!fn) 4268 goto out; 4269 4270 for_each_fib6_node_rt_rcu(fn) { 4271 /* these routes do not use nexthops */ 4272 if (rt->nh) 4273 continue; 4274 if (rt->fib6_nh->fib_nh_dev->ifindex != ifindex) 4275 continue; 4276 if (!(rt->fib6_flags & RTF_ROUTEINFO) || 4277 !rt->fib6_nh->fib_nh_gw_family) 4278 continue; 4279 if (!ipv6_addr_equal(&rt->fib6_nh->fib_nh_gw6, gwaddr)) 4280 continue; 4281 if (!fib6_info_hold_safe(rt)) 4282 continue; 4283 break; 4284 } 4285 out: 4286 rcu_read_unlock(); 4287 return rt; 4288 } 4289 4290 static struct fib6_info *rt6_add_route_info(struct net *net, 4291 const struct in6_addr *prefix, int prefixlen, 4292 const struct in6_addr *gwaddr, 4293 struct net_device *dev, 4294 unsigned int pref) 4295 { 4296 struct fib6_config cfg = { 4297 .fc_metric = IP6_RT_PRIO_USER, 4298 .fc_ifindex = dev->ifindex, 4299 .fc_dst_len = prefixlen, 4300 .fc_flags = RTF_GATEWAY | RTF_ADDRCONF | RTF_ROUTEINFO | 4301 RTF_UP | RTF_PREF(pref), 4302 .fc_protocol = RTPROT_RA, 4303 .fc_type = RTN_UNICAST, 4304 .fc_nlinfo.portid = 0, 4305 .fc_nlinfo.nlh = NULL, 4306 .fc_nlinfo.nl_net = net, 4307 }; 4308 4309 cfg.fc_table = l3mdev_fib_table(dev) ? : RT6_TABLE_INFO; 4310 cfg.fc_dst = *prefix; 4311 cfg.fc_gateway = *gwaddr; 4312 4313 /* We should treat it as a default route if prefix length is 0. */ 4314 if (!prefixlen) 4315 cfg.fc_flags |= RTF_DEFAULT; 4316 4317 ip6_route_add(&cfg, GFP_ATOMIC, NULL); 4318 4319 return rt6_get_route_info(net, prefix, prefixlen, gwaddr, dev); 4320 } 4321 #endif 4322 4323 struct fib6_info *rt6_get_dflt_router(struct net *net, 4324 const struct in6_addr *addr, 4325 struct net_device *dev) 4326 { 4327 u32 tb_id = l3mdev_fib_table(dev) ? : RT6_TABLE_DFLT; 4328 struct fib6_info *rt; 4329 struct fib6_table *table; 4330 4331 table = fib6_get_table(net, tb_id); 4332 if (!table) 4333 return NULL; 4334 4335 rcu_read_lock(); 4336 for_each_fib6_node_rt_rcu(&table->tb6_root) { 4337 struct fib6_nh *nh; 4338 4339 /* RA routes do not use nexthops */ 4340 if (rt->nh) 4341 continue; 4342 4343 nh = rt->fib6_nh; 4344 if (dev == nh->fib_nh_dev && 4345 ((rt->fib6_flags & (RTF_ADDRCONF | RTF_DEFAULT)) == (RTF_ADDRCONF | RTF_DEFAULT)) && 4346 ipv6_addr_equal(&nh->fib_nh_gw6, addr)) 4347 break; 4348 } 4349 if (rt && !fib6_info_hold_safe(rt)) 4350 rt = NULL; 4351 rcu_read_unlock(); 4352 return rt; 4353 } 4354 4355 struct fib6_info *rt6_add_dflt_router(struct net *net, 4356 const struct in6_addr *gwaddr, 4357 struct net_device *dev, 4358 unsigned int pref, 4359 u32 defrtr_usr_metric) 4360 { 4361 struct fib6_config cfg = { 4362 .fc_table = l3mdev_fib_table(dev) ? : RT6_TABLE_DFLT, 4363 .fc_metric = defrtr_usr_metric, 4364 .fc_ifindex = dev->ifindex, 4365 .fc_flags = RTF_GATEWAY | RTF_ADDRCONF | RTF_DEFAULT | 4366 RTF_UP | RTF_EXPIRES | RTF_PREF(pref), 4367 .fc_protocol = RTPROT_RA, 4368 .fc_type = RTN_UNICAST, 4369 .fc_nlinfo.portid = 0, 4370 .fc_nlinfo.nlh = NULL, 4371 .fc_nlinfo.nl_net = net, 4372 }; 4373 4374 cfg.fc_gateway = *gwaddr; 4375 4376 if (!ip6_route_add(&cfg, GFP_ATOMIC, NULL)) { 4377 struct fib6_table *table; 4378 4379 table = fib6_get_table(dev_net(dev), cfg.fc_table); 4380 if (table) 4381 table->flags |= RT6_TABLE_HAS_DFLT_ROUTER; 4382 } 4383 4384 return rt6_get_dflt_router(net, gwaddr, dev); 4385 } 4386 4387 static void __rt6_purge_dflt_routers(struct net *net, 4388 struct fib6_table *table) 4389 { 4390 struct fib6_info *rt; 4391 4392 restart: 4393 rcu_read_lock(); 4394 for_each_fib6_node_rt_rcu(&table->tb6_root) { 4395 struct net_device *dev = fib6_info_nh_dev(rt); 4396 struct inet6_dev *idev = dev ? __in6_dev_get(dev) : NULL; 4397 4398 if (rt->fib6_flags & (RTF_DEFAULT | RTF_ADDRCONF) && 4399 (!idev || idev->cnf.accept_ra != 2) && 4400 fib6_info_hold_safe(rt)) { 4401 rcu_read_unlock(); 4402 ip6_del_rt(net, rt, false); 4403 goto restart; 4404 } 4405 } 4406 rcu_read_unlock(); 4407 4408 table->flags &= ~RT6_TABLE_HAS_DFLT_ROUTER; 4409 } 4410 4411 void rt6_purge_dflt_routers(struct net *net) 4412 { 4413 struct fib6_table *table; 4414 struct hlist_head *head; 4415 unsigned int h; 4416 4417 rcu_read_lock(); 4418 4419 for (h = 0; h < FIB6_TABLE_HASHSZ; h++) { 4420 head = &net->ipv6.fib_table_hash[h]; 4421 hlist_for_each_entry_rcu(table, head, tb6_hlist) { 4422 if (table->flags & RT6_TABLE_HAS_DFLT_ROUTER) 4423 __rt6_purge_dflt_routers(net, table); 4424 } 4425 } 4426 4427 rcu_read_unlock(); 4428 } 4429 4430 static void rtmsg_to_fib6_config(struct net *net, 4431 struct in6_rtmsg *rtmsg, 4432 struct fib6_config *cfg) 4433 { 4434 *cfg = (struct fib6_config){ 4435 .fc_table = l3mdev_fib_table_by_index(net, rtmsg->rtmsg_ifindex) ? 4436 : RT6_TABLE_MAIN, 4437 .fc_ifindex = rtmsg->rtmsg_ifindex, 4438 .fc_metric = rtmsg->rtmsg_metric, 4439 .fc_expires = rtmsg->rtmsg_info, 4440 .fc_dst_len = rtmsg->rtmsg_dst_len, 4441 .fc_src_len = rtmsg->rtmsg_src_len, 4442 .fc_flags = rtmsg->rtmsg_flags, 4443 .fc_type = rtmsg->rtmsg_type, 4444 4445 .fc_nlinfo.nl_net = net, 4446 4447 .fc_dst = rtmsg->rtmsg_dst, 4448 .fc_src = rtmsg->rtmsg_src, 4449 .fc_gateway = rtmsg->rtmsg_gateway, 4450 }; 4451 } 4452 4453 int ipv6_route_ioctl(struct net *net, unsigned int cmd, struct in6_rtmsg *rtmsg) 4454 { 4455 struct fib6_config cfg; 4456 int err; 4457 4458 if (cmd != SIOCADDRT && cmd != SIOCDELRT) 4459 return -EINVAL; 4460 if (!ns_capable(net->user_ns, CAP_NET_ADMIN)) 4461 return -EPERM; 4462 4463 rtmsg_to_fib6_config(net, rtmsg, &cfg); 4464 4465 rtnl_lock(); 4466 switch (cmd) { 4467 case SIOCADDRT: 4468 /* Only do the default setting of fc_metric in route adding */ 4469 if (cfg.fc_metric == 0) 4470 cfg.fc_metric = IP6_RT_PRIO_USER; 4471 err = ip6_route_add(&cfg, GFP_KERNEL, NULL); 4472 break; 4473 case SIOCDELRT: 4474 err = ip6_route_del(&cfg, NULL); 4475 break; 4476 } 4477 rtnl_unlock(); 4478 return err; 4479 } 4480 4481 /* 4482 * Drop the packet on the floor 4483 */ 4484 4485 static int ip6_pkt_drop(struct sk_buff *skb, u8 code, int ipstats_mib_noroutes) 4486 { 4487 struct dst_entry *dst = skb_dst(skb); 4488 struct net *net = dev_net(dst->dev); 4489 struct inet6_dev *idev; 4490 SKB_DR(reason); 4491 int type; 4492 4493 if (netif_is_l3_master(skb->dev) || 4494 dst->dev == net->loopback_dev) 4495 idev = __in6_dev_get_safely(dev_get_by_index_rcu(net, IP6CB(skb)->iif)); 4496 else 4497 idev = ip6_dst_idev(dst); 4498 4499 switch (ipstats_mib_noroutes) { 4500 case IPSTATS_MIB_INNOROUTES: 4501 type = ipv6_addr_type(&ipv6_hdr(skb)->daddr); 4502 if (type == IPV6_ADDR_ANY) { 4503 SKB_DR_SET(reason, IP_INADDRERRORS); 4504 IP6_INC_STATS(net, idev, IPSTATS_MIB_INADDRERRORS); 4505 break; 4506 } 4507 SKB_DR_SET(reason, IP_INNOROUTES); 4508 fallthrough; 4509 case IPSTATS_MIB_OUTNOROUTES: 4510 SKB_DR_OR(reason, IP_OUTNOROUTES); 4511 IP6_INC_STATS(net, idev, ipstats_mib_noroutes); 4512 break; 4513 } 4514 4515 /* Start over by dropping the dst for l3mdev case */ 4516 if (netif_is_l3_master(skb->dev)) 4517 skb_dst_drop(skb); 4518 4519 icmpv6_send(skb, ICMPV6_DEST_UNREACH, code, 0); 4520 kfree_skb_reason(skb, reason); 4521 return 0; 4522 } 4523 4524 static int ip6_pkt_discard(struct sk_buff *skb) 4525 { 4526 return ip6_pkt_drop(skb, ICMPV6_NOROUTE, IPSTATS_MIB_INNOROUTES); 4527 } 4528 4529 static int ip6_pkt_discard_out(struct net *net, struct sock *sk, struct sk_buff *skb) 4530 { 4531 skb->dev = skb_dst(skb)->dev; 4532 return ip6_pkt_drop(skb, ICMPV6_NOROUTE, IPSTATS_MIB_OUTNOROUTES); 4533 } 4534 4535 static int ip6_pkt_prohibit(struct sk_buff *skb) 4536 { 4537 return ip6_pkt_drop(skb, ICMPV6_ADM_PROHIBITED, IPSTATS_MIB_INNOROUTES); 4538 } 4539 4540 static int ip6_pkt_prohibit_out(struct net *net, struct sock *sk, struct sk_buff *skb) 4541 { 4542 skb->dev = skb_dst(skb)->dev; 4543 return ip6_pkt_drop(skb, ICMPV6_ADM_PROHIBITED, IPSTATS_MIB_OUTNOROUTES); 4544 } 4545 4546 /* 4547 * Allocate a dst for local (unicast / anycast) address. 4548 */ 4549 4550 struct fib6_info *addrconf_f6i_alloc(struct net *net, 4551 struct inet6_dev *idev, 4552 const struct in6_addr *addr, 4553 bool anycast, gfp_t gfp_flags, 4554 struct netlink_ext_ack *extack) 4555 { 4556 struct fib6_config cfg = { 4557 .fc_table = l3mdev_fib_table(idev->dev) ? : RT6_TABLE_LOCAL, 4558 .fc_ifindex = idev->dev->ifindex, 4559 .fc_flags = RTF_UP | RTF_NONEXTHOP, 4560 .fc_dst = *addr, 4561 .fc_dst_len = 128, 4562 .fc_protocol = RTPROT_KERNEL, 4563 .fc_nlinfo.nl_net = net, 4564 .fc_ignore_dev_down = true, 4565 }; 4566 struct fib6_info *f6i; 4567 4568 if (anycast) { 4569 cfg.fc_type = RTN_ANYCAST; 4570 cfg.fc_flags |= RTF_ANYCAST; 4571 } else { 4572 cfg.fc_type = RTN_LOCAL; 4573 cfg.fc_flags |= RTF_LOCAL; 4574 } 4575 4576 f6i = ip6_route_info_create(&cfg, gfp_flags, extack); 4577 if (!IS_ERR(f6i)) { 4578 f6i->dst_nocount = true; 4579 4580 if (!anycast && 4581 (net->ipv6.devconf_all->disable_policy || 4582 idev->cnf.disable_policy)) 4583 f6i->dst_nopolicy = true; 4584 } 4585 4586 return f6i; 4587 } 4588 4589 /* remove deleted ip from prefsrc entries */ 4590 struct arg_dev_net_ip { 4591 struct net *net; 4592 struct in6_addr *addr; 4593 }; 4594 4595 static int fib6_remove_prefsrc(struct fib6_info *rt, void *arg) 4596 { 4597 struct net *net = ((struct arg_dev_net_ip *)arg)->net; 4598 struct in6_addr *addr = ((struct arg_dev_net_ip *)arg)->addr; 4599 4600 if (!rt->nh && 4601 rt != net->ipv6.fib6_null_entry && 4602 ipv6_addr_equal(addr, &rt->fib6_prefsrc.addr) && 4603 !ipv6_chk_addr(net, addr, rt->fib6_nh->fib_nh_dev, 0)) { 4604 spin_lock_bh(&rt6_exception_lock); 4605 /* remove prefsrc entry */ 4606 rt->fib6_prefsrc.plen = 0; 4607 spin_unlock_bh(&rt6_exception_lock); 4608 } 4609 return 0; 4610 } 4611 4612 void rt6_remove_prefsrc(struct inet6_ifaddr *ifp) 4613 { 4614 struct net *net = dev_net(ifp->idev->dev); 4615 struct arg_dev_net_ip adni = { 4616 .net = net, 4617 .addr = &ifp->addr, 4618 }; 4619 fib6_clean_all(net, fib6_remove_prefsrc, &adni); 4620 } 4621 4622 #define RTF_RA_ROUTER (RTF_ADDRCONF | RTF_DEFAULT) 4623 4624 /* Remove routers and update dst entries when gateway turn into host. */ 4625 static int fib6_clean_tohost(struct fib6_info *rt, void *arg) 4626 { 4627 struct in6_addr *gateway = (struct in6_addr *)arg; 4628 struct fib6_nh *nh; 4629 4630 /* RA routes do not use nexthops */ 4631 if (rt->nh) 4632 return 0; 4633 4634 nh = rt->fib6_nh; 4635 if (((rt->fib6_flags & RTF_RA_ROUTER) == RTF_RA_ROUTER) && 4636 nh->fib_nh_gw_family && ipv6_addr_equal(gateway, &nh->fib_nh_gw6)) 4637 return -1; 4638 4639 /* Further clean up cached routes in exception table. 4640 * This is needed because cached route may have a different 4641 * gateway than its 'parent' in the case of an ip redirect. 4642 */ 4643 fib6_nh_exceptions_clean_tohost(nh, gateway); 4644 4645 return 0; 4646 } 4647 4648 void rt6_clean_tohost(struct net *net, struct in6_addr *gateway) 4649 { 4650 fib6_clean_all(net, fib6_clean_tohost, gateway); 4651 } 4652 4653 struct arg_netdev_event { 4654 const struct net_device *dev; 4655 union { 4656 unsigned char nh_flags; 4657 unsigned long event; 4658 }; 4659 }; 4660 4661 static struct fib6_info *rt6_multipath_first_sibling(const struct fib6_info *rt) 4662 { 4663 struct fib6_info *iter; 4664 struct fib6_node *fn; 4665 4666 fn = rcu_dereference_protected(rt->fib6_node, 4667 lockdep_is_held(&rt->fib6_table->tb6_lock)); 4668 iter = rcu_dereference_protected(fn->leaf, 4669 lockdep_is_held(&rt->fib6_table->tb6_lock)); 4670 while (iter) { 4671 if (iter->fib6_metric == rt->fib6_metric && 4672 rt6_qualify_for_ecmp(iter)) 4673 return iter; 4674 iter = rcu_dereference_protected(iter->fib6_next, 4675 lockdep_is_held(&rt->fib6_table->tb6_lock)); 4676 } 4677 4678 return NULL; 4679 } 4680 4681 /* only called for fib entries with builtin fib6_nh */ 4682 static bool rt6_is_dead(const struct fib6_info *rt) 4683 { 4684 if (rt->fib6_nh->fib_nh_flags & RTNH_F_DEAD || 4685 (rt->fib6_nh->fib_nh_flags & RTNH_F_LINKDOWN && 4686 ip6_ignore_linkdown(rt->fib6_nh->fib_nh_dev))) 4687 return true; 4688 4689 return false; 4690 } 4691 4692 static int rt6_multipath_total_weight(const struct fib6_info *rt) 4693 { 4694 struct fib6_info *iter; 4695 int total = 0; 4696 4697 if (!rt6_is_dead(rt)) 4698 total += rt->fib6_nh->fib_nh_weight; 4699 4700 list_for_each_entry(iter, &rt->fib6_siblings, fib6_siblings) { 4701 if (!rt6_is_dead(iter)) 4702 total += iter->fib6_nh->fib_nh_weight; 4703 } 4704 4705 return total; 4706 } 4707 4708 static void rt6_upper_bound_set(struct fib6_info *rt, int *weight, int total) 4709 { 4710 int upper_bound = -1; 4711 4712 if (!rt6_is_dead(rt)) { 4713 *weight += rt->fib6_nh->fib_nh_weight; 4714 upper_bound = DIV_ROUND_CLOSEST_ULL((u64) (*weight) << 31, 4715 total) - 1; 4716 } 4717 atomic_set(&rt->fib6_nh->fib_nh_upper_bound, upper_bound); 4718 } 4719 4720 static void rt6_multipath_upper_bound_set(struct fib6_info *rt, int total) 4721 { 4722 struct fib6_info *iter; 4723 int weight = 0; 4724 4725 rt6_upper_bound_set(rt, &weight, total); 4726 4727 list_for_each_entry(iter, &rt->fib6_siblings, fib6_siblings) 4728 rt6_upper_bound_set(iter, &weight, total); 4729 } 4730 4731 void rt6_multipath_rebalance(struct fib6_info *rt) 4732 { 4733 struct fib6_info *first; 4734 int total; 4735 4736 /* In case the entire multipath route was marked for flushing, 4737 * then there is no need to rebalance upon the removal of every 4738 * sibling route. 4739 */ 4740 if (!rt->fib6_nsiblings || rt->should_flush) 4741 return; 4742 4743 /* During lookup routes are evaluated in order, so we need to 4744 * make sure upper bounds are assigned from the first sibling 4745 * onwards. 4746 */ 4747 first = rt6_multipath_first_sibling(rt); 4748 if (WARN_ON_ONCE(!first)) 4749 return; 4750 4751 total = rt6_multipath_total_weight(first); 4752 rt6_multipath_upper_bound_set(first, total); 4753 } 4754 4755 static int fib6_ifup(struct fib6_info *rt, void *p_arg) 4756 { 4757 const struct arg_netdev_event *arg = p_arg; 4758 struct net *net = dev_net(arg->dev); 4759 4760 if (rt != net->ipv6.fib6_null_entry && !rt->nh && 4761 rt->fib6_nh->fib_nh_dev == arg->dev) { 4762 rt->fib6_nh->fib_nh_flags &= ~arg->nh_flags; 4763 fib6_update_sernum_upto_root(net, rt); 4764 rt6_multipath_rebalance(rt); 4765 } 4766 4767 return 0; 4768 } 4769 4770 void rt6_sync_up(struct net_device *dev, unsigned char nh_flags) 4771 { 4772 struct arg_netdev_event arg = { 4773 .dev = dev, 4774 { 4775 .nh_flags = nh_flags, 4776 }, 4777 }; 4778 4779 if (nh_flags & RTNH_F_DEAD && netif_carrier_ok(dev)) 4780 arg.nh_flags |= RTNH_F_LINKDOWN; 4781 4782 fib6_clean_all(dev_net(dev), fib6_ifup, &arg); 4783 } 4784 4785 /* only called for fib entries with inline fib6_nh */ 4786 static bool rt6_multipath_uses_dev(const struct fib6_info *rt, 4787 const struct net_device *dev) 4788 { 4789 struct fib6_info *iter; 4790 4791 if (rt->fib6_nh->fib_nh_dev == dev) 4792 return true; 4793 list_for_each_entry(iter, &rt->fib6_siblings, fib6_siblings) 4794 if (iter->fib6_nh->fib_nh_dev == dev) 4795 return true; 4796 4797 return false; 4798 } 4799 4800 static void rt6_multipath_flush(struct fib6_info *rt) 4801 { 4802 struct fib6_info *iter; 4803 4804 rt->should_flush = 1; 4805 list_for_each_entry(iter, &rt->fib6_siblings, fib6_siblings) 4806 iter->should_flush = 1; 4807 } 4808 4809 static unsigned int rt6_multipath_dead_count(const struct fib6_info *rt, 4810 const struct net_device *down_dev) 4811 { 4812 struct fib6_info *iter; 4813 unsigned int dead = 0; 4814 4815 if (rt->fib6_nh->fib_nh_dev == down_dev || 4816 rt->fib6_nh->fib_nh_flags & RTNH_F_DEAD) 4817 dead++; 4818 list_for_each_entry(iter, &rt->fib6_siblings, fib6_siblings) 4819 if (iter->fib6_nh->fib_nh_dev == down_dev || 4820 iter->fib6_nh->fib_nh_flags & RTNH_F_DEAD) 4821 dead++; 4822 4823 return dead; 4824 } 4825 4826 static void rt6_multipath_nh_flags_set(struct fib6_info *rt, 4827 const struct net_device *dev, 4828 unsigned char nh_flags) 4829 { 4830 struct fib6_info *iter; 4831 4832 if (rt->fib6_nh->fib_nh_dev == dev) 4833 rt->fib6_nh->fib_nh_flags |= nh_flags; 4834 list_for_each_entry(iter, &rt->fib6_siblings, fib6_siblings) 4835 if (iter->fib6_nh->fib_nh_dev == dev) 4836 iter->fib6_nh->fib_nh_flags |= nh_flags; 4837 } 4838 4839 /* called with write lock held for table with rt */ 4840 static int fib6_ifdown(struct fib6_info *rt, void *p_arg) 4841 { 4842 const struct arg_netdev_event *arg = p_arg; 4843 const struct net_device *dev = arg->dev; 4844 struct net *net = dev_net(dev); 4845 4846 if (rt == net->ipv6.fib6_null_entry || rt->nh) 4847 return 0; 4848 4849 switch (arg->event) { 4850 case NETDEV_UNREGISTER: 4851 return rt->fib6_nh->fib_nh_dev == dev ? -1 : 0; 4852 case NETDEV_DOWN: 4853 if (rt->should_flush) 4854 return -1; 4855 if (!rt->fib6_nsiblings) 4856 return rt->fib6_nh->fib_nh_dev == dev ? -1 : 0; 4857 if (rt6_multipath_uses_dev(rt, dev)) { 4858 unsigned int count; 4859 4860 count = rt6_multipath_dead_count(rt, dev); 4861 if (rt->fib6_nsiblings + 1 == count) { 4862 rt6_multipath_flush(rt); 4863 return -1; 4864 } 4865 rt6_multipath_nh_flags_set(rt, dev, RTNH_F_DEAD | 4866 RTNH_F_LINKDOWN); 4867 fib6_update_sernum(net, rt); 4868 rt6_multipath_rebalance(rt); 4869 } 4870 return -2; 4871 case NETDEV_CHANGE: 4872 if (rt->fib6_nh->fib_nh_dev != dev || 4873 rt->fib6_flags & (RTF_LOCAL | RTF_ANYCAST)) 4874 break; 4875 rt->fib6_nh->fib_nh_flags |= RTNH_F_LINKDOWN; 4876 rt6_multipath_rebalance(rt); 4877 break; 4878 } 4879 4880 return 0; 4881 } 4882 4883 void rt6_sync_down_dev(struct net_device *dev, unsigned long event) 4884 { 4885 struct arg_netdev_event arg = { 4886 .dev = dev, 4887 { 4888 .event = event, 4889 }, 4890 }; 4891 struct net *net = dev_net(dev); 4892 4893 if (net->ipv6.sysctl.skip_notify_on_dev_down) 4894 fib6_clean_all_skip_notify(net, fib6_ifdown, &arg); 4895 else 4896 fib6_clean_all(net, fib6_ifdown, &arg); 4897 } 4898 4899 void rt6_disable_ip(struct net_device *dev, unsigned long event) 4900 { 4901 rt6_sync_down_dev(dev, event); 4902 rt6_uncached_list_flush_dev(dev); 4903 neigh_ifdown(&nd_tbl, dev); 4904 } 4905 4906 struct rt6_mtu_change_arg { 4907 struct net_device *dev; 4908 unsigned int mtu; 4909 struct fib6_info *f6i; 4910 }; 4911 4912 static int fib6_nh_mtu_change(struct fib6_nh *nh, void *_arg) 4913 { 4914 struct rt6_mtu_change_arg *arg = (struct rt6_mtu_change_arg *)_arg; 4915 struct fib6_info *f6i = arg->f6i; 4916 4917 /* For administrative MTU increase, there is no way to discover 4918 * IPv6 PMTU increase, so PMTU increase should be updated here. 4919 * Since RFC 1981 doesn't include administrative MTU increase 4920 * update PMTU increase is a MUST. (i.e. jumbo frame) 4921 */ 4922 if (nh->fib_nh_dev == arg->dev) { 4923 struct inet6_dev *idev = __in6_dev_get(arg->dev); 4924 u32 mtu = f6i->fib6_pmtu; 4925 4926 if (mtu >= arg->mtu || 4927 (mtu < arg->mtu && mtu == idev->cnf.mtu6)) 4928 fib6_metric_set(f6i, RTAX_MTU, arg->mtu); 4929 4930 spin_lock_bh(&rt6_exception_lock); 4931 rt6_exceptions_update_pmtu(idev, nh, arg->mtu); 4932 spin_unlock_bh(&rt6_exception_lock); 4933 } 4934 4935 return 0; 4936 } 4937 4938 static int rt6_mtu_change_route(struct fib6_info *f6i, void *p_arg) 4939 { 4940 struct rt6_mtu_change_arg *arg = (struct rt6_mtu_change_arg *) p_arg; 4941 struct inet6_dev *idev; 4942 4943 /* In IPv6 pmtu discovery is not optional, 4944 so that RTAX_MTU lock cannot disable it. 4945 We still use this lock to block changes 4946 caused by addrconf/ndisc. 4947 */ 4948 4949 idev = __in6_dev_get(arg->dev); 4950 if (!idev) 4951 return 0; 4952 4953 if (fib6_metric_locked(f6i, RTAX_MTU)) 4954 return 0; 4955 4956 arg->f6i = f6i; 4957 if (f6i->nh) { 4958 /* fib6_nh_mtu_change only returns 0, so this is safe */ 4959 return nexthop_for_each_fib6_nh(f6i->nh, fib6_nh_mtu_change, 4960 arg); 4961 } 4962 4963 return fib6_nh_mtu_change(f6i->fib6_nh, arg); 4964 } 4965 4966 void rt6_mtu_change(struct net_device *dev, unsigned int mtu) 4967 { 4968 struct rt6_mtu_change_arg arg = { 4969 .dev = dev, 4970 .mtu = mtu, 4971 }; 4972 4973 fib6_clean_all(dev_net(dev), rt6_mtu_change_route, &arg); 4974 } 4975 4976 static const struct nla_policy rtm_ipv6_policy[RTA_MAX+1] = { 4977 [RTA_UNSPEC] = { .strict_start_type = RTA_DPORT + 1 }, 4978 [RTA_GATEWAY] = { .len = sizeof(struct in6_addr) }, 4979 [RTA_PREFSRC] = { .len = sizeof(struct in6_addr) }, 4980 [RTA_OIF] = { .type = NLA_U32 }, 4981 [RTA_IIF] = { .type = NLA_U32 }, 4982 [RTA_PRIORITY] = { .type = NLA_U32 }, 4983 [RTA_METRICS] = { .type = NLA_NESTED }, 4984 [RTA_MULTIPATH] = { .len = sizeof(struct rtnexthop) }, 4985 [RTA_PREF] = { .type = NLA_U8 }, 4986 [RTA_ENCAP_TYPE] = { .type = NLA_U16 }, 4987 [RTA_ENCAP] = { .type = NLA_NESTED }, 4988 [RTA_EXPIRES] = { .type = NLA_U32 }, 4989 [RTA_UID] = { .type = NLA_U32 }, 4990 [RTA_MARK] = { .type = NLA_U32 }, 4991 [RTA_TABLE] = { .type = NLA_U32 }, 4992 [RTA_IP_PROTO] = { .type = NLA_U8 }, 4993 [RTA_SPORT] = { .type = NLA_U16 }, 4994 [RTA_DPORT] = { .type = NLA_U16 }, 4995 [RTA_NH_ID] = { .type = NLA_U32 }, 4996 }; 4997 4998 static int rtm_to_fib6_config(struct sk_buff *skb, struct nlmsghdr *nlh, 4999 struct fib6_config *cfg, 5000 struct netlink_ext_ack *extack) 5001 { 5002 struct rtmsg *rtm; 5003 struct nlattr *tb[RTA_MAX+1]; 5004 unsigned int pref; 5005 int err; 5006 5007 err = nlmsg_parse_deprecated(nlh, sizeof(*rtm), tb, RTA_MAX, 5008 rtm_ipv6_policy, extack); 5009 if (err < 0) 5010 goto errout; 5011 5012 err = -EINVAL; 5013 rtm = nlmsg_data(nlh); 5014 5015 if (rtm->rtm_tos) { 5016 NL_SET_ERR_MSG(extack, 5017 "Invalid dsfield (tos): option not available for IPv6"); 5018 goto errout; 5019 } 5020 5021 *cfg = (struct fib6_config){ 5022 .fc_table = rtm->rtm_table, 5023 .fc_dst_len = rtm->rtm_dst_len, 5024 .fc_src_len = rtm->rtm_src_len, 5025 .fc_flags = RTF_UP, 5026 .fc_protocol = rtm->rtm_protocol, 5027 .fc_type = rtm->rtm_type, 5028 5029 .fc_nlinfo.portid = NETLINK_CB(skb).portid, 5030 .fc_nlinfo.nlh = nlh, 5031 .fc_nlinfo.nl_net = sock_net(skb->sk), 5032 }; 5033 5034 if (rtm->rtm_type == RTN_UNREACHABLE || 5035 rtm->rtm_type == RTN_BLACKHOLE || 5036 rtm->rtm_type == RTN_PROHIBIT || 5037 rtm->rtm_type == RTN_THROW) 5038 cfg->fc_flags |= RTF_REJECT; 5039 5040 if (rtm->rtm_type == RTN_LOCAL) 5041 cfg->fc_flags |= RTF_LOCAL; 5042 5043 if (rtm->rtm_flags & RTM_F_CLONED) 5044 cfg->fc_flags |= RTF_CACHE; 5045 5046 cfg->fc_flags |= (rtm->rtm_flags & RTNH_F_ONLINK); 5047 5048 if (tb[RTA_NH_ID]) { 5049 if (tb[RTA_GATEWAY] || tb[RTA_OIF] || 5050 tb[RTA_MULTIPATH] || tb[RTA_ENCAP]) { 5051 NL_SET_ERR_MSG(extack, 5052 "Nexthop specification and nexthop id are mutually exclusive"); 5053 goto errout; 5054 } 5055 cfg->fc_nh_id = nla_get_u32(tb[RTA_NH_ID]); 5056 } 5057 5058 if (tb[RTA_GATEWAY]) { 5059 cfg->fc_gateway = nla_get_in6_addr(tb[RTA_GATEWAY]); 5060 cfg->fc_flags |= RTF_GATEWAY; 5061 } 5062 if (tb[RTA_VIA]) { 5063 NL_SET_ERR_MSG(extack, "IPv6 does not support RTA_VIA attribute"); 5064 goto errout; 5065 } 5066 5067 if (tb[RTA_DST]) { 5068 int plen = (rtm->rtm_dst_len + 7) >> 3; 5069 5070 if (nla_len(tb[RTA_DST]) < plen) 5071 goto errout; 5072 5073 nla_memcpy(&cfg->fc_dst, tb[RTA_DST], plen); 5074 } 5075 5076 if (tb[RTA_SRC]) { 5077 int plen = (rtm->rtm_src_len + 7) >> 3; 5078 5079 if (nla_len(tb[RTA_SRC]) < plen) 5080 goto errout; 5081 5082 nla_memcpy(&cfg->fc_src, tb[RTA_SRC], plen); 5083 } 5084 5085 if (tb[RTA_PREFSRC]) 5086 cfg->fc_prefsrc = nla_get_in6_addr(tb[RTA_PREFSRC]); 5087 5088 if (tb[RTA_OIF]) 5089 cfg->fc_ifindex = nla_get_u32(tb[RTA_OIF]); 5090 5091 if (tb[RTA_PRIORITY]) 5092 cfg->fc_metric = nla_get_u32(tb[RTA_PRIORITY]); 5093 5094 if (tb[RTA_METRICS]) { 5095 cfg->fc_mx = nla_data(tb[RTA_METRICS]); 5096 cfg->fc_mx_len = nla_len(tb[RTA_METRICS]); 5097 } 5098 5099 if (tb[RTA_TABLE]) 5100 cfg->fc_table = nla_get_u32(tb[RTA_TABLE]); 5101 5102 if (tb[RTA_MULTIPATH]) { 5103 cfg->fc_mp = nla_data(tb[RTA_MULTIPATH]); 5104 cfg->fc_mp_len = nla_len(tb[RTA_MULTIPATH]); 5105 5106 err = lwtunnel_valid_encap_type_attr(cfg->fc_mp, 5107 cfg->fc_mp_len, extack); 5108 if (err < 0) 5109 goto errout; 5110 } 5111 5112 if (tb[RTA_PREF]) { 5113 pref = nla_get_u8(tb[RTA_PREF]); 5114 if (pref != ICMPV6_ROUTER_PREF_LOW && 5115 pref != ICMPV6_ROUTER_PREF_HIGH) 5116 pref = ICMPV6_ROUTER_PREF_MEDIUM; 5117 cfg->fc_flags |= RTF_PREF(pref); 5118 } 5119 5120 if (tb[RTA_ENCAP]) 5121 cfg->fc_encap = tb[RTA_ENCAP]; 5122 5123 if (tb[RTA_ENCAP_TYPE]) { 5124 cfg->fc_encap_type = nla_get_u16(tb[RTA_ENCAP_TYPE]); 5125 5126 err = lwtunnel_valid_encap_type(cfg->fc_encap_type, extack); 5127 if (err < 0) 5128 goto errout; 5129 } 5130 5131 if (tb[RTA_EXPIRES]) { 5132 unsigned long timeout = addrconf_timeout_fixup(nla_get_u32(tb[RTA_EXPIRES]), HZ); 5133 5134 if (addrconf_finite_timeout(timeout)) { 5135 cfg->fc_expires = jiffies_to_clock_t(timeout * HZ); 5136 cfg->fc_flags |= RTF_EXPIRES; 5137 } 5138 } 5139 5140 err = 0; 5141 errout: 5142 return err; 5143 } 5144 5145 struct rt6_nh { 5146 struct fib6_info *fib6_info; 5147 struct fib6_config r_cfg; 5148 struct list_head next; 5149 }; 5150 5151 static int ip6_route_info_append(struct net *net, 5152 struct list_head *rt6_nh_list, 5153 struct fib6_info *rt, 5154 struct fib6_config *r_cfg) 5155 { 5156 struct rt6_nh *nh; 5157 int err = -EEXIST; 5158 5159 list_for_each_entry(nh, rt6_nh_list, next) { 5160 /* check if fib6_info already exists */ 5161 if (rt6_duplicate_nexthop(nh->fib6_info, rt)) 5162 return err; 5163 } 5164 5165 nh = kzalloc(sizeof(*nh), GFP_KERNEL); 5166 if (!nh) 5167 return -ENOMEM; 5168 nh->fib6_info = rt; 5169 memcpy(&nh->r_cfg, r_cfg, sizeof(*r_cfg)); 5170 list_add_tail(&nh->next, rt6_nh_list); 5171 5172 return 0; 5173 } 5174 5175 static void ip6_route_mpath_notify(struct fib6_info *rt, 5176 struct fib6_info *rt_last, 5177 struct nl_info *info, 5178 __u16 nlflags) 5179 { 5180 /* if this is an APPEND route, then rt points to the first route 5181 * inserted and rt_last points to last route inserted. Userspace 5182 * wants a consistent dump of the route which starts at the first 5183 * nexthop. Since sibling routes are always added at the end of 5184 * the list, find the first sibling of the last route appended 5185 */ 5186 if ((nlflags & NLM_F_APPEND) && rt_last && rt_last->fib6_nsiblings) { 5187 rt = list_first_entry(&rt_last->fib6_siblings, 5188 struct fib6_info, 5189 fib6_siblings); 5190 } 5191 5192 if (rt) 5193 inet6_rt_notify(RTM_NEWROUTE, rt, info, nlflags); 5194 } 5195 5196 static bool ip6_route_mpath_should_notify(const struct fib6_info *rt) 5197 { 5198 bool rt_can_ecmp = rt6_qualify_for_ecmp(rt); 5199 bool should_notify = false; 5200 struct fib6_info *leaf; 5201 struct fib6_node *fn; 5202 5203 rcu_read_lock(); 5204 fn = rcu_dereference(rt->fib6_node); 5205 if (!fn) 5206 goto out; 5207 5208 leaf = rcu_dereference(fn->leaf); 5209 if (!leaf) 5210 goto out; 5211 5212 if (rt == leaf || 5213 (rt_can_ecmp && rt->fib6_metric == leaf->fib6_metric && 5214 rt6_qualify_for_ecmp(leaf))) 5215 should_notify = true; 5216 out: 5217 rcu_read_unlock(); 5218 5219 return should_notify; 5220 } 5221 5222 static int fib6_gw_from_attr(struct in6_addr *gw, struct nlattr *nla, 5223 struct netlink_ext_ack *extack) 5224 { 5225 if (nla_len(nla) < sizeof(*gw)) { 5226 NL_SET_ERR_MSG(extack, "Invalid IPv6 address in RTA_GATEWAY"); 5227 return -EINVAL; 5228 } 5229 5230 *gw = nla_get_in6_addr(nla); 5231 5232 return 0; 5233 } 5234 5235 static int ip6_route_multipath_add(struct fib6_config *cfg, 5236 struct netlink_ext_ack *extack) 5237 { 5238 struct fib6_info *rt_notif = NULL, *rt_last = NULL; 5239 struct nl_info *info = &cfg->fc_nlinfo; 5240 struct fib6_config r_cfg; 5241 struct rtnexthop *rtnh; 5242 struct fib6_info *rt; 5243 struct rt6_nh *err_nh; 5244 struct rt6_nh *nh, *nh_safe; 5245 __u16 nlflags; 5246 int remaining; 5247 int attrlen; 5248 int err = 1; 5249 int nhn = 0; 5250 int replace = (cfg->fc_nlinfo.nlh && 5251 (cfg->fc_nlinfo.nlh->nlmsg_flags & NLM_F_REPLACE)); 5252 LIST_HEAD(rt6_nh_list); 5253 5254 nlflags = replace ? NLM_F_REPLACE : NLM_F_CREATE; 5255 if (info->nlh && info->nlh->nlmsg_flags & NLM_F_APPEND) 5256 nlflags |= NLM_F_APPEND; 5257 5258 remaining = cfg->fc_mp_len; 5259 rtnh = (struct rtnexthop *)cfg->fc_mp; 5260 5261 /* Parse a Multipath Entry and build a list (rt6_nh_list) of 5262 * fib6_info structs per nexthop 5263 */ 5264 while (rtnh_ok(rtnh, remaining)) { 5265 memcpy(&r_cfg, cfg, sizeof(*cfg)); 5266 if (rtnh->rtnh_ifindex) 5267 r_cfg.fc_ifindex = rtnh->rtnh_ifindex; 5268 5269 attrlen = rtnh_attrlen(rtnh); 5270 if (attrlen > 0) { 5271 struct nlattr *nla, *attrs = rtnh_attrs(rtnh); 5272 5273 nla = nla_find(attrs, attrlen, RTA_GATEWAY); 5274 if (nla) { 5275 err = fib6_gw_from_attr(&r_cfg.fc_gateway, nla, 5276 extack); 5277 if (err) 5278 goto cleanup; 5279 5280 r_cfg.fc_flags |= RTF_GATEWAY; 5281 } 5282 r_cfg.fc_encap = nla_find(attrs, attrlen, RTA_ENCAP); 5283 5284 /* RTA_ENCAP_TYPE length checked in 5285 * lwtunnel_valid_encap_type_attr 5286 */ 5287 nla = nla_find(attrs, attrlen, RTA_ENCAP_TYPE); 5288 if (nla) 5289 r_cfg.fc_encap_type = nla_get_u16(nla); 5290 } 5291 5292 r_cfg.fc_flags |= (rtnh->rtnh_flags & RTNH_F_ONLINK); 5293 rt = ip6_route_info_create(&r_cfg, GFP_KERNEL, extack); 5294 if (IS_ERR(rt)) { 5295 err = PTR_ERR(rt); 5296 rt = NULL; 5297 goto cleanup; 5298 } 5299 if (!rt6_qualify_for_ecmp(rt)) { 5300 err = -EINVAL; 5301 NL_SET_ERR_MSG(extack, 5302 "Device only routes can not be added for IPv6 using the multipath API."); 5303 fib6_info_release(rt); 5304 goto cleanup; 5305 } 5306 5307 rt->fib6_nh->fib_nh_weight = rtnh->rtnh_hops + 1; 5308 5309 err = ip6_route_info_append(info->nl_net, &rt6_nh_list, 5310 rt, &r_cfg); 5311 if (err) { 5312 fib6_info_release(rt); 5313 goto cleanup; 5314 } 5315 5316 rtnh = rtnh_next(rtnh, &remaining); 5317 } 5318 5319 if (list_empty(&rt6_nh_list)) { 5320 NL_SET_ERR_MSG(extack, 5321 "Invalid nexthop configuration - no valid nexthops"); 5322 return -EINVAL; 5323 } 5324 5325 /* for add and replace send one notification with all nexthops. 5326 * Skip the notification in fib6_add_rt2node and send one with 5327 * the full route when done 5328 */ 5329 info->skip_notify = 1; 5330 5331 /* For add and replace, send one notification with all nexthops. For 5332 * append, send one notification with all appended nexthops. 5333 */ 5334 info->skip_notify_kernel = 1; 5335 5336 err_nh = NULL; 5337 list_for_each_entry(nh, &rt6_nh_list, next) { 5338 err = __ip6_ins_rt(nh->fib6_info, info, extack); 5339 5340 if (err) { 5341 if (replace && nhn) 5342 NL_SET_ERR_MSG_MOD(extack, 5343 "multipath route replace failed (check consistency of installed routes)"); 5344 err_nh = nh; 5345 goto add_errout; 5346 } 5347 /* save reference to last route successfully inserted */ 5348 rt_last = nh->fib6_info; 5349 5350 /* save reference to first route for notification */ 5351 if (!rt_notif) 5352 rt_notif = nh->fib6_info; 5353 5354 /* Because each route is added like a single route we remove 5355 * these flags after the first nexthop: if there is a collision, 5356 * we have already failed to add the first nexthop: 5357 * fib6_add_rt2node() has rejected it; when replacing, old 5358 * nexthops have been replaced by first new, the rest should 5359 * be added to it. 5360 */ 5361 if (cfg->fc_nlinfo.nlh) { 5362 cfg->fc_nlinfo.nlh->nlmsg_flags &= ~(NLM_F_EXCL | 5363 NLM_F_REPLACE); 5364 cfg->fc_nlinfo.nlh->nlmsg_flags |= NLM_F_CREATE; 5365 } 5366 nhn++; 5367 } 5368 5369 /* An in-kernel notification should only be sent in case the new 5370 * multipath route is added as the first route in the node, or if 5371 * it was appended to it. We pass 'rt_notif' since it is the first 5372 * sibling and might allow us to skip some checks in the replace case. 5373 */ 5374 if (ip6_route_mpath_should_notify(rt_notif)) { 5375 enum fib_event_type fib_event; 5376 5377 if (rt_notif->fib6_nsiblings != nhn - 1) 5378 fib_event = FIB_EVENT_ENTRY_APPEND; 5379 else 5380 fib_event = FIB_EVENT_ENTRY_REPLACE; 5381 5382 err = call_fib6_multipath_entry_notifiers(info->nl_net, 5383 fib_event, rt_notif, 5384 nhn - 1, extack); 5385 if (err) { 5386 /* Delete all the siblings that were just added */ 5387 err_nh = NULL; 5388 goto add_errout; 5389 } 5390 } 5391 5392 /* success ... tell user about new route */ 5393 ip6_route_mpath_notify(rt_notif, rt_last, info, nlflags); 5394 goto cleanup; 5395 5396 add_errout: 5397 /* send notification for routes that were added so that 5398 * the delete notifications sent by ip6_route_del are 5399 * coherent 5400 */ 5401 if (rt_notif) 5402 ip6_route_mpath_notify(rt_notif, rt_last, info, nlflags); 5403 5404 /* Delete routes that were already added */ 5405 list_for_each_entry(nh, &rt6_nh_list, next) { 5406 if (err_nh == nh) 5407 break; 5408 ip6_route_del(&nh->r_cfg, extack); 5409 } 5410 5411 cleanup: 5412 list_for_each_entry_safe(nh, nh_safe, &rt6_nh_list, next) { 5413 fib6_info_release(nh->fib6_info); 5414 list_del(&nh->next); 5415 kfree(nh); 5416 } 5417 5418 return err; 5419 } 5420 5421 static int ip6_route_multipath_del(struct fib6_config *cfg, 5422 struct netlink_ext_ack *extack) 5423 { 5424 struct fib6_config r_cfg; 5425 struct rtnexthop *rtnh; 5426 int last_err = 0; 5427 int remaining; 5428 int attrlen; 5429 int err; 5430 5431 remaining = cfg->fc_mp_len; 5432 rtnh = (struct rtnexthop *)cfg->fc_mp; 5433 5434 /* Parse a Multipath Entry */ 5435 while (rtnh_ok(rtnh, remaining)) { 5436 memcpy(&r_cfg, cfg, sizeof(*cfg)); 5437 if (rtnh->rtnh_ifindex) 5438 r_cfg.fc_ifindex = rtnh->rtnh_ifindex; 5439 5440 attrlen = rtnh_attrlen(rtnh); 5441 if (attrlen > 0) { 5442 struct nlattr *nla, *attrs = rtnh_attrs(rtnh); 5443 5444 nla = nla_find(attrs, attrlen, RTA_GATEWAY); 5445 if (nla) { 5446 err = fib6_gw_from_attr(&r_cfg.fc_gateway, nla, 5447 extack); 5448 if (err) { 5449 last_err = err; 5450 goto next_rtnh; 5451 } 5452 5453 r_cfg.fc_flags |= RTF_GATEWAY; 5454 } 5455 } 5456 err = ip6_route_del(&r_cfg, extack); 5457 if (err) 5458 last_err = err; 5459 5460 next_rtnh: 5461 rtnh = rtnh_next(rtnh, &remaining); 5462 } 5463 5464 return last_err; 5465 } 5466 5467 static int inet6_rtm_delroute(struct sk_buff *skb, struct nlmsghdr *nlh, 5468 struct netlink_ext_ack *extack) 5469 { 5470 struct fib6_config cfg; 5471 int err; 5472 5473 err = rtm_to_fib6_config(skb, nlh, &cfg, extack); 5474 if (err < 0) 5475 return err; 5476 5477 if (cfg.fc_nh_id && 5478 !nexthop_find_by_id(sock_net(skb->sk), cfg.fc_nh_id)) { 5479 NL_SET_ERR_MSG(extack, "Nexthop id does not exist"); 5480 return -EINVAL; 5481 } 5482 5483 if (cfg.fc_mp) 5484 return ip6_route_multipath_del(&cfg, extack); 5485 else { 5486 cfg.fc_delete_all_nh = 1; 5487 return ip6_route_del(&cfg, extack); 5488 } 5489 } 5490 5491 static int inet6_rtm_newroute(struct sk_buff *skb, struct nlmsghdr *nlh, 5492 struct netlink_ext_ack *extack) 5493 { 5494 struct fib6_config cfg; 5495 int err; 5496 5497 err = rtm_to_fib6_config(skb, nlh, &cfg, extack); 5498 if (err < 0) 5499 return err; 5500 5501 if (cfg.fc_metric == 0) 5502 cfg.fc_metric = IP6_RT_PRIO_USER; 5503 5504 if (cfg.fc_mp) 5505 return ip6_route_multipath_add(&cfg, extack); 5506 else 5507 return ip6_route_add(&cfg, GFP_KERNEL, extack); 5508 } 5509 5510 /* add the overhead of this fib6_nh to nexthop_len */ 5511 static int rt6_nh_nlmsg_size(struct fib6_nh *nh, void *arg) 5512 { 5513 int *nexthop_len = arg; 5514 5515 *nexthop_len += nla_total_size(0) /* RTA_MULTIPATH */ 5516 + NLA_ALIGN(sizeof(struct rtnexthop)) 5517 + nla_total_size(16); /* RTA_GATEWAY */ 5518 5519 if (nh->fib_nh_lws) { 5520 /* RTA_ENCAP_TYPE */ 5521 *nexthop_len += lwtunnel_get_encap_size(nh->fib_nh_lws); 5522 /* RTA_ENCAP */ 5523 *nexthop_len += nla_total_size(2); 5524 } 5525 5526 return 0; 5527 } 5528 5529 static size_t rt6_nlmsg_size(struct fib6_info *f6i) 5530 { 5531 int nexthop_len; 5532 5533 if (f6i->nh) { 5534 nexthop_len = nla_total_size(4); /* RTA_NH_ID */ 5535 nexthop_for_each_fib6_nh(f6i->nh, rt6_nh_nlmsg_size, 5536 &nexthop_len); 5537 } else { 5538 struct fib6_info *sibling, *next_sibling; 5539 struct fib6_nh *nh = f6i->fib6_nh; 5540 5541 nexthop_len = 0; 5542 if (f6i->fib6_nsiblings) { 5543 rt6_nh_nlmsg_size(nh, &nexthop_len); 5544 5545 list_for_each_entry_safe(sibling, next_sibling, 5546 &f6i->fib6_siblings, fib6_siblings) { 5547 rt6_nh_nlmsg_size(sibling->fib6_nh, &nexthop_len); 5548 } 5549 } 5550 nexthop_len += lwtunnel_get_encap_size(nh->fib_nh_lws); 5551 } 5552 5553 return NLMSG_ALIGN(sizeof(struct rtmsg)) 5554 + nla_total_size(16) /* RTA_SRC */ 5555 + nla_total_size(16) /* RTA_DST */ 5556 + nla_total_size(16) /* RTA_GATEWAY */ 5557 + nla_total_size(16) /* RTA_PREFSRC */ 5558 + nla_total_size(4) /* RTA_TABLE */ 5559 + nla_total_size(4) /* RTA_IIF */ 5560 + nla_total_size(4) /* RTA_OIF */ 5561 + nla_total_size(4) /* RTA_PRIORITY */ 5562 + RTAX_MAX * nla_total_size(4) /* RTA_METRICS */ 5563 + nla_total_size(sizeof(struct rta_cacheinfo)) 5564 + nla_total_size(TCP_CA_NAME_MAX) /* RTAX_CC_ALGO */ 5565 + nla_total_size(1) /* RTA_PREF */ 5566 + nexthop_len; 5567 } 5568 5569 static int rt6_fill_node_nexthop(struct sk_buff *skb, struct nexthop *nh, 5570 unsigned char *flags) 5571 { 5572 if (nexthop_is_multipath(nh)) { 5573 struct nlattr *mp; 5574 5575 mp = nla_nest_start_noflag(skb, RTA_MULTIPATH); 5576 if (!mp) 5577 goto nla_put_failure; 5578 5579 if (nexthop_mpath_fill_node(skb, nh, AF_INET6)) 5580 goto nla_put_failure; 5581 5582 nla_nest_end(skb, mp); 5583 } else { 5584 struct fib6_nh *fib6_nh; 5585 5586 fib6_nh = nexthop_fib6_nh(nh); 5587 if (fib_nexthop_info(skb, &fib6_nh->nh_common, AF_INET6, 5588 flags, false) < 0) 5589 goto nla_put_failure; 5590 } 5591 5592 return 0; 5593 5594 nla_put_failure: 5595 return -EMSGSIZE; 5596 } 5597 5598 static int rt6_fill_node(struct net *net, struct sk_buff *skb, 5599 struct fib6_info *rt, struct dst_entry *dst, 5600 struct in6_addr *dest, struct in6_addr *src, 5601 int iif, int type, u32 portid, u32 seq, 5602 unsigned int flags) 5603 { 5604 struct rt6_info *rt6 = (struct rt6_info *)dst; 5605 struct rt6key *rt6_dst, *rt6_src; 5606 u32 *pmetrics, table, rt6_flags; 5607 unsigned char nh_flags = 0; 5608 struct nlmsghdr *nlh; 5609 struct rtmsg *rtm; 5610 long expires = 0; 5611 5612 nlh = nlmsg_put(skb, portid, seq, type, sizeof(*rtm), flags); 5613 if (!nlh) 5614 return -EMSGSIZE; 5615 5616 if (rt6) { 5617 rt6_dst = &rt6->rt6i_dst; 5618 rt6_src = &rt6->rt6i_src; 5619 rt6_flags = rt6->rt6i_flags; 5620 } else { 5621 rt6_dst = &rt->fib6_dst; 5622 rt6_src = &rt->fib6_src; 5623 rt6_flags = rt->fib6_flags; 5624 } 5625 5626 rtm = nlmsg_data(nlh); 5627 rtm->rtm_family = AF_INET6; 5628 rtm->rtm_dst_len = rt6_dst->plen; 5629 rtm->rtm_src_len = rt6_src->plen; 5630 rtm->rtm_tos = 0; 5631 if (rt->fib6_table) 5632 table = rt->fib6_table->tb6_id; 5633 else 5634 table = RT6_TABLE_UNSPEC; 5635 rtm->rtm_table = table < 256 ? table : RT_TABLE_COMPAT; 5636 if (nla_put_u32(skb, RTA_TABLE, table)) 5637 goto nla_put_failure; 5638 5639 rtm->rtm_type = rt->fib6_type; 5640 rtm->rtm_flags = 0; 5641 rtm->rtm_scope = RT_SCOPE_UNIVERSE; 5642 rtm->rtm_protocol = rt->fib6_protocol; 5643 5644 if (rt6_flags & RTF_CACHE) 5645 rtm->rtm_flags |= RTM_F_CLONED; 5646 5647 if (dest) { 5648 if (nla_put_in6_addr(skb, RTA_DST, dest)) 5649 goto nla_put_failure; 5650 rtm->rtm_dst_len = 128; 5651 } else if (rtm->rtm_dst_len) 5652 if (nla_put_in6_addr(skb, RTA_DST, &rt6_dst->addr)) 5653 goto nla_put_failure; 5654 #ifdef CONFIG_IPV6_SUBTREES 5655 if (src) { 5656 if (nla_put_in6_addr(skb, RTA_SRC, src)) 5657 goto nla_put_failure; 5658 rtm->rtm_src_len = 128; 5659 } else if (rtm->rtm_src_len && 5660 nla_put_in6_addr(skb, RTA_SRC, &rt6_src->addr)) 5661 goto nla_put_failure; 5662 #endif 5663 if (iif) { 5664 #ifdef CONFIG_IPV6_MROUTE 5665 if (ipv6_addr_is_multicast(&rt6_dst->addr)) { 5666 int err = ip6mr_get_route(net, skb, rtm, portid); 5667 5668 if (err == 0) 5669 return 0; 5670 if (err < 0) 5671 goto nla_put_failure; 5672 } else 5673 #endif 5674 if (nla_put_u32(skb, RTA_IIF, iif)) 5675 goto nla_put_failure; 5676 } else if (dest) { 5677 struct in6_addr saddr_buf; 5678 if (ip6_route_get_saddr(net, rt, dest, 0, &saddr_buf) == 0 && 5679 nla_put_in6_addr(skb, RTA_PREFSRC, &saddr_buf)) 5680 goto nla_put_failure; 5681 } 5682 5683 if (rt->fib6_prefsrc.plen) { 5684 struct in6_addr saddr_buf; 5685 saddr_buf = rt->fib6_prefsrc.addr; 5686 if (nla_put_in6_addr(skb, RTA_PREFSRC, &saddr_buf)) 5687 goto nla_put_failure; 5688 } 5689 5690 pmetrics = dst ? dst_metrics_ptr(dst) : rt->fib6_metrics->metrics; 5691 if (rtnetlink_put_metrics(skb, pmetrics) < 0) 5692 goto nla_put_failure; 5693 5694 if (nla_put_u32(skb, RTA_PRIORITY, rt->fib6_metric)) 5695 goto nla_put_failure; 5696 5697 /* For multipath routes, walk the siblings list and add 5698 * each as a nexthop within RTA_MULTIPATH. 5699 */ 5700 if (rt6) { 5701 if (rt6_flags & RTF_GATEWAY && 5702 nla_put_in6_addr(skb, RTA_GATEWAY, &rt6->rt6i_gateway)) 5703 goto nla_put_failure; 5704 5705 if (dst->dev && nla_put_u32(skb, RTA_OIF, dst->dev->ifindex)) 5706 goto nla_put_failure; 5707 5708 if (dst->lwtstate && 5709 lwtunnel_fill_encap(skb, dst->lwtstate, RTA_ENCAP, RTA_ENCAP_TYPE) < 0) 5710 goto nla_put_failure; 5711 } else if (rt->fib6_nsiblings) { 5712 struct fib6_info *sibling, *next_sibling; 5713 struct nlattr *mp; 5714 5715 mp = nla_nest_start_noflag(skb, RTA_MULTIPATH); 5716 if (!mp) 5717 goto nla_put_failure; 5718 5719 if (fib_add_nexthop(skb, &rt->fib6_nh->nh_common, 5720 rt->fib6_nh->fib_nh_weight, AF_INET6, 5721 0) < 0) 5722 goto nla_put_failure; 5723 5724 list_for_each_entry_safe(sibling, next_sibling, 5725 &rt->fib6_siblings, fib6_siblings) { 5726 if (fib_add_nexthop(skb, &sibling->fib6_nh->nh_common, 5727 sibling->fib6_nh->fib_nh_weight, 5728 AF_INET6, 0) < 0) 5729 goto nla_put_failure; 5730 } 5731 5732 nla_nest_end(skb, mp); 5733 } else if (rt->nh) { 5734 if (nla_put_u32(skb, RTA_NH_ID, rt->nh->id)) 5735 goto nla_put_failure; 5736 5737 if (nexthop_is_blackhole(rt->nh)) 5738 rtm->rtm_type = RTN_BLACKHOLE; 5739 5740 if (READ_ONCE(net->ipv4.sysctl_nexthop_compat_mode) && 5741 rt6_fill_node_nexthop(skb, rt->nh, &nh_flags) < 0) 5742 goto nla_put_failure; 5743 5744 rtm->rtm_flags |= nh_flags; 5745 } else { 5746 if (fib_nexthop_info(skb, &rt->fib6_nh->nh_common, AF_INET6, 5747 &nh_flags, false) < 0) 5748 goto nla_put_failure; 5749 5750 rtm->rtm_flags |= nh_flags; 5751 } 5752 5753 if (rt6_flags & RTF_EXPIRES) { 5754 expires = dst ? dst->expires : rt->expires; 5755 expires -= jiffies; 5756 } 5757 5758 if (!dst) { 5759 if (READ_ONCE(rt->offload)) 5760 rtm->rtm_flags |= RTM_F_OFFLOAD; 5761 if (READ_ONCE(rt->trap)) 5762 rtm->rtm_flags |= RTM_F_TRAP; 5763 if (READ_ONCE(rt->offload_failed)) 5764 rtm->rtm_flags |= RTM_F_OFFLOAD_FAILED; 5765 } 5766 5767 if (rtnl_put_cacheinfo(skb, dst, 0, expires, dst ? dst->error : 0) < 0) 5768 goto nla_put_failure; 5769 5770 if (nla_put_u8(skb, RTA_PREF, IPV6_EXTRACT_PREF(rt6_flags))) 5771 goto nla_put_failure; 5772 5773 5774 nlmsg_end(skb, nlh); 5775 return 0; 5776 5777 nla_put_failure: 5778 nlmsg_cancel(skb, nlh); 5779 return -EMSGSIZE; 5780 } 5781 5782 static int fib6_info_nh_uses_dev(struct fib6_nh *nh, void *arg) 5783 { 5784 const struct net_device *dev = arg; 5785 5786 if (nh->fib_nh_dev == dev) 5787 return 1; 5788 5789 return 0; 5790 } 5791 5792 static bool fib6_info_uses_dev(const struct fib6_info *f6i, 5793 const struct net_device *dev) 5794 { 5795 if (f6i->nh) { 5796 struct net_device *_dev = (struct net_device *)dev; 5797 5798 return !!nexthop_for_each_fib6_nh(f6i->nh, 5799 fib6_info_nh_uses_dev, 5800 _dev); 5801 } 5802 5803 if (f6i->fib6_nh->fib_nh_dev == dev) 5804 return true; 5805 5806 if (f6i->fib6_nsiblings) { 5807 struct fib6_info *sibling, *next_sibling; 5808 5809 list_for_each_entry_safe(sibling, next_sibling, 5810 &f6i->fib6_siblings, fib6_siblings) { 5811 if (sibling->fib6_nh->fib_nh_dev == dev) 5812 return true; 5813 } 5814 } 5815 5816 return false; 5817 } 5818 5819 struct fib6_nh_exception_dump_walker { 5820 struct rt6_rtnl_dump_arg *dump; 5821 struct fib6_info *rt; 5822 unsigned int flags; 5823 unsigned int skip; 5824 unsigned int count; 5825 }; 5826 5827 static int rt6_nh_dump_exceptions(struct fib6_nh *nh, void *arg) 5828 { 5829 struct fib6_nh_exception_dump_walker *w = arg; 5830 struct rt6_rtnl_dump_arg *dump = w->dump; 5831 struct rt6_exception_bucket *bucket; 5832 struct rt6_exception *rt6_ex; 5833 int i, err; 5834 5835 bucket = fib6_nh_get_excptn_bucket(nh, NULL); 5836 if (!bucket) 5837 return 0; 5838 5839 for (i = 0; i < FIB6_EXCEPTION_BUCKET_SIZE; i++) { 5840 hlist_for_each_entry(rt6_ex, &bucket->chain, hlist) { 5841 if (w->skip) { 5842 w->skip--; 5843 continue; 5844 } 5845 5846 /* Expiration of entries doesn't bump sernum, insertion 5847 * does. Removal is triggered by insertion, so we can 5848 * rely on the fact that if entries change between two 5849 * partial dumps, this node is scanned again completely, 5850 * see rt6_insert_exception() and fib6_dump_table(). 5851 * 5852 * Count expired entries we go through as handled 5853 * entries that we'll skip next time, in case of partial 5854 * node dump. Otherwise, if entries expire meanwhile, 5855 * we'll skip the wrong amount. 5856 */ 5857 if (rt6_check_expired(rt6_ex->rt6i)) { 5858 w->count++; 5859 continue; 5860 } 5861 5862 err = rt6_fill_node(dump->net, dump->skb, w->rt, 5863 &rt6_ex->rt6i->dst, NULL, NULL, 0, 5864 RTM_NEWROUTE, 5865 NETLINK_CB(dump->cb->skb).portid, 5866 dump->cb->nlh->nlmsg_seq, w->flags); 5867 if (err) 5868 return err; 5869 5870 w->count++; 5871 } 5872 bucket++; 5873 } 5874 5875 return 0; 5876 } 5877 5878 /* Return -1 if done with node, number of handled routes on partial dump */ 5879 int rt6_dump_route(struct fib6_info *rt, void *p_arg, unsigned int skip) 5880 { 5881 struct rt6_rtnl_dump_arg *arg = (struct rt6_rtnl_dump_arg *) p_arg; 5882 struct fib_dump_filter *filter = &arg->filter; 5883 unsigned int flags = NLM_F_MULTI; 5884 struct net *net = arg->net; 5885 int count = 0; 5886 5887 if (rt == net->ipv6.fib6_null_entry) 5888 return -1; 5889 5890 if ((filter->flags & RTM_F_PREFIX) && 5891 !(rt->fib6_flags & RTF_PREFIX_RT)) { 5892 /* success since this is not a prefix route */ 5893 return -1; 5894 } 5895 if (filter->filter_set && 5896 ((filter->rt_type && rt->fib6_type != filter->rt_type) || 5897 (filter->dev && !fib6_info_uses_dev(rt, filter->dev)) || 5898 (filter->protocol && rt->fib6_protocol != filter->protocol))) { 5899 return -1; 5900 } 5901 5902 if (filter->filter_set || 5903 !filter->dump_routes || !filter->dump_exceptions) { 5904 flags |= NLM_F_DUMP_FILTERED; 5905 } 5906 5907 if (filter->dump_routes) { 5908 if (skip) { 5909 skip--; 5910 } else { 5911 if (rt6_fill_node(net, arg->skb, rt, NULL, NULL, NULL, 5912 0, RTM_NEWROUTE, 5913 NETLINK_CB(arg->cb->skb).portid, 5914 arg->cb->nlh->nlmsg_seq, flags)) { 5915 return 0; 5916 } 5917 count++; 5918 } 5919 } 5920 5921 if (filter->dump_exceptions) { 5922 struct fib6_nh_exception_dump_walker w = { .dump = arg, 5923 .rt = rt, 5924 .flags = flags, 5925 .skip = skip, 5926 .count = 0 }; 5927 int err; 5928 5929 rcu_read_lock(); 5930 if (rt->nh) { 5931 err = nexthop_for_each_fib6_nh(rt->nh, 5932 rt6_nh_dump_exceptions, 5933 &w); 5934 } else { 5935 err = rt6_nh_dump_exceptions(rt->fib6_nh, &w); 5936 } 5937 rcu_read_unlock(); 5938 5939 if (err) 5940 return count + w.count; 5941 } 5942 5943 return -1; 5944 } 5945 5946 static int inet6_rtm_valid_getroute_req(struct sk_buff *skb, 5947 const struct nlmsghdr *nlh, 5948 struct nlattr **tb, 5949 struct netlink_ext_ack *extack) 5950 { 5951 struct rtmsg *rtm; 5952 int i, err; 5953 5954 if (nlh->nlmsg_len < nlmsg_msg_size(sizeof(*rtm))) { 5955 NL_SET_ERR_MSG_MOD(extack, 5956 "Invalid header for get route request"); 5957 return -EINVAL; 5958 } 5959 5960 if (!netlink_strict_get_check(skb)) 5961 return nlmsg_parse_deprecated(nlh, sizeof(*rtm), tb, RTA_MAX, 5962 rtm_ipv6_policy, extack); 5963 5964 rtm = nlmsg_data(nlh); 5965 if ((rtm->rtm_src_len && rtm->rtm_src_len != 128) || 5966 (rtm->rtm_dst_len && rtm->rtm_dst_len != 128) || 5967 rtm->rtm_table || rtm->rtm_protocol || rtm->rtm_scope || 5968 rtm->rtm_type) { 5969 NL_SET_ERR_MSG_MOD(extack, "Invalid values in header for get route request"); 5970 return -EINVAL; 5971 } 5972 if (rtm->rtm_flags & ~RTM_F_FIB_MATCH) { 5973 NL_SET_ERR_MSG_MOD(extack, 5974 "Invalid flags for get route request"); 5975 return -EINVAL; 5976 } 5977 5978 err = nlmsg_parse_deprecated_strict(nlh, sizeof(*rtm), tb, RTA_MAX, 5979 rtm_ipv6_policy, extack); 5980 if (err) 5981 return err; 5982 5983 if ((tb[RTA_SRC] && !rtm->rtm_src_len) || 5984 (tb[RTA_DST] && !rtm->rtm_dst_len)) { 5985 NL_SET_ERR_MSG_MOD(extack, "rtm_src_len and rtm_dst_len must be 128 for IPv6"); 5986 return -EINVAL; 5987 } 5988 5989 for (i = 0; i <= RTA_MAX; i++) { 5990 if (!tb[i]) 5991 continue; 5992 5993 switch (i) { 5994 case RTA_SRC: 5995 case RTA_DST: 5996 case RTA_IIF: 5997 case RTA_OIF: 5998 case RTA_MARK: 5999 case RTA_UID: 6000 case RTA_SPORT: 6001 case RTA_DPORT: 6002 case RTA_IP_PROTO: 6003 break; 6004 default: 6005 NL_SET_ERR_MSG_MOD(extack, "Unsupported attribute in get route request"); 6006 return -EINVAL; 6007 } 6008 } 6009 6010 return 0; 6011 } 6012 6013 static int inet6_rtm_getroute(struct sk_buff *in_skb, struct nlmsghdr *nlh, 6014 struct netlink_ext_ack *extack) 6015 { 6016 struct net *net = sock_net(in_skb->sk); 6017 struct nlattr *tb[RTA_MAX+1]; 6018 int err, iif = 0, oif = 0; 6019 struct fib6_info *from; 6020 struct dst_entry *dst; 6021 struct rt6_info *rt; 6022 struct sk_buff *skb; 6023 struct rtmsg *rtm; 6024 struct flowi6 fl6 = {}; 6025 bool fibmatch; 6026 6027 err = inet6_rtm_valid_getroute_req(in_skb, nlh, tb, extack); 6028 if (err < 0) 6029 goto errout; 6030 6031 err = -EINVAL; 6032 rtm = nlmsg_data(nlh); 6033 fl6.flowlabel = ip6_make_flowinfo(rtm->rtm_tos, 0); 6034 fibmatch = !!(rtm->rtm_flags & RTM_F_FIB_MATCH); 6035 6036 if (tb[RTA_SRC]) { 6037 if (nla_len(tb[RTA_SRC]) < sizeof(struct in6_addr)) 6038 goto errout; 6039 6040 fl6.saddr = *(struct in6_addr *)nla_data(tb[RTA_SRC]); 6041 } 6042 6043 if (tb[RTA_DST]) { 6044 if (nla_len(tb[RTA_DST]) < sizeof(struct in6_addr)) 6045 goto errout; 6046 6047 fl6.daddr = *(struct in6_addr *)nla_data(tb[RTA_DST]); 6048 } 6049 6050 if (tb[RTA_IIF]) 6051 iif = nla_get_u32(tb[RTA_IIF]); 6052 6053 if (tb[RTA_OIF]) 6054 oif = nla_get_u32(tb[RTA_OIF]); 6055 6056 if (tb[RTA_MARK]) 6057 fl6.flowi6_mark = nla_get_u32(tb[RTA_MARK]); 6058 6059 if (tb[RTA_UID]) 6060 fl6.flowi6_uid = make_kuid(current_user_ns(), 6061 nla_get_u32(tb[RTA_UID])); 6062 else 6063 fl6.flowi6_uid = iif ? INVALID_UID : current_uid(); 6064 6065 if (tb[RTA_SPORT]) 6066 fl6.fl6_sport = nla_get_be16(tb[RTA_SPORT]); 6067 6068 if (tb[RTA_DPORT]) 6069 fl6.fl6_dport = nla_get_be16(tb[RTA_DPORT]); 6070 6071 if (tb[RTA_IP_PROTO]) { 6072 err = rtm_getroute_parse_ip_proto(tb[RTA_IP_PROTO], 6073 &fl6.flowi6_proto, AF_INET6, 6074 extack); 6075 if (err) 6076 goto errout; 6077 } 6078 6079 if (iif) { 6080 struct net_device *dev; 6081 int flags = 0; 6082 6083 rcu_read_lock(); 6084 6085 dev = dev_get_by_index_rcu(net, iif); 6086 if (!dev) { 6087 rcu_read_unlock(); 6088 err = -ENODEV; 6089 goto errout; 6090 } 6091 6092 fl6.flowi6_iif = iif; 6093 6094 if (!ipv6_addr_any(&fl6.saddr)) 6095 flags |= RT6_LOOKUP_F_HAS_SADDR; 6096 6097 dst = ip6_route_input_lookup(net, dev, &fl6, NULL, flags); 6098 6099 rcu_read_unlock(); 6100 } else { 6101 fl6.flowi6_oif = oif; 6102 6103 dst = ip6_route_output(net, NULL, &fl6); 6104 } 6105 6106 6107 rt = container_of(dst, struct rt6_info, dst); 6108 if (rt->dst.error) { 6109 err = rt->dst.error; 6110 ip6_rt_put(rt); 6111 goto errout; 6112 } 6113 6114 if (rt == net->ipv6.ip6_null_entry) { 6115 err = rt->dst.error; 6116 ip6_rt_put(rt); 6117 goto errout; 6118 } 6119 6120 skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL); 6121 if (!skb) { 6122 ip6_rt_put(rt); 6123 err = -ENOBUFS; 6124 goto errout; 6125 } 6126 6127 skb_dst_set(skb, &rt->dst); 6128 6129 rcu_read_lock(); 6130 from = rcu_dereference(rt->from); 6131 if (from) { 6132 if (fibmatch) 6133 err = rt6_fill_node(net, skb, from, NULL, NULL, NULL, 6134 iif, RTM_NEWROUTE, 6135 NETLINK_CB(in_skb).portid, 6136 nlh->nlmsg_seq, 0); 6137 else 6138 err = rt6_fill_node(net, skb, from, dst, &fl6.daddr, 6139 &fl6.saddr, iif, RTM_NEWROUTE, 6140 NETLINK_CB(in_skb).portid, 6141 nlh->nlmsg_seq, 0); 6142 } else { 6143 err = -ENETUNREACH; 6144 } 6145 rcu_read_unlock(); 6146 6147 if (err < 0) { 6148 kfree_skb(skb); 6149 goto errout; 6150 } 6151 6152 err = rtnl_unicast(skb, net, NETLINK_CB(in_skb).portid); 6153 errout: 6154 return err; 6155 } 6156 6157 void inet6_rt_notify(int event, struct fib6_info *rt, struct nl_info *info, 6158 unsigned int nlm_flags) 6159 { 6160 struct sk_buff *skb; 6161 struct net *net = info->nl_net; 6162 u32 seq; 6163 int err; 6164 6165 err = -ENOBUFS; 6166 seq = info->nlh ? info->nlh->nlmsg_seq : 0; 6167 6168 skb = nlmsg_new(rt6_nlmsg_size(rt), gfp_any()); 6169 if (!skb) 6170 goto errout; 6171 6172 err = rt6_fill_node(net, skb, rt, NULL, NULL, NULL, 0, 6173 event, info->portid, seq, nlm_flags); 6174 if (err < 0) { 6175 /* -EMSGSIZE implies BUG in rt6_nlmsg_size() */ 6176 WARN_ON(err == -EMSGSIZE); 6177 kfree_skb(skb); 6178 goto errout; 6179 } 6180 rtnl_notify(skb, net, info->portid, RTNLGRP_IPV6_ROUTE, 6181 info->nlh, gfp_any()); 6182 return; 6183 errout: 6184 if (err < 0) 6185 rtnl_set_sk_err(net, RTNLGRP_IPV6_ROUTE, err); 6186 } 6187 6188 void fib6_rt_update(struct net *net, struct fib6_info *rt, 6189 struct nl_info *info) 6190 { 6191 u32 seq = info->nlh ? info->nlh->nlmsg_seq : 0; 6192 struct sk_buff *skb; 6193 int err = -ENOBUFS; 6194 6195 skb = nlmsg_new(rt6_nlmsg_size(rt), gfp_any()); 6196 if (!skb) 6197 goto errout; 6198 6199 err = rt6_fill_node(net, skb, rt, NULL, NULL, NULL, 0, 6200 RTM_NEWROUTE, info->portid, seq, NLM_F_REPLACE); 6201 if (err < 0) { 6202 /* -EMSGSIZE implies BUG in rt6_nlmsg_size() */ 6203 WARN_ON(err == -EMSGSIZE); 6204 kfree_skb(skb); 6205 goto errout; 6206 } 6207 rtnl_notify(skb, net, info->portid, RTNLGRP_IPV6_ROUTE, 6208 info->nlh, gfp_any()); 6209 return; 6210 errout: 6211 if (err < 0) 6212 rtnl_set_sk_err(net, RTNLGRP_IPV6_ROUTE, err); 6213 } 6214 6215 void fib6_info_hw_flags_set(struct net *net, struct fib6_info *f6i, 6216 bool offload, bool trap, bool offload_failed) 6217 { 6218 struct sk_buff *skb; 6219 int err; 6220 6221 if (READ_ONCE(f6i->offload) == offload && 6222 READ_ONCE(f6i->trap) == trap && 6223 READ_ONCE(f6i->offload_failed) == offload_failed) 6224 return; 6225 6226 WRITE_ONCE(f6i->offload, offload); 6227 WRITE_ONCE(f6i->trap, trap); 6228 6229 /* 2 means send notifications only if offload_failed was changed. */ 6230 if (net->ipv6.sysctl.fib_notify_on_flag_change == 2 && 6231 READ_ONCE(f6i->offload_failed) == offload_failed) 6232 return; 6233 6234 WRITE_ONCE(f6i->offload_failed, offload_failed); 6235 6236 if (!rcu_access_pointer(f6i->fib6_node)) 6237 /* The route was removed from the tree, do not send 6238 * notification. 6239 */ 6240 return; 6241 6242 if (!net->ipv6.sysctl.fib_notify_on_flag_change) 6243 return; 6244 6245 skb = nlmsg_new(rt6_nlmsg_size(f6i), GFP_KERNEL); 6246 if (!skb) { 6247 err = -ENOBUFS; 6248 goto errout; 6249 } 6250 6251 err = rt6_fill_node(net, skb, f6i, NULL, NULL, NULL, 0, RTM_NEWROUTE, 0, 6252 0, 0); 6253 if (err < 0) { 6254 /* -EMSGSIZE implies BUG in rt6_nlmsg_size() */ 6255 WARN_ON(err == -EMSGSIZE); 6256 kfree_skb(skb); 6257 goto errout; 6258 } 6259 6260 rtnl_notify(skb, net, 0, RTNLGRP_IPV6_ROUTE, NULL, GFP_KERNEL); 6261 return; 6262 6263 errout: 6264 rtnl_set_sk_err(net, RTNLGRP_IPV6_ROUTE, err); 6265 } 6266 EXPORT_SYMBOL(fib6_info_hw_flags_set); 6267 6268 static int ip6_route_dev_notify(struct notifier_block *this, 6269 unsigned long event, void *ptr) 6270 { 6271 struct net_device *dev = netdev_notifier_info_to_dev(ptr); 6272 struct net *net = dev_net(dev); 6273 6274 if (!(dev->flags & IFF_LOOPBACK)) 6275 return NOTIFY_OK; 6276 6277 if (event == NETDEV_REGISTER) { 6278 net->ipv6.fib6_null_entry->fib6_nh->fib_nh_dev = dev; 6279 net->ipv6.ip6_null_entry->dst.dev = dev; 6280 net->ipv6.ip6_null_entry->rt6i_idev = in6_dev_get(dev); 6281 #ifdef CONFIG_IPV6_MULTIPLE_TABLES 6282 net->ipv6.ip6_prohibit_entry->dst.dev = dev; 6283 net->ipv6.ip6_prohibit_entry->rt6i_idev = in6_dev_get(dev); 6284 net->ipv6.ip6_blk_hole_entry->dst.dev = dev; 6285 net->ipv6.ip6_blk_hole_entry->rt6i_idev = in6_dev_get(dev); 6286 #endif 6287 } else if (event == NETDEV_UNREGISTER && 6288 dev->reg_state != NETREG_UNREGISTERED) { 6289 /* NETDEV_UNREGISTER could be fired for multiple times by 6290 * netdev_wait_allrefs(). Make sure we only call this once. 6291 */ 6292 in6_dev_put_clear(&net->ipv6.ip6_null_entry->rt6i_idev); 6293 #ifdef CONFIG_IPV6_MULTIPLE_TABLES 6294 in6_dev_put_clear(&net->ipv6.ip6_prohibit_entry->rt6i_idev); 6295 in6_dev_put_clear(&net->ipv6.ip6_blk_hole_entry->rt6i_idev); 6296 #endif 6297 } 6298 6299 return NOTIFY_OK; 6300 } 6301 6302 /* 6303 * /proc 6304 */ 6305 6306 #ifdef CONFIG_PROC_FS 6307 static int rt6_stats_seq_show(struct seq_file *seq, void *v) 6308 { 6309 struct net *net = (struct net *)seq->private; 6310 seq_printf(seq, "%04x %04x %04x %04x %04x %04x %04x\n", 6311 net->ipv6.rt6_stats->fib_nodes, 6312 net->ipv6.rt6_stats->fib_route_nodes, 6313 atomic_read(&net->ipv6.rt6_stats->fib_rt_alloc), 6314 net->ipv6.rt6_stats->fib_rt_entries, 6315 net->ipv6.rt6_stats->fib_rt_cache, 6316 dst_entries_get_slow(&net->ipv6.ip6_dst_ops), 6317 net->ipv6.rt6_stats->fib_discarded_routes); 6318 6319 return 0; 6320 } 6321 #endif /* CONFIG_PROC_FS */ 6322 6323 #ifdef CONFIG_SYSCTL 6324 6325 static int ipv6_sysctl_rtcache_flush(struct ctl_table *ctl, int write, 6326 void *buffer, size_t *lenp, loff_t *ppos) 6327 { 6328 struct net *net; 6329 int delay; 6330 int ret; 6331 if (!write) 6332 return -EINVAL; 6333 6334 net = (struct net *)ctl->extra1; 6335 delay = net->ipv6.sysctl.flush_delay; 6336 ret = proc_dointvec(ctl, write, buffer, lenp, ppos); 6337 if (ret) 6338 return ret; 6339 6340 fib6_run_gc(delay <= 0 ? 0 : (unsigned long)delay, net, delay > 0); 6341 return 0; 6342 } 6343 6344 static struct ctl_table ipv6_route_table_template[] = { 6345 { 6346 .procname = "max_size", 6347 .data = &init_net.ipv6.sysctl.ip6_rt_max_size, 6348 .maxlen = sizeof(int), 6349 .mode = 0644, 6350 .proc_handler = proc_dointvec, 6351 }, 6352 { 6353 .procname = "gc_thresh", 6354 .data = &ip6_dst_ops_template.gc_thresh, 6355 .maxlen = sizeof(int), 6356 .mode = 0644, 6357 .proc_handler = proc_dointvec, 6358 }, 6359 { 6360 .procname = "flush", 6361 .data = &init_net.ipv6.sysctl.flush_delay, 6362 .maxlen = sizeof(int), 6363 .mode = 0200, 6364 .proc_handler = ipv6_sysctl_rtcache_flush 6365 }, 6366 { 6367 .procname = "gc_min_interval", 6368 .data = &init_net.ipv6.sysctl.ip6_rt_gc_min_interval, 6369 .maxlen = sizeof(int), 6370 .mode = 0644, 6371 .proc_handler = proc_dointvec_jiffies, 6372 }, 6373 { 6374 .procname = "gc_timeout", 6375 .data = &init_net.ipv6.sysctl.ip6_rt_gc_timeout, 6376 .maxlen = sizeof(int), 6377 .mode = 0644, 6378 .proc_handler = proc_dointvec_jiffies, 6379 }, 6380 { 6381 .procname = "gc_interval", 6382 .data = &init_net.ipv6.sysctl.ip6_rt_gc_interval, 6383 .maxlen = sizeof(int), 6384 .mode = 0644, 6385 .proc_handler = proc_dointvec_jiffies, 6386 }, 6387 { 6388 .procname = "gc_elasticity", 6389 .data = &init_net.ipv6.sysctl.ip6_rt_gc_elasticity, 6390 .maxlen = sizeof(int), 6391 .mode = 0644, 6392 .proc_handler = proc_dointvec, 6393 }, 6394 { 6395 .procname = "mtu_expires", 6396 .data = &init_net.ipv6.sysctl.ip6_rt_mtu_expires, 6397 .maxlen = sizeof(int), 6398 .mode = 0644, 6399 .proc_handler = proc_dointvec_jiffies, 6400 }, 6401 { 6402 .procname = "min_adv_mss", 6403 .data = &init_net.ipv6.sysctl.ip6_rt_min_advmss, 6404 .maxlen = sizeof(int), 6405 .mode = 0644, 6406 .proc_handler = proc_dointvec, 6407 }, 6408 { 6409 .procname = "gc_min_interval_ms", 6410 .data = &init_net.ipv6.sysctl.ip6_rt_gc_min_interval, 6411 .maxlen = sizeof(int), 6412 .mode = 0644, 6413 .proc_handler = proc_dointvec_ms_jiffies, 6414 }, 6415 { 6416 .procname = "skip_notify_on_dev_down", 6417 .data = &init_net.ipv6.sysctl.skip_notify_on_dev_down, 6418 .maxlen = sizeof(u8), 6419 .mode = 0644, 6420 .proc_handler = proc_dou8vec_minmax, 6421 .extra1 = SYSCTL_ZERO, 6422 .extra2 = SYSCTL_ONE, 6423 }, 6424 { } 6425 }; 6426 6427 struct ctl_table * __net_init ipv6_route_sysctl_init(struct net *net) 6428 { 6429 struct ctl_table *table; 6430 6431 table = kmemdup(ipv6_route_table_template, 6432 sizeof(ipv6_route_table_template), 6433 GFP_KERNEL); 6434 6435 if (table) { 6436 table[0].data = &net->ipv6.sysctl.ip6_rt_max_size; 6437 table[1].data = &net->ipv6.ip6_dst_ops.gc_thresh; 6438 table[2].data = &net->ipv6.sysctl.flush_delay; 6439 table[2].extra1 = net; 6440 table[3].data = &net->ipv6.sysctl.ip6_rt_gc_min_interval; 6441 table[4].data = &net->ipv6.sysctl.ip6_rt_gc_timeout; 6442 table[5].data = &net->ipv6.sysctl.ip6_rt_gc_interval; 6443 table[6].data = &net->ipv6.sysctl.ip6_rt_gc_elasticity; 6444 table[7].data = &net->ipv6.sysctl.ip6_rt_mtu_expires; 6445 table[8].data = &net->ipv6.sysctl.ip6_rt_min_advmss; 6446 table[9].data = &net->ipv6.sysctl.ip6_rt_gc_min_interval; 6447 table[10].data = &net->ipv6.sysctl.skip_notify_on_dev_down; 6448 6449 /* Don't export sysctls to unprivileged users */ 6450 if (net->user_ns != &init_user_ns) 6451 table[1].procname = NULL; 6452 } 6453 6454 return table; 6455 } 6456 6457 size_t ipv6_route_sysctl_table_size(struct net *net) 6458 { 6459 /* Don't export sysctls to unprivileged users */ 6460 if (net->user_ns != &init_user_ns) 6461 return 1; 6462 6463 return ARRAY_SIZE(ipv6_route_table_template); 6464 } 6465 #endif 6466 6467 static int __net_init ip6_route_net_init(struct net *net) 6468 { 6469 int ret = -ENOMEM; 6470 6471 memcpy(&net->ipv6.ip6_dst_ops, &ip6_dst_ops_template, 6472 sizeof(net->ipv6.ip6_dst_ops)); 6473 6474 if (dst_entries_init(&net->ipv6.ip6_dst_ops) < 0) 6475 goto out_ip6_dst_ops; 6476 6477 net->ipv6.fib6_null_entry = fib6_info_alloc(GFP_KERNEL, true); 6478 if (!net->ipv6.fib6_null_entry) 6479 goto out_ip6_dst_entries; 6480 memcpy(net->ipv6.fib6_null_entry, &fib6_null_entry_template, 6481 sizeof(*net->ipv6.fib6_null_entry)); 6482 6483 net->ipv6.ip6_null_entry = kmemdup(&ip6_null_entry_template, 6484 sizeof(*net->ipv6.ip6_null_entry), 6485 GFP_KERNEL); 6486 if (!net->ipv6.ip6_null_entry) 6487 goto out_fib6_null_entry; 6488 net->ipv6.ip6_null_entry->dst.ops = &net->ipv6.ip6_dst_ops; 6489 dst_init_metrics(&net->ipv6.ip6_null_entry->dst, 6490 ip6_template_metrics, true); 6491 INIT_LIST_HEAD(&net->ipv6.ip6_null_entry->dst.rt_uncached); 6492 6493 #ifdef CONFIG_IPV6_MULTIPLE_TABLES 6494 net->ipv6.fib6_has_custom_rules = false; 6495 net->ipv6.ip6_prohibit_entry = kmemdup(&ip6_prohibit_entry_template, 6496 sizeof(*net->ipv6.ip6_prohibit_entry), 6497 GFP_KERNEL); 6498 if (!net->ipv6.ip6_prohibit_entry) 6499 goto out_ip6_null_entry; 6500 net->ipv6.ip6_prohibit_entry->dst.ops = &net->ipv6.ip6_dst_ops; 6501 dst_init_metrics(&net->ipv6.ip6_prohibit_entry->dst, 6502 ip6_template_metrics, true); 6503 INIT_LIST_HEAD(&net->ipv6.ip6_prohibit_entry->dst.rt_uncached); 6504 6505 net->ipv6.ip6_blk_hole_entry = kmemdup(&ip6_blk_hole_entry_template, 6506 sizeof(*net->ipv6.ip6_blk_hole_entry), 6507 GFP_KERNEL); 6508 if (!net->ipv6.ip6_blk_hole_entry) 6509 goto out_ip6_prohibit_entry; 6510 net->ipv6.ip6_blk_hole_entry->dst.ops = &net->ipv6.ip6_dst_ops; 6511 dst_init_metrics(&net->ipv6.ip6_blk_hole_entry->dst, 6512 ip6_template_metrics, true); 6513 INIT_LIST_HEAD(&net->ipv6.ip6_blk_hole_entry->dst.rt_uncached); 6514 #ifdef CONFIG_IPV6_SUBTREES 6515 net->ipv6.fib6_routes_require_src = 0; 6516 #endif 6517 #endif 6518 6519 net->ipv6.sysctl.flush_delay = 0; 6520 net->ipv6.sysctl.ip6_rt_max_size = INT_MAX; 6521 net->ipv6.sysctl.ip6_rt_gc_min_interval = HZ / 2; 6522 net->ipv6.sysctl.ip6_rt_gc_timeout = 60*HZ; 6523 net->ipv6.sysctl.ip6_rt_gc_interval = 30*HZ; 6524 net->ipv6.sysctl.ip6_rt_gc_elasticity = 9; 6525 net->ipv6.sysctl.ip6_rt_mtu_expires = 10*60*HZ; 6526 net->ipv6.sysctl.ip6_rt_min_advmss = IPV6_MIN_MTU - 20 - 40; 6527 net->ipv6.sysctl.skip_notify_on_dev_down = 0; 6528 6529 atomic_set(&net->ipv6.ip6_rt_gc_expire, 30*HZ); 6530 6531 ret = 0; 6532 out: 6533 return ret; 6534 6535 #ifdef CONFIG_IPV6_MULTIPLE_TABLES 6536 out_ip6_prohibit_entry: 6537 kfree(net->ipv6.ip6_prohibit_entry); 6538 out_ip6_null_entry: 6539 kfree(net->ipv6.ip6_null_entry); 6540 #endif 6541 out_fib6_null_entry: 6542 kfree(net->ipv6.fib6_null_entry); 6543 out_ip6_dst_entries: 6544 dst_entries_destroy(&net->ipv6.ip6_dst_ops); 6545 out_ip6_dst_ops: 6546 goto out; 6547 } 6548 6549 static void __net_exit ip6_route_net_exit(struct net *net) 6550 { 6551 kfree(net->ipv6.fib6_null_entry); 6552 kfree(net->ipv6.ip6_null_entry); 6553 #ifdef CONFIG_IPV6_MULTIPLE_TABLES 6554 kfree(net->ipv6.ip6_prohibit_entry); 6555 kfree(net->ipv6.ip6_blk_hole_entry); 6556 #endif 6557 dst_entries_destroy(&net->ipv6.ip6_dst_ops); 6558 } 6559 6560 static int __net_init ip6_route_net_init_late(struct net *net) 6561 { 6562 #ifdef CONFIG_PROC_FS 6563 if (!proc_create_net("ipv6_route", 0, net->proc_net, 6564 &ipv6_route_seq_ops, 6565 sizeof(struct ipv6_route_iter))) 6566 return -ENOMEM; 6567 6568 if (!proc_create_net_single("rt6_stats", 0444, net->proc_net, 6569 rt6_stats_seq_show, NULL)) { 6570 remove_proc_entry("ipv6_route", net->proc_net); 6571 return -ENOMEM; 6572 } 6573 #endif 6574 return 0; 6575 } 6576 6577 static void __net_exit ip6_route_net_exit_late(struct net *net) 6578 { 6579 #ifdef CONFIG_PROC_FS 6580 remove_proc_entry("ipv6_route", net->proc_net); 6581 remove_proc_entry("rt6_stats", net->proc_net); 6582 #endif 6583 } 6584 6585 static struct pernet_operations ip6_route_net_ops = { 6586 .init = ip6_route_net_init, 6587 .exit = ip6_route_net_exit, 6588 }; 6589 6590 static int __net_init ipv6_inetpeer_init(struct net *net) 6591 { 6592 struct inet_peer_base *bp = kmalloc(sizeof(*bp), GFP_KERNEL); 6593 6594 if (!bp) 6595 return -ENOMEM; 6596 inet_peer_base_init(bp); 6597 net->ipv6.peers = bp; 6598 return 0; 6599 } 6600 6601 static void __net_exit ipv6_inetpeer_exit(struct net *net) 6602 { 6603 struct inet_peer_base *bp = net->ipv6.peers; 6604 6605 net->ipv6.peers = NULL; 6606 inetpeer_invalidate_tree(bp); 6607 kfree(bp); 6608 } 6609 6610 static struct pernet_operations ipv6_inetpeer_ops = { 6611 .init = ipv6_inetpeer_init, 6612 .exit = ipv6_inetpeer_exit, 6613 }; 6614 6615 static struct pernet_operations ip6_route_net_late_ops = { 6616 .init = ip6_route_net_init_late, 6617 .exit = ip6_route_net_exit_late, 6618 }; 6619 6620 static struct notifier_block ip6_route_dev_notifier = { 6621 .notifier_call = ip6_route_dev_notify, 6622 .priority = ADDRCONF_NOTIFY_PRIORITY - 10, 6623 }; 6624 6625 void __init ip6_route_init_special_entries(void) 6626 { 6627 /* Registering of the loopback is done before this portion of code, 6628 * the loopback reference in rt6_info will not be taken, do it 6629 * manually for init_net */ 6630 init_net.ipv6.fib6_null_entry->fib6_nh->fib_nh_dev = init_net.loopback_dev; 6631 init_net.ipv6.ip6_null_entry->dst.dev = init_net.loopback_dev; 6632 init_net.ipv6.ip6_null_entry->rt6i_idev = in6_dev_get(init_net.loopback_dev); 6633 #ifdef CONFIG_IPV6_MULTIPLE_TABLES 6634 init_net.ipv6.ip6_prohibit_entry->dst.dev = init_net.loopback_dev; 6635 init_net.ipv6.ip6_prohibit_entry->rt6i_idev = in6_dev_get(init_net.loopback_dev); 6636 init_net.ipv6.ip6_blk_hole_entry->dst.dev = init_net.loopback_dev; 6637 init_net.ipv6.ip6_blk_hole_entry->rt6i_idev = in6_dev_get(init_net.loopback_dev); 6638 #endif 6639 } 6640 6641 #if IS_BUILTIN(CONFIG_IPV6) 6642 #if defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_PROC_FS) 6643 DEFINE_BPF_ITER_FUNC(ipv6_route, struct bpf_iter_meta *meta, struct fib6_info *rt) 6644 6645 BTF_ID_LIST(btf_fib6_info_id) 6646 BTF_ID(struct, fib6_info) 6647 6648 static const struct bpf_iter_seq_info ipv6_route_seq_info = { 6649 .seq_ops = &ipv6_route_seq_ops, 6650 .init_seq_private = bpf_iter_init_seq_net, 6651 .fini_seq_private = bpf_iter_fini_seq_net, 6652 .seq_priv_size = sizeof(struct ipv6_route_iter), 6653 }; 6654 6655 static struct bpf_iter_reg ipv6_route_reg_info = { 6656 .target = "ipv6_route", 6657 .ctx_arg_info_size = 1, 6658 .ctx_arg_info = { 6659 { offsetof(struct bpf_iter__ipv6_route, rt), 6660 PTR_TO_BTF_ID_OR_NULL }, 6661 }, 6662 .seq_info = &ipv6_route_seq_info, 6663 }; 6664 6665 static int __init bpf_iter_register(void) 6666 { 6667 ipv6_route_reg_info.ctx_arg_info[0].btf_id = *btf_fib6_info_id; 6668 return bpf_iter_reg_target(&ipv6_route_reg_info); 6669 } 6670 6671 static void bpf_iter_unregister(void) 6672 { 6673 bpf_iter_unreg_target(&ipv6_route_reg_info); 6674 } 6675 #endif 6676 #endif 6677 6678 int __init ip6_route_init(void) 6679 { 6680 int ret; 6681 int cpu; 6682 6683 ret = -ENOMEM; 6684 ip6_dst_ops_template.kmem_cachep = 6685 kmem_cache_create("ip6_dst_cache", sizeof(struct rt6_info), 0, 6686 SLAB_HWCACHE_ALIGN | SLAB_ACCOUNT, NULL); 6687 if (!ip6_dst_ops_template.kmem_cachep) 6688 goto out; 6689 6690 ret = dst_entries_init(&ip6_dst_blackhole_ops); 6691 if (ret) 6692 goto out_kmem_cache; 6693 6694 ret = register_pernet_subsys(&ipv6_inetpeer_ops); 6695 if (ret) 6696 goto out_dst_entries; 6697 6698 ret = register_pernet_subsys(&ip6_route_net_ops); 6699 if (ret) 6700 goto out_register_inetpeer; 6701 6702 ip6_dst_blackhole_ops.kmem_cachep = ip6_dst_ops_template.kmem_cachep; 6703 6704 ret = fib6_init(); 6705 if (ret) 6706 goto out_register_subsys; 6707 6708 ret = xfrm6_init(); 6709 if (ret) 6710 goto out_fib6_init; 6711 6712 ret = fib6_rules_init(); 6713 if (ret) 6714 goto xfrm6_init; 6715 6716 ret = register_pernet_subsys(&ip6_route_net_late_ops); 6717 if (ret) 6718 goto fib6_rules_init; 6719 6720 ret = rtnl_register_module(THIS_MODULE, PF_INET6, RTM_NEWROUTE, 6721 inet6_rtm_newroute, NULL, 0); 6722 if (ret < 0) 6723 goto out_register_late_subsys; 6724 6725 ret = rtnl_register_module(THIS_MODULE, PF_INET6, RTM_DELROUTE, 6726 inet6_rtm_delroute, NULL, 0); 6727 if (ret < 0) 6728 goto out_register_late_subsys; 6729 6730 ret = rtnl_register_module(THIS_MODULE, PF_INET6, RTM_GETROUTE, 6731 inet6_rtm_getroute, NULL, 6732 RTNL_FLAG_DOIT_UNLOCKED); 6733 if (ret < 0) 6734 goto out_register_late_subsys; 6735 6736 ret = register_netdevice_notifier(&ip6_route_dev_notifier); 6737 if (ret) 6738 goto out_register_late_subsys; 6739 6740 #if IS_BUILTIN(CONFIG_IPV6) 6741 #if defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_PROC_FS) 6742 ret = bpf_iter_register(); 6743 if (ret) 6744 goto out_register_late_subsys; 6745 #endif 6746 #endif 6747 6748 for_each_possible_cpu(cpu) { 6749 struct uncached_list *ul = per_cpu_ptr(&rt6_uncached_list, cpu); 6750 6751 INIT_LIST_HEAD(&ul->head); 6752 INIT_LIST_HEAD(&ul->quarantine); 6753 spin_lock_init(&ul->lock); 6754 } 6755 6756 out: 6757 return ret; 6758 6759 out_register_late_subsys: 6760 rtnl_unregister_all(PF_INET6); 6761 unregister_pernet_subsys(&ip6_route_net_late_ops); 6762 fib6_rules_init: 6763 fib6_rules_cleanup(); 6764 xfrm6_init: 6765 xfrm6_fini(); 6766 out_fib6_init: 6767 fib6_gc_cleanup(); 6768 out_register_subsys: 6769 unregister_pernet_subsys(&ip6_route_net_ops); 6770 out_register_inetpeer: 6771 unregister_pernet_subsys(&ipv6_inetpeer_ops); 6772 out_dst_entries: 6773 dst_entries_destroy(&ip6_dst_blackhole_ops); 6774 out_kmem_cache: 6775 kmem_cache_destroy(ip6_dst_ops_template.kmem_cachep); 6776 goto out; 6777 } 6778 6779 void ip6_route_cleanup(void) 6780 { 6781 #if IS_BUILTIN(CONFIG_IPV6) 6782 #if defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_PROC_FS) 6783 bpf_iter_unregister(); 6784 #endif 6785 #endif 6786 unregister_netdevice_notifier(&ip6_route_dev_notifier); 6787 unregister_pernet_subsys(&ip6_route_net_late_ops); 6788 fib6_rules_cleanup(); 6789 xfrm6_fini(); 6790 fib6_gc_cleanup(); 6791 unregister_pernet_subsys(&ipv6_inetpeer_ops); 6792 unregister_pernet_subsys(&ip6_route_net_ops); 6793 dst_entries_destroy(&ip6_dst_blackhole_ops); 6794 kmem_cache_destroy(ip6_dst_ops_template.kmem_cachep); 6795 } 6796