1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * net/dst.h Protocol independent destination cache definitions. 4 * 5 * Authors: Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru> 6 * 7 */ 8 9 #ifndef _NET_DST_H 10 #define _NET_DST_H 11 12 #include <net/dst_ops.h> 13 #include <linux/netdevice.h> 14 #include <linux/rtnetlink.h> 15 #include <linux/rcupdate.h> 16 #include <linux/bug.h> 17 #include <linux/jiffies.h> 18 #include <linux/refcount.h> 19 #include <net/neighbour.h> 20 #include <asm/processor.h> 21 22 #define DST_GC_MIN (HZ/10) 23 #define DST_GC_INC (HZ/2) 24 #define DST_GC_MAX (120*HZ) 25 26 /* Each dst_entry has reference count and sits in some parent list(s). 27 * When it is removed from parent list, it is "freed" (dst_free). 28 * After this it enters dead state (dst->obsolete > 0) and if its refcnt 29 * is zero, it can be destroyed immediately, otherwise it is added 30 * to gc list and garbage collector periodically checks the refcnt. 31 */ 32 33 struct sk_buff; 34 35 struct dst_entry { 36 struct net_device *dev; 37 struct dst_ops *ops; 38 unsigned long _metrics; 39 unsigned long expires; 40 #ifdef CONFIG_XFRM 41 struct xfrm_state *xfrm; 42 #else 43 void *__pad1; 44 #endif 45 int (*input)(struct sk_buff *); 46 int (*output)(struct net *net, struct sock *sk, struct sk_buff *skb); 47 48 unsigned short flags; 49 #define DST_HOST 0x0001 50 #define DST_NOXFRM 0x0002 51 #define DST_NOPOLICY 0x0004 52 #define DST_NOCOUNT 0x0008 53 #define DST_FAKE_RTABLE 0x0010 54 #define DST_XFRM_TUNNEL 0x0020 55 #define DST_XFRM_QUEUE 0x0040 56 #define DST_METADATA 0x0080 57 58 /* A non-zero value of dst->obsolete forces by-hand validation 59 * of the route entry. Positive values are set by the generic 60 * dst layer to indicate that the entry has been forcefully 61 * destroyed. 62 * 63 * Negative values are used by the implementation layer code to 64 * force invocation of the dst_ops->check() method. 65 */ 66 short obsolete; 67 #define DST_OBSOLETE_NONE 0 68 #define DST_OBSOLETE_DEAD 2 69 #define DST_OBSOLETE_FORCE_CHK -1 70 #define DST_OBSOLETE_KILL -2 71 unsigned short header_len; /* more space at head required */ 72 unsigned short trailer_len; /* space to reserve at tail */ 73 74 /* 75 * __refcnt wants to be on a different cache line from 76 * input/output/ops or performance tanks badly 77 */ 78 #ifdef CONFIG_64BIT 79 atomic_t __refcnt; /* 64-bit offset 64 */ 80 #endif 81 int __use; 82 unsigned long lastuse; 83 struct lwtunnel_state *lwtstate; 84 struct rcu_head rcu_head; 85 short error; 86 short __pad; 87 __u32 tclassid; 88 #ifndef CONFIG_64BIT 89 atomic_t __refcnt; /* 32-bit offset 64 */ 90 #endif 91 }; 92 93 struct dst_metrics { 94 u32 metrics[RTAX_MAX]; 95 refcount_t refcnt; 96 }; 97 extern const struct dst_metrics dst_default_metrics; 98 99 u32 *dst_cow_metrics_generic(struct dst_entry *dst, unsigned long old); 100 101 #define DST_METRICS_READ_ONLY 0x1UL 102 #define DST_METRICS_REFCOUNTED 0x2UL 103 #define DST_METRICS_FLAGS 0x3UL 104 #define __DST_METRICS_PTR(Y) \ 105 ((u32 *)((Y) & ~DST_METRICS_FLAGS)) 106 #define DST_METRICS_PTR(X) __DST_METRICS_PTR((X)->_metrics) 107 108 static inline bool dst_metrics_read_only(const struct dst_entry *dst) 109 { 110 return dst->_metrics & DST_METRICS_READ_ONLY; 111 } 112 113 void __dst_destroy_metrics_generic(struct dst_entry *dst, unsigned long old); 114 115 static inline void dst_destroy_metrics_generic(struct dst_entry *dst) 116 { 117 unsigned long val = dst->_metrics; 118 if (!(val & DST_METRICS_READ_ONLY)) 119 __dst_destroy_metrics_generic(dst, val); 120 } 121 122 static inline u32 *dst_metrics_write_ptr(struct dst_entry *dst) 123 { 124 unsigned long p = dst->_metrics; 125 126 BUG_ON(!p); 127 128 if (p & DST_METRICS_READ_ONLY) 129 return dst->ops->cow_metrics(dst, p); 130 return __DST_METRICS_PTR(p); 131 } 132 133 /* This may only be invoked before the entry has reached global 134 * visibility. 135 */ 136 static inline void dst_init_metrics(struct dst_entry *dst, 137 const u32 *src_metrics, 138 bool read_only) 139 { 140 dst->_metrics = ((unsigned long) src_metrics) | 141 (read_only ? DST_METRICS_READ_ONLY : 0); 142 } 143 144 static inline void dst_copy_metrics(struct dst_entry *dest, const struct dst_entry *src) 145 { 146 u32 *dst_metrics = dst_metrics_write_ptr(dest); 147 148 if (dst_metrics) { 149 u32 *src_metrics = DST_METRICS_PTR(src); 150 151 memcpy(dst_metrics, src_metrics, RTAX_MAX * sizeof(u32)); 152 } 153 } 154 155 static inline u32 *dst_metrics_ptr(struct dst_entry *dst) 156 { 157 return DST_METRICS_PTR(dst); 158 } 159 160 static inline u32 161 dst_metric_raw(const struct dst_entry *dst, const int metric) 162 { 163 u32 *p = DST_METRICS_PTR(dst); 164 165 return p[metric-1]; 166 } 167 168 static inline u32 169 dst_metric(const struct dst_entry *dst, const int metric) 170 { 171 WARN_ON_ONCE(metric == RTAX_HOPLIMIT || 172 metric == RTAX_ADVMSS || 173 metric == RTAX_MTU); 174 return dst_metric_raw(dst, metric); 175 } 176 177 static inline u32 178 dst_metric_advmss(const struct dst_entry *dst) 179 { 180 u32 advmss = dst_metric_raw(dst, RTAX_ADVMSS); 181 182 if (!advmss) 183 advmss = dst->ops->default_advmss(dst); 184 185 return advmss; 186 } 187 188 static inline void dst_metric_set(struct dst_entry *dst, int metric, u32 val) 189 { 190 u32 *p = dst_metrics_write_ptr(dst); 191 192 if (p) 193 p[metric-1] = val; 194 } 195 196 /* Kernel-internal feature bits that are unallocated in user space. */ 197 #define DST_FEATURE_ECN_CA (1 << 31) 198 199 #define DST_FEATURE_MASK (DST_FEATURE_ECN_CA) 200 #define DST_FEATURE_ECN_MASK (DST_FEATURE_ECN_CA | RTAX_FEATURE_ECN) 201 202 static inline u32 203 dst_feature(const struct dst_entry *dst, u32 feature) 204 { 205 return dst_metric(dst, RTAX_FEATURES) & feature; 206 } 207 208 static inline u32 dst_mtu(const struct dst_entry *dst) 209 { 210 return dst->ops->mtu(dst); 211 } 212 213 /* RTT metrics are stored in milliseconds for user ABI, but used as jiffies */ 214 static inline unsigned long dst_metric_rtt(const struct dst_entry *dst, int metric) 215 { 216 return msecs_to_jiffies(dst_metric(dst, metric)); 217 } 218 219 static inline u32 220 dst_allfrag(const struct dst_entry *dst) 221 { 222 int ret = dst_feature(dst, RTAX_FEATURE_ALLFRAG); 223 return ret; 224 } 225 226 static inline int 227 dst_metric_locked(const struct dst_entry *dst, int metric) 228 { 229 return dst_metric(dst, RTAX_LOCK) & (1<<metric); 230 } 231 232 static inline void dst_hold(struct dst_entry *dst) 233 { 234 /* 235 * If your kernel compilation stops here, please check 236 * the placement of __refcnt in struct dst_entry 237 */ 238 BUILD_BUG_ON(offsetof(struct dst_entry, __refcnt) & 63); 239 WARN_ON(atomic_inc_not_zero(&dst->__refcnt) == 0); 240 } 241 242 static inline void dst_use_noref(struct dst_entry *dst, unsigned long time) 243 { 244 if (unlikely(time != dst->lastuse)) { 245 dst->__use++; 246 dst->lastuse = time; 247 } 248 } 249 250 static inline void dst_hold_and_use(struct dst_entry *dst, unsigned long time) 251 { 252 dst_hold(dst); 253 dst_use_noref(dst, time); 254 } 255 256 static inline struct dst_entry *dst_clone(struct dst_entry *dst) 257 { 258 if (dst) 259 dst_hold(dst); 260 return dst; 261 } 262 263 void dst_release(struct dst_entry *dst); 264 265 void dst_release_immediate(struct dst_entry *dst); 266 267 static inline void refdst_drop(unsigned long refdst) 268 { 269 if (!(refdst & SKB_DST_NOREF)) 270 dst_release((struct dst_entry *)(refdst & SKB_DST_PTRMASK)); 271 } 272 273 /** 274 * skb_dst_drop - drops skb dst 275 * @skb: buffer 276 * 277 * Drops dst reference count if a reference was taken. 278 */ 279 static inline void skb_dst_drop(struct sk_buff *skb) 280 { 281 if (skb->_skb_refdst) { 282 refdst_drop(skb->_skb_refdst); 283 skb->_skb_refdst = 0UL; 284 } 285 } 286 287 static inline void __skb_dst_copy(struct sk_buff *nskb, unsigned long refdst) 288 { 289 nskb->_skb_refdst = refdst; 290 if (!(nskb->_skb_refdst & SKB_DST_NOREF)) 291 dst_clone(skb_dst(nskb)); 292 } 293 294 static inline void skb_dst_copy(struct sk_buff *nskb, const struct sk_buff *oskb) 295 { 296 __skb_dst_copy(nskb, oskb->_skb_refdst); 297 } 298 299 /** 300 * dst_hold_safe - Take a reference on a dst if possible 301 * @dst: pointer to dst entry 302 * 303 * This helper returns false if it could not safely 304 * take a reference on a dst. 305 */ 306 static inline bool dst_hold_safe(struct dst_entry *dst) 307 { 308 return atomic_inc_not_zero(&dst->__refcnt); 309 } 310 311 /** 312 * skb_dst_force - makes sure skb dst is refcounted 313 * @skb: buffer 314 * 315 * If dst is not yet refcounted and not destroyed, grab a ref on it. 316 */ 317 static inline void skb_dst_force(struct sk_buff *skb) 318 { 319 if (skb_dst_is_noref(skb)) { 320 struct dst_entry *dst = skb_dst(skb); 321 322 WARN_ON(!rcu_read_lock_held()); 323 if (!dst_hold_safe(dst)) 324 dst = NULL; 325 326 skb->_skb_refdst = (unsigned long)dst; 327 } 328 } 329 330 331 /** 332 * __skb_tunnel_rx - prepare skb for rx reinsert 333 * @skb: buffer 334 * @dev: tunnel device 335 * @net: netns for packet i/o 336 * 337 * After decapsulation, packet is going to re-enter (netif_rx()) our stack, 338 * so make some cleanups. (no accounting done) 339 */ 340 static inline void __skb_tunnel_rx(struct sk_buff *skb, struct net_device *dev, 341 struct net *net) 342 { 343 skb->dev = dev; 344 345 /* 346 * Clear hash so that we can recalulate the hash for the 347 * encapsulated packet, unless we have already determine the hash 348 * over the L4 4-tuple. 349 */ 350 skb_clear_hash_if_not_l4(skb); 351 skb_set_queue_mapping(skb, 0); 352 skb_scrub_packet(skb, !net_eq(net, dev_net(dev))); 353 } 354 355 /** 356 * skb_tunnel_rx - prepare skb for rx reinsert 357 * @skb: buffer 358 * @dev: tunnel device 359 * @net: netns for packet i/o 360 * 361 * After decapsulation, packet is going to re-enter (netif_rx()) our stack, 362 * so make some cleanups, and perform accounting. 363 * Note: this accounting is not SMP safe. 364 */ 365 static inline void skb_tunnel_rx(struct sk_buff *skb, struct net_device *dev, 366 struct net *net) 367 { 368 /* TODO : stats should be SMP safe */ 369 dev->stats.rx_packets++; 370 dev->stats.rx_bytes += skb->len; 371 __skb_tunnel_rx(skb, dev, net); 372 } 373 374 static inline u32 dst_tclassid(const struct sk_buff *skb) 375 { 376 #ifdef CONFIG_IP_ROUTE_CLASSID 377 const struct dst_entry *dst; 378 379 dst = skb_dst(skb); 380 if (dst) 381 return dst->tclassid; 382 #endif 383 return 0; 384 } 385 386 int dst_discard_out(struct net *net, struct sock *sk, struct sk_buff *skb); 387 static inline int dst_discard(struct sk_buff *skb) 388 { 389 return dst_discard_out(&init_net, skb->sk, skb); 390 } 391 void *dst_alloc(struct dst_ops *ops, struct net_device *dev, int initial_ref, 392 int initial_obsolete, unsigned short flags); 393 void dst_init(struct dst_entry *dst, struct dst_ops *ops, 394 struct net_device *dev, int initial_ref, int initial_obsolete, 395 unsigned short flags); 396 struct dst_entry *dst_destroy(struct dst_entry *dst); 397 void dst_dev_put(struct dst_entry *dst); 398 399 static inline void dst_confirm(struct dst_entry *dst) 400 { 401 } 402 403 static inline struct neighbour *dst_neigh_lookup(const struct dst_entry *dst, const void *daddr) 404 { 405 struct neighbour *n = dst->ops->neigh_lookup(dst, NULL, daddr); 406 return IS_ERR(n) ? NULL : n; 407 } 408 409 static inline struct neighbour *dst_neigh_lookup_skb(const struct dst_entry *dst, 410 struct sk_buff *skb) 411 { 412 struct neighbour *n = dst->ops->neigh_lookup(dst, skb, NULL); 413 return IS_ERR(n) ? NULL : n; 414 } 415 416 static inline void dst_confirm_neigh(const struct dst_entry *dst, 417 const void *daddr) 418 { 419 if (dst->ops->confirm_neigh) 420 dst->ops->confirm_neigh(dst, daddr); 421 } 422 423 static inline void dst_link_failure(struct sk_buff *skb) 424 { 425 struct dst_entry *dst = skb_dst(skb); 426 if (dst && dst->ops && dst->ops->link_failure) 427 dst->ops->link_failure(skb); 428 } 429 430 static inline void dst_set_expires(struct dst_entry *dst, int timeout) 431 { 432 unsigned long expires = jiffies + timeout; 433 434 if (expires == 0) 435 expires = 1; 436 437 if (dst->expires == 0 || time_before(expires, dst->expires)) 438 dst->expires = expires; 439 } 440 441 /* Output packet to network from transport. */ 442 static inline int dst_output(struct net *net, struct sock *sk, struct sk_buff *skb) 443 { 444 return skb_dst(skb)->output(net, sk, skb); 445 } 446 447 /* Input packet from network to transport. */ 448 static inline int dst_input(struct sk_buff *skb) 449 { 450 return skb_dst(skb)->input(skb); 451 } 452 453 static inline struct dst_entry *dst_check(struct dst_entry *dst, u32 cookie) 454 { 455 if (dst->obsolete) 456 dst = dst->ops->check(dst, cookie); 457 return dst; 458 } 459 460 /* Flags for xfrm_lookup flags argument. */ 461 enum { 462 XFRM_LOOKUP_ICMP = 1 << 0, 463 XFRM_LOOKUP_QUEUE = 1 << 1, 464 XFRM_LOOKUP_KEEP_DST_REF = 1 << 2, 465 }; 466 467 struct flowi; 468 #ifndef CONFIG_XFRM 469 static inline struct dst_entry *xfrm_lookup(struct net *net, 470 struct dst_entry *dst_orig, 471 const struct flowi *fl, 472 const struct sock *sk, 473 int flags) 474 { 475 return dst_orig; 476 } 477 478 static inline struct dst_entry * 479 xfrm_lookup_with_ifid(struct net *net, struct dst_entry *dst_orig, 480 const struct flowi *fl, const struct sock *sk, 481 int flags, u32 if_id) 482 { 483 return dst_orig; 484 } 485 486 static inline struct dst_entry *xfrm_lookup_route(struct net *net, 487 struct dst_entry *dst_orig, 488 const struct flowi *fl, 489 const struct sock *sk, 490 int flags) 491 { 492 return dst_orig; 493 } 494 495 static inline struct xfrm_state *dst_xfrm(const struct dst_entry *dst) 496 { 497 return NULL; 498 } 499 500 #else 501 struct dst_entry *xfrm_lookup(struct net *net, struct dst_entry *dst_orig, 502 const struct flowi *fl, const struct sock *sk, 503 int flags); 504 505 struct dst_entry *xfrm_lookup_with_ifid(struct net *net, 506 struct dst_entry *dst_orig, 507 const struct flowi *fl, 508 const struct sock *sk, int flags, 509 u32 if_id); 510 511 struct dst_entry *xfrm_lookup_route(struct net *net, struct dst_entry *dst_orig, 512 const struct flowi *fl, const struct sock *sk, 513 int flags); 514 515 /* skb attached with this dst needs transformation if dst->xfrm is valid */ 516 static inline struct xfrm_state *dst_xfrm(const struct dst_entry *dst) 517 { 518 return dst->xfrm; 519 } 520 #endif 521 522 static inline void skb_dst_update_pmtu(struct sk_buff *skb, u32 mtu) 523 { 524 struct dst_entry *dst = skb_dst(skb); 525 526 if (dst && dst->ops->update_pmtu) 527 dst->ops->update_pmtu(dst, NULL, skb, mtu); 528 } 529 530 static inline void skb_tunnel_check_pmtu(struct sk_buff *skb, 531 struct dst_entry *encap_dst, 532 int headroom) 533 { 534 u32 encap_mtu = dst_mtu(encap_dst); 535 536 if (skb->len > encap_mtu - headroom) 537 skb_dst_update_pmtu(skb, encap_mtu - headroom); 538 } 539 540 #endif /* _NET_DST_H */ 541