1 /* 2 * xfrm_policy.c 3 * 4 * Changes: 5 * Mitsuru KANDA @USAGI 6 * Kazunori MIYAZAWA @USAGI 7 * Kunihiro Ishiguro <kunihiro@ipinfusion.com> 8 * IPv6 support 9 * Kazunori MIYAZAWA @USAGI 10 * YOSHIFUJI Hideaki 11 * Split up af-specific portion 12 * Derek Atkins <derek@ihtfp.com> Add the post_input processor 13 * 14 */ 15 16 #include <linux/err.h> 17 #include <linux/slab.h> 18 #include <linux/kmod.h> 19 #include <linux/list.h> 20 #include <linux/spinlock.h> 21 #include <linux/workqueue.h> 22 #include <linux/notifier.h> 23 #include <linux/netdevice.h> 24 #include <linux/netfilter.h> 25 #include <linux/module.h> 26 #include <linux/cache.h> 27 #include <linux/audit.h> 28 #include <net/dst.h> 29 #include <net/flow.h> 30 #include <net/xfrm.h> 31 #include <net/ip.h> 32 #ifdef CONFIG_XFRM_STATISTICS 33 #include <net/snmp.h> 34 #endif 35 36 #include "xfrm_hash.h" 37 38 #define XFRM_QUEUE_TMO_MIN ((unsigned)(HZ/10)) 39 #define XFRM_QUEUE_TMO_MAX ((unsigned)(60*HZ)) 40 #define XFRM_MAX_QUEUE_LEN 100 41 42 DEFINE_MUTEX(xfrm_cfg_mutex); 43 EXPORT_SYMBOL(xfrm_cfg_mutex); 44 45 static DEFINE_SPINLOCK(xfrm_policy_sk_bundle_lock); 46 static struct dst_entry *xfrm_policy_sk_bundles; 47 static DEFINE_RWLOCK(xfrm_policy_lock); 48 49 static DEFINE_SPINLOCK(xfrm_policy_afinfo_lock); 50 static struct xfrm_policy_afinfo __rcu *xfrm_policy_afinfo[NPROTO] 51 __read_mostly; 52 53 static struct kmem_cache *xfrm_dst_cache __read_mostly; 54 55 static void xfrm_init_pmtu(struct dst_entry *dst); 56 static int stale_bundle(struct dst_entry *dst); 57 static int xfrm_bundle_ok(struct xfrm_dst *xdst); 58 static void xfrm_policy_queue_process(unsigned long arg); 59 60 static struct xfrm_policy *__xfrm_policy_unlink(struct xfrm_policy *pol, 61 int dir); 62 63 static inline bool 64 __xfrm4_selector_match(const struct xfrm_selector *sel, const struct flowi *fl) 65 { 66 const struct flowi4 *fl4 = &fl->u.ip4; 67 68 return addr4_match(fl4->daddr, sel->daddr.a4, sel->prefixlen_d) && 69 addr4_match(fl4->saddr, sel->saddr.a4, sel->prefixlen_s) && 70 !((xfrm_flowi_dport(fl, &fl4->uli) ^ sel->dport) & sel->dport_mask) && 71 !((xfrm_flowi_sport(fl, &fl4->uli) ^ sel->sport) & sel->sport_mask) && 72 (fl4->flowi4_proto == sel->proto || !sel->proto) && 73 (fl4->flowi4_oif == sel->ifindex || !sel->ifindex); 74 } 75 76 static inline bool 77 __xfrm6_selector_match(const struct xfrm_selector *sel, const struct flowi *fl) 78 { 79 const struct flowi6 *fl6 = &fl->u.ip6; 80 81 return addr_match(&fl6->daddr, &sel->daddr, sel->prefixlen_d) && 82 addr_match(&fl6->saddr, &sel->saddr, sel->prefixlen_s) && 83 !((xfrm_flowi_dport(fl, &fl6->uli) ^ sel->dport) & sel->dport_mask) && 84 !((xfrm_flowi_sport(fl, &fl6->uli) ^ sel->sport) & sel->sport_mask) && 85 (fl6->flowi6_proto == sel->proto || !sel->proto) && 86 (fl6->flowi6_oif == sel->ifindex || !sel->ifindex); 87 } 88 89 bool xfrm_selector_match(const struct xfrm_selector *sel, const struct flowi *fl, 90 unsigned short family) 91 { 92 switch (family) { 93 case AF_INET: 94 return __xfrm4_selector_match(sel, fl); 95 case AF_INET6: 96 return __xfrm6_selector_match(sel, fl); 97 } 98 return false; 99 } 100 101 static struct xfrm_policy_afinfo *xfrm_policy_get_afinfo(unsigned short family) 102 { 103 struct xfrm_policy_afinfo *afinfo; 104 105 if (unlikely(family >= NPROTO)) 106 return NULL; 107 rcu_read_lock(); 108 afinfo = rcu_dereference(xfrm_policy_afinfo[family]); 109 if (unlikely(!afinfo)) 110 rcu_read_unlock(); 111 return afinfo; 112 } 113 114 static void xfrm_policy_put_afinfo(struct xfrm_policy_afinfo *afinfo) 115 { 116 rcu_read_unlock(); 117 } 118 119 static inline struct dst_entry *__xfrm_dst_lookup(struct net *net, int tos, 120 const xfrm_address_t *saddr, 121 const xfrm_address_t *daddr, 122 int family) 123 { 124 struct xfrm_policy_afinfo *afinfo; 125 struct dst_entry *dst; 126 127 afinfo = xfrm_policy_get_afinfo(family); 128 if (unlikely(afinfo == NULL)) 129 return ERR_PTR(-EAFNOSUPPORT); 130 131 dst = afinfo->dst_lookup(net, tos, saddr, daddr); 132 133 xfrm_policy_put_afinfo(afinfo); 134 135 return dst; 136 } 137 138 static inline struct dst_entry *xfrm_dst_lookup(struct xfrm_state *x, int tos, 139 xfrm_address_t *prev_saddr, 140 xfrm_address_t *prev_daddr, 141 int family) 142 { 143 struct net *net = xs_net(x); 144 xfrm_address_t *saddr = &x->props.saddr; 145 xfrm_address_t *daddr = &x->id.daddr; 146 struct dst_entry *dst; 147 148 if (x->type->flags & XFRM_TYPE_LOCAL_COADDR) { 149 saddr = x->coaddr; 150 daddr = prev_daddr; 151 } 152 if (x->type->flags & XFRM_TYPE_REMOTE_COADDR) { 153 saddr = prev_saddr; 154 daddr = x->coaddr; 155 } 156 157 dst = __xfrm_dst_lookup(net, tos, saddr, daddr, family); 158 159 if (!IS_ERR(dst)) { 160 if (prev_saddr != saddr) 161 memcpy(prev_saddr, saddr, sizeof(*prev_saddr)); 162 if (prev_daddr != daddr) 163 memcpy(prev_daddr, daddr, sizeof(*prev_daddr)); 164 } 165 166 return dst; 167 } 168 169 static inline unsigned long make_jiffies(long secs) 170 { 171 if (secs >= (MAX_SCHEDULE_TIMEOUT-1)/HZ) 172 return MAX_SCHEDULE_TIMEOUT-1; 173 else 174 return secs*HZ; 175 } 176 177 static void xfrm_policy_timer(unsigned long data) 178 { 179 struct xfrm_policy *xp = (struct xfrm_policy*)data; 180 unsigned long now = get_seconds(); 181 long next = LONG_MAX; 182 int warn = 0; 183 int dir; 184 185 read_lock(&xp->lock); 186 187 if (unlikely(xp->walk.dead)) 188 goto out; 189 190 dir = xfrm_policy_id2dir(xp->index); 191 192 if (xp->lft.hard_add_expires_seconds) { 193 long tmo = xp->lft.hard_add_expires_seconds + 194 xp->curlft.add_time - now; 195 if (tmo <= 0) 196 goto expired; 197 if (tmo < next) 198 next = tmo; 199 } 200 if (xp->lft.hard_use_expires_seconds) { 201 long tmo = xp->lft.hard_use_expires_seconds + 202 (xp->curlft.use_time ? : xp->curlft.add_time) - now; 203 if (tmo <= 0) 204 goto expired; 205 if (tmo < next) 206 next = tmo; 207 } 208 if (xp->lft.soft_add_expires_seconds) { 209 long tmo = xp->lft.soft_add_expires_seconds + 210 xp->curlft.add_time - now; 211 if (tmo <= 0) { 212 warn = 1; 213 tmo = XFRM_KM_TIMEOUT; 214 } 215 if (tmo < next) 216 next = tmo; 217 } 218 if (xp->lft.soft_use_expires_seconds) { 219 long tmo = xp->lft.soft_use_expires_seconds + 220 (xp->curlft.use_time ? : xp->curlft.add_time) - now; 221 if (tmo <= 0) { 222 warn = 1; 223 tmo = XFRM_KM_TIMEOUT; 224 } 225 if (tmo < next) 226 next = tmo; 227 } 228 229 if (warn) 230 km_policy_expired(xp, dir, 0, 0); 231 if (next != LONG_MAX && 232 !mod_timer(&xp->timer, jiffies + make_jiffies(next))) 233 xfrm_pol_hold(xp); 234 235 out: 236 read_unlock(&xp->lock); 237 xfrm_pol_put(xp); 238 return; 239 240 expired: 241 read_unlock(&xp->lock); 242 if (!xfrm_policy_delete(xp, dir)) 243 km_policy_expired(xp, dir, 1, 0); 244 xfrm_pol_put(xp); 245 } 246 247 static struct flow_cache_object *xfrm_policy_flo_get(struct flow_cache_object *flo) 248 { 249 struct xfrm_policy *pol = container_of(flo, struct xfrm_policy, flo); 250 251 if (unlikely(pol->walk.dead)) 252 flo = NULL; 253 else 254 xfrm_pol_hold(pol); 255 256 return flo; 257 } 258 259 static int xfrm_policy_flo_check(struct flow_cache_object *flo) 260 { 261 struct xfrm_policy *pol = container_of(flo, struct xfrm_policy, flo); 262 263 return !pol->walk.dead; 264 } 265 266 static void xfrm_policy_flo_delete(struct flow_cache_object *flo) 267 { 268 xfrm_pol_put(container_of(flo, struct xfrm_policy, flo)); 269 } 270 271 static const struct flow_cache_ops xfrm_policy_fc_ops = { 272 .get = xfrm_policy_flo_get, 273 .check = xfrm_policy_flo_check, 274 .delete = xfrm_policy_flo_delete, 275 }; 276 277 /* Allocate xfrm_policy. Not used here, it is supposed to be used by pfkeyv2 278 * SPD calls. 279 */ 280 281 struct xfrm_policy *xfrm_policy_alloc(struct net *net, gfp_t gfp) 282 { 283 struct xfrm_policy *policy; 284 285 policy = kzalloc(sizeof(struct xfrm_policy), gfp); 286 287 if (policy) { 288 write_pnet(&policy->xp_net, net); 289 INIT_LIST_HEAD(&policy->walk.all); 290 INIT_HLIST_NODE(&policy->bydst); 291 INIT_HLIST_NODE(&policy->byidx); 292 rwlock_init(&policy->lock); 293 atomic_set(&policy->refcnt, 1); 294 skb_queue_head_init(&policy->polq.hold_queue); 295 setup_timer(&policy->timer, xfrm_policy_timer, 296 (unsigned long)policy); 297 setup_timer(&policy->polq.hold_timer, xfrm_policy_queue_process, 298 (unsigned long)policy); 299 policy->flo.ops = &xfrm_policy_fc_ops; 300 } 301 return policy; 302 } 303 EXPORT_SYMBOL(xfrm_policy_alloc); 304 305 /* Destroy xfrm_policy: descendant resources must be released to this moment. */ 306 307 void xfrm_policy_destroy(struct xfrm_policy *policy) 308 { 309 BUG_ON(!policy->walk.dead); 310 311 if (del_timer(&policy->timer)) 312 BUG(); 313 314 security_xfrm_policy_free(policy->security); 315 kfree(policy); 316 } 317 EXPORT_SYMBOL(xfrm_policy_destroy); 318 319 static void xfrm_queue_purge(struct sk_buff_head *list) 320 { 321 struct sk_buff *skb; 322 323 while ((skb = skb_dequeue(list)) != NULL) { 324 dev_put(skb->dev); 325 kfree_skb(skb); 326 } 327 } 328 329 /* Rule must be locked. Release descentant resources, announce 330 * entry dead. The rule must be unlinked from lists to the moment. 331 */ 332 333 static void xfrm_policy_kill(struct xfrm_policy *policy) 334 { 335 policy->walk.dead = 1; 336 337 atomic_inc(&policy->genid); 338 339 del_timer(&policy->polq.hold_timer); 340 xfrm_queue_purge(&policy->polq.hold_queue); 341 342 if (del_timer(&policy->timer)) 343 xfrm_pol_put(policy); 344 345 xfrm_pol_put(policy); 346 } 347 348 static unsigned int xfrm_policy_hashmax __read_mostly = 1 * 1024 * 1024; 349 350 static inline unsigned int idx_hash(struct net *net, u32 index) 351 { 352 return __idx_hash(index, net->xfrm.policy_idx_hmask); 353 } 354 355 static struct hlist_head *policy_hash_bysel(struct net *net, 356 const struct xfrm_selector *sel, 357 unsigned short family, int dir) 358 { 359 unsigned int hmask = net->xfrm.policy_bydst[dir].hmask; 360 unsigned int hash = __sel_hash(sel, family, hmask); 361 362 return (hash == hmask + 1 ? 363 &net->xfrm.policy_inexact[dir] : 364 net->xfrm.policy_bydst[dir].table + hash); 365 } 366 367 static struct hlist_head *policy_hash_direct(struct net *net, 368 const xfrm_address_t *daddr, 369 const xfrm_address_t *saddr, 370 unsigned short family, int dir) 371 { 372 unsigned int hmask = net->xfrm.policy_bydst[dir].hmask; 373 unsigned int hash = __addr_hash(daddr, saddr, family, hmask); 374 375 return net->xfrm.policy_bydst[dir].table + hash; 376 } 377 378 static void xfrm_dst_hash_transfer(struct hlist_head *list, 379 struct hlist_head *ndsttable, 380 unsigned int nhashmask) 381 { 382 struct hlist_node *tmp, *entry0 = NULL; 383 struct xfrm_policy *pol; 384 unsigned int h0 = 0; 385 386 redo: 387 hlist_for_each_entry_safe(pol, tmp, list, bydst) { 388 unsigned int h; 389 390 h = __addr_hash(&pol->selector.daddr, &pol->selector.saddr, 391 pol->family, nhashmask); 392 if (!entry0) { 393 hlist_del(&pol->bydst); 394 hlist_add_head(&pol->bydst, ndsttable+h); 395 h0 = h; 396 } else { 397 if (h != h0) 398 continue; 399 hlist_del(&pol->bydst); 400 hlist_add_after(entry0, &pol->bydst); 401 } 402 entry0 = &pol->bydst; 403 } 404 if (!hlist_empty(list)) { 405 entry0 = NULL; 406 goto redo; 407 } 408 } 409 410 static void xfrm_idx_hash_transfer(struct hlist_head *list, 411 struct hlist_head *nidxtable, 412 unsigned int nhashmask) 413 { 414 struct hlist_node *tmp; 415 struct xfrm_policy *pol; 416 417 hlist_for_each_entry_safe(pol, tmp, list, byidx) { 418 unsigned int h; 419 420 h = __idx_hash(pol->index, nhashmask); 421 hlist_add_head(&pol->byidx, nidxtable+h); 422 } 423 } 424 425 static unsigned long xfrm_new_hash_mask(unsigned int old_hmask) 426 { 427 return ((old_hmask + 1) << 1) - 1; 428 } 429 430 static void xfrm_bydst_resize(struct net *net, int dir) 431 { 432 unsigned int hmask = net->xfrm.policy_bydst[dir].hmask; 433 unsigned int nhashmask = xfrm_new_hash_mask(hmask); 434 unsigned int nsize = (nhashmask + 1) * sizeof(struct hlist_head); 435 struct hlist_head *odst = net->xfrm.policy_bydst[dir].table; 436 struct hlist_head *ndst = xfrm_hash_alloc(nsize); 437 int i; 438 439 if (!ndst) 440 return; 441 442 write_lock_bh(&xfrm_policy_lock); 443 444 for (i = hmask; i >= 0; i--) 445 xfrm_dst_hash_transfer(odst + i, ndst, nhashmask); 446 447 net->xfrm.policy_bydst[dir].table = ndst; 448 net->xfrm.policy_bydst[dir].hmask = nhashmask; 449 450 write_unlock_bh(&xfrm_policy_lock); 451 452 xfrm_hash_free(odst, (hmask + 1) * sizeof(struct hlist_head)); 453 } 454 455 static void xfrm_byidx_resize(struct net *net, int total) 456 { 457 unsigned int hmask = net->xfrm.policy_idx_hmask; 458 unsigned int nhashmask = xfrm_new_hash_mask(hmask); 459 unsigned int nsize = (nhashmask + 1) * sizeof(struct hlist_head); 460 struct hlist_head *oidx = net->xfrm.policy_byidx; 461 struct hlist_head *nidx = xfrm_hash_alloc(nsize); 462 int i; 463 464 if (!nidx) 465 return; 466 467 write_lock_bh(&xfrm_policy_lock); 468 469 for (i = hmask; i >= 0; i--) 470 xfrm_idx_hash_transfer(oidx + i, nidx, nhashmask); 471 472 net->xfrm.policy_byidx = nidx; 473 net->xfrm.policy_idx_hmask = nhashmask; 474 475 write_unlock_bh(&xfrm_policy_lock); 476 477 xfrm_hash_free(oidx, (hmask + 1) * sizeof(struct hlist_head)); 478 } 479 480 static inline int xfrm_bydst_should_resize(struct net *net, int dir, int *total) 481 { 482 unsigned int cnt = net->xfrm.policy_count[dir]; 483 unsigned int hmask = net->xfrm.policy_bydst[dir].hmask; 484 485 if (total) 486 *total += cnt; 487 488 if ((hmask + 1) < xfrm_policy_hashmax && 489 cnt > hmask) 490 return 1; 491 492 return 0; 493 } 494 495 static inline int xfrm_byidx_should_resize(struct net *net, int total) 496 { 497 unsigned int hmask = net->xfrm.policy_idx_hmask; 498 499 if ((hmask + 1) < xfrm_policy_hashmax && 500 total > hmask) 501 return 1; 502 503 return 0; 504 } 505 506 void xfrm_spd_getinfo(struct net *net, struct xfrmk_spdinfo *si) 507 { 508 read_lock_bh(&xfrm_policy_lock); 509 si->incnt = net->xfrm.policy_count[XFRM_POLICY_IN]; 510 si->outcnt = net->xfrm.policy_count[XFRM_POLICY_OUT]; 511 si->fwdcnt = net->xfrm.policy_count[XFRM_POLICY_FWD]; 512 si->inscnt = net->xfrm.policy_count[XFRM_POLICY_IN+XFRM_POLICY_MAX]; 513 si->outscnt = net->xfrm.policy_count[XFRM_POLICY_OUT+XFRM_POLICY_MAX]; 514 si->fwdscnt = net->xfrm.policy_count[XFRM_POLICY_FWD+XFRM_POLICY_MAX]; 515 si->spdhcnt = net->xfrm.policy_idx_hmask; 516 si->spdhmcnt = xfrm_policy_hashmax; 517 read_unlock_bh(&xfrm_policy_lock); 518 } 519 EXPORT_SYMBOL(xfrm_spd_getinfo); 520 521 static DEFINE_MUTEX(hash_resize_mutex); 522 static void xfrm_hash_resize(struct work_struct *work) 523 { 524 struct net *net = container_of(work, struct net, xfrm.policy_hash_work); 525 int dir, total; 526 527 mutex_lock(&hash_resize_mutex); 528 529 total = 0; 530 for (dir = 0; dir < XFRM_POLICY_MAX * 2; dir++) { 531 if (xfrm_bydst_should_resize(net, dir, &total)) 532 xfrm_bydst_resize(net, dir); 533 } 534 if (xfrm_byidx_should_resize(net, total)) 535 xfrm_byidx_resize(net, total); 536 537 mutex_unlock(&hash_resize_mutex); 538 } 539 540 /* Generate new index... KAME seems to generate them ordered by cost 541 * of an absolute inpredictability of ordering of rules. This will not pass. */ 542 static u32 xfrm_gen_index(struct net *net, int dir) 543 { 544 static u32 idx_generator; 545 546 for (;;) { 547 struct hlist_head *list; 548 struct xfrm_policy *p; 549 u32 idx; 550 int found; 551 552 idx = (idx_generator | dir); 553 idx_generator += 8; 554 if (idx == 0) 555 idx = 8; 556 list = net->xfrm.policy_byidx + idx_hash(net, idx); 557 found = 0; 558 hlist_for_each_entry(p, list, byidx) { 559 if (p->index == idx) { 560 found = 1; 561 break; 562 } 563 } 564 if (!found) 565 return idx; 566 } 567 } 568 569 static inline int selector_cmp(struct xfrm_selector *s1, struct xfrm_selector *s2) 570 { 571 u32 *p1 = (u32 *) s1; 572 u32 *p2 = (u32 *) s2; 573 int len = sizeof(struct xfrm_selector) / sizeof(u32); 574 int i; 575 576 for (i = 0; i < len; i++) { 577 if (p1[i] != p2[i]) 578 return 1; 579 } 580 581 return 0; 582 } 583 584 static void xfrm_policy_requeue(struct xfrm_policy *old, 585 struct xfrm_policy *new) 586 { 587 struct xfrm_policy_queue *pq = &old->polq; 588 struct sk_buff_head list; 589 590 __skb_queue_head_init(&list); 591 592 spin_lock_bh(&pq->hold_queue.lock); 593 skb_queue_splice_init(&pq->hold_queue, &list); 594 del_timer(&pq->hold_timer); 595 spin_unlock_bh(&pq->hold_queue.lock); 596 597 if (skb_queue_empty(&list)) 598 return; 599 600 pq = &new->polq; 601 602 spin_lock_bh(&pq->hold_queue.lock); 603 skb_queue_splice(&list, &pq->hold_queue); 604 pq->timeout = XFRM_QUEUE_TMO_MIN; 605 mod_timer(&pq->hold_timer, jiffies); 606 spin_unlock_bh(&pq->hold_queue.lock); 607 } 608 609 static bool xfrm_policy_mark_match(struct xfrm_policy *policy, 610 struct xfrm_policy *pol) 611 { 612 u32 mark = policy->mark.v & policy->mark.m; 613 614 if (policy->mark.v == pol->mark.v && policy->mark.m == pol->mark.m) 615 return true; 616 617 if ((mark & pol->mark.m) == pol->mark.v && 618 policy->priority == pol->priority) 619 return true; 620 621 return false; 622 } 623 624 int xfrm_policy_insert(int dir, struct xfrm_policy *policy, int excl) 625 { 626 struct net *net = xp_net(policy); 627 struct xfrm_policy *pol; 628 struct xfrm_policy *delpol; 629 struct hlist_head *chain; 630 struct hlist_node *newpos; 631 632 write_lock_bh(&xfrm_policy_lock); 633 chain = policy_hash_bysel(net, &policy->selector, policy->family, dir); 634 delpol = NULL; 635 newpos = NULL; 636 hlist_for_each_entry(pol, chain, bydst) { 637 if (pol->type == policy->type && 638 !selector_cmp(&pol->selector, &policy->selector) && 639 xfrm_policy_mark_match(policy, pol) && 640 xfrm_sec_ctx_match(pol->security, policy->security) && 641 !WARN_ON(delpol)) { 642 if (excl) { 643 write_unlock_bh(&xfrm_policy_lock); 644 return -EEXIST; 645 } 646 delpol = pol; 647 if (policy->priority > pol->priority) 648 continue; 649 } else if (policy->priority >= pol->priority) { 650 newpos = &pol->bydst; 651 continue; 652 } 653 if (delpol) 654 break; 655 } 656 if (newpos) 657 hlist_add_after(newpos, &policy->bydst); 658 else 659 hlist_add_head(&policy->bydst, chain); 660 xfrm_pol_hold(policy); 661 net->xfrm.policy_count[dir]++; 662 atomic_inc(&flow_cache_genid); 663 rt_genid_bump(net); 664 if (delpol) { 665 xfrm_policy_requeue(delpol, policy); 666 __xfrm_policy_unlink(delpol, dir); 667 } 668 policy->index = delpol ? delpol->index : xfrm_gen_index(net, dir); 669 hlist_add_head(&policy->byidx, net->xfrm.policy_byidx+idx_hash(net, policy->index)); 670 policy->curlft.add_time = get_seconds(); 671 policy->curlft.use_time = 0; 672 if (!mod_timer(&policy->timer, jiffies + HZ)) 673 xfrm_pol_hold(policy); 674 list_add(&policy->walk.all, &net->xfrm.policy_all); 675 write_unlock_bh(&xfrm_policy_lock); 676 677 if (delpol) 678 xfrm_policy_kill(delpol); 679 else if (xfrm_bydst_should_resize(net, dir, NULL)) 680 schedule_work(&net->xfrm.policy_hash_work); 681 682 return 0; 683 } 684 EXPORT_SYMBOL(xfrm_policy_insert); 685 686 struct xfrm_policy *xfrm_policy_bysel_ctx(struct net *net, u32 mark, u8 type, 687 int dir, struct xfrm_selector *sel, 688 struct xfrm_sec_ctx *ctx, int delete, 689 int *err) 690 { 691 struct xfrm_policy *pol, *ret; 692 struct hlist_head *chain; 693 694 *err = 0; 695 write_lock_bh(&xfrm_policy_lock); 696 chain = policy_hash_bysel(net, sel, sel->family, dir); 697 ret = NULL; 698 hlist_for_each_entry(pol, chain, bydst) { 699 if (pol->type == type && 700 (mark & pol->mark.m) == pol->mark.v && 701 !selector_cmp(sel, &pol->selector) && 702 xfrm_sec_ctx_match(ctx, pol->security)) { 703 xfrm_pol_hold(pol); 704 if (delete) { 705 *err = security_xfrm_policy_delete( 706 pol->security); 707 if (*err) { 708 write_unlock_bh(&xfrm_policy_lock); 709 return pol; 710 } 711 __xfrm_policy_unlink(pol, dir); 712 } 713 ret = pol; 714 break; 715 } 716 } 717 write_unlock_bh(&xfrm_policy_lock); 718 719 if (ret && delete) 720 xfrm_policy_kill(ret); 721 return ret; 722 } 723 EXPORT_SYMBOL(xfrm_policy_bysel_ctx); 724 725 struct xfrm_policy *xfrm_policy_byid(struct net *net, u32 mark, u8 type, 726 int dir, u32 id, int delete, int *err) 727 { 728 struct xfrm_policy *pol, *ret; 729 struct hlist_head *chain; 730 731 *err = -ENOENT; 732 if (xfrm_policy_id2dir(id) != dir) 733 return NULL; 734 735 *err = 0; 736 write_lock_bh(&xfrm_policy_lock); 737 chain = net->xfrm.policy_byidx + idx_hash(net, id); 738 ret = NULL; 739 hlist_for_each_entry(pol, chain, byidx) { 740 if (pol->type == type && pol->index == id && 741 (mark & pol->mark.m) == pol->mark.v) { 742 xfrm_pol_hold(pol); 743 if (delete) { 744 *err = security_xfrm_policy_delete( 745 pol->security); 746 if (*err) { 747 write_unlock_bh(&xfrm_policy_lock); 748 return pol; 749 } 750 __xfrm_policy_unlink(pol, dir); 751 } 752 ret = pol; 753 break; 754 } 755 } 756 write_unlock_bh(&xfrm_policy_lock); 757 758 if (ret && delete) 759 xfrm_policy_kill(ret); 760 return ret; 761 } 762 EXPORT_SYMBOL(xfrm_policy_byid); 763 764 #ifdef CONFIG_SECURITY_NETWORK_XFRM 765 static inline int 766 xfrm_policy_flush_secctx_check(struct net *net, u8 type, struct xfrm_audit *audit_info) 767 { 768 int dir, err = 0; 769 770 for (dir = 0; dir < XFRM_POLICY_MAX; dir++) { 771 struct xfrm_policy *pol; 772 int i; 773 774 hlist_for_each_entry(pol, 775 &net->xfrm.policy_inexact[dir], bydst) { 776 if (pol->type != type) 777 continue; 778 err = security_xfrm_policy_delete(pol->security); 779 if (err) { 780 xfrm_audit_policy_delete(pol, 0, 781 audit_info->loginuid, 782 audit_info->sessionid, 783 audit_info->secid); 784 return err; 785 } 786 } 787 for (i = net->xfrm.policy_bydst[dir].hmask; i >= 0; i--) { 788 hlist_for_each_entry(pol, 789 net->xfrm.policy_bydst[dir].table + i, 790 bydst) { 791 if (pol->type != type) 792 continue; 793 err = security_xfrm_policy_delete( 794 pol->security); 795 if (err) { 796 xfrm_audit_policy_delete(pol, 0, 797 audit_info->loginuid, 798 audit_info->sessionid, 799 audit_info->secid); 800 return err; 801 } 802 } 803 } 804 } 805 return err; 806 } 807 #else 808 static inline int 809 xfrm_policy_flush_secctx_check(struct net *net, u8 type, struct xfrm_audit *audit_info) 810 { 811 return 0; 812 } 813 #endif 814 815 int xfrm_policy_flush(struct net *net, u8 type, struct xfrm_audit *audit_info) 816 { 817 int dir, err = 0, cnt = 0; 818 819 write_lock_bh(&xfrm_policy_lock); 820 821 err = xfrm_policy_flush_secctx_check(net, type, audit_info); 822 if (err) 823 goto out; 824 825 for (dir = 0; dir < XFRM_POLICY_MAX; dir++) { 826 struct xfrm_policy *pol; 827 int i; 828 829 again1: 830 hlist_for_each_entry(pol, 831 &net->xfrm.policy_inexact[dir], bydst) { 832 if (pol->type != type) 833 continue; 834 __xfrm_policy_unlink(pol, dir); 835 write_unlock_bh(&xfrm_policy_lock); 836 cnt++; 837 838 xfrm_audit_policy_delete(pol, 1, audit_info->loginuid, 839 audit_info->sessionid, 840 audit_info->secid); 841 842 xfrm_policy_kill(pol); 843 844 write_lock_bh(&xfrm_policy_lock); 845 goto again1; 846 } 847 848 for (i = net->xfrm.policy_bydst[dir].hmask; i >= 0; i--) { 849 again2: 850 hlist_for_each_entry(pol, 851 net->xfrm.policy_bydst[dir].table + i, 852 bydst) { 853 if (pol->type != type) 854 continue; 855 __xfrm_policy_unlink(pol, dir); 856 write_unlock_bh(&xfrm_policy_lock); 857 cnt++; 858 859 xfrm_audit_policy_delete(pol, 1, 860 audit_info->loginuid, 861 audit_info->sessionid, 862 audit_info->secid); 863 xfrm_policy_kill(pol); 864 865 write_lock_bh(&xfrm_policy_lock); 866 goto again2; 867 } 868 } 869 870 } 871 if (!cnt) 872 err = -ESRCH; 873 out: 874 write_unlock_bh(&xfrm_policy_lock); 875 return err; 876 } 877 EXPORT_SYMBOL(xfrm_policy_flush); 878 879 int xfrm_policy_walk(struct net *net, struct xfrm_policy_walk *walk, 880 int (*func)(struct xfrm_policy *, int, int, void*), 881 void *data) 882 { 883 struct xfrm_policy *pol; 884 struct xfrm_policy_walk_entry *x; 885 int error = 0; 886 887 if (walk->type >= XFRM_POLICY_TYPE_MAX && 888 walk->type != XFRM_POLICY_TYPE_ANY) 889 return -EINVAL; 890 891 if (list_empty(&walk->walk.all) && walk->seq != 0) 892 return 0; 893 894 write_lock_bh(&xfrm_policy_lock); 895 if (list_empty(&walk->walk.all)) 896 x = list_first_entry(&net->xfrm.policy_all, struct xfrm_policy_walk_entry, all); 897 else 898 x = list_entry(&walk->walk.all, struct xfrm_policy_walk_entry, all); 899 list_for_each_entry_from(x, &net->xfrm.policy_all, all) { 900 if (x->dead) 901 continue; 902 pol = container_of(x, struct xfrm_policy, walk); 903 if (walk->type != XFRM_POLICY_TYPE_ANY && 904 walk->type != pol->type) 905 continue; 906 error = func(pol, xfrm_policy_id2dir(pol->index), 907 walk->seq, data); 908 if (error) { 909 list_move_tail(&walk->walk.all, &x->all); 910 goto out; 911 } 912 walk->seq++; 913 } 914 if (walk->seq == 0) { 915 error = -ENOENT; 916 goto out; 917 } 918 list_del_init(&walk->walk.all); 919 out: 920 write_unlock_bh(&xfrm_policy_lock); 921 return error; 922 } 923 EXPORT_SYMBOL(xfrm_policy_walk); 924 925 void xfrm_policy_walk_init(struct xfrm_policy_walk *walk, u8 type) 926 { 927 INIT_LIST_HEAD(&walk->walk.all); 928 walk->walk.dead = 1; 929 walk->type = type; 930 walk->seq = 0; 931 } 932 EXPORT_SYMBOL(xfrm_policy_walk_init); 933 934 void xfrm_policy_walk_done(struct xfrm_policy_walk *walk) 935 { 936 if (list_empty(&walk->walk.all)) 937 return; 938 939 write_lock_bh(&xfrm_policy_lock); 940 list_del(&walk->walk.all); 941 write_unlock_bh(&xfrm_policy_lock); 942 } 943 EXPORT_SYMBOL(xfrm_policy_walk_done); 944 945 /* 946 * Find policy to apply to this flow. 947 * 948 * Returns 0 if policy found, else an -errno. 949 */ 950 static int xfrm_policy_match(const struct xfrm_policy *pol, 951 const struct flowi *fl, 952 u8 type, u16 family, int dir) 953 { 954 const struct xfrm_selector *sel = &pol->selector; 955 int ret = -ESRCH; 956 bool match; 957 958 if (pol->family != family || 959 (fl->flowi_mark & pol->mark.m) != pol->mark.v || 960 pol->type != type) 961 return ret; 962 963 match = xfrm_selector_match(sel, fl, family); 964 if (match) 965 ret = security_xfrm_policy_lookup(pol->security, fl->flowi_secid, 966 dir); 967 968 return ret; 969 } 970 971 static struct xfrm_policy *xfrm_policy_lookup_bytype(struct net *net, u8 type, 972 const struct flowi *fl, 973 u16 family, u8 dir) 974 { 975 int err; 976 struct xfrm_policy *pol, *ret; 977 const xfrm_address_t *daddr, *saddr; 978 struct hlist_head *chain; 979 u32 priority = ~0U; 980 981 daddr = xfrm_flowi_daddr(fl, family); 982 saddr = xfrm_flowi_saddr(fl, family); 983 if (unlikely(!daddr || !saddr)) 984 return NULL; 985 986 read_lock_bh(&xfrm_policy_lock); 987 chain = policy_hash_direct(net, daddr, saddr, family, dir); 988 ret = NULL; 989 hlist_for_each_entry(pol, chain, bydst) { 990 err = xfrm_policy_match(pol, fl, type, family, dir); 991 if (err) { 992 if (err == -ESRCH) 993 continue; 994 else { 995 ret = ERR_PTR(err); 996 goto fail; 997 } 998 } else { 999 ret = pol; 1000 priority = ret->priority; 1001 break; 1002 } 1003 } 1004 chain = &net->xfrm.policy_inexact[dir]; 1005 hlist_for_each_entry(pol, chain, bydst) { 1006 err = xfrm_policy_match(pol, fl, type, family, dir); 1007 if (err) { 1008 if (err == -ESRCH) 1009 continue; 1010 else { 1011 ret = ERR_PTR(err); 1012 goto fail; 1013 } 1014 } else if (pol->priority < priority) { 1015 ret = pol; 1016 break; 1017 } 1018 } 1019 if (ret) 1020 xfrm_pol_hold(ret); 1021 fail: 1022 read_unlock_bh(&xfrm_policy_lock); 1023 1024 return ret; 1025 } 1026 1027 static struct xfrm_policy * 1028 __xfrm_policy_lookup(struct net *net, const struct flowi *fl, u16 family, u8 dir) 1029 { 1030 #ifdef CONFIG_XFRM_SUB_POLICY 1031 struct xfrm_policy *pol; 1032 1033 pol = xfrm_policy_lookup_bytype(net, XFRM_POLICY_TYPE_SUB, fl, family, dir); 1034 if (pol != NULL) 1035 return pol; 1036 #endif 1037 return xfrm_policy_lookup_bytype(net, XFRM_POLICY_TYPE_MAIN, fl, family, dir); 1038 } 1039 1040 static int flow_to_policy_dir(int dir) 1041 { 1042 if (XFRM_POLICY_IN == FLOW_DIR_IN && 1043 XFRM_POLICY_OUT == FLOW_DIR_OUT && 1044 XFRM_POLICY_FWD == FLOW_DIR_FWD) 1045 return dir; 1046 1047 switch (dir) { 1048 default: 1049 case FLOW_DIR_IN: 1050 return XFRM_POLICY_IN; 1051 case FLOW_DIR_OUT: 1052 return XFRM_POLICY_OUT; 1053 case FLOW_DIR_FWD: 1054 return XFRM_POLICY_FWD; 1055 } 1056 } 1057 1058 static struct flow_cache_object * 1059 xfrm_policy_lookup(struct net *net, const struct flowi *fl, u16 family, 1060 u8 dir, struct flow_cache_object *old_obj, void *ctx) 1061 { 1062 struct xfrm_policy *pol; 1063 1064 if (old_obj) 1065 xfrm_pol_put(container_of(old_obj, struct xfrm_policy, flo)); 1066 1067 pol = __xfrm_policy_lookup(net, fl, family, flow_to_policy_dir(dir)); 1068 if (IS_ERR_OR_NULL(pol)) 1069 return ERR_CAST(pol); 1070 1071 /* Resolver returns two references: 1072 * one for cache and one for caller of flow_cache_lookup() */ 1073 xfrm_pol_hold(pol); 1074 1075 return &pol->flo; 1076 } 1077 1078 static inline int policy_to_flow_dir(int dir) 1079 { 1080 if (XFRM_POLICY_IN == FLOW_DIR_IN && 1081 XFRM_POLICY_OUT == FLOW_DIR_OUT && 1082 XFRM_POLICY_FWD == FLOW_DIR_FWD) 1083 return dir; 1084 switch (dir) { 1085 default: 1086 case XFRM_POLICY_IN: 1087 return FLOW_DIR_IN; 1088 case XFRM_POLICY_OUT: 1089 return FLOW_DIR_OUT; 1090 case XFRM_POLICY_FWD: 1091 return FLOW_DIR_FWD; 1092 } 1093 } 1094 1095 static struct xfrm_policy *xfrm_sk_policy_lookup(struct sock *sk, int dir, 1096 const struct flowi *fl) 1097 { 1098 struct xfrm_policy *pol; 1099 1100 read_lock_bh(&xfrm_policy_lock); 1101 if ((pol = sk->sk_policy[dir]) != NULL) { 1102 bool match = xfrm_selector_match(&pol->selector, fl, 1103 sk->sk_family); 1104 int err = 0; 1105 1106 if (match) { 1107 if ((sk->sk_mark & pol->mark.m) != pol->mark.v) { 1108 pol = NULL; 1109 goto out; 1110 } 1111 err = security_xfrm_policy_lookup(pol->security, 1112 fl->flowi_secid, 1113 policy_to_flow_dir(dir)); 1114 if (!err) 1115 xfrm_pol_hold(pol); 1116 else if (err == -ESRCH) 1117 pol = NULL; 1118 else 1119 pol = ERR_PTR(err); 1120 } else 1121 pol = NULL; 1122 } 1123 out: 1124 read_unlock_bh(&xfrm_policy_lock); 1125 return pol; 1126 } 1127 1128 static void __xfrm_policy_link(struct xfrm_policy *pol, int dir) 1129 { 1130 struct net *net = xp_net(pol); 1131 struct hlist_head *chain = policy_hash_bysel(net, &pol->selector, 1132 pol->family, dir); 1133 1134 list_add(&pol->walk.all, &net->xfrm.policy_all); 1135 hlist_add_head(&pol->bydst, chain); 1136 hlist_add_head(&pol->byidx, net->xfrm.policy_byidx+idx_hash(net, pol->index)); 1137 net->xfrm.policy_count[dir]++; 1138 xfrm_pol_hold(pol); 1139 1140 if (xfrm_bydst_should_resize(net, dir, NULL)) 1141 schedule_work(&net->xfrm.policy_hash_work); 1142 } 1143 1144 static struct xfrm_policy *__xfrm_policy_unlink(struct xfrm_policy *pol, 1145 int dir) 1146 { 1147 struct net *net = xp_net(pol); 1148 1149 if (hlist_unhashed(&pol->bydst)) 1150 return NULL; 1151 1152 hlist_del(&pol->bydst); 1153 hlist_del(&pol->byidx); 1154 list_del(&pol->walk.all); 1155 net->xfrm.policy_count[dir]--; 1156 1157 return pol; 1158 } 1159 1160 int xfrm_policy_delete(struct xfrm_policy *pol, int dir) 1161 { 1162 write_lock_bh(&xfrm_policy_lock); 1163 pol = __xfrm_policy_unlink(pol, dir); 1164 write_unlock_bh(&xfrm_policy_lock); 1165 if (pol) { 1166 xfrm_policy_kill(pol); 1167 return 0; 1168 } 1169 return -ENOENT; 1170 } 1171 EXPORT_SYMBOL(xfrm_policy_delete); 1172 1173 int xfrm_sk_policy_insert(struct sock *sk, int dir, struct xfrm_policy *pol) 1174 { 1175 struct net *net = xp_net(pol); 1176 struct xfrm_policy *old_pol; 1177 1178 #ifdef CONFIG_XFRM_SUB_POLICY 1179 if (pol && pol->type != XFRM_POLICY_TYPE_MAIN) 1180 return -EINVAL; 1181 #endif 1182 1183 write_lock_bh(&xfrm_policy_lock); 1184 old_pol = sk->sk_policy[dir]; 1185 sk->sk_policy[dir] = pol; 1186 if (pol) { 1187 pol->curlft.add_time = get_seconds(); 1188 pol->index = xfrm_gen_index(net, XFRM_POLICY_MAX+dir); 1189 __xfrm_policy_link(pol, XFRM_POLICY_MAX+dir); 1190 } 1191 if (old_pol) { 1192 if (pol) 1193 xfrm_policy_requeue(old_pol, pol); 1194 1195 /* Unlinking succeeds always. This is the only function 1196 * allowed to delete or replace socket policy. 1197 */ 1198 __xfrm_policy_unlink(old_pol, XFRM_POLICY_MAX+dir); 1199 } 1200 write_unlock_bh(&xfrm_policy_lock); 1201 1202 if (old_pol) { 1203 xfrm_policy_kill(old_pol); 1204 } 1205 return 0; 1206 } 1207 1208 static struct xfrm_policy *clone_policy(const struct xfrm_policy *old, int dir) 1209 { 1210 struct xfrm_policy *newp = xfrm_policy_alloc(xp_net(old), GFP_ATOMIC); 1211 1212 if (newp) { 1213 newp->selector = old->selector; 1214 if (security_xfrm_policy_clone(old->security, 1215 &newp->security)) { 1216 kfree(newp); 1217 return NULL; /* ENOMEM */ 1218 } 1219 newp->lft = old->lft; 1220 newp->curlft = old->curlft; 1221 newp->mark = old->mark; 1222 newp->action = old->action; 1223 newp->flags = old->flags; 1224 newp->xfrm_nr = old->xfrm_nr; 1225 newp->index = old->index; 1226 newp->type = old->type; 1227 memcpy(newp->xfrm_vec, old->xfrm_vec, 1228 newp->xfrm_nr*sizeof(struct xfrm_tmpl)); 1229 write_lock_bh(&xfrm_policy_lock); 1230 __xfrm_policy_link(newp, XFRM_POLICY_MAX+dir); 1231 write_unlock_bh(&xfrm_policy_lock); 1232 xfrm_pol_put(newp); 1233 } 1234 return newp; 1235 } 1236 1237 int __xfrm_sk_clone_policy(struct sock *sk) 1238 { 1239 struct xfrm_policy *p0 = sk->sk_policy[0], 1240 *p1 = sk->sk_policy[1]; 1241 1242 sk->sk_policy[0] = sk->sk_policy[1] = NULL; 1243 if (p0 && (sk->sk_policy[0] = clone_policy(p0, 0)) == NULL) 1244 return -ENOMEM; 1245 if (p1 && (sk->sk_policy[1] = clone_policy(p1, 1)) == NULL) 1246 return -ENOMEM; 1247 return 0; 1248 } 1249 1250 static int 1251 xfrm_get_saddr(struct net *net, xfrm_address_t *local, xfrm_address_t *remote, 1252 unsigned short family) 1253 { 1254 int err; 1255 struct xfrm_policy_afinfo *afinfo = xfrm_policy_get_afinfo(family); 1256 1257 if (unlikely(afinfo == NULL)) 1258 return -EINVAL; 1259 err = afinfo->get_saddr(net, local, remote); 1260 xfrm_policy_put_afinfo(afinfo); 1261 return err; 1262 } 1263 1264 /* Resolve list of templates for the flow, given policy. */ 1265 1266 static int 1267 xfrm_tmpl_resolve_one(struct xfrm_policy *policy, const struct flowi *fl, 1268 struct xfrm_state **xfrm, unsigned short family) 1269 { 1270 struct net *net = xp_net(policy); 1271 int nx; 1272 int i, error; 1273 xfrm_address_t *daddr = xfrm_flowi_daddr(fl, family); 1274 xfrm_address_t *saddr = xfrm_flowi_saddr(fl, family); 1275 xfrm_address_t tmp; 1276 1277 for (nx=0, i = 0; i < policy->xfrm_nr; i++) { 1278 struct xfrm_state *x; 1279 xfrm_address_t *remote = daddr; 1280 xfrm_address_t *local = saddr; 1281 struct xfrm_tmpl *tmpl = &policy->xfrm_vec[i]; 1282 1283 if (tmpl->mode == XFRM_MODE_TUNNEL || 1284 tmpl->mode == XFRM_MODE_BEET) { 1285 remote = &tmpl->id.daddr; 1286 local = &tmpl->saddr; 1287 if (xfrm_addr_any(local, tmpl->encap_family)) { 1288 error = xfrm_get_saddr(net, &tmp, remote, tmpl->encap_family); 1289 if (error) 1290 goto fail; 1291 local = &tmp; 1292 } 1293 } 1294 1295 x = xfrm_state_find(remote, local, fl, tmpl, policy, &error, family); 1296 1297 if (x && x->km.state == XFRM_STATE_VALID) { 1298 xfrm[nx++] = x; 1299 daddr = remote; 1300 saddr = local; 1301 continue; 1302 } 1303 if (x) { 1304 error = (x->km.state == XFRM_STATE_ERROR ? 1305 -EINVAL : -EAGAIN); 1306 xfrm_state_put(x); 1307 } 1308 else if (error == -ESRCH) 1309 error = -EAGAIN; 1310 1311 if (!tmpl->optional) 1312 goto fail; 1313 } 1314 return nx; 1315 1316 fail: 1317 for (nx--; nx>=0; nx--) 1318 xfrm_state_put(xfrm[nx]); 1319 return error; 1320 } 1321 1322 static int 1323 xfrm_tmpl_resolve(struct xfrm_policy **pols, int npols, const struct flowi *fl, 1324 struct xfrm_state **xfrm, unsigned short family) 1325 { 1326 struct xfrm_state *tp[XFRM_MAX_DEPTH]; 1327 struct xfrm_state **tpp = (npols > 1) ? tp : xfrm; 1328 int cnx = 0; 1329 int error; 1330 int ret; 1331 int i; 1332 1333 for (i = 0; i < npols; i++) { 1334 if (cnx + pols[i]->xfrm_nr >= XFRM_MAX_DEPTH) { 1335 error = -ENOBUFS; 1336 goto fail; 1337 } 1338 1339 ret = xfrm_tmpl_resolve_one(pols[i], fl, &tpp[cnx], family); 1340 if (ret < 0) { 1341 error = ret; 1342 goto fail; 1343 } else 1344 cnx += ret; 1345 } 1346 1347 /* found states are sorted for outbound processing */ 1348 if (npols > 1) 1349 xfrm_state_sort(xfrm, tpp, cnx, family); 1350 1351 return cnx; 1352 1353 fail: 1354 for (cnx--; cnx>=0; cnx--) 1355 xfrm_state_put(tpp[cnx]); 1356 return error; 1357 1358 } 1359 1360 /* Check that the bundle accepts the flow and its components are 1361 * still valid. 1362 */ 1363 1364 static inline int xfrm_get_tos(const struct flowi *fl, int family) 1365 { 1366 struct xfrm_policy_afinfo *afinfo = xfrm_policy_get_afinfo(family); 1367 int tos; 1368 1369 if (!afinfo) 1370 return -EINVAL; 1371 1372 tos = afinfo->get_tos(fl); 1373 1374 xfrm_policy_put_afinfo(afinfo); 1375 1376 return tos; 1377 } 1378 1379 static struct flow_cache_object *xfrm_bundle_flo_get(struct flow_cache_object *flo) 1380 { 1381 struct xfrm_dst *xdst = container_of(flo, struct xfrm_dst, flo); 1382 struct dst_entry *dst = &xdst->u.dst; 1383 1384 if (xdst->route == NULL) { 1385 /* Dummy bundle - if it has xfrms we were not 1386 * able to build bundle as template resolution failed. 1387 * It means we need to try again resolving. */ 1388 if (xdst->num_xfrms > 0) 1389 return NULL; 1390 } else if (dst->flags & DST_XFRM_QUEUE) { 1391 return NULL; 1392 } else { 1393 /* Real bundle */ 1394 if (stale_bundle(dst)) 1395 return NULL; 1396 } 1397 1398 dst_hold(dst); 1399 return flo; 1400 } 1401 1402 static int xfrm_bundle_flo_check(struct flow_cache_object *flo) 1403 { 1404 struct xfrm_dst *xdst = container_of(flo, struct xfrm_dst, flo); 1405 struct dst_entry *dst = &xdst->u.dst; 1406 1407 if (!xdst->route) 1408 return 0; 1409 if (stale_bundle(dst)) 1410 return 0; 1411 1412 return 1; 1413 } 1414 1415 static void xfrm_bundle_flo_delete(struct flow_cache_object *flo) 1416 { 1417 struct xfrm_dst *xdst = container_of(flo, struct xfrm_dst, flo); 1418 struct dst_entry *dst = &xdst->u.dst; 1419 1420 dst_free(dst); 1421 } 1422 1423 static const struct flow_cache_ops xfrm_bundle_fc_ops = { 1424 .get = xfrm_bundle_flo_get, 1425 .check = xfrm_bundle_flo_check, 1426 .delete = xfrm_bundle_flo_delete, 1427 }; 1428 1429 static inline struct xfrm_dst *xfrm_alloc_dst(struct net *net, int family) 1430 { 1431 struct xfrm_policy_afinfo *afinfo = xfrm_policy_get_afinfo(family); 1432 struct dst_ops *dst_ops; 1433 struct xfrm_dst *xdst; 1434 1435 if (!afinfo) 1436 return ERR_PTR(-EINVAL); 1437 1438 switch (family) { 1439 case AF_INET: 1440 dst_ops = &net->xfrm.xfrm4_dst_ops; 1441 break; 1442 #if IS_ENABLED(CONFIG_IPV6) 1443 case AF_INET6: 1444 dst_ops = &net->xfrm.xfrm6_dst_ops; 1445 break; 1446 #endif 1447 default: 1448 BUG(); 1449 } 1450 xdst = dst_alloc(dst_ops, NULL, 0, DST_OBSOLETE_NONE, 0); 1451 1452 if (likely(xdst)) { 1453 struct dst_entry *dst = &xdst->u.dst; 1454 1455 memset(dst + 1, 0, sizeof(*xdst) - sizeof(*dst)); 1456 xdst->flo.ops = &xfrm_bundle_fc_ops; 1457 if (afinfo->init_dst) 1458 afinfo->init_dst(net, xdst); 1459 } else 1460 xdst = ERR_PTR(-ENOBUFS); 1461 1462 xfrm_policy_put_afinfo(afinfo); 1463 1464 return xdst; 1465 } 1466 1467 static inline int xfrm_init_path(struct xfrm_dst *path, struct dst_entry *dst, 1468 int nfheader_len) 1469 { 1470 struct xfrm_policy_afinfo *afinfo = 1471 xfrm_policy_get_afinfo(dst->ops->family); 1472 int err; 1473 1474 if (!afinfo) 1475 return -EINVAL; 1476 1477 err = afinfo->init_path(path, dst, nfheader_len); 1478 1479 xfrm_policy_put_afinfo(afinfo); 1480 1481 return err; 1482 } 1483 1484 static inline int xfrm_fill_dst(struct xfrm_dst *xdst, struct net_device *dev, 1485 const struct flowi *fl) 1486 { 1487 struct xfrm_policy_afinfo *afinfo = 1488 xfrm_policy_get_afinfo(xdst->u.dst.ops->family); 1489 int err; 1490 1491 if (!afinfo) 1492 return -EINVAL; 1493 1494 err = afinfo->fill_dst(xdst, dev, fl); 1495 1496 xfrm_policy_put_afinfo(afinfo); 1497 1498 return err; 1499 } 1500 1501 1502 /* Allocate chain of dst_entry's, attach known xfrm's, calculate 1503 * all the metrics... Shortly, bundle a bundle. 1504 */ 1505 1506 static struct dst_entry *xfrm_bundle_create(struct xfrm_policy *policy, 1507 struct xfrm_state **xfrm, int nx, 1508 const struct flowi *fl, 1509 struct dst_entry *dst) 1510 { 1511 struct net *net = xp_net(policy); 1512 unsigned long now = jiffies; 1513 struct net_device *dev; 1514 struct xfrm_mode *inner_mode; 1515 struct dst_entry *dst_prev = NULL; 1516 struct dst_entry *dst0 = NULL; 1517 int i = 0; 1518 int err; 1519 int header_len = 0; 1520 int nfheader_len = 0; 1521 int trailer_len = 0; 1522 int tos; 1523 int family = policy->selector.family; 1524 xfrm_address_t saddr, daddr; 1525 1526 xfrm_flowi_addr_get(fl, &saddr, &daddr, family); 1527 1528 tos = xfrm_get_tos(fl, family); 1529 err = tos; 1530 if (tos < 0) 1531 goto put_states; 1532 1533 dst_hold(dst); 1534 1535 for (; i < nx; i++) { 1536 struct xfrm_dst *xdst = xfrm_alloc_dst(net, family); 1537 struct dst_entry *dst1 = &xdst->u.dst; 1538 1539 err = PTR_ERR(xdst); 1540 if (IS_ERR(xdst)) { 1541 dst_release(dst); 1542 goto put_states; 1543 } 1544 1545 if (xfrm[i]->sel.family == AF_UNSPEC) { 1546 inner_mode = xfrm_ip2inner_mode(xfrm[i], 1547 xfrm_af2proto(family)); 1548 if (!inner_mode) { 1549 err = -EAFNOSUPPORT; 1550 dst_release(dst); 1551 goto put_states; 1552 } 1553 } else 1554 inner_mode = xfrm[i]->inner_mode; 1555 1556 if (!dst_prev) 1557 dst0 = dst1; 1558 else { 1559 dst_prev->child = dst_clone(dst1); 1560 dst1->flags |= DST_NOHASH; 1561 } 1562 1563 xdst->route = dst; 1564 dst_copy_metrics(dst1, dst); 1565 1566 if (xfrm[i]->props.mode != XFRM_MODE_TRANSPORT) { 1567 family = xfrm[i]->props.family; 1568 dst = xfrm_dst_lookup(xfrm[i], tos, &saddr, &daddr, 1569 family); 1570 err = PTR_ERR(dst); 1571 if (IS_ERR(dst)) 1572 goto put_states; 1573 } else 1574 dst_hold(dst); 1575 1576 dst1->xfrm = xfrm[i]; 1577 xdst->xfrm_genid = xfrm[i]->genid; 1578 1579 dst1->obsolete = DST_OBSOLETE_FORCE_CHK; 1580 dst1->flags |= DST_HOST; 1581 dst1->lastuse = now; 1582 1583 dst1->input = dst_discard; 1584 dst1->output = inner_mode->afinfo->output; 1585 1586 dst1->next = dst_prev; 1587 dst_prev = dst1; 1588 1589 header_len += xfrm[i]->props.header_len; 1590 if (xfrm[i]->type->flags & XFRM_TYPE_NON_FRAGMENT) 1591 nfheader_len += xfrm[i]->props.header_len; 1592 trailer_len += xfrm[i]->props.trailer_len; 1593 } 1594 1595 dst_prev->child = dst; 1596 dst0->path = dst; 1597 1598 err = -ENODEV; 1599 dev = dst->dev; 1600 if (!dev) 1601 goto free_dst; 1602 1603 xfrm_init_path((struct xfrm_dst *)dst0, dst, nfheader_len); 1604 xfrm_init_pmtu(dst_prev); 1605 1606 for (dst_prev = dst0; dst_prev != dst; dst_prev = dst_prev->child) { 1607 struct xfrm_dst *xdst = (struct xfrm_dst *)dst_prev; 1608 1609 err = xfrm_fill_dst(xdst, dev, fl); 1610 if (err) 1611 goto free_dst; 1612 1613 dst_prev->header_len = header_len; 1614 dst_prev->trailer_len = trailer_len; 1615 header_len -= xdst->u.dst.xfrm->props.header_len; 1616 trailer_len -= xdst->u.dst.xfrm->props.trailer_len; 1617 } 1618 1619 out: 1620 return dst0; 1621 1622 put_states: 1623 for (; i < nx; i++) 1624 xfrm_state_put(xfrm[i]); 1625 free_dst: 1626 if (dst0) 1627 dst_free(dst0); 1628 dst0 = ERR_PTR(err); 1629 goto out; 1630 } 1631 1632 static int inline 1633 xfrm_dst_alloc_copy(void **target, const void *src, int size) 1634 { 1635 if (!*target) { 1636 *target = kmalloc(size, GFP_ATOMIC); 1637 if (!*target) 1638 return -ENOMEM; 1639 } 1640 memcpy(*target, src, size); 1641 return 0; 1642 } 1643 1644 static int inline 1645 xfrm_dst_update_parent(struct dst_entry *dst, const struct xfrm_selector *sel) 1646 { 1647 #ifdef CONFIG_XFRM_SUB_POLICY 1648 struct xfrm_dst *xdst = (struct xfrm_dst *)dst; 1649 return xfrm_dst_alloc_copy((void **)&(xdst->partner), 1650 sel, sizeof(*sel)); 1651 #else 1652 return 0; 1653 #endif 1654 } 1655 1656 static int inline 1657 xfrm_dst_update_origin(struct dst_entry *dst, const struct flowi *fl) 1658 { 1659 #ifdef CONFIG_XFRM_SUB_POLICY 1660 struct xfrm_dst *xdst = (struct xfrm_dst *)dst; 1661 return xfrm_dst_alloc_copy((void **)&(xdst->origin), fl, sizeof(*fl)); 1662 #else 1663 return 0; 1664 #endif 1665 } 1666 1667 static int xfrm_expand_policies(const struct flowi *fl, u16 family, 1668 struct xfrm_policy **pols, 1669 int *num_pols, int *num_xfrms) 1670 { 1671 int i; 1672 1673 if (*num_pols == 0 || !pols[0]) { 1674 *num_pols = 0; 1675 *num_xfrms = 0; 1676 return 0; 1677 } 1678 if (IS_ERR(pols[0])) 1679 return PTR_ERR(pols[0]); 1680 1681 *num_xfrms = pols[0]->xfrm_nr; 1682 1683 #ifdef CONFIG_XFRM_SUB_POLICY 1684 if (pols[0] && pols[0]->action == XFRM_POLICY_ALLOW && 1685 pols[0]->type != XFRM_POLICY_TYPE_MAIN) { 1686 pols[1] = xfrm_policy_lookup_bytype(xp_net(pols[0]), 1687 XFRM_POLICY_TYPE_MAIN, 1688 fl, family, 1689 XFRM_POLICY_OUT); 1690 if (pols[1]) { 1691 if (IS_ERR(pols[1])) { 1692 xfrm_pols_put(pols, *num_pols); 1693 return PTR_ERR(pols[1]); 1694 } 1695 (*num_pols) ++; 1696 (*num_xfrms) += pols[1]->xfrm_nr; 1697 } 1698 } 1699 #endif 1700 for (i = 0; i < *num_pols; i++) { 1701 if (pols[i]->action != XFRM_POLICY_ALLOW) { 1702 *num_xfrms = -1; 1703 break; 1704 } 1705 } 1706 1707 return 0; 1708 1709 } 1710 1711 static struct xfrm_dst * 1712 xfrm_resolve_and_create_bundle(struct xfrm_policy **pols, int num_pols, 1713 const struct flowi *fl, u16 family, 1714 struct dst_entry *dst_orig) 1715 { 1716 struct net *net = xp_net(pols[0]); 1717 struct xfrm_state *xfrm[XFRM_MAX_DEPTH]; 1718 struct dst_entry *dst; 1719 struct xfrm_dst *xdst; 1720 int err; 1721 1722 /* Try to instantiate a bundle */ 1723 err = xfrm_tmpl_resolve(pols, num_pols, fl, xfrm, family); 1724 if (err <= 0) { 1725 if (err != 0 && err != -EAGAIN) 1726 XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTPOLERROR); 1727 return ERR_PTR(err); 1728 } 1729 1730 dst = xfrm_bundle_create(pols[0], xfrm, err, fl, dst_orig); 1731 if (IS_ERR(dst)) { 1732 XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTBUNDLEGENERROR); 1733 return ERR_CAST(dst); 1734 } 1735 1736 xdst = (struct xfrm_dst *)dst; 1737 xdst->num_xfrms = err; 1738 if (num_pols > 1) 1739 err = xfrm_dst_update_parent(dst, &pols[1]->selector); 1740 else 1741 err = xfrm_dst_update_origin(dst, fl); 1742 if (unlikely(err)) { 1743 dst_free(dst); 1744 XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTBUNDLECHECKERROR); 1745 return ERR_PTR(err); 1746 } 1747 1748 xdst->num_pols = num_pols; 1749 memcpy(xdst->pols, pols, sizeof(struct xfrm_policy*) * num_pols); 1750 xdst->policy_genid = atomic_read(&pols[0]->genid); 1751 1752 return xdst; 1753 } 1754 1755 static void xfrm_policy_queue_process(unsigned long arg) 1756 { 1757 int err = 0; 1758 struct sk_buff *skb; 1759 struct sock *sk; 1760 struct dst_entry *dst; 1761 struct net_device *dev; 1762 struct xfrm_policy *pol = (struct xfrm_policy *)arg; 1763 struct xfrm_policy_queue *pq = &pol->polq; 1764 struct flowi fl; 1765 struct sk_buff_head list; 1766 1767 spin_lock(&pq->hold_queue.lock); 1768 skb = skb_peek(&pq->hold_queue); 1769 dst = skb_dst(skb); 1770 sk = skb->sk; 1771 xfrm_decode_session(skb, &fl, dst->ops->family); 1772 spin_unlock(&pq->hold_queue.lock); 1773 1774 dst_hold(dst->path); 1775 dst = xfrm_lookup(xp_net(pol), dst->path, &fl, 1776 sk, 0); 1777 if (IS_ERR(dst)) 1778 goto purge_queue; 1779 1780 if (dst->flags & DST_XFRM_QUEUE) { 1781 dst_release(dst); 1782 1783 if (pq->timeout >= XFRM_QUEUE_TMO_MAX) 1784 goto purge_queue; 1785 1786 pq->timeout = pq->timeout << 1; 1787 mod_timer(&pq->hold_timer, jiffies + pq->timeout); 1788 return; 1789 } 1790 1791 dst_release(dst); 1792 1793 __skb_queue_head_init(&list); 1794 1795 spin_lock(&pq->hold_queue.lock); 1796 pq->timeout = 0; 1797 skb_queue_splice_init(&pq->hold_queue, &list); 1798 spin_unlock(&pq->hold_queue.lock); 1799 1800 while (!skb_queue_empty(&list)) { 1801 skb = __skb_dequeue(&list); 1802 1803 xfrm_decode_session(skb, &fl, skb_dst(skb)->ops->family); 1804 dst_hold(skb_dst(skb)->path); 1805 dst = xfrm_lookup(xp_net(pol), skb_dst(skb)->path, 1806 &fl, skb->sk, 0); 1807 if (IS_ERR(dst)) { 1808 dev_put(skb->dev); 1809 kfree_skb(skb); 1810 continue; 1811 } 1812 1813 nf_reset(skb); 1814 skb_dst_drop(skb); 1815 skb_dst_set(skb, dst); 1816 1817 dev = skb->dev; 1818 err = dst_output(skb); 1819 dev_put(dev); 1820 } 1821 1822 return; 1823 1824 purge_queue: 1825 pq->timeout = 0; 1826 xfrm_queue_purge(&pq->hold_queue); 1827 } 1828 1829 static int xdst_queue_output(struct sk_buff *skb) 1830 { 1831 unsigned long sched_next; 1832 struct dst_entry *dst = skb_dst(skb); 1833 struct xfrm_dst *xdst = (struct xfrm_dst *) dst; 1834 struct xfrm_policy_queue *pq = &xdst->pols[0]->polq; 1835 1836 if (pq->hold_queue.qlen > XFRM_MAX_QUEUE_LEN) { 1837 kfree_skb(skb); 1838 return -EAGAIN; 1839 } 1840 1841 skb_dst_force(skb); 1842 dev_hold(skb->dev); 1843 1844 spin_lock_bh(&pq->hold_queue.lock); 1845 1846 if (!pq->timeout) 1847 pq->timeout = XFRM_QUEUE_TMO_MIN; 1848 1849 sched_next = jiffies + pq->timeout; 1850 1851 if (del_timer(&pq->hold_timer)) { 1852 if (time_before(pq->hold_timer.expires, sched_next)) 1853 sched_next = pq->hold_timer.expires; 1854 } 1855 1856 __skb_queue_tail(&pq->hold_queue, skb); 1857 mod_timer(&pq->hold_timer, sched_next); 1858 1859 spin_unlock_bh(&pq->hold_queue.lock); 1860 1861 return 0; 1862 } 1863 1864 static struct xfrm_dst *xfrm_create_dummy_bundle(struct net *net, 1865 struct dst_entry *dst, 1866 const struct flowi *fl, 1867 int num_xfrms, 1868 u16 family) 1869 { 1870 int err; 1871 struct net_device *dev; 1872 struct dst_entry *dst1; 1873 struct xfrm_dst *xdst; 1874 1875 xdst = xfrm_alloc_dst(net, family); 1876 if (IS_ERR(xdst)) 1877 return xdst; 1878 1879 if (net->xfrm.sysctl_larval_drop || num_xfrms <= 0 || 1880 (fl->flowi_flags & FLOWI_FLAG_CAN_SLEEP)) 1881 return xdst; 1882 1883 dst1 = &xdst->u.dst; 1884 dst_hold(dst); 1885 xdst->route = dst; 1886 1887 dst_copy_metrics(dst1, dst); 1888 1889 dst1->obsolete = DST_OBSOLETE_FORCE_CHK; 1890 dst1->flags |= DST_HOST | DST_XFRM_QUEUE; 1891 dst1->lastuse = jiffies; 1892 1893 dst1->input = dst_discard; 1894 dst1->output = xdst_queue_output; 1895 1896 dst_hold(dst); 1897 dst1->child = dst; 1898 dst1->path = dst; 1899 1900 xfrm_init_path((struct xfrm_dst *)dst1, dst, 0); 1901 1902 err = -ENODEV; 1903 dev = dst->dev; 1904 if (!dev) 1905 goto free_dst; 1906 1907 err = xfrm_fill_dst(xdst, dev, fl); 1908 if (err) 1909 goto free_dst; 1910 1911 out: 1912 return xdst; 1913 1914 free_dst: 1915 dst_release(dst1); 1916 xdst = ERR_PTR(err); 1917 goto out; 1918 } 1919 1920 static struct flow_cache_object * 1921 xfrm_bundle_lookup(struct net *net, const struct flowi *fl, u16 family, u8 dir, 1922 struct flow_cache_object *oldflo, void *ctx) 1923 { 1924 struct dst_entry *dst_orig = (struct dst_entry *)ctx; 1925 struct xfrm_policy *pols[XFRM_POLICY_TYPE_MAX]; 1926 struct xfrm_dst *xdst, *new_xdst; 1927 int num_pols = 0, num_xfrms = 0, i, err, pol_dead; 1928 1929 /* Check if the policies from old bundle are usable */ 1930 xdst = NULL; 1931 if (oldflo) { 1932 xdst = container_of(oldflo, struct xfrm_dst, flo); 1933 num_pols = xdst->num_pols; 1934 num_xfrms = xdst->num_xfrms; 1935 pol_dead = 0; 1936 for (i = 0; i < num_pols; i++) { 1937 pols[i] = xdst->pols[i]; 1938 pol_dead |= pols[i]->walk.dead; 1939 } 1940 if (pol_dead) { 1941 dst_free(&xdst->u.dst); 1942 xdst = NULL; 1943 num_pols = 0; 1944 num_xfrms = 0; 1945 oldflo = NULL; 1946 } 1947 } 1948 1949 /* Resolve policies to use if we couldn't get them from 1950 * previous cache entry */ 1951 if (xdst == NULL) { 1952 num_pols = 1; 1953 pols[0] = __xfrm_policy_lookup(net, fl, family, 1954 flow_to_policy_dir(dir)); 1955 err = xfrm_expand_policies(fl, family, pols, 1956 &num_pols, &num_xfrms); 1957 if (err < 0) 1958 goto inc_error; 1959 if (num_pols == 0) 1960 return NULL; 1961 if (num_xfrms <= 0) 1962 goto make_dummy_bundle; 1963 } 1964 1965 new_xdst = xfrm_resolve_and_create_bundle(pols, num_pols, fl, family, dst_orig); 1966 if (IS_ERR(new_xdst)) { 1967 err = PTR_ERR(new_xdst); 1968 if (err != -EAGAIN) 1969 goto error; 1970 if (oldflo == NULL) 1971 goto make_dummy_bundle; 1972 dst_hold(&xdst->u.dst); 1973 return oldflo; 1974 } else if (new_xdst == NULL) { 1975 num_xfrms = 0; 1976 if (oldflo == NULL) 1977 goto make_dummy_bundle; 1978 xdst->num_xfrms = 0; 1979 dst_hold(&xdst->u.dst); 1980 return oldflo; 1981 } 1982 1983 /* Kill the previous bundle */ 1984 if (xdst) { 1985 /* The policies were stolen for newly generated bundle */ 1986 xdst->num_pols = 0; 1987 dst_free(&xdst->u.dst); 1988 } 1989 1990 /* Flow cache does not have reference, it dst_free()'s, 1991 * but we do need to return one reference for original caller */ 1992 dst_hold(&new_xdst->u.dst); 1993 return &new_xdst->flo; 1994 1995 make_dummy_bundle: 1996 /* We found policies, but there's no bundles to instantiate: 1997 * either because the policy blocks, has no transformations or 1998 * we could not build template (no xfrm_states).*/ 1999 xdst = xfrm_create_dummy_bundle(net, dst_orig, fl, num_xfrms, family); 2000 if (IS_ERR(xdst)) { 2001 xfrm_pols_put(pols, num_pols); 2002 return ERR_CAST(xdst); 2003 } 2004 xdst->num_pols = num_pols; 2005 xdst->num_xfrms = num_xfrms; 2006 memcpy(xdst->pols, pols, sizeof(struct xfrm_policy*) * num_pols); 2007 2008 dst_hold(&xdst->u.dst); 2009 return &xdst->flo; 2010 2011 inc_error: 2012 XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTPOLERROR); 2013 error: 2014 if (xdst != NULL) 2015 dst_free(&xdst->u.dst); 2016 else 2017 xfrm_pols_put(pols, num_pols); 2018 return ERR_PTR(err); 2019 } 2020 2021 static struct dst_entry *make_blackhole(struct net *net, u16 family, 2022 struct dst_entry *dst_orig) 2023 { 2024 struct xfrm_policy_afinfo *afinfo = xfrm_policy_get_afinfo(family); 2025 struct dst_entry *ret; 2026 2027 if (!afinfo) { 2028 dst_release(dst_orig); 2029 return ERR_PTR(-EINVAL); 2030 } else { 2031 ret = afinfo->blackhole_route(net, dst_orig); 2032 } 2033 xfrm_policy_put_afinfo(afinfo); 2034 2035 return ret; 2036 } 2037 2038 /* Main function: finds/creates a bundle for given flow. 2039 * 2040 * At the moment we eat a raw IP route. Mostly to speed up lookups 2041 * on interfaces with disabled IPsec. 2042 */ 2043 struct dst_entry *xfrm_lookup(struct net *net, struct dst_entry *dst_orig, 2044 const struct flowi *fl, 2045 struct sock *sk, int flags) 2046 { 2047 struct xfrm_policy *pols[XFRM_POLICY_TYPE_MAX]; 2048 struct flow_cache_object *flo; 2049 struct xfrm_dst *xdst; 2050 struct dst_entry *dst, *route; 2051 u16 family = dst_orig->ops->family; 2052 u8 dir = policy_to_flow_dir(XFRM_POLICY_OUT); 2053 int i, err, num_pols, num_xfrms = 0, drop_pols = 0; 2054 2055 restart: 2056 dst = NULL; 2057 xdst = NULL; 2058 route = NULL; 2059 2060 if (sk && sk->sk_policy[XFRM_POLICY_OUT]) { 2061 num_pols = 1; 2062 pols[0] = xfrm_sk_policy_lookup(sk, XFRM_POLICY_OUT, fl); 2063 err = xfrm_expand_policies(fl, family, pols, 2064 &num_pols, &num_xfrms); 2065 if (err < 0) 2066 goto dropdst; 2067 2068 if (num_pols) { 2069 if (num_xfrms <= 0) { 2070 drop_pols = num_pols; 2071 goto no_transform; 2072 } 2073 2074 xdst = xfrm_resolve_and_create_bundle( 2075 pols, num_pols, fl, 2076 family, dst_orig); 2077 if (IS_ERR(xdst)) { 2078 xfrm_pols_put(pols, num_pols); 2079 err = PTR_ERR(xdst); 2080 goto dropdst; 2081 } else if (xdst == NULL) { 2082 num_xfrms = 0; 2083 drop_pols = num_pols; 2084 goto no_transform; 2085 } 2086 2087 dst_hold(&xdst->u.dst); 2088 2089 spin_lock_bh(&xfrm_policy_sk_bundle_lock); 2090 xdst->u.dst.next = xfrm_policy_sk_bundles; 2091 xfrm_policy_sk_bundles = &xdst->u.dst; 2092 spin_unlock_bh(&xfrm_policy_sk_bundle_lock); 2093 2094 route = xdst->route; 2095 } 2096 } 2097 2098 if (xdst == NULL) { 2099 /* To accelerate a bit... */ 2100 if ((dst_orig->flags & DST_NOXFRM) || 2101 !net->xfrm.policy_count[XFRM_POLICY_OUT]) 2102 goto nopol; 2103 2104 flo = flow_cache_lookup(net, fl, family, dir, 2105 xfrm_bundle_lookup, dst_orig); 2106 if (flo == NULL) 2107 goto nopol; 2108 if (IS_ERR(flo)) { 2109 err = PTR_ERR(flo); 2110 goto dropdst; 2111 } 2112 xdst = container_of(flo, struct xfrm_dst, flo); 2113 2114 num_pols = xdst->num_pols; 2115 num_xfrms = xdst->num_xfrms; 2116 memcpy(pols, xdst->pols, sizeof(struct xfrm_policy*) * num_pols); 2117 route = xdst->route; 2118 } 2119 2120 dst = &xdst->u.dst; 2121 if (route == NULL && num_xfrms > 0) { 2122 /* The only case when xfrm_bundle_lookup() returns a 2123 * bundle with null route, is when the template could 2124 * not be resolved. It means policies are there, but 2125 * bundle could not be created, since we don't yet 2126 * have the xfrm_state's. We need to wait for KM to 2127 * negotiate new SA's or bail out with error.*/ 2128 if (net->xfrm.sysctl_larval_drop) { 2129 /* EREMOTE tells the caller to generate 2130 * a one-shot blackhole route. */ 2131 dst_release(dst); 2132 xfrm_pols_put(pols, drop_pols); 2133 XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTNOSTATES); 2134 2135 return make_blackhole(net, family, dst_orig); 2136 } 2137 if (fl->flowi_flags & FLOWI_FLAG_CAN_SLEEP) { 2138 DECLARE_WAITQUEUE(wait, current); 2139 2140 add_wait_queue(&net->xfrm.km_waitq, &wait); 2141 set_current_state(TASK_INTERRUPTIBLE); 2142 schedule(); 2143 set_current_state(TASK_RUNNING); 2144 remove_wait_queue(&net->xfrm.km_waitq, &wait); 2145 2146 if (!signal_pending(current)) { 2147 dst_release(dst); 2148 goto restart; 2149 } 2150 2151 err = -ERESTART; 2152 } else 2153 err = -EAGAIN; 2154 2155 XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTNOSTATES); 2156 goto error; 2157 } 2158 2159 no_transform: 2160 if (num_pols == 0) 2161 goto nopol; 2162 2163 if ((flags & XFRM_LOOKUP_ICMP) && 2164 !(pols[0]->flags & XFRM_POLICY_ICMP)) { 2165 err = -ENOENT; 2166 goto error; 2167 } 2168 2169 for (i = 0; i < num_pols; i++) 2170 pols[i]->curlft.use_time = get_seconds(); 2171 2172 if (num_xfrms < 0) { 2173 /* Prohibit the flow */ 2174 XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTPOLBLOCK); 2175 err = -EPERM; 2176 goto error; 2177 } else if (num_xfrms > 0) { 2178 /* Flow transformed */ 2179 dst_release(dst_orig); 2180 } else { 2181 /* Flow passes untransformed */ 2182 dst_release(dst); 2183 dst = dst_orig; 2184 } 2185 ok: 2186 xfrm_pols_put(pols, drop_pols); 2187 if (dst && dst->xfrm && 2188 dst->xfrm->props.mode == XFRM_MODE_TUNNEL) 2189 dst->flags |= DST_XFRM_TUNNEL; 2190 return dst; 2191 2192 nopol: 2193 if (!(flags & XFRM_LOOKUP_ICMP)) { 2194 dst = dst_orig; 2195 goto ok; 2196 } 2197 err = -ENOENT; 2198 error: 2199 dst_release(dst); 2200 dropdst: 2201 dst_release(dst_orig); 2202 xfrm_pols_put(pols, drop_pols); 2203 return ERR_PTR(err); 2204 } 2205 EXPORT_SYMBOL(xfrm_lookup); 2206 2207 static inline int 2208 xfrm_secpath_reject(int idx, struct sk_buff *skb, const struct flowi *fl) 2209 { 2210 struct xfrm_state *x; 2211 2212 if (!skb->sp || idx < 0 || idx >= skb->sp->len) 2213 return 0; 2214 x = skb->sp->xvec[idx]; 2215 if (!x->type->reject) 2216 return 0; 2217 return x->type->reject(x, skb, fl); 2218 } 2219 2220 /* When skb is transformed back to its "native" form, we have to 2221 * check policy restrictions. At the moment we make this in maximally 2222 * stupid way. Shame on me. :-) Of course, connected sockets must 2223 * have policy cached at them. 2224 */ 2225 2226 static inline int 2227 xfrm_state_ok(const struct xfrm_tmpl *tmpl, const struct xfrm_state *x, 2228 unsigned short family) 2229 { 2230 if (xfrm_state_kern(x)) 2231 return tmpl->optional && !xfrm_state_addr_cmp(tmpl, x, tmpl->encap_family); 2232 return x->id.proto == tmpl->id.proto && 2233 (x->id.spi == tmpl->id.spi || !tmpl->id.spi) && 2234 (x->props.reqid == tmpl->reqid || !tmpl->reqid) && 2235 x->props.mode == tmpl->mode && 2236 (tmpl->allalgs || (tmpl->aalgos & (1<<x->props.aalgo)) || 2237 !(xfrm_id_proto_match(tmpl->id.proto, IPSEC_PROTO_ANY))) && 2238 !(x->props.mode != XFRM_MODE_TRANSPORT && 2239 xfrm_state_addr_cmp(tmpl, x, family)); 2240 } 2241 2242 /* 2243 * 0 or more than 0 is returned when validation is succeeded (either bypass 2244 * because of optional transport mode, or next index of the mathced secpath 2245 * state with the template. 2246 * -1 is returned when no matching template is found. 2247 * Otherwise "-2 - errored_index" is returned. 2248 */ 2249 static inline int 2250 xfrm_policy_ok(const struct xfrm_tmpl *tmpl, const struct sec_path *sp, int start, 2251 unsigned short family) 2252 { 2253 int idx = start; 2254 2255 if (tmpl->optional) { 2256 if (tmpl->mode == XFRM_MODE_TRANSPORT) 2257 return start; 2258 } else 2259 start = -1; 2260 for (; idx < sp->len; idx++) { 2261 if (xfrm_state_ok(tmpl, sp->xvec[idx], family)) 2262 return ++idx; 2263 if (sp->xvec[idx]->props.mode != XFRM_MODE_TRANSPORT) { 2264 if (start == -1) 2265 start = -2-idx; 2266 break; 2267 } 2268 } 2269 return start; 2270 } 2271 2272 int __xfrm_decode_session(struct sk_buff *skb, struct flowi *fl, 2273 unsigned int family, int reverse) 2274 { 2275 struct xfrm_policy_afinfo *afinfo = xfrm_policy_get_afinfo(family); 2276 int err; 2277 2278 if (unlikely(afinfo == NULL)) 2279 return -EAFNOSUPPORT; 2280 2281 afinfo->decode_session(skb, fl, reverse); 2282 err = security_xfrm_decode_session(skb, &fl->flowi_secid); 2283 xfrm_policy_put_afinfo(afinfo); 2284 return err; 2285 } 2286 EXPORT_SYMBOL(__xfrm_decode_session); 2287 2288 static inline int secpath_has_nontransport(const struct sec_path *sp, int k, int *idxp) 2289 { 2290 for (; k < sp->len; k++) { 2291 if (sp->xvec[k]->props.mode != XFRM_MODE_TRANSPORT) { 2292 *idxp = k; 2293 return 1; 2294 } 2295 } 2296 2297 return 0; 2298 } 2299 2300 int __xfrm_policy_check(struct sock *sk, int dir, struct sk_buff *skb, 2301 unsigned short family) 2302 { 2303 struct net *net = dev_net(skb->dev); 2304 struct xfrm_policy *pol; 2305 struct xfrm_policy *pols[XFRM_POLICY_TYPE_MAX]; 2306 int npols = 0; 2307 int xfrm_nr; 2308 int pi; 2309 int reverse; 2310 struct flowi fl; 2311 u8 fl_dir; 2312 int xerr_idx = -1; 2313 2314 reverse = dir & ~XFRM_POLICY_MASK; 2315 dir &= XFRM_POLICY_MASK; 2316 fl_dir = policy_to_flow_dir(dir); 2317 2318 if (__xfrm_decode_session(skb, &fl, family, reverse) < 0) { 2319 XFRM_INC_STATS(net, LINUX_MIB_XFRMINHDRERROR); 2320 return 0; 2321 } 2322 2323 nf_nat_decode_session(skb, &fl, family); 2324 2325 /* First, check used SA against their selectors. */ 2326 if (skb->sp) { 2327 int i; 2328 2329 for (i=skb->sp->len-1; i>=0; i--) { 2330 struct xfrm_state *x = skb->sp->xvec[i]; 2331 if (!xfrm_selector_match(&x->sel, &fl, family)) { 2332 XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEMISMATCH); 2333 return 0; 2334 } 2335 } 2336 } 2337 2338 pol = NULL; 2339 if (sk && sk->sk_policy[dir]) { 2340 pol = xfrm_sk_policy_lookup(sk, dir, &fl); 2341 if (IS_ERR(pol)) { 2342 XFRM_INC_STATS(net, LINUX_MIB_XFRMINPOLERROR); 2343 return 0; 2344 } 2345 } 2346 2347 if (!pol) { 2348 struct flow_cache_object *flo; 2349 2350 flo = flow_cache_lookup(net, &fl, family, fl_dir, 2351 xfrm_policy_lookup, NULL); 2352 if (IS_ERR_OR_NULL(flo)) 2353 pol = ERR_CAST(flo); 2354 else 2355 pol = container_of(flo, struct xfrm_policy, flo); 2356 } 2357 2358 if (IS_ERR(pol)) { 2359 XFRM_INC_STATS(net, LINUX_MIB_XFRMINPOLERROR); 2360 return 0; 2361 } 2362 2363 if (!pol) { 2364 if (skb->sp && secpath_has_nontransport(skb->sp, 0, &xerr_idx)) { 2365 xfrm_secpath_reject(xerr_idx, skb, &fl); 2366 XFRM_INC_STATS(net, LINUX_MIB_XFRMINNOPOLS); 2367 return 0; 2368 } 2369 return 1; 2370 } 2371 2372 pol->curlft.use_time = get_seconds(); 2373 2374 pols[0] = pol; 2375 npols ++; 2376 #ifdef CONFIG_XFRM_SUB_POLICY 2377 if (pols[0]->type != XFRM_POLICY_TYPE_MAIN) { 2378 pols[1] = xfrm_policy_lookup_bytype(net, XFRM_POLICY_TYPE_MAIN, 2379 &fl, family, 2380 XFRM_POLICY_IN); 2381 if (pols[1]) { 2382 if (IS_ERR(pols[1])) { 2383 XFRM_INC_STATS(net, LINUX_MIB_XFRMINPOLERROR); 2384 return 0; 2385 } 2386 pols[1]->curlft.use_time = get_seconds(); 2387 npols ++; 2388 } 2389 } 2390 #endif 2391 2392 if (pol->action == XFRM_POLICY_ALLOW) { 2393 struct sec_path *sp; 2394 static struct sec_path dummy; 2395 struct xfrm_tmpl *tp[XFRM_MAX_DEPTH]; 2396 struct xfrm_tmpl *stp[XFRM_MAX_DEPTH]; 2397 struct xfrm_tmpl **tpp = tp; 2398 int ti = 0; 2399 int i, k; 2400 2401 if ((sp = skb->sp) == NULL) 2402 sp = &dummy; 2403 2404 for (pi = 0; pi < npols; pi++) { 2405 if (pols[pi] != pol && 2406 pols[pi]->action != XFRM_POLICY_ALLOW) { 2407 XFRM_INC_STATS(net, LINUX_MIB_XFRMINPOLBLOCK); 2408 goto reject; 2409 } 2410 if (ti + pols[pi]->xfrm_nr >= XFRM_MAX_DEPTH) { 2411 XFRM_INC_STATS(net, LINUX_MIB_XFRMINBUFFERERROR); 2412 goto reject_error; 2413 } 2414 for (i = 0; i < pols[pi]->xfrm_nr; i++) 2415 tpp[ti++] = &pols[pi]->xfrm_vec[i]; 2416 } 2417 xfrm_nr = ti; 2418 if (npols > 1) { 2419 xfrm_tmpl_sort(stp, tpp, xfrm_nr, family); 2420 tpp = stp; 2421 } 2422 2423 /* For each tunnel xfrm, find the first matching tmpl. 2424 * For each tmpl before that, find corresponding xfrm. 2425 * Order is _important_. Later we will implement 2426 * some barriers, but at the moment barriers 2427 * are implied between each two transformations. 2428 */ 2429 for (i = xfrm_nr-1, k = 0; i >= 0; i--) { 2430 k = xfrm_policy_ok(tpp[i], sp, k, family); 2431 if (k < 0) { 2432 if (k < -1) 2433 /* "-2 - errored_index" returned */ 2434 xerr_idx = -(2+k); 2435 XFRM_INC_STATS(net, LINUX_MIB_XFRMINTMPLMISMATCH); 2436 goto reject; 2437 } 2438 } 2439 2440 if (secpath_has_nontransport(sp, k, &xerr_idx)) { 2441 XFRM_INC_STATS(net, LINUX_MIB_XFRMINTMPLMISMATCH); 2442 goto reject; 2443 } 2444 2445 xfrm_pols_put(pols, npols); 2446 return 1; 2447 } 2448 XFRM_INC_STATS(net, LINUX_MIB_XFRMINPOLBLOCK); 2449 2450 reject: 2451 xfrm_secpath_reject(xerr_idx, skb, &fl); 2452 reject_error: 2453 xfrm_pols_put(pols, npols); 2454 return 0; 2455 } 2456 EXPORT_SYMBOL(__xfrm_policy_check); 2457 2458 int __xfrm_route_forward(struct sk_buff *skb, unsigned short family) 2459 { 2460 struct net *net = dev_net(skb->dev); 2461 struct flowi fl; 2462 struct dst_entry *dst; 2463 int res = 1; 2464 2465 if (xfrm_decode_session(skb, &fl, family) < 0) { 2466 XFRM_INC_STATS(net, LINUX_MIB_XFRMFWDHDRERROR); 2467 return 0; 2468 } 2469 2470 skb_dst_force(skb); 2471 2472 dst = xfrm_lookup(net, skb_dst(skb), &fl, NULL, 0); 2473 if (IS_ERR(dst)) { 2474 res = 0; 2475 dst = NULL; 2476 } 2477 skb_dst_set(skb, dst); 2478 return res; 2479 } 2480 EXPORT_SYMBOL(__xfrm_route_forward); 2481 2482 /* Optimize later using cookies and generation ids. */ 2483 2484 static struct dst_entry *xfrm_dst_check(struct dst_entry *dst, u32 cookie) 2485 { 2486 /* Code (such as __xfrm4_bundle_create()) sets dst->obsolete 2487 * to DST_OBSOLETE_FORCE_CHK to force all XFRM destinations to 2488 * get validated by dst_ops->check on every use. We do this 2489 * because when a normal route referenced by an XFRM dst is 2490 * obsoleted we do not go looking around for all parent 2491 * referencing XFRM dsts so that we can invalidate them. It 2492 * is just too much work. Instead we make the checks here on 2493 * every use. For example: 2494 * 2495 * XFRM dst A --> IPv4 dst X 2496 * 2497 * X is the "xdst->route" of A (X is also the "dst->path" of A 2498 * in this example). If X is marked obsolete, "A" will not 2499 * notice. That's what we are validating here via the 2500 * stale_bundle() check. 2501 * 2502 * When a policy's bundle is pruned, we dst_free() the XFRM 2503 * dst which causes it's ->obsolete field to be set to 2504 * DST_OBSOLETE_DEAD. If an XFRM dst has been pruned like 2505 * this, we want to force a new route lookup. 2506 */ 2507 if (dst->obsolete < 0 && !stale_bundle(dst)) 2508 return dst; 2509 2510 return NULL; 2511 } 2512 2513 static int stale_bundle(struct dst_entry *dst) 2514 { 2515 return !xfrm_bundle_ok((struct xfrm_dst *)dst); 2516 } 2517 2518 void xfrm_dst_ifdown(struct dst_entry *dst, struct net_device *dev) 2519 { 2520 while ((dst = dst->child) && dst->xfrm && dst->dev == dev) { 2521 dst->dev = dev_net(dev)->loopback_dev; 2522 dev_hold(dst->dev); 2523 dev_put(dev); 2524 } 2525 } 2526 EXPORT_SYMBOL(xfrm_dst_ifdown); 2527 2528 static void xfrm_link_failure(struct sk_buff *skb) 2529 { 2530 /* Impossible. Such dst must be popped before reaches point of failure. */ 2531 } 2532 2533 static struct dst_entry *xfrm_negative_advice(struct dst_entry *dst) 2534 { 2535 if (dst) { 2536 if (dst->obsolete) { 2537 dst_release(dst); 2538 dst = NULL; 2539 } 2540 } 2541 return dst; 2542 } 2543 2544 static void __xfrm_garbage_collect(struct net *net) 2545 { 2546 struct dst_entry *head, *next; 2547 2548 spin_lock_bh(&xfrm_policy_sk_bundle_lock); 2549 head = xfrm_policy_sk_bundles; 2550 xfrm_policy_sk_bundles = NULL; 2551 spin_unlock_bh(&xfrm_policy_sk_bundle_lock); 2552 2553 while (head) { 2554 next = head->next; 2555 dst_free(head); 2556 head = next; 2557 } 2558 } 2559 2560 static void xfrm_garbage_collect(struct net *net) 2561 { 2562 flow_cache_flush(); 2563 __xfrm_garbage_collect(net); 2564 } 2565 2566 static void xfrm_garbage_collect_deferred(struct net *net) 2567 { 2568 flow_cache_flush_deferred(); 2569 __xfrm_garbage_collect(net); 2570 } 2571 2572 static void xfrm_init_pmtu(struct dst_entry *dst) 2573 { 2574 do { 2575 struct xfrm_dst *xdst = (struct xfrm_dst *)dst; 2576 u32 pmtu, route_mtu_cached; 2577 2578 pmtu = dst_mtu(dst->child); 2579 xdst->child_mtu_cached = pmtu; 2580 2581 pmtu = xfrm_state_mtu(dst->xfrm, pmtu); 2582 2583 route_mtu_cached = dst_mtu(xdst->route); 2584 xdst->route_mtu_cached = route_mtu_cached; 2585 2586 if (pmtu > route_mtu_cached) 2587 pmtu = route_mtu_cached; 2588 2589 dst_metric_set(dst, RTAX_MTU, pmtu); 2590 } while ((dst = dst->next)); 2591 } 2592 2593 /* Check that the bundle accepts the flow and its components are 2594 * still valid. 2595 */ 2596 2597 static int xfrm_bundle_ok(struct xfrm_dst *first) 2598 { 2599 struct dst_entry *dst = &first->u.dst; 2600 struct xfrm_dst *last; 2601 u32 mtu; 2602 2603 if (!dst_check(dst->path, ((struct xfrm_dst *)dst)->path_cookie) || 2604 (dst->dev && !netif_running(dst->dev))) 2605 return 0; 2606 2607 if (dst->flags & DST_XFRM_QUEUE) 2608 return 1; 2609 2610 last = NULL; 2611 2612 do { 2613 struct xfrm_dst *xdst = (struct xfrm_dst *)dst; 2614 2615 if (dst->xfrm->km.state != XFRM_STATE_VALID) 2616 return 0; 2617 if (xdst->xfrm_genid != dst->xfrm->genid) 2618 return 0; 2619 if (xdst->num_pols > 0 && 2620 xdst->policy_genid != atomic_read(&xdst->pols[0]->genid)) 2621 return 0; 2622 2623 mtu = dst_mtu(dst->child); 2624 if (xdst->child_mtu_cached != mtu) { 2625 last = xdst; 2626 xdst->child_mtu_cached = mtu; 2627 } 2628 2629 if (!dst_check(xdst->route, xdst->route_cookie)) 2630 return 0; 2631 mtu = dst_mtu(xdst->route); 2632 if (xdst->route_mtu_cached != mtu) { 2633 last = xdst; 2634 xdst->route_mtu_cached = mtu; 2635 } 2636 2637 dst = dst->child; 2638 } while (dst->xfrm); 2639 2640 if (likely(!last)) 2641 return 1; 2642 2643 mtu = last->child_mtu_cached; 2644 for (;;) { 2645 dst = &last->u.dst; 2646 2647 mtu = xfrm_state_mtu(dst->xfrm, mtu); 2648 if (mtu > last->route_mtu_cached) 2649 mtu = last->route_mtu_cached; 2650 dst_metric_set(dst, RTAX_MTU, mtu); 2651 2652 if (last == first) 2653 break; 2654 2655 last = (struct xfrm_dst *)last->u.dst.next; 2656 last->child_mtu_cached = mtu; 2657 } 2658 2659 return 1; 2660 } 2661 2662 static unsigned int xfrm_default_advmss(const struct dst_entry *dst) 2663 { 2664 return dst_metric_advmss(dst->path); 2665 } 2666 2667 static unsigned int xfrm_mtu(const struct dst_entry *dst) 2668 { 2669 unsigned int mtu = dst_metric_raw(dst, RTAX_MTU); 2670 2671 return mtu ? : dst_mtu(dst->path); 2672 } 2673 2674 static struct neighbour *xfrm_neigh_lookup(const struct dst_entry *dst, 2675 struct sk_buff *skb, 2676 const void *daddr) 2677 { 2678 return dst->path->ops->neigh_lookup(dst, skb, daddr); 2679 } 2680 2681 int xfrm_policy_register_afinfo(struct xfrm_policy_afinfo *afinfo) 2682 { 2683 struct net *net; 2684 int err = 0; 2685 if (unlikely(afinfo == NULL)) 2686 return -EINVAL; 2687 if (unlikely(afinfo->family >= NPROTO)) 2688 return -EAFNOSUPPORT; 2689 spin_lock(&xfrm_policy_afinfo_lock); 2690 if (unlikely(xfrm_policy_afinfo[afinfo->family] != NULL)) 2691 err = -ENOBUFS; 2692 else { 2693 struct dst_ops *dst_ops = afinfo->dst_ops; 2694 if (likely(dst_ops->kmem_cachep == NULL)) 2695 dst_ops->kmem_cachep = xfrm_dst_cache; 2696 if (likely(dst_ops->check == NULL)) 2697 dst_ops->check = xfrm_dst_check; 2698 if (likely(dst_ops->default_advmss == NULL)) 2699 dst_ops->default_advmss = xfrm_default_advmss; 2700 if (likely(dst_ops->mtu == NULL)) 2701 dst_ops->mtu = xfrm_mtu; 2702 if (likely(dst_ops->negative_advice == NULL)) 2703 dst_ops->negative_advice = xfrm_negative_advice; 2704 if (likely(dst_ops->link_failure == NULL)) 2705 dst_ops->link_failure = xfrm_link_failure; 2706 if (likely(dst_ops->neigh_lookup == NULL)) 2707 dst_ops->neigh_lookup = xfrm_neigh_lookup; 2708 if (likely(afinfo->garbage_collect == NULL)) 2709 afinfo->garbage_collect = xfrm_garbage_collect_deferred; 2710 rcu_assign_pointer(xfrm_policy_afinfo[afinfo->family], afinfo); 2711 } 2712 spin_unlock(&xfrm_policy_afinfo_lock); 2713 2714 rtnl_lock(); 2715 for_each_net(net) { 2716 struct dst_ops *xfrm_dst_ops; 2717 2718 switch (afinfo->family) { 2719 case AF_INET: 2720 xfrm_dst_ops = &net->xfrm.xfrm4_dst_ops; 2721 break; 2722 #if IS_ENABLED(CONFIG_IPV6) 2723 case AF_INET6: 2724 xfrm_dst_ops = &net->xfrm.xfrm6_dst_ops; 2725 break; 2726 #endif 2727 default: 2728 BUG(); 2729 } 2730 *xfrm_dst_ops = *afinfo->dst_ops; 2731 } 2732 rtnl_unlock(); 2733 2734 return err; 2735 } 2736 EXPORT_SYMBOL(xfrm_policy_register_afinfo); 2737 2738 int xfrm_policy_unregister_afinfo(struct xfrm_policy_afinfo *afinfo) 2739 { 2740 int err = 0; 2741 if (unlikely(afinfo == NULL)) 2742 return -EINVAL; 2743 if (unlikely(afinfo->family >= NPROTO)) 2744 return -EAFNOSUPPORT; 2745 spin_lock(&xfrm_policy_afinfo_lock); 2746 if (likely(xfrm_policy_afinfo[afinfo->family] != NULL)) { 2747 if (unlikely(xfrm_policy_afinfo[afinfo->family] != afinfo)) 2748 err = -EINVAL; 2749 else 2750 RCU_INIT_POINTER(xfrm_policy_afinfo[afinfo->family], 2751 NULL); 2752 } 2753 spin_unlock(&xfrm_policy_afinfo_lock); 2754 if (!err) { 2755 struct dst_ops *dst_ops = afinfo->dst_ops; 2756 2757 synchronize_rcu(); 2758 2759 dst_ops->kmem_cachep = NULL; 2760 dst_ops->check = NULL; 2761 dst_ops->negative_advice = NULL; 2762 dst_ops->link_failure = NULL; 2763 afinfo->garbage_collect = NULL; 2764 } 2765 return err; 2766 } 2767 EXPORT_SYMBOL(xfrm_policy_unregister_afinfo); 2768 2769 static void __net_init xfrm_dst_ops_init(struct net *net) 2770 { 2771 struct xfrm_policy_afinfo *afinfo; 2772 2773 rcu_read_lock(); 2774 afinfo = rcu_dereference(xfrm_policy_afinfo[AF_INET]); 2775 if (afinfo) 2776 net->xfrm.xfrm4_dst_ops = *afinfo->dst_ops; 2777 #if IS_ENABLED(CONFIG_IPV6) 2778 afinfo = rcu_dereference(xfrm_policy_afinfo[AF_INET6]); 2779 if (afinfo) 2780 net->xfrm.xfrm6_dst_ops = *afinfo->dst_ops; 2781 #endif 2782 rcu_read_unlock(); 2783 } 2784 2785 static int xfrm_dev_event(struct notifier_block *this, unsigned long event, void *ptr) 2786 { 2787 struct net_device *dev = ptr; 2788 2789 switch (event) { 2790 case NETDEV_DOWN: 2791 xfrm_garbage_collect(dev_net(dev)); 2792 } 2793 return NOTIFY_DONE; 2794 } 2795 2796 static struct notifier_block xfrm_dev_notifier = { 2797 .notifier_call = xfrm_dev_event, 2798 }; 2799 2800 #ifdef CONFIG_XFRM_STATISTICS 2801 static int __net_init xfrm_statistics_init(struct net *net) 2802 { 2803 int rv; 2804 2805 if (snmp_mib_init((void __percpu **)net->mib.xfrm_statistics, 2806 sizeof(struct linux_xfrm_mib), 2807 __alignof__(struct linux_xfrm_mib)) < 0) 2808 return -ENOMEM; 2809 rv = xfrm_proc_init(net); 2810 if (rv < 0) 2811 snmp_mib_free((void __percpu **)net->mib.xfrm_statistics); 2812 return rv; 2813 } 2814 2815 static void xfrm_statistics_fini(struct net *net) 2816 { 2817 xfrm_proc_fini(net); 2818 snmp_mib_free((void __percpu **)net->mib.xfrm_statistics); 2819 } 2820 #else 2821 static int __net_init xfrm_statistics_init(struct net *net) 2822 { 2823 return 0; 2824 } 2825 2826 static void xfrm_statistics_fini(struct net *net) 2827 { 2828 } 2829 #endif 2830 2831 static int __net_init xfrm_policy_init(struct net *net) 2832 { 2833 unsigned int hmask, sz; 2834 int dir; 2835 2836 if (net_eq(net, &init_net)) 2837 xfrm_dst_cache = kmem_cache_create("xfrm_dst_cache", 2838 sizeof(struct xfrm_dst), 2839 0, SLAB_HWCACHE_ALIGN|SLAB_PANIC, 2840 NULL); 2841 2842 hmask = 8 - 1; 2843 sz = (hmask+1) * sizeof(struct hlist_head); 2844 2845 net->xfrm.policy_byidx = xfrm_hash_alloc(sz); 2846 if (!net->xfrm.policy_byidx) 2847 goto out_byidx; 2848 net->xfrm.policy_idx_hmask = hmask; 2849 2850 for (dir = 0; dir < XFRM_POLICY_MAX * 2; dir++) { 2851 struct xfrm_policy_hash *htab; 2852 2853 net->xfrm.policy_count[dir] = 0; 2854 INIT_HLIST_HEAD(&net->xfrm.policy_inexact[dir]); 2855 2856 htab = &net->xfrm.policy_bydst[dir]; 2857 htab->table = xfrm_hash_alloc(sz); 2858 if (!htab->table) 2859 goto out_bydst; 2860 htab->hmask = hmask; 2861 } 2862 2863 INIT_LIST_HEAD(&net->xfrm.policy_all); 2864 INIT_WORK(&net->xfrm.policy_hash_work, xfrm_hash_resize); 2865 if (net_eq(net, &init_net)) 2866 register_netdevice_notifier(&xfrm_dev_notifier); 2867 return 0; 2868 2869 out_bydst: 2870 for (dir--; dir >= 0; dir--) { 2871 struct xfrm_policy_hash *htab; 2872 2873 htab = &net->xfrm.policy_bydst[dir]; 2874 xfrm_hash_free(htab->table, sz); 2875 } 2876 xfrm_hash_free(net->xfrm.policy_byidx, sz); 2877 out_byidx: 2878 return -ENOMEM; 2879 } 2880 2881 static void xfrm_policy_fini(struct net *net) 2882 { 2883 struct xfrm_audit audit_info; 2884 unsigned int sz; 2885 int dir; 2886 2887 flush_work(&net->xfrm.policy_hash_work); 2888 #ifdef CONFIG_XFRM_SUB_POLICY 2889 audit_info.loginuid = INVALID_UID; 2890 audit_info.sessionid = -1; 2891 audit_info.secid = 0; 2892 xfrm_policy_flush(net, XFRM_POLICY_TYPE_SUB, &audit_info); 2893 #endif 2894 audit_info.loginuid = INVALID_UID; 2895 audit_info.sessionid = -1; 2896 audit_info.secid = 0; 2897 xfrm_policy_flush(net, XFRM_POLICY_TYPE_MAIN, &audit_info); 2898 2899 WARN_ON(!list_empty(&net->xfrm.policy_all)); 2900 2901 for (dir = 0; dir < XFRM_POLICY_MAX * 2; dir++) { 2902 struct xfrm_policy_hash *htab; 2903 2904 WARN_ON(!hlist_empty(&net->xfrm.policy_inexact[dir])); 2905 2906 htab = &net->xfrm.policy_bydst[dir]; 2907 sz = (htab->hmask + 1) * sizeof(struct hlist_head); 2908 WARN_ON(!hlist_empty(htab->table)); 2909 xfrm_hash_free(htab->table, sz); 2910 } 2911 2912 sz = (net->xfrm.policy_idx_hmask + 1) * sizeof(struct hlist_head); 2913 WARN_ON(!hlist_empty(net->xfrm.policy_byidx)); 2914 xfrm_hash_free(net->xfrm.policy_byidx, sz); 2915 } 2916 2917 static int __net_init xfrm_net_init(struct net *net) 2918 { 2919 int rv; 2920 2921 rv = xfrm_statistics_init(net); 2922 if (rv < 0) 2923 goto out_statistics; 2924 rv = xfrm_state_init(net); 2925 if (rv < 0) 2926 goto out_state; 2927 rv = xfrm_policy_init(net); 2928 if (rv < 0) 2929 goto out_policy; 2930 xfrm_dst_ops_init(net); 2931 rv = xfrm_sysctl_init(net); 2932 if (rv < 0) 2933 goto out_sysctl; 2934 return 0; 2935 2936 out_sysctl: 2937 xfrm_policy_fini(net); 2938 out_policy: 2939 xfrm_state_fini(net); 2940 out_state: 2941 xfrm_statistics_fini(net); 2942 out_statistics: 2943 return rv; 2944 } 2945 2946 static void __net_exit xfrm_net_exit(struct net *net) 2947 { 2948 xfrm_sysctl_fini(net); 2949 xfrm_policy_fini(net); 2950 xfrm_state_fini(net); 2951 xfrm_statistics_fini(net); 2952 } 2953 2954 static struct pernet_operations __net_initdata xfrm_net_ops = { 2955 .init = xfrm_net_init, 2956 .exit = xfrm_net_exit, 2957 }; 2958 2959 void __init xfrm_init(void) 2960 { 2961 register_pernet_subsys(&xfrm_net_ops); 2962 xfrm_input_init(); 2963 } 2964 2965 #ifdef CONFIG_AUDITSYSCALL 2966 static void xfrm_audit_common_policyinfo(struct xfrm_policy *xp, 2967 struct audit_buffer *audit_buf) 2968 { 2969 struct xfrm_sec_ctx *ctx = xp->security; 2970 struct xfrm_selector *sel = &xp->selector; 2971 2972 if (ctx) 2973 audit_log_format(audit_buf, " sec_alg=%u sec_doi=%u sec_obj=%s", 2974 ctx->ctx_alg, ctx->ctx_doi, ctx->ctx_str); 2975 2976 switch(sel->family) { 2977 case AF_INET: 2978 audit_log_format(audit_buf, " src=%pI4", &sel->saddr.a4); 2979 if (sel->prefixlen_s != 32) 2980 audit_log_format(audit_buf, " src_prefixlen=%d", 2981 sel->prefixlen_s); 2982 audit_log_format(audit_buf, " dst=%pI4", &sel->daddr.a4); 2983 if (sel->prefixlen_d != 32) 2984 audit_log_format(audit_buf, " dst_prefixlen=%d", 2985 sel->prefixlen_d); 2986 break; 2987 case AF_INET6: 2988 audit_log_format(audit_buf, " src=%pI6", sel->saddr.a6); 2989 if (sel->prefixlen_s != 128) 2990 audit_log_format(audit_buf, " src_prefixlen=%d", 2991 sel->prefixlen_s); 2992 audit_log_format(audit_buf, " dst=%pI6", sel->daddr.a6); 2993 if (sel->prefixlen_d != 128) 2994 audit_log_format(audit_buf, " dst_prefixlen=%d", 2995 sel->prefixlen_d); 2996 break; 2997 } 2998 } 2999 3000 void xfrm_audit_policy_add(struct xfrm_policy *xp, int result, 3001 kuid_t auid, u32 sessionid, u32 secid) 3002 { 3003 struct audit_buffer *audit_buf; 3004 3005 audit_buf = xfrm_audit_start("SPD-add"); 3006 if (audit_buf == NULL) 3007 return; 3008 xfrm_audit_helper_usrinfo(auid, sessionid, secid, audit_buf); 3009 audit_log_format(audit_buf, " res=%u", result); 3010 xfrm_audit_common_policyinfo(xp, audit_buf); 3011 audit_log_end(audit_buf); 3012 } 3013 EXPORT_SYMBOL_GPL(xfrm_audit_policy_add); 3014 3015 void xfrm_audit_policy_delete(struct xfrm_policy *xp, int result, 3016 kuid_t auid, u32 sessionid, u32 secid) 3017 { 3018 struct audit_buffer *audit_buf; 3019 3020 audit_buf = xfrm_audit_start("SPD-delete"); 3021 if (audit_buf == NULL) 3022 return; 3023 xfrm_audit_helper_usrinfo(auid, sessionid, secid, audit_buf); 3024 audit_log_format(audit_buf, " res=%u", result); 3025 xfrm_audit_common_policyinfo(xp, audit_buf); 3026 audit_log_end(audit_buf); 3027 } 3028 EXPORT_SYMBOL_GPL(xfrm_audit_policy_delete); 3029 #endif 3030 3031 #ifdef CONFIG_XFRM_MIGRATE 3032 static bool xfrm_migrate_selector_match(const struct xfrm_selector *sel_cmp, 3033 const struct xfrm_selector *sel_tgt) 3034 { 3035 if (sel_cmp->proto == IPSEC_ULPROTO_ANY) { 3036 if (sel_tgt->family == sel_cmp->family && 3037 xfrm_addr_equal(&sel_tgt->daddr, &sel_cmp->daddr, 3038 sel_cmp->family) && 3039 xfrm_addr_equal(&sel_tgt->saddr, &sel_cmp->saddr, 3040 sel_cmp->family) && 3041 sel_tgt->prefixlen_d == sel_cmp->prefixlen_d && 3042 sel_tgt->prefixlen_s == sel_cmp->prefixlen_s) { 3043 return true; 3044 } 3045 } else { 3046 if (memcmp(sel_tgt, sel_cmp, sizeof(*sel_tgt)) == 0) { 3047 return true; 3048 } 3049 } 3050 return false; 3051 } 3052 3053 static struct xfrm_policy * xfrm_migrate_policy_find(const struct xfrm_selector *sel, 3054 u8 dir, u8 type) 3055 { 3056 struct xfrm_policy *pol, *ret = NULL; 3057 struct hlist_head *chain; 3058 u32 priority = ~0U; 3059 3060 read_lock_bh(&xfrm_policy_lock); 3061 chain = policy_hash_direct(&init_net, &sel->daddr, &sel->saddr, sel->family, dir); 3062 hlist_for_each_entry(pol, chain, bydst) { 3063 if (xfrm_migrate_selector_match(sel, &pol->selector) && 3064 pol->type == type) { 3065 ret = pol; 3066 priority = ret->priority; 3067 break; 3068 } 3069 } 3070 chain = &init_net.xfrm.policy_inexact[dir]; 3071 hlist_for_each_entry(pol, chain, bydst) { 3072 if (xfrm_migrate_selector_match(sel, &pol->selector) && 3073 pol->type == type && 3074 pol->priority < priority) { 3075 ret = pol; 3076 break; 3077 } 3078 } 3079 3080 if (ret) 3081 xfrm_pol_hold(ret); 3082 3083 read_unlock_bh(&xfrm_policy_lock); 3084 3085 return ret; 3086 } 3087 3088 static int migrate_tmpl_match(const struct xfrm_migrate *m, const struct xfrm_tmpl *t) 3089 { 3090 int match = 0; 3091 3092 if (t->mode == m->mode && t->id.proto == m->proto && 3093 (m->reqid == 0 || t->reqid == m->reqid)) { 3094 switch (t->mode) { 3095 case XFRM_MODE_TUNNEL: 3096 case XFRM_MODE_BEET: 3097 if (xfrm_addr_equal(&t->id.daddr, &m->old_daddr, 3098 m->old_family) && 3099 xfrm_addr_equal(&t->saddr, &m->old_saddr, 3100 m->old_family)) { 3101 match = 1; 3102 } 3103 break; 3104 case XFRM_MODE_TRANSPORT: 3105 /* in case of transport mode, template does not store 3106 any IP addresses, hence we just compare mode and 3107 protocol */ 3108 match = 1; 3109 break; 3110 default: 3111 break; 3112 } 3113 } 3114 return match; 3115 } 3116 3117 /* update endpoint address(es) of template(s) */ 3118 static int xfrm_policy_migrate(struct xfrm_policy *pol, 3119 struct xfrm_migrate *m, int num_migrate) 3120 { 3121 struct xfrm_migrate *mp; 3122 int i, j, n = 0; 3123 3124 write_lock_bh(&pol->lock); 3125 if (unlikely(pol->walk.dead)) { 3126 /* target policy has been deleted */ 3127 write_unlock_bh(&pol->lock); 3128 return -ENOENT; 3129 } 3130 3131 for (i = 0; i < pol->xfrm_nr; i++) { 3132 for (j = 0, mp = m; j < num_migrate; j++, mp++) { 3133 if (!migrate_tmpl_match(mp, &pol->xfrm_vec[i])) 3134 continue; 3135 n++; 3136 if (pol->xfrm_vec[i].mode != XFRM_MODE_TUNNEL && 3137 pol->xfrm_vec[i].mode != XFRM_MODE_BEET) 3138 continue; 3139 /* update endpoints */ 3140 memcpy(&pol->xfrm_vec[i].id.daddr, &mp->new_daddr, 3141 sizeof(pol->xfrm_vec[i].id.daddr)); 3142 memcpy(&pol->xfrm_vec[i].saddr, &mp->new_saddr, 3143 sizeof(pol->xfrm_vec[i].saddr)); 3144 pol->xfrm_vec[i].encap_family = mp->new_family; 3145 /* flush bundles */ 3146 atomic_inc(&pol->genid); 3147 } 3148 } 3149 3150 write_unlock_bh(&pol->lock); 3151 3152 if (!n) 3153 return -ENODATA; 3154 3155 return 0; 3156 } 3157 3158 static int xfrm_migrate_check(const struct xfrm_migrate *m, int num_migrate) 3159 { 3160 int i, j; 3161 3162 if (num_migrate < 1 || num_migrate > XFRM_MAX_DEPTH) 3163 return -EINVAL; 3164 3165 for (i = 0; i < num_migrate; i++) { 3166 if (xfrm_addr_equal(&m[i].old_daddr, &m[i].new_daddr, 3167 m[i].old_family) && 3168 xfrm_addr_equal(&m[i].old_saddr, &m[i].new_saddr, 3169 m[i].old_family)) 3170 return -EINVAL; 3171 if (xfrm_addr_any(&m[i].new_daddr, m[i].new_family) || 3172 xfrm_addr_any(&m[i].new_saddr, m[i].new_family)) 3173 return -EINVAL; 3174 3175 /* check if there is any duplicated entry */ 3176 for (j = i + 1; j < num_migrate; j++) { 3177 if (!memcmp(&m[i].old_daddr, &m[j].old_daddr, 3178 sizeof(m[i].old_daddr)) && 3179 !memcmp(&m[i].old_saddr, &m[j].old_saddr, 3180 sizeof(m[i].old_saddr)) && 3181 m[i].proto == m[j].proto && 3182 m[i].mode == m[j].mode && 3183 m[i].reqid == m[j].reqid && 3184 m[i].old_family == m[j].old_family) 3185 return -EINVAL; 3186 } 3187 } 3188 3189 return 0; 3190 } 3191 3192 int xfrm_migrate(const struct xfrm_selector *sel, u8 dir, u8 type, 3193 struct xfrm_migrate *m, int num_migrate, 3194 struct xfrm_kmaddress *k) 3195 { 3196 int i, err, nx_cur = 0, nx_new = 0; 3197 struct xfrm_policy *pol = NULL; 3198 struct xfrm_state *x, *xc; 3199 struct xfrm_state *x_cur[XFRM_MAX_DEPTH]; 3200 struct xfrm_state *x_new[XFRM_MAX_DEPTH]; 3201 struct xfrm_migrate *mp; 3202 3203 if ((err = xfrm_migrate_check(m, num_migrate)) < 0) 3204 goto out; 3205 3206 /* Stage 1 - find policy */ 3207 if ((pol = xfrm_migrate_policy_find(sel, dir, type)) == NULL) { 3208 err = -ENOENT; 3209 goto out; 3210 } 3211 3212 /* Stage 2 - find and update state(s) */ 3213 for (i = 0, mp = m; i < num_migrate; i++, mp++) { 3214 if ((x = xfrm_migrate_state_find(mp))) { 3215 x_cur[nx_cur] = x; 3216 nx_cur++; 3217 if ((xc = xfrm_state_migrate(x, mp))) { 3218 x_new[nx_new] = xc; 3219 nx_new++; 3220 } else { 3221 err = -ENODATA; 3222 goto restore_state; 3223 } 3224 } 3225 } 3226 3227 /* Stage 3 - update policy */ 3228 if ((err = xfrm_policy_migrate(pol, m, num_migrate)) < 0) 3229 goto restore_state; 3230 3231 /* Stage 4 - delete old state(s) */ 3232 if (nx_cur) { 3233 xfrm_states_put(x_cur, nx_cur); 3234 xfrm_states_delete(x_cur, nx_cur); 3235 } 3236 3237 /* Stage 5 - announce */ 3238 km_migrate(sel, dir, type, m, num_migrate, k); 3239 3240 xfrm_pol_put(pol); 3241 3242 return 0; 3243 out: 3244 return err; 3245 3246 restore_state: 3247 if (pol) 3248 xfrm_pol_put(pol); 3249 if (nx_cur) 3250 xfrm_states_put(x_cur, nx_cur); 3251 if (nx_new) 3252 xfrm_states_delete(x_new, nx_new); 3253 3254 return err; 3255 } 3256 EXPORT_SYMBOL(xfrm_migrate); 3257 #endif 3258