1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * xfrm_state.c 4 * 5 * Changes: 6 * Mitsuru KANDA @USAGI 7 * Kazunori MIYAZAWA @USAGI 8 * Kunihiro Ishiguro <kunihiro@ipinfusion.com> 9 * IPv6 support 10 * YOSHIFUJI Hideaki @USAGI 11 * Split up af-specific functions 12 * Derek Atkins <derek@ihtfp.com> 13 * Add UDP Encapsulation 14 * 15 */ 16 17 #include <linux/compat.h> 18 #include <linux/workqueue.h> 19 #include <net/xfrm.h> 20 #include <linux/pfkeyv2.h> 21 #include <linux/ipsec.h> 22 #include <linux/module.h> 23 #include <linux/cache.h> 24 #include <linux/audit.h> 25 #include <linux/uaccess.h> 26 #include <linux/ktime.h> 27 #include <linux/slab.h> 28 #include <linux/interrupt.h> 29 #include <linux/kernel.h> 30 31 #include <crypto/aead.h> 32 33 #include "xfrm_hash.h" 34 35 #define xfrm_state_deref_prot(table, net) \ 36 rcu_dereference_protected((table), lockdep_is_held(&(net)->xfrm.xfrm_state_lock)) 37 38 static void xfrm_state_gc_task(struct work_struct *work); 39 40 /* Each xfrm_state may be linked to two tables: 41 42 1. Hash table by (spi,daddr,ah/esp) to find SA by SPI. (input,ctl) 43 2. Hash table by (daddr,family,reqid) to find what SAs exist for given 44 destination/tunnel endpoint. (output) 45 */ 46 47 static unsigned int xfrm_state_hashmax __read_mostly = 1 * 1024 * 1024; 48 static struct kmem_cache *xfrm_state_cache __ro_after_init; 49 50 static DECLARE_WORK(xfrm_state_gc_work, xfrm_state_gc_task); 51 static HLIST_HEAD(xfrm_state_gc_list); 52 static HLIST_HEAD(xfrm_state_dev_gc_list); 53 54 static inline bool xfrm_state_hold_rcu(struct xfrm_state __rcu *x) 55 { 56 return refcount_inc_not_zero(&x->refcnt); 57 } 58 59 static inline unsigned int xfrm_dst_hash(struct net *net, 60 const xfrm_address_t *daddr, 61 const xfrm_address_t *saddr, 62 u32 reqid, 63 unsigned short family) 64 { 65 return __xfrm_dst_hash(daddr, saddr, reqid, family, net->xfrm.state_hmask); 66 } 67 68 static inline unsigned int xfrm_src_hash(struct net *net, 69 const xfrm_address_t *daddr, 70 const xfrm_address_t *saddr, 71 unsigned short family) 72 { 73 return __xfrm_src_hash(daddr, saddr, family, net->xfrm.state_hmask); 74 } 75 76 static inline unsigned int 77 xfrm_spi_hash(struct net *net, const xfrm_address_t *daddr, 78 __be32 spi, u8 proto, unsigned short family) 79 { 80 return __xfrm_spi_hash(daddr, spi, proto, family, net->xfrm.state_hmask); 81 } 82 83 static unsigned int xfrm_seq_hash(struct net *net, u32 seq) 84 { 85 return __xfrm_seq_hash(seq, net->xfrm.state_hmask); 86 } 87 88 #define XFRM_STATE_INSERT(by, _n, _h, _type) \ 89 { \ 90 struct xfrm_state *_x = NULL; \ 91 \ 92 if (_type != XFRM_DEV_OFFLOAD_PACKET) { \ 93 hlist_for_each_entry_rcu(_x, _h, by) { \ 94 if (_x->xso.type == XFRM_DEV_OFFLOAD_PACKET) \ 95 continue; \ 96 break; \ 97 } \ 98 } \ 99 \ 100 if (!_x || _x->xso.type == XFRM_DEV_OFFLOAD_PACKET) \ 101 /* SAD is empty or consist from HW SAs only */ \ 102 hlist_add_head_rcu(_n, _h); \ 103 else \ 104 hlist_add_before_rcu(_n, &_x->by); \ 105 } 106 107 static void xfrm_hash_transfer(struct hlist_head *list, 108 struct hlist_head *ndsttable, 109 struct hlist_head *nsrctable, 110 struct hlist_head *nspitable, 111 struct hlist_head *nseqtable, 112 unsigned int nhashmask) 113 { 114 struct hlist_node *tmp; 115 struct xfrm_state *x; 116 117 hlist_for_each_entry_safe(x, tmp, list, bydst) { 118 unsigned int h; 119 120 h = __xfrm_dst_hash(&x->id.daddr, &x->props.saddr, 121 x->props.reqid, x->props.family, 122 nhashmask); 123 XFRM_STATE_INSERT(bydst, &x->bydst, ndsttable + h, x->xso.type); 124 125 h = __xfrm_src_hash(&x->id.daddr, &x->props.saddr, 126 x->props.family, 127 nhashmask); 128 XFRM_STATE_INSERT(bysrc, &x->bysrc, nsrctable + h, x->xso.type); 129 130 if (x->id.spi) { 131 h = __xfrm_spi_hash(&x->id.daddr, x->id.spi, 132 x->id.proto, x->props.family, 133 nhashmask); 134 XFRM_STATE_INSERT(byspi, &x->byspi, nspitable + h, 135 x->xso.type); 136 } 137 138 if (x->km.seq) { 139 h = __xfrm_seq_hash(x->km.seq, nhashmask); 140 XFRM_STATE_INSERT(byseq, &x->byseq, nseqtable + h, 141 x->xso.type); 142 } 143 } 144 } 145 146 static unsigned long xfrm_hash_new_size(unsigned int state_hmask) 147 { 148 return ((state_hmask + 1) << 1) * sizeof(struct hlist_head); 149 } 150 151 static void xfrm_hash_resize(struct work_struct *work) 152 { 153 struct net *net = container_of(work, struct net, xfrm.state_hash_work); 154 struct hlist_head *ndst, *nsrc, *nspi, *nseq, *odst, *osrc, *ospi, *oseq; 155 unsigned long nsize, osize; 156 unsigned int nhashmask, ohashmask; 157 int i; 158 159 nsize = xfrm_hash_new_size(net->xfrm.state_hmask); 160 ndst = xfrm_hash_alloc(nsize); 161 if (!ndst) 162 return; 163 nsrc = xfrm_hash_alloc(nsize); 164 if (!nsrc) { 165 xfrm_hash_free(ndst, nsize); 166 return; 167 } 168 nspi = xfrm_hash_alloc(nsize); 169 if (!nspi) { 170 xfrm_hash_free(ndst, nsize); 171 xfrm_hash_free(nsrc, nsize); 172 return; 173 } 174 nseq = xfrm_hash_alloc(nsize); 175 if (!nseq) { 176 xfrm_hash_free(ndst, nsize); 177 xfrm_hash_free(nsrc, nsize); 178 xfrm_hash_free(nspi, nsize); 179 return; 180 } 181 182 spin_lock_bh(&net->xfrm.xfrm_state_lock); 183 write_seqcount_begin(&net->xfrm.xfrm_state_hash_generation); 184 185 nhashmask = (nsize / sizeof(struct hlist_head)) - 1U; 186 odst = xfrm_state_deref_prot(net->xfrm.state_bydst, net); 187 for (i = net->xfrm.state_hmask; i >= 0; i--) 188 xfrm_hash_transfer(odst + i, ndst, nsrc, nspi, nseq, nhashmask); 189 190 osrc = xfrm_state_deref_prot(net->xfrm.state_bysrc, net); 191 ospi = xfrm_state_deref_prot(net->xfrm.state_byspi, net); 192 oseq = xfrm_state_deref_prot(net->xfrm.state_byseq, net); 193 ohashmask = net->xfrm.state_hmask; 194 195 rcu_assign_pointer(net->xfrm.state_bydst, ndst); 196 rcu_assign_pointer(net->xfrm.state_bysrc, nsrc); 197 rcu_assign_pointer(net->xfrm.state_byspi, nspi); 198 rcu_assign_pointer(net->xfrm.state_byseq, nseq); 199 net->xfrm.state_hmask = nhashmask; 200 201 write_seqcount_end(&net->xfrm.xfrm_state_hash_generation); 202 spin_unlock_bh(&net->xfrm.xfrm_state_lock); 203 204 osize = (ohashmask + 1) * sizeof(struct hlist_head); 205 206 synchronize_rcu(); 207 208 xfrm_hash_free(odst, osize); 209 xfrm_hash_free(osrc, osize); 210 xfrm_hash_free(ospi, osize); 211 xfrm_hash_free(oseq, osize); 212 } 213 214 static DEFINE_SPINLOCK(xfrm_state_afinfo_lock); 215 static struct xfrm_state_afinfo __rcu *xfrm_state_afinfo[NPROTO]; 216 217 static DEFINE_SPINLOCK(xfrm_state_gc_lock); 218 static DEFINE_SPINLOCK(xfrm_state_dev_gc_lock); 219 220 int __xfrm_state_delete(struct xfrm_state *x); 221 222 int km_query(struct xfrm_state *x, struct xfrm_tmpl *t, struct xfrm_policy *pol); 223 static bool km_is_alive(const struct km_event *c); 224 void km_state_expired(struct xfrm_state *x, int hard, u32 portid); 225 226 int xfrm_register_type(const struct xfrm_type *type, unsigned short family) 227 { 228 struct xfrm_state_afinfo *afinfo = xfrm_state_get_afinfo(family); 229 int err = 0; 230 231 if (!afinfo) 232 return -EAFNOSUPPORT; 233 234 #define X(afi, T, name) do { \ 235 WARN_ON((afi)->type_ ## name); \ 236 (afi)->type_ ## name = (T); \ 237 } while (0) 238 239 switch (type->proto) { 240 case IPPROTO_COMP: 241 X(afinfo, type, comp); 242 break; 243 case IPPROTO_AH: 244 X(afinfo, type, ah); 245 break; 246 case IPPROTO_ESP: 247 X(afinfo, type, esp); 248 break; 249 case IPPROTO_IPIP: 250 X(afinfo, type, ipip); 251 break; 252 case IPPROTO_DSTOPTS: 253 X(afinfo, type, dstopts); 254 break; 255 case IPPROTO_ROUTING: 256 X(afinfo, type, routing); 257 break; 258 case IPPROTO_IPV6: 259 X(afinfo, type, ipip6); 260 break; 261 default: 262 WARN_ON(1); 263 err = -EPROTONOSUPPORT; 264 break; 265 } 266 #undef X 267 rcu_read_unlock(); 268 return err; 269 } 270 EXPORT_SYMBOL(xfrm_register_type); 271 272 void xfrm_unregister_type(const struct xfrm_type *type, unsigned short family) 273 { 274 struct xfrm_state_afinfo *afinfo = xfrm_state_get_afinfo(family); 275 276 if (unlikely(afinfo == NULL)) 277 return; 278 279 #define X(afi, T, name) do { \ 280 WARN_ON((afi)->type_ ## name != (T)); \ 281 (afi)->type_ ## name = NULL; \ 282 } while (0) 283 284 switch (type->proto) { 285 case IPPROTO_COMP: 286 X(afinfo, type, comp); 287 break; 288 case IPPROTO_AH: 289 X(afinfo, type, ah); 290 break; 291 case IPPROTO_ESP: 292 X(afinfo, type, esp); 293 break; 294 case IPPROTO_IPIP: 295 X(afinfo, type, ipip); 296 break; 297 case IPPROTO_DSTOPTS: 298 X(afinfo, type, dstopts); 299 break; 300 case IPPROTO_ROUTING: 301 X(afinfo, type, routing); 302 break; 303 case IPPROTO_IPV6: 304 X(afinfo, type, ipip6); 305 break; 306 default: 307 WARN_ON(1); 308 break; 309 } 310 #undef X 311 rcu_read_unlock(); 312 } 313 EXPORT_SYMBOL(xfrm_unregister_type); 314 315 static const struct xfrm_type *xfrm_get_type(u8 proto, unsigned short family) 316 { 317 const struct xfrm_type *type = NULL; 318 struct xfrm_state_afinfo *afinfo; 319 int modload_attempted = 0; 320 321 retry: 322 afinfo = xfrm_state_get_afinfo(family); 323 if (unlikely(afinfo == NULL)) 324 return NULL; 325 326 switch (proto) { 327 case IPPROTO_COMP: 328 type = afinfo->type_comp; 329 break; 330 case IPPROTO_AH: 331 type = afinfo->type_ah; 332 break; 333 case IPPROTO_ESP: 334 type = afinfo->type_esp; 335 break; 336 case IPPROTO_IPIP: 337 type = afinfo->type_ipip; 338 break; 339 case IPPROTO_DSTOPTS: 340 type = afinfo->type_dstopts; 341 break; 342 case IPPROTO_ROUTING: 343 type = afinfo->type_routing; 344 break; 345 case IPPROTO_IPV6: 346 type = afinfo->type_ipip6; 347 break; 348 default: 349 break; 350 } 351 352 if (unlikely(type && !try_module_get(type->owner))) 353 type = NULL; 354 355 rcu_read_unlock(); 356 357 if (!type && !modload_attempted) { 358 request_module("xfrm-type-%d-%d", family, proto); 359 modload_attempted = 1; 360 goto retry; 361 } 362 363 return type; 364 } 365 366 static void xfrm_put_type(const struct xfrm_type *type) 367 { 368 module_put(type->owner); 369 } 370 371 int xfrm_register_type_offload(const struct xfrm_type_offload *type, 372 unsigned short family) 373 { 374 struct xfrm_state_afinfo *afinfo = xfrm_state_get_afinfo(family); 375 int err = 0; 376 377 if (unlikely(afinfo == NULL)) 378 return -EAFNOSUPPORT; 379 380 switch (type->proto) { 381 case IPPROTO_ESP: 382 WARN_ON(afinfo->type_offload_esp); 383 afinfo->type_offload_esp = type; 384 break; 385 default: 386 WARN_ON(1); 387 err = -EPROTONOSUPPORT; 388 break; 389 } 390 391 rcu_read_unlock(); 392 return err; 393 } 394 EXPORT_SYMBOL(xfrm_register_type_offload); 395 396 void xfrm_unregister_type_offload(const struct xfrm_type_offload *type, 397 unsigned short family) 398 { 399 struct xfrm_state_afinfo *afinfo = xfrm_state_get_afinfo(family); 400 401 if (unlikely(afinfo == NULL)) 402 return; 403 404 switch (type->proto) { 405 case IPPROTO_ESP: 406 WARN_ON(afinfo->type_offload_esp != type); 407 afinfo->type_offload_esp = NULL; 408 break; 409 default: 410 WARN_ON(1); 411 break; 412 } 413 rcu_read_unlock(); 414 } 415 EXPORT_SYMBOL(xfrm_unregister_type_offload); 416 417 static const struct xfrm_type_offload * 418 xfrm_get_type_offload(u8 proto, unsigned short family, bool try_load) 419 { 420 const struct xfrm_type_offload *type = NULL; 421 struct xfrm_state_afinfo *afinfo; 422 423 retry: 424 afinfo = xfrm_state_get_afinfo(family); 425 if (unlikely(afinfo == NULL)) 426 return NULL; 427 428 switch (proto) { 429 case IPPROTO_ESP: 430 type = afinfo->type_offload_esp; 431 break; 432 default: 433 break; 434 } 435 436 if ((type && !try_module_get(type->owner))) 437 type = NULL; 438 439 rcu_read_unlock(); 440 441 if (!type && try_load) { 442 request_module("xfrm-offload-%d-%d", family, proto); 443 try_load = false; 444 goto retry; 445 } 446 447 return type; 448 } 449 450 static void xfrm_put_type_offload(const struct xfrm_type_offload *type) 451 { 452 module_put(type->owner); 453 } 454 455 static const struct xfrm_mode xfrm4_mode_map[XFRM_MODE_MAX] = { 456 [XFRM_MODE_BEET] = { 457 .encap = XFRM_MODE_BEET, 458 .flags = XFRM_MODE_FLAG_TUNNEL, 459 .family = AF_INET, 460 }, 461 [XFRM_MODE_TRANSPORT] = { 462 .encap = XFRM_MODE_TRANSPORT, 463 .family = AF_INET, 464 }, 465 [XFRM_MODE_TUNNEL] = { 466 .encap = XFRM_MODE_TUNNEL, 467 .flags = XFRM_MODE_FLAG_TUNNEL, 468 .family = AF_INET, 469 }, 470 }; 471 472 static const struct xfrm_mode xfrm6_mode_map[XFRM_MODE_MAX] = { 473 [XFRM_MODE_BEET] = { 474 .encap = XFRM_MODE_BEET, 475 .flags = XFRM_MODE_FLAG_TUNNEL, 476 .family = AF_INET6, 477 }, 478 [XFRM_MODE_ROUTEOPTIMIZATION] = { 479 .encap = XFRM_MODE_ROUTEOPTIMIZATION, 480 .family = AF_INET6, 481 }, 482 [XFRM_MODE_TRANSPORT] = { 483 .encap = XFRM_MODE_TRANSPORT, 484 .family = AF_INET6, 485 }, 486 [XFRM_MODE_TUNNEL] = { 487 .encap = XFRM_MODE_TUNNEL, 488 .flags = XFRM_MODE_FLAG_TUNNEL, 489 .family = AF_INET6, 490 }, 491 }; 492 493 static const struct xfrm_mode *xfrm_get_mode(unsigned int encap, int family) 494 { 495 const struct xfrm_mode *mode; 496 497 if (unlikely(encap >= XFRM_MODE_MAX)) 498 return NULL; 499 500 switch (family) { 501 case AF_INET: 502 mode = &xfrm4_mode_map[encap]; 503 if (mode->family == family) 504 return mode; 505 break; 506 case AF_INET6: 507 mode = &xfrm6_mode_map[encap]; 508 if (mode->family == family) 509 return mode; 510 break; 511 default: 512 break; 513 } 514 515 return NULL; 516 } 517 518 void xfrm_state_free(struct xfrm_state *x) 519 { 520 kmem_cache_free(xfrm_state_cache, x); 521 } 522 EXPORT_SYMBOL(xfrm_state_free); 523 524 static void ___xfrm_state_destroy(struct xfrm_state *x) 525 { 526 hrtimer_cancel(&x->mtimer); 527 del_timer_sync(&x->rtimer); 528 kfree(x->aead); 529 kfree(x->aalg); 530 kfree(x->ealg); 531 kfree(x->calg); 532 kfree(x->encap); 533 kfree(x->coaddr); 534 kfree(x->replay_esn); 535 kfree(x->preplay_esn); 536 if (x->type_offload) 537 xfrm_put_type_offload(x->type_offload); 538 if (x->type) { 539 x->type->destructor(x); 540 xfrm_put_type(x->type); 541 } 542 if (x->xfrag.page) 543 put_page(x->xfrag.page); 544 xfrm_dev_state_free(x); 545 security_xfrm_state_free(x); 546 xfrm_state_free(x); 547 } 548 549 static void xfrm_state_gc_task(struct work_struct *work) 550 { 551 struct xfrm_state *x; 552 struct hlist_node *tmp; 553 struct hlist_head gc_list; 554 555 spin_lock_bh(&xfrm_state_gc_lock); 556 hlist_move_list(&xfrm_state_gc_list, &gc_list); 557 spin_unlock_bh(&xfrm_state_gc_lock); 558 559 synchronize_rcu(); 560 561 hlist_for_each_entry_safe(x, tmp, &gc_list, gclist) 562 ___xfrm_state_destroy(x); 563 } 564 565 static enum hrtimer_restart xfrm_timer_handler(struct hrtimer *me) 566 { 567 struct xfrm_state *x = container_of(me, struct xfrm_state, mtimer); 568 enum hrtimer_restart ret = HRTIMER_NORESTART; 569 time64_t now = ktime_get_real_seconds(); 570 time64_t next = TIME64_MAX; 571 int warn = 0; 572 int err = 0; 573 574 spin_lock(&x->lock); 575 xfrm_dev_state_update_curlft(x); 576 577 if (x->km.state == XFRM_STATE_DEAD) 578 goto out; 579 if (x->km.state == XFRM_STATE_EXPIRED) 580 goto expired; 581 if (x->lft.hard_add_expires_seconds) { 582 time64_t tmo = x->lft.hard_add_expires_seconds + 583 x->curlft.add_time - now; 584 if (tmo <= 0) { 585 if (x->xflags & XFRM_SOFT_EXPIRE) { 586 /* enter hard expire without soft expire first?! 587 * setting a new date could trigger this. 588 * workaround: fix x->curflt.add_time by below: 589 */ 590 x->curlft.add_time = now - x->saved_tmo - 1; 591 tmo = x->lft.hard_add_expires_seconds - x->saved_tmo; 592 } else 593 goto expired; 594 } 595 if (tmo < next) 596 next = tmo; 597 } 598 if (x->lft.hard_use_expires_seconds) { 599 time64_t tmo = x->lft.hard_use_expires_seconds + 600 (READ_ONCE(x->curlft.use_time) ? : now) - now; 601 if (tmo <= 0) 602 goto expired; 603 if (tmo < next) 604 next = tmo; 605 } 606 if (x->km.dying) 607 goto resched; 608 if (x->lft.soft_add_expires_seconds) { 609 time64_t tmo = x->lft.soft_add_expires_seconds + 610 x->curlft.add_time - now; 611 if (tmo <= 0) { 612 warn = 1; 613 x->xflags &= ~XFRM_SOFT_EXPIRE; 614 } else if (tmo < next) { 615 next = tmo; 616 x->xflags |= XFRM_SOFT_EXPIRE; 617 x->saved_tmo = tmo; 618 } 619 } 620 if (x->lft.soft_use_expires_seconds) { 621 time64_t tmo = x->lft.soft_use_expires_seconds + 622 (READ_ONCE(x->curlft.use_time) ? : now) - now; 623 if (tmo <= 0) 624 warn = 1; 625 else if (tmo < next) 626 next = tmo; 627 } 628 629 x->km.dying = warn; 630 if (warn) 631 km_state_expired(x, 0, 0); 632 resched: 633 if (next != TIME64_MAX) { 634 hrtimer_forward_now(&x->mtimer, ktime_set(next, 0)); 635 ret = HRTIMER_RESTART; 636 } 637 638 goto out; 639 640 expired: 641 if (x->km.state == XFRM_STATE_ACQ && x->id.spi == 0) 642 x->km.state = XFRM_STATE_EXPIRED; 643 644 err = __xfrm_state_delete(x); 645 if (!err) 646 km_state_expired(x, 1, 0); 647 648 xfrm_audit_state_delete(x, err ? 0 : 1, true); 649 650 out: 651 spin_unlock(&x->lock); 652 return ret; 653 } 654 655 static void xfrm_replay_timer_handler(struct timer_list *t); 656 657 struct xfrm_state *xfrm_state_alloc(struct net *net) 658 { 659 struct xfrm_state *x; 660 661 x = kmem_cache_zalloc(xfrm_state_cache, GFP_ATOMIC); 662 663 if (x) { 664 write_pnet(&x->xs_net, net); 665 refcount_set(&x->refcnt, 1); 666 atomic_set(&x->tunnel_users, 0); 667 INIT_LIST_HEAD(&x->km.all); 668 INIT_HLIST_NODE(&x->bydst); 669 INIT_HLIST_NODE(&x->bysrc); 670 INIT_HLIST_NODE(&x->byspi); 671 INIT_HLIST_NODE(&x->byseq); 672 hrtimer_init(&x->mtimer, CLOCK_BOOTTIME, HRTIMER_MODE_ABS_SOFT); 673 x->mtimer.function = xfrm_timer_handler; 674 timer_setup(&x->rtimer, xfrm_replay_timer_handler, 0); 675 x->curlft.add_time = ktime_get_real_seconds(); 676 x->lft.soft_byte_limit = XFRM_INF; 677 x->lft.soft_packet_limit = XFRM_INF; 678 x->lft.hard_byte_limit = XFRM_INF; 679 x->lft.hard_packet_limit = XFRM_INF; 680 x->replay_maxage = 0; 681 x->replay_maxdiff = 0; 682 spin_lock_init(&x->lock); 683 } 684 return x; 685 } 686 EXPORT_SYMBOL(xfrm_state_alloc); 687 688 #ifdef CONFIG_XFRM_OFFLOAD 689 void xfrm_dev_state_delete(struct xfrm_state *x) 690 { 691 struct xfrm_dev_offload *xso = &x->xso; 692 struct net_device *dev = READ_ONCE(xso->dev); 693 694 if (dev) { 695 dev->xfrmdev_ops->xdo_dev_state_delete(x); 696 spin_lock_bh(&xfrm_state_dev_gc_lock); 697 hlist_add_head(&x->dev_gclist, &xfrm_state_dev_gc_list); 698 spin_unlock_bh(&xfrm_state_dev_gc_lock); 699 } 700 } 701 EXPORT_SYMBOL_GPL(xfrm_dev_state_delete); 702 703 void xfrm_dev_state_free(struct xfrm_state *x) 704 { 705 struct xfrm_dev_offload *xso = &x->xso; 706 struct net_device *dev = READ_ONCE(xso->dev); 707 708 if (dev && dev->xfrmdev_ops) { 709 spin_lock_bh(&xfrm_state_dev_gc_lock); 710 if (!hlist_unhashed(&x->dev_gclist)) 711 hlist_del(&x->dev_gclist); 712 spin_unlock_bh(&xfrm_state_dev_gc_lock); 713 714 if (dev->xfrmdev_ops->xdo_dev_state_free) 715 dev->xfrmdev_ops->xdo_dev_state_free(x); 716 WRITE_ONCE(xso->dev, NULL); 717 xso->type = XFRM_DEV_OFFLOAD_UNSPECIFIED; 718 netdev_put(dev, &xso->dev_tracker); 719 } 720 } 721 #endif 722 723 void __xfrm_state_destroy(struct xfrm_state *x, bool sync) 724 { 725 WARN_ON(x->km.state != XFRM_STATE_DEAD); 726 727 if (sync) { 728 synchronize_rcu(); 729 ___xfrm_state_destroy(x); 730 } else { 731 spin_lock_bh(&xfrm_state_gc_lock); 732 hlist_add_head(&x->gclist, &xfrm_state_gc_list); 733 spin_unlock_bh(&xfrm_state_gc_lock); 734 schedule_work(&xfrm_state_gc_work); 735 } 736 } 737 EXPORT_SYMBOL(__xfrm_state_destroy); 738 739 int __xfrm_state_delete(struct xfrm_state *x) 740 { 741 struct net *net = xs_net(x); 742 int err = -ESRCH; 743 744 if (x->km.state != XFRM_STATE_DEAD) { 745 x->km.state = XFRM_STATE_DEAD; 746 spin_lock(&net->xfrm.xfrm_state_lock); 747 list_del(&x->km.all); 748 hlist_del_rcu(&x->bydst); 749 hlist_del_rcu(&x->bysrc); 750 if (x->km.seq) 751 hlist_del_rcu(&x->byseq); 752 if (x->id.spi) 753 hlist_del_rcu(&x->byspi); 754 net->xfrm.state_num--; 755 spin_unlock(&net->xfrm.xfrm_state_lock); 756 757 if (x->encap_sk) 758 sock_put(rcu_dereference_raw(x->encap_sk)); 759 760 xfrm_dev_state_delete(x); 761 762 /* All xfrm_state objects are created by xfrm_state_alloc. 763 * The xfrm_state_alloc call gives a reference, and that 764 * is what we are dropping here. 765 */ 766 xfrm_state_put(x); 767 err = 0; 768 } 769 770 return err; 771 } 772 EXPORT_SYMBOL(__xfrm_state_delete); 773 774 int xfrm_state_delete(struct xfrm_state *x) 775 { 776 int err; 777 778 spin_lock_bh(&x->lock); 779 err = __xfrm_state_delete(x); 780 spin_unlock_bh(&x->lock); 781 782 return err; 783 } 784 EXPORT_SYMBOL(xfrm_state_delete); 785 786 #ifdef CONFIG_SECURITY_NETWORK_XFRM 787 static inline int 788 xfrm_state_flush_secctx_check(struct net *net, u8 proto, bool task_valid) 789 { 790 int i, err = 0; 791 792 for (i = 0; i <= net->xfrm.state_hmask; i++) { 793 struct xfrm_state *x; 794 795 hlist_for_each_entry(x, net->xfrm.state_bydst+i, bydst) { 796 if (xfrm_id_proto_match(x->id.proto, proto) && 797 (err = security_xfrm_state_delete(x)) != 0) { 798 xfrm_audit_state_delete(x, 0, task_valid); 799 return err; 800 } 801 } 802 } 803 804 return err; 805 } 806 807 static inline int 808 xfrm_dev_state_flush_secctx_check(struct net *net, struct net_device *dev, bool task_valid) 809 { 810 int i, err = 0; 811 812 for (i = 0; i <= net->xfrm.state_hmask; i++) { 813 struct xfrm_state *x; 814 struct xfrm_dev_offload *xso; 815 816 hlist_for_each_entry(x, net->xfrm.state_bydst+i, bydst) { 817 xso = &x->xso; 818 819 if (xso->dev == dev && 820 (err = security_xfrm_state_delete(x)) != 0) { 821 xfrm_audit_state_delete(x, 0, task_valid); 822 return err; 823 } 824 } 825 } 826 827 return err; 828 } 829 #else 830 static inline int 831 xfrm_state_flush_secctx_check(struct net *net, u8 proto, bool task_valid) 832 { 833 return 0; 834 } 835 836 static inline int 837 xfrm_dev_state_flush_secctx_check(struct net *net, struct net_device *dev, bool task_valid) 838 { 839 return 0; 840 } 841 #endif 842 843 int xfrm_state_flush(struct net *net, u8 proto, bool task_valid, bool sync) 844 { 845 int i, err = 0, cnt = 0; 846 847 spin_lock_bh(&net->xfrm.xfrm_state_lock); 848 err = xfrm_state_flush_secctx_check(net, proto, task_valid); 849 if (err) 850 goto out; 851 852 err = -ESRCH; 853 for (i = 0; i <= net->xfrm.state_hmask; i++) { 854 struct xfrm_state *x; 855 restart: 856 hlist_for_each_entry(x, net->xfrm.state_bydst+i, bydst) { 857 if (!xfrm_state_kern(x) && 858 xfrm_id_proto_match(x->id.proto, proto)) { 859 xfrm_state_hold(x); 860 spin_unlock_bh(&net->xfrm.xfrm_state_lock); 861 862 err = xfrm_state_delete(x); 863 xfrm_audit_state_delete(x, err ? 0 : 1, 864 task_valid); 865 if (sync) 866 xfrm_state_put_sync(x); 867 else 868 xfrm_state_put(x); 869 if (!err) 870 cnt++; 871 872 spin_lock_bh(&net->xfrm.xfrm_state_lock); 873 goto restart; 874 } 875 } 876 } 877 out: 878 spin_unlock_bh(&net->xfrm.xfrm_state_lock); 879 if (cnt) 880 err = 0; 881 882 return err; 883 } 884 EXPORT_SYMBOL(xfrm_state_flush); 885 886 int xfrm_dev_state_flush(struct net *net, struct net_device *dev, bool task_valid) 887 { 888 struct xfrm_state *x; 889 struct hlist_node *tmp; 890 struct xfrm_dev_offload *xso; 891 int i, err = 0, cnt = 0; 892 893 spin_lock_bh(&net->xfrm.xfrm_state_lock); 894 err = xfrm_dev_state_flush_secctx_check(net, dev, task_valid); 895 if (err) 896 goto out; 897 898 err = -ESRCH; 899 for (i = 0; i <= net->xfrm.state_hmask; i++) { 900 restart: 901 hlist_for_each_entry(x, net->xfrm.state_bydst+i, bydst) { 902 xso = &x->xso; 903 904 if (!xfrm_state_kern(x) && xso->dev == dev) { 905 xfrm_state_hold(x); 906 spin_unlock_bh(&net->xfrm.xfrm_state_lock); 907 908 err = xfrm_state_delete(x); 909 xfrm_dev_state_free(x); 910 911 xfrm_audit_state_delete(x, err ? 0 : 1, 912 task_valid); 913 xfrm_state_put(x); 914 if (!err) 915 cnt++; 916 917 spin_lock_bh(&net->xfrm.xfrm_state_lock); 918 goto restart; 919 } 920 } 921 } 922 if (cnt) 923 err = 0; 924 925 out: 926 spin_unlock_bh(&net->xfrm.xfrm_state_lock); 927 928 spin_lock_bh(&xfrm_state_dev_gc_lock); 929 restart_gc: 930 hlist_for_each_entry_safe(x, tmp, &xfrm_state_dev_gc_list, dev_gclist) { 931 xso = &x->xso; 932 933 if (xso->dev == dev) { 934 spin_unlock_bh(&xfrm_state_dev_gc_lock); 935 xfrm_dev_state_free(x); 936 spin_lock_bh(&xfrm_state_dev_gc_lock); 937 goto restart_gc; 938 } 939 940 } 941 spin_unlock_bh(&xfrm_state_dev_gc_lock); 942 943 xfrm_flush_gc(); 944 945 return err; 946 } 947 EXPORT_SYMBOL(xfrm_dev_state_flush); 948 949 void xfrm_sad_getinfo(struct net *net, struct xfrmk_sadinfo *si) 950 { 951 spin_lock_bh(&net->xfrm.xfrm_state_lock); 952 si->sadcnt = net->xfrm.state_num; 953 si->sadhcnt = net->xfrm.state_hmask + 1; 954 si->sadhmcnt = xfrm_state_hashmax; 955 spin_unlock_bh(&net->xfrm.xfrm_state_lock); 956 } 957 EXPORT_SYMBOL(xfrm_sad_getinfo); 958 959 static void 960 __xfrm4_init_tempsel(struct xfrm_selector *sel, const struct flowi *fl) 961 { 962 const struct flowi4 *fl4 = &fl->u.ip4; 963 964 sel->daddr.a4 = fl4->daddr; 965 sel->saddr.a4 = fl4->saddr; 966 sel->dport = xfrm_flowi_dport(fl, &fl4->uli); 967 sel->dport_mask = htons(0xffff); 968 sel->sport = xfrm_flowi_sport(fl, &fl4->uli); 969 sel->sport_mask = htons(0xffff); 970 sel->family = AF_INET; 971 sel->prefixlen_d = 32; 972 sel->prefixlen_s = 32; 973 sel->proto = fl4->flowi4_proto; 974 sel->ifindex = fl4->flowi4_oif; 975 } 976 977 static void 978 __xfrm6_init_tempsel(struct xfrm_selector *sel, const struct flowi *fl) 979 { 980 const struct flowi6 *fl6 = &fl->u.ip6; 981 982 /* Initialize temporary selector matching only to current session. */ 983 *(struct in6_addr *)&sel->daddr = fl6->daddr; 984 *(struct in6_addr *)&sel->saddr = fl6->saddr; 985 sel->dport = xfrm_flowi_dport(fl, &fl6->uli); 986 sel->dport_mask = htons(0xffff); 987 sel->sport = xfrm_flowi_sport(fl, &fl6->uli); 988 sel->sport_mask = htons(0xffff); 989 sel->family = AF_INET6; 990 sel->prefixlen_d = 128; 991 sel->prefixlen_s = 128; 992 sel->proto = fl6->flowi6_proto; 993 sel->ifindex = fl6->flowi6_oif; 994 } 995 996 static void 997 xfrm_init_tempstate(struct xfrm_state *x, const struct flowi *fl, 998 const struct xfrm_tmpl *tmpl, 999 const xfrm_address_t *daddr, const xfrm_address_t *saddr, 1000 unsigned short family) 1001 { 1002 switch (family) { 1003 case AF_INET: 1004 __xfrm4_init_tempsel(&x->sel, fl); 1005 break; 1006 case AF_INET6: 1007 __xfrm6_init_tempsel(&x->sel, fl); 1008 break; 1009 } 1010 1011 x->id = tmpl->id; 1012 1013 switch (tmpl->encap_family) { 1014 case AF_INET: 1015 if (x->id.daddr.a4 == 0) 1016 x->id.daddr.a4 = daddr->a4; 1017 x->props.saddr = tmpl->saddr; 1018 if (x->props.saddr.a4 == 0) 1019 x->props.saddr.a4 = saddr->a4; 1020 break; 1021 case AF_INET6: 1022 if (ipv6_addr_any((struct in6_addr *)&x->id.daddr)) 1023 memcpy(&x->id.daddr, daddr, sizeof(x->sel.daddr)); 1024 memcpy(&x->props.saddr, &tmpl->saddr, sizeof(x->props.saddr)); 1025 if (ipv6_addr_any((struct in6_addr *)&x->props.saddr)) 1026 memcpy(&x->props.saddr, saddr, sizeof(x->props.saddr)); 1027 break; 1028 } 1029 1030 x->props.mode = tmpl->mode; 1031 x->props.reqid = tmpl->reqid; 1032 x->props.family = tmpl->encap_family; 1033 } 1034 1035 static struct xfrm_state *__xfrm_state_lookup_all(struct net *net, u32 mark, 1036 const xfrm_address_t *daddr, 1037 __be32 spi, u8 proto, 1038 unsigned short family, 1039 struct xfrm_dev_offload *xdo) 1040 { 1041 unsigned int h = xfrm_spi_hash(net, daddr, spi, proto, family); 1042 struct xfrm_state *x; 1043 1044 hlist_for_each_entry_rcu(x, net->xfrm.state_byspi + h, byspi) { 1045 #ifdef CONFIG_XFRM_OFFLOAD 1046 if (xdo->type == XFRM_DEV_OFFLOAD_PACKET) { 1047 if (x->xso.type != XFRM_DEV_OFFLOAD_PACKET) 1048 /* HW states are in the head of list, there is 1049 * no need to iterate further. 1050 */ 1051 break; 1052 1053 /* Packet offload: both policy and SA should 1054 * have same device. 1055 */ 1056 if (xdo->dev != x->xso.dev) 1057 continue; 1058 } else if (x->xso.type == XFRM_DEV_OFFLOAD_PACKET) 1059 /* Skip HW policy for SW lookups */ 1060 continue; 1061 #endif 1062 if (x->props.family != family || 1063 x->id.spi != spi || 1064 x->id.proto != proto || 1065 !xfrm_addr_equal(&x->id.daddr, daddr, family)) 1066 continue; 1067 1068 if ((mark & x->mark.m) != x->mark.v) 1069 continue; 1070 if (!xfrm_state_hold_rcu(x)) 1071 continue; 1072 return x; 1073 } 1074 1075 return NULL; 1076 } 1077 1078 static struct xfrm_state *__xfrm_state_lookup(struct net *net, u32 mark, 1079 const xfrm_address_t *daddr, 1080 __be32 spi, u8 proto, 1081 unsigned short family) 1082 { 1083 unsigned int h = xfrm_spi_hash(net, daddr, spi, proto, family); 1084 struct xfrm_state *x; 1085 1086 hlist_for_each_entry_rcu(x, net->xfrm.state_byspi + h, byspi) { 1087 if (x->props.family != family || 1088 x->id.spi != spi || 1089 x->id.proto != proto || 1090 !xfrm_addr_equal(&x->id.daddr, daddr, family)) 1091 continue; 1092 1093 if ((mark & x->mark.m) != x->mark.v) 1094 continue; 1095 if (!xfrm_state_hold_rcu(x)) 1096 continue; 1097 return x; 1098 } 1099 1100 return NULL; 1101 } 1102 1103 static struct xfrm_state *__xfrm_state_lookup_byaddr(struct net *net, u32 mark, 1104 const xfrm_address_t *daddr, 1105 const xfrm_address_t *saddr, 1106 u8 proto, unsigned short family) 1107 { 1108 unsigned int h = xfrm_src_hash(net, daddr, saddr, family); 1109 struct xfrm_state *x; 1110 1111 hlist_for_each_entry_rcu(x, net->xfrm.state_bysrc + h, bysrc) { 1112 if (x->props.family != family || 1113 x->id.proto != proto || 1114 !xfrm_addr_equal(&x->id.daddr, daddr, family) || 1115 !xfrm_addr_equal(&x->props.saddr, saddr, family)) 1116 continue; 1117 1118 if ((mark & x->mark.m) != x->mark.v) 1119 continue; 1120 if (!xfrm_state_hold_rcu(x)) 1121 continue; 1122 return x; 1123 } 1124 1125 return NULL; 1126 } 1127 1128 static inline struct xfrm_state * 1129 __xfrm_state_locate(struct xfrm_state *x, int use_spi, int family) 1130 { 1131 struct net *net = xs_net(x); 1132 u32 mark = x->mark.v & x->mark.m; 1133 1134 if (use_spi) 1135 return __xfrm_state_lookup(net, mark, &x->id.daddr, 1136 x->id.spi, x->id.proto, family); 1137 else 1138 return __xfrm_state_lookup_byaddr(net, mark, 1139 &x->id.daddr, 1140 &x->props.saddr, 1141 x->id.proto, family); 1142 } 1143 1144 static void xfrm_hash_grow_check(struct net *net, int have_hash_collision) 1145 { 1146 if (have_hash_collision && 1147 (net->xfrm.state_hmask + 1) < xfrm_state_hashmax && 1148 net->xfrm.state_num > net->xfrm.state_hmask) 1149 schedule_work(&net->xfrm.state_hash_work); 1150 } 1151 1152 static void xfrm_state_look_at(struct xfrm_policy *pol, struct xfrm_state *x, 1153 const struct flowi *fl, unsigned short family, 1154 struct xfrm_state **best, int *acq_in_progress, 1155 int *error) 1156 { 1157 /* Resolution logic: 1158 * 1. There is a valid state with matching selector. Done. 1159 * 2. Valid state with inappropriate selector. Skip. 1160 * 1161 * Entering area of "sysdeps". 1162 * 1163 * 3. If state is not valid, selector is temporary, it selects 1164 * only session which triggered previous resolution. Key 1165 * manager will do something to install a state with proper 1166 * selector. 1167 */ 1168 if (x->km.state == XFRM_STATE_VALID) { 1169 if ((x->sel.family && 1170 (x->sel.family != family || 1171 !xfrm_selector_match(&x->sel, fl, family))) || 1172 !security_xfrm_state_pol_flow_match(x, pol, 1173 &fl->u.__fl_common)) 1174 return; 1175 1176 if (!*best || 1177 (*best)->km.dying > x->km.dying || 1178 ((*best)->km.dying == x->km.dying && 1179 (*best)->curlft.add_time < x->curlft.add_time)) 1180 *best = x; 1181 } else if (x->km.state == XFRM_STATE_ACQ) { 1182 *acq_in_progress = 1; 1183 } else if (x->km.state == XFRM_STATE_ERROR || 1184 x->km.state == XFRM_STATE_EXPIRED) { 1185 if ((!x->sel.family || 1186 (x->sel.family == family && 1187 xfrm_selector_match(&x->sel, fl, family))) && 1188 security_xfrm_state_pol_flow_match(x, pol, 1189 &fl->u.__fl_common)) 1190 *error = -ESRCH; 1191 } 1192 } 1193 1194 struct xfrm_state * 1195 xfrm_state_find(const xfrm_address_t *daddr, const xfrm_address_t *saddr, 1196 const struct flowi *fl, struct xfrm_tmpl *tmpl, 1197 struct xfrm_policy *pol, int *err, 1198 unsigned short family, u32 if_id) 1199 { 1200 static xfrm_address_t saddr_wildcard = { }; 1201 struct net *net = xp_net(pol); 1202 unsigned int h, h_wildcard; 1203 struct xfrm_state *x, *x0, *to_put; 1204 int acquire_in_progress = 0; 1205 int error = 0; 1206 struct xfrm_state *best = NULL; 1207 u32 mark = pol->mark.v & pol->mark.m; 1208 unsigned short encap_family = tmpl->encap_family; 1209 unsigned int sequence; 1210 struct km_event c; 1211 1212 to_put = NULL; 1213 1214 sequence = read_seqcount_begin(&net->xfrm.xfrm_state_hash_generation); 1215 1216 rcu_read_lock(); 1217 h = xfrm_dst_hash(net, daddr, saddr, tmpl->reqid, encap_family); 1218 hlist_for_each_entry_rcu(x, net->xfrm.state_bydst + h, bydst) { 1219 #ifdef CONFIG_XFRM_OFFLOAD 1220 if (pol->xdo.type == XFRM_DEV_OFFLOAD_PACKET) { 1221 if (x->xso.type != XFRM_DEV_OFFLOAD_PACKET) 1222 /* HW states are in the head of list, there is 1223 * no need to iterate further. 1224 */ 1225 break; 1226 1227 /* Packet offload: both policy and SA should 1228 * have same device. 1229 */ 1230 if (pol->xdo.dev != x->xso.dev) 1231 continue; 1232 } else if (x->xso.type == XFRM_DEV_OFFLOAD_PACKET) 1233 /* Skip HW policy for SW lookups */ 1234 continue; 1235 #endif 1236 if (x->props.family == encap_family && 1237 x->props.reqid == tmpl->reqid && 1238 (mark & x->mark.m) == x->mark.v && 1239 x->if_id == if_id && 1240 !(x->props.flags & XFRM_STATE_WILDRECV) && 1241 xfrm_state_addr_check(x, daddr, saddr, encap_family) && 1242 tmpl->mode == x->props.mode && 1243 tmpl->id.proto == x->id.proto && 1244 (tmpl->id.spi == x->id.spi || !tmpl->id.spi)) 1245 xfrm_state_look_at(pol, x, fl, family, 1246 &best, &acquire_in_progress, &error); 1247 } 1248 if (best || acquire_in_progress) 1249 goto found; 1250 1251 h_wildcard = xfrm_dst_hash(net, daddr, &saddr_wildcard, tmpl->reqid, encap_family); 1252 hlist_for_each_entry_rcu(x, net->xfrm.state_bydst + h_wildcard, bydst) { 1253 #ifdef CONFIG_XFRM_OFFLOAD 1254 if (pol->xdo.type == XFRM_DEV_OFFLOAD_PACKET) { 1255 if (x->xso.type != XFRM_DEV_OFFLOAD_PACKET) 1256 /* HW states are in the head of list, there is 1257 * no need to iterate further. 1258 */ 1259 break; 1260 1261 /* Packet offload: both policy and SA should 1262 * have same device. 1263 */ 1264 if (pol->xdo.dev != x->xso.dev) 1265 continue; 1266 } else if (x->xso.type == XFRM_DEV_OFFLOAD_PACKET) 1267 /* Skip HW policy for SW lookups */ 1268 continue; 1269 #endif 1270 if (x->props.family == encap_family && 1271 x->props.reqid == tmpl->reqid && 1272 (mark & x->mark.m) == x->mark.v && 1273 x->if_id == if_id && 1274 !(x->props.flags & XFRM_STATE_WILDRECV) && 1275 xfrm_addr_equal(&x->id.daddr, daddr, encap_family) && 1276 tmpl->mode == x->props.mode && 1277 tmpl->id.proto == x->id.proto && 1278 (tmpl->id.spi == x->id.spi || !tmpl->id.spi)) 1279 xfrm_state_look_at(pol, x, fl, family, 1280 &best, &acquire_in_progress, &error); 1281 } 1282 1283 found: 1284 x = best; 1285 if (!x && !error && !acquire_in_progress) { 1286 if (tmpl->id.spi && 1287 (x0 = __xfrm_state_lookup_all(net, mark, daddr, 1288 tmpl->id.spi, tmpl->id.proto, 1289 encap_family, 1290 &pol->xdo)) != NULL) { 1291 to_put = x0; 1292 error = -EEXIST; 1293 goto out; 1294 } 1295 1296 c.net = net; 1297 /* If the KMs have no listeners (yet...), avoid allocating an SA 1298 * for each and every packet - garbage collection might not 1299 * handle the flood. 1300 */ 1301 if (!km_is_alive(&c)) { 1302 error = -ESRCH; 1303 goto out; 1304 } 1305 1306 x = xfrm_state_alloc(net); 1307 if (x == NULL) { 1308 error = -ENOMEM; 1309 goto out; 1310 } 1311 /* Initialize temporary state matching only 1312 * to current session. */ 1313 xfrm_init_tempstate(x, fl, tmpl, daddr, saddr, family); 1314 memcpy(&x->mark, &pol->mark, sizeof(x->mark)); 1315 x->if_id = if_id; 1316 1317 error = security_xfrm_state_alloc_acquire(x, pol->security, fl->flowi_secid); 1318 if (error) { 1319 x->km.state = XFRM_STATE_DEAD; 1320 to_put = x; 1321 x = NULL; 1322 goto out; 1323 } 1324 #ifdef CONFIG_XFRM_OFFLOAD 1325 if (pol->xdo.type == XFRM_DEV_OFFLOAD_PACKET) { 1326 struct xfrm_dev_offload *xdo = &pol->xdo; 1327 struct xfrm_dev_offload *xso = &x->xso; 1328 1329 xso->type = XFRM_DEV_OFFLOAD_PACKET; 1330 xso->dir = xdo->dir; 1331 xso->dev = xdo->dev; 1332 xso->real_dev = xdo->real_dev; 1333 xso->flags = XFRM_DEV_OFFLOAD_FLAG_ACQ; 1334 netdev_hold(xso->dev, &xso->dev_tracker, GFP_ATOMIC); 1335 error = xso->dev->xfrmdev_ops->xdo_dev_state_add(x, NULL); 1336 if (error) { 1337 xso->dir = 0; 1338 netdev_put(xso->dev, &xso->dev_tracker); 1339 xso->dev = NULL; 1340 xso->real_dev = NULL; 1341 xso->type = XFRM_DEV_OFFLOAD_UNSPECIFIED; 1342 x->km.state = XFRM_STATE_DEAD; 1343 to_put = x; 1344 x = NULL; 1345 goto out; 1346 } 1347 } 1348 #endif 1349 if (km_query(x, tmpl, pol) == 0) { 1350 spin_lock_bh(&net->xfrm.xfrm_state_lock); 1351 x->km.state = XFRM_STATE_ACQ; 1352 list_add(&x->km.all, &net->xfrm.state_all); 1353 XFRM_STATE_INSERT(bydst, &x->bydst, 1354 net->xfrm.state_bydst + h, 1355 x->xso.type); 1356 h = xfrm_src_hash(net, daddr, saddr, encap_family); 1357 XFRM_STATE_INSERT(bysrc, &x->bysrc, 1358 net->xfrm.state_bysrc + h, 1359 x->xso.type); 1360 if (x->id.spi) { 1361 h = xfrm_spi_hash(net, &x->id.daddr, x->id.spi, x->id.proto, encap_family); 1362 XFRM_STATE_INSERT(byspi, &x->byspi, 1363 net->xfrm.state_byspi + h, 1364 x->xso.type); 1365 } 1366 if (x->km.seq) { 1367 h = xfrm_seq_hash(net, x->km.seq); 1368 XFRM_STATE_INSERT(byseq, &x->byseq, 1369 net->xfrm.state_byseq + h, 1370 x->xso.type); 1371 } 1372 x->lft.hard_add_expires_seconds = net->xfrm.sysctl_acq_expires; 1373 hrtimer_start(&x->mtimer, 1374 ktime_set(net->xfrm.sysctl_acq_expires, 0), 1375 HRTIMER_MODE_REL_SOFT); 1376 net->xfrm.state_num++; 1377 xfrm_hash_grow_check(net, x->bydst.next != NULL); 1378 spin_unlock_bh(&net->xfrm.xfrm_state_lock); 1379 } else { 1380 #ifdef CONFIG_XFRM_OFFLOAD 1381 struct xfrm_dev_offload *xso = &x->xso; 1382 1383 if (xso->type == XFRM_DEV_OFFLOAD_PACKET) { 1384 xfrm_dev_state_delete(x); 1385 xfrm_dev_state_free(x); 1386 } 1387 #endif 1388 x->km.state = XFRM_STATE_DEAD; 1389 to_put = x; 1390 x = NULL; 1391 error = -ESRCH; 1392 } 1393 } 1394 out: 1395 if (x) { 1396 if (!xfrm_state_hold_rcu(x)) { 1397 *err = -EAGAIN; 1398 x = NULL; 1399 } 1400 } else { 1401 *err = acquire_in_progress ? -EAGAIN : error; 1402 } 1403 rcu_read_unlock(); 1404 if (to_put) 1405 xfrm_state_put(to_put); 1406 1407 if (read_seqcount_retry(&net->xfrm.xfrm_state_hash_generation, sequence)) { 1408 *err = -EAGAIN; 1409 if (x) { 1410 xfrm_state_put(x); 1411 x = NULL; 1412 } 1413 } 1414 1415 return x; 1416 } 1417 1418 struct xfrm_state * 1419 xfrm_stateonly_find(struct net *net, u32 mark, u32 if_id, 1420 xfrm_address_t *daddr, xfrm_address_t *saddr, 1421 unsigned short family, u8 mode, u8 proto, u32 reqid) 1422 { 1423 unsigned int h; 1424 struct xfrm_state *rx = NULL, *x = NULL; 1425 1426 spin_lock_bh(&net->xfrm.xfrm_state_lock); 1427 h = xfrm_dst_hash(net, daddr, saddr, reqid, family); 1428 hlist_for_each_entry(x, net->xfrm.state_bydst+h, bydst) { 1429 if (x->props.family == family && 1430 x->props.reqid == reqid && 1431 (mark & x->mark.m) == x->mark.v && 1432 x->if_id == if_id && 1433 !(x->props.flags & XFRM_STATE_WILDRECV) && 1434 xfrm_state_addr_check(x, daddr, saddr, family) && 1435 mode == x->props.mode && 1436 proto == x->id.proto && 1437 x->km.state == XFRM_STATE_VALID) { 1438 rx = x; 1439 break; 1440 } 1441 } 1442 1443 if (rx) 1444 xfrm_state_hold(rx); 1445 spin_unlock_bh(&net->xfrm.xfrm_state_lock); 1446 1447 1448 return rx; 1449 } 1450 EXPORT_SYMBOL(xfrm_stateonly_find); 1451 1452 struct xfrm_state *xfrm_state_lookup_byspi(struct net *net, __be32 spi, 1453 unsigned short family) 1454 { 1455 struct xfrm_state *x; 1456 struct xfrm_state_walk *w; 1457 1458 spin_lock_bh(&net->xfrm.xfrm_state_lock); 1459 list_for_each_entry(w, &net->xfrm.state_all, all) { 1460 x = container_of(w, struct xfrm_state, km); 1461 if (x->props.family != family || 1462 x->id.spi != spi) 1463 continue; 1464 1465 xfrm_state_hold(x); 1466 spin_unlock_bh(&net->xfrm.xfrm_state_lock); 1467 return x; 1468 } 1469 spin_unlock_bh(&net->xfrm.xfrm_state_lock); 1470 return NULL; 1471 } 1472 EXPORT_SYMBOL(xfrm_state_lookup_byspi); 1473 1474 static void __xfrm_state_insert(struct xfrm_state *x) 1475 { 1476 struct net *net = xs_net(x); 1477 unsigned int h; 1478 1479 list_add(&x->km.all, &net->xfrm.state_all); 1480 1481 h = xfrm_dst_hash(net, &x->id.daddr, &x->props.saddr, 1482 x->props.reqid, x->props.family); 1483 XFRM_STATE_INSERT(bydst, &x->bydst, net->xfrm.state_bydst + h, 1484 x->xso.type); 1485 1486 h = xfrm_src_hash(net, &x->id.daddr, &x->props.saddr, x->props.family); 1487 XFRM_STATE_INSERT(bysrc, &x->bysrc, net->xfrm.state_bysrc + h, 1488 x->xso.type); 1489 1490 if (x->id.spi) { 1491 h = xfrm_spi_hash(net, &x->id.daddr, x->id.spi, x->id.proto, 1492 x->props.family); 1493 1494 XFRM_STATE_INSERT(byspi, &x->byspi, net->xfrm.state_byspi + h, 1495 x->xso.type); 1496 } 1497 1498 if (x->km.seq) { 1499 h = xfrm_seq_hash(net, x->km.seq); 1500 1501 XFRM_STATE_INSERT(byseq, &x->byseq, net->xfrm.state_byseq + h, 1502 x->xso.type); 1503 } 1504 1505 hrtimer_start(&x->mtimer, ktime_set(1, 0), HRTIMER_MODE_REL_SOFT); 1506 if (x->replay_maxage) 1507 mod_timer(&x->rtimer, jiffies + x->replay_maxage); 1508 1509 net->xfrm.state_num++; 1510 1511 xfrm_hash_grow_check(net, x->bydst.next != NULL); 1512 } 1513 1514 /* net->xfrm.xfrm_state_lock is held */ 1515 static void __xfrm_state_bump_genids(struct xfrm_state *xnew) 1516 { 1517 struct net *net = xs_net(xnew); 1518 unsigned short family = xnew->props.family; 1519 u32 reqid = xnew->props.reqid; 1520 struct xfrm_state *x; 1521 unsigned int h; 1522 u32 mark = xnew->mark.v & xnew->mark.m; 1523 u32 if_id = xnew->if_id; 1524 1525 h = xfrm_dst_hash(net, &xnew->id.daddr, &xnew->props.saddr, reqid, family); 1526 hlist_for_each_entry(x, net->xfrm.state_bydst+h, bydst) { 1527 if (x->props.family == family && 1528 x->props.reqid == reqid && 1529 x->if_id == if_id && 1530 (mark & x->mark.m) == x->mark.v && 1531 xfrm_addr_equal(&x->id.daddr, &xnew->id.daddr, family) && 1532 xfrm_addr_equal(&x->props.saddr, &xnew->props.saddr, family)) 1533 x->genid++; 1534 } 1535 } 1536 1537 void xfrm_state_insert(struct xfrm_state *x) 1538 { 1539 struct net *net = xs_net(x); 1540 1541 spin_lock_bh(&net->xfrm.xfrm_state_lock); 1542 __xfrm_state_bump_genids(x); 1543 __xfrm_state_insert(x); 1544 spin_unlock_bh(&net->xfrm.xfrm_state_lock); 1545 } 1546 EXPORT_SYMBOL(xfrm_state_insert); 1547 1548 /* net->xfrm.xfrm_state_lock is held */ 1549 static struct xfrm_state *__find_acq_core(struct net *net, 1550 const struct xfrm_mark *m, 1551 unsigned short family, u8 mode, 1552 u32 reqid, u32 if_id, u8 proto, 1553 const xfrm_address_t *daddr, 1554 const xfrm_address_t *saddr, 1555 int create) 1556 { 1557 unsigned int h = xfrm_dst_hash(net, daddr, saddr, reqid, family); 1558 struct xfrm_state *x; 1559 u32 mark = m->v & m->m; 1560 1561 hlist_for_each_entry(x, net->xfrm.state_bydst+h, bydst) { 1562 if (x->props.reqid != reqid || 1563 x->props.mode != mode || 1564 x->props.family != family || 1565 x->km.state != XFRM_STATE_ACQ || 1566 x->id.spi != 0 || 1567 x->id.proto != proto || 1568 (mark & x->mark.m) != x->mark.v || 1569 !xfrm_addr_equal(&x->id.daddr, daddr, family) || 1570 !xfrm_addr_equal(&x->props.saddr, saddr, family)) 1571 continue; 1572 1573 xfrm_state_hold(x); 1574 return x; 1575 } 1576 1577 if (!create) 1578 return NULL; 1579 1580 x = xfrm_state_alloc(net); 1581 if (likely(x)) { 1582 switch (family) { 1583 case AF_INET: 1584 x->sel.daddr.a4 = daddr->a4; 1585 x->sel.saddr.a4 = saddr->a4; 1586 x->sel.prefixlen_d = 32; 1587 x->sel.prefixlen_s = 32; 1588 x->props.saddr.a4 = saddr->a4; 1589 x->id.daddr.a4 = daddr->a4; 1590 break; 1591 1592 case AF_INET6: 1593 x->sel.daddr.in6 = daddr->in6; 1594 x->sel.saddr.in6 = saddr->in6; 1595 x->sel.prefixlen_d = 128; 1596 x->sel.prefixlen_s = 128; 1597 x->props.saddr.in6 = saddr->in6; 1598 x->id.daddr.in6 = daddr->in6; 1599 break; 1600 } 1601 1602 x->km.state = XFRM_STATE_ACQ; 1603 x->id.proto = proto; 1604 x->props.family = family; 1605 x->props.mode = mode; 1606 x->props.reqid = reqid; 1607 x->if_id = if_id; 1608 x->mark.v = m->v; 1609 x->mark.m = m->m; 1610 x->lft.hard_add_expires_seconds = net->xfrm.sysctl_acq_expires; 1611 xfrm_state_hold(x); 1612 hrtimer_start(&x->mtimer, 1613 ktime_set(net->xfrm.sysctl_acq_expires, 0), 1614 HRTIMER_MODE_REL_SOFT); 1615 list_add(&x->km.all, &net->xfrm.state_all); 1616 XFRM_STATE_INSERT(bydst, &x->bydst, net->xfrm.state_bydst + h, 1617 x->xso.type); 1618 h = xfrm_src_hash(net, daddr, saddr, family); 1619 XFRM_STATE_INSERT(bysrc, &x->bysrc, net->xfrm.state_bysrc + h, 1620 x->xso.type); 1621 1622 net->xfrm.state_num++; 1623 1624 xfrm_hash_grow_check(net, x->bydst.next != NULL); 1625 } 1626 1627 return x; 1628 } 1629 1630 static struct xfrm_state *__xfrm_find_acq_byseq(struct net *net, u32 mark, u32 seq); 1631 1632 int xfrm_state_add(struct xfrm_state *x) 1633 { 1634 struct net *net = xs_net(x); 1635 struct xfrm_state *x1, *to_put; 1636 int family; 1637 int err; 1638 u32 mark = x->mark.v & x->mark.m; 1639 int use_spi = xfrm_id_proto_match(x->id.proto, IPSEC_PROTO_ANY); 1640 1641 family = x->props.family; 1642 1643 to_put = NULL; 1644 1645 spin_lock_bh(&net->xfrm.xfrm_state_lock); 1646 1647 x1 = __xfrm_state_locate(x, use_spi, family); 1648 if (x1) { 1649 to_put = x1; 1650 x1 = NULL; 1651 err = -EEXIST; 1652 goto out; 1653 } 1654 1655 if (use_spi && x->km.seq) { 1656 x1 = __xfrm_find_acq_byseq(net, mark, x->km.seq); 1657 if (x1 && ((x1->id.proto != x->id.proto) || 1658 !xfrm_addr_equal(&x1->id.daddr, &x->id.daddr, family))) { 1659 to_put = x1; 1660 x1 = NULL; 1661 } 1662 } 1663 1664 if (use_spi && !x1) 1665 x1 = __find_acq_core(net, &x->mark, family, x->props.mode, 1666 x->props.reqid, x->if_id, x->id.proto, 1667 &x->id.daddr, &x->props.saddr, 0); 1668 1669 __xfrm_state_bump_genids(x); 1670 __xfrm_state_insert(x); 1671 err = 0; 1672 1673 out: 1674 spin_unlock_bh(&net->xfrm.xfrm_state_lock); 1675 1676 if (x1) { 1677 xfrm_state_delete(x1); 1678 xfrm_state_put(x1); 1679 } 1680 1681 if (to_put) 1682 xfrm_state_put(to_put); 1683 1684 return err; 1685 } 1686 EXPORT_SYMBOL(xfrm_state_add); 1687 1688 #ifdef CONFIG_XFRM_MIGRATE 1689 static inline int clone_security(struct xfrm_state *x, struct xfrm_sec_ctx *security) 1690 { 1691 struct xfrm_user_sec_ctx *uctx; 1692 int size = sizeof(*uctx) + security->ctx_len; 1693 int err; 1694 1695 uctx = kmalloc(size, GFP_KERNEL); 1696 if (!uctx) 1697 return -ENOMEM; 1698 1699 uctx->exttype = XFRMA_SEC_CTX; 1700 uctx->len = size; 1701 uctx->ctx_doi = security->ctx_doi; 1702 uctx->ctx_alg = security->ctx_alg; 1703 uctx->ctx_len = security->ctx_len; 1704 memcpy(uctx + 1, security->ctx_str, security->ctx_len); 1705 err = security_xfrm_state_alloc(x, uctx); 1706 kfree(uctx); 1707 if (err) 1708 return err; 1709 1710 return 0; 1711 } 1712 1713 static struct xfrm_state *xfrm_state_clone(struct xfrm_state *orig, 1714 struct xfrm_encap_tmpl *encap) 1715 { 1716 struct net *net = xs_net(orig); 1717 struct xfrm_state *x = xfrm_state_alloc(net); 1718 if (!x) 1719 goto out; 1720 1721 memcpy(&x->id, &orig->id, sizeof(x->id)); 1722 memcpy(&x->sel, &orig->sel, sizeof(x->sel)); 1723 memcpy(&x->lft, &orig->lft, sizeof(x->lft)); 1724 x->props.mode = orig->props.mode; 1725 x->props.replay_window = orig->props.replay_window; 1726 x->props.reqid = orig->props.reqid; 1727 x->props.family = orig->props.family; 1728 x->props.saddr = orig->props.saddr; 1729 1730 if (orig->aalg) { 1731 x->aalg = xfrm_algo_auth_clone(orig->aalg); 1732 if (!x->aalg) 1733 goto error; 1734 } 1735 x->props.aalgo = orig->props.aalgo; 1736 1737 if (orig->aead) { 1738 x->aead = xfrm_algo_aead_clone(orig->aead); 1739 x->geniv = orig->geniv; 1740 if (!x->aead) 1741 goto error; 1742 } 1743 if (orig->ealg) { 1744 x->ealg = xfrm_algo_clone(orig->ealg); 1745 if (!x->ealg) 1746 goto error; 1747 } 1748 x->props.ealgo = orig->props.ealgo; 1749 1750 if (orig->calg) { 1751 x->calg = xfrm_algo_clone(orig->calg); 1752 if (!x->calg) 1753 goto error; 1754 } 1755 x->props.calgo = orig->props.calgo; 1756 1757 if (encap || orig->encap) { 1758 if (encap) 1759 x->encap = kmemdup(encap, sizeof(*x->encap), 1760 GFP_KERNEL); 1761 else 1762 x->encap = kmemdup(orig->encap, sizeof(*x->encap), 1763 GFP_KERNEL); 1764 1765 if (!x->encap) 1766 goto error; 1767 } 1768 1769 if (orig->security) 1770 if (clone_security(x, orig->security)) 1771 goto error; 1772 1773 if (orig->coaddr) { 1774 x->coaddr = kmemdup(orig->coaddr, sizeof(*x->coaddr), 1775 GFP_KERNEL); 1776 if (!x->coaddr) 1777 goto error; 1778 } 1779 1780 if (orig->replay_esn) { 1781 if (xfrm_replay_clone(x, orig)) 1782 goto error; 1783 } 1784 1785 memcpy(&x->mark, &orig->mark, sizeof(x->mark)); 1786 memcpy(&x->props.smark, &orig->props.smark, sizeof(x->props.smark)); 1787 1788 x->props.flags = orig->props.flags; 1789 x->props.extra_flags = orig->props.extra_flags; 1790 1791 x->if_id = orig->if_id; 1792 x->tfcpad = orig->tfcpad; 1793 x->replay_maxdiff = orig->replay_maxdiff; 1794 x->replay_maxage = orig->replay_maxage; 1795 memcpy(&x->curlft, &orig->curlft, sizeof(x->curlft)); 1796 x->km.state = orig->km.state; 1797 x->km.seq = orig->km.seq; 1798 x->replay = orig->replay; 1799 x->preplay = orig->preplay; 1800 x->mapping_maxage = orig->mapping_maxage; 1801 x->lastused = orig->lastused; 1802 x->new_mapping = 0; 1803 x->new_mapping_sport = 0; 1804 1805 return x; 1806 1807 error: 1808 xfrm_state_put(x); 1809 out: 1810 return NULL; 1811 } 1812 1813 struct xfrm_state *xfrm_migrate_state_find(struct xfrm_migrate *m, struct net *net, 1814 u32 if_id) 1815 { 1816 unsigned int h; 1817 struct xfrm_state *x = NULL; 1818 1819 spin_lock_bh(&net->xfrm.xfrm_state_lock); 1820 1821 if (m->reqid) { 1822 h = xfrm_dst_hash(net, &m->old_daddr, &m->old_saddr, 1823 m->reqid, m->old_family); 1824 hlist_for_each_entry(x, net->xfrm.state_bydst+h, bydst) { 1825 if (x->props.mode != m->mode || 1826 x->id.proto != m->proto) 1827 continue; 1828 if (m->reqid && x->props.reqid != m->reqid) 1829 continue; 1830 if (if_id != 0 && x->if_id != if_id) 1831 continue; 1832 if (!xfrm_addr_equal(&x->id.daddr, &m->old_daddr, 1833 m->old_family) || 1834 !xfrm_addr_equal(&x->props.saddr, &m->old_saddr, 1835 m->old_family)) 1836 continue; 1837 xfrm_state_hold(x); 1838 break; 1839 } 1840 } else { 1841 h = xfrm_src_hash(net, &m->old_daddr, &m->old_saddr, 1842 m->old_family); 1843 hlist_for_each_entry(x, net->xfrm.state_bysrc+h, bysrc) { 1844 if (x->props.mode != m->mode || 1845 x->id.proto != m->proto) 1846 continue; 1847 if (if_id != 0 && x->if_id != if_id) 1848 continue; 1849 if (!xfrm_addr_equal(&x->id.daddr, &m->old_daddr, 1850 m->old_family) || 1851 !xfrm_addr_equal(&x->props.saddr, &m->old_saddr, 1852 m->old_family)) 1853 continue; 1854 xfrm_state_hold(x); 1855 break; 1856 } 1857 } 1858 1859 spin_unlock_bh(&net->xfrm.xfrm_state_lock); 1860 1861 return x; 1862 } 1863 EXPORT_SYMBOL(xfrm_migrate_state_find); 1864 1865 struct xfrm_state *xfrm_state_migrate(struct xfrm_state *x, 1866 struct xfrm_migrate *m, 1867 struct xfrm_encap_tmpl *encap) 1868 { 1869 struct xfrm_state *xc; 1870 1871 xc = xfrm_state_clone(x, encap); 1872 if (!xc) 1873 return NULL; 1874 1875 xc->props.family = m->new_family; 1876 1877 if (xfrm_init_state(xc) < 0) 1878 goto error; 1879 1880 memcpy(&xc->id.daddr, &m->new_daddr, sizeof(xc->id.daddr)); 1881 memcpy(&xc->props.saddr, &m->new_saddr, sizeof(xc->props.saddr)); 1882 1883 /* add state */ 1884 if (xfrm_addr_equal(&x->id.daddr, &m->new_daddr, m->new_family)) { 1885 /* a care is needed when the destination address of the 1886 state is to be updated as it is a part of triplet */ 1887 xfrm_state_insert(xc); 1888 } else { 1889 if (xfrm_state_add(xc) < 0) 1890 goto error; 1891 } 1892 1893 return xc; 1894 error: 1895 xfrm_state_put(xc); 1896 return NULL; 1897 } 1898 EXPORT_SYMBOL(xfrm_state_migrate); 1899 #endif 1900 1901 int xfrm_state_update(struct xfrm_state *x) 1902 { 1903 struct xfrm_state *x1, *to_put; 1904 int err; 1905 int use_spi = xfrm_id_proto_match(x->id.proto, IPSEC_PROTO_ANY); 1906 struct net *net = xs_net(x); 1907 1908 to_put = NULL; 1909 1910 spin_lock_bh(&net->xfrm.xfrm_state_lock); 1911 x1 = __xfrm_state_locate(x, use_spi, x->props.family); 1912 1913 err = -ESRCH; 1914 if (!x1) 1915 goto out; 1916 1917 if (xfrm_state_kern(x1)) { 1918 to_put = x1; 1919 err = -EEXIST; 1920 goto out; 1921 } 1922 1923 if (x1->km.state == XFRM_STATE_ACQ) { 1924 __xfrm_state_insert(x); 1925 x = NULL; 1926 } 1927 err = 0; 1928 1929 out: 1930 spin_unlock_bh(&net->xfrm.xfrm_state_lock); 1931 1932 if (to_put) 1933 xfrm_state_put(to_put); 1934 1935 if (err) 1936 return err; 1937 1938 if (!x) { 1939 xfrm_state_delete(x1); 1940 xfrm_state_put(x1); 1941 return 0; 1942 } 1943 1944 err = -EINVAL; 1945 spin_lock_bh(&x1->lock); 1946 if (likely(x1->km.state == XFRM_STATE_VALID)) { 1947 if (x->encap && x1->encap && 1948 x->encap->encap_type == x1->encap->encap_type) 1949 memcpy(x1->encap, x->encap, sizeof(*x1->encap)); 1950 else if (x->encap || x1->encap) 1951 goto fail; 1952 1953 if (x->coaddr && x1->coaddr) { 1954 memcpy(x1->coaddr, x->coaddr, sizeof(*x1->coaddr)); 1955 } 1956 if (!use_spi && memcmp(&x1->sel, &x->sel, sizeof(x1->sel))) 1957 memcpy(&x1->sel, &x->sel, sizeof(x1->sel)); 1958 memcpy(&x1->lft, &x->lft, sizeof(x1->lft)); 1959 x1->km.dying = 0; 1960 1961 hrtimer_start(&x1->mtimer, ktime_set(1, 0), 1962 HRTIMER_MODE_REL_SOFT); 1963 if (READ_ONCE(x1->curlft.use_time)) 1964 xfrm_state_check_expire(x1); 1965 1966 if (x->props.smark.m || x->props.smark.v || x->if_id) { 1967 spin_lock_bh(&net->xfrm.xfrm_state_lock); 1968 1969 if (x->props.smark.m || x->props.smark.v) 1970 x1->props.smark = x->props.smark; 1971 1972 if (x->if_id) 1973 x1->if_id = x->if_id; 1974 1975 __xfrm_state_bump_genids(x1); 1976 spin_unlock_bh(&net->xfrm.xfrm_state_lock); 1977 } 1978 1979 err = 0; 1980 x->km.state = XFRM_STATE_DEAD; 1981 __xfrm_state_put(x); 1982 } 1983 1984 fail: 1985 spin_unlock_bh(&x1->lock); 1986 1987 xfrm_state_put(x1); 1988 1989 return err; 1990 } 1991 EXPORT_SYMBOL(xfrm_state_update); 1992 1993 int xfrm_state_check_expire(struct xfrm_state *x) 1994 { 1995 xfrm_dev_state_update_curlft(x); 1996 1997 if (!READ_ONCE(x->curlft.use_time)) 1998 WRITE_ONCE(x->curlft.use_time, ktime_get_real_seconds()); 1999 2000 if (x->curlft.bytes >= x->lft.hard_byte_limit || 2001 x->curlft.packets >= x->lft.hard_packet_limit) { 2002 x->km.state = XFRM_STATE_EXPIRED; 2003 hrtimer_start(&x->mtimer, 0, HRTIMER_MODE_REL_SOFT); 2004 return -EINVAL; 2005 } 2006 2007 if (!x->km.dying && 2008 (x->curlft.bytes >= x->lft.soft_byte_limit || 2009 x->curlft.packets >= x->lft.soft_packet_limit)) { 2010 x->km.dying = 1; 2011 km_state_expired(x, 0, 0); 2012 } 2013 return 0; 2014 } 2015 EXPORT_SYMBOL(xfrm_state_check_expire); 2016 2017 struct xfrm_state * 2018 xfrm_state_lookup(struct net *net, u32 mark, const xfrm_address_t *daddr, __be32 spi, 2019 u8 proto, unsigned short family) 2020 { 2021 struct xfrm_state *x; 2022 2023 rcu_read_lock(); 2024 x = __xfrm_state_lookup(net, mark, daddr, spi, proto, family); 2025 rcu_read_unlock(); 2026 return x; 2027 } 2028 EXPORT_SYMBOL(xfrm_state_lookup); 2029 2030 struct xfrm_state * 2031 xfrm_state_lookup_byaddr(struct net *net, u32 mark, 2032 const xfrm_address_t *daddr, const xfrm_address_t *saddr, 2033 u8 proto, unsigned short family) 2034 { 2035 struct xfrm_state *x; 2036 2037 spin_lock_bh(&net->xfrm.xfrm_state_lock); 2038 x = __xfrm_state_lookup_byaddr(net, mark, daddr, saddr, proto, family); 2039 spin_unlock_bh(&net->xfrm.xfrm_state_lock); 2040 return x; 2041 } 2042 EXPORT_SYMBOL(xfrm_state_lookup_byaddr); 2043 2044 struct xfrm_state * 2045 xfrm_find_acq(struct net *net, const struct xfrm_mark *mark, u8 mode, u32 reqid, 2046 u32 if_id, u8 proto, const xfrm_address_t *daddr, 2047 const xfrm_address_t *saddr, int create, unsigned short family) 2048 { 2049 struct xfrm_state *x; 2050 2051 spin_lock_bh(&net->xfrm.xfrm_state_lock); 2052 x = __find_acq_core(net, mark, family, mode, reqid, if_id, proto, daddr, saddr, create); 2053 spin_unlock_bh(&net->xfrm.xfrm_state_lock); 2054 2055 return x; 2056 } 2057 EXPORT_SYMBOL(xfrm_find_acq); 2058 2059 #ifdef CONFIG_XFRM_SUB_POLICY 2060 #if IS_ENABLED(CONFIG_IPV6) 2061 /* distribution counting sort function for xfrm_state and xfrm_tmpl */ 2062 static void 2063 __xfrm6_sort(void **dst, void **src, int n, 2064 int (*cmp)(const void *p), int maxclass) 2065 { 2066 int count[XFRM_MAX_DEPTH] = { }; 2067 int class[XFRM_MAX_DEPTH]; 2068 int i; 2069 2070 for (i = 0; i < n; i++) { 2071 int c = cmp(src[i]); 2072 2073 class[i] = c; 2074 count[c]++; 2075 } 2076 2077 for (i = 2; i < maxclass; i++) 2078 count[i] += count[i - 1]; 2079 2080 for (i = 0; i < n; i++) { 2081 dst[count[class[i] - 1]++] = src[i]; 2082 src[i] = NULL; 2083 } 2084 } 2085 2086 /* Rule for xfrm_state: 2087 * 2088 * rule 1: select IPsec transport except AH 2089 * rule 2: select MIPv6 RO or inbound trigger 2090 * rule 3: select IPsec transport AH 2091 * rule 4: select IPsec tunnel 2092 * rule 5: others 2093 */ 2094 static int __xfrm6_state_sort_cmp(const void *p) 2095 { 2096 const struct xfrm_state *v = p; 2097 2098 switch (v->props.mode) { 2099 case XFRM_MODE_TRANSPORT: 2100 if (v->id.proto != IPPROTO_AH) 2101 return 1; 2102 else 2103 return 3; 2104 #if IS_ENABLED(CONFIG_IPV6_MIP6) 2105 case XFRM_MODE_ROUTEOPTIMIZATION: 2106 case XFRM_MODE_IN_TRIGGER: 2107 return 2; 2108 #endif 2109 case XFRM_MODE_TUNNEL: 2110 case XFRM_MODE_BEET: 2111 return 4; 2112 } 2113 return 5; 2114 } 2115 2116 /* Rule for xfrm_tmpl: 2117 * 2118 * rule 1: select IPsec transport 2119 * rule 2: select MIPv6 RO or inbound trigger 2120 * rule 3: select IPsec tunnel 2121 * rule 4: others 2122 */ 2123 static int __xfrm6_tmpl_sort_cmp(const void *p) 2124 { 2125 const struct xfrm_tmpl *v = p; 2126 2127 switch (v->mode) { 2128 case XFRM_MODE_TRANSPORT: 2129 return 1; 2130 #if IS_ENABLED(CONFIG_IPV6_MIP6) 2131 case XFRM_MODE_ROUTEOPTIMIZATION: 2132 case XFRM_MODE_IN_TRIGGER: 2133 return 2; 2134 #endif 2135 case XFRM_MODE_TUNNEL: 2136 case XFRM_MODE_BEET: 2137 return 3; 2138 } 2139 return 4; 2140 } 2141 #else 2142 static inline int __xfrm6_state_sort_cmp(const void *p) { return 5; } 2143 static inline int __xfrm6_tmpl_sort_cmp(const void *p) { return 4; } 2144 2145 static inline void 2146 __xfrm6_sort(void **dst, void **src, int n, 2147 int (*cmp)(const void *p), int maxclass) 2148 { 2149 int i; 2150 2151 for (i = 0; i < n; i++) 2152 dst[i] = src[i]; 2153 } 2154 #endif /* CONFIG_IPV6 */ 2155 2156 void 2157 xfrm_tmpl_sort(struct xfrm_tmpl **dst, struct xfrm_tmpl **src, int n, 2158 unsigned short family) 2159 { 2160 int i; 2161 2162 if (family == AF_INET6) 2163 __xfrm6_sort((void **)dst, (void **)src, n, 2164 __xfrm6_tmpl_sort_cmp, 5); 2165 else 2166 for (i = 0; i < n; i++) 2167 dst[i] = src[i]; 2168 } 2169 2170 void 2171 xfrm_state_sort(struct xfrm_state **dst, struct xfrm_state **src, int n, 2172 unsigned short family) 2173 { 2174 int i; 2175 2176 if (family == AF_INET6) 2177 __xfrm6_sort((void **)dst, (void **)src, n, 2178 __xfrm6_state_sort_cmp, 6); 2179 else 2180 for (i = 0; i < n; i++) 2181 dst[i] = src[i]; 2182 } 2183 #endif 2184 2185 /* Silly enough, but I'm lazy to build resolution list */ 2186 2187 static struct xfrm_state *__xfrm_find_acq_byseq(struct net *net, u32 mark, u32 seq) 2188 { 2189 unsigned int h = xfrm_seq_hash(net, seq); 2190 struct xfrm_state *x; 2191 2192 hlist_for_each_entry_rcu(x, net->xfrm.state_byseq + h, byseq) { 2193 if (x->km.seq == seq && 2194 (mark & x->mark.m) == x->mark.v && 2195 x->km.state == XFRM_STATE_ACQ) { 2196 xfrm_state_hold(x); 2197 return x; 2198 } 2199 } 2200 2201 return NULL; 2202 } 2203 2204 struct xfrm_state *xfrm_find_acq_byseq(struct net *net, u32 mark, u32 seq) 2205 { 2206 struct xfrm_state *x; 2207 2208 spin_lock_bh(&net->xfrm.xfrm_state_lock); 2209 x = __xfrm_find_acq_byseq(net, mark, seq); 2210 spin_unlock_bh(&net->xfrm.xfrm_state_lock); 2211 return x; 2212 } 2213 EXPORT_SYMBOL(xfrm_find_acq_byseq); 2214 2215 u32 xfrm_get_acqseq(void) 2216 { 2217 u32 res; 2218 static atomic_t acqseq; 2219 2220 do { 2221 res = atomic_inc_return(&acqseq); 2222 } while (!res); 2223 2224 return res; 2225 } 2226 EXPORT_SYMBOL(xfrm_get_acqseq); 2227 2228 int verify_spi_info(u8 proto, u32 min, u32 max, struct netlink_ext_ack *extack) 2229 { 2230 switch (proto) { 2231 case IPPROTO_AH: 2232 case IPPROTO_ESP: 2233 break; 2234 2235 case IPPROTO_COMP: 2236 /* IPCOMP spi is 16-bits. */ 2237 if (max >= 0x10000) { 2238 NL_SET_ERR_MSG(extack, "IPCOMP SPI must be <= 65535"); 2239 return -EINVAL; 2240 } 2241 break; 2242 2243 default: 2244 NL_SET_ERR_MSG(extack, "Invalid protocol, must be one of AH, ESP, IPCOMP"); 2245 return -EINVAL; 2246 } 2247 2248 if (min > max) { 2249 NL_SET_ERR_MSG(extack, "Invalid SPI range: min > max"); 2250 return -EINVAL; 2251 } 2252 2253 return 0; 2254 } 2255 EXPORT_SYMBOL(verify_spi_info); 2256 2257 int xfrm_alloc_spi(struct xfrm_state *x, u32 low, u32 high, 2258 struct netlink_ext_ack *extack) 2259 { 2260 struct net *net = xs_net(x); 2261 unsigned int h; 2262 struct xfrm_state *x0; 2263 int err = -ENOENT; 2264 __be32 minspi = htonl(low); 2265 __be32 maxspi = htonl(high); 2266 __be32 newspi = 0; 2267 u32 mark = x->mark.v & x->mark.m; 2268 2269 spin_lock_bh(&x->lock); 2270 if (x->km.state == XFRM_STATE_DEAD) { 2271 NL_SET_ERR_MSG(extack, "Target ACQUIRE is in DEAD state"); 2272 goto unlock; 2273 } 2274 2275 err = 0; 2276 if (x->id.spi) 2277 goto unlock; 2278 2279 err = -ENOENT; 2280 2281 if (minspi == maxspi) { 2282 x0 = xfrm_state_lookup(net, mark, &x->id.daddr, minspi, x->id.proto, x->props.family); 2283 if (x0) { 2284 NL_SET_ERR_MSG(extack, "Requested SPI is already in use"); 2285 xfrm_state_put(x0); 2286 goto unlock; 2287 } 2288 newspi = minspi; 2289 } else { 2290 u32 spi = 0; 2291 for (h = 0; h < high-low+1; h++) { 2292 spi = get_random_u32_inclusive(low, high); 2293 x0 = xfrm_state_lookup(net, mark, &x->id.daddr, htonl(spi), x->id.proto, x->props.family); 2294 if (x0 == NULL) { 2295 newspi = htonl(spi); 2296 break; 2297 } 2298 xfrm_state_put(x0); 2299 } 2300 } 2301 if (newspi) { 2302 spin_lock_bh(&net->xfrm.xfrm_state_lock); 2303 x->id.spi = newspi; 2304 h = xfrm_spi_hash(net, &x->id.daddr, x->id.spi, x->id.proto, x->props.family); 2305 XFRM_STATE_INSERT(byspi, &x->byspi, net->xfrm.state_byspi + h, 2306 x->xso.type); 2307 spin_unlock_bh(&net->xfrm.xfrm_state_lock); 2308 2309 err = 0; 2310 } else { 2311 NL_SET_ERR_MSG(extack, "No SPI available in the requested range"); 2312 } 2313 2314 unlock: 2315 spin_unlock_bh(&x->lock); 2316 2317 return err; 2318 } 2319 EXPORT_SYMBOL(xfrm_alloc_spi); 2320 2321 static bool __xfrm_state_filter_match(struct xfrm_state *x, 2322 struct xfrm_address_filter *filter) 2323 { 2324 if (filter) { 2325 if ((filter->family == AF_INET || 2326 filter->family == AF_INET6) && 2327 x->props.family != filter->family) 2328 return false; 2329 2330 return addr_match(&x->props.saddr, &filter->saddr, 2331 filter->splen) && 2332 addr_match(&x->id.daddr, &filter->daddr, 2333 filter->dplen); 2334 } 2335 return true; 2336 } 2337 2338 int xfrm_state_walk(struct net *net, struct xfrm_state_walk *walk, 2339 int (*func)(struct xfrm_state *, int, void*), 2340 void *data) 2341 { 2342 struct xfrm_state *state; 2343 struct xfrm_state_walk *x; 2344 int err = 0; 2345 2346 if (walk->seq != 0 && list_empty(&walk->all)) 2347 return 0; 2348 2349 spin_lock_bh(&net->xfrm.xfrm_state_lock); 2350 if (list_empty(&walk->all)) 2351 x = list_first_entry(&net->xfrm.state_all, struct xfrm_state_walk, all); 2352 else 2353 x = list_first_entry(&walk->all, struct xfrm_state_walk, all); 2354 list_for_each_entry_from(x, &net->xfrm.state_all, all) { 2355 if (x->state == XFRM_STATE_DEAD) 2356 continue; 2357 state = container_of(x, struct xfrm_state, km); 2358 if (!xfrm_id_proto_match(state->id.proto, walk->proto)) 2359 continue; 2360 if (!__xfrm_state_filter_match(state, walk->filter)) 2361 continue; 2362 err = func(state, walk->seq, data); 2363 if (err) { 2364 list_move_tail(&walk->all, &x->all); 2365 goto out; 2366 } 2367 walk->seq++; 2368 } 2369 if (walk->seq == 0) { 2370 err = -ENOENT; 2371 goto out; 2372 } 2373 list_del_init(&walk->all); 2374 out: 2375 spin_unlock_bh(&net->xfrm.xfrm_state_lock); 2376 return err; 2377 } 2378 EXPORT_SYMBOL(xfrm_state_walk); 2379 2380 void xfrm_state_walk_init(struct xfrm_state_walk *walk, u8 proto, 2381 struct xfrm_address_filter *filter) 2382 { 2383 INIT_LIST_HEAD(&walk->all); 2384 walk->proto = proto; 2385 walk->state = XFRM_STATE_DEAD; 2386 walk->seq = 0; 2387 walk->filter = filter; 2388 } 2389 EXPORT_SYMBOL(xfrm_state_walk_init); 2390 2391 void xfrm_state_walk_done(struct xfrm_state_walk *walk, struct net *net) 2392 { 2393 kfree(walk->filter); 2394 2395 if (list_empty(&walk->all)) 2396 return; 2397 2398 spin_lock_bh(&net->xfrm.xfrm_state_lock); 2399 list_del(&walk->all); 2400 spin_unlock_bh(&net->xfrm.xfrm_state_lock); 2401 } 2402 EXPORT_SYMBOL(xfrm_state_walk_done); 2403 2404 static void xfrm_replay_timer_handler(struct timer_list *t) 2405 { 2406 struct xfrm_state *x = from_timer(x, t, rtimer); 2407 2408 spin_lock(&x->lock); 2409 2410 if (x->km.state == XFRM_STATE_VALID) { 2411 if (xfrm_aevent_is_on(xs_net(x))) 2412 xfrm_replay_notify(x, XFRM_REPLAY_TIMEOUT); 2413 else 2414 x->xflags |= XFRM_TIME_DEFER; 2415 } 2416 2417 spin_unlock(&x->lock); 2418 } 2419 2420 static LIST_HEAD(xfrm_km_list); 2421 2422 void km_policy_notify(struct xfrm_policy *xp, int dir, const struct km_event *c) 2423 { 2424 struct xfrm_mgr *km; 2425 2426 rcu_read_lock(); 2427 list_for_each_entry_rcu(km, &xfrm_km_list, list) 2428 if (km->notify_policy) 2429 km->notify_policy(xp, dir, c); 2430 rcu_read_unlock(); 2431 } 2432 2433 void km_state_notify(struct xfrm_state *x, const struct km_event *c) 2434 { 2435 struct xfrm_mgr *km; 2436 rcu_read_lock(); 2437 list_for_each_entry_rcu(km, &xfrm_km_list, list) 2438 if (km->notify) 2439 km->notify(x, c); 2440 rcu_read_unlock(); 2441 } 2442 2443 EXPORT_SYMBOL(km_policy_notify); 2444 EXPORT_SYMBOL(km_state_notify); 2445 2446 void km_state_expired(struct xfrm_state *x, int hard, u32 portid) 2447 { 2448 struct km_event c; 2449 2450 c.data.hard = hard; 2451 c.portid = portid; 2452 c.event = XFRM_MSG_EXPIRE; 2453 km_state_notify(x, &c); 2454 } 2455 2456 EXPORT_SYMBOL(km_state_expired); 2457 /* 2458 * We send to all registered managers regardless of failure 2459 * We are happy with one success 2460 */ 2461 int km_query(struct xfrm_state *x, struct xfrm_tmpl *t, struct xfrm_policy *pol) 2462 { 2463 int err = -EINVAL, acqret; 2464 struct xfrm_mgr *km; 2465 2466 rcu_read_lock(); 2467 list_for_each_entry_rcu(km, &xfrm_km_list, list) { 2468 acqret = km->acquire(x, t, pol); 2469 if (!acqret) 2470 err = acqret; 2471 } 2472 rcu_read_unlock(); 2473 return err; 2474 } 2475 EXPORT_SYMBOL(km_query); 2476 2477 static int __km_new_mapping(struct xfrm_state *x, xfrm_address_t *ipaddr, __be16 sport) 2478 { 2479 int err = -EINVAL; 2480 struct xfrm_mgr *km; 2481 2482 rcu_read_lock(); 2483 list_for_each_entry_rcu(km, &xfrm_km_list, list) { 2484 if (km->new_mapping) 2485 err = km->new_mapping(x, ipaddr, sport); 2486 if (!err) 2487 break; 2488 } 2489 rcu_read_unlock(); 2490 return err; 2491 } 2492 2493 int km_new_mapping(struct xfrm_state *x, xfrm_address_t *ipaddr, __be16 sport) 2494 { 2495 int ret = 0; 2496 2497 if (x->mapping_maxage) { 2498 if ((jiffies / HZ - x->new_mapping) > x->mapping_maxage || 2499 x->new_mapping_sport != sport) { 2500 x->new_mapping_sport = sport; 2501 x->new_mapping = jiffies / HZ; 2502 ret = __km_new_mapping(x, ipaddr, sport); 2503 } 2504 } else { 2505 ret = __km_new_mapping(x, ipaddr, sport); 2506 } 2507 2508 return ret; 2509 } 2510 EXPORT_SYMBOL(km_new_mapping); 2511 2512 void km_policy_expired(struct xfrm_policy *pol, int dir, int hard, u32 portid) 2513 { 2514 struct km_event c; 2515 2516 c.data.hard = hard; 2517 c.portid = portid; 2518 c.event = XFRM_MSG_POLEXPIRE; 2519 km_policy_notify(pol, dir, &c); 2520 } 2521 EXPORT_SYMBOL(km_policy_expired); 2522 2523 #ifdef CONFIG_XFRM_MIGRATE 2524 int km_migrate(const struct xfrm_selector *sel, u8 dir, u8 type, 2525 const struct xfrm_migrate *m, int num_migrate, 2526 const struct xfrm_kmaddress *k, 2527 const struct xfrm_encap_tmpl *encap) 2528 { 2529 int err = -EINVAL; 2530 int ret; 2531 struct xfrm_mgr *km; 2532 2533 rcu_read_lock(); 2534 list_for_each_entry_rcu(km, &xfrm_km_list, list) { 2535 if (km->migrate) { 2536 ret = km->migrate(sel, dir, type, m, num_migrate, k, 2537 encap); 2538 if (!ret) 2539 err = ret; 2540 } 2541 } 2542 rcu_read_unlock(); 2543 return err; 2544 } 2545 EXPORT_SYMBOL(km_migrate); 2546 #endif 2547 2548 int km_report(struct net *net, u8 proto, struct xfrm_selector *sel, xfrm_address_t *addr) 2549 { 2550 int err = -EINVAL; 2551 int ret; 2552 struct xfrm_mgr *km; 2553 2554 rcu_read_lock(); 2555 list_for_each_entry_rcu(km, &xfrm_km_list, list) { 2556 if (km->report) { 2557 ret = km->report(net, proto, sel, addr); 2558 if (!ret) 2559 err = ret; 2560 } 2561 } 2562 rcu_read_unlock(); 2563 return err; 2564 } 2565 EXPORT_SYMBOL(km_report); 2566 2567 static bool km_is_alive(const struct km_event *c) 2568 { 2569 struct xfrm_mgr *km; 2570 bool is_alive = false; 2571 2572 rcu_read_lock(); 2573 list_for_each_entry_rcu(km, &xfrm_km_list, list) { 2574 if (km->is_alive && km->is_alive(c)) { 2575 is_alive = true; 2576 break; 2577 } 2578 } 2579 rcu_read_unlock(); 2580 2581 return is_alive; 2582 } 2583 2584 #if IS_ENABLED(CONFIG_XFRM_USER_COMPAT) 2585 static DEFINE_SPINLOCK(xfrm_translator_lock); 2586 static struct xfrm_translator __rcu *xfrm_translator; 2587 2588 struct xfrm_translator *xfrm_get_translator(void) 2589 { 2590 struct xfrm_translator *xtr; 2591 2592 rcu_read_lock(); 2593 xtr = rcu_dereference(xfrm_translator); 2594 if (unlikely(!xtr)) 2595 goto out; 2596 if (!try_module_get(xtr->owner)) 2597 xtr = NULL; 2598 out: 2599 rcu_read_unlock(); 2600 return xtr; 2601 } 2602 EXPORT_SYMBOL_GPL(xfrm_get_translator); 2603 2604 void xfrm_put_translator(struct xfrm_translator *xtr) 2605 { 2606 module_put(xtr->owner); 2607 } 2608 EXPORT_SYMBOL_GPL(xfrm_put_translator); 2609 2610 int xfrm_register_translator(struct xfrm_translator *xtr) 2611 { 2612 int err = 0; 2613 2614 spin_lock_bh(&xfrm_translator_lock); 2615 if (unlikely(xfrm_translator != NULL)) 2616 err = -EEXIST; 2617 else 2618 rcu_assign_pointer(xfrm_translator, xtr); 2619 spin_unlock_bh(&xfrm_translator_lock); 2620 2621 return err; 2622 } 2623 EXPORT_SYMBOL_GPL(xfrm_register_translator); 2624 2625 int xfrm_unregister_translator(struct xfrm_translator *xtr) 2626 { 2627 int err = 0; 2628 2629 spin_lock_bh(&xfrm_translator_lock); 2630 if (likely(xfrm_translator != NULL)) { 2631 if (rcu_access_pointer(xfrm_translator) != xtr) 2632 err = -EINVAL; 2633 else 2634 RCU_INIT_POINTER(xfrm_translator, NULL); 2635 } 2636 spin_unlock_bh(&xfrm_translator_lock); 2637 synchronize_rcu(); 2638 2639 return err; 2640 } 2641 EXPORT_SYMBOL_GPL(xfrm_unregister_translator); 2642 #endif 2643 2644 int xfrm_user_policy(struct sock *sk, int optname, sockptr_t optval, int optlen) 2645 { 2646 int err; 2647 u8 *data; 2648 struct xfrm_mgr *km; 2649 struct xfrm_policy *pol = NULL; 2650 2651 if (sockptr_is_null(optval) && !optlen) { 2652 xfrm_sk_policy_insert(sk, XFRM_POLICY_IN, NULL); 2653 xfrm_sk_policy_insert(sk, XFRM_POLICY_OUT, NULL); 2654 __sk_dst_reset(sk); 2655 return 0; 2656 } 2657 2658 if (optlen <= 0 || optlen > PAGE_SIZE) 2659 return -EMSGSIZE; 2660 2661 data = memdup_sockptr(optval, optlen); 2662 if (IS_ERR(data)) 2663 return PTR_ERR(data); 2664 2665 if (in_compat_syscall()) { 2666 struct xfrm_translator *xtr = xfrm_get_translator(); 2667 2668 if (!xtr) { 2669 kfree(data); 2670 return -EOPNOTSUPP; 2671 } 2672 2673 err = xtr->xlate_user_policy_sockptr(&data, optlen); 2674 xfrm_put_translator(xtr); 2675 if (err) { 2676 kfree(data); 2677 return err; 2678 } 2679 } 2680 2681 err = -EINVAL; 2682 rcu_read_lock(); 2683 list_for_each_entry_rcu(km, &xfrm_km_list, list) { 2684 pol = km->compile_policy(sk, optname, data, 2685 optlen, &err); 2686 if (err >= 0) 2687 break; 2688 } 2689 rcu_read_unlock(); 2690 2691 if (err >= 0) { 2692 xfrm_sk_policy_insert(sk, err, pol); 2693 xfrm_pol_put(pol); 2694 __sk_dst_reset(sk); 2695 err = 0; 2696 } 2697 2698 kfree(data); 2699 return err; 2700 } 2701 EXPORT_SYMBOL(xfrm_user_policy); 2702 2703 static DEFINE_SPINLOCK(xfrm_km_lock); 2704 2705 void xfrm_register_km(struct xfrm_mgr *km) 2706 { 2707 spin_lock_bh(&xfrm_km_lock); 2708 list_add_tail_rcu(&km->list, &xfrm_km_list); 2709 spin_unlock_bh(&xfrm_km_lock); 2710 } 2711 EXPORT_SYMBOL(xfrm_register_km); 2712 2713 void xfrm_unregister_km(struct xfrm_mgr *km) 2714 { 2715 spin_lock_bh(&xfrm_km_lock); 2716 list_del_rcu(&km->list); 2717 spin_unlock_bh(&xfrm_km_lock); 2718 synchronize_rcu(); 2719 } 2720 EXPORT_SYMBOL(xfrm_unregister_km); 2721 2722 int xfrm_state_register_afinfo(struct xfrm_state_afinfo *afinfo) 2723 { 2724 int err = 0; 2725 2726 if (WARN_ON(afinfo->family >= NPROTO)) 2727 return -EAFNOSUPPORT; 2728 2729 spin_lock_bh(&xfrm_state_afinfo_lock); 2730 if (unlikely(xfrm_state_afinfo[afinfo->family] != NULL)) 2731 err = -EEXIST; 2732 else 2733 rcu_assign_pointer(xfrm_state_afinfo[afinfo->family], afinfo); 2734 spin_unlock_bh(&xfrm_state_afinfo_lock); 2735 return err; 2736 } 2737 EXPORT_SYMBOL(xfrm_state_register_afinfo); 2738 2739 int xfrm_state_unregister_afinfo(struct xfrm_state_afinfo *afinfo) 2740 { 2741 int err = 0, family = afinfo->family; 2742 2743 if (WARN_ON(family >= NPROTO)) 2744 return -EAFNOSUPPORT; 2745 2746 spin_lock_bh(&xfrm_state_afinfo_lock); 2747 if (likely(xfrm_state_afinfo[afinfo->family] != NULL)) { 2748 if (rcu_access_pointer(xfrm_state_afinfo[family]) != afinfo) 2749 err = -EINVAL; 2750 else 2751 RCU_INIT_POINTER(xfrm_state_afinfo[afinfo->family], NULL); 2752 } 2753 spin_unlock_bh(&xfrm_state_afinfo_lock); 2754 synchronize_rcu(); 2755 return err; 2756 } 2757 EXPORT_SYMBOL(xfrm_state_unregister_afinfo); 2758 2759 struct xfrm_state_afinfo *xfrm_state_afinfo_get_rcu(unsigned int family) 2760 { 2761 if (unlikely(family >= NPROTO)) 2762 return NULL; 2763 2764 return rcu_dereference(xfrm_state_afinfo[family]); 2765 } 2766 EXPORT_SYMBOL_GPL(xfrm_state_afinfo_get_rcu); 2767 2768 struct xfrm_state_afinfo *xfrm_state_get_afinfo(unsigned int family) 2769 { 2770 struct xfrm_state_afinfo *afinfo; 2771 if (unlikely(family >= NPROTO)) 2772 return NULL; 2773 rcu_read_lock(); 2774 afinfo = rcu_dereference(xfrm_state_afinfo[family]); 2775 if (unlikely(!afinfo)) 2776 rcu_read_unlock(); 2777 return afinfo; 2778 } 2779 2780 void xfrm_flush_gc(void) 2781 { 2782 flush_work(&xfrm_state_gc_work); 2783 } 2784 EXPORT_SYMBOL(xfrm_flush_gc); 2785 2786 /* Temporarily located here until net/xfrm/xfrm_tunnel.c is created */ 2787 void xfrm_state_delete_tunnel(struct xfrm_state *x) 2788 { 2789 if (x->tunnel) { 2790 struct xfrm_state *t = x->tunnel; 2791 2792 if (atomic_read(&t->tunnel_users) == 2) 2793 xfrm_state_delete(t); 2794 atomic_dec(&t->tunnel_users); 2795 xfrm_state_put_sync(t); 2796 x->tunnel = NULL; 2797 } 2798 } 2799 EXPORT_SYMBOL(xfrm_state_delete_tunnel); 2800 2801 u32 xfrm_state_mtu(struct xfrm_state *x, int mtu) 2802 { 2803 const struct xfrm_type *type = READ_ONCE(x->type); 2804 struct crypto_aead *aead; 2805 u32 blksize, net_adj = 0; 2806 2807 if (x->km.state != XFRM_STATE_VALID || 2808 !type || type->proto != IPPROTO_ESP) 2809 return mtu - x->props.header_len; 2810 2811 aead = x->data; 2812 blksize = ALIGN(crypto_aead_blocksize(aead), 4); 2813 2814 switch (x->props.mode) { 2815 case XFRM_MODE_TRANSPORT: 2816 case XFRM_MODE_BEET: 2817 if (x->props.family == AF_INET) 2818 net_adj = sizeof(struct iphdr); 2819 else if (x->props.family == AF_INET6) 2820 net_adj = sizeof(struct ipv6hdr); 2821 break; 2822 case XFRM_MODE_TUNNEL: 2823 break; 2824 default: 2825 WARN_ON_ONCE(1); 2826 break; 2827 } 2828 2829 return ((mtu - x->props.header_len - crypto_aead_authsize(aead) - 2830 net_adj) & ~(blksize - 1)) + net_adj - 2; 2831 } 2832 EXPORT_SYMBOL_GPL(xfrm_state_mtu); 2833 2834 int __xfrm_init_state(struct xfrm_state *x, bool init_replay, bool offload, 2835 struct netlink_ext_ack *extack) 2836 { 2837 const struct xfrm_mode *inner_mode; 2838 const struct xfrm_mode *outer_mode; 2839 int family = x->props.family; 2840 int err; 2841 2842 if (family == AF_INET && 2843 READ_ONCE(xs_net(x)->ipv4.sysctl_ip_no_pmtu_disc)) 2844 x->props.flags |= XFRM_STATE_NOPMTUDISC; 2845 2846 err = -EPROTONOSUPPORT; 2847 2848 if (x->sel.family != AF_UNSPEC) { 2849 inner_mode = xfrm_get_mode(x->props.mode, x->sel.family); 2850 if (inner_mode == NULL) { 2851 NL_SET_ERR_MSG(extack, "Requested mode not found"); 2852 goto error; 2853 } 2854 2855 if (!(inner_mode->flags & XFRM_MODE_FLAG_TUNNEL) && 2856 family != x->sel.family) { 2857 NL_SET_ERR_MSG(extack, "Only tunnel modes can accommodate a change of family"); 2858 goto error; 2859 } 2860 2861 x->inner_mode = *inner_mode; 2862 } else { 2863 const struct xfrm_mode *inner_mode_iaf; 2864 int iafamily = AF_INET; 2865 2866 inner_mode = xfrm_get_mode(x->props.mode, x->props.family); 2867 if (inner_mode == NULL) { 2868 NL_SET_ERR_MSG(extack, "Requested mode not found"); 2869 goto error; 2870 } 2871 2872 x->inner_mode = *inner_mode; 2873 2874 if (x->props.family == AF_INET) 2875 iafamily = AF_INET6; 2876 2877 inner_mode_iaf = xfrm_get_mode(x->props.mode, iafamily); 2878 if (inner_mode_iaf) { 2879 if (inner_mode_iaf->flags & XFRM_MODE_FLAG_TUNNEL) 2880 x->inner_mode_iaf = *inner_mode_iaf; 2881 } 2882 } 2883 2884 x->type = xfrm_get_type(x->id.proto, family); 2885 if (x->type == NULL) { 2886 NL_SET_ERR_MSG(extack, "Requested type not found"); 2887 goto error; 2888 } 2889 2890 x->type_offload = xfrm_get_type_offload(x->id.proto, family, offload); 2891 2892 err = x->type->init_state(x, extack); 2893 if (err) 2894 goto error; 2895 2896 outer_mode = xfrm_get_mode(x->props.mode, family); 2897 if (!outer_mode) { 2898 NL_SET_ERR_MSG(extack, "Requested mode not found"); 2899 err = -EPROTONOSUPPORT; 2900 goto error; 2901 } 2902 2903 x->outer_mode = *outer_mode; 2904 if (init_replay) { 2905 err = xfrm_init_replay(x, extack); 2906 if (err) 2907 goto error; 2908 } 2909 2910 error: 2911 return err; 2912 } 2913 2914 EXPORT_SYMBOL(__xfrm_init_state); 2915 2916 int xfrm_init_state(struct xfrm_state *x) 2917 { 2918 int err; 2919 2920 err = __xfrm_init_state(x, true, false, NULL); 2921 if (!err) 2922 x->km.state = XFRM_STATE_VALID; 2923 2924 return err; 2925 } 2926 2927 EXPORT_SYMBOL(xfrm_init_state); 2928 2929 int __net_init xfrm_state_init(struct net *net) 2930 { 2931 unsigned int sz; 2932 2933 if (net_eq(net, &init_net)) 2934 xfrm_state_cache = KMEM_CACHE(xfrm_state, 2935 SLAB_HWCACHE_ALIGN | SLAB_PANIC); 2936 2937 INIT_LIST_HEAD(&net->xfrm.state_all); 2938 2939 sz = sizeof(struct hlist_head) * 8; 2940 2941 net->xfrm.state_bydst = xfrm_hash_alloc(sz); 2942 if (!net->xfrm.state_bydst) 2943 goto out_bydst; 2944 net->xfrm.state_bysrc = xfrm_hash_alloc(sz); 2945 if (!net->xfrm.state_bysrc) 2946 goto out_bysrc; 2947 net->xfrm.state_byspi = xfrm_hash_alloc(sz); 2948 if (!net->xfrm.state_byspi) 2949 goto out_byspi; 2950 net->xfrm.state_byseq = xfrm_hash_alloc(sz); 2951 if (!net->xfrm.state_byseq) 2952 goto out_byseq; 2953 net->xfrm.state_hmask = ((sz / sizeof(struct hlist_head)) - 1); 2954 2955 net->xfrm.state_num = 0; 2956 INIT_WORK(&net->xfrm.state_hash_work, xfrm_hash_resize); 2957 spin_lock_init(&net->xfrm.xfrm_state_lock); 2958 seqcount_spinlock_init(&net->xfrm.xfrm_state_hash_generation, 2959 &net->xfrm.xfrm_state_lock); 2960 return 0; 2961 2962 out_byseq: 2963 xfrm_hash_free(net->xfrm.state_byspi, sz); 2964 out_byspi: 2965 xfrm_hash_free(net->xfrm.state_bysrc, sz); 2966 out_bysrc: 2967 xfrm_hash_free(net->xfrm.state_bydst, sz); 2968 out_bydst: 2969 return -ENOMEM; 2970 } 2971 2972 void xfrm_state_fini(struct net *net) 2973 { 2974 unsigned int sz; 2975 2976 flush_work(&net->xfrm.state_hash_work); 2977 flush_work(&xfrm_state_gc_work); 2978 xfrm_state_flush(net, 0, false, true); 2979 2980 WARN_ON(!list_empty(&net->xfrm.state_all)); 2981 2982 sz = (net->xfrm.state_hmask + 1) * sizeof(struct hlist_head); 2983 WARN_ON(!hlist_empty(net->xfrm.state_byseq)); 2984 xfrm_hash_free(net->xfrm.state_byseq, sz); 2985 WARN_ON(!hlist_empty(net->xfrm.state_byspi)); 2986 xfrm_hash_free(net->xfrm.state_byspi, sz); 2987 WARN_ON(!hlist_empty(net->xfrm.state_bysrc)); 2988 xfrm_hash_free(net->xfrm.state_bysrc, sz); 2989 WARN_ON(!hlist_empty(net->xfrm.state_bydst)); 2990 xfrm_hash_free(net->xfrm.state_bydst, sz); 2991 } 2992 2993 #ifdef CONFIG_AUDITSYSCALL 2994 static void xfrm_audit_helper_sainfo(struct xfrm_state *x, 2995 struct audit_buffer *audit_buf) 2996 { 2997 struct xfrm_sec_ctx *ctx = x->security; 2998 u32 spi = ntohl(x->id.spi); 2999 3000 if (ctx) 3001 audit_log_format(audit_buf, " sec_alg=%u sec_doi=%u sec_obj=%s", 3002 ctx->ctx_alg, ctx->ctx_doi, ctx->ctx_str); 3003 3004 switch (x->props.family) { 3005 case AF_INET: 3006 audit_log_format(audit_buf, " src=%pI4 dst=%pI4", 3007 &x->props.saddr.a4, &x->id.daddr.a4); 3008 break; 3009 case AF_INET6: 3010 audit_log_format(audit_buf, " src=%pI6 dst=%pI6", 3011 x->props.saddr.a6, x->id.daddr.a6); 3012 break; 3013 } 3014 3015 audit_log_format(audit_buf, " spi=%u(0x%x)", spi, spi); 3016 } 3017 3018 static void xfrm_audit_helper_pktinfo(struct sk_buff *skb, u16 family, 3019 struct audit_buffer *audit_buf) 3020 { 3021 const struct iphdr *iph4; 3022 const struct ipv6hdr *iph6; 3023 3024 switch (family) { 3025 case AF_INET: 3026 iph4 = ip_hdr(skb); 3027 audit_log_format(audit_buf, " src=%pI4 dst=%pI4", 3028 &iph4->saddr, &iph4->daddr); 3029 break; 3030 case AF_INET6: 3031 iph6 = ipv6_hdr(skb); 3032 audit_log_format(audit_buf, 3033 " src=%pI6 dst=%pI6 flowlbl=0x%x%02x%02x", 3034 &iph6->saddr, &iph6->daddr, 3035 iph6->flow_lbl[0] & 0x0f, 3036 iph6->flow_lbl[1], 3037 iph6->flow_lbl[2]); 3038 break; 3039 } 3040 } 3041 3042 void xfrm_audit_state_add(struct xfrm_state *x, int result, bool task_valid) 3043 { 3044 struct audit_buffer *audit_buf; 3045 3046 audit_buf = xfrm_audit_start("SAD-add"); 3047 if (audit_buf == NULL) 3048 return; 3049 xfrm_audit_helper_usrinfo(task_valid, audit_buf); 3050 xfrm_audit_helper_sainfo(x, audit_buf); 3051 audit_log_format(audit_buf, " res=%u", result); 3052 audit_log_end(audit_buf); 3053 } 3054 EXPORT_SYMBOL_GPL(xfrm_audit_state_add); 3055 3056 void xfrm_audit_state_delete(struct xfrm_state *x, int result, bool task_valid) 3057 { 3058 struct audit_buffer *audit_buf; 3059 3060 audit_buf = xfrm_audit_start("SAD-delete"); 3061 if (audit_buf == NULL) 3062 return; 3063 xfrm_audit_helper_usrinfo(task_valid, audit_buf); 3064 xfrm_audit_helper_sainfo(x, audit_buf); 3065 audit_log_format(audit_buf, " res=%u", result); 3066 audit_log_end(audit_buf); 3067 } 3068 EXPORT_SYMBOL_GPL(xfrm_audit_state_delete); 3069 3070 void xfrm_audit_state_replay_overflow(struct xfrm_state *x, 3071 struct sk_buff *skb) 3072 { 3073 struct audit_buffer *audit_buf; 3074 u32 spi; 3075 3076 audit_buf = xfrm_audit_start("SA-replay-overflow"); 3077 if (audit_buf == NULL) 3078 return; 3079 xfrm_audit_helper_pktinfo(skb, x->props.family, audit_buf); 3080 /* don't record the sequence number because it's inherent in this kind 3081 * of audit message */ 3082 spi = ntohl(x->id.spi); 3083 audit_log_format(audit_buf, " spi=%u(0x%x)", spi, spi); 3084 audit_log_end(audit_buf); 3085 } 3086 EXPORT_SYMBOL_GPL(xfrm_audit_state_replay_overflow); 3087 3088 void xfrm_audit_state_replay(struct xfrm_state *x, 3089 struct sk_buff *skb, __be32 net_seq) 3090 { 3091 struct audit_buffer *audit_buf; 3092 u32 spi; 3093 3094 audit_buf = xfrm_audit_start("SA-replayed-pkt"); 3095 if (audit_buf == NULL) 3096 return; 3097 xfrm_audit_helper_pktinfo(skb, x->props.family, audit_buf); 3098 spi = ntohl(x->id.spi); 3099 audit_log_format(audit_buf, " spi=%u(0x%x) seqno=%u", 3100 spi, spi, ntohl(net_seq)); 3101 audit_log_end(audit_buf); 3102 } 3103 EXPORT_SYMBOL_GPL(xfrm_audit_state_replay); 3104 3105 void xfrm_audit_state_notfound_simple(struct sk_buff *skb, u16 family) 3106 { 3107 struct audit_buffer *audit_buf; 3108 3109 audit_buf = xfrm_audit_start("SA-notfound"); 3110 if (audit_buf == NULL) 3111 return; 3112 xfrm_audit_helper_pktinfo(skb, family, audit_buf); 3113 audit_log_end(audit_buf); 3114 } 3115 EXPORT_SYMBOL_GPL(xfrm_audit_state_notfound_simple); 3116 3117 void xfrm_audit_state_notfound(struct sk_buff *skb, u16 family, 3118 __be32 net_spi, __be32 net_seq) 3119 { 3120 struct audit_buffer *audit_buf; 3121 u32 spi; 3122 3123 audit_buf = xfrm_audit_start("SA-notfound"); 3124 if (audit_buf == NULL) 3125 return; 3126 xfrm_audit_helper_pktinfo(skb, family, audit_buf); 3127 spi = ntohl(net_spi); 3128 audit_log_format(audit_buf, " spi=%u(0x%x) seqno=%u", 3129 spi, spi, ntohl(net_seq)); 3130 audit_log_end(audit_buf); 3131 } 3132 EXPORT_SYMBOL_GPL(xfrm_audit_state_notfound); 3133 3134 void xfrm_audit_state_icvfail(struct xfrm_state *x, 3135 struct sk_buff *skb, u8 proto) 3136 { 3137 struct audit_buffer *audit_buf; 3138 __be32 net_spi; 3139 __be32 net_seq; 3140 3141 audit_buf = xfrm_audit_start("SA-icv-failure"); 3142 if (audit_buf == NULL) 3143 return; 3144 xfrm_audit_helper_pktinfo(skb, x->props.family, audit_buf); 3145 if (xfrm_parse_spi(skb, proto, &net_spi, &net_seq) == 0) { 3146 u32 spi = ntohl(net_spi); 3147 audit_log_format(audit_buf, " spi=%u(0x%x) seqno=%u", 3148 spi, spi, ntohl(net_seq)); 3149 } 3150 audit_log_end(audit_buf); 3151 } 3152 EXPORT_SYMBOL_GPL(xfrm_audit_state_icvfail); 3153 #endif /* CONFIG_AUDITSYSCALL */ 3154