1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * xfrm_input.c 4 * 5 * Changes: 6 * YOSHIFUJI Hideaki @USAGI 7 * Split up af-specific portion 8 * 9 */ 10 11 #include <linux/slab.h> 12 #include <linux/module.h> 13 #include <linux/netdevice.h> 14 #include <net/dst.h> 15 #include <net/ip.h> 16 #include <net/xfrm.h> 17 #include <net/ip_tunnels.h> 18 #include <net/ip6_tunnel.h> 19 20 static struct kmem_cache *secpath_cachep __read_mostly; 21 22 static DEFINE_SPINLOCK(xfrm_input_afinfo_lock); 23 static struct xfrm_input_afinfo const __rcu *xfrm_input_afinfo[AF_INET6 + 1]; 24 25 static struct gro_cells gro_cells; 26 static struct net_device xfrm_napi_dev; 27 28 int xfrm_input_register_afinfo(const struct xfrm_input_afinfo *afinfo) 29 { 30 int err = 0; 31 32 if (WARN_ON(afinfo->family >= ARRAY_SIZE(xfrm_input_afinfo))) 33 return -EAFNOSUPPORT; 34 35 spin_lock_bh(&xfrm_input_afinfo_lock); 36 if (unlikely(xfrm_input_afinfo[afinfo->family] != NULL)) 37 err = -EEXIST; 38 else 39 rcu_assign_pointer(xfrm_input_afinfo[afinfo->family], afinfo); 40 spin_unlock_bh(&xfrm_input_afinfo_lock); 41 return err; 42 } 43 EXPORT_SYMBOL(xfrm_input_register_afinfo); 44 45 int xfrm_input_unregister_afinfo(const struct xfrm_input_afinfo *afinfo) 46 { 47 int err = 0; 48 49 spin_lock_bh(&xfrm_input_afinfo_lock); 50 if (likely(xfrm_input_afinfo[afinfo->family] != NULL)) { 51 if (unlikely(xfrm_input_afinfo[afinfo->family] != afinfo)) 52 err = -EINVAL; 53 else 54 RCU_INIT_POINTER(xfrm_input_afinfo[afinfo->family], NULL); 55 } 56 spin_unlock_bh(&xfrm_input_afinfo_lock); 57 synchronize_rcu(); 58 return err; 59 } 60 EXPORT_SYMBOL(xfrm_input_unregister_afinfo); 61 62 static const struct xfrm_input_afinfo *xfrm_input_get_afinfo(unsigned int family) 63 { 64 const struct xfrm_input_afinfo *afinfo; 65 66 if (WARN_ON_ONCE(family >= ARRAY_SIZE(xfrm_input_afinfo))) 67 return NULL; 68 69 rcu_read_lock(); 70 afinfo = rcu_dereference(xfrm_input_afinfo[family]); 71 if (unlikely(!afinfo)) 72 rcu_read_unlock(); 73 return afinfo; 74 } 75 76 static int xfrm_rcv_cb(struct sk_buff *skb, unsigned int family, u8 protocol, 77 int err) 78 { 79 int ret; 80 const struct xfrm_input_afinfo *afinfo = xfrm_input_get_afinfo(family); 81 82 if (!afinfo) 83 return -EAFNOSUPPORT; 84 85 ret = afinfo->callback(skb, protocol, err); 86 rcu_read_unlock(); 87 88 return ret; 89 } 90 91 void __secpath_destroy(struct sec_path *sp) 92 { 93 int i; 94 for (i = 0; i < sp->len; i++) 95 xfrm_state_put(sp->xvec[i]); 96 kmem_cache_free(secpath_cachep, sp); 97 } 98 EXPORT_SYMBOL(__secpath_destroy); 99 100 struct sec_path *secpath_dup(struct sec_path *src) 101 { 102 struct sec_path *sp; 103 104 sp = kmem_cache_alloc(secpath_cachep, GFP_ATOMIC); 105 if (!sp) 106 return NULL; 107 108 sp->len = 0; 109 sp->olen = 0; 110 111 memset(sp->ovec, 0, sizeof(sp->ovec[XFRM_MAX_OFFLOAD_DEPTH])); 112 113 if (src) { 114 int i; 115 116 memcpy(sp, src, sizeof(*sp)); 117 for (i = 0; i < sp->len; i++) 118 xfrm_state_hold(sp->xvec[i]); 119 } 120 refcount_set(&sp->refcnt, 1); 121 return sp; 122 } 123 EXPORT_SYMBOL(secpath_dup); 124 125 int secpath_set(struct sk_buff *skb) 126 { 127 struct sec_path *sp; 128 129 /* Allocate new secpath or COW existing one. */ 130 if (!skb->sp || refcount_read(&skb->sp->refcnt) != 1) { 131 sp = secpath_dup(skb->sp); 132 if (!sp) 133 return -ENOMEM; 134 135 if (skb->sp) 136 secpath_put(skb->sp); 137 skb->sp = sp; 138 } 139 return 0; 140 } 141 EXPORT_SYMBOL(secpath_set); 142 143 /* Fetch spi and seq from ipsec header */ 144 145 int xfrm_parse_spi(struct sk_buff *skb, u8 nexthdr, __be32 *spi, __be32 *seq) 146 { 147 int offset, offset_seq; 148 int hlen; 149 150 switch (nexthdr) { 151 case IPPROTO_AH: 152 hlen = sizeof(struct ip_auth_hdr); 153 offset = offsetof(struct ip_auth_hdr, spi); 154 offset_seq = offsetof(struct ip_auth_hdr, seq_no); 155 break; 156 case IPPROTO_ESP: 157 hlen = sizeof(struct ip_esp_hdr); 158 offset = offsetof(struct ip_esp_hdr, spi); 159 offset_seq = offsetof(struct ip_esp_hdr, seq_no); 160 break; 161 case IPPROTO_COMP: 162 if (!pskb_may_pull(skb, sizeof(struct ip_comp_hdr))) 163 return -EINVAL; 164 *spi = htonl(ntohs(*(__be16 *)(skb_transport_header(skb) + 2))); 165 *seq = 0; 166 return 0; 167 default: 168 return 1; 169 } 170 171 if (!pskb_may_pull(skb, hlen)) 172 return -EINVAL; 173 174 *spi = *(__be32 *)(skb_transport_header(skb) + offset); 175 *seq = *(__be32 *)(skb_transport_header(skb) + offset_seq); 176 return 0; 177 } 178 EXPORT_SYMBOL(xfrm_parse_spi); 179 180 int xfrm_prepare_input(struct xfrm_state *x, struct sk_buff *skb) 181 { 182 struct xfrm_mode *inner_mode = x->inner_mode; 183 int err; 184 185 err = x->outer_mode->afinfo->extract_input(x, skb); 186 if (err) 187 return err; 188 189 if (x->sel.family == AF_UNSPEC) { 190 inner_mode = xfrm_ip2inner_mode(x, XFRM_MODE_SKB_CB(skb)->protocol); 191 if (inner_mode == NULL) 192 return -EAFNOSUPPORT; 193 } 194 195 skb->protocol = inner_mode->afinfo->eth_proto; 196 return inner_mode->input2(x, skb); 197 } 198 EXPORT_SYMBOL(xfrm_prepare_input); 199 200 int xfrm_input(struct sk_buff *skb, int nexthdr, __be32 spi, int encap_type) 201 { 202 struct net *net = dev_net(skb->dev); 203 int err; 204 __be32 seq; 205 __be32 seq_hi; 206 struct xfrm_state *x = NULL; 207 xfrm_address_t *daddr; 208 struct xfrm_mode *inner_mode; 209 u32 mark = skb->mark; 210 unsigned int family; 211 int decaps = 0; 212 int async = 0; 213 bool xfrm_gro = false; 214 bool crypto_done = false; 215 struct xfrm_offload *xo = xfrm_offload(skb); 216 217 if (encap_type < 0) { 218 x = xfrm_input_state(skb); 219 family = x->outer_mode->afinfo->family; 220 221 /* An encap_type of -1 indicates async resumption. */ 222 if (encap_type == -1) { 223 async = 1; 224 seq = XFRM_SKB_CB(skb)->seq.input.low; 225 goto resume; 226 } 227 228 /* encap_type < -1 indicates a GRO call. */ 229 encap_type = 0; 230 seq = XFRM_SPI_SKB_CB(skb)->seq; 231 232 if (xo && (xo->flags & CRYPTO_DONE)) { 233 crypto_done = true; 234 x = xfrm_input_state(skb); 235 family = XFRM_SPI_SKB_CB(skb)->family; 236 237 if (!(xo->status & CRYPTO_SUCCESS)) { 238 if (xo->status & 239 (CRYPTO_TRANSPORT_AH_AUTH_FAILED | 240 CRYPTO_TRANSPORT_ESP_AUTH_FAILED | 241 CRYPTO_TUNNEL_AH_AUTH_FAILED | 242 CRYPTO_TUNNEL_ESP_AUTH_FAILED)) { 243 244 xfrm_audit_state_icvfail(x, skb, 245 x->type->proto); 246 x->stats.integrity_failed++; 247 XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEPROTOERROR); 248 goto drop; 249 } 250 251 if (xo->status & CRYPTO_INVALID_PROTOCOL) { 252 XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEPROTOERROR); 253 goto drop; 254 } 255 256 XFRM_INC_STATS(net, LINUX_MIB_XFRMINBUFFERERROR); 257 goto drop; 258 } 259 260 if ((err = xfrm_parse_spi(skb, nexthdr, &spi, &seq)) != 0) { 261 XFRM_INC_STATS(net, LINUX_MIB_XFRMINHDRERROR); 262 goto drop; 263 } 264 } 265 266 goto lock; 267 } 268 269 family = XFRM_SPI_SKB_CB(skb)->family; 270 271 /* if tunnel is present override skb->mark value with tunnel i_key */ 272 switch (family) { 273 case AF_INET: 274 if (XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip4) 275 mark = be32_to_cpu(XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip4->parms.i_key); 276 break; 277 case AF_INET6: 278 if (XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip6) 279 mark = be32_to_cpu(XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip6->parms.i_key); 280 break; 281 } 282 283 err = secpath_set(skb); 284 if (err) { 285 XFRM_INC_STATS(net, LINUX_MIB_XFRMINERROR); 286 goto drop; 287 } 288 289 seq = 0; 290 if (!spi && (err = xfrm_parse_spi(skb, nexthdr, &spi, &seq)) != 0) { 291 XFRM_INC_STATS(net, LINUX_MIB_XFRMINHDRERROR); 292 goto drop; 293 } 294 295 daddr = (xfrm_address_t *)(skb_network_header(skb) + 296 XFRM_SPI_SKB_CB(skb)->daddroff); 297 do { 298 if (skb->sp->len == XFRM_MAX_DEPTH) { 299 XFRM_INC_STATS(net, LINUX_MIB_XFRMINBUFFERERROR); 300 goto drop; 301 } 302 303 x = xfrm_state_lookup(net, mark, daddr, spi, nexthdr, family); 304 if (x == NULL) { 305 XFRM_INC_STATS(net, LINUX_MIB_XFRMINNOSTATES); 306 xfrm_audit_state_notfound(skb, family, spi, seq); 307 goto drop; 308 } 309 310 skb->sp->xvec[skb->sp->len++] = x; 311 312 lock: 313 spin_lock(&x->lock); 314 315 if (unlikely(x->km.state != XFRM_STATE_VALID)) { 316 if (x->km.state == XFRM_STATE_ACQ) 317 XFRM_INC_STATS(net, LINUX_MIB_XFRMACQUIREERROR); 318 else 319 XFRM_INC_STATS(net, 320 LINUX_MIB_XFRMINSTATEINVALID); 321 goto drop_unlock; 322 } 323 324 if ((x->encap ? x->encap->encap_type : 0) != encap_type) { 325 XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEMISMATCH); 326 goto drop_unlock; 327 } 328 329 if (x->repl->check(x, skb, seq)) { 330 XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATESEQERROR); 331 goto drop_unlock; 332 } 333 334 if (xfrm_state_check_expire(x)) { 335 XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEEXPIRED); 336 goto drop_unlock; 337 } 338 339 spin_unlock(&x->lock); 340 341 if (xfrm_tunnel_check(skb, x, family)) { 342 XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEMODEERROR); 343 goto drop; 344 } 345 346 seq_hi = htonl(xfrm_replay_seqhi(x, seq)); 347 348 XFRM_SKB_CB(skb)->seq.input.low = seq; 349 XFRM_SKB_CB(skb)->seq.input.hi = seq_hi; 350 351 skb_dst_force(skb); 352 dev_hold(skb->dev); 353 354 if (crypto_done) 355 nexthdr = x->type_offload->input_tail(x, skb); 356 else 357 nexthdr = x->type->input(x, skb); 358 359 if (nexthdr == -EINPROGRESS) 360 return 0; 361 resume: 362 dev_put(skb->dev); 363 364 spin_lock(&x->lock); 365 if (nexthdr <= 0) { 366 if (nexthdr == -EBADMSG) { 367 xfrm_audit_state_icvfail(x, skb, 368 x->type->proto); 369 x->stats.integrity_failed++; 370 } 371 XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEPROTOERROR); 372 goto drop_unlock; 373 } 374 375 /* only the first xfrm gets the encap type */ 376 encap_type = 0; 377 378 if (async && x->repl->recheck(x, skb, seq)) { 379 XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATESEQERROR); 380 goto drop_unlock; 381 } 382 383 x->repl->advance(x, seq); 384 385 x->curlft.bytes += skb->len; 386 x->curlft.packets++; 387 388 spin_unlock(&x->lock); 389 390 XFRM_MODE_SKB_CB(skb)->protocol = nexthdr; 391 392 inner_mode = x->inner_mode; 393 394 if (x->sel.family == AF_UNSPEC) { 395 inner_mode = xfrm_ip2inner_mode(x, XFRM_MODE_SKB_CB(skb)->protocol); 396 if (inner_mode == NULL) { 397 XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEMODEERROR); 398 goto drop; 399 } 400 } 401 402 if (inner_mode->input(x, skb)) { 403 XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEMODEERROR); 404 goto drop; 405 } 406 407 if (x->outer_mode->flags & XFRM_MODE_FLAG_TUNNEL) { 408 decaps = 1; 409 break; 410 } 411 412 /* 413 * We need the inner address. However, we only get here for 414 * transport mode so the outer address is identical. 415 */ 416 daddr = &x->id.daddr; 417 family = x->outer_mode->afinfo->family; 418 419 err = xfrm_parse_spi(skb, nexthdr, &spi, &seq); 420 if (err < 0) { 421 XFRM_INC_STATS(net, LINUX_MIB_XFRMINHDRERROR); 422 goto drop; 423 } 424 } while (!err); 425 426 err = xfrm_rcv_cb(skb, family, x->type->proto, 0); 427 if (err) 428 goto drop; 429 430 nf_reset(skb); 431 432 if (decaps) { 433 if (skb->sp) 434 skb->sp->olen = 0; 435 skb_dst_drop(skb); 436 gro_cells_receive(&gro_cells, skb); 437 return 0; 438 } else { 439 xo = xfrm_offload(skb); 440 if (xo) 441 xfrm_gro = xo->flags & XFRM_GRO; 442 443 err = x->inner_mode->afinfo->transport_finish(skb, xfrm_gro || async); 444 if (xfrm_gro) { 445 if (skb->sp) 446 skb->sp->olen = 0; 447 skb_dst_drop(skb); 448 gro_cells_receive(&gro_cells, skb); 449 return err; 450 } 451 452 return err; 453 } 454 455 drop_unlock: 456 spin_unlock(&x->lock); 457 drop: 458 xfrm_rcv_cb(skb, family, x && x->type ? x->type->proto : nexthdr, -1); 459 kfree_skb(skb); 460 return 0; 461 } 462 EXPORT_SYMBOL(xfrm_input); 463 464 int xfrm_input_resume(struct sk_buff *skb, int nexthdr) 465 { 466 return xfrm_input(skb, nexthdr, 0, -1); 467 } 468 EXPORT_SYMBOL(xfrm_input_resume); 469 470 void __init xfrm_input_init(void) 471 { 472 int err; 473 474 init_dummy_netdev(&xfrm_napi_dev); 475 err = gro_cells_init(&gro_cells, &xfrm_napi_dev); 476 if (err) 477 gro_cells.cells = NULL; 478 479 secpath_cachep = kmem_cache_create("secpath_cache", 480 sizeof(struct sec_path), 481 0, SLAB_HWCACHE_ALIGN|SLAB_PANIC, 482 NULL); 483 } 484