1 /* 2 * xfrm_input.c 3 * 4 * Changes: 5 * YOSHIFUJI Hideaki @USAGI 6 * Split up af-specific portion 7 * 8 */ 9 10 #include <linux/slab.h> 11 #include <linux/module.h> 12 #include <linux/netdevice.h> 13 #include <net/dst.h> 14 #include <net/ip.h> 15 #include <net/xfrm.h> 16 #include <net/ip_tunnels.h> 17 #include <net/ip6_tunnel.h> 18 19 static struct kmem_cache *secpath_cachep __read_mostly; 20 21 static DEFINE_SPINLOCK(xfrm_input_afinfo_lock); 22 static struct xfrm_input_afinfo const __rcu *xfrm_input_afinfo[AF_INET6 + 1]; 23 24 static struct gro_cells gro_cells; 25 static struct net_device xfrm_napi_dev; 26 27 int xfrm_input_register_afinfo(const struct xfrm_input_afinfo *afinfo) 28 { 29 int err = 0; 30 31 if (WARN_ON(afinfo->family >= ARRAY_SIZE(xfrm_input_afinfo))) 32 return -EAFNOSUPPORT; 33 34 spin_lock_bh(&xfrm_input_afinfo_lock); 35 if (unlikely(xfrm_input_afinfo[afinfo->family] != NULL)) 36 err = -EEXIST; 37 else 38 rcu_assign_pointer(xfrm_input_afinfo[afinfo->family], afinfo); 39 spin_unlock_bh(&xfrm_input_afinfo_lock); 40 return err; 41 } 42 EXPORT_SYMBOL(xfrm_input_register_afinfo); 43 44 int xfrm_input_unregister_afinfo(const struct xfrm_input_afinfo *afinfo) 45 { 46 int err = 0; 47 48 spin_lock_bh(&xfrm_input_afinfo_lock); 49 if (likely(xfrm_input_afinfo[afinfo->family] != NULL)) { 50 if (unlikely(xfrm_input_afinfo[afinfo->family] != afinfo)) 51 err = -EINVAL; 52 else 53 RCU_INIT_POINTER(xfrm_input_afinfo[afinfo->family], NULL); 54 } 55 spin_unlock_bh(&xfrm_input_afinfo_lock); 56 synchronize_rcu(); 57 return err; 58 } 59 EXPORT_SYMBOL(xfrm_input_unregister_afinfo); 60 61 static const struct xfrm_input_afinfo *xfrm_input_get_afinfo(unsigned int family) 62 { 63 const struct xfrm_input_afinfo *afinfo; 64 65 if (WARN_ON_ONCE(family >= ARRAY_SIZE(xfrm_input_afinfo))) 66 return NULL; 67 68 rcu_read_lock(); 69 afinfo = rcu_dereference(xfrm_input_afinfo[family]); 70 if (unlikely(!afinfo)) 71 rcu_read_unlock(); 72 return afinfo; 73 } 74 75 static int xfrm_rcv_cb(struct sk_buff *skb, unsigned int family, u8 protocol, 76 int err) 77 { 78 int ret; 79 const struct xfrm_input_afinfo *afinfo = xfrm_input_get_afinfo(family); 80 81 if (!afinfo) 82 return -EAFNOSUPPORT; 83 84 ret = afinfo->callback(skb, protocol, err); 85 rcu_read_unlock(); 86 87 return ret; 88 } 89 90 void __secpath_destroy(struct sec_path *sp) 91 { 92 int i; 93 for (i = 0; i < sp->len; i++) 94 xfrm_state_put(sp->xvec[i]); 95 kmem_cache_free(secpath_cachep, sp); 96 } 97 EXPORT_SYMBOL(__secpath_destroy); 98 99 struct sec_path *secpath_dup(struct sec_path *src) 100 { 101 struct sec_path *sp; 102 103 sp = kmem_cache_alloc(secpath_cachep, GFP_ATOMIC); 104 if (!sp) 105 return NULL; 106 107 sp->len = 0; 108 sp->olen = 0; 109 110 memset(sp->ovec, 0, sizeof(sp->ovec[XFRM_MAX_OFFLOAD_DEPTH])); 111 112 if (src) { 113 int i; 114 115 memcpy(sp, src, sizeof(*sp)); 116 for (i = 0; i < sp->len; i++) 117 xfrm_state_hold(sp->xvec[i]); 118 } 119 refcount_set(&sp->refcnt, 1); 120 return sp; 121 } 122 EXPORT_SYMBOL(secpath_dup); 123 124 int secpath_set(struct sk_buff *skb) 125 { 126 struct sec_path *sp; 127 128 /* Allocate new secpath or COW existing one. */ 129 if (!skb->sp || refcount_read(&skb->sp->refcnt) != 1) { 130 sp = secpath_dup(skb->sp); 131 if (!sp) 132 return -ENOMEM; 133 134 if (skb->sp) 135 secpath_put(skb->sp); 136 skb->sp = sp; 137 } 138 return 0; 139 } 140 EXPORT_SYMBOL(secpath_set); 141 142 /* Fetch spi and seq from ipsec header */ 143 144 int xfrm_parse_spi(struct sk_buff *skb, u8 nexthdr, __be32 *spi, __be32 *seq) 145 { 146 int offset, offset_seq; 147 int hlen; 148 149 switch (nexthdr) { 150 case IPPROTO_AH: 151 hlen = sizeof(struct ip_auth_hdr); 152 offset = offsetof(struct ip_auth_hdr, spi); 153 offset_seq = offsetof(struct ip_auth_hdr, seq_no); 154 break; 155 case IPPROTO_ESP: 156 hlen = sizeof(struct ip_esp_hdr); 157 offset = offsetof(struct ip_esp_hdr, spi); 158 offset_seq = offsetof(struct ip_esp_hdr, seq_no); 159 break; 160 case IPPROTO_COMP: 161 if (!pskb_may_pull(skb, sizeof(struct ip_comp_hdr))) 162 return -EINVAL; 163 *spi = htonl(ntohs(*(__be16 *)(skb_transport_header(skb) + 2))); 164 *seq = 0; 165 return 0; 166 default: 167 return 1; 168 } 169 170 if (!pskb_may_pull(skb, hlen)) 171 return -EINVAL; 172 173 *spi = *(__be32 *)(skb_transport_header(skb) + offset); 174 *seq = *(__be32 *)(skb_transport_header(skb) + offset_seq); 175 return 0; 176 } 177 EXPORT_SYMBOL(xfrm_parse_spi); 178 179 int xfrm_prepare_input(struct xfrm_state *x, struct sk_buff *skb) 180 { 181 struct xfrm_mode *inner_mode = x->inner_mode; 182 int err; 183 184 err = x->outer_mode->afinfo->extract_input(x, skb); 185 if (err) 186 return err; 187 188 if (x->sel.family == AF_UNSPEC) { 189 inner_mode = xfrm_ip2inner_mode(x, XFRM_MODE_SKB_CB(skb)->protocol); 190 if (inner_mode == NULL) 191 return -EAFNOSUPPORT; 192 } 193 194 skb->protocol = inner_mode->afinfo->eth_proto; 195 return inner_mode->input2(x, skb); 196 } 197 EXPORT_SYMBOL(xfrm_prepare_input); 198 199 int xfrm_input(struct sk_buff *skb, int nexthdr, __be32 spi, int encap_type) 200 { 201 struct net *net = dev_net(skb->dev); 202 int err; 203 __be32 seq; 204 __be32 seq_hi; 205 struct xfrm_state *x = NULL; 206 xfrm_address_t *daddr; 207 struct xfrm_mode *inner_mode; 208 u32 mark = skb->mark; 209 unsigned int family; 210 int decaps = 0; 211 int async = 0; 212 bool xfrm_gro = false; 213 bool crypto_done = false; 214 struct xfrm_offload *xo = xfrm_offload(skb); 215 216 if (encap_type < 0) { 217 x = xfrm_input_state(skb); 218 family = x->outer_mode->afinfo->family; 219 220 /* An encap_type of -1 indicates async resumption. */ 221 if (encap_type == -1) { 222 async = 1; 223 seq = XFRM_SKB_CB(skb)->seq.input.low; 224 goto resume; 225 } 226 227 /* encap_type < -1 indicates a GRO call. */ 228 encap_type = 0; 229 seq = XFRM_SPI_SKB_CB(skb)->seq; 230 231 if (xo && (xo->flags & CRYPTO_DONE)) { 232 crypto_done = true; 233 x = xfrm_input_state(skb); 234 family = XFRM_SPI_SKB_CB(skb)->family; 235 236 if (!(xo->status & CRYPTO_SUCCESS)) { 237 if (xo->status & 238 (CRYPTO_TRANSPORT_AH_AUTH_FAILED | 239 CRYPTO_TRANSPORT_ESP_AUTH_FAILED | 240 CRYPTO_TUNNEL_AH_AUTH_FAILED | 241 CRYPTO_TUNNEL_ESP_AUTH_FAILED)) { 242 243 xfrm_audit_state_icvfail(x, skb, 244 x->type->proto); 245 x->stats.integrity_failed++; 246 XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEPROTOERROR); 247 goto drop; 248 } 249 250 if (xo->status & CRYPTO_INVALID_PROTOCOL) { 251 XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEPROTOERROR); 252 goto drop; 253 } 254 255 XFRM_INC_STATS(net, LINUX_MIB_XFRMINBUFFERERROR); 256 goto drop; 257 } 258 259 if ((err = xfrm_parse_spi(skb, nexthdr, &spi, &seq)) != 0) { 260 XFRM_INC_STATS(net, LINUX_MIB_XFRMINHDRERROR); 261 goto drop; 262 } 263 } 264 265 goto lock; 266 } 267 268 daddr = (xfrm_address_t *)(skb_network_header(skb) + 269 XFRM_SPI_SKB_CB(skb)->daddroff); 270 family = XFRM_SPI_SKB_CB(skb)->family; 271 272 /* if tunnel is present override skb->mark value with tunnel i_key */ 273 switch (family) { 274 case AF_INET: 275 if (XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip4) 276 mark = be32_to_cpu(XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip4->parms.i_key); 277 break; 278 case AF_INET6: 279 if (XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip6) 280 mark = be32_to_cpu(XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip6->parms.i_key); 281 break; 282 } 283 284 err = secpath_set(skb); 285 if (err) { 286 XFRM_INC_STATS(net, LINUX_MIB_XFRMINERROR); 287 goto drop; 288 } 289 290 seq = 0; 291 if (!spi && (err = xfrm_parse_spi(skb, nexthdr, &spi, &seq)) != 0) { 292 XFRM_INC_STATS(net, LINUX_MIB_XFRMINHDRERROR); 293 goto drop; 294 } 295 296 do { 297 if (skb->sp->len == XFRM_MAX_DEPTH) { 298 XFRM_INC_STATS(net, LINUX_MIB_XFRMINBUFFERERROR); 299 goto drop; 300 } 301 302 x = xfrm_state_lookup(net, mark, daddr, spi, nexthdr, family); 303 if (x == NULL) { 304 XFRM_INC_STATS(net, LINUX_MIB_XFRMINNOSTATES); 305 xfrm_audit_state_notfound(skb, family, spi, seq); 306 goto drop; 307 } 308 309 skb->sp->xvec[skb->sp->len++] = x; 310 311 lock: 312 spin_lock(&x->lock); 313 314 if (unlikely(x->km.state != XFRM_STATE_VALID)) { 315 if (x->km.state == XFRM_STATE_ACQ) 316 XFRM_INC_STATS(net, LINUX_MIB_XFRMACQUIREERROR); 317 else 318 XFRM_INC_STATS(net, 319 LINUX_MIB_XFRMINSTATEINVALID); 320 goto drop_unlock; 321 } 322 323 if ((x->encap ? x->encap->encap_type : 0) != encap_type) { 324 XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEMISMATCH); 325 goto drop_unlock; 326 } 327 328 if (x->repl->check(x, skb, seq)) { 329 XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATESEQERROR); 330 goto drop_unlock; 331 } 332 333 if (xfrm_state_check_expire(x)) { 334 XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEEXPIRED); 335 goto drop_unlock; 336 } 337 338 spin_unlock(&x->lock); 339 340 if (xfrm_tunnel_check(skb, x, family)) { 341 XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEMODEERROR); 342 goto drop; 343 } 344 345 seq_hi = htonl(xfrm_replay_seqhi(x, seq)); 346 347 XFRM_SKB_CB(skb)->seq.input.low = seq; 348 XFRM_SKB_CB(skb)->seq.input.hi = seq_hi; 349 350 skb_dst_force(skb); 351 dev_hold(skb->dev); 352 353 if (crypto_done) 354 nexthdr = x->type_offload->input_tail(x, skb); 355 else 356 nexthdr = x->type->input(x, skb); 357 358 if (nexthdr == -EINPROGRESS) 359 return 0; 360 resume: 361 dev_put(skb->dev); 362 363 spin_lock(&x->lock); 364 if (nexthdr <= 0) { 365 if (nexthdr == -EBADMSG) { 366 xfrm_audit_state_icvfail(x, skb, 367 x->type->proto); 368 x->stats.integrity_failed++; 369 } 370 XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEPROTOERROR); 371 goto drop_unlock; 372 } 373 374 /* only the first xfrm gets the encap type */ 375 encap_type = 0; 376 377 if (async && x->repl->recheck(x, skb, seq)) { 378 XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATESEQERROR); 379 goto drop_unlock; 380 } 381 382 x->repl->advance(x, seq); 383 384 x->curlft.bytes += skb->len; 385 x->curlft.packets++; 386 387 spin_unlock(&x->lock); 388 389 XFRM_MODE_SKB_CB(skb)->protocol = nexthdr; 390 391 inner_mode = x->inner_mode; 392 393 if (x->sel.family == AF_UNSPEC) { 394 inner_mode = xfrm_ip2inner_mode(x, XFRM_MODE_SKB_CB(skb)->protocol); 395 if (inner_mode == NULL) { 396 XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEMODEERROR); 397 goto drop; 398 } 399 } 400 401 if (inner_mode->input(x, skb)) { 402 XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEMODEERROR); 403 goto drop; 404 } 405 406 if (x->outer_mode->flags & XFRM_MODE_FLAG_TUNNEL) { 407 decaps = 1; 408 break; 409 } 410 411 /* 412 * We need the inner address. However, we only get here for 413 * transport mode so the outer address is identical. 414 */ 415 daddr = &x->id.daddr; 416 family = x->outer_mode->afinfo->family; 417 418 err = xfrm_parse_spi(skb, nexthdr, &spi, &seq); 419 if (err < 0) { 420 XFRM_INC_STATS(net, LINUX_MIB_XFRMINHDRERROR); 421 goto drop; 422 } 423 } while (!err); 424 425 err = xfrm_rcv_cb(skb, family, x->type->proto, 0); 426 if (err) 427 goto drop; 428 429 nf_reset(skb); 430 431 if (decaps) { 432 skb->sp->olen = 0; 433 skb_dst_drop(skb); 434 gro_cells_receive(&gro_cells, skb); 435 return 0; 436 } else { 437 xo = xfrm_offload(skb); 438 if (xo) 439 xfrm_gro = xo->flags & XFRM_GRO; 440 441 err = x->inner_mode->afinfo->transport_finish(skb, xfrm_gro || async); 442 if (xfrm_gro) { 443 skb->sp->olen = 0; 444 skb_dst_drop(skb); 445 gro_cells_receive(&gro_cells, skb); 446 return err; 447 } 448 449 return err; 450 } 451 452 drop_unlock: 453 spin_unlock(&x->lock); 454 drop: 455 xfrm_rcv_cb(skb, family, x && x->type ? x->type->proto : nexthdr, -1); 456 kfree_skb(skb); 457 return 0; 458 } 459 EXPORT_SYMBOL(xfrm_input); 460 461 int xfrm_input_resume(struct sk_buff *skb, int nexthdr) 462 { 463 return xfrm_input(skb, nexthdr, 0, -1); 464 } 465 EXPORT_SYMBOL(xfrm_input_resume); 466 467 void __init xfrm_input_init(void) 468 { 469 int err; 470 471 init_dummy_netdev(&xfrm_napi_dev); 472 err = gro_cells_init(&gro_cells, &xfrm_napi_dev); 473 if (err) 474 gro_cells.cells = NULL; 475 476 secpath_cachep = kmem_cache_create("secpath_cache", 477 sizeof(struct sec_path), 478 0, SLAB_HWCACHE_ALIGN|SLAB_PANIC, 479 NULL); 480 } 481