xref: /openbmc/linux/net/xfrm/xfrm_policy.c (revision ae3473231e77a3f1909d48cd144cebe5e1d049b3)
1 /*
2  * xfrm_policy.c
3  *
4  * Changes:
5  *	Mitsuru KANDA @USAGI
6  * 	Kazunori MIYAZAWA @USAGI
7  * 	Kunihiro Ishiguro <kunihiro@ipinfusion.com>
8  * 		IPv6 support
9  * 	Kazunori MIYAZAWA @USAGI
10  * 	YOSHIFUJI Hideaki
11  * 		Split up af-specific portion
12  *	Derek Atkins <derek@ihtfp.com>		Add the post_input processor
13  *
14  */
15 
16 #include <linux/err.h>
17 #include <linux/slab.h>
18 #include <linux/kmod.h>
19 #include <linux/list.h>
20 #include <linux/spinlock.h>
21 #include <linux/workqueue.h>
22 #include <linux/notifier.h>
23 #include <linux/netdevice.h>
24 #include <linux/netfilter.h>
25 #include <linux/module.h>
26 #include <linux/cache.h>
27 #include <linux/audit.h>
28 #include <net/dst.h>
29 #include <net/flow.h>
30 #include <net/xfrm.h>
31 #include <net/ip.h>
32 #ifdef CONFIG_XFRM_STATISTICS
33 #include <net/snmp.h>
34 #endif
35 
36 #include "xfrm_hash.h"
37 
38 #define XFRM_QUEUE_TMO_MIN ((unsigned)(HZ/10))
39 #define XFRM_QUEUE_TMO_MAX ((unsigned)(60*HZ))
40 #define XFRM_MAX_QUEUE_LEN	100
41 
42 struct xfrm_flo {
43 	struct dst_entry *dst_orig;
44 	u8 flags;
45 };
46 
47 static DEFINE_SPINLOCK(xfrm_policy_afinfo_lock);
48 static struct xfrm_policy_afinfo __rcu *xfrm_policy_afinfo[NPROTO]
49 						__read_mostly;
50 
51 static struct kmem_cache *xfrm_dst_cache __read_mostly;
52 static __read_mostly seqcount_t xfrm_policy_hash_generation;
53 
54 static void xfrm_init_pmtu(struct dst_entry *dst);
55 static int stale_bundle(struct dst_entry *dst);
56 static int xfrm_bundle_ok(struct xfrm_dst *xdst);
57 static void xfrm_policy_queue_process(unsigned long arg);
58 
59 static void __xfrm_policy_link(struct xfrm_policy *pol, int dir);
60 static struct xfrm_policy *__xfrm_policy_unlink(struct xfrm_policy *pol,
61 						int dir);
62 
63 static inline bool xfrm_pol_hold_rcu(struct xfrm_policy *policy)
64 {
65 	return atomic_inc_not_zero(&policy->refcnt);
66 }
67 
68 static inline bool
69 __xfrm4_selector_match(const struct xfrm_selector *sel, const struct flowi *fl)
70 {
71 	const struct flowi4 *fl4 = &fl->u.ip4;
72 
73 	return  addr4_match(fl4->daddr, sel->daddr.a4, sel->prefixlen_d) &&
74 		addr4_match(fl4->saddr, sel->saddr.a4, sel->prefixlen_s) &&
75 		!((xfrm_flowi_dport(fl, &fl4->uli) ^ sel->dport) & sel->dport_mask) &&
76 		!((xfrm_flowi_sport(fl, &fl4->uli) ^ sel->sport) & sel->sport_mask) &&
77 		(fl4->flowi4_proto == sel->proto || !sel->proto) &&
78 		(fl4->flowi4_oif == sel->ifindex || !sel->ifindex);
79 }
80 
81 static inline bool
82 __xfrm6_selector_match(const struct xfrm_selector *sel, const struct flowi *fl)
83 {
84 	const struct flowi6 *fl6 = &fl->u.ip6;
85 
86 	return  addr_match(&fl6->daddr, &sel->daddr, sel->prefixlen_d) &&
87 		addr_match(&fl6->saddr, &sel->saddr, sel->prefixlen_s) &&
88 		!((xfrm_flowi_dport(fl, &fl6->uli) ^ sel->dport) & sel->dport_mask) &&
89 		!((xfrm_flowi_sport(fl, &fl6->uli) ^ sel->sport) & sel->sport_mask) &&
90 		(fl6->flowi6_proto == sel->proto || !sel->proto) &&
91 		(fl6->flowi6_oif == sel->ifindex || !sel->ifindex);
92 }
93 
94 bool xfrm_selector_match(const struct xfrm_selector *sel, const struct flowi *fl,
95 			 unsigned short family)
96 {
97 	switch (family) {
98 	case AF_INET:
99 		return __xfrm4_selector_match(sel, fl);
100 	case AF_INET6:
101 		return __xfrm6_selector_match(sel, fl);
102 	}
103 	return false;
104 }
105 
106 static struct xfrm_policy_afinfo *xfrm_policy_get_afinfo(unsigned short family)
107 {
108 	struct xfrm_policy_afinfo *afinfo;
109 
110 	if (unlikely(family >= NPROTO))
111 		return NULL;
112 	rcu_read_lock();
113 	afinfo = rcu_dereference(xfrm_policy_afinfo[family]);
114 	if (unlikely(!afinfo))
115 		rcu_read_unlock();
116 	return afinfo;
117 }
118 
119 static void xfrm_policy_put_afinfo(struct xfrm_policy_afinfo *afinfo)
120 {
121 	rcu_read_unlock();
122 }
123 
124 static inline struct dst_entry *__xfrm_dst_lookup(struct net *net,
125 						  int tos, int oif,
126 						  const xfrm_address_t *saddr,
127 						  const xfrm_address_t *daddr,
128 						  int family)
129 {
130 	struct xfrm_policy_afinfo *afinfo;
131 	struct dst_entry *dst;
132 
133 	afinfo = xfrm_policy_get_afinfo(family);
134 	if (unlikely(afinfo == NULL))
135 		return ERR_PTR(-EAFNOSUPPORT);
136 
137 	dst = afinfo->dst_lookup(net, tos, oif, saddr, daddr);
138 
139 	xfrm_policy_put_afinfo(afinfo);
140 
141 	return dst;
142 }
143 
144 static inline struct dst_entry *xfrm_dst_lookup(struct xfrm_state *x,
145 						int tos, int oif,
146 						xfrm_address_t *prev_saddr,
147 						xfrm_address_t *prev_daddr,
148 						int family)
149 {
150 	struct net *net = xs_net(x);
151 	xfrm_address_t *saddr = &x->props.saddr;
152 	xfrm_address_t *daddr = &x->id.daddr;
153 	struct dst_entry *dst;
154 
155 	if (x->type->flags & XFRM_TYPE_LOCAL_COADDR) {
156 		saddr = x->coaddr;
157 		daddr = prev_daddr;
158 	}
159 	if (x->type->flags & XFRM_TYPE_REMOTE_COADDR) {
160 		saddr = prev_saddr;
161 		daddr = x->coaddr;
162 	}
163 
164 	dst = __xfrm_dst_lookup(net, tos, oif, saddr, daddr, family);
165 
166 	if (!IS_ERR(dst)) {
167 		if (prev_saddr != saddr)
168 			memcpy(prev_saddr, saddr,  sizeof(*prev_saddr));
169 		if (prev_daddr != daddr)
170 			memcpy(prev_daddr, daddr,  sizeof(*prev_daddr));
171 	}
172 
173 	return dst;
174 }
175 
176 static inline unsigned long make_jiffies(long secs)
177 {
178 	if (secs >= (MAX_SCHEDULE_TIMEOUT-1)/HZ)
179 		return MAX_SCHEDULE_TIMEOUT-1;
180 	else
181 		return secs*HZ;
182 }
183 
184 static void xfrm_policy_timer(unsigned long data)
185 {
186 	struct xfrm_policy *xp = (struct xfrm_policy *)data;
187 	unsigned long now = get_seconds();
188 	long next = LONG_MAX;
189 	int warn = 0;
190 	int dir;
191 
192 	read_lock(&xp->lock);
193 
194 	if (unlikely(xp->walk.dead))
195 		goto out;
196 
197 	dir = xfrm_policy_id2dir(xp->index);
198 
199 	if (xp->lft.hard_add_expires_seconds) {
200 		long tmo = xp->lft.hard_add_expires_seconds +
201 			xp->curlft.add_time - now;
202 		if (tmo <= 0)
203 			goto expired;
204 		if (tmo < next)
205 			next = tmo;
206 	}
207 	if (xp->lft.hard_use_expires_seconds) {
208 		long tmo = xp->lft.hard_use_expires_seconds +
209 			(xp->curlft.use_time ? : xp->curlft.add_time) - now;
210 		if (tmo <= 0)
211 			goto expired;
212 		if (tmo < next)
213 			next = tmo;
214 	}
215 	if (xp->lft.soft_add_expires_seconds) {
216 		long tmo = xp->lft.soft_add_expires_seconds +
217 			xp->curlft.add_time - now;
218 		if (tmo <= 0) {
219 			warn = 1;
220 			tmo = XFRM_KM_TIMEOUT;
221 		}
222 		if (tmo < next)
223 			next = tmo;
224 	}
225 	if (xp->lft.soft_use_expires_seconds) {
226 		long tmo = xp->lft.soft_use_expires_seconds +
227 			(xp->curlft.use_time ? : xp->curlft.add_time) - now;
228 		if (tmo <= 0) {
229 			warn = 1;
230 			tmo = XFRM_KM_TIMEOUT;
231 		}
232 		if (tmo < next)
233 			next = tmo;
234 	}
235 
236 	if (warn)
237 		km_policy_expired(xp, dir, 0, 0);
238 	if (next != LONG_MAX &&
239 	    !mod_timer(&xp->timer, jiffies + make_jiffies(next)))
240 		xfrm_pol_hold(xp);
241 
242 out:
243 	read_unlock(&xp->lock);
244 	xfrm_pol_put(xp);
245 	return;
246 
247 expired:
248 	read_unlock(&xp->lock);
249 	if (!xfrm_policy_delete(xp, dir))
250 		km_policy_expired(xp, dir, 1, 0);
251 	xfrm_pol_put(xp);
252 }
253 
254 static struct flow_cache_object *xfrm_policy_flo_get(struct flow_cache_object *flo)
255 {
256 	struct xfrm_policy *pol = container_of(flo, struct xfrm_policy, flo);
257 
258 	if (unlikely(pol->walk.dead))
259 		flo = NULL;
260 	else
261 		xfrm_pol_hold(pol);
262 
263 	return flo;
264 }
265 
266 static int xfrm_policy_flo_check(struct flow_cache_object *flo)
267 {
268 	struct xfrm_policy *pol = container_of(flo, struct xfrm_policy, flo);
269 
270 	return !pol->walk.dead;
271 }
272 
273 static void xfrm_policy_flo_delete(struct flow_cache_object *flo)
274 {
275 	xfrm_pol_put(container_of(flo, struct xfrm_policy, flo));
276 }
277 
278 static const struct flow_cache_ops xfrm_policy_fc_ops = {
279 	.get = xfrm_policy_flo_get,
280 	.check = xfrm_policy_flo_check,
281 	.delete = xfrm_policy_flo_delete,
282 };
283 
284 /* Allocate xfrm_policy. Not used here, it is supposed to be used by pfkeyv2
285  * SPD calls.
286  */
287 
288 struct xfrm_policy *xfrm_policy_alloc(struct net *net, gfp_t gfp)
289 {
290 	struct xfrm_policy *policy;
291 
292 	policy = kzalloc(sizeof(struct xfrm_policy), gfp);
293 
294 	if (policy) {
295 		write_pnet(&policy->xp_net, net);
296 		INIT_LIST_HEAD(&policy->walk.all);
297 		INIT_HLIST_NODE(&policy->bydst);
298 		INIT_HLIST_NODE(&policy->byidx);
299 		rwlock_init(&policy->lock);
300 		atomic_set(&policy->refcnt, 1);
301 		skb_queue_head_init(&policy->polq.hold_queue);
302 		setup_timer(&policy->timer, xfrm_policy_timer,
303 				(unsigned long)policy);
304 		setup_timer(&policy->polq.hold_timer, xfrm_policy_queue_process,
305 			    (unsigned long)policy);
306 		policy->flo.ops = &xfrm_policy_fc_ops;
307 	}
308 	return policy;
309 }
310 EXPORT_SYMBOL(xfrm_policy_alloc);
311 
312 static void xfrm_policy_destroy_rcu(struct rcu_head *head)
313 {
314 	struct xfrm_policy *policy = container_of(head, struct xfrm_policy, rcu);
315 
316 	security_xfrm_policy_free(policy->security);
317 	kfree(policy);
318 }
319 
320 /* Destroy xfrm_policy: descendant resources must be released to this moment. */
321 
322 void xfrm_policy_destroy(struct xfrm_policy *policy)
323 {
324 	BUG_ON(!policy->walk.dead);
325 
326 	if (del_timer(&policy->timer) || del_timer(&policy->polq.hold_timer))
327 		BUG();
328 
329 	call_rcu(&policy->rcu, xfrm_policy_destroy_rcu);
330 }
331 EXPORT_SYMBOL(xfrm_policy_destroy);
332 
333 /* Rule must be locked. Release descentant resources, announce
334  * entry dead. The rule must be unlinked from lists to the moment.
335  */
336 
337 static void xfrm_policy_kill(struct xfrm_policy *policy)
338 {
339 	policy->walk.dead = 1;
340 
341 	atomic_inc(&policy->genid);
342 
343 	if (del_timer(&policy->polq.hold_timer))
344 		xfrm_pol_put(policy);
345 	skb_queue_purge(&policy->polq.hold_queue);
346 
347 	if (del_timer(&policy->timer))
348 		xfrm_pol_put(policy);
349 
350 	xfrm_pol_put(policy);
351 }
352 
353 static unsigned int xfrm_policy_hashmax __read_mostly = 1 * 1024 * 1024;
354 
355 static inline unsigned int idx_hash(struct net *net, u32 index)
356 {
357 	return __idx_hash(index, net->xfrm.policy_idx_hmask);
358 }
359 
360 /* calculate policy hash thresholds */
361 static void __get_hash_thresh(struct net *net,
362 			      unsigned short family, int dir,
363 			      u8 *dbits, u8 *sbits)
364 {
365 	switch (family) {
366 	case AF_INET:
367 		*dbits = net->xfrm.policy_bydst[dir].dbits4;
368 		*sbits = net->xfrm.policy_bydst[dir].sbits4;
369 		break;
370 
371 	case AF_INET6:
372 		*dbits = net->xfrm.policy_bydst[dir].dbits6;
373 		*sbits = net->xfrm.policy_bydst[dir].sbits6;
374 		break;
375 
376 	default:
377 		*dbits = 0;
378 		*sbits = 0;
379 	}
380 }
381 
382 static struct hlist_head *policy_hash_bysel(struct net *net,
383 					    const struct xfrm_selector *sel,
384 					    unsigned short family, int dir)
385 {
386 	unsigned int hmask = net->xfrm.policy_bydst[dir].hmask;
387 	unsigned int hash;
388 	u8 dbits;
389 	u8 sbits;
390 
391 	__get_hash_thresh(net, family, dir, &dbits, &sbits);
392 	hash = __sel_hash(sel, family, hmask, dbits, sbits);
393 
394 	if (hash == hmask + 1)
395 		return &net->xfrm.policy_inexact[dir];
396 
397 	return rcu_dereference_check(net->xfrm.policy_bydst[dir].table,
398 		     lockdep_is_held(&net->xfrm.xfrm_policy_lock)) + hash;
399 }
400 
401 static struct hlist_head *policy_hash_direct(struct net *net,
402 					     const xfrm_address_t *daddr,
403 					     const xfrm_address_t *saddr,
404 					     unsigned short family, int dir)
405 {
406 	unsigned int hmask = net->xfrm.policy_bydst[dir].hmask;
407 	unsigned int hash;
408 	u8 dbits;
409 	u8 sbits;
410 
411 	__get_hash_thresh(net, family, dir, &dbits, &sbits);
412 	hash = __addr_hash(daddr, saddr, family, hmask, dbits, sbits);
413 
414 	return rcu_dereference_check(net->xfrm.policy_bydst[dir].table,
415 		     lockdep_is_held(&net->xfrm.xfrm_policy_lock)) + hash;
416 }
417 
418 static void xfrm_dst_hash_transfer(struct net *net,
419 				   struct hlist_head *list,
420 				   struct hlist_head *ndsttable,
421 				   unsigned int nhashmask,
422 				   int dir)
423 {
424 	struct hlist_node *tmp, *entry0 = NULL;
425 	struct xfrm_policy *pol;
426 	unsigned int h0 = 0;
427 	u8 dbits;
428 	u8 sbits;
429 
430 redo:
431 	hlist_for_each_entry_safe(pol, tmp, list, bydst) {
432 		unsigned int h;
433 
434 		__get_hash_thresh(net, pol->family, dir, &dbits, &sbits);
435 		h = __addr_hash(&pol->selector.daddr, &pol->selector.saddr,
436 				pol->family, nhashmask, dbits, sbits);
437 		if (!entry0) {
438 			hlist_del_rcu(&pol->bydst);
439 			hlist_add_head_rcu(&pol->bydst, ndsttable + h);
440 			h0 = h;
441 		} else {
442 			if (h != h0)
443 				continue;
444 			hlist_del_rcu(&pol->bydst);
445 			hlist_add_behind_rcu(&pol->bydst, entry0);
446 		}
447 		entry0 = &pol->bydst;
448 	}
449 	if (!hlist_empty(list)) {
450 		entry0 = NULL;
451 		goto redo;
452 	}
453 }
454 
455 static void xfrm_idx_hash_transfer(struct hlist_head *list,
456 				   struct hlist_head *nidxtable,
457 				   unsigned int nhashmask)
458 {
459 	struct hlist_node *tmp;
460 	struct xfrm_policy *pol;
461 
462 	hlist_for_each_entry_safe(pol, tmp, list, byidx) {
463 		unsigned int h;
464 
465 		h = __idx_hash(pol->index, nhashmask);
466 		hlist_add_head(&pol->byidx, nidxtable+h);
467 	}
468 }
469 
470 static unsigned long xfrm_new_hash_mask(unsigned int old_hmask)
471 {
472 	return ((old_hmask + 1) << 1) - 1;
473 }
474 
475 static void xfrm_bydst_resize(struct net *net, int dir)
476 {
477 	unsigned int hmask = net->xfrm.policy_bydst[dir].hmask;
478 	unsigned int nhashmask = xfrm_new_hash_mask(hmask);
479 	unsigned int nsize = (nhashmask + 1) * sizeof(struct hlist_head);
480 	struct hlist_head *ndst = xfrm_hash_alloc(nsize);
481 	struct hlist_head *odst;
482 	int i;
483 
484 	if (!ndst)
485 		return;
486 
487 	spin_lock_bh(&net->xfrm.xfrm_policy_lock);
488 	write_seqcount_begin(&xfrm_policy_hash_generation);
489 
490 	odst = rcu_dereference_protected(net->xfrm.policy_bydst[dir].table,
491 				lockdep_is_held(&net->xfrm.xfrm_policy_lock));
492 
493 	odst = rcu_dereference_protected(net->xfrm.policy_bydst[dir].table,
494 				lockdep_is_held(&net->xfrm.xfrm_policy_lock));
495 
496 	for (i = hmask; i >= 0; i--)
497 		xfrm_dst_hash_transfer(net, odst + i, ndst, nhashmask, dir);
498 
499 	rcu_assign_pointer(net->xfrm.policy_bydst[dir].table, ndst);
500 	net->xfrm.policy_bydst[dir].hmask = nhashmask;
501 
502 	write_seqcount_end(&xfrm_policy_hash_generation);
503 	spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
504 
505 	synchronize_rcu();
506 
507 	xfrm_hash_free(odst, (hmask + 1) * sizeof(struct hlist_head));
508 }
509 
510 static void xfrm_byidx_resize(struct net *net, int total)
511 {
512 	unsigned int hmask = net->xfrm.policy_idx_hmask;
513 	unsigned int nhashmask = xfrm_new_hash_mask(hmask);
514 	unsigned int nsize = (nhashmask + 1) * sizeof(struct hlist_head);
515 	struct hlist_head *oidx = net->xfrm.policy_byidx;
516 	struct hlist_head *nidx = xfrm_hash_alloc(nsize);
517 	int i;
518 
519 	if (!nidx)
520 		return;
521 
522 	spin_lock_bh(&net->xfrm.xfrm_policy_lock);
523 
524 	for (i = hmask; i >= 0; i--)
525 		xfrm_idx_hash_transfer(oidx + i, nidx, nhashmask);
526 
527 	net->xfrm.policy_byidx = nidx;
528 	net->xfrm.policy_idx_hmask = nhashmask;
529 
530 	spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
531 
532 	xfrm_hash_free(oidx, (hmask + 1) * sizeof(struct hlist_head));
533 }
534 
535 static inline int xfrm_bydst_should_resize(struct net *net, int dir, int *total)
536 {
537 	unsigned int cnt = net->xfrm.policy_count[dir];
538 	unsigned int hmask = net->xfrm.policy_bydst[dir].hmask;
539 
540 	if (total)
541 		*total += cnt;
542 
543 	if ((hmask + 1) < xfrm_policy_hashmax &&
544 	    cnt > hmask)
545 		return 1;
546 
547 	return 0;
548 }
549 
550 static inline int xfrm_byidx_should_resize(struct net *net, int total)
551 {
552 	unsigned int hmask = net->xfrm.policy_idx_hmask;
553 
554 	if ((hmask + 1) < xfrm_policy_hashmax &&
555 	    total > hmask)
556 		return 1;
557 
558 	return 0;
559 }
560 
561 void xfrm_spd_getinfo(struct net *net, struct xfrmk_spdinfo *si)
562 {
563 	si->incnt = net->xfrm.policy_count[XFRM_POLICY_IN];
564 	si->outcnt = net->xfrm.policy_count[XFRM_POLICY_OUT];
565 	si->fwdcnt = net->xfrm.policy_count[XFRM_POLICY_FWD];
566 	si->inscnt = net->xfrm.policy_count[XFRM_POLICY_IN+XFRM_POLICY_MAX];
567 	si->outscnt = net->xfrm.policy_count[XFRM_POLICY_OUT+XFRM_POLICY_MAX];
568 	si->fwdscnt = net->xfrm.policy_count[XFRM_POLICY_FWD+XFRM_POLICY_MAX];
569 	si->spdhcnt = net->xfrm.policy_idx_hmask;
570 	si->spdhmcnt = xfrm_policy_hashmax;
571 }
572 EXPORT_SYMBOL(xfrm_spd_getinfo);
573 
574 static DEFINE_MUTEX(hash_resize_mutex);
575 static void xfrm_hash_resize(struct work_struct *work)
576 {
577 	struct net *net = container_of(work, struct net, xfrm.policy_hash_work);
578 	int dir, total;
579 
580 	mutex_lock(&hash_resize_mutex);
581 
582 	total = 0;
583 	for (dir = 0; dir < XFRM_POLICY_MAX; dir++) {
584 		if (xfrm_bydst_should_resize(net, dir, &total))
585 			xfrm_bydst_resize(net, dir);
586 	}
587 	if (xfrm_byidx_should_resize(net, total))
588 		xfrm_byidx_resize(net, total);
589 
590 	mutex_unlock(&hash_resize_mutex);
591 }
592 
593 static void xfrm_hash_rebuild(struct work_struct *work)
594 {
595 	struct net *net = container_of(work, struct net,
596 				       xfrm.policy_hthresh.work);
597 	unsigned int hmask;
598 	struct xfrm_policy *pol;
599 	struct xfrm_policy *policy;
600 	struct hlist_head *chain;
601 	struct hlist_head *odst;
602 	struct hlist_node *newpos;
603 	int i;
604 	int dir;
605 	unsigned seq;
606 	u8 lbits4, rbits4, lbits6, rbits6;
607 
608 	mutex_lock(&hash_resize_mutex);
609 
610 	/* read selector prefixlen thresholds */
611 	do {
612 		seq = read_seqbegin(&net->xfrm.policy_hthresh.lock);
613 
614 		lbits4 = net->xfrm.policy_hthresh.lbits4;
615 		rbits4 = net->xfrm.policy_hthresh.rbits4;
616 		lbits6 = net->xfrm.policy_hthresh.lbits6;
617 		rbits6 = net->xfrm.policy_hthresh.rbits6;
618 	} while (read_seqretry(&net->xfrm.policy_hthresh.lock, seq));
619 
620 	spin_lock_bh(&net->xfrm.xfrm_policy_lock);
621 
622 	/* reset the bydst and inexact table in all directions */
623 	for (dir = 0; dir < XFRM_POLICY_MAX; dir++) {
624 		INIT_HLIST_HEAD(&net->xfrm.policy_inexact[dir]);
625 		hmask = net->xfrm.policy_bydst[dir].hmask;
626 		odst = net->xfrm.policy_bydst[dir].table;
627 		for (i = hmask; i >= 0; i--)
628 			INIT_HLIST_HEAD(odst + i);
629 		if ((dir & XFRM_POLICY_MASK) == XFRM_POLICY_OUT) {
630 			/* dir out => dst = remote, src = local */
631 			net->xfrm.policy_bydst[dir].dbits4 = rbits4;
632 			net->xfrm.policy_bydst[dir].sbits4 = lbits4;
633 			net->xfrm.policy_bydst[dir].dbits6 = rbits6;
634 			net->xfrm.policy_bydst[dir].sbits6 = lbits6;
635 		} else {
636 			/* dir in/fwd => dst = local, src = remote */
637 			net->xfrm.policy_bydst[dir].dbits4 = lbits4;
638 			net->xfrm.policy_bydst[dir].sbits4 = rbits4;
639 			net->xfrm.policy_bydst[dir].dbits6 = lbits6;
640 			net->xfrm.policy_bydst[dir].sbits6 = rbits6;
641 		}
642 	}
643 
644 	/* re-insert all policies by order of creation */
645 	list_for_each_entry_reverse(policy, &net->xfrm.policy_all, walk.all) {
646 		if (xfrm_policy_id2dir(policy->index) >= XFRM_POLICY_MAX) {
647 			/* skip socket policies */
648 			continue;
649 		}
650 		newpos = NULL;
651 		chain = policy_hash_bysel(net, &policy->selector,
652 					  policy->family,
653 					  xfrm_policy_id2dir(policy->index));
654 		hlist_for_each_entry(pol, chain, bydst) {
655 			if (policy->priority >= pol->priority)
656 				newpos = &pol->bydst;
657 			else
658 				break;
659 		}
660 		if (newpos)
661 			hlist_add_behind(&policy->bydst, newpos);
662 		else
663 			hlist_add_head(&policy->bydst, chain);
664 	}
665 
666 	spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
667 
668 	mutex_unlock(&hash_resize_mutex);
669 }
670 
671 void xfrm_policy_hash_rebuild(struct net *net)
672 {
673 	schedule_work(&net->xfrm.policy_hthresh.work);
674 }
675 EXPORT_SYMBOL(xfrm_policy_hash_rebuild);
676 
677 /* Generate new index... KAME seems to generate them ordered by cost
678  * of an absolute inpredictability of ordering of rules. This will not pass. */
679 static u32 xfrm_gen_index(struct net *net, int dir, u32 index)
680 {
681 	static u32 idx_generator;
682 
683 	for (;;) {
684 		struct hlist_head *list;
685 		struct xfrm_policy *p;
686 		u32 idx;
687 		int found;
688 
689 		if (!index) {
690 			idx = (idx_generator | dir);
691 			idx_generator += 8;
692 		} else {
693 			idx = index;
694 			index = 0;
695 		}
696 
697 		if (idx == 0)
698 			idx = 8;
699 		list = net->xfrm.policy_byidx + idx_hash(net, idx);
700 		found = 0;
701 		hlist_for_each_entry(p, list, byidx) {
702 			if (p->index == idx) {
703 				found = 1;
704 				break;
705 			}
706 		}
707 		if (!found)
708 			return idx;
709 	}
710 }
711 
712 static inline int selector_cmp(struct xfrm_selector *s1, struct xfrm_selector *s2)
713 {
714 	u32 *p1 = (u32 *) s1;
715 	u32 *p2 = (u32 *) s2;
716 	int len = sizeof(struct xfrm_selector) / sizeof(u32);
717 	int i;
718 
719 	for (i = 0; i < len; i++) {
720 		if (p1[i] != p2[i])
721 			return 1;
722 	}
723 
724 	return 0;
725 }
726 
727 static void xfrm_policy_requeue(struct xfrm_policy *old,
728 				struct xfrm_policy *new)
729 {
730 	struct xfrm_policy_queue *pq = &old->polq;
731 	struct sk_buff_head list;
732 
733 	if (skb_queue_empty(&pq->hold_queue))
734 		return;
735 
736 	__skb_queue_head_init(&list);
737 
738 	spin_lock_bh(&pq->hold_queue.lock);
739 	skb_queue_splice_init(&pq->hold_queue, &list);
740 	if (del_timer(&pq->hold_timer))
741 		xfrm_pol_put(old);
742 	spin_unlock_bh(&pq->hold_queue.lock);
743 
744 	pq = &new->polq;
745 
746 	spin_lock_bh(&pq->hold_queue.lock);
747 	skb_queue_splice(&list, &pq->hold_queue);
748 	pq->timeout = XFRM_QUEUE_TMO_MIN;
749 	if (!mod_timer(&pq->hold_timer, jiffies))
750 		xfrm_pol_hold(new);
751 	spin_unlock_bh(&pq->hold_queue.lock);
752 }
753 
754 static bool xfrm_policy_mark_match(struct xfrm_policy *policy,
755 				   struct xfrm_policy *pol)
756 {
757 	u32 mark = policy->mark.v & policy->mark.m;
758 
759 	if (policy->mark.v == pol->mark.v && policy->mark.m == pol->mark.m)
760 		return true;
761 
762 	if ((mark & pol->mark.m) == pol->mark.v &&
763 	    policy->priority == pol->priority)
764 		return true;
765 
766 	return false;
767 }
768 
769 int xfrm_policy_insert(int dir, struct xfrm_policy *policy, int excl)
770 {
771 	struct net *net = xp_net(policy);
772 	struct xfrm_policy *pol;
773 	struct xfrm_policy *delpol;
774 	struct hlist_head *chain;
775 	struct hlist_node *newpos;
776 
777 	spin_lock_bh(&net->xfrm.xfrm_policy_lock);
778 	chain = policy_hash_bysel(net, &policy->selector, policy->family, dir);
779 	delpol = NULL;
780 	newpos = NULL;
781 	hlist_for_each_entry(pol, chain, bydst) {
782 		if (pol->type == policy->type &&
783 		    !selector_cmp(&pol->selector, &policy->selector) &&
784 		    xfrm_policy_mark_match(policy, pol) &&
785 		    xfrm_sec_ctx_match(pol->security, policy->security) &&
786 		    !WARN_ON(delpol)) {
787 			if (excl) {
788 				spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
789 				return -EEXIST;
790 			}
791 			delpol = pol;
792 			if (policy->priority > pol->priority)
793 				continue;
794 		} else if (policy->priority >= pol->priority) {
795 			newpos = &pol->bydst;
796 			continue;
797 		}
798 		if (delpol)
799 			break;
800 	}
801 	if (newpos)
802 		hlist_add_behind(&policy->bydst, newpos);
803 	else
804 		hlist_add_head(&policy->bydst, chain);
805 	__xfrm_policy_link(policy, dir);
806 	atomic_inc(&net->xfrm.flow_cache_genid);
807 
808 	/* After previous checking, family can either be AF_INET or AF_INET6 */
809 	if (policy->family == AF_INET)
810 		rt_genid_bump_ipv4(net);
811 	else
812 		rt_genid_bump_ipv6(net);
813 
814 	if (delpol) {
815 		xfrm_policy_requeue(delpol, policy);
816 		__xfrm_policy_unlink(delpol, dir);
817 	}
818 	policy->index = delpol ? delpol->index : xfrm_gen_index(net, dir, policy->index);
819 	hlist_add_head(&policy->byidx, net->xfrm.policy_byidx+idx_hash(net, policy->index));
820 	policy->curlft.add_time = get_seconds();
821 	policy->curlft.use_time = 0;
822 	if (!mod_timer(&policy->timer, jiffies + HZ))
823 		xfrm_pol_hold(policy);
824 	spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
825 
826 	if (delpol)
827 		xfrm_policy_kill(delpol);
828 	else if (xfrm_bydst_should_resize(net, dir, NULL))
829 		schedule_work(&net->xfrm.policy_hash_work);
830 
831 	return 0;
832 }
833 EXPORT_SYMBOL(xfrm_policy_insert);
834 
835 struct xfrm_policy *xfrm_policy_bysel_ctx(struct net *net, u32 mark, u8 type,
836 					  int dir, struct xfrm_selector *sel,
837 					  struct xfrm_sec_ctx *ctx, int delete,
838 					  int *err)
839 {
840 	struct xfrm_policy *pol, *ret;
841 	struct hlist_head *chain;
842 
843 	*err = 0;
844 	spin_lock_bh(&net->xfrm.xfrm_policy_lock);
845 	chain = policy_hash_bysel(net, sel, sel->family, dir);
846 	ret = NULL;
847 	hlist_for_each_entry(pol, chain, bydst) {
848 		if (pol->type == type &&
849 		    (mark & pol->mark.m) == pol->mark.v &&
850 		    !selector_cmp(sel, &pol->selector) &&
851 		    xfrm_sec_ctx_match(ctx, pol->security)) {
852 			xfrm_pol_hold(pol);
853 			if (delete) {
854 				*err = security_xfrm_policy_delete(
855 								pol->security);
856 				if (*err) {
857 					spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
858 					return pol;
859 				}
860 				__xfrm_policy_unlink(pol, dir);
861 			}
862 			ret = pol;
863 			break;
864 		}
865 	}
866 	spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
867 
868 	if (ret && delete)
869 		xfrm_policy_kill(ret);
870 	return ret;
871 }
872 EXPORT_SYMBOL(xfrm_policy_bysel_ctx);
873 
874 struct xfrm_policy *xfrm_policy_byid(struct net *net, u32 mark, u8 type,
875 				     int dir, u32 id, int delete, int *err)
876 {
877 	struct xfrm_policy *pol, *ret;
878 	struct hlist_head *chain;
879 
880 	*err = -ENOENT;
881 	if (xfrm_policy_id2dir(id) != dir)
882 		return NULL;
883 
884 	*err = 0;
885 	spin_lock_bh(&net->xfrm.xfrm_policy_lock);
886 	chain = net->xfrm.policy_byidx + idx_hash(net, id);
887 	ret = NULL;
888 	hlist_for_each_entry(pol, chain, byidx) {
889 		if (pol->type == type && pol->index == id &&
890 		    (mark & pol->mark.m) == pol->mark.v) {
891 			xfrm_pol_hold(pol);
892 			if (delete) {
893 				*err = security_xfrm_policy_delete(
894 								pol->security);
895 				if (*err) {
896 					spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
897 					return pol;
898 				}
899 				__xfrm_policy_unlink(pol, dir);
900 			}
901 			ret = pol;
902 			break;
903 		}
904 	}
905 	spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
906 
907 	if (ret && delete)
908 		xfrm_policy_kill(ret);
909 	return ret;
910 }
911 EXPORT_SYMBOL(xfrm_policy_byid);
912 
913 #ifdef CONFIG_SECURITY_NETWORK_XFRM
914 static inline int
915 xfrm_policy_flush_secctx_check(struct net *net, u8 type, bool task_valid)
916 {
917 	int dir, err = 0;
918 
919 	for (dir = 0; dir < XFRM_POLICY_MAX; dir++) {
920 		struct xfrm_policy *pol;
921 		int i;
922 
923 		hlist_for_each_entry(pol,
924 				     &net->xfrm.policy_inexact[dir], bydst) {
925 			if (pol->type != type)
926 				continue;
927 			err = security_xfrm_policy_delete(pol->security);
928 			if (err) {
929 				xfrm_audit_policy_delete(pol, 0, task_valid);
930 				return err;
931 			}
932 		}
933 		for (i = net->xfrm.policy_bydst[dir].hmask; i >= 0; i--) {
934 			hlist_for_each_entry(pol,
935 					     net->xfrm.policy_bydst[dir].table + i,
936 					     bydst) {
937 				if (pol->type != type)
938 					continue;
939 				err = security_xfrm_policy_delete(
940 								pol->security);
941 				if (err) {
942 					xfrm_audit_policy_delete(pol, 0,
943 								 task_valid);
944 					return err;
945 				}
946 			}
947 		}
948 	}
949 	return err;
950 }
951 #else
952 static inline int
953 xfrm_policy_flush_secctx_check(struct net *net, u8 type, bool task_valid)
954 {
955 	return 0;
956 }
957 #endif
958 
959 int xfrm_policy_flush(struct net *net, u8 type, bool task_valid)
960 {
961 	int dir, err = 0, cnt = 0;
962 
963 	spin_lock_bh(&net->xfrm.xfrm_policy_lock);
964 
965 	err = xfrm_policy_flush_secctx_check(net, type, task_valid);
966 	if (err)
967 		goto out;
968 
969 	for (dir = 0; dir < XFRM_POLICY_MAX; dir++) {
970 		struct xfrm_policy *pol;
971 		int i;
972 
973 	again1:
974 		hlist_for_each_entry(pol,
975 				     &net->xfrm.policy_inexact[dir], bydst) {
976 			if (pol->type != type)
977 				continue;
978 			__xfrm_policy_unlink(pol, dir);
979 			spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
980 			cnt++;
981 
982 			xfrm_audit_policy_delete(pol, 1, task_valid);
983 
984 			xfrm_policy_kill(pol);
985 
986 			spin_lock_bh(&net->xfrm.xfrm_policy_lock);
987 			goto again1;
988 		}
989 
990 		for (i = net->xfrm.policy_bydst[dir].hmask; i >= 0; i--) {
991 	again2:
992 			hlist_for_each_entry(pol,
993 					     net->xfrm.policy_bydst[dir].table + i,
994 					     bydst) {
995 				if (pol->type != type)
996 					continue;
997 				__xfrm_policy_unlink(pol, dir);
998 				spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
999 				cnt++;
1000 
1001 				xfrm_audit_policy_delete(pol, 1, task_valid);
1002 				xfrm_policy_kill(pol);
1003 
1004 				spin_lock_bh(&net->xfrm.xfrm_policy_lock);
1005 				goto again2;
1006 			}
1007 		}
1008 
1009 	}
1010 	if (!cnt)
1011 		err = -ESRCH;
1012 out:
1013 	spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
1014 	return err;
1015 }
1016 EXPORT_SYMBOL(xfrm_policy_flush);
1017 
1018 int xfrm_policy_walk(struct net *net, struct xfrm_policy_walk *walk,
1019 		     int (*func)(struct xfrm_policy *, int, int, void*),
1020 		     void *data)
1021 {
1022 	struct xfrm_policy *pol;
1023 	struct xfrm_policy_walk_entry *x;
1024 	int error = 0;
1025 
1026 	if (walk->type >= XFRM_POLICY_TYPE_MAX &&
1027 	    walk->type != XFRM_POLICY_TYPE_ANY)
1028 		return -EINVAL;
1029 
1030 	if (list_empty(&walk->walk.all) && walk->seq != 0)
1031 		return 0;
1032 
1033 	spin_lock_bh(&net->xfrm.xfrm_policy_lock);
1034 	if (list_empty(&walk->walk.all))
1035 		x = list_first_entry(&net->xfrm.policy_all, struct xfrm_policy_walk_entry, all);
1036 	else
1037 		x = list_first_entry(&walk->walk.all,
1038 				     struct xfrm_policy_walk_entry, all);
1039 
1040 	list_for_each_entry_from(x, &net->xfrm.policy_all, all) {
1041 		if (x->dead)
1042 			continue;
1043 		pol = container_of(x, struct xfrm_policy, walk);
1044 		if (walk->type != XFRM_POLICY_TYPE_ANY &&
1045 		    walk->type != pol->type)
1046 			continue;
1047 		error = func(pol, xfrm_policy_id2dir(pol->index),
1048 			     walk->seq, data);
1049 		if (error) {
1050 			list_move_tail(&walk->walk.all, &x->all);
1051 			goto out;
1052 		}
1053 		walk->seq++;
1054 	}
1055 	if (walk->seq == 0) {
1056 		error = -ENOENT;
1057 		goto out;
1058 	}
1059 	list_del_init(&walk->walk.all);
1060 out:
1061 	spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
1062 	return error;
1063 }
1064 EXPORT_SYMBOL(xfrm_policy_walk);
1065 
1066 void xfrm_policy_walk_init(struct xfrm_policy_walk *walk, u8 type)
1067 {
1068 	INIT_LIST_HEAD(&walk->walk.all);
1069 	walk->walk.dead = 1;
1070 	walk->type = type;
1071 	walk->seq = 0;
1072 }
1073 EXPORT_SYMBOL(xfrm_policy_walk_init);
1074 
1075 void xfrm_policy_walk_done(struct xfrm_policy_walk *walk, struct net *net)
1076 {
1077 	if (list_empty(&walk->walk.all))
1078 		return;
1079 
1080 	spin_lock_bh(&net->xfrm.xfrm_policy_lock); /*FIXME where is net? */
1081 	list_del(&walk->walk.all);
1082 	spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
1083 }
1084 EXPORT_SYMBOL(xfrm_policy_walk_done);
1085 
1086 /*
1087  * Find policy to apply to this flow.
1088  *
1089  * Returns 0 if policy found, else an -errno.
1090  */
1091 static int xfrm_policy_match(const struct xfrm_policy *pol,
1092 			     const struct flowi *fl,
1093 			     u8 type, u16 family, int dir)
1094 {
1095 	const struct xfrm_selector *sel = &pol->selector;
1096 	int ret = -ESRCH;
1097 	bool match;
1098 
1099 	if (pol->family != family ||
1100 	    (fl->flowi_mark & pol->mark.m) != pol->mark.v ||
1101 	    pol->type != type)
1102 		return ret;
1103 
1104 	match = xfrm_selector_match(sel, fl, family);
1105 	if (match)
1106 		ret = security_xfrm_policy_lookup(pol->security, fl->flowi_secid,
1107 						  dir);
1108 
1109 	return ret;
1110 }
1111 
1112 static struct xfrm_policy *xfrm_policy_lookup_bytype(struct net *net, u8 type,
1113 						     const struct flowi *fl,
1114 						     u16 family, u8 dir)
1115 {
1116 	int err;
1117 	struct xfrm_policy *pol, *ret;
1118 	const xfrm_address_t *daddr, *saddr;
1119 	struct hlist_head *chain;
1120 	unsigned int sequence;
1121 	u32 priority;
1122 
1123 	daddr = xfrm_flowi_daddr(fl, family);
1124 	saddr = xfrm_flowi_saddr(fl, family);
1125 	if (unlikely(!daddr || !saddr))
1126 		return NULL;
1127 
1128 	rcu_read_lock();
1129  retry:
1130 	do {
1131 		sequence = read_seqcount_begin(&xfrm_policy_hash_generation);
1132 		chain = policy_hash_direct(net, daddr, saddr, family, dir);
1133 	} while (read_seqcount_retry(&xfrm_policy_hash_generation, sequence));
1134 
1135 	priority = ~0U;
1136 	ret = NULL;
1137 	hlist_for_each_entry_rcu(pol, chain, bydst) {
1138 		err = xfrm_policy_match(pol, fl, type, family, dir);
1139 		if (err) {
1140 			if (err == -ESRCH)
1141 				continue;
1142 			else {
1143 				ret = ERR_PTR(err);
1144 				goto fail;
1145 			}
1146 		} else {
1147 			ret = pol;
1148 			priority = ret->priority;
1149 			break;
1150 		}
1151 	}
1152 	chain = &net->xfrm.policy_inexact[dir];
1153 	hlist_for_each_entry_rcu(pol, chain, bydst) {
1154 		if ((pol->priority >= priority) && ret)
1155 			break;
1156 
1157 		err = xfrm_policy_match(pol, fl, type, family, dir);
1158 		if (err) {
1159 			if (err == -ESRCH)
1160 				continue;
1161 			else {
1162 				ret = ERR_PTR(err);
1163 				goto fail;
1164 			}
1165 		} else {
1166 			ret = pol;
1167 			break;
1168 		}
1169 	}
1170 
1171 	if (read_seqcount_retry(&xfrm_policy_hash_generation, sequence))
1172 		goto retry;
1173 
1174 	if (ret && !xfrm_pol_hold_rcu(ret))
1175 		goto retry;
1176 fail:
1177 	rcu_read_unlock();
1178 
1179 	return ret;
1180 }
1181 
1182 static struct xfrm_policy *
1183 __xfrm_policy_lookup(struct net *net, const struct flowi *fl, u16 family, u8 dir)
1184 {
1185 #ifdef CONFIG_XFRM_SUB_POLICY
1186 	struct xfrm_policy *pol;
1187 
1188 	pol = xfrm_policy_lookup_bytype(net, XFRM_POLICY_TYPE_SUB, fl, family, dir);
1189 	if (pol != NULL)
1190 		return pol;
1191 #endif
1192 	return xfrm_policy_lookup_bytype(net, XFRM_POLICY_TYPE_MAIN, fl, family, dir);
1193 }
1194 
1195 static int flow_to_policy_dir(int dir)
1196 {
1197 	if (XFRM_POLICY_IN == FLOW_DIR_IN &&
1198 	    XFRM_POLICY_OUT == FLOW_DIR_OUT &&
1199 	    XFRM_POLICY_FWD == FLOW_DIR_FWD)
1200 		return dir;
1201 
1202 	switch (dir) {
1203 	default:
1204 	case FLOW_DIR_IN:
1205 		return XFRM_POLICY_IN;
1206 	case FLOW_DIR_OUT:
1207 		return XFRM_POLICY_OUT;
1208 	case FLOW_DIR_FWD:
1209 		return XFRM_POLICY_FWD;
1210 	}
1211 }
1212 
1213 static struct flow_cache_object *
1214 xfrm_policy_lookup(struct net *net, const struct flowi *fl, u16 family,
1215 		   u8 dir, struct flow_cache_object *old_obj, void *ctx)
1216 {
1217 	struct xfrm_policy *pol;
1218 
1219 	if (old_obj)
1220 		xfrm_pol_put(container_of(old_obj, struct xfrm_policy, flo));
1221 
1222 	pol = __xfrm_policy_lookup(net, fl, family, flow_to_policy_dir(dir));
1223 	if (IS_ERR_OR_NULL(pol))
1224 		return ERR_CAST(pol);
1225 
1226 	/* Resolver returns two references:
1227 	 * one for cache and one for caller of flow_cache_lookup() */
1228 	xfrm_pol_hold(pol);
1229 
1230 	return &pol->flo;
1231 }
1232 
1233 static inline int policy_to_flow_dir(int dir)
1234 {
1235 	if (XFRM_POLICY_IN == FLOW_DIR_IN &&
1236 	    XFRM_POLICY_OUT == FLOW_DIR_OUT &&
1237 	    XFRM_POLICY_FWD == FLOW_DIR_FWD)
1238 		return dir;
1239 	switch (dir) {
1240 	default:
1241 	case XFRM_POLICY_IN:
1242 		return FLOW_DIR_IN;
1243 	case XFRM_POLICY_OUT:
1244 		return FLOW_DIR_OUT;
1245 	case XFRM_POLICY_FWD:
1246 		return FLOW_DIR_FWD;
1247 	}
1248 }
1249 
1250 static struct xfrm_policy *xfrm_sk_policy_lookup(const struct sock *sk, int dir,
1251 						 const struct flowi *fl)
1252 {
1253 	struct xfrm_policy *pol;
1254 
1255 	rcu_read_lock();
1256  again:
1257 	pol = rcu_dereference(sk->sk_policy[dir]);
1258 	if (pol != NULL) {
1259 		bool match = xfrm_selector_match(&pol->selector, fl,
1260 						 sk->sk_family);
1261 		int err = 0;
1262 
1263 		if (match) {
1264 			if ((sk->sk_mark & pol->mark.m) != pol->mark.v) {
1265 				pol = NULL;
1266 				goto out;
1267 			}
1268 			err = security_xfrm_policy_lookup(pol->security,
1269 						      fl->flowi_secid,
1270 						      policy_to_flow_dir(dir));
1271 			if (!err) {
1272 				if (!xfrm_pol_hold_rcu(pol))
1273 					goto again;
1274 			} else if (err == -ESRCH) {
1275 				pol = NULL;
1276 			} else {
1277 				pol = ERR_PTR(err);
1278 			}
1279 		} else
1280 			pol = NULL;
1281 	}
1282 out:
1283 	rcu_read_unlock();
1284 	return pol;
1285 }
1286 
1287 static void __xfrm_policy_link(struct xfrm_policy *pol, int dir)
1288 {
1289 	struct net *net = xp_net(pol);
1290 
1291 	list_add(&pol->walk.all, &net->xfrm.policy_all);
1292 	net->xfrm.policy_count[dir]++;
1293 	xfrm_pol_hold(pol);
1294 }
1295 
1296 static struct xfrm_policy *__xfrm_policy_unlink(struct xfrm_policy *pol,
1297 						int dir)
1298 {
1299 	struct net *net = xp_net(pol);
1300 
1301 	if (list_empty(&pol->walk.all))
1302 		return NULL;
1303 
1304 	/* Socket policies are not hashed. */
1305 	if (!hlist_unhashed(&pol->bydst)) {
1306 		hlist_del_rcu(&pol->bydst);
1307 		hlist_del(&pol->byidx);
1308 	}
1309 
1310 	list_del_init(&pol->walk.all);
1311 	net->xfrm.policy_count[dir]--;
1312 
1313 	return pol;
1314 }
1315 
1316 static void xfrm_sk_policy_link(struct xfrm_policy *pol, int dir)
1317 {
1318 	__xfrm_policy_link(pol, XFRM_POLICY_MAX + dir);
1319 }
1320 
1321 static void xfrm_sk_policy_unlink(struct xfrm_policy *pol, int dir)
1322 {
1323 	__xfrm_policy_unlink(pol, XFRM_POLICY_MAX + dir);
1324 }
1325 
1326 int xfrm_policy_delete(struct xfrm_policy *pol, int dir)
1327 {
1328 	struct net *net = xp_net(pol);
1329 
1330 	spin_lock_bh(&net->xfrm.xfrm_policy_lock);
1331 	pol = __xfrm_policy_unlink(pol, dir);
1332 	spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
1333 	if (pol) {
1334 		xfrm_policy_kill(pol);
1335 		return 0;
1336 	}
1337 	return -ENOENT;
1338 }
1339 EXPORT_SYMBOL(xfrm_policy_delete);
1340 
1341 int xfrm_sk_policy_insert(struct sock *sk, int dir, struct xfrm_policy *pol)
1342 {
1343 	struct net *net = xp_net(pol);
1344 	struct xfrm_policy *old_pol;
1345 
1346 #ifdef CONFIG_XFRM_SUB_POLICY
1347 	if (pol && pol->type != XFRM_POLICY_TYPE_MAIN)
1348 		return -EINVAL;
1349 #endif
1350 
1351 	spin_lock_bh(&net->xfrm.xfrm_policy_lock);
1352 	old_pol = rcu_dereference_protected(sk->sk_policy[dir],
1353 				lockdep_is_held(&net->xfrm.xfrm_policy_lock));
1354 	if (pol) {
1355 		pol->curlft.add_time = get_seconds();
1356 		pol->index = xfrm_gen_index(net, XFRM_POLICY_MAX+dir, 0);
1357 		xfrm_sk_policy_link(pol, dir);
1358 	}
1359 	rcu_assign_pointer(sk->sk_policy[dir], pol);
1360 	if (old_pol) {
1361 		if (pol)
1362 			xfrm_policy_requeue(old_pol, pol);
1363 
1364 		/* Unlinking succeeds always. This is the only function
1365 		 * allowed to delete or replace socket policy.
1366 		 */
1367 		xfrm_sk_policy_unlink(old_pol, dir);
1368 	}
1369 	spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
1370 
1371 	if (old_pol) {
1372 		xfrm_policy_kill(old_pol);
1373 	}
1374 	return 0;
1375 }
1376 
1377 static struct xfrm_policy *clone_policy(const struct xfrm_policy *old, int dir)
1378 {
1379 	struct xfrm_policy *newp = xfrm_policy_alloc(xp_net(old), GFP_ATOMIC);
1380 	struct net *net = xp_net(old);
1381 
1382 	if (newp) {
1383 		newp->selector = old->selector;
1384 		if (security_xfrm_policy_clone(old->security,
1385 					       &newp->security)) {
1386 			kfree(newp);
1387 			return NULL;  /* ENOMEM */
1388 		}
1389 		newp->lft = old->lft;
1390 		newp->curlft = old->curlft;
1391 		newp->mark = old->mark;
1392 		newp->action = old->action;
1393 		newp->flags = old->flags;
1394 		newp->xfrm_nr = old->xfrm_nr;
1395 		newp->index = old->index;
1396 		newp->type = old->type;
1397 		memcpy(newp->xfrm_vec, old->xfrm_vec,
1398 		       newp->xfrm_nr*sizeof(struct xfrm_tmpl));
1399 		spin_lock_bh(&net->xfrm.xfrm_policy_lock);
1400 		xfrm_sk_policy_link(newp, dir);
1401 		spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
1402 		xfrm_pol_put(newp);
1403 	}
1404 	return newp;
1405 }
1406 
1407 int __xfrm_sk_clone_policy(struct sock *sk, const struct sock *osk)
1408 {
1409 	const struct xfrm_policy *p;
1410 	struct xfrm_policy *np;
1411 	int i, ret = 0;
1412 
1413 	rcu_read_lock();
1414 	for (i = 0; i < 2; i++) {
1415 		p = rcu_dereference(osk->sk_policy[i]);
1416 		if (p) {
1417 			np = clone_policy(p, i);
1418 			if (unlikely(!np)) {
1419 				ret = -ENOMEM;
1420 				break;
1421 			}
1422 			rcu_assign_pointer(sk->sk_policy[i], np);
1423 		}
1424 	}
1425 	rcu_read_unlock();
1426 	return ret;
1427 }
1428 
1429 static int
1430 xfrm_get_saddr(struct net *net, int oif, xfrm_address_t *local,
1431 	       xfrm_address_t *remote, unsigned short family)
1432 {
1433 	int err;
1434 	struct xfrm_policy_afinfo *afinfo = xfrm_policy_get_afinfo(family);
1435 
1436 	if (unlikely(afinfo == NULL))
1437 		return -EINVAL;
1438 	err = afinfo->get_saddr(net, oif, local, remote);
1439 	xfrm_policy_put_afinfo(afinfo);
1440 	return err;
1441 }
1442 
1443 /* Resolve list of templates for the flow, given policy. */
1444 
1445 static int
1446 xfrm_tmpl_resolve_one(struct xfrm_policy *policy, const struct flowi *fl,
1447 		      struct xfrm_state **xfrm, unsigned short family)
1448 {
1449 	struct net *net = xp_net(policy);
1450 	int nx;
1451 	int i, error;
1452 	xfrm_address_t *daddr = xfrm_flowi_daddr(fl, family);
1453 	xfrm_address_t *saddr = xfrm_flowi_saddr(fl, family);
1454 	xfrm_address_t tmp;
1455 
1456 	for (nx = 0, i = 0; i < policy->xfrm_nr; i++) {
1457 		struct xfrm_state *x;
1458 		xfrm_address_t *remote = daddr;
1459 		xfrm_address_t *local  = saddr;
1460 		struct xfrm_tmpl *tmpl = &policy->xfrm_vec[i];
1461 
1462 		if (tmpl->mode == XFRM_MODE_TUNNEL ||
1463 		    tmpl->mode == XFRM_MODE_BEET) {
1464 			remote = &tmpl->id.daddr;
1465 			local = &tmpl->saddr;
1466 			if (xfrm_addr_any(local, tmpl->encap_family)) {
1467 				error = xfrm_get_saddr(net, fl->flowi_oif,
1468 						       &tmp, remote,
1469 						       tmpl->encap_family);
1470 				if (error)
1471 					goto fail;
1472 				local = &tmp;
1473 			}
1474 		}
1475 
1476 		x = xfrm_state_find(remote, local, fl, tmpl, policy, &error, family);
1477 
1478 		if (x && x->km.state == XFRM_STATE_VALID) {
1479 			xfrm[nx++] = x;
1480 			daddr = remote;
1481 			saddr = local;
1482 			continue;
1483 		}
1484 		if (x) {
1485 			error = (x->km.state == XFRM_STATE_ERROR ?
1486 				 -EINVAL : -EAGAIN);
1487 			xfrm_state_put(x);
1488 		} else if (error == -ESRCH) {
1489 			error = -EAGAIN;
1490 		}
1491 
1492 		if (!tmpl->optional)
1493 			goto fail;
1494 	}
1495 	return nx;
1496 
1497 fail:
1498 	for (nx--; nx >= 0; nx--)
1499 		xfrm_state_put(xfrm[nx]);
1500 	return error;
1501 }
1502 
1503 static int
1504 xfrm_tmpl_resolve(struct xfrm_policy **pols, int npols, const struct flowi *fl,
1505 		  struct xfrm_state **xfrm, unsigned short family)
1506 {
1507 	struct xfrm_state *tp[XFRM_MAX_DEPTH];
1508 	struct xfrm_state **tpp = (npols > 1) ? tp : xfrm;
1509 	int cnx = 0;
1510 	int error;
1511 	int ret;
1512 	int i;
1513 
1514 	for (i = 0; i < npols; i++) {
1515 		if (cnx + pols[i]->xfrm_nr >= XFRM_MAX_DEPTH) {
1516 			error = -ENOBUFS;
1517 			goto fail;
1518 		}
1519 
1520 		ret = xfrm_tmpl_resolve_one(pols[i], fl, &tpp[cnx], family);
1521 		if (ret < 0) {
1522 			error = ret;
1523 			goto fail;
1524 		} else
1525 			cnx += ret;
1526 	}
1527 
1528 	/* found states are sorted for outbound processing */
1529 	if (npols > 1)
1530 		xfrm_state_sort(xfrm, tpp, cnx, family);
1531 
1532 	return cnx;
1533 
1534  fail:
1535 	for (cnx--; cnx >= 0; cnx--)
1536 		xfrm_state_put(tpp[cnx]);
1537 	return error;
1538 
1539 }
1540 
1541 /* Check that the bundle accepts the flow and its components are
1542  * still valid.
1543  */
1544 
1545 static inline int xfrm_get_tos(const struct flowi *fl, int family)
1546 {
1547 	struct xfrm_policy_afinfo *afinfo = xfrm_policy_get_afinfo(family);
1548 	int tos;
1549 
1550 	if (!afinfo)
1551 		return -EINVAL;
1552 
1553 	tos = afinfo->get_tos(fl);
1554 
1555 	xfrm_policy_put_afinfo(afinfo);
1556 
1557 	return tos;
1558 }
1559 
1560 static struct flow_cache_object *xfrm_bundle_flo_get(struct flow_cache_object *flo)
1561 {
1562 	struct xfrm_dst *xdst = container_of(flo, struct xfrm_dst, flo);
1563 	struct dst_entry *dst = &xdst->u.dst;
1564 
1565 	if (xdst->route == NULL) {
1566 		/* Dummy bundle - if it has xfrms we were not
1567 		 * able to build bundle as template resolution failed.
1568 		 * It means we need to try again resolving. */
1569 		if (xdst->num_xfrms > 0)
1570 			return NULL;
1571 	} else if (dst->flags & DST_XFRM_QUEUE) {
1572 		return NULL;
1573 	} else {
1574 		/* Real bundle */
1575 		if (stale_bundle(dst))
1576 			return NULL;
1577 	}
1578 
1579 	dst_hold(dst);
1580 	return flo;
1581 }
1582 
1583 static int xfrm_bundle_flo_check(struct flow_cache_object *flo)
1584 {
1585 	struct xfrm_dst *xdst = container_of(flo, struct xfrm_dst, flo);
1586 	struct dst_entry *dst = &xdst->u.dst;
1587 
1588 	if (!xdst->route)
1589 		return 0;
1590 	if (stale_bundle(dst))
1591 		return 0;
1592 
1593 	return 1;
1594 }
1595 
1596 static void xfrm_bundle_flo_delete(struct flow_cache_object *flo)
1597 {
1598 	struct xfrm_dst *xdst = container_of(flo, struct xfrm_dst, flo);
1599 	struct dst_entry *dst = &xdst->u.dst;
1600 
1601 	dst_free(dst);
1602 }
1603 
1604 static const struct flow_cache_ops xfrm_bundle_fc_ops = {
1605 	.get = xfrm_bundle_flo_get,
1606 	.check = xfrm_bundle_flo_check,
1607 	.delete = xfrm_bundle_flo_delete,
1608 };
1609 
1610 static inline struct xfrm_dst *xfrm_alloc_dst(struct net *net, int family)
1611 {
1612 	struct xfrm_policy_afinfo *afinfo = xfrm_policy_get_afinfo(family);
1613 	struct dst_ops *dst_ops;
1614 	struct xfrm_dst *xdst;
1615 
1616 	if (!afinfo)
1617 		return ERR_PTR(-EINVAL);
1618 
1619 	switch (family) {
1620 	case AF_INET:
1621 		dst_ops = &net->xfrm.xfrm4_dst_ops;
1622 		break;
1623 #if IS_ENABLED(CONFIG_IPV6)
1624 	case AF_INET6:
1625 		dst_ops = &net->xfrm.xfrm6_dst_ops;
1626 		break;
1627 #endif
1628 	default:
1629 		BUG();
1630 	}
1631 	xdst = dst_alloc(dst_ops, NULL, 0, DST_OBSOLETE_NONE, 0);
1632 
1633 	if (likely(xdst)) {
1634 		struct dst_entry *dst = &xdst->u.dst;
1635 
1636 		memset(dst + 1, 0, sizeof(*xdst) - sizeof(*dst));
1637 		xdst->flo.ops = &xfrm_bundle_fc_ops;
1638 	} else
1639 		xdst = ERR_PTR(-ENOBUFS);
1640 
1641 	xfrm_policy_put_afinfo(afinfo);
1642 
1643 	return xdst;
1644 }
1645 
1646 static inline int xfrm_init_path(struct xfrm_dst *path, struct dst_entry *dst,
1647 				 int nfheader_len)
1648 {
1649 	struct xfrm_policy_afinfo *afinfo =
1650 		xfrm_policy_get_afinfo(dst->ops->family);
1651 	int err;
1652 
1653 	if (!afinfo)
1654 		return -EINVAL;
1655 
1656 	err = afinfo->init_path(path, dst, nfheader_len);
1657 
1658 	xfrm_policy_put_afinfo(afinfo);
1659 
1660 	return err;
1661 }
1662 
1663 static inline int xfrm_fill_dst(struct xfrm_dst *xdst, struct net_device *dev,
1664 				const struct flowi *fl)
1665 {
1666 	struct xfrm_policy_afinfo *afinfo =
1667 		xfrm_policy_get_afinfo(xdst->u.dst.ops->family);
1668 	int err;
1669 
1670 	if (!afinfo)
1671 		return -EINVAL;
1672 
1673 	err = afinfo->fill_dst(xdst, dev, fl);
1674 
1675 	xfrm_policy_put_afinfo(afinfo);
1676 
1677 	return err;
1678 }
1679 
1680 
1681 /* Allocate chain of dst_entry's, attach known xfrm's, calculate
1682  * all the metrics... Shortly, bundle a bundle.
1683  */
1684 
1685 static struct dst_entry *xfrm_bundle_create(struct xfrm_policy *policy,
1686 					    struct xfrm_state **xfrm, int nx,
1687 					    const struct flowi *fl,
1688 					    struct dst_entry *dst)
1689 {
1690 	struct net *net = xp_net(policy);
1691 	unsigned long now = jiffies;
1692 	struct net_device *dev;
1693 	struct xfrm_mode *inner_mode;
1694 	struct dst_entry *dst_prev = NULL;
1695 	struct dst_entry *dst0 = NULL;
1696 	int i = 0;
1697 	int err;
1698 	int header_len = 0;
1699 	int nfheader_len = 0;
1700 	int trailer_len = 0;
1701 	int tos;
1702 	int family = policy->selector.family;
1703 	xfrm_address_t saddr, daddr;
1704 
1705 	xfrm_flowi_addr_get(fl, &saddr, &daddr, family);
1706 
1707 	tos = xfrm_get_tos(fl, family);
1708 	err = tos;
1709 	if (tos < 0)
1710 		goto put_states;
1711 
1712 	dst_hold(dst);
1713 
1714 	for (; i < nx; i++) {
1715 		struct xfrm_dst *xdst = xfrm_alloc_dst(net, family);
1716 		struct dst_entry *dst1 = &xdst->u.dst;
1717 
1718 		err = PTR_ERR(xdst);
1719 		if (IS_ERR(xdst)) {
1720 			dst_release(dst);
1721 			goto put_states;
1722 		}
1723 
1724 		if (xfrm[i]->sel.family == AF_UNSPEC) {
1725 			inner_mode = xfrm_ip2inner_mode(xfrm[i],
1726 							xfrm_af2proto(family));
1727 			if (!inner_mode) {
1728 				err = -EAFNOSUPPORT;
1729 				dst_release(dst);
1730 				goto put_states;
1731 			}
1732 		} else
1733 			inner_mode = xfrm[i]->inner_mode;
1734 
1735 		if (!dst_prev)
1736 			dst0 = dst1;
1737 		else {
1738 			dst_prev->child = dst_clone(dst1);
1739 			dst1->flags |= DST_NOHASH;
1740 		}
1741 
1742 		xdst->route = dst;
1743 		dst_copy_metrics(dst1, dst);
1744 
1745 		if (xfrm[i]->props.mode != XFRM_MODE_TRANSPORT) {
1746 			family = xfrm[i]->props.family;
1747 			dst = xfrm_dst_lookup(xfrm[i], tos, fl->flowi_oif,
1748 					      &saddr, &daddr, family);
1749 			err = PTR_ERR(dst);
1750 			if (IS_ERR(dst))
1751 				goto put_states;
1752 		} else
1753 			dst_hold(dst);
1754 
1755 		dst1->xfrm = xfrm[i];
1756 		xdst->xfrm_genid = xfrm[i]->genid;
1757 
1758 		dst1->obsolete = DST_OBSOLETE_FORCE_CHK;
1759 		dst1->flags |= DST_HOST;
1760 		dst1->lastuse = now;
1761 
1762 		dst1->input = dst_discard;
1763 		dst1->output = inner_mode->afinfo->output;
1764 
1765 		dst1->next = dst_prev;
1766 		dst_prev = dst1;
1767 
1768 		header_len += xfrm[i]->props.header_len;
1769 		if (xfrm[i]->type->flags & XFRM_TYPE_NON_FRAGMENT)
1770 			nfheader_len += xfrm[i]->props.header_len;
1771 		trailer_len += xfrm[i]->props.trailer_len;
1772 	}
1773 
1774 	dst_prev->child = dst;
1775 	dst0->path = dst;
1776 
1777 	err = -ENODEV;
1778 	dev = dst->dev;
1779 	if (!dev)
1780 		goto free_dst;
1781 
1782 	xfrm_init_path((struct xfrm_dst *)dst0, dst, nfheader_len);
1783 	xfrm_init_pmtu(dst_prev);
1784 
1785 	for (dst_prev = dst0; dst_prev != dst; dst_prev = dst_prev->child) {
1786 		struct xfrm_dst *xdst = (struct xfrm_dst *)dst_prev;
1787 
1788 		err = xfrm_fill_dst(xdst, dev, fl);
1789 		if (err)
1790 			goto free_dst;
1791 
1792 		dst_prev->header_len = header_len;
1793 		dst_prev->trailer_len = trailer_len;
1794 		header_len -= xdst->u.dst.xfrm->props.header_len;
1795 		trailer_len -= xdst->u.dst.xfrm->props.trailer_len;
1796 	}
1797 
1798 out:
1799 	return dst0;
1800 
1801 put_states:
1802 	for (; i < nx; i++)
1803 		xfrm_state_put(xfrm[i]);
1804 free_dst:
1805 	if (dst0)
1806 		dst_free(dst0);
1807 	dst0 = ERR_PTR(err);
1808 	goto out;
1809 }
1810 
1811 #ifdef CONFIG_XFRM_SUB_POLICY
1812 static int xfrm_dst_alloc_copy(void **target, const void *src, int size)
1813 {
1814 	if (!*target) {
1815 		*target = kmalloc(size, GFP_ATOMIC);
1816 		if (!*target)
1817 			return -ENOMEM;
1818 	}
1819 
1820 	memcpy(*target, src, size);
1821 	return 0;
1822 }
1823 #endif
1824 
1825 static int xfrm_dst_update_parent(struct dst_entry *dst,
1826 				  const struct xfrm_selector *sel)
1827 {
1828 #ifdef CONFIG_XFRM_SUB_POLICY
1829 	struct xfrm_dst *xdst = (struct xfrm_dst *)dst;
1830 	return xfrm_dst_alloc_copy((void **)&(xdst->partner),
1831 				   sel, sizeof(*sel));
1832 #else
1833 	return 0;
1834 #endif
1835 }
1836 
1837 static int xfrm_dst_update_origin(struct dst_entry *dst,
1838 				  const struct flowi *fl)
1839 {
1840 #ifdef CONFIG_XFRM_SUB_POLICY
1841 	struct xfrm_dst *xdst = (struct xfrm_dst *)dst;
1842 	return xfrm_dst_alloc_copy((void **)&(xdst->origin), fl, sizeof(*fl));
1843 #else
1844 	return 0;
1845 #endif
1846 }
1847 
1848 static int xfrm_expand_policies(const struct flowi *fl, u16 family,
1849 				struct xfrm_policy **pols,
1850 				int *num_pols, int *num_xfrms)
1851 {
1852 	int i;
1853 
1854 	if (*num_pols == 0 || !pols[0]) {
1855 		*num_pols = 0;
1856 		*num_xfrms = 0;
1857 		return 0;
1858 	}
1859 	if (IS_ERR(pols[0]))
1860 		return PTR_ERR(pols[0]);
1861 
1862 	*num_xfrms = pols[0]->xfrm_nr;
1863 
1864 #ifdef CONFIG_XFRM_SUB_POLICY
1865 	if (pols[0] && pols[0]->action == XFRM_POLICY_ALLOW &&
1866 	    pols[0]->type != XFRM_POLICY_TYPE_MAIN) {
1867 		pols[1] = xfrm_policy_lookup_bytype(xp_net(pols[0]),
1868 						    XFRM_POLICY_TYPE_MAIN,
1869 						    fl, family,
1870 						    XFRM_POLICY_OUT);
1871 		if (pols[1]) {
1872 			if (IS_ERR(pols[1])) {
1873 				xfrm_pols_put(pols, *num_pols);
1874 				return PTR_ERR(pols[1]);
1875 			}
1876 			(*num_pols)++;
1877 			(*num_xfrms) += pols[1]->xfrm_nr;
1878 		}
1879 	}
1880 #endif
1881 	for (i = 0; i < *num_pols; i++) {
1882 		if (pols[i]->action != XFRM_POLICY_ALLOW) {
1883 			*num_xfrms = -1;
1884 			break;
1885 		}
1886 	}
1887 
1888 	return 0;
1889 
1890 }
1891 
1892 static struct xfrm_dst *
1893 xfrm_resolve_and_create_bundle(struct xfrm_policy **pols, int num_pols,
1894 			       const struct flowi *fl, u16 family,
1895 			       struct dst_entry *dst_orig)
1896 {
1897 	struct net *net = xp_net(pols[0]);
1898 	struct xfrm_state *xfrm[XFRM_MAX_DEPTH];
1899 	struct dst_entry *dst;
1900 	struct xfrm_dst *xdst;
1901 	int err;
1902 
1903 	/* Try to instantiate a bundle */
1904 	err = xfrm_tmpl_resolve(pols, num_pols, fl, xfrm, family);
1905 	if (err <= 0) {
1906 		if (err != 0 && err != -EAGAIN)
1907 			XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTPOLERROR);
1908 		return ERR_PTR(err);
1909 	}
1910 
1911 	dst = xfrm_bundle_create(pols[0], xfrm, err, fl, dst_orig);
1912 	if (IS_ERR(dst)) {
1913 		XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTBUNDLEGENERROR);
1914 		return ERR_CAST(dst);
1915 	}
1916 
1917 	xdst = (struct xfrm_dst *)dst;
1918 	xdst->num_xfrms = err;
1919 	if (num_pols > 1)
1920 		err = xfrm_dst_update_parent(dst, &pols[1]->selector);
1921 	else
1922 		err = xfrm_dst_update_origin(dst, fl);
1923 	if (unlikely(err)) {
1924 		dst_free(dst);
1925 		XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTBUNDLECHECKERROR);
1926 		return ERR_PTR(err);
1927 	}
1928 
1929 	xdst->num_pols = num_pols;
1930 	memcpy(xdst->pols, pols, sizeof(struct xfrm_policy *) * num_pols);
1931 	xdst->policy_genid = atomic_read(&pols[0]->genid);
1932 
1933 	return xdst;
1934 }
1935 
1936 static void xfrm_policy_queue_process(unsigned long arg)
1937 {
1938 	struct sk_buff *skb;
1939 	struct sock *sk;
1940 	struct dst_entry *dst;
1941 	struct xfrm_policy *pol = (struct xfrm_policy *)arg;
1942 	struct net *net = xp_net(pol);
1943 	struct xfrm_policy_queue *pq = &pol->polq;
1944 	struct flowi fl;
1945 	struct sk_buff_head list;
1946 
1947 	spin_lock(&pq->hold_queue.lock);
1948 	skb = skb_peek(&pq->hold_queue);
1949 	if (!skb) {
1950 		spin_unlock(&pq->hold_queue.lock);
1951 		goto out;
1952 	}
1953 	dst = skb_dst(skb);
1954 	sk = skb->sk;
1955 	xfrm_decode_session(skb, &fl, dst->ops->family);
1956 	spin_unlock(&pq->hold_queue.lock);
1957 
1958 	dst_hold(dst->path);
1959 	dst = xfrm_lookup(net, dst->path, &fl, sk, 0);
1960 	if (IS_ERR(dst))
1961 		goto purge_queue;
1962 
1963 	if (dst->flags & DST_XFRM_QUEUE) {
1964 		dst_release(dst);
1965 
1966 		if (pq->timeout >= XFRM_QUEUE_TMO_MAX)
1967 			goto purge_queue;
1968 
1969 		pq->timeout = pq->timeout << 1;
1970 		if (!mod_timer(&pq->hold_timer, jiffies + pq->timeout))
1971 			xfrm_pol_hold(pol);
1972 	goto out;
1973 	}
1974 
1975 	dst_release(dst);
1976 
1977 	__skb_queue_head_init(&list);
1978 
1979 	spin_lock(&pq->hold_queue.lock);
1980 	pq->timeout = 0;
1981 	skb_queue_splice_init(&pq->hold_queue, &list);
1982 	spin_unlock(&pq->hold_queue.lock);
1983 
1984 	while (!skb_queue_empty(&list)) {
1985 		skb = __skb_dequeue(&list);
1986 
1987 		xfrm_decode_session(skb, &fl, skb_dst(skb)->ops->family);
1988 		dst_hold(skb_dst(skb)->path);
1989 		dst = xfrm_lookup(net, skb_dst(skb)->path, &fl, skb->sk, 0);
1990 		if (IS_ERR(dst)) {
1991 			kfree_skb(skb);
1992 			continue;
1993 		}
1994 
1995 		nf_reset(skb);
1996 		skb_dst_drop(skb);
1997 		skb_dst_set(skb, dst);
1998 
1999 		dst_output(net, skb->sk, skb);
2000 	}
2001 
2002 out:
2003 	xfrm_pol_put(pol);
2004 	return;
2005 
2006 purge_queue:
2007 	pq->timeout = 0;
2008 	skb_queue_purge(&pq->hold_queue);
2009 	xfrm_pol_put(pol);
2010 }
2011 
2012 static int xdst_queue_output(struct net *net, struct sock *sk, struct sk_buff *skb)
2013 {
2014 	unsigned long sched_next;
2015 	struct dst_entry *dst = skb_dst(skb);
2016 	struct xfrm_dst *xdst = (struct xfrm_dst *) dst;
2017 	struct xfrm_policy *pol = xdst->pols[0];
2018 	struct xfrm_policy_queue *pq = &pol->polq;
2019 
2020 	if (unlikely(skb_fclone_busy(sk, skb))) {
2021 		kfree_skb(skb);
2022 		return 0;
2023 	}
2024 
2025 	if (pq->hold_queue.qlen > XFRM_MAX_QUEUE_LEN) {
2026 		kfree_skb(skb);
2027 		return -EAGAIN;
2028 	}
2029 
2030 	skb_dst_force(skb);
2031 
2032 	spin_lock_bh(&pq->hold_queue.lock);
2033 
2034 	if (!pq->timeout)
2035 		pq->timeout = XFRM_QUEUE_TMO_MIN;
2036 
2037 	sched_next = jiffies + pq->timeout;
2038 
2039 	if (del_timer(&pq->hold_timer)) {
2040 		if (time_before(pq->hold_timer.expires, sched_next))
2041 			sched_next = pq->hold_timer.expires;
2042 		xfrm_pol_put(pol);
2043 	}
2044 
2045 	__skb_queue_tail(&pq->hold_queue, skb);
2046 	if (!mod_timer(&pq->hold_timer, sched_next))
2047 		xfrm_pol_hold(pol);
2048 
2049 	spin_unlock_bh(&pq->hold_queue.lock);
2050 
2051 	return 0;
2052 }
2053 
2054 static struct xfrm_dst *xfrm_create_dummy_bundle(struct net *net,
2055 						 struct xfrm_flo *xflo,
2056 						 const struct flowi *fl,
2057 						 int num_xfrms,
2058 						 u16 family)
2059 {
2060 	int err;
2061 	struct net_device *dev;
2062 	struct dst_entry *dst;
2063 	struct dst_entry *dst1;
2064 	struct xfrm_dst *xdst;
2065 
2066 	xdst = xfrm_alloc_dst(net, family);
2067 	if (IS_ERR(xdst))
2068 		return xdst;
2069 
2070 	if (!(xflo->flags & XFRM_LOOKUP_QUEUE) ||
2071 	    net->xfrm.sysctl_larval_drop ||
2072 	    num_xfrms <= 0)
2073 		return xdst;
2074 
2075 	dst = xflo->dst_orig;
2076 	dst1 = &xdst->u.dst;
2077 	dst_hold(dst);
2078 	xdst->route = dst;
2079 
2080 	dst_copy_metrics(dst1, dst);
2081 
2082 	dst1->obsolete = DST_OBSOLETE_FORCE_CHK;
2083 	dst1->flags |= DST_HOST | DST_XFRM_QUEUE;
2084 	dst1->lastuse = jiffies;
2085 
2086 	dst1->input = dst_discard;
2087 	dst1->output = xdst_queue_output;
2088 
2089 	dst_hold(dst);
2090 	dst1->child = dst;
2091 	dst1->path = dst;
2092 
2093 	xfrm_init_path((struct xfrm_dst *)dst1, dst, 0);
2094 
2095 	err = -ENODEV;
2096 	dev = dst->dev;
2097 	if (!dev)
2098 		goto free_dst;
2099 
2100 	err = xfrm_fill_dst(xdst, dev, fl);
2101 	if (err)
2102 		goto free_dst;
2103 
2104 out:
2105 	return xdst;
2106 
2107 free_dst:
2108 	dst_release(dst1);
2109 	xdst = ERR_PTR(err);
2110 	goto out;
2111 }
2112 
2113 static struct flow_cache_object *
2114 xfrm_bundle_lookup(struct net *net, const struct flowi *fl, u16 family, u8 dir,
2115 		   struct flow_cache_object *oldflo, void *ctx)
2116 {
2117 	struct xfrm_flo *xflo = (struct xfrm_flo *)ctx;
2118 	struct xfrm_policy *pols[XFRM_POLICY_TYPE_MAX];
2119 	struct xfrm_dst *xdst, *new_xdst;
2120 	int num_pols = 0, num_xfrms = 0, i, err, pol_dead;
2121 
2122 	/* Check if the policies from old bundle are usable */
2123 	xdst = NULL;
2124 	if (oldflo) {
2125 		xdst = container_of(oldflo, struct xfrm_dst, flo);
2126 		num_pols = xdst->num_pols;
2127 		num_xfrms = xdst->num_xfrms;
2128 		pol_dead = 0;
2129 		for (i = 0; i < num_pols; i++) {
2130 			pols[i] = xdst->pols[i];
2131 			pol_dead |= pols[i]->walk.dead;
2132 		}
2133 		if (pol_dead) {
2134 			dst_free(&xdst->u.dst);
2135 			xdst = NULL;
2136 			num_pols = 0;
2137 			num_xfrms = 0;
2138 			oldflo = NULL;
2139 		}
2140 	}
2141 
2142 	/* Resolve policies to use if we couldn't get them from
2143 	 * previous cache entry */
2144 	if (xdst == NULL) {
2145 		num_pols = 1;
2146 		pols[0] = __xfrm_policy_lookup(net, fl, family,
2147 					       flow_to_policy_dir(dir));
2148 		err = xfrm_expand_policies(fl, family, pols,
2149 					   &num_pols, &num_xfrms);
2150 		if (err < 0)
2151 			goto inc_error;
2152 		if (num_pols == 0)
2153 			return NULL;
2154 		if (num_xfrms <= 0)
2155 			goto make_dummy_bundle;
2156 	}
2157 
2158 	new_xdst = xfrm_resolve_and_create_bundle(pols, num_pols, fl, family,
2159 						  xflo->dst_orig);
2160 	if (IS_ERR(new_xdst)) {
2161 		err = PTR_ERR(new_xdst);
2162 		if (err != -EAGAIN)
2163 			goto error;
2164 		if (oldflo == NULL)
2165 			goto make_dummy_bundle;
2166 		dst_hold(&xdst->u.dst);
2167 		return oldflo;
2168 	} else if (new_xdst == NULL) {
2169 		num_xfrms = 0;
2170 		if (oldflo == NULL)
2171 			goto make_dummy_bundle;
2172 		xdst->num_xfrms = 0;
2173 		dst_hold(&xdst->u.dst);
2174 		return oldflo;
2175 	}
2176 
2177 	/* Kill the previous bundle */
2178 	if (xdst) {
2179 		/* The policies were stolen for newly generated bundle */
2180 		xdst->num_pols = 0;
2181 		dst_free(&xdst->u.dst);
2182 	}
2183 
2184 	/* Flow cache does not have reference, it dst_free()'s,
2185 	 * but we do need to return one reference for original caller */
2186 	dst_hold(&new_xdst->u.dst);
2187 	return &new_xdst->flo;
2188 
2189 make_dummy_bundle:
2190 	/* We found policies, but there's no bundles to instantiate:
2191 	 * either because the policy blocks, has no transformations or
2192 	 * we could not build template (no xfrm_states).*/
2193 	xdst = xfrm_create_dummy_bundle(net, xflo, fl, num_xfrms, family);
2194 	if (IS_ERR(xdst)) {
2195 		xfrm_pols_put(pols, num_pols);
2196 		return ERR_CAST(xdst);
2197 	}
2198 	xdst->num_pols = num_pols;
2199 	xdst->num_xfrms = num_xfrms;
2200 	memcpy(xdst->pols, pols, sizeof(struct xfrm_policy *) * num_pols);
2201 
2202 	dst_hold(&xdst->u.dst);
2203 	return &xdst->flo;
2204 
2205 inc_error:
2206 	XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTPOLERROR);
2207 error:
2208 	if (xdst != NULL)
2209 		dst_free(&xdst->u.dst);
2210 	else
2211 		xfrm_pols_put(pols, num_pols);
2212 	return ERR_PTR(err);
2213 }
2214 
2215 static struct dst_entry *make_blackhole(struct net *net, u16 family,
2216 					struct dst_entry *dst_orig)
2217 {
2218 	struct xfrm_policy_afinfo *afinfo = xfrm_policy_get_afinfo(family);
2219 	struct dst_entry *ret;
2220 
2221 	if (!afinfo) {
2222 		dst_release(dst_orig);
2223 		return ERR_PTR(-EINVAL);
2224 	} else {
2225 		ret = afinfo->blackhole_route(net, dst_orig);
2226 	}
2227 	xfrm_policy_put_afinfo(afinfo);
2228 
2229 	return ret;
2230 }
2231 
2232 /* Main function: finds/creates a bundle for given flow.
2233  *
2234  * At the moment we eat a raw IP route. Mostly to speed up lookups
2235  * on interfaces with disabled IPsec.
2236  */
2237 struct dst_entry *xfrm_lookup(struct net *net, struct dst_entry *dst_orig,
2238 			      const struct flowi *fl,
2239 			      const struct sock *sk, int flags)
2240 {
2241 	struct xfrm_policy *pols[XFRM_POLICY_TYPE_MAX];
2242 	struct flow_cache_object *flo;
2243 	struct xfrm_dst *xdst;
2244 	struct dst_entry *dst, *route;
2245 	u16 family = dst_orig->ops->family;
2246 	u8 dir = policy_to_flow_dir(XFRM_POLICY_OUT);
2247 	int i, err, num_pols, num_xfrms = 0, drop_pols = 0;
2248 
2249 	dst = NULL;
2250 	xdst = NULL;
2251 	route = NULL;
2252 
2253 	sk = sk_const_to_full_sk(sk);
2254 	if (sk && sk->sk_policy[XFRM_POLICY_OUT]) {
2255 		num_pols = 1;
2256 		pols[0] = xfrm_sk_policy_lookup(sk, XFRM_POLICY_OUT, fl);
2257 		err = xfrm_expand_policies(fl, family, pols,
2258 					   &num_pols, &num_xfrms);
2259 		if (err < 0)
2260 			goto dropdst;
2261 
2262 		if (num_pols) {
2263 			if (num_xfrms <= 0) {
2264 				drop_pols = num_pols;
2265 				goto no_transform;
2266 			}
2267 
2268 			xdst = xfrm_resolve_and_create_bundle(
2269 					pols, num_pols, fl,
2270 					family, dst_orig);
2271 			if (IS_ERR(xdst)) {
2272 				xfrm_pols_put(pols, num_pols);
2273 				err = PTR_ERR(xdst);
2274 				goto dropdst;
2275 			} else if (xdst == NULL) {
2276 				num_xfrms = 0;
2277 				drop_pols = num_pols;
2278 				goto no_transform;
2279 			}
2280 
2281 			dst_hold(&xdst->u.dst);
2282 			xdst->u.dst.flags |= DST_NOCACHE;
2283 			route = xdst->route;
2284 		}
2285 	}
2286 
2287 	if (xdst == NULL) {
2288 		struct xfrm_flo xflo;
2289 
2290 		xflo.dst_orig = dst_orig;
2291 		xflo.flags = flags;
2292 
2293 		/* To accelerate a bit...  */
2294 		if ((dst_orig->flags & DST_NOXFRM) ||
2295 		    !net->xfrm.policy_count[XFRM_POLICY_OUT])
2296 			goto nopol;
2297 
2298 		flo = flow_cache_lookup(net, fl, family, dir,
2299 					xfrm_bundle_lookup, &xflo);
2300 		if (flo == NULL)
2301 			goto nopol;
2302 		if (IS_ERR(flo)) {
2303 			err = PTR_ERR(flo);
2304 			goto dropdst;
2305 		}
2306 		xdst = container_of(flo, struct xfrm_dst, flo);
2307 
2308 		num_pols = xdst->num_pols;
2309 		num_xfrms = xdst->num_xfrms;
2310 		memcpy(pols, xdst->pols, sizeof(struct xfrm_policy *) * num_pols);
2311 		route = xdst->route;
2312 	}
2313 
2314 	dst = &xdst->u.dst;
2315 	if (route == NULL && num_xfrms > 0) {
2316 		/* The only case when xfrm_bundle_lookup() returns a
2317 		 * bundle with null route, is when the template could
2318 		 * not be resolved. It means policies are there, but
2319 		 * bundle could not be created, since we don't yet
2320 		 * have the xfrm_state's. We need to wait for KM to
2321 		 * negotiate new SA's or bail out with error.*/
2322 		if (net->xfrm.sysctl_larval_drop) {
2323 			XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTNOSTATES);
2324 			err = -EREMOTE;
2325 			goto error;
2326 		}
2327 
2328 		err = -EAGAIN;
2329 
2330 		XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTNOSTATES);
2331 		goto error;
2332 	}
2333 
2334 no_transform:
2335 	if (num_pols == 0)
2336 		goto nopol;
2337 
2338 	if ((flags & XFRM_LOOKUP_ICMP) &&
2339 	    !(pols[0]->flags & XFRM_POLICY_ICMP)) {
2340 		err = -ENOENT;
2341 		goto error;
2342 	}
2343 
2344 	for (i = 0; i < num_pols; i++)
2345 		pols[i]->curlft.use_time = get_seconds();
2346 
2347 	if (num_xfrms < 0) {
2348 		/* Prohibit the flow */
2349 		XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTPOLBLOCK);
2350 		err = -EPERM;
2351 		goto error;
2352 	} else if (num_xfrms > 0) {
2353 		/* Flow transformed */
2354 		dst_release(dst_orig);
2355 	} else {
2356 		/* Flow passes untransformed */
2357 		dst_release(dst);
2358 		dst = dst_orig;
2359 	}
2360 ok:
2361 	xfrm_pols_put(pols, drop_pols);
2362 	if (dst && dst->xfrm &&
2363 	    dst->xfrm->props.mode == XFRM_MODE_TUNNEL)
2364 		dst->flags |= DST_XFRM_TUNNEL;
2365 	return dst;
2366 
2367 nopol:
2368 	if (!(flags & XFRM_LOOKUP_ICMP)) {
2369 		dst = dst_orig;
2370 		goto ok;
2371 	}
2372 	err = -ENOENT;
2373 error:
2374 	dst_release(dst);
2375 dropdst:
2376 	if (!(flags & XFRM_LOOKUP_KEEP_DST_REF))
2377 		dst_release(dst_orig);
2378 	xfrm_pols_put(pols, drop_pols);
2379 	return ERR_PTR(err);
2380 }
2381 EXPORT_SYMBOL(xfrm_lookup);
2382 
2383 /* Callers of xfrm_lookup_route() must ensure a call to dst_output().
2384  * Otherwise we may send out blackholed packets.
2385  */
2386 struct dst_entry *xfrm_lookup_route(struct net *net, struct dst_entry *dst_orig,
2387 				    const struct flowi *fl,
2388 				    const struct sock *sk, int flags)
2389 {
2390 	struct dst_entry *dst = xfrm_lookup(net, dst_orig, fl, sk,
2391 					    flags | XFRM_LOOKUP_QUEUE |
2392 					    XFRM_LOOKUP_KEEP_DST_REF);
2393 
2394 	if (IS_ERR(dst) && PTR_ERR(dst) == -EREMOTE)
2395 		return make_blackhole(net, dst_orig->ops->family, dst_orig);
2396 
2397 	return dst;
2398 }
2399 EXPORT_SYMBOL(xfrm_lookup_route);
2400 
2401 static inline int
2402 xfrm_secpath_reject(int idx, struct sk_buff *skb, const struct flowi *fl)
2403 {
2404 	struct xfrm_state *x;
2405 
2406 	if (!skb->sp || idx < 0 || idx >= skb->sp->len)
2407 		return 0;
2408 	x = skb->sp->xvec[idx];
2409 	if (!x->type->reject)
2410 		return 0;
2411 	return x->type->reject(x, skb, fl);
2412 }
2413 
2414 /* When skb is transformed back to its "native" form, we have to
2415  * check policy restrictions. At the moment we make this in maximally
2416  * stupid way. Shame on me. :-) Of course, connected sockets must
2417  * have policy cached at them.
2418  */
2419 
2420 static inline int
2421 xfrm_state_ok(const struct xfrm_tmpl *tmpl, const struct xfrm_state *x,
2422 	      unsigned short family)
2423 {
2424 	if (xfrm_state_kern(x))
2425 		return tmpl->optional && !xfrm_state_addr_cmp(tmpl, x, tmpl->encap_family);
2426 	return	x->id.proto == tmpl->id.proto &&
2427 		(x->id.spi == tmpl->id.spi || !tmpl->id.spi) &&
2428 		(x->props.reqid == tmpl->reqid || !tmpl->reqid) &&
2429 		x->props.mode == tmpl->mode &&
2430 		(tmpl->allalgs || (tmpl->aalgos & (1<<x->props.aalgo)) ||
2431 		 !(xfrm_id_proto_match(tmpl->id.proto, IPSEC_PROTO_ANY))) &&
2432 		!(x->props.mode != XFRM_MODE_TRANSPORT &&
2433 		  xfrm_state_addr_cmp(tmpl, x, family));
2434 }
2435 
2436 /*
2437  * 0 or more than 0 is returned when validation is succeeded (either bypass
2438  * because of optional transport mode, or next index of the mathced secpath
2439  * state with the template.
2440  * -1 is returned when no matching template is found.
2441  * Otherwise "-2 - errored_index" is returned.
2442  */
2443 static inline int
2444 xfrm_policy_ok(const struct xfrm_tmpl *tmpl, const struct sec_path *sp, int start,
2445 	       unsigned short family)
2446 {
2447 	int idx = start;
2448 
2449 	if (tmpl->optional) {
2450 		if (tmpl->mode == XFRM_MODE_TRANSPORT)
2451 			return start;
2452 	} else
2453 		start = -1;
2454 	for (; idx < sp->len; idx++) {
2455 		if (xfrm_state_ok(tmpl, sp->xvec[idx], family))
2456 			return ++idx;
2457 		if (sp->xvec[idx]->props.mode != XFRM_MODE_TRANSPORT) {
2458 			if (start == -1)
2459 				start = -2-idx;
2460 			break;
2461 		}
2462 	}
2463 	return start;
2464 }
2465 
2466 int __xfrm_decode_session(struct sk_buff *skb, struct flowi *fl,
2467 			  unsigned int family, int reverse)
2468 {
2469 	struct xfrm_policy_afinfo *afinfo = xfrm_policy_get_afinfo(family);
2470 	int err;
2471 
2472 	if (unlikely(afinfo == NULL))
2473 		return -EAFNOSUPPORT;
2474 
2475 	afinfo->decode_session(skb, fl, reverse);
2476 	err = security_xfrm_decode_session(skb, &fl->flowi_secid);
2477 	xfrm_policy_put_afinfo(afinfo);
2478 	return err;
2479 }
2480 EXPORT_SYMBOL(__xfrm_decode_session);
2481 
2482 static inline int secpath_has_nontransport(const struct sec_path *sp, int k, int *idxp)
2483 {
2484 	for (; k < sp->len; k++) {
2485 		if (sp->xvec[k]->props.mode != XFRM_MODE_TRANSPORT) {
2486 			*idxp = k;
2487 			return 1;
2488 		}
2489 	}
2490 
2491 	return 0;
2492 }
2493 
2494 int __xfrm_policy_check(struct sock *sk, int dir, struct sk_buff *skb,
2495 			unsigned short family)
2496 {
2497 	struct net *net = dev_net(skb->dev);
2498 	struct xfrm_policy *pol;
2499 	struct xfrm_policy *pols[XFRM_POLICY_TYPE_MAX];
2500 	int npols = 0;
2501 	int xfrm_nr;
2502 	int pi;
2503 	int reverse;
2504 	struct flowi fl;
2505 	u8 fl_dir;
2506 	int xerr_idx = -1;
2507 
2508 	reverse = dir & ~XFRM_POLICY_MASK;
2509 	dir &= XFRM_POLICY_MASK;
2510 	fl_dir = policy_to_flow_dir(dir);
2511 
2512 	if (__xfrm_decode_session(skb, &fl, family, reverse) < 0) {
2513 		XFRM_INC_STATS(net, LINUX_MIB_XFRMINHDRERROR);
2514 		return 0;
2515 	}
2516 
2517 	nf_nat_decode_session(skb, &fl, family);
2518 
2519 	/* First, check used SA against their selectors. */
2520 	if (skb->sp) {
2521 		int i;
2522 
2523 		for (i = skb->sp->len-1; i >= 0; i--) {
2524 			struct xfrm_state *x = skb->sp->xvec[i];
2525 			if (!xfrm_selector_match(&x->sel, &fl, family)) {
2526 				XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEMISMATCH);
2527 				return 0;
2528 			}
2529 		}
2530 	}
2531 
2532 	pol = NULL;
2533 	sk = sk_to_full_sk(sk);
2534 	if (sk && sk->sk_policy[dir]) {
2535 		pol = xfrm_sk_policy_lookup(sk, dir, &fl);
2536 		if (IS_ERR(pol)) {
2537 			XFRM_INC_STATS(net, LINUX_MIB_XFRMINPOLERROR);
2538 			return 0;
2539 		}
2540 	}
2541 
2542 	if (!pol) {
2543 		struct flow_cache_object *flo;
2544 
2545 		flo = flow_cache_lookup(net, &fl, family, fl_dir,
2546 					xfrm_policy_lookup, NULL);
2547 		if (IS_ERR_OR_NULL(flo))
2548 			pol = ERR_CAST(flo);
2549 		else
2550 			pol = container_of(flo, struct xfrm_policy, flo);
2551 	}
2552 
2553 	if (IS_ERR(pol)) {
2554 		XFRM_INC_STATS(net, LINUX_MIB_XFRMINPOLERROR);
2555 		return 0;
2556 	}
2557 
2558 	if (!pol) {
2559 		if (skb->sp && secpath_has_nontransport(skb->sp, 0, &xerr_idx)) {
2560 			xfrm_secpath_reject(xerr_idx, skb, &fl);
2561 			XFRM_INC_STATS(net, LINUX_MIB_XFRMINNOPOLS);
2562 			return 0;
2563 		}
2564 		return 1;
2565 	}
2566 
2567 	pol->curlft.use_time = get_seconds();
2568 
2569 	pols[0] = pol;
2570 	npols++;
2571 #ifdef CONFIG_XFRM_SUB_POLICY
2572 	if (pols[0]->type != XFRM_POLICY_TYPE_MAIN) {
2573 		pols[1] = xfrm_policy_lookup_bytype(net, XFRM_POLICY_TYPE_MAIN,
2574 						    &fl, family,
2575 						    XFRM_POLICY_IN);
2576 		if (pols[1]) {
2577 			if (IS_ERR(pols[1])) {
2578 				XFRM_INC_STATS(net, LINUX_MIB_XFRMINPOLERROR);
2579 				return 0;
2580 			}
2581 			pols[1]->curlft.use_time = get_seconds();
2582 			npols++;
2583 		}
2584 	}
2585 #endif
2586 
2587 	if (pol->action == XFRM_POLICY_ALLOW) {
2588 		struct sec_path *sp;
2589 		static struct sec_path dummy;
2590 		struct xfrm_tmpl *tp[XFRM_MAX_DEPTH];
2591 		struct xfrm_tmpl *stp[XFRM_MAX_DEPTH];
2592 		struct xfrm_tmpl **tpp = tp;
2593 		int ti = 0;
2594 		int i, k;
2595 
2596 		if ((sp = skb->sp) == NULL)
2597 			sp = &dummy;
2598 
2599 		for (pi = 0; pi < npols; pi++) {
2600 			if (pols[pi] != pol &&
2601 			    pols[pi]->action != XFRM_POLICY_ALLOW) {
2602 				XFRM_INC_STATS(net, LINUX_MIB_XFRMINPOLBLOCK);
2603 				goto reject;
2604 			}
2605 			if (ti + pols[pi]->xfrm_nr >= XFRM_MAX_DEPTH) {
2606 				XFRM_INC_STATS(net, LINUX_MIB_XFRMINBUFFERERROR);
2607 				goto reject_error;
2608 			}
2609 			for (i = 0; i < pols[pi]->xfrm_nr; i++)
2610 				tpp[ti++] = &pols[pi]->xfrm_vec[i];
2611 		}
2612 		xfrm_nr = ti;
2613 		if (npols > 1) {
2614 			xfrm_tmpl_sort(stp, tpp, xfrm_nr, family, net);
2615 			tpp = stp;
2616 		}
2617 
2618 		/* For each tunnel xfrm, find the first matching tmpl.
2619 		 * For each tmpl before that, find corresponding xfrm.
2620 		 * Order is _important_. Later we will implement
2621 		 * some barriers, but at the moment barriers
2622 		 * are implied between each two transformations.
2623 		 */
2624 		for (i = xfrm_nr-1, k = 0; i >= 0; i--) {
2625 			k = xfrm_policy_ok(tpp[i], sp, k, family);
2626 			if (k < 0) {
2627 				if (k < -1)
2628 					/* "-2 - errored_index" returned */
2629 					xerr_idx = -(2+k);
2630 				XFRM_INC_STATS(net, LINUX_MIB_XFRMINTMPLMISMATCH);
2631 				goto reject;
2632 			}
2633 		}
2634 
2635 		if (secpath_has_nontransport(sp, k, &xerr_idx)) {
2636 			XFRM_INC_STATS(net, LINUX_MIB_XFRMINTMPLMISMATCH);
2637 			goto reject;
2638 		}
2639 
2640 		xfrm_pols_put(pols, npols);
2641 		return 1;
2642 	}
2643 	XFRM_INC_STATS(net, LINUX_MIB_XFRMINPOLBLOCK);
2644 
2645 reject:
2646 	xfrm_secpath_reject(xerr_idx, skb, &fl);
2647 reject_error:
2648 	xfrm_pols_put(pols, npols);
2649 	return 0;
2650 }
2651 EXPORT_SYMBOL(__xfrm_policy_check);
2652 
2653 int __xfrm_route_forward(struct sk_buff *skb, unsigned short family)
2654 {
2655 	struct net *net = dev_net(skb->dev);
2656 	struct flowi fl;
2657 	struct dst_entry *dst;
2658 	int res = 1;
2659 
2660 	if (xfrm_decode_session(skb, &fl, family) < 0) {
2661 		XFRM_INC_STATS(net, LINUX_MIB_XFRMFWDHDRERROR);
2662 		return 0;
2663 	}
2664 
2665 	skb_dst_force(skb);
2666 
2667 	dst = xfrm_lookup(net, skb_dst(skb), &fl, NULL, XFRM_LOOKUP_QUEUE);
2668 	if (IS_ERR(dst)) {
2669 		res = 0;
2670 		dst = NULL;
2671 	}
2672 	skb_dst_set(skb, dst);
2673 	return res;
2674 }
2675 EXPORT_SYMBOL(__xfrm_route_forward);
2676 
2677 /* Optimize later using cookies and generation ids. */
2678 
2679 static struct dst_entry *xfrm_dst_check(struct dst_entry *dst, u32 cookie)
2680 {
2681 	/* Code (such as __xfrm4_bundle_create()) sets dst->obsolete
2682 	 * to DST_OBSOLETE_FORCE_CHK to force all XFRM destinations to
2683 	 * get validated by dst_ops->check on every use.  We do this
2684 	 * because when a normal route referenced by an XFRM dst is
2685 	 * obsoleted we do not go looking around for all parent
2686 	 * referencing XFRM dsts so that we can invalidate them.  It
2687 	 * is just too much work.  Instead we make the checks here on
2688 	 * every use.  For example:
2689 	 *
2690 	 *	XFRM dst A --> IPv4 dst X
2691 	 *
2692 	 * X is the "xdst->route" of A (X is also the "dst->path" of A
2693 	 * in this example).  If X is marked obsolete, "A" will not
2694 	 * notice.  That's what we are validating here via the
2695 	 * stale_bundle() check.
2696 	 *
2697 	 * When a policy's bundle is pruned, we dst_free() the XFRM
2698 	 * dst which causes it's ->obsolete field to be set to
2699 	 * DST_OBSOLETE_DEAD.  If an XFRM dst has been pruned like
2700 	 * this, we want to force a new route lookup.
2701 	 */
2702 	if (dst->obsolete < 0 && !stale_bundle(dst))
2703 		return dst;
2704 
2705 	return NULL;
2706 }
2707 
2708 static int stale_bundle(struct dst_entry *dst)
2709 {
2710 	return !xfrm_bundle_ok((struct xfrm_dst *)dst);
2711 }
2712 
2713 void xfrm_dst_ifdown(struct dst_entry *dst, struct net_device *dev)
2714 {
2715 	while ((dst = dst->child) && dst->xfrm && dst->dev == dev) {
2716 		dst->dev = dev_net(dev)->loopback_dev;
2717 		dev_hold(dst->dev);
2718 		dev_put(dev);
2719 	}
2720 }
2721 EXPORT_SYMBOL(xfrm_dst_ifdown);
2722 
2723 static void xfrm_link_failure(struct sk_buff *skb)
2724 {
2725 	/* Impossible. Such dst must be popped before reaches point of failure. */
2726 }
2727 
2728 static struct dst_entry *xfrm_negative_advice(struct dst_entry *dst)
2729 {
2730 	if (dst) {
2731 		if (dst->obsolete) {
2732 			dst_release(dst);
2733 			dst = NULL;
2734 		}
2735 	}
2736 	return dst;
2737 }
2738 
2739 void xfrm_garbage_collect(struct net *net)
2740 {
2741 	flow_cache_flush(net);
2742 }
2743 EXPORT_SYMBOL(xfrm_garbage_collect);
2744 
2745 static void xfrm_garbage_collect_deferred(struct net *net)
2746 {
2747 	flow_cache_flush_deferred(net);
2748 }
2749 
2750 static void xfrm_init_pmtu(struct dst_entry *dst)
2751 {
2752 	do {
2753 		struct xfrm_dst *xdst = (struct xfrm_dst *)dst;
2754 		u32 pmtu, route_mtu_cached;
2755 
2756 		pmtu = dst_mtu(dst->child);
2757 		xdst->child_mtu_cached = pmtu;
2758 
2759 		pmtu = xfrm_state_mtu(dst->xfrm, pmtu);
2760 
2761 		route_mtu_cached = dst_mtu(xdst->route);
2762 		xdst->route_mtu_cached = route_mtu_cached;
2763 
2764 		if (pmtu > route_mtu_cached)
2765 			pmtu = route_mtu_cached;
2766 
2767 		dst_metric_set(dst, RTAX_MTU, pmtu);
2768 	} while ((dst = dst->next));
2769 }
2770 
2771 /* Check that the bundle accepts the flow and its components are
2772  * still valid.
2773  */
2774 
2775 static int xfrm_bundle_ok(struct xfrm_dst *first)
2776 {
2777 	struct dst_entry *dst = &first->u.dst;
2778 	struct xfrm_dst *last;
2779 	u32 mtu;
2780 
2781 	if (!dst_check(dst->path, ((struct xfrm_dst *)dst)->path_cookie) ||
2782 	    (dst->dev && !netif_running(dst->dev)))
2783 		return 0;
2784 
2785 	if (dst->flags & DST_XFRM_QUEUE)
2786 		return 1;
2787 
2788 	last = NULL;
2789 
2790 	do {
2791 		struct xfrm_dst *xdst = (struct xfrm_dst *)dst;
2792 
2793 		if (dst->xfrm->km.state != XFRM_STATE_VALID)
2794 			return 0;
2795 		if (xdst->xfrm_genid != dst->xfrm->genid)
2796 			return 0;
2797 		if (xdst->num_pols > 0 &&
2798 		    xdst->policy_genid != atomic_read(&xdst->pols[0]->genid))
2799 			return 0;
2800 
2801 		mtu = dst_mtu(dst->child);
2802 		if (xdst->child_mtu_cached != mtu) {
2803 			last = xdst;
2804 			xdst->child_mtu_cached = mtu;
2805 		}
2806 
2807 		if (!dst_check(xdst->route, xdst->route_cookie))
2808 			return 0;
2809 		mtu = dst_mtu(xdst->route);
2810 		if (xdst->route_mtu_cached != mtu) {
2811 			last = xdst;
2812 			xdst->route_mtu_cached = mtu;
2813 		}
2814 
2815 		dst = dst->child;
2816 	} while (dst->xfrm);
2817 
2818 	if (likely(!last))
2819 		return 1;
2820 
2821 	mtu = last->child_mtu_cached;
2822 	for (;;) {
2823 		dst = &last->u.dst;
2824 
2825 		mtu = xfrm_state_mtu(dst->xfrm, mtu);
2826 		if (mtu > last->route_mtu_cached)
2827 			mtu = last->route_mtu_cached;
2828 		dst_metric_set(dst, RTAX_MTU, mtu);
2829 
2830 		if (last == first)
2831 			break;
2832 
2833 		last = (struct xfrm_dst *)last->u.dst.next;
2834 		last->child_mtu_cached = mtu;
2835 	}
2836 
2837 	return 1;
2838 }
2839 
2840 static unsigned int xfrm_default_advmss(const struct dst_entry *dst)
2841 {
2842 	return dst_metric_advmss(dst->path);
2843 }
2844 
2845 static unsigned int xfrm_mtu(const struct dst_entry *dst)
2846 {
2847 	unsigned int mtu = dst_metric_raw(dst, RTAX_MTU);
2848 
2849 	return mtu ? : dst_mtu(dst->path);
2850 }
2851 
2852 static struct neighbour *xfrm_neigh_lookup(const struct dst_entry *dst,
2853 					   struct sk_buff *skb,
2854 					   const void *daddr)
2855 {
2856 	return dst->path->ops->neigh_lookup(dst, skb, daddr);
2857 }
2858 
2859 int xfrm_policy_register_afinfo(struct xfrm_policy_afinfo *afinfo)
2860 {
2861 	int err = 0;
2862 	if (unlikely(afinfo == NULL))
2863 		return -EINVAL;
2864 	if (unlikely(afinfo->family >= NPROTO))
2865 		return -EAFNOSUPPORT;
2866 	spin_lock(&xfrm_policy_afinfo_lock);
2867 	if (unlikely(xfrm_policy_afinfo[afinfo->family] != NULL))
2868 		err = -EEXIST;
2869 	else {
2870 		struct dst_ops *dst_ops = afinfo->dst_ops;
2871 		if (likely(dst_ops->kmem_cachep == NULL))
2872 			dst_ops->kmem_cachep = xfrm_dst_cache;
2873 		if (likely(dst_ops->check == NULL))
2874 			dst_ops->check = xfrm_dst_check;
2875 		if (likely(dst_ops->default_advmss == NULL))
2876 			dst_ops->default_advmss = xfrm_default_advmss;
2877 		if (likely(dst_ops->mtu == NULL))
2878 			dst_ops->mtu = xfrm_mtu;
2879 		if (likely(dst_ops->negative_advice == NULL))
2880 			dst_ops->negative_advice = xfrm_negative_advice;
2881 		if (likely(dst_ops->link_failure == NULL))
2882 			dst_ops->link_failure = xfrm_link_failure;
2883 		if (likely(dst_ops->neigh_lookup == NULL))
2884 			dst_ops->neigh_lookup = xfrm_neigh_lookup;
2885 		if (likely(afinfo->garbage_collect == NULL))
2886 			afinfo->garbage_collect = xfrm_garbage_collect_deferred;
2887 		rcu_assign_pointer(xfrm_policy_afinfo[afinfo->family], afinfo);
2888 	}
2889 	spin_unlock(&xfrm_policy_afinfo_lock);
2890 
2891 	return err;
2892 }
2893 EXPORT_SYMBOL(xfrm_policy_register_afinfo);
2894 
2895 int xfrm_policy_unregister_afinfo(struct xfrm_policy_afinfo *afinfo)
2896 {
2897 	int err = 0;
2898 	if (unlikely(afinfo == NULL))
2899 		return -EINVAL;
2900 	if (unlikely(afinfo->family >= NPROTO))
2901 		return -EAFNOSUPPORT;
2902 	spin_lock(&xfrm_policy_afinfo_lock);
2903 	if (likely(xfrm_policy_afinfo[afinfo->family] != NULL)) {
2904 		if (unlikely(xfrm_policy_afinfo[afinfo->family] != afinfo))
2905 			err = -EINVAL;
2906 		else
2907 			RCU_INIT_POINTER(xfrm_policy_afinfo[afinfo->family],
2908 					 NULL);
2909 	}
2910 	spin_unlock(&xfrm_policy_afinfo_lock);
2911 	if (!err) {
2912 		struct dst_ops *dst_ops = afinfo->dst_ops;
2913 
2914 		synchronize_rcu();
2915 
2916 		dst_ops->kmem_cachep = NULL;
2917 		dst_ops->check = NULL;
2918 		dst_ops->negative_advice = NULL;
2919 		dst_ops->link_failure = NULL;
2920 		afinfo->garbage_collect = NULL;
2921 	}
2922 	return err;
2923 }
2924 EXPORT_SYMBOL(xfrm_policy_unregister_afinfo);
2925 
2926 static int xfrm_dev_event(struct notifier_block *this, unsigned long event, void *ptr)
2927 {
2928 	struct net_device *dev = netdev_notifier_info_to_dev(ptr);
2929 
2930 	switch (event) {
2931 	case NETDEV_DOWN:
2932 		xfrm_garbage_collect(dev_net(dev));
2933 	}
2934 	return NOTIFY_DONE;
2935 }
2936 
2937 static struct notifier_block xfrm_dev_notifier = {
2938 	.notifier_call	= xfrm_dev_event,
2939 };
2940 
2941 #ifdef CONFIG_XFRM_STATISTICS
2942 static int __net_init xfrm_statistics_init(struct net *net)
2943 {
2944 	int rv;
2945 	net->mib.xfrm_statistics = alloc_percpu(struct linux_xfrm_mib);
2946 	if (!net->mib.xfrm_statistics)
2947 		return -ENOMEM;
2948 	rv = xfrm_proc_init(net);
2949 	if (rv < 0)
2950 		free_percpu(net->mib.xfrm_statistics);
2951 	return rv;
2952 }
2953 
2954 static void xfrm_statistics_fini(struct net *net)
2955 {
2956 	xfrm_proc_fini(net);
2957 	free_percpu(net->mib.xfrm_statistics);
2958 }
2959 #else
2960 static int __net_init xfrm_statistics_init(struct net *net)
2961 {
2962 	return 0;
2963 }
2964 
2965 static void xfrm_statistics_fini(struct net *net)
2966 {
2967 }
2968 #endif
2969 
2970 static int __net_init xfrm_policy_init(struct net *net)
2971 {
2972 	unsigned int hmask, sz;
2973 	int dir;
2974 
2975 	if (net_eq(net, &init_net))
2976 		xfrm_dst_cache = kmem_cache_create("xfrm_dst_cache",
2977 					   sizeof(struct xfrm_dst),
2978 					   0, SLAB_HWCACHE_ALIGN|SLAB_PANIC,
2979 					   NULL);
2980 
2981 	hmask = 8 - 1;
2982 	sz = (hmask+1) * sizeof(struct hlist_head);
2983 
2984 	net->xfrm.policy_byidx = xfrm_hash_alloc(sz);
2985 	if (!net->xfrm.policy_byidx)
2986 		goto out_byidx;
2987 	net->xfrm.policy_idx_hmask = hmask;
2988 
2989 	for (dir = 0; dir < XFRM_POLICY_MAX; dir++) {
2990 		struct xfrm_policy_hash *htab;
2991 
2992 		net->xfrm.policy_count[dir] = 0;
2993 		net->xfrm.policy_count[XFRM_POLICY_MAX + dir] = 0;
2994 		INIT_HLIST_HEAD(&net->xfrm.policy_inexact[dir]);
2995 
2996 		htab = &net->xfrm.policy_bydst[dir];
2997 		htab->table = xfrm_hash_alloc(sz);
2998 		if (!htab->table)
2999 			goto out_bydst;
3000 		htab->hmask = hmask;
3001 		htab->dbits4 = 32;
3002 		htab->sbits4 = 32;
3003 		htab->dbits6 = 128;
3004 		htab->sbits6 = 128;
3005 	}
3006 	net->xfrm.policy_hthresh.lbits4 = 32;
3007 	net->xfrm.policy_hthresh.rbits4 = 32;
3008 	net->xfrm.policy_hthresh.lbits6 = 128;
3009 	net->xfrm.policy_hthresh.rbits6 = 128;
3010 
3011 	seqlock_init(&net->xfrm.policy_hthresh.lock);
3012 
3013 	INIT_LIST_HEAD(&net->xfrm.policy_all);
3014 	INIT_WORK(&net->xfrm.policy_hash_work, xfrm_hash_resize);
3015 	INIT_WORK(&net->xfrm.policy_hthresh.work, xfrm_hash_rebuild);
3016 	if (net_eq(net, &init_net))
3017 		register_netdevice_notifier(&xfrm_dev_notifier);
3018 	return 0;
3019 
3020 out_bydst:
3021 	for (dir--; dir >= 0; dir--) {
3022 		struct xfrm_policy_hash *htab;
3023 
3024 		htab = &net->xfrm.policy_bydst[dir];
3025 		xfrm_hash_free(htab->table, sz);
3026 	}
3027 	xfrm_hash_free(net->xfrm.policy_byidx, sz);
3028 out_byidx:
3029 	return -ENOMEM;
3030 }
3031 
3032 static void xfrm_policy_fini(struct net *net)
3033 {
3034 	unsigned int sz;
3035 	int dir;
3036 
3037 	flush_work(&net->xfrm.policy_hash_work);
3038 #ifdef CONFIG_XFRM_SUB_POLICY
3039 	xfrm_policy_flush(net, XFRM_POLICY_TYPE_SUB, false);
3040 #endif
3041 	xfrm_policy_flush(net, XFRM_POLICY_TYPE_MAIN, false);
3042 
3043 	WARN_ON(!list_empty(&net->xfrm.policy_all));
3044 
3045 	for (dir = 0; dir < XFRM_POLICY_MAX; dir++) {
3046 		struct xfrm_policy_hash *htab;
3047 
3048 		WARN_ON(!hlist_empty(&net->xfrm.policy_inexact[dir]));
3049 
3050 		htab = &net->xfrm.policy_bydst[dir];
3051 		sz = (htab->hmask + 1) * sizeof(struct hlist_head);
3052 		WARN_ON(!hlist_empty(htab->table));
3053 		xfrm_hash_free(htab->table, sz);
3054 	}
3055 
3056 	sz = (net->xfrm.policy_idx_hmask + 1) * sizeof(struct hlist_head);
3057 	WARN_ON(!hlist_empty(net->xfrm.policy_byidx));
3058 	xfrm_hash_free(net->xfrm.policy_byidx, sz);
3059 }
3060 
3061 static int __net_init xfrm_net_init(struct net *net)
3062 {
3063 	int rv;
3064 
3065 	rv = xfrm_statistics_init(net);
3066 	if (rv < 0)
3067 		goto out_statistics;
3068 	rv = xfrm_state_init(net);
3069 	if (rv < 0)
3070 		goto out_state;
3071 	rv = xfrm_policy_init(net);
3072 	if (rv < 0)
3073 		goto out_policy;
3074 	rv = xfrm_sysctl_init(net);
3075 	if (rv < 0)
3076 		goto out_sysctl;
3077 	rv = flow_cache_init(net);
3078 	if (rv < 0)
3079 		goto out;
3080 
3081 	/* Initialize the per-net locks here */
3082 	spin_lock_init(&net->xfrm.xfrm_state_lock);
3083 	spin_lock_init(&net->xfrm.xfrm_policy_lock);
3084 	mutex_init(&net->xfrm.xfrm_cfg_mutex);
3085 
3086 	return 0;
3087 
3088 out:
3089 	xfrm_sysctl_fini(net);
3090 out_sysctl:
3091 	xfrm_policy_fini(net);
3092 out_policy:
3093 	xfrm_state_fini(net);
3094 out_state:
3095 	xfrm_statistics_fini(net);
3096 out_statistics:
3097 	return rv;
3098 }
3099 
3100 static void __net_exit xfrm_net_exit(struct net *net)
3101 {
3102 	flow_cache_fini(net);
3103 	xfrm_sysctl_fini(net);
3104 	xfrm_policy_fini(net);
3105 	xfrm_state_fini(net);
3106 	xfrm_statistics_fini(net);
3107 }
3108 
3109 static struct pernet_operations __net_initdata xfrm_net_ops = {
3110 	.init = xfrm_net_init,
3111 	.exit = xfrm_net_exit,
3112 };
3113 
3114 void __init xfrm_init(void)
3115 {
3116 	flow_cache_hp_init();
3117 	register_pernet_subsys(&xfrm_net_ops);
3118 	seqcount_init(&xfrm_policy_hash_generation);
3119 	xfrm_input_init();
3120 }
3121 
3122 #ifdef CONFIG_AUDITSYSCALL
3123 static void xfrm_audit_common_policyinfo(struct xfrm_policy *xp,
3124 					 struct audit_buffer *audit_buf)
3125 {
3126 	struct xfrm_sec_ctx *ctx = xp->security;
3127 	struct xfrm_selector *sel = &xp->selector;
3128 
3129 	if (ctx)
3130 		audit_log_format(audit_buf, " sec_alg=%u sec_doi=%u sec_obj=%s",
3131 				 ctx->ctx_alg, ctx->ctx_doi, ctx->ctx_str);
3132 
3133 	switch (sel->family) {
3134 	case AF_INET:
3135 		audit_log_format(audit_buf, " src=%pI4", &sel->saddr.a4);
3136 		if (sel->prefixlen_s != 32)
3137 			audit_log_format(audit_buf, " src_prefixlen=%d",
3138 					 sel->prefixlen_s);
3139 		audit_log_format(audit_buf, " dst=%pI4", &sel->daddr.a4);
3140 		if (sel->prefixlen_d != 32)
3141 			audit_log_format(audit_buf, " dst_prefixlen=%d",
3142 					 sel->prefixlen_d);
3143 		break;
3144 	case AF_INET6:
3145 		audit_log_format(audit_buf, " src=%pI6", sel->saddr.a6);
3146 		if (sel->prefixlen_s != 128)
3147 			audit_log_format(audit_buf, " src_prefixlen=%d",
3148 					 sel->prefixlen_s);
3149 		audit_log_format(audit_buf, " dst=%pI6", sel->daddr.a6);
3150 		if (sel->prefixlen_d != 128)
3151 			audit_log_format(audit_buf, " dst_prefixlen=%d",
3152 					 sel->prefixlen_d);
3153 		break;
3154 	}
3155 }
3156 
3157 void xfrm_audit_policy_add(struct xfrm_policy *xp, int result, bool task_valid)
3158 {
3159 	struct audit_buffer *audit_buf;
3160 
3161 	audit_buf = xfrm_audit_start("SPD-add");
3162 	if (audit_buf == NULL)
3163 		return;
3164 	xfrm_audit_helper_usrinfo(task_valid, audit_buf);
3165 	audit_log_format(audit_buf, " res=%u", result);
3166 	xfrm_audit_common_policyinfo(xp, audit_buf);
3167 	audit_log_end(audit_buf);
3168 }
3169 EXPORT_SYMBOL_GPL(xfrm_audit_policy_add);
3170 
3171 void xfrm_audit_policy_delete(struct xfrm_policy *xp, int result,
3172 			      bool task_valid)
3173 {
3174 	struct audit_buffer *audit_buf;
3175 
3176 	audit_buf = xfrm_audit_start("SPD-delete");
3177 	if (audit_buf == NULL)
3178 		return;
3179 	xfrm_audit_helper_usrinfo(task_valid, audit_buf);
3180 	audit_log_format(audit_buf, " res=%u", result);
3181 	xfrm_audit_common_policyinfo(xp, audit_buf);
3182 	audit_log_end(audit_buf);
3183 }
3184 EXPORT_SYMBOL_GPL(xfrm_audit_policy_delete);
3185 #endif
3186 
3187 #ifdef CONFIG_XFRM_MIGRATE
3188 static bool xfrm_migrate_selector_match(const struct xfrm_selector *sel_cmp,
3189 					const struct xfrm_selector *sel_tgt)
3190 {
3191 	if (sel_cmp->proto == IPSEC_ULPROTO_ANY) {
3192 		if (sel_tgt->family == sel_cmp->family &&
3193 		    xfrm_addr_equal(&sel_tgt->daddr, &sel_cmp->daddr,
3194 				    sel_cmp->family) &&
3195 		    xfrm_addr_equal(&sel_tgt->saddr, &sel_cmp->saddr,
3196 				    sel_cmp->family) &&
3197 		    sel_tgt->prefixlen_d == sel_cmp->prefixlen_d &&
3198 		    sel_tgt->prefixlen_s == sel_cmp->prefixlen_s) {
3199 			return true;
3200 		}
3201 	} else {
3202 		if (memcmp(sel_tgt, sel_cmp, sizeof(*sel_tgt)) == 0) {
3203 			return true;
3204 		}
3205 	}
3206 	return false;
3207 }
3208 
3209 static struct xfrm_policy *xfrm_migrate_policy_find(const struct xfrm_selector *sel,
3210 						    u8 dir, u8 type, struct net *net)
3211 {
3212 	struct xfrm_policy *pol, *ret = NULL;
3213 	struct hlist_head *chain;
3214 	u32 priority = ~0U;
3215 
3216 	spin_lock_bh(&net->xfrm.xfrm_policy_lock);
3217 	chain = policy_hash_direct(net, &sel->daddr, &sel->saddr, sel->family, dir);
3218 	hlist_for_each_entry(pol, chain, bydst) {
3219 		if (xfrm_migrate_selector_match(sel, &pol->selector) &&
3220 		    pol->type == type) {
3221 			ret = pol;
3222 			priority = ret->priority;
3223 			break;
3224 		}
3225 	}
3226 	chain = &net->xfrm.policy_inexact[dir];
3227 	hlist_for_each_entry(pol, chain, bydst) {
3228 		if ((pol->priority >= priority) && ret)
3229 			break;
3230 
3231 		if (xfrm_migrate_selector_match(sel, &pol->selector) &&
3232 		    pol->type == type) {
3233 			ret = pol;
3234 			break;
3235 		}
3236 	}
3237 
3238 	xfrm_pol_hold(ret);
3239 
3240 	spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
3241 
3242 	return ret;
3243 }
3244 
3245 static int migrate_tmpl_match(const struct xfrm_migrate *m, const struct xfrm_tmpl *t)
3246 {
3247 	int match = 0;
3248 
3249 	if (t->mode == m->mode && t->id.proto == m->proto &&
3250 	    (m->reqid == 0 || t->reqid == m->reqid)) {
3251 		switch (t->mode) {
3252 		case XFRM_MODE_TUNNEL:
3253 		case XFRM_MODE_BEET:
3254 			if (xfrm_addr_equal(&t->id.daddr, &m->old_daddr,
3255 					    m->old_family) &&
3256 			    xfrm_addr_equal(&t->saddr, &m->old_saddr,
3257 					    m->old_family)) {
3258 				match = 1;
3259 			}
3260 			break;
3261 		case XFRM_MODE_TRANSPORT:
3262 			/* in case of transport mode, template does not store
3263 			   any IP addresses, hence we just compare mode and
3264 			   protocol */
3265 			match = 1;
3266 			break;
3267 		default:
3268 			break;
3269 		}
3270 	}
3271 	return match;
3272 }
3273 
3274 /* update endpoint address(es) of template(s) */
3275 static int xfrm_policy_migrate(struct xfrm_policy *pol,
3276 			       struct xfrm_migrate *m, int num_migrate)
3277 {
3278 	struct xfrm_migrate *mp;
3279 	int i, j, n = 0;
3280 
3281 	write_lock_bh(&pol->lock);
3282 	if (unlikely(pol->walk.dead)) {
3283 		/* target policy has been deleted */
3284 		write_unlock_bh(&pol->lock);
3285 		return -ENOENT;
3286 	}
3287 
3288 	for (i = 0; i < pol->xfrm_nr; i++) {
3289 		for (j = 0, mp = m; j < num_migrate; j++, mp++) {
3290 			if (!migrate_tmpl_match(mp, &pol->xfrm_vec[i]))
3291 				continue;
3292 			n++;
3293 			if (pol->xfrm_vec[i].mode != XFRM_MODE_TUNNEL &&
3294 			    pol->xfrm_vec[i].mode != XFRM_MODE_BEET)
3295 				continue;
3296 			/* update endpoints */
3297 			memcpy(&pol->xfrm_vec[i].id.daddr, &mp->new_daddr,
3298 			       sizeof(pol->xfrm_vec[i].id.daddr));
3299 			memcpy(&pol->xfrm_vec[i].saddr, &mp->new_saddr,
3300 			       sizeof(pol->xfrm_vec[i].saddr));
3301 			pol->xfrm_vec[i].encap_family = mp->new_family;
3302 			/* flush bundles */
3303 			atomic_inc(&pol->genid);
3304 		}
3305 	}
3306 
3307 	write_unlock_bh(&pol->lock);
3308 
3309 	if (!n)
3310 		return -ENODATA;
3311 
3312 	return 0;
3313 }
3314 
3315 static int xfrm_migrate_check(const struct xfrm_migrate *m, int num_migrate)
3316 {
3317 	int i, j;
3318 
3319 	if (num_migrate < 1 || num_migrate > XFRM_MAX_DEPTH)
3320 		return -EINVAL;
3321 
3322 	for (i = 0; i < num_migrate; i++) {
3323 		if (xfrm_addr_equal(&m[i].old_daddr, &m[i].new_daddr,
3324 				    m[i].old_family) &&
3325 		    xfrm_addr_equal(&m[i].old_saddr, &m[i].new_saddr,
3326 				    m[i].old_family))
3327 			return -EINVAL;
3328 		if (xfrm_addr_any(&m[i].new_daddr, m[i].new_family) ||
3329 		    xfrm_addr_any(&m[i].new_saddr, m[i].new_family))
3330 			return -EINVAL;
3331 
3332 		/* check if there is any duplicated entry */
3333 		for (j = i + 1; j < num_migrate; j++) {
3334 			if (!memcmp(&m[i].old_daddr, &m[j].old_daddr,
3335 				    sizeof(m[i].old_daddr)) &&
3336 			    !memcmp(&m[i].old_saddr, &m[j].old_saddr,
3337 				    sizeof(m[i].old_saddr)) &&
3338 			    m[i].proto == m[j].proto &&
3339 			    m[i].mode == m[j].mode &&
3340 			    m[i].reqid == m[j].reqid &&
3341 			    m[i].old_family == m[j].old_family)
3342 				return -EINVAL;
3343 		}
3344 	}
3345 
3346 	return 0;
3347 }
3348 
3349 int xfrm_migrate(const struct xfrm_selector *sel, u8 dir, u8 type,
3350 		 struct xfrm_migrate *m, int num_migrate,
3351 		 struct xfrm_kmaddress *k, struct net *net)
3352 {
3353 	int i, err, nx_cur = 0, nx_new = 0;
3354 	struct xfrm_policy *pol = NULL;
3355 	struct xfrm_state *x, *xc;
3356 	struct xfrm_state *x_cur[XFRM_MAX_DEPTH];
3357 	struct xfrm_state *x_new[XFRM_MAX_DEPTH];
3358 	struct xfrm_migrate *mp;
3359 
3360 	if ((err = xfrm_migrate_check(m, num_migrate)) < 0)
3361 		goto out;
3362 
3363 	/* Stage 1 - find policy */
3364 	if ((pol = xfrm_migrate_policy_find(sel, dir, type, net)) == NULL) {
3365 		err = -ENOENT;
3366 		goto out;
3367 	}
3368 
3369 	/* Stage 2 - find and update state(s) */
3370 	for (i = 0, mp = m; i < num_migrate; i++, mp++) {
3371 		if ((x = xfrm_migrate_state_find(mp, net))) {
3372 			x_cur[nx_cur] = x;
3373 			nx_cur++;
3374 			if ((xc = xfrm_state_migrate(x, mp))) {
3375 				x_new[nx_new] = xc;
3376 				nx_new++;
3377 			} else {
3378 				err = -ENODATA;
3379 				goto restore_state;
3380 			}
3381 		}
3382 	}
3383 
3384 	/* Stage 3 - update policy */
3385 	if ((err = xfrm_policy_migrate(pol, m, num_migrate)) < 0)
3386 		goto restore_state;
3387 
3388 	/* Stage 4 - delete old state(s) */
3389 	if (nx_cur) {
3390 		xfrm_states_put(x_cur, nx_cur);
3391 		xfrm_states_delete(x_cur, nx_cur);
3392 	}
3393 
3394 	/* Stage 5 - announce */
3395 	km_migrate(sel, dir, type, m, num_migrate, k);
3396 
3397 	xfrm_pol_put(pol);
3398 
3399 	return 0;
3400 out:
3401 	return err;
3402 
3403 restore_state:
3404 	if (pol)
3405 		xfrm_pol_put(pol);
3406 	if (nx_cur)
3407 		xfrm_states_put(x_cur, nx_cur);
3408 	if (nx_new)
3409 		xfrm_states_delete(x_new, nx_new);
3410 
3411 	return err;
3412 }
3413 EXPORT_SYMBOL(xfrm_migrate);
3414 #endif
3415