1 // SPDX-License-Identifier: GPL-2.0
2 #include <linux/jhash.h>
3 #include <linux/netfilter.h>
4 #include <linux/rcupdate.h>
5 #include <linux/rhashtable.h>
6 #include <linux/vmalloc.h>
7 #include <net/genetlink.h>
8 #include <net/netns/generic.h>
9 #include <uapi/linux/genetlink.h>
10 #include "ila.h"
11
12 struct ila_xlat_params {
13 struct ila_params ip;
14 int ifindex;
15 };
16
17 struct ila_map {
18 struct ila_xlat_params xp;
19 struct rhash_head node;
20 struct ila_map __rcu *next;
21 struct rcu_head rcu;
22 };
23
24 #define MAX_LOCKS 1024
25 #define LOCKS_PER_CPU 10
26
alloc_ila_locks(struct ila_net * ilan)27 static int alloc_ila_locks(struct ila_net *ilan)
28 {
29 return alloc_bucket_spinlocks(&ilan->xlat.locks, &ilan->xlat.locks_mask,
30 MAX_LOCKS, LOCKS_PER_CPU,
31 GFP_KERNEL);
32 }
33
34 static u32 hashrnd __read_mostly;
__ila_hash_secret_init(void)35 static __always_inline void __ila_hash_secret_init(void)
36 {
37 net_get_random_once(&hashrnd, sizeof(hashrnd));
38 }
39
ila_locator_hash(struct ila_locator loc)40 static inline u32 ila_locator_hash(struct ila_locator loc)
41 {
42 u32 *v = (u32 *)loc.v32;
43
44 __ila_hash_secret_init();
45 return jhash_2words(v[0], v[1], hashrnd);
46 }
47
ila_get_lock(struct ila_net * ilan,struct ila_locator loc)48 static inline spinlock_t *ila_get_lock(struct ila_net *ilan,
49 struct ila_locator loc)
50 {
51 return &ilan->xlat.locks[ila_locator_hash(loc) & ilan->xlat.locks_mask];
52 }
53
ila_cmp_wildcards(struct ila_map * ila,struct ila_addr * iaddr,int ifindex)54 static inline int ila_cmp_wildcards(struct ila_map *ila,
55 struct ila_addr *iaddr, int ifindex)
56 {
57 return (ila->xp.ifindex && ila->xp.ifindex != ifindex);
58 }
59
ila_cmp_params(struct ila_map * ila,struct ila_xlat_params * xp)60 static inline int ila_cmp_params(struct ila_map *ila,
61 struct ila_xlat_params *xp)
62 {
63 return (ila->xp.ifindex != xp->ifindex);
64 }
65
ila_cmpfn(struct rhashtable_compare_arg * arg,const void * obj)66 static int ila_cmpfn(struct rhashtable_compare_arg *arg,
67 const void *obj)
68 {
69 const struct ila_map *ila = obj;
70
71 return (ila->xp.ip.locator_match.v64 != *(__be64 *)arg->key);
72 }
73
ila_order(struct ila_map * ila)74 static inline int ila_order(struct ila_map *ila)
75 {
76 int score = 0;
77
78 if (ila->xp.ifindex)
79 score += 1 << 1;
80
81 return score;
82 }
83
84 static const struct rhashtable_params rht_params = {
85 .nelem_hint = 1024,
86 .head_offset = offsetof(struct ila_map, node),
87 .key_offset = offsetof(struct ila_map, xp.ip.locator_match),
88 .key_len = sizeof(u64), /* identifier */
89 .max_size = 1048576,
90 .min_size = 256,
91 .automatic_shrinking = true,
92 .obj_cmpfn = ila_cmpfn,
93 };
94
parse_nl_config(struct genl_info * info,struct ila_xlat_params * xp)95 static int parse_nl_config(struct genl_info *info,
96 struct ila_xlat_params *xp)
97 {
98 memset(xp, 0, sizeof(*xp));
99
100 if (info->attrs[ILA_ATTR_LOCATOR])
101 xp->ip.locator.v64 = (__force __be64)nla_get_u64(
102 info->attrs[ILA_ATTR_LOCATOR]);
103
104 if (info->attrs[ILA_ATTR_LOCATOR_MATCH])
105 xp->ip.locator_match.v64 = (__force __be64)nla_get_u64(
106 info->attrs[ILA_ATTR_LOCATOR_MATCH]);
107
108 if (info->attrs[ILA_ATTR_CSUM_MODE])
109 xp->ip.csum_mode = nla_get_u8(info->attrs[ILA_ATTR_CSUM_MODE]);
110 else
111 xp->ip.csum_mode = ILA_CSUM_NO_ACTION;
112
113 if (info->attrs[ILA_ATTR_IDENT_TYPE])
114 xp->ip.ident_type = nla_get_u8(
115 info->attrs[ILA_ATTR_IDENT_TYPE]);
116 else
117 xp->ip.ident_type = ILA_ATYPE_USE_FORMAT;
118
119 if (info->attrs[ILA_ATTR_IFINDEX])
120 xp->ifindex = nla_get_s32(info->attrs[ILA_ATTR_IFINDEX]);
121
122 return 0;
123 }
124
125 /* Must be called with rcu readlock */
ila_lookup_wildcards(struct ila_addr * iaddr,int ifindex,struct ila_net * ilan)126 static inline struct ila_map *ila_lookup_wildcards(struct ila_addr *iaddr,
127 int ifindex,
128 struct ila_net *ilan)
129 {
130 struct ila_map *ila;
131
132 ila = rhashtable_lookup_fast(&ilan->xlat.rhash_table, &iaddr->loc,
133 rht_params);
134 while (ila) {
135 if (!ila_cmp_wildcards(ila, iaddr, ifindex))
136 return ila;
137 ila = rcu_access_pointer(ila->next);
138 }
139
140 return NULL;
141 }
142
143 /* Must be called with rcu readlock */
ila_lookup_by_params(struct ila_xlat_params * xp,struct ila_net * ilan)144 static inline struct ila_map *ila_lookup_by_params(struct ila_xlat_params *xp,
145 struct ila_net *ilan)
146 {
147 struct ila_map *ila;
148
149 ila = rhashtable_lookup_fast(&ilan->xlat.rhash_table,
150 &xp->ip.locator_match,
151 rht_params);
152 while (ila) {
153 if (!ila_cmp_params(ila, xp))
154 return ila;
155 ila = rcu_access_pointer(ila->next);
156 }
157
158 return NULL;
159 }
160
ila_release(struct ila_map * ila)161 static inline void ila_release(struct ila_map *ila)
162 {
163 kfree_rcu(ila, rcu);
164 }
165
ila_free_node(struct ila_map * ila)166 static void ila_free_node(struct ila_map *ila)
167 {
168 struct ila_map *next;
169
170 /* Assume rcu_readlock held */
171 while (ila) {
172 next = rcu_access_pointer(ila->next);
173 ila_release(ila);
174 ila = next;
175 }
176 }
177
ila_free_cb(void * ptr,void * arg)178 static void ila_free_cb(void *ptr, void *arg)
179 {
180 ila_free_node((struct ila_map *)ptr);
181 }
182
183 static int ila_xlat_addr(struct sk_buff *skb, bool sir2ila);
184
185 static unsigned int
ila_nf_input(void * priv,struct sk_buff * skb,const struct nf_hook_state * state)186 ila_nf_input(void *priv,
187 struct sk_buff *skb,
188 const struct nf_hook_state *state)
189 {
190 ila_xlat_addr(skb, false);
191 return NF_ACCEPT;
192 }
193
194 static const struct nf_hook_ops ila_nf_hook_ops[] = {
195 {
196 .hook = ila_nf_input,
197 .pf = NFPROTO_IPV6,
198 .hooknum = NF_INET_PRE_ROUTING,
199 .priority = -1,
200 },
201 };
202
203 static DEFINE_MUTEX(ila_mutex);
204
ila_add_mapping(struct net * net,struct ila_xlat_params * xp)205 static int ila_add_mapping(struct net *net, struct ila_xlat_params *xp)
206 {
207 struct ila_net *ilan = net_generic(net, ila_net_id);
208 struct ila_map *ila, *head;
209 spinlock_t *lock = ila_get_lock(ilan, xp->ip.locator_match);
210 int err = 0, order;
211
212 if (!READ_ONCE(ilan->xlat.hooks_registered)) {
213 /* We defer registering net hooks in the namespace until the
214 * first mapping is added.
215 */
216 mutex_lock(&ila_mutex);
217 if (!ilan->xlat.hooks_registered) {
218 err = nf_register_net_hooks(net, ila_nf_hook_ops,
219 ARRAY_SIZE(ila_nf_hook_ops));
220 if (!err)
221 WRITE_ONCE(ilan->xlat.hooks_registered, true);
222 }
223 mutex_unlock(&ila_mutex);
224 if (err)
225 return err;
226 }
227
228 ila = kzalloc(sizeof(*ila), GFP_KERNEL);
229 if (!ila)
230 return -ENOMEM;
231
232 ila_init_saved_csum(&xp->ip);
233
234 ila->xp = *xp;
235
236 order = ila_order(ila);
237
238 spin_lock(lock);
239
240 head = rhashtable_lookup_fast(&ilan->xlat.rhash_table,
241 &xp->ip.locator_match,
242 rht_params);
243 if (!head) {
244 /* New entry for the rhash_table */
245 err = rhashtable_lookup_insert_fast(&ilan->xlat.rhash_table,
246 &ila->node, rht_params);
247 } else {
248 struct ila_map *tila = head, *prev = NULL;
249
250 do {
251 if (!ila_cmp_params(tila, xp)) {
252 err = -EEXIST;
253 goto out;
254 }
255
256 if (order > ila_order(tila))
257 break;
258
259 prev = tila;
260 tila = rcu_dereference_protected(tila->next,
261 lockdep_is_held(lock));
262 } while (tila);
263
264 if (prev) {
265 /* Insert in sub list of head */
266 RCU_INIT_POINTER(ila->next, tila);
267 rcu_assign_pointer(prev->next, ila);
268 } else {
269 /* Make this ila new head */
270 RCU_INIT_POINTER(ila->next, head);
271 err = rhashtable_replace_fast(&ilan->xlat.rhash_table,
272 &head->node,
273 &ila->node, rht_params);
274 if (err)
275 goto out;
276 }
277 }
278
279 out:
280 spin_unlock(lock);
281
282 if (err)
283 kfree(ila);
284
285 return err;
286 }
287
ila_del_mapping(struct net * net,struct ila_xlat_params * xp)288 static int ila_del_mapping(struct net *net, struct ila_xlat_params *xp)
289 {
290 struct ila_net *ilan = net_generic(net, ila_net_id);
291 struct ila_map *ila, *head, *prev;
292 spinlock_t *lock = ila_get_lock(ilan, xp->ip.locator_match);
293 int err = -ENOENT;
294
295 spin_lock(lock);
296
297 head = rhashtable_lookup_fast(&ilan->xlat.rhash_table,
298 &xp->ip.locator_match, rht_params);
299 ila = head;
300
301 prev = NULL;
302
303 while (ila) {
304 if (ila_cmp_params(ila, xp)) {
305 prev = ila;
306 ila = rcu_dereference_protected(ila->next,
307 lockdep_is_held(lock));
308 continue;
309 }
310
311 err = 0;
312
313 if (prev) {
314 /* Not head, just delete from list */
315 rcu_assign_pointer(prev->next, ila->next);
316 } else {
317 /* It is the head. If there is something in the
318 * sublist we need to make a new head.
319 */
320 head = rcu_dereference_protected(ila->next,
321 lockdep_is_held(lock));
322 if (head) {
323 /* Put first entry in the sublist into the
324 * table
325 */
326 err = rhashtable_replace_fast(
327 &ilan->xlat.rhash_table, &ila->node,
328 &head->node, rht_params);
329 if (err)
330 goto out;
331 } else {
332 /* Entry no longer used */
333 err = rhashtable_remove_fast(
334 &ilan->xlat.rhash_table,
335 &ila->node, rht_params);
336 }
337 }
338
339 ila_release(ila);
340
341 break;
342 }
343
344 out:
345 spin_unlock(lock);
346
347 return err;
348 }
349
ila_xlat_nl_cmd_add_mapping(struct sk_buff * skb,struct genl_info * info)350 int ila_xlat_nl_cmd_add_mapping(struct sk_buff *skb, struct genl_info *info)
351 {
352 struct net *net = genl_info_net(info);
353 struct ila_xlat_params p;
354 int err;
355
356 err = parse_nl_config(info, &p);
357 if (err)
358 return err;
359
360 return ila_add_mapping(net, &p);
361 }
362
ila_xlat_nl_cmd_del_mapping(struct sk_buff * skb,struct genl_info * info)363 int ila_xlat_nl_cmd_del_mapping(struct sk_buff *skb, struct genl_info *info)
364 {
365 struct net *net = genl_info_net(info);
366 struct ila_xlat_params xp;
367 int err;
368
369 err = parse_nl_config(info, &xp);
370 if (err)
371 return err;
372
373 ila_del_mapping(net, &xp);
374
375 return 0;
376 }
377
lock_from_ila_map(struct ila_net * ilan,struct ila_map * ila)378 static inline spinlock_t *lock_from_ila_map(struct ila_net *ilan,
379 struct ila_map *ila)
380 {
381 return ila_get_lock(ilan, ila->xp.ip.locator_match);
382 }
383
ila_xlat_nl_cmd_flush(struct sk_buff * skb,struct genl_info * info)384 int ila_xlat_nl_cmd_flush(struct sk_buff *skb, struct genl_info *info)
385 {
386 struct net *net = genl_info_net(info);
387 struct ila_net *ilan = net_generic(net, ila_net_id);
388 struct rhashtable_iter iter;
389 struct ila_map *ila;
390 spinlock_t *lock;
391 int ret = 0;
392
393 rhashtable_walk_enter(&ilan->xlat.rhash_table, &iter);
394 rhashtable_walk_start(&iter);
395
396 for (;;) {
397 ila = rhashtable_walk_next(&iter);
398
399 if (IS_ERR(ila)) {
400 if (PTR_ERR(ila) == -EAGAIN)
401 continue;
402 ret = PTR_ERR(ila);
403 goto done;
404 } else if (!ila) {
405 break;
406 }
407
408 lock = lock_from_ila_map(ilan, ila);
409
410 spin_lock(lock);
411
412 ret = rhashtable_remove_fast(&ilan->xlat.rhash_table,
413 &ila->node, rht_params);
414 if (!ret)
415 ila_free_node(ila);
416
417 spin_unlock(lock);
418
419 if (ret)
420 break;
421 }
422
423 done:
424 rhashtable_walk_stop(&iter);
425 rhashtable_walk_exit(&iter);
426 return ret;
427 }
428
ila_fill_info(struct ila_map * ila,struct sk_buff * msg)429 static int ila_fill_info(struct ila_map *ila, struct sk_buff *msg)
430 {
431 if (nla_put_u64_64bit(msg, ILA_ATTR_LOCATOR,
432 (__force u64)ila->xp.ip.locator.v64,
433 ILA_ATTR_PAD) ||
434 nla_put_u64_64bit(msg, ILA_ATTR_LOCATOR_MATCH,
435 (__force u64)ila->xp.ip.locator_match.v64,
436 ILA_ATTR_PAD) ||
437 nla_put_s32(msg, ILA_ATTR_IFINDEX, ila->xp.ifindex) ||
438 nla_put_u8(msg, ILA_ATTR_CSUM_MODE, ila->xp.ip.csum_mode) ||
439 nla_put_u8(msg, ILA_ATTR_IDENT_TYPE, ila->xp.ip.ident_type))
440 return -1;
441
442 return 0;
443 }
444
ila_dump_info(struct ila_map * ila,u32 portid,u32 seq,u32 flags,struct sk_buff * skb,u8 cmd)445 static int ila_dump_info(struct ila_map *ila,
446 u32 portid, u32 seq, u32 flags,
447 struct sk_buff *skb, u8 cmd)
448 {
449 void *hdr;
450
451 hdr = genlmsg_put(skb, portid, seq, &ila_nl_family, flags, cmd);
452 if (!hdr)
453 return -ENOMEM;
454
455 if (ila_fill_info(ila, skb) < 0)
456 goto nla_put_failure;
457
458 genlmsg_end(skb, hdr);
459 return 0;
460
461 nla_put_failure:
462 genlmsg_cancel(skb, hdr);
463 return -EMSGSIZE;
464 }
465
ila_xlat_nl_cmd_get_mapping(struct sk_buff * skb,struct genl_info * info)466 int ila_xlat_nl_cmd_get_mapping(struct sk_buff *skb, struct genl_info *info)
467 {
468 struct net *net = genl_info_net(info);
469 struct ila_net *ilan = net_generic(net, ila_net_id);
470 struct sk_buff *msg;
471 struct ila_xlat_params xp;
472 struct ila_map *ila;
473 int ret;
474
475 ret = parse_nl_config(info, &xp);
476 if (ret)
477 return ret;
478
479 msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
480 if (!msg)
481 return -ENOMEM;
482
483 rcu_read_lock();
484
485 ret = -ESRCH;
486 ila = ila_lookup_by_params(&xp, ilan);
487 if (ila) {
488 ret = ila_dump_info(ila,
489 info->snd_portid,
490 info->snd_seq, 0, msg,
491 info->genlhdr->cmd);
492 }
493
494 rcu_read_unlock();
495
496 if (ret < 0)
497 goto out_free;
498
499 return genlmsg_reply(msg, info);
500
501 out_free:
502 nlmsg_free(msg);
503 return ret;
504 }
505
506 struct ila_dump_iter {
507 struct rhashtable_iter rhiter;
508 int skip;
509 };
510
ila_xlat_nl_dump_start(struct netlink_callback * cb)511 int ila_xlat_nl_dump_start(struct netlink_callback *cb)
512 {
513 struct net *net = sock_net(cb->skb->sk);
514 struct ila_net *ilan = net_generic(net, ila_net_id);
515 struct ila_dump_iter *iter;
516
517 iter = kmalloc(sizeof(*iter), GFP_KERNEL);
518 if (!iter)
519 return -ENOMEM;
520
521 rhashtable_walk_enter(&ilan->xlat.rhash_table, &iter->rhiter);
522
523 iter->skip = 0;
524 cb->args[0] = (long)iter;
525
526 return 0;
527 }
528
ila_xlat_nl_dump_done(struct netlink_callback * cb)529 int ila_xlat_nl_dump_done(struct netlink_callback *cb)
530 {
531 struct ila_dump_iter *iter = (struct ila_dump_iter *)cb->args[0];
532
533 rhashtable_walk_exit(&iter->rhiter);
534
535 kfree(iter);
536
537 return 0;
538 }
539
ila_xlat_nl_dump(struct sk_buff * skb,struct netlink_callback * cb)540 int ila_xlat_nl_dump(struct sk_buff *skb, struct netlink_callback *cb)
541 {
542 struct ila_dump_iter *iter = (struct ila_dump_iter *)cb->args[0];
543 struct rhashtable_iter *rhiter = &iter->rhiter;
544 int skip = iter->skip;
545 struct ila_map *ila;
546 int ret;
547
548 rhashtable_walk_start(rhiter);
549
550 /* Get first entry */
551 ila = rhashtable_walk_peek(rhiter);
552
553 if (ila && !IS_ERR(ila) && skip) {
554 /* Skip over visited entries */
555
556 while (ila && skip) {
557 /* Skip over any ila entries in this list that we
558 * have already dumped.
559 */
560 ila = rcu_access_pointer(ila->next);
561 skip--;
562 }
563 }
564
565 skip = 0;
566
567 for (;;) {
568 if (IS_ERR(ila)) {
569 ret = PTR_ERR(ila);
570 if (ret == -EAGAIN) {
571 /* Table has changed and iter has reset. Return
572 * -EAGAIN to the application even if we have
573 * written data to the skb. The application
574 * needs to deal with this.
575 */
576
577 goto out_ret;
578 } else {
579 break;
580 }
581 } else if (!ila) {
582 ret = 0;
583 break;
584 }
585
586 while (ila) {
587 ret = ila_dump_info(ila, NETLINK_CB(cb->skb).portid,
588 cb->nlh->nlmsg_seq, NLM_F_MULTI,
589 skb, ILA_CMD_GET);
590 if (ret)
591 goto out;
592
593 skip++;
594 ila = rcu_access_pointer(ila->next);
595 }
596
597 skip = 0;
598 ila = rhashtable_walk_next(rhiter);
599 }
600
601 out:
602 iter->skip = skip;
603 ret = (skb->len ? : ret);
604
605 out_ret:
606 rhashtable_walk_stop(rhiter);
607 return ret;
608 }
609
ila_xlat_init_net(struct net * net)610 int ila_xlat_init_net(struct net *net)
611 {
612 struct ila_net *ilan = net_generic(net, ila_net_id);
613 int err;
614
615 err = alloc_ila_locks(ilan);
616 if (err)
617 return err;
618
619 err = rhashtable_init(&ilan->xlat.rhash_table, &rht_params);
620 if (err) {
621 free_bucket_spinlocks(ilan->xlat.locks);
622 return err;
623 }
624
625 return 0;
626 }
627
ila_xlat_pre_exit_net(struct net * net)628 void ila_xlat_pre_exit_net(struct net *net)
629 {
630 struct ila_net *ilan = net_generic(net, ila_net_id);
631
632 if (ilan->xlat.hooks_registered)
633 nf_unregister_net_hooks(net, ila_nf_hook_ops,
634 ARRAY_SIZE(ila_nf_hook_ops));
635 }
636
ila_xlat_exit_net(struct net * net)637 void ila_xlat_exit_net(struct net *net)
638 {
639 struct ila_net *ilan = net_generic(net, ila_net_id);
640
641 rhashtable_free_and_destroy(&ilan->xlat.rhash_table, ila_free_cb, NULL);
642
643 free_bucket_spinlocks(ilan->xlat.locks);
644 }
645
ila_xlat_addr(struct sk_buff * skb,bool sir2ila)646 static int ila_xlat_addr(struct sk_buff *skb, bool sir2ila)
647 {
648 struct ila_map *ila;
649 struct ipv6hdr *ip6h = ipv6_hdr(skb);
650 struct net *net = dev_net(skb->dev);
651 struct ila_net *ilan = net_generic(net, ila_net_id);
652 struct ila_addr *iaddr = ila_a2i(&ip6h->daddr);
653
654 /* Assumes skb contains a valid IPv6 header that is pulled */
655
656 /* No check here that ILA type in the mapping matches what is in the
657 * address. We assume that whatever sender gaves us can be translated.
658 * The checksum mode however is relevant.
659 */
660
661 rcu_read_lock();
662
663 ila = ila_lookup_wildcards(iaddr, skb->dev->ifindex, ilan);
664 if (ila)
665 ila_update_ipv6_locator(skb, &ila->xp.ip, sir2ila);
666
667 rcu_read_unlock();
668
669 return 0;
670 }
671