1 // SPDX-License-Identifier: GPL-2.0 2 #include <linux/jhash.h> 3 #include <linux/netfilter.h> 4 #include <linux/rcupdate.h> 5 #include <linux/rhashtable.h> 6 #include <linux/vmalloc.h> 7 #include <net/genetlink.h> 8 #include <net/ila.h> 9 #include <net/netns/generic.h> 10 #include <uapi/linux/genetlink.h> 11 #include "ila.h" 12 13 struct ila_xlat_params { 14 struct ila_params ip; 15 int ifindex; 16 }; 17 18 struct ila_map { 19 struct ila_xlat_params xp; 20 struct rhash_head node; 21 struct ila_map __rcu *next; 22 struct rcu_head rcu; 23 }; 24 25 static unsigned int ila_net_id; 26 27 struct ila_net { 28 struct rhashtable rhash_table; 29 spinlock_t *locks; /* Bucket locks for entry manipulation */ 30 unsigned int locks_mask; 31 bool hooks_registered; 32 }; 33 34 #define LOCKS_PER_CPU 10 35 36 static int alloc_ila_locks(struct ila_net *ilan) 37 { 38 unsigned int i, size; 39 unsigned int nr_pcpus = num_possible_cpus(); 40 41 nr_pcpus = min_t(unsigned int, nr_pcpus, 32UL); 42 size = roundup_pow_of_two(nr_pcpus * LOCKS_PER_CPU); 43 44 if (sizeof(spinlock_t) != 0) { 45 ilan->locks = kvmalloc(size * sizeof(spinlock_t), GFP_KERNEL); 46 if (!ilan->locks) 47 return -ENOMEM; 48 for (i = 0; i < size; i++) 49 spin_lock_init(&ilan->locks[i]); 50 } 51 ilan->locks_mask = size - 1; 52 53 return 0; 54 } 55 56 static u32 hashrnd __read_mostly; 57 static __always_inline void __ila_hash_secret_init(void) 58 { 59 net_get_random_once(&hashrnd, sizeof(hashrnd)); 60 } 61 62 static inline u32 ila_locator_hash(struct ila_locator loc) 63 { 64 u32 *v = (u32 *)loc.v32; 65 66 __ila_hash_secret_init(); 67 return jhash_2words(v[0], v[1], hashrnd); 68 } 69 70 static inline spinlock_t *ila_get_lock(struct ila_net *ilan, 71 struct ila_locator loc) 72 { 73 return &ilan->locks[ila_locator_hash(loc) & ilan->locks_mask]; 74 } 75 76 static inline int ila_cmp_wildcards(struct ila_map *ila, 77 struct ila_addr *iaddr, int ifindex) 78 { 79 return (ila->xp.ifindex && ila->xp.ifindex != ifindex); 80 } 81 82 static inline int ila_cmp_params(struct ila_map *ila, 83 struct ila_xlat_params *xp) 84 { 85 return (ila->xp.ifindex != xp->ifindex); 86 } 87 88 static int ila_cmpfn(struct rhashtable_compare_arg *arg, 89 const void *obj) 90 { 91 const struct ila_map *ila = obj; 92 93 return (ila->xp.ip.locator_match.v64 != *(__be64 *)arg->key); 94 } 95 96 static inline int ila_order(struct ila_map *ila) 97 { 98 int score = 0; 99 100 if (ila->xp.ifindex) 101 score += 1 << 1; 102 103 return score; 104 } 105 106 static const struct rhashtable_params rht_params = { 107 .nelem_hint = 1024, 108 .head_offset = offsetof(struct ila_map, node), 109 .key_offset = offsetof(struct ila_map, xp.ip.locator_match), 110 .key_len = sizeof(u64), /* identifier */ 111 .max_size = 1048576, 112 .min_size = 256, 113 .automatic_shrinking = true, 114 .obj_cmpfn = ila_cmpfn, 115 }; 116 117 static struct genl_family ila_nl_family; 118 119 static const struct nla_policy ila_nl_policy[ILA_ATTR_MAX + 1] = { 120 [ILA_ATTR_LOCATOR] = { .type = NLA_U64, }, 121 [ILA_ATTR_LOCATOR_MATCH] = { .type = NLA_U64, }, 122 [ILA_ATTR_IFINDEX] = { .type = NLA_U32, }, 123 [ILA_ATTR_CSUM_MODE] = { .type = NLA_U8, }, 124 [ILA_ATTR_IDENT_TYPE] = { .type = NLA_U8, }, 125 }; 126 127 static int parse_nl_config(struct genl_info *info, 128 struct ila_xlat_params *xp) 129 { 130 memset(xp, 0, sizeof(*xp)); 131 132 if (info->attrs[ILA_ATTR_LOCATOR]) 133 xp->ip.locator.v64 = (__force __be64)nla_get_u64( 134 info->attrs[ILA_ATTR_LOCATOR]); 135 136 if (info->attrs[ILA_ATTR_LOCATOR_MATCH]) 137 xp->ip.locator_match.v64 = (__force __be64)nla_get_u64( 138 info->attrs[ILA_ATTR_LOCATOR_MATCH]); 139 140 if (info->attrs[ILA_ATTR_CSUM_MODE]) 141 xp->ip.csum_mode = nla_get_u8(info->attrs[ILA_ATTR_CSUM_MODE]); 142 else 143 xp->ip.csum_mode = ILA_CSUM_NO_ACTION; 144 145 if (info->attrs[ILA_ATTR_IDENT_TYPE]) 146 xp->ip.ident_type = nla_get_u8( 147 info->attrs[ILA_ATTR_IDENT_TYPE]); 148 else 149 xp->ip.ident_type = ILA_ATYPE_USE_FORMAT; 150 151 if (info->attrs[ILA_ATTR_IFINDEX]) 152 xp->ifindex = nla_get_s32(info->attrs[ILA_ATTR_IFINDEX]); 153 154 return 0; 155 } 156 157 /* Must be called with rcu readlock */ 158 static inline struct ila_map *ila_lookup_wildcards(struct ila_addr *iaddr, 159 int ifindex, 160 struct ila_net *ilan) 161 { 162 struct ila_map *ila; 163 164 ila = rhashtable_lookup_fast(&ilan->rhash_table, &iaddr->loc, 165 rht_params); 166 while (ila) { 167 if (!ila_cmp_wildcards(ila, iaddr, ifindex)) 168 return ila; 169 ila = rcu_access_pointer(ila->next); 170 } 171 172 return NULL; 173 } 174 175 /* Must be called with rcu readlock */ 176 static inline struct ila_map *ila_lookup_by_params(struct ila_xlat_params *xp, 177 struct ila_net *ilan) 178 { 179 struct ila_map *ila; 180 181 ila = rhashtable_lookup_fast(&ilan->rhash_table, 182 &xp->ip.locator_match, 183 rht_params); 184 while (ila) { 185 if (!ila_cmp_params(ila, xp)) 186 return ila; 187 ila = rcu_access_pointer(ila->next); 188 } 189 190 return NULL; 191 } 192 193 static inline void ila_release(struct ila_map *ila) 194 { 195 kfree_rcu(ila, rcu); 196 } 197 198 static void ila_free_cb(void *ptr, void *arg) 199 { 200 struct ila_map *ila = (struct ila_map *)ptr, *next; 201 202 /* Assume rcu_readlock held */ 203 while (ila) { 204 next = rcu_access_pointer(ila->next); 205 ila_release(ila); 206 ila = next; 207 } 208 } 209 210 static int ila_xlat_addr(struct sk_buff *skb, bool sir2ila); 211 212 static unsigned int 213 ila_nf_input(void *priv, 214 struct sk_buff *skb, 215 const struct nf_hook_state *state) 216 { 217 ila_xlat_addr(skb, false); 218 return NF_ACCEPT; 219 } 220 221 static const struct nf_hook_ops ila_nf_hook_ops[] = { 222 { 223 .hook = ila_nf_input, 224 .pf = NFPROTO_IPV6, 225 .hooknum = NF_INET_PRE_ROUTING, 226 .priority = -1, 227 }, 228 }; 229 230 static int ila_add_mapping(struct net *net, struct ila_xlat_params *xp) 231 { 232 struct ila_net *ilan = net_generic(net, ila_net_id); 233 struct ila_map *ila, *head; 234 spinlock_t *lock = ila_get_lock(ilan, xp->ip.locator_match); 235 int err = 0, order; 236 237 if (!ilan->hooks_registered) { 238 /* We defer registering net hooks in the namespace until the 239 * first mapping is added. 240 */ 241 err = nf_register_net_hooks(net, ila_nf_hook_ops, 242 ARRAY_SIZE(ila_nf_hook_ops)); 243 if (err) 244 return err; 245 246 ilan->hooks_registered = true; 247 } 248 249 ila = kzalloc(sizeof(*ila), GFP_KERNEL); 250 if (!ila) 251 return -ENOMEM; 252 253 ila_init_saved_csum(&xp->ip); 254 255 ila->xp = *xp; 256 257 order = ila_order(ila); 258 259 spin_lock(lock); 260 261 head = rhashtable_lookup_fast(&ilan->rhash_table, 262 &xp->ip.locator_match, 263 rht_params); 264 if (!head) { 265 /* New entry for the rhash_table */ 266 err = rhashtable_lookup_insert_fast(&ilan->rhash_table, 267 &ila->node, rht_params); 268 } else { 269 struct ila_map *tila = head, *prev = NULL; 270 271 do { 272 if (!ila_cmp_params(tila, xp)) { 273 err = -EEXIST; 274 goto out; 275 } 276 277 if (order > ila_order(tila)) 278 break; 279 280 prev = tila; 281 tila = rcu_dereference_protected(tila->next, 282 lockdep_is_held(lock)); 283 } while (tila); 284 285 if (prev) { 286 /* Insert in sub list of head */ 287 RCU_INIT_POINTER(ila->next, tila); 288 rcu_assign_pointer(prev->next, ila); 289 } else { 290 /* Make this ila new head */ 291 RCU_INIT_POINTER(ila->next, head); 292 err = rhashtable_replace_fast(&ilan->rhash_table, 293 &head->node, 294 &ila->node, rht_params); 295 if (err) 296 goto out; 297 } 298 } 299 300 out: 301 spin_unlock(lock); 302 303 if (err) 304 kfree(ila); 305 306 return err; 307 } 308 309 static int ila_del_mapping(struct net *net, struct ila_xlat_params *xp) 310 { 311 struct ila_net *ilan = net_generic(net, ila_net_id); 312 struct ila_map *ila, *head, *prev; 313 spinlock_t *lock = ila_get_lock(ilan, xp->ip.locator_match); 314 int err = -ENOENT; 315 316 spin_lock(lock); 317 318 head = rhashtable_lookup_fast(&ilan->rhash_table, 319 &xp->ip.locator_match, rht_params); 320 ila = head; 321 322 prev = NULL; 323 324 while (ila) { 325 if (ila_cmp_params(ila, xp)) { 326 prev = ila; 327 ila = rcu_dereference_protected(ila->next, 328 lockdep_is_held(lock)); 329 continue; 330 } 331 332 err = 0; 333 334 if (prev) { 335 /* Not head, just delete from list */ 336 rcu_assign_pointer(prev->next, ila->next); 337 } else { 338 /* It is the head. If there is something in the 339 * sublist we need to make a new head. 340 */ 341 head = rcu_dereference_protected(ila->next, 342 lockdep_is_held(lock)); 343 if (head) { 344 /* Put first entry in the sublist into the 345 * table 346 */ 347 err = rhashtable_replace_fast( 348 &ilan->rhash_table, &ila->node, 349 &head->node, rht_params); 350 if (err) 351 goto out; 352 } else { 353 /* Entry no longer used */ 354 err = rhashtable_remove_fast(&ilan->rhash_table, 355 &ila->node, 356 rht_params); 357 } 358 } 359 360 ila_release(ila); 361 362 break; 363 } 364 365 out: 366 spin_unlock(lock); 367 368 return err; 369 } 370 371 static int ila_nl_cmd_add_mapping(struct sk_buff *skb, struct genl_info *info) 372 { 373 struct net *net = genl_info_net(info); 374 struct ila_xlat_params p; 375 int err; 376 377 err = parse_nl_config(info, &p); 378 if (err) 379 return err; 380 381 return ila_add_mapping(net, &p); 382 } 383 384 static int ila_nl_cmd_del_mapping(struct sk_buff *skb, struct genl_info *info) 385 { 386 struct net *net = genl_info_net(info); 387 struct ila_xlat_params xp; 388 int err; 389 390 err = parse_nl_config(info, &xp); 391 if (err) 392 return err; 393 394 ila_del_mapping(net, &xp); 395 396 return 0; 397 } 398 399 static int ila_fill_info(struct ila_map *ila, struct sk_buff *msg) 400 { 401 if (nla_put_u64_64bit(msg, ILA_ATTR_LOCATOR, 402 (__force u64)ila->xp.ip.locator.v64, 403 ILA_ATTR_PAD) || 404 nla_put_u64_64bit(msg, ILA_ATTR_LOCATOR_MATCH, 405 (__force u64)ila->xp.ip.locator_match.v64, 406 ILA_ATTR_PAD) || 407 nla_put_s32(msg, ILA_ATTR_IFINDEX, ila->xp.ifindex) || 408 nla_put_u8(msg, ILA_ATTR_CSUM_MODE, ila->xp.ip.csum_mode) || 409 nla_put_u8(msg, ILA_ATTR_IDENT_TYPE, ila->xp.ip.ident_type)) 410 return -1; 411 412 return 0; 413 } 414 415 static int ila_dump_info(struct ila_map *ila, 416 u32 portid, u32 seq, u32 flags, 417 struct sk_buff *skb, u8 cmd) 418 { 419 void *hdr; 420 421 hdr = genlmsg_put(skb, portid, seq, &ila_nl_family, flags, cmd); 422 if (!hdr) 423 return -ENOMEM; 424 425 if (ila_fill_info(ila, skb) < 0) 426 goto nla_put_failure; 427 428 genlmsg_end(skb, hdr); 429 return 0; 430 431 nla_put_failure: 432 genlmsg_cancel(skb, hdr); 433 return -EMSGSIZE; 434 } 435 436 static int ila_nl_cmd_get_mapping(struct sk_buff *skb, struct genl_info *info) 437 { 438 struct net *net = genl_info_net(info); 439 struct ila_net *ilan = net_generic(net, ila_net_id); 440 struct sk_buff *msg; 441 struct ila_xlat_params xp; 442 struct ila_map *ila; 443 int ret; 444 445 ret = parse_nl_config(info, &xp); 446 if (ret) 447 return ret; 448 449 msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL); 450 if (!msg) 451 return -ENOMEM; 452 453 rcu_read_lock(); 454 455 ila = ila_lookup_by_params(&xp, ilan); 456 if (ila) { 457 ret = ila_dump_info(ila, 458 info->snd_portid, 459 info->snd_seq, 0, msg, 460 info->genlhdr->cmd); 461 } 462 463 rcu_read_unlock(); 464 465 if (ret < 0) 466 goto out_free; 467 468 return genlmsg_reply(msg, info); 469 470 out_free: 471 nlmsg_free(msg); 472 return ret; 473 } 474 475 struct ila_dump_iter { 476 struct rhashtable_iter rhiter; 477 }; 478 479 static int ila_nl_dump_start(struct netlink_callback *cb) 480 { 481 struct net *net = sock_net(cb->skb->sk); 482 struct ila_net *ilan = net_generic(net, ila_net_id); 483 struct ila_dump_iter *iter = (struct ila_dump_iter *)cb->args[0]; 484 485 if (!iter) { 486 iter = kmalloc(sizeof(*iter), GFP_KERNEL); 487 if (!iter) 488 return -ENOMEM; 489 490 cb->args[0] = (long)iter; 491 } 492 493 return rhashtable_walk_init(&ilan->rhash_table, &iter->rhiter, 494 GFP_KERNEL); 495 } 496 497 static int ila_nl_dump_done(struct netlink_callback *cb) 498 { 499 struct ila_dump_iter *iter = (struct ila_dump_iter *)cb->args[0]; 500 501 rhashtable_walk_exit(&iter->rhiter); 502 503 kfree(iter); 504 505 return 0; 506 } 507 508 static int ila_nl_dump(struct sk_buff *skb, struct netlink_callback *cb) 509 { 510 struct ila_dump_iter *iter = (struct ila_dump_iter *)cb->args[0]; 511 struct rhashtable_iter *rhiter = &iter->rhiter; 512 struct ila_map *ila; 513 int ret; 514 515 rhashtable_walk_start(rhiter); 516 517 for (;;) { 518 ila = rhashtable_walk_next(rhiter); 519 520 if (IS_ERR(ila)) { 521 if (PTR_ERR(ila) == -EAGAIN) 522 continue; 523 ret = PTR_ERR(ila); 524 goto done; 525 } else if (!ila) { 526 break; 527 } 528 529 while (ila) { 530 ret = ila_dump_info(ila, NETLINK_CB(cb->skb).portid, 531 cb->nlh->nlmsg_seq, NLM_F_MULTI, 532 skb, ILA_CMD_GET); 533 if (ret) 534 goto done; 535 536 ila = rcu_access_pointer(ila->next); 537 } 538 } 539 540 ret = skb->len; 541 542 done: 543 rhashtable_walk_stop(rhiter); 544 return ret; 545 } 546 547 static const struct genl_ops ila_nl_ops[] = { 548 { 549 .cmd = ILA_CMD_ADD, 550 .doit = ila_nl_cmd_add_mapping, 551 .policy = ila_nl_policy, 552 .flags = GENL_ADMIN_PERM, 553 }, 554 { 555 .cmd = ILA_CMD_DEL, 556 .doit = ila_nl_cmd_del_mapping, 557 .policy = ila_nl_policy, 558 .flags = GENL_ADMIN_PERM, 559 }, 560 { 561 .cmd = ILA_CMD_GET, 562 .doit = ila_nl_cmd_get_mapping, 563 .start = ila_nl_dump_start, 564 .dumpit = ila_nl_dump, 565 .done = ila_nl_dump_done, 566 .policy = ila_nl_policy, 567 }, 568 }; 569 570 static struct genl_family ila_nl_family __ro_after_init = { 571 .hdrsize = 0, 572 .name = ILA_GENL_NAME, 573 .version = ILA_GENL_VERSION, 574 .maxattr = ILA_ATTR_MAX, 575 .netnsok = true, 576 .parallel_ops = true, 577 .module = THIS_MODULE, 578 .ops = ila_nl_ops, 579 .n_ops = ARRAY_SIZE(ila_nl_ops), 580 }; 581 582 #define ILA_HASH_TABLE_SIZE 1024 583 584 static __net_init int ila_init_net(struct net *net) 585 { 586 int err; 587 struct ila_net *ilan = net_generic(net, ila_net_id); 588 589 err = alloc_ila_locks(ilan); 590 if (err) 591 return err; 592 593 rhashtable_init(&ilan->rhash_table, &rht_params); 594 595 return 0; 596 } 597 598 static __net_exit void ila_exit_net(struct net *net) 599 { 600 struct ila_net *ilan = net_generic(net, ila_net_id); 601 602 rhashtable_free_and_destroy(&ilan->rhash_table, ila_free_cb, NULL); 603 604 kvfree(ilan->locks); 605 606 if (ilan->hooks_registered) 607 nf_unregister_net_hooks(net, ila_nf_hook_ops, 608 ARRAY_SIZE(ila_nf_hook_ops)); 609 } 610 611 static struct pernet_operations ila_net_ops = { 612 .init = ila_init_net, 613 .exit = ila_exit_net, 614 .id = &ila_net_id, 615 .size = sizeof(struct ila_net), 616 }; 617 618 static int ila_xlat_addr(struct sk_buff *skb, bool sir2ila) 619 { 620 struct ila_map *ila; 621 struct ipv6hdr *ip6h = ipv6_hdr(skb); 622 struct net *net = dev_net(skb->dev); 623 struct ila_net *ilan = net_generic(net, ila_net_id); 624 struct ila_addr *iaddr = ila_a2i(&ip6h->daddr); 625 626 /* Assumes skb contains a valid IPv6 header that is pulled */ 627 628 /* No check here that ILA type in the mapping matches what is in the 629 * address. We assume that whatever sender gaves us can be translated. 630 * The checksum mode however is relevant. 631 */ 632 633 rcu_read_lock(); 634 635 ila = ila_lookup_wildcards(iaddr, skb->dev->ifindex, ilan); 636 if (ila) 637 ila_update_ipv6_locator(skb, &ila->xp.ip, sir2ila); 638 639 rcu_read_unlock(); 640 641 return 0; 642 } 643 644 int __init ila_xlat_init(void) 645 { 646 int ret; 647 648 ret = register_pernet_device(&ila_net_ops); 649 if (ret) 650 goto exit; 651 652 ret = genl_register_family(&ila_nl_family); 653 if (ret < 0) 654 goto unregister; 655 656 return 0; 657 658 unregister: 659 unregister_pernet_device(&ila_net_ops); 660 exit: 661 return ret; 662 } 663 664 void ila_xlat_fini(void) 665 { 666 genl_unregister_family(&ila_nl_family); 667 unregister_pernet_device(&ila_net_ops); 668 } 669