1 // SPDX-License-Identifier: GPL-2.0 2 #include <linux/jhash.h> 3 #include <linux/netfilter.h> 4 #include <linux/rcupdate.h> 5 #include <linux/rhashtable.h> 6 #include <linux/vmalloc.h> 7 #include <net/genetlink.h> 8 #include <net/ila.h> 9 #include <net/netns/generic.h> 10 #include <uapi/linux/genetlink.h> 11 #include "ila.h" 12 13 struct ila_xlat_params { 14 struct ila_params ip; 15 int ifindex; 16 }; 17 18 struct ila_map { 19 struct ila_xlat_params xp; 20 struct rhash_head node; 21 struct ila_map __rcu *next; 22 struct rcu_head rcu; 23 }; 24 25 static unsigned int ila_net_id; 26 27 struct ila_net { 28 struct rhashtable rhash_table; 29 spinlock_t *locks; /* Bucket locks for entry manipulation */ 30 unsigned int locks_mask; 31 bool hooks_registered; 32 }; 33 34 #define LOCKS_PER_CPU 10 35 36 static int alloc_ila_locks(struct ila_net *ilan) 37 { 38 unsigned int i, size; 39 unsigned int nr_pcpus = num_possible_cpus(); 40 41 nr_pcpus = min_t(unsigned int, nr_pcpus, 32UL); 42 size = roundup_pow_of_two(nr_pcpus * LOCKS_PER_CPU); 43 44 if (sizeof(spinlock_t) != 0) { 45 ilan->locks = kvmalloc(size * sizeof(spinlock_t), GFP_KERNEL); 46 if (!ilan->locks) 47 return -ENOMEM; 48 for (i = 0; i < size; i++) 49 spin_lock_init(&ilan->locks[i]); 50 } 51 ilan->locks_mask = size - 1; 52 53 return 0; 54 } 55 56 static u32 hashrnd __read_mostly; 57 static __always_inline void __ila_hash_secret_init(void) 58 { 59 net_get_random_once(&hashrnd, sizeof(hashrnd)); 60 } 61 62 static inline u32 ila_locator_hash(struct ila_locator loc) 63 { 64 u32 *v = (u32 *)loc.v32; 65 66 __ila_hash_secret_init(); 67 return jhash_2words(v[0], v[1], hashrnd); 68 } 69 70 static inline spinlock_t *ila_get_lock(struct ila_net *ilan, 71 struct ila_locator loc) 72 { 73 return &ilan->locks[ila_locator_hash(loc) & ilan->locks_mask]; 74 } 75 76 static inline int ila_cmp_wildcards(struct ila_map *ila, 77 struct ila_addr *iaddr, int ifindex) 78 { 79 return (ila->xp.ifindex && ila->xp.ifindex != ifindex); 80 } 81 82 static inline int ila_cmp_params(struct ila_map *ila, 83 struct ila_xlat_params *xp) 84 { 85 return (ila->xp.ifindex != xp->ifindex); 86 } 87 88 static int ila_cmpfn(struct rhashtable_compare_arg *arg, 89 const void *obj) 90 { 91 const struct ila_map *ila = obj; 92 93 return (ila->xp.ip.locator_match.v64 != *(__be64 *)arg->key); 94 } 95 96 static inline int ila_order(struct ila_map *ila) 97 { 98 int score = 0; 99 100 if (ila->xp.ifindex) 101 score += 1 << 1; 102 103 return score; 104 } 105 106 static const struct rhashtable_params rht_params = { 107 .nelem_hint = 1024, 108 .head_offset = offsetof(struct ila_map, node), 109 .key_offset = offsetof(struct ila_map, xp.ip.locator_match), 110 .key_len = sizeof(u64), /* identifier */ 111 .max_size = 1048576, 112 .min_size = 256, 113 .automatic_shrinking = true, 114 .obj_cmpfn = ila_cmpfn, 115 }; 116 117 static struct genl_family ila_nl_family; 118 119 static const struct nla_policy ila_nl_policy[ILA_ATTR_MAX + 1] = { 120 [ILA_ATTR_LOCATOR] = { .type = NLA_U64, }, 121 [ILA_ATTR_LOCATOR_MATCH] = { .type = NLA_U64, }, 122 [ILA_ATTR_IFINDEX] = { .type = NLA_U32, }, 123 [ILA_ATTR_CSUM_MODE] = { .type = NLA_U8, }, 124 }; 125 126 static int parse_nl_config(struct genl_info *info, 127 struct ila_xlat_params *xp) 128 { 129 memset(xp, 0, sizeof(*xp)); 130 131 if (info->attrs[ILA_ATTR_LOCATOR]) 132 xp->ip.locator.v64 = (__force __be64)nla_get_u64( 133 info->attrs[ILA_ATTR_LOCATOR]); 134 135 if (info->attrs[ILA_ATTR_LOCATOR_MATCH]) 136 xp->ip.locator_match.v64 = (__force __be64)nla_get_u64( 137 info->attrs[ILA_ATTR_LOCATOR_MATCH]); 138 139 if (info->attrs[ILA_ATTR_CSUM_MODE]) 140 xp->ip.csum_mode = nla_get_u8(info->attrs[ILA_ATTR_CSUM_MODE]); 141 142 if (info->attrs[ILA_ATTR_IFINDEX]) 143 xp->ifindex = nla_get_s32(info->attrs[ILA_ATTR_IFINDEX]); 144 145 return 0; 146 } 147 148 /* Must be called with rcu readlock */ 149 static inline struct ila_map *ila_lookup_wildcards(struct ila_addr *iaddr, 150 int ifindex, 151 struct ila_net *ilan) 152 { 153 struct ila_map *ila; 154 155 ila = rhashtable_lookup_fast(&ilan->rhash_table, &iaddr->loc, 156 rht_params); 157 while (ila) { 158 if (!ila_cmp_wildcards(ila, iaddr, ifindex)) 159 return ila; 160 ila = rcu_access_pointer(ila->next); 161 } 162 163 return NULL; 164 } 165 166 /* Must be called with rcu readlock */ 167 static inline struct ila_map *ila_lookup_by_params(struct ila_xlat_params *xp, 168 struct ila_net *ilan) 169 { 170 struct ila_map *ila; 171 172 ila = rhashtable_lookup_fast(&ilan->rhash_table, 173 &xp->ip.locator_match, 174 rht_params); 175 while (ila) { 176 if (!ila_cmp_params(ila, xp)) 177 return ila; 178 ila = rcu_access_pointer(ila->next); 179 } 180 181 return NULL; 182 } 183 184 static inline void ila_release(struct ila_map *ila) 185 { 186 kfree_rcu(ila, rcu); 187 } 188 189 static void ila_free_cb(void *ptr, void *arg) 190 { 191 struct ila_map *ila = (struct ila_map *)ptr, *next; 192 193 /* Assume rcu_readlock held */ 194 while (ila) { 195 next = rcu_access_pointer(ila->next); 196 ila_release(ila); 197 ila = next; 198 } 199 } 200 201 static int ila_xlat_addr(struct sk_buff *skb, bool set_csum_neutral); 202 203 static unsigned int 204 ila_nf_input(void *priv, 205 struct sk_buff *skb, 206 const struct nf_hook_state *state) 207 { 208 ila_xlat_addr(skb, false); 209 return NF_ACCEPT; 210 } 211 212 static const struct nf_hook_ops ila_nf_hook_ops[] = { 213 { 214 .hook = ila_nf_input, 215 .pf = NFPROTO_IPV6, 216 .hooknum = NF_INET_PRE_ROUTING, 217 .priority = -1, 218 }, 219 }; 220 221 static int ila_add_mapping(struct net *net, struct ila_xlat_params *xp) 222 { 223 struct ila_net *ilan = net_generic(net, ila_net_id); 224 struct ila_map *ila, *head; 225 spinlock_t *lock = ila_get_lock(ilan, xp->ip.locator_match); 226 int err = 0, order; 227 228 if (!ilan->hooks_registered) { 229 /* We defer registering net hooks in the namespace until the 230 * first mapping is added. 231 */ 232 err = nf_register_net_hooks(net, ila_nf_hook_ops, 233 ARRAY_SIZE(ila_nf_hook_ops)); 234 if (err) 235 return err; 236 237 ilan->hooks_registered = true; 238 } 239 240 ila = kzalloc(sizeof(*ila), GFP_KERNEL); 241 if (!ila) 242 return -ENOMEM; 243 244 ila_init_saved_csum(&xp->ip); 245 246 ila->xp = *xp; 247 248 order = ila_order(ila); 249 250 spin_lock(lock); 251 252 head = rhashtable_lookup_fast(&ilan->rhash_table, 253 &xp->ip.locator_match, 254 rht_params); 255 if (!head) { 256 /* New entry for the rhash_table */ 257 err = rhashtable_lookup_insert_fast(&ilan->rhash_table, 258 &ila->node, rht_params); 259 } else { 260 struct ila_map *tila = head, *prev = NULL; 261 262 do { 263 if (!ila_cmp_params(tila, xp)) { 264 err = -EEXIST; 265 goto out; 266 } 267 268 if (order > ila_order(tila)) 269 break; 270 271 prev = tila; 272 tila = rcu_dereference_protected(tila->next, 273 lockdep_is_held(lock)); 274 } while (tila); 275 276 if (prev) { 277 /* Insert in sub list of head */ 278 RCU_INIT_POINTER(ila->next, tila); 279 rcu_assign_pointer(prev->next, ila); 280 } else { 281 /* Make this ila new head */ 282 RCU_INIT_POINTER(ila->next, head); 283 err = rhashtable_replace_fast(&ilan->rhash_table, 284 &head->node, 285 &ila->node, rht_params); 286 if (err) 287 goto out; 288 } 289 } 290 291 out: 292 spin_unlock(lock); 293 294 if (err) 295 kfree(ila); 296 297 return err; 298 } 299 300 static int ila_del_mapping(struct net *net, struct ila_xlat_params *xp) 301 { 302 struct ila_net *ilan = net_generic(net, ila_net_id); 303 struct ila_map *ila, *head, *prev; 304 spinlock_t *lock = ila_get_lock(ilan, xp->ip.locator_match); 305 int err = -ENOENT; 306 307 spin_lock(lock); 308 309 head = rhashtable_lookup_fast(&ilan->rhash_table, 310 &xp->ip.locator_match, rht_params); 311 ila = head; 312 313 prev = NULL; 314 315 while (ila) { 316 if (ila_cmp_params(ila, xp)) { 317 prev = ila; 318 ila = rcu_dereference_protected(ila->next, 319 lockdep_is_held(lock)); 320 continue; 321 } 322 323 err = 0; 324 325 if (prev) { 326 /* Not head, just delete from list */ 327 rcu_assign_pointer(prev->next, ila->next); 328 } else { 329 /* It is the head. If there is something in the 330 * sublist we need to make a new head. 331 */ 332 head = rcu_dereference_protected(ila->next, 333 lockdep_is_held(lock)); 334 if (head) { 335 /* Put first entry in the sublist into the 336 * table 337 */ 338 err = rhashtable_replace_fast( 339 &ilan->rhash_table, &ila->node, 340 &head->node, rht_params); 341 if (err) 342 goto out; 343 } else { 344 /* Entry no longer used */ 345 err = rhashtable_remove_fast(&ilan->rhash_table, 346 &ila->node, 347 rht_params); 348 } 349 } 350 351 ila_release(ila); 352 353 break; 354 } 355 356 out: 357 spin_unlock(lock); 358 359 return err; 360 } 361 362 static int ila_nl_cmd_add_mapping(struct sk_buff *skb, struct genl_info *info) 363 { 364 struct net *net = genl_info_net(info); 365 struct ila_xlat_params p; 366 int err; 367 368 err = parse_nl_config(info, &p); 369 if (err) 370 return err; 371 372 return ila_add_mapping(net, &p); 373 } 374 375 static int ila_nl_cmd_del_mapping(struct sk_buff *skb, struct genl_info *info) 376 { 377 struct net *net = genl_info_net(info); 378 struct ila_xlat_params xp; 379 int err; 380 381 err = parse_nl_config(info, &xp); 382 if (err) 383 return err; 384 385 ila_del_mapping(net, &xp); 386 387 return 0; 388 } 389 390 static int ila_fill_info(struct ila_map *ila, struct sk_buff *msg) 391 { 392 if (nla_put_u64_64bit(msg, ILA_ATTR_LOCATOR, 393 (__force u64)ila->xp.ip.locator.v64, 394 ILA_ATTR_PAD) || 395 nla_put_u64_64bit(msg, ILA_ATTR_LOCATOR_MATCH, 396 (__force u64)ila->xp.ip.locator_match.v64, 397 ILA_ATTR_PAD) || 398 nla_put_s32(msg, ILA_ATTR_IFINDEX, ila->xp.ifindex) || 399 nla_put_u32(msg, ILA_ATTR_CSUM_MODE, ila->xp.ip.csum_mode)) 400 return -1; 401 402 return 0; 403 } 404 405 static int ila_dump_info(struct ila_map *ila, 406 u32 portid, u32 seq, u32 flags, 407 struct sk_buff *skb, u8 cmd) 408 { 409 void *hdr; 410 411 hdr = genlmsg_put(skb, portid, seq, &ila_nl_family, flags, cmd); 412 if (!hdr) 413 return -ENOMEM; 414 415 if (ila_fill_info(ila, skb) < 0) 416 goto nla_put_failure; 417 418 genlmsg_end(skb, hdr); 419 return 0; 420 421 nla_put_failure: 422 genlmsg_cancel(skb, hdr); 423 return -EMSGSIZE; 424 } 425 426 static int ila_nl_cmd_get_mapping(struct sk_buff *skb, struct genl_info *info) 427 { 428 struct net *net = genl_info_net(info); 429 struct ila_net *ilan = net_generic(net, ila_net_id); 430 struct sk_buff *msg; 431 struct ila_xlat_params xp; 432 struct ila_map *ila; 433 int ret; 434 435 ret = parse_nl_config(info, &xp); 436 if (ret) 437 return ret; 438 439 msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL); 440 if (!msg) 441 return -ENOMEM; 442 443 rcu_read_lock(); 444 445 ila = ila_lookup_by_params(&xp, ilan); 446 if (ila) { 447 ret = ila_dump_info(ila, 448 info->snd_portid, 449 info->snd_seq, 0, msg, 450 info->genlhdr->cmd); 451 } 452 453 rcu_read_unlock(); 454 455 if (ret < 0) 456 goto out_free; 457 458 return genlmsg_reply(msg, info); 459 460 out_free: 461 nlmsg_free(msg); 462 return ret; 463 } 464 465 struct ila_dump_iter { 466 struct rhashtable_iter rhiter; 467 }; 468 469 static int ila_nl_dump_start(struct netlink_callback *cb) 470 { 471 struct net *net = sock_net(cb->skb->sk); 472 struct ila_net *ilan = net_generic(net, ila_net_id); 473 struct ila_dump_iter *iter = (struct ila_dump_iter *)cb->args[0]; 474 475 if (!iter) { 476 iter = kmalloc(sizeof(*iter), GFP_KERNEL); 477 if (!iter) 478 return -ENOMEM; 479 480 cb->args[0] = (long)iter; 481 } 482 483 return rhashtable_walk_init(&ilan->rhash_table, &iter->rhiter, 484 GFP_KERNEL); 485 } 486 487 static int ila_nl_dump_done(struct netlink_callback *cb) 488 { 489 struct ila_dump_iter *iter = (struct ila_dump_iter *)cb->args[0]; 490 491 rhashtable_walk_exit(&iter->rhiter); 492 493 kfree(iter); 494 495 return 0; 496 } 497 498 static int ila_nl_dump(struct sk_buff *skb, struct netlink_callback *cb) 499 { 500 struct ila_dump_iter *iter = (struct ila_dump_iter *)cb->args[0]; 501 struct rhashtable_iter *rhiter = &iter->rhiter; 502 struct ila_map *ila; 503 int ret; 504 505 ret = rhashtable_walk_start(rhiter); 506 if (ret && ret != -EAGAIN) 507 goto done; 508 509 for (;;) { 510 ila = rhashtable_walk_next(rhiter); 511 512 if (IS_ERR(ila)) { 513 if (PTR_ERR(ila) == -EAGAIN) 514 continue; 515 ret = PTR_ERR(ila); 516 goto done; 517 } else if (!ila) { 518 break; 519 } 520 521 while (ila) { 522 ret = ila_dump_info(ila, NETLINK_CB(cb->skb).portid, 523 cb->nlh->nlmsg_seq, NLM_F_MULTI, 524 skb, ILA_CMD_GET); 525 if (ret) 526 goto done; 527 528 ila = rcu_access_pointer(ila->next); 529 } 530 } 531 532 ret = skb->len; 533 534 done: 535 rhashtable_walk_stop(rhiter); 536 return ret; 537 } 538 539 static const struct genl_ops ila_nl_ops[] = { 540 { 541 .cmd = ILA_CMD_ADD, 542 .doit = ila_nl_cmd_add_mapping, 543 .policy = ila_nl_policy, 544 .flags = GENL_ADMIN_PERM, 545 }, 546 { 547 .cmd = ILA_CMD_DEL, 548 .doit = ila_nl_cmd_del_mapping, 549 .policy = ila_nl_policy, 550 .flags = GENL_ADMIN_PERM, 551 }, 552 { 553 .cmd = ILA_CMD_GET, 554 .doit = ila_nl_cmd_get_mapping, 555 .start = ila_nl_dump_start, 556 .dumpit = ila_nl_dump, 557 .done = ila_nl_dump_done, 558 .policy = ila_nl_policy, 559 }, 560 }; 561 562 static struct genl_family ila_nl_family __ro_after_init = { 563 .hdrsize = 0, 564 .name = ILA_GENL_NAME, 565 .version = ILA_GENL_VERSION, 566 .maxattr = ILA_ATTR_MAX, 567 .netnsok = true, 568 .parallel_ops = true, 569 .module = THIS_MODULE, 570 .ops = ila_nl_ops, 571 .n_ops = ARRAY_SIZE(ila_nl_ops), 572 }; 573 574 #define ILA_HASH_TABLE_SIZE 1024 575 576 static __net_init int ila_init_net(struct net *net) 577 { 578 int err; 579 struct ila_net *ilan = net_generic(net, ila_net_id); 580 581 err = alloc_ila_locks(ilan); 582 if (err) 583 return err; 584 585 rhashtable_init(&ilan->rhash_table, &rht_params); 586 587 return 0; 588 } 589 590 static __net_exit void ila_exit_net(struct net *net) 591 { 592 struct ila_net *ilan = net_generic(net, ila_net_id); 593 594 rhashtable_free_and_destroy(&ilan->rhash_table, ila_free_cb, NULL); 595 596 kvfree(ilan->locks); 597 598 if (ilan->hooks_registered) 599 nf_unregister_net_hooks(net, ila_nf_hook_ops, 600 ARRAY_SIZE(ila_nf_hook_ops)); 601 } 602 603 static struct pernet_operations ila_net_ops = { 604 .init = ila_init_net, 605 .exit = ila_exit_net, 606 .id = &ila_net_id, 607 .size = sizeof(struct ila_net), 608 }; 609 610 static int ila_xlat_addr(struct sk_buff *skb, bool set_csum_neutral) 611 { 612 struct ila_map *ila; 613 struct ipv6hdr *ip6h = ipv6_hdr(skb); 614 struct net *net = dev_net(skb->dev); 615 struct ila_net *ilan = net_generic(net, ila_net_id); 616 struct ila_addr *iaddr = ila_a2i(&ip6h->daddr); 617 618 /* Assumes skb contains a valid IPv6 header that is pulled */ 619 620 if (!ila_addr_is_ila(iaddr)) { 621 /* Type indicates this is not an ILA address */ 622 return 0; 623 } 624 625 rcu_read_lock(); 626 627 ila = ila_lookup_wildcards(iaddr, skb->dev->ifindex, ilan); 628 if (ila) 629 ila_update_ipv6_locator(skb, &ila->xp.ip, set_csum_neutral); 630 631 rcu_read_unlock(); 632 633 return 0; 634 } 635 636 int __init ila_xlat_init(void) 637 { 638 int ret; 639 640 ret = register_pernet_device(&ila_net_ops); 641 if (ret) 642 goto exit; 643 644 ret = genl_register_family(&ila_nl_family); 645 if (ret < 0) 646 goto unregister; 647 648 return 0; 649 650 unregister: 651 unregister_pernet_device(&ila_net_ops); 652 exit: 653 return ret; 654 } 655 656 void ila_xlat_fini(void) 657 { 658 genl_unregister_family(&ila_nl_family); 659 unregister_pernet_device(&ila_net_ops); 660 } 661