1 // SPDX-License-Identifier: GPL-2.0 2 #include <linux/jhash.h> 3 #include <linux/netfilter.h> 4 #include <linux/rcupdate.h> 5 #include <linux/rhashtable.h> 6 #include <linux/vmalloc.h> 7 #include <net/genetlink.h> 8 #include <net/ila.h> 9 #include <net/netns/generic.h> 10 #include <uapi/linux/genetlink.h> 11 #include "ila.h" 12 13 struct ila_xlat_params { 14 struct ila_params ip; 15 int ifindex; 16 }; 17 18 struct ila_map { 19 struct ila_xlat_params xp; 20 struct rhash_head node; 21 struct ila_map __rcu *next; 22 struct rcu_head rcu; 23 }; 24 25 static unsigned int ila_net_id; 26 27 struct ila_net { 28 struct rhashtable rhash_table; 29 spinlock_t *locks; /* Bucket locks for entry manipulation */ 30 unsigned int locks_mask; 31 bool hooks_registered; 32 }; 33 34 #define LOCKS_PER_CPU 10 35 36 static int alloc_ila_locks(struct ila_net *ilan) 37 { 38 unsigned int i, size; 39 unsigned int nr_pcpus = num_possible_cpus(); 40 41 nr_pcpus = min_t(unsigned int, nr_pcpus, 32UL); 42 size = roundup_pow_of_two(nr_pcpus * LOCKS_PER_CPU); 43 44 if (sizeof(spinlock_t) != 0) { 45 ilan->locks = kvmalloc_array(size, sizeof(spinlock_t), 46 GFP_KERNEL); 47 if (!ilan->locks) 48 return -ENOMEM; 49 for (i = 0; i < size; i++) 50 spin_lock_init(&ilan->locks[i]); 51 } 52 ilan->locks_mask = size - 1; 53 54 return 0; 55 } 56 57 static u32 hashrnd __read_mostly; 58 static __always_inline void __ila_hash_secret_init(void) 59 { 60 net_get_random_once(&hashrnd, sizeof(hashrnd)); 61 } 62 63 static inline u32 ila_locator_hash(struct ila_locator loc) 64 { 65 u32 *v = (u32 *)loc.v32; 66 67 __ila_hash_secret_init(); 68 return jhash_2words(v[0], v[1], hashrnd); 69 } 70 71 static inline spinlock_t *ila_get_lock(struct ila_net *ilan, 72 struct ila_locator loc) 73 { 74 return &ilan->locks[ila_locator_hash(loc) & ilan->locks_mask]; 75 } 76 77 static inline int ila_cmp_wildcards(struct ila_map *ila, 78 struct ila_addr *iaddr, int ifindex) 79 { 80 return (ila->xp.ifindex && ila->xp.ifindex != ifindex); 81 } 82 83 static inline int ila_cmp_params(struct ila_map *ila, 84 struct ila_xlat_params *xp) 85 { 86 return (ila->xp.ifindex != xp->ifindex); 87 } 88 89 static int ila_cmpfn(struct rhashtable_compare_arg *arg, 90 const void *obj) 91 { 92 const struct ila_map *ila = obj; 93 94 return (ila->xp.ip.locator_match.v64 != *(__be64 *)arg->key); 95 } 96 97 static inline int ila_order(struct ila_map *ila) 98 { 99 int score = 0; 100 101 if (ila->xp.ifindex) 102 score += 1 << 1; 103 104 return score; 105 } 106 107 static const struct rhashtable_params rht_params = { 108 .nelem_hint = 1024, 109 .head_offset = offsetof(struct ila_map, node), 110 .key_offset = offsetof(struct ila_map, xp.ip.locator_match), 111 .key_len = sizeof(u64), /* identifier */ 112 .max_size = 1048576, 113 .min_size = 256, 114 .automatic_shrinking = true, 115 .obj_cmpfn = ila_cmpfn, 116 }; 117 118 static struct genl_family ila_nl_family; 119 120 static const struct nla_policy ila_nl_policy[ILA_ATTR_MAX + 1] = { 121 [ILA_ATTR_LOCATOR] = { .type = NLA_U64, }, 122 [ILA_ATTR_LOCATOR_MATCH] = { .type = NLA_U64, }, 123 [ILA_ATTR_IFINDEX] = { .type = NLA_U32, }, 124 [ILA_ATTR_CSUM_MODE] = { .type = NLA_U8, }, 125 [ILA_ATTR_IDENT_TYPE] = { .type = NLA_U8, }, 126 }; 127 128 static int parse_nl_config(struct genl_info *info, 129 struct ila_xlat_params *xp) 130 { 131 memset(xp, 0, sizeof(*xp)); 132 133 if (info->attrs[ILA_ATTR_LOCATOR]) 134 xp->ip.locator.v64 = (__force __be64)nla_get_u64( 135 info->attrs[ILA_ATTR_LOCATOR]); 136 137 if (info->attrs[ILA_ATTR_LOCATOR_MATCH]) 138 xp->ip.locator_match.v64 = (__force __be64)nla_get_u64( 139 info->attrs[ILA_ATTR_LOCATOR_MATCH]); 140 141 if (info->attrs[ILA_ATTR_CSUM_MODE]) 142 xp->ip.csum_mode = nla_get_u8(info->attrs[ILA_ATTR_CSUM_MODE]); 143 else 144 xp->ip.csum_mode = ILA_CSUM_NO_ACTION; 145 146 if (info->attrs[ILA_ATTR_IDENT_TYPE]) 147 xp->ip.ident_type = nla_get_u8( 148 info->attrs[ILA_ATTR_IDENT_TYPE]); 149 else 150 xp->ip.ident_type = ILA_ATYPE_USE_FORMAT; 151 152 if (info->attrs[ILA_ATTR_IFINDEX]) 153 xp->ifindex = nla_get_s32(info->attrs[ILA_ATTR_IFINDEX]); 154 155 return 0; 156 } 157 158 /* Must be called with rcu readlock */ 159 static inline struct ila_map *ila_lookup_wildcards(struct ila_addr *iaddr, 160 int ifindex, 161 struct ila_net *ilan) 162 { 163 struct ila_map *ila; 164 165 ila = rhashtable_lookup_fast(&ilan->rhash_table, &iaddr->loc, 166 rht_params); 167 while (ila) { 168 if (!ila_cmp_wildcards(ila, iaddr, ifindex)) 169 return ila; 170 ila = rcu_access_pointer(ila->next); 171 } 172 173 return NULL; 174 } 175 176 /* Must be called with rcu readlock */ 177 static inline struct ila_map *ila_lookup_by_params(struct ila_xlat_params *xp, 178 struct ila_net *ilan) 179 { 180 struct ila_map *ila; 181 182 ila = rhashtable_lookup_fast(&ilan->rhash_table, 183 &xp->ip.locator_match, 184 rht_params); 185 while (ila) { 186 if (!ila_cmp_params(ila, xp)) 187 return ila; 188 ila = rcu_access_pointer(ila->next); 189 } 190 191 return NULL; 192 } 193 194 static inline void ila_release(struct ila_map *ila) 195 { 196 kfree_rcu(ila, rcu); 197 } 198 199 static void ila_free_cb(void *ptr, void *arg) 200 { 201 struct ila_map *ila = (struct ila_map *)ptr, *next; 202 203 /* Assume rcu_readlock held */ 204 while (ila) { 205 next = rcu_access_pointer(ila->next); 206 ila_release(ila); 207 ila = next; 208 } 209 } 210 211 static int ila_xlat_addr(struct sk_buff *skb, bool sir2ila); 212 213 static unsigned int 214 ila_nf_input(void *priv, 215 struct sk_buff *skb, 216 const struct nf_hook_state *state) 217 { 218 ila_xlat_addr(skb, false); 219 return NF_ACCEPT; 220 } 221 222 static const struct nf_hook_ops ila_nf_hook_ops[] = { 223 { 224 .hook = ila_nf_input, 225 .pf = NFPROTO_IPV6, 226 .hooknum = NF_INET_PRE_ROUTING, 227 .priority = -1, 228 }, 229 }; 230 231 static int ila_add_mapping(struct net *net, struct ila_xlat_params *xp) 232 { 233 struct ila_net *ilan = net_generic(net, ila_net_id); 234 struct ila_map *ila, *head; 235 spinlock_t *lock = ila_get_lock(ilan, xp->ip.locator_match); 236 int err = 0, order; 237 238 if (!ilan->hooks_registered) { 239 /* We defer registering net hooks in the namespace until the 240 * first mapping is added. 241 */ 242 err = nf_register_net_hooks(net, ila_nf_hook_ops, 243 ARRAY_SIZE(ila_nf_hook_ops)); 244 if (err) 245 return err; 246 247 ilan->hooks_registered = true; 248 } 249 250 ila = kzalloc(sizeof(*ila), GFP_KERNEL); 251 if (!ila) 252 return -ENOMEM; 253 254 ila_init_saved_csum(&xp->ip); 255 256 ila->xp = *xp; 257 258 order = ila_order(ila); 259 260 spin_lock(lock); 261 262 head = rhashtable_lookup_fast(&ilan->rhash_table, 263 &xp->ip.locator_match, 264 rht_params); 265 if (!head) { 266 /* New entry for the rhash_table */ 267 err = rhashtable_lookup_insert_fast(&ilan->rhash_table, 268 &ila->node, rht_params); 269 } else { 270 struct ila_map *tila = head, *prev = NULL; 271 272 do { 273 if (!ila_cmp_params(tila, xp)) { 274 err = -EEXIST; 275 goto out; 276 } 277 278 if (order > ila_order(tila)) 279 break; 280 281 prev = tila; 282 tila = rcu_dereference_protected(tila->next, 283 lockdep_is_held(lock)); 284 } while (tila); 285 286 if (prev) { 287 /* Insert in sub list of head */ 288 RCU_INIT_POINTER(ila->next, tila); 289 rcu_assign_pointer(prev->next, ila); 290 } else { 291 /* Make this ila new head */ 292 RCU_INIT_POINTER(ila->next, head); 293 err = rhashtable_replace_fast(&ilan->rhash_table, 294 &head->node, 295 &ila->node, rht_params); 296 if (err) 297 goto out; 298 } 299 } 300 301 out: 302 spin_unlock(lock); 303 304 if (err) 305 kfree(ila); 306 307 return err; 308 } 309 310 static int ila_del_mapping(struct net *net, struct ila_xlat_params *xp) 311 { 312 struct ila_net *ilan = net_generic(net, ila_net_id); 313 struct ila_map *ila, *head, *prev; 314 spinlock_t *lock = ila_get_lock(ilan, xp->ip.locator_match); 315 int err = -ENOENT; 316 317 spin_lock(lock); 318 319 head = rhashtable_lookup_fast(&ilan->rhash_table, 320 &xp->ip.locator_match, rht_params); 321 ila = head; 322 323 prev = NULL; 324 325 while (ila) { 326 if (ila_cmp_params(ila, xp)) { 327 prev = ila; 328 ila = rcu_dereference_protected(ila->next, 329 lockdep_is_held(lock)); 330 continue; 331 } 332 333 err = 0; 334 335 if (prev) { 336 /* Not head, just delete from list */ 337 rcu_assign_pointer(prev->next, ila->next); 338 } else { 339 /* It is the head. If there is something in the 340 * sublist we need to make a new head. 341 */ 342 head = rcu_dereference_protected(ila->next, 343 lockdep_is_held(lock)); 344 if (head) { 345 /* Put first entry in the sublist into the 346 * table 347 */ 348 err = rhashtable_replace_fast( 349 &ilan->rhash_table, &ila->node, 350 &head->node, rht_params); 351 if (err) 352 goto out; 353 } else { 354 /* Entry no longer used */ 355 err = rhashtable_remove_fast(&ilan->rhash_table, 356 &ila->node, 357 rht_params); 358 } 359 } 360 361 ila_release(ila); 362 363 break; 364 } 365 366 out: 367 spin_unlock(lock); 368 369 return err; 370 } 371 372 static int ila_nl_cmd_add_mapping(struct sk_buff *skb, struct genl_info *info) 373 { 374 struct net *net = genl_info_net(info); 375 struct ila_xlat_params p; 376 int err; 377 378 err = parse_nl_config(info, &p); 379 if (err) 380 return err; 381 382 return ila_add_mapping(net, &p); 383 } 384 385 static int ila_nl_cmd_del_mapping(struct sk_buff *skb, struct genl_info *info) 386 { 387 struct net *net = genl_info_net(info); 388 struct ila_xlat_params xp; 389 int err; 390 391 err = parse_nl_config(info, &xp); 392 if (err) 393 return err; 394 395 ila_del_mapping(net, &xp); 396 397 return 0; 398 } 399 400 static int ila_fill_info(struct ila_map *ila, struct sk_buff *msg) 401 { 402 if (nla_put_u64_64bit(msg, ILA_ATTR_LOCATOR, 403 (__force u64)ila->xp.ip.locator.v64, 404 ILA_ATTR_PAD) || 405 nla_put_u64_64bit(msg, ILA_ATTR_LOCATOR_MATCH, 406 (__force u64)ila->xp.ip.locator_match.v64, 407 ILA_ATTR_PAD) || 408 nla_put_s32(msg, ILA_ATTR_IFINDEX, ila->xp.ifindex) || 409 nla_put_u8(msg, ILA_ATTR_CSUM_MODE, ila->xp.ip.csum_mode) || 410 nla_put_u8(msg, ILA_ATTR_IDENT_TYPE, ila->xp.ip.ident_type)) 411 return -1; 412 413 return 0; 414 } 415 416 static int ila_dump_info(struct ila_map *ila, 417 u32 portid, u32 seq, u32 flags, 418 struct sk_buff *skb, u8 cmd) 419 { 420 void *hdr; 421 422 hdr = genlmsg_put(skb, portid, seq, &ila_nl_family, flags, cmd); 423 if (!hdr) 424 return -ENOMEM; 425 426 if (ila_fill_info(ila, skb) < 0) 427 goto nla_put_failure; 428 429 genlmsg_end(skb, hdr); 430 return 0; 431 432 nla_put_failure: 433 genlmsg_cancel(skb, hdr); 434 return -EMSGSIZE; 435 } 436 437 static int ila_nl_cmd_get_mapping(struct sk_buff *skb, struct genl_info *info) 438 { 439 struct net *net = genl_info_net(info); 440 struct ila_net *ilan = net_generic(net, ila_net_id); 441 struct sk_buff *msg; 442 struct ila_xlat_params xp; 443 struct ila_map *ila; 444 int ret; 445 446 ret = parse_nl_config(info, &xp); 447 if (ret) 448 return ret; 449 450 msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL); 451 if (!msg) 452 return -ENOMEM; 453 454 rcu_read_lock(); 455 456 ila = ila_lookup_by_params(&xp, ilan); 457 if (ila) { 458 ret = ila_dump_info(ila, 459 info->snd_portid, 460 info->snd_seq, 0, msg, 461 info->genlhdr->cmd); 462 } 463 464 rcu_read_unlock(); 465 466 if (ret < 0) 467 goto out_free; 468 469 return genlmsg_reply(msg, info); 470 471 out_free: 472 nlmsg_free(msg); 473 return ret; 474 } 475 476 struct ila_dump_iter { 477 struct rhashtable_iter rhiter; 478 }; 479 480 static int ila_nl_dump_start(struct netlink_callback *cb) 481 { 482 struct net *net = sock_net(cb->skb->sk); 483 struct ila_net *ilan = net_generic(net, ila_net_id); 484 struct ila_dump_iter *iter = (struct ila_dump_iter *)cb->args[0]; 485 486 if (!iter) { 487 iter = kmalloc(sizeof(*iter), GFP_KERNEL); 488 if (!iter) 489 return -ENOMEM; 490 491 cb->args[0] = (long)iter; 492 } 493 494 return rhashtable_walk_init(&ilan->rhash_table, &iter->rhiter, 495 GFP_KERNEL); 496 } 497 498 static int ila_nl_dump_done(struct netlink_callback *cb) 499 { 500 struct ila_dump_iter *iter = (struct ila_dump_iter *)cb->args[0]; 501 502 rhashtable_walk_exit(&iter->rhiter); 503 504 kfree(iter); 505 506 return 0; 507 } 508 509 static int ila_nl_dump(struct sk_buff *skb, struct netlink_callback *cb) 510 { 511 struct ila_dump_iter *iter = (struct ila_dump_iter *)cb->args[0]; 512 struct rhashtable_iter *rhiter = &iter->rhiter; 513 struct ila_map *ila; 514 int ret; 515 516 rhashtable_walk_start(rhiter); 517 518 for (;;) { 519 ila = rhashtable_walk_next(rhiter); 520 521 if (IS_ERR(ila)) { 522 if (PTR_ERR(ila) == -EAGAIN) 523 continue; 524 ret = PTR_ERR(ila); 525 goto done; 526 } else if (!ila) { 527 break; 528 } 529 530 while (ila) { 531 ret = ila_dump_info(ila, NETLINK_CB(cb->skb).portid, 532 cb->nlh->nlmsg_seq, NLM_F_MULTI, 533 skb, ILA_CMD_GET); 534 if (ret) 535 goto done; 536 537 ila = rcu_access_pointer(ila->next); 538 } 539 } 540 541 ret = skb->len; 542 543 done: 544 rhashtable_walk_stop(rhiter); 545 return ret; 546 } 547 548 static const struct genl_ops ila_nl_ops[] = { 549 { 550 .cmd = ILA_CMD_ADD, 551 .doit = ila_nl_cmd_add_mapping, 552 .policy = ila_nl_policy, 553 .flags = GENL_ADMIN_PERM, 554 }, 555 { 556 .cmd = ILA_CMD_DEL, 557 .doit = ila_nl_cmd_del_mapping, 558 .policy = ila_nl_policy, 559 .flags = GENL_ADMIN_PERM, 560 }, 561 { 562 .cmd = ILA_CMD_GET, 563 .doit = ila_nl_cmd_get_mapping, 564 .start = ila_nl_dump_start, 565 .dumpit = ila_nl_dump, 566 .done = ila_nl_dump_done, 567 .policy = ila_nl_policy, 568 }, 569 }; 570 571 static struct genl_family ila_nl_family __ro_after_init = { 572 .hdrsize = 0, 573 .name = ILA_GENL_NAME, 574 .version = ILA_GENL_VERSION, 575 .maxattr = ILA_ATTR_MAX, 576 .netnsok = true, 577 .parallel_ops = true, 578 .module = THIS_MODULE, 579 .ops = ila_nl_ops, 580 .n_ops = ARRAY_SIZE(ila_nl_ops), 581 }; 582 583 #define ILA_HASH_TABLE_SIZE 1024 584 585 static __net_init int ila_init_net(struct net *net) 586 { 587 int err; 588 struct ila_net *ilan = net_generic(net, ila_net_id); 589 590 err = alloc_ila_locks(ilan); 591 if (err) 592 return err; 593 594 rhashtable_init(&ilan->rhash_table, &rht_params); 595 596 return 0; 597 } 598 599 static __net_exit void ila_exit_net(struct net *net) 600 { 601 struct ila_net *ilan = net_generic(net, ila_net_id); 602 603 rhashtable_free_and_destroy(&ilan->rhash_table, ila_free_cb, NULL); 604 605 kvfree(ilan->locks); 606 607 if (ilan->hooks_registered) 608 nf_unregister_net_hooks(net, ila_nf_hook_ops, 609 ARRAY_SIZE(ila_nf_hook_ops)); 610 } 611 612 static struct pernet_operations ila_net_ops = { 613 .init = ila_init_net, 614 .exit = ila_exit_net, 615 .id = &ila_net_id, 616 .size = sizeof(struct ila_net), 617 }; 618 619 static int ila_xlat_addr(struct sk_buff *skb, bool sir2ila) 620 { 621 struct ila_map *ila; 622 struct ipv6hdr *ip6h = ipv6_hdr(skb); 623 struct net *net = dev_net(skb->dev); 624 struct ila_net *ilan = net_generic(net, ila_net_id); 625 struct ila_addr *iaddr = ila_a2i(&ip6h->daddr); 626 627 /* Assumes skb contains a valid IPv6 header that is pulled */ 628 629 /* No check here that ILA type in the mapping matches what is in the 630 * address. We assume that whatever sender gaves us can be translated. 631 * The checksum mode however is relevant. 632 */ 633 634 rcu_read_lock(); 635 636 ila = ila_lookup_wildcards(iaddr, skb->dev->ifindex, ilan); 637 if (ila) 638 ila_update_ipv6_locator(skb, &ila->xp.ip, sir2ila); 639 640 rcu_read_unlock(); 641 642 return 0; 643 } 644 645 int __init ila_xlat_init(void) 646 { 647 int ret; 648 649 ret = register_pernet_device(&ila_net_ops); 650 if (ret) 651 goto exit; 652 653 ret = genl_register_family(&ila_nl_family); 654 if (ret < 0) 655 goto unregister; 656 657 return 0; 658 659 unregister: 660 unregister_pernet_device(&ila_net_ops); 661 exit: 662 return ret; 663 } 664 665 void ila_xlat_fini(void) 666 { 667 genl_unregister_family(&ila_nl_family); 668 unregister_pernet_device(&ila_net_ops); 669 } 670