1 #include <linux/jhash.h> 2 #include <linux/netfilter.h> 3 #include <linux/rcupdate.h> 4 #include <linux/rhashtable.h> 5 #include <linux/vmalloc.h> 6 #include <net/genetlink.h> 7 #include <net/ila.h> 8 #include <net/netns/generic.h> 9 #include <uapi/linux/genetlink.h> 10 #include "ila.h" 11 12 struct ila_xlat_params { 13 struct ila_params ip; 14 int ifindex; 15 }; 16 17 struct ila_map { 18 struct ila_xlat_params xp; 19 struct rhash_head node; 20 struct ila_map __rcu *next; 21 struct rcu_head rcu; 22 }; 23 24 static unsigned int ila_net_id; 25 26 struct ila_net { 27 struct rhashtable rhash_table; 28 spinlock_t *locks; /* Bucket locks for entry manipulation */ 29 unsigned int locks_mask; 30 bool hooks_registered; 31 }; 32 33 #define LOCKS_PER_CPU 10 34 35 static int alloc_ila_locks(struct ila_net *ilan) 36 { 37 unsigned int i, size; 38 unsigned int nr_pcpus = num_possible_cpus(); 39 40 nr_pcpus = min_t(unsigned int, nr_pcpus, 32UL); 41 size = roundup_pow_of_two(nr_pcpus * LOCKS_PER_CPU); 42 43 if (sizeof(spinlock_t) != 0) { 44 #ifdef CONFIG_NUMA 45 if (size * sizeof(spinlock_t) > PAGE_SIZE) 46 ilan->locks = vmalloc(size * sizeof(spinlock_t)); 47 else 48 #endif 49 ilan->locks = kmalloc_array(size, sizeof(spinlock_t), 50 GFP_KERNEL); 51 if (!ilan->locks) 52 return -ENOMEM; 53 for (i = 0; i < size; i++) 54 spin_lock_init(&ilan->locks[i]); 55 } 56 ilan->locks_mask = size - 1; 57 58 return 0; 59 } 60 61 static u32 hashrnd __read_mostly; 62 static __always_inline void __ila_hash_secret_init(void) 63 { 64 net_get_random_once(&hashrnd, sizeof(hashrnd)); 65 } 66 67 static inline u32 ila_locator_hash(struct ila_locator loc) 68 { 69 u32 *v = (u32 *)loc.v32; 70 71 return jhash_2words(v[0], v[1], hashrnd); 72 } 73 74 static inline spinlock_t *ila_get_lock(struct ila_net *ilan, 75 struct ila_locator loc) 76 { 77 return &ilan->locks[ila_locator_hash(loc) & ilan->locks_mask]; 78 } 79 80 static inline int ila_cmp_wildcards(struct ila_map *ila, 81 struct ila_addr *iaddr, int ifindex) 82 { 83 return (ila->xp.ifindex && ila->xp.ifindex != ifindex); 84 } 85 86 static inline int ila_cmp_params(struct ila_map *ila, 87 struct ila_xlat_params *xp) 88 { 89 return (ila->xp.ifindex != xp->ifindex); 90 } 91 92 static int ila_cmpfn(struct rhashtable_compare_arg *arg, 93 const void *obj) 94 { 95 const struct ila_map *ila = obj; 96 97 return (ila->xp.ip.locator_match.v64 != *(__be64 *)arg->key); 98 } 99 100 static inline int ila_order(struct ila_map *ila) 101 { 102 int score = 0; 103 104 if (ila->xp.ifindex) 105 score += 1 << 1; 106 107 return score; 108 } 109 110 static const struct rhashtable_params rht_params = { 111 .nelem_hint = 1024, 112 .head_offset = offsetof(struct ila_map, node), 113 .key_offset = offsetof(struct ila_map, xp.ip.locator_match), 114 .key_len = sizeof(u64), /* identifier */ 115 .max_size = 1048576, 116 .min_size = 256, 117 .automatic_shrinking = true, 118 .obj_cmpfn = ila_cmpfn, 119 }; 120 121 static struct genl_family ila_nl_family; 122 123 static const struct nla_policy ila_nl_policy[ILA_ATTR_MAX + 1] = { 124 [ILA_ATTR_LOCATOR] = { .type = NLA_U64, }, 125 [ILA_ATTR_LOCATOR_MATCH] = { .type = NLA_U64, }, 126 [ILA_ATTR_IFINDEX] = { .type = NLA_U32, }, 127 [ILA_ATTR_CSUM_MODE] = { .type = NLA_U8, }, 128 }; 129 130 static int parse_nl_config(struct genl_info *info, 131 struct ila_xlat_params *xp) 132 { 133 memset(xp, 0, sizeof(*xp)); 134 135 if (info->attrs[ILA_ATTR_LOCATOR]) 136 xp->ip.locator.v64 = (__force __be64)nla_get_u64( 137 info->attrs[ILA_ATTR_LOCATOR]); 138 139 if (info->attrs[ILA_ATTR_LOCATOR_MATCH]) 140 xp->ip.locator_match.v64 = (__force __be64)nla_get_u64( 141 info->attrs[ILA_ATTR_LOCATOR_MATCH]); 142 143 if (info->attrs[ILA_ATTR_CSUM_MODE]) 144 xp->ip.csum_mode = nla_get_u8(info->attrs[ILA_ATTR_CSUM_MODE]); 145 146 if (info->attrs[ILA_ATTR_IFINDEX]) 147 xp->ifindex = nla_get_s32(info->attrs[ILA_ATTR_IFINDEX]); 148 149 return 0; 150 } 151 152 /* Must be called with rcu readlock */ 153 static inline struct ila_map *ila_lookup_wildcards(struct ila_addr *iaddr, 154 int ifindex, 155 struct ila_net *ilan) 156 { 157 struct ila_map *ila; 158 159 ila = rhashtable_lookup_fast(&ilan->rhash_table, &iaddr->loc, 160 rht_params); 161 while (ila) { 162 if (!ila_cmp_wildcards(ila, iaddr, ifindex)) 163 return ila; 164 ila = rcu_access_pointer(ila->next); 165 } 166 167 return NULL; 168 } 169 170 /* Must be called with rcu readlock */ 171 static inline struct ila_map *ila_lookup_by_params(struct ila_xlat_params *xp, 172 struct ila_net *ilan) 173 { 174 struct ila_map *ila; 175 176 ila = rhashtable_lookup_fast(&ilan->rhash_table, 177 &xp->ip.locator_match, 178 rht_params); 179 while (ila) { 180 if (!ila_cmp_params(ila, xp)) 181 return ila; 182 ila = rcu_access_pointer(ila->next); 183 } 184 185 return NULL; 186 } 187 188 static inline void ila_release(struct ila_map *ila) 189 { 190 kfree_rcu(ila, rcu); 191 } 192 193 static void ila_free_cb(void *ptr, void *arg) 194 { 195 struct ila_map *ila = (struct ila_map *)ptr, *next; 196 197 /* Assume rcu_readlock held */ 198 while (ila) { 199 next = rcu_access_pointer(ila->next); 200 ila_release(ila); 201 ila = next; 202 } 203 } 204 205 static int ila_xlat_addr(struct sk_buff *skb, bool set_csum_neutral); 206 207 static unsigned int 208 ila_nf_input(void *priv, 209 struct sk_buff *skb, 210 const struct nf_hook_state *state) 211 { 212 ila_xlat_addr(skb, false); 213 return NF_ACCEPT; 214 } 215 216 static struct nf_hook_ops ila_nf_hook_ops[] __read_mostly = { 217 { 218 .hook = ila_nf_input, 219 .pf = NFPROTO_IPV6, 220 .hooknum = NF_INET_PRE_ROUTING, 221 .priority = -1, 222 }, 223 }; 224 225 static int ila_add_mapping(struct net *net, struct ila_xlat_params *xp) 226 { 227 struct ila_net *ilan = net_generic(net, ila_net_id); 228 struct ila_map *ila, *head; 229 spinlock_t *lock = ila_get_lock(ilan, xp->ip.locator_match); 230 int err = 0, order; 231 232 if (!ilan->hooks_registered) { 233 /* We defer registering net hooks in the namespace until the 234 * first mapping is added. 235 */ 236 err = nf_register_net_hooks(net, ila_nf_hook_ops, 237 ARRAY_SIZE(ila_nf_hook_ops)); 238 if (err) 239 return err; 240 241 ilan->hooks_registered = true; 242 } 243 244 ila = kzalloc(sizeof(*ila), GFP_KERNEL); 245 if (!ila) 246 return -ENOMEM; 247 248 ila_init_saved_csum(&xp->ip); 249 250 ila->xp = *xp; 251 252 order = ila_order(ila); 253 254 spin_lock(lock); 255 256 head = rhashtable_lookup_fast(&ilan->rhash_table, 257 &xp->ip.locator_match, 258 rht_params); 259 if (!head) { 260 /* New entry for the rhash_table */ 261 err = rhashtable_lookup_insert_fast(&ilan->rhash_table, 262 &ila->node, rht_params); 263 } else { 264 struct ila_map *tila = head, *prev = NULL; 265 266 do { 267 if (!ila_cmp_params(tila, xp)) { 268 err = -EEXIST; 269 goto out; 270 } 271 272 if (order > ila_order(tila)) 273 break; 274 275 prev = tila; 276 tila = rcu_dereference_protected(tila->next, 277 lockdep_is_held(lock)); 278 } while (tila); 279 280 if (prev) { 281 /* Insert in sub list of head */ 282 RCU_INIT_POINTER(ila->next, tila); 283 rcu_assign_pointer(prev->next, ila); 284 } else { 285 /* Make this ila new head */ 286 RCU_INIT_POINTER(ila->next, head); 287 err = rhashtable_replace_fast(&ilan->rhash_table, 288 &head->node, 289 &ila->node, rht_params); 290 if (err) 291 goto out; 292 } 293 } 294 295 out: 296 spin_unlock(lock); 297 298 if (err) 299 kfree(ila); 300 301 return err; 302 } 303 304 static int ila_del_mapping(struct net *net, struct ila_xlat_params *xp) 305 { 306 struct ila_net *ilan = net_generic(net, ila_net_id); 307 struct ila_map *ila, *head, *prev; 308 spinlock_t *lock = ila_get_lock(ilan, xp->ip.locator_match); 309 int err = -ENOENT; 310 311 spin_lock(lock); 312 313 head = rhashtable_lookup_fast(&ilan->rhash_table, 314 &xp->ip.locator_match, rht_params); 315 ila = head; 316 317 prev = NULL; 318 319 while (ila) { 320 if (ila_cmp_params(ila, xp)) { 321 prev = ila; 322 ila = rcu_dereference_protected(ila->next, 323 lockdep_is_held(lock)); 324 continue; 325 } 326 327 err = 0; 328 329 if (prev) { 330 /* Not head, just delete from list */ 331 rcu_assign_pointer(prev->next, ila->next); 332 } else { 333 /* It is the head. If there is something in the 334 * sublist we need to make a new head. 335 */ 336 head = rcu_dereference_protected(ila->next, 337 lockdep_is_held(lock)); 338 if (head) { 339 /* Put first entry in the sublist into the 340 * table 341 */ 342 err = rhashtable_replace_fast( 343 &ilan->rhash_table, &ila->node, 344 &head->node, rht_params); 345 if (err) 346 goto out; 347 } else { 348 /* Entry no longer used */ 349 err = rhashtable_remove_fast(&ilan->rhash_table, 350 &ila->node, 351 rht_params); 352 } 353 } 354 355 ila_release(ila); 356 357 break; 358 } 359 360 out: 361 spin_unlock(lock); 362 363 return err; 364 } 365 366 static int ila_nl_cmd_add_mapping(struct sk_buff *skb, struct genl_info *info) 367 { 368 struct net *net = genl_info_net(info); 369 struct ila_xlat_params p; 370 int err; 371 372 err = parse_nl_config(info, &p); 373 if (err) 374 return err; 375 376 return ila_add_mapping(net, &p); 377 } 378 379 static int ila_nl_cmd_del_mapping(struct sk_buff *skb, struct genl_info *info) 380 { 381 struct net *net = genl_info_net(info); 382 struct ila_xlat_params xp; 383 int err; 384 385 err = parse_nl_config(info, &xp); 386 if (err) 387 return err; 388 389 ila_del_mapping(net, &xp); 390 391 return 0; 392 } 393 394 static int ila_fill_info(struct ila_map *ila, struct sk_buff *msg) 395 { 396 if (nla_put_u64_64bit(msg, ILA_ATTR_LOCATOR, 397 (__force u64)ila->xp.ip.locator.v64, 398 ILA_ATTR_PAD) || 399 nla_put_u64_64bit(msg, ILA_ATTR_LOCATOR_MATCH, 400 (__force u64)ila->xp.ip.locator_match.v64, 401 ILA_ATTR_PAD) || 402 nla_put_s32(msg, ILA_ATTR_IFINDEX, ila->xp.ifindex) || 403 nla_put_u32(msg, ILA_ATTR_CSUM_MODE, ila->xp.ip.csum_mode)) 404 return -1; 405 406 return 0; 407 } 408 409 static int ila_dump_info(struct ila_map *ila, 410 u32 portid, u32 seq, u32 flags, 411 struct sk_buff *skb, u8 cmd) 412 { 413 void *hdr; 414 415 hdr = genlmsg_put(skb, portid, seq, &ila_nl_family, flags, cmd); 416 if (!hdr) 417 return -ENOMEM; 418 419 if (ila_fill_info(ila, skb) < 0) 420 goto nla_put_failure; 421 422 genlmsg_end(skb, hdr); 423 return 0; 424 425 nla_put_failure: 426 genlmsg_cancel(skb, hdr); 427 return -EMSGSIZE; 428 } 429 430 static int ila_nl_cmd_get_mapping(struct sk_buff *skb, struct genl_info *info) 431 { 432 struct net *net = genl_info_net(info); 433 struct ila_net *ilan = net_generic(net, ila_net_id); 434 struct sk_buff *msg; 435 struct ila_xlat_params xp; 436 struct ila_map *ila; 437 int ret; 438 439 ret = parse_nl_config(info, &xp); 440 if (ret) 441 return ret; 442 443 msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL); 444 if (!msg) 445 return -ENOMEM; 446 447 rcu_read_lock(); 448 449 ila = ila_lookup_by_params(&xp, ilan); 450 if (ila) { 451 ret = ila_dump_info(ila, 452 info->snd_portid, 453 info->snd_seq, 0, msg, 454 info->genlhdr->cmd); 455 } 456 457 rcu_read_unlock(); 458 459 if (ret < 0) 460 goto out_free; 461 462 return genlmsg_reply(msg, info); 463 464 out_free: 465 nlmsg_free(msg); 466 return ret; 467 } 468 469 struct ila_dump_iter { 470 struct rhashtable_iter rhiter; 471 }; 472 473 static int ila_nl_dump_start(struct netlink_callback *cb) 474 { 475 struct net *net = sock_net(cb->skb->sk); 476 struct ila_net *ilan = net_generic(net, ila_net_id); 477 struct ila_dump_iter *iter = (struct ila_dump_iter *)cb->args[0]; 478 479 if (!iter) { 480 iter = kmalloc(sizeof(*iter), GFP_KERNEL); 481 if (!iter) 482 return -ENOMEM; 483 484 cb->args[0] = (long)iter; 485 } 486 487 return rhashtable_walk_init(&ilan->rhash_table, &iter->rhiter, 488 GFP_KERNEL); 489 } 490 491 static int ila_nl_dump_done(struct netlink_callback *cb) 492 { 493 struct ila_dump_iter *iter = (struct ila_dump_iter *)cb->args[0]; 494 495 rhashtable_walk_exit(&iter->rhiter); 496 497 kfree(iter); 498 499 return 0; 500 } 501 502 static int ila_nl_dump(struct sk_buff *skb, struct netlink_callback *cb) 503 { 504 struct ila_dump_iter *iter = (struct ila_dump_iter *)cb->args[0]; 505 struct rhashtable_iter *rhiter = &iter->rhiter; 506 struct ila_map *ila; 507 int ret; 508 509 ret = rhashtable_walk_start(rhiter); 510 if (ret && ret != -EAGAIN) 511 goto done; 512 513 for (;;) { 514 ila = rhashtable_walk_next(rhiter); 515 516 if (IS_ERR(ila)) { 517 if (PTR_ERR(ila) == -EAGAIN) 518 continue; 519 ret = PTR_ERR(ila); 520 goto done; 521 } else if (!ila) { 522 break; 523 } 524 525 while (ila) { 526 ret = ila_dump_info(ila, NETLINK_CB(cb->skb).portid, 527 cb->nlh->nlmsg_seq, NLM_F_MULTI, 528 skb, ILA_CMD_GET); 529 if (ret) 530 goto done; 531 532 ila = rcu_access_pointer(ila->next); 533 } 534 } 535 536 ret = skb->len; 537 538 done: 539 rhashtable_walk_stop(rhiter); 540 return ret; 541 } 542 543 static const struct genl_ops ila_nl_ops[] = { 544 { 545 .cmd = ILA_CMD_ADD, 546 .doit = ila_nl_cmd_add_mapping, 547 .policy = ila_nl_policy, 548 .flags = GENL_ADMIN_PERM, 549 }, 550 { 551 .cmd = ILA_CMD_DEL, 552 .doit = ila_nl_cmd_del_mapping, 553 .policy = ila_nl_policy, 554 .flags = GENL_ADMIN_PERM, 555 }, 556 { 557 .cmd = ILA_CMD_GET, 558 .doit = ila_nl_cmd_get_mapping, 559 .start = ila_nl_dump_start, 560 .dumpit = ila_nl_dump, 561 .done = ila_nl_dump_done, 562 .policy = ila_nl_policy, 563 }, 564 }; 565 566 static struct genl_family ila_nl_family __ro_after_init = { 567 .hdrsize = 0, 568 .name = ILA_GENL_NAME, 569 .version = ILA_GENL_VERSION, 570 .maxattr = ILA_ATTR_MAX, 571 .netnsok = true, 572 .parallel_ops = true, 573 .module = THIS_MODULE, 574 .ops = ila_nl_ops, 575 .n_ops = ARRAY_SIZE(ila_nl_ops), 576 }; 577 578 #define ILA_HASH_TABLE_SIZE 1024 579 580 static __net_init int ila_init_net(struct net *net) 581 { 582 int err; 583 struct ila_net *ilan = net_generic(net, ila_net_id); 584 585 err = alloc_ila_locks(ilan); 586 if (err) 587 return err; 588 589 rhashtable_init(&ilan->rhash_table, &rht_params); 590 591 return 0; 592 } 593 594 static __net_exit void ila_exit_net(struct net *net) 595 { 596 struct ila_net *ilan = net_generic(net, ila_net_id); 597 598 rhashtable_free_and_destroy(&ilan->rhash_table, ila_free_cb, NULL); 599 600 kvfree(ilan->locks); 601 602 if (ilan->hooks_registered) 603 nf_unregister_net_hooks(net, ila_nf_hook_ops, 604 ARRAY_SIZE(ila_nf_hook_ops)); 605 } 606 607 static struct pernet_operations ila_net_ops = { 608 .init = ila_init_net, 609 .exit = ila_exit_net, 610 .id = &ila_net_id, 611 .size = sizeof(struct ila_net), 612 }; 613 614 static int ila_xlat_addr(struct sk_buff *skb, bool set_csum_neutral) 615 { 616 struct ila_map *ila; 617 struct ipv6hdr *ip6h = ipv6_hdr(skb); 618 struct net *net = dev_net(skb->dev); 619 struct ila_net *ilan = net_generic(net, ila_net_id); 620 struct ila_addr *iaddr = ila_a2i(&ip6h->daddr); 621 622 /* Assumes skb contains a valid IPv6 header that is pulled */ 623 624 if (!ila_addr_is_ila(iaddr)) { 625 /* Type indicates this is not an ILA address */ 626 return 0; 627 } 628 629 rcu_read_lock(); 630 631 ila = ila_lookup_wildcards(iaddr, skb->dev->ifindex, ilan); 632 if (ila) 633 ila_update_ipv6_locator(skb, &ila->xp.ip, set_csum_neutral); 634 635 rcu_read_unlock(); 636 637 return 0; 638 } 639 640 int __init ila_xlat_init(void) 641 { 642 int ret; 643 644 ret = register_pernet_device(&ila_net_ops); 645 if (ret) 646 goto exit; 647 648 ret = genl_register_family(&ila_nl_family); 649 if (ret < 0) 650 goto unregister; 651 652 return 0; 653 654 unregister: 655 unregister_pernet_device(&ila_net_ops); 656 exit: 657 return ret; 658 } 659 660 void ila_xlat_fini(void) 661 { 662 genl_unregister_family(&ila_nl_family); 663 unregister_pernet_device(&ila_net_ops); 664 } 665