1 #include <linux/jhash.h> 2 #include <linux/netfilter.h> 3 #include <linux/rcupdate.h> 4 #include <linux/rhashtable.h> 5 #include <linux/vmalloc.h> 6 #include <net/genetlink.h> 7 #include <net/ila.h> 8 #include <net/netns/generic.h> 9 #include <uapi/linux/genetlink.h> 10 #include "ila.h" 11 12 struct ila_xlat_params { 13 struct ila_params ip; 14 int ifindex; 15 }; 16 17 struct ila_map { 18 struct ila_xlat_params xp; 19 struct rhash_head node; 20 struct ila_map __rcu *next; 21 struct rcu_head rcu; 22 }; 23 24 static unsigned int ila_net_id; 25 26 struct ila_net { 27 struct rhashtable rhash_table; 28 spinlock_t *locks; /* Bucket locks for entry manipulation */ 29 unsigned int locks_mask; 30 bool hooks_registered; 31 }; 32 33 #define LOCKS_PER_CPU 10 34 35 static int alloc_ila_locks(struct ila_net *ilan) 36 { 37 unsigned int i, size; 38 unsigned int nr_pcpus = num_possible_cpus(); 39 40 nr_pcpus = min_t(unsigned int, nr_pcpus, 32UL); 41 size = roundup_pow_of_two(nr_pcpus * LOCKS_PER_CPU); 42 43 if (sizeof(spinlock_t) != 0) { 44 ilan->locks = kvmalloc(size * sizeof(spinlock_t), GFP_KERNEL); 45 if (!ilan->locks) 46 return -ENOMEM; 47 for (i = 0; i < size; i++) 48 spin_lock_init(&ilan->locks[i]); 49 } 50 ilan->locks_mask = size - 1; 51 52 return 0; 53 } 54 55 static u32 hashrnd __read_mostly; 56 static __always_inline void __ila_hash_secret_init(void) 57 { 58 net_get_random_once(&hashrnd, sizeof(hashrnd)); 59 } 60 61 static inline u32 ila_locator_hash(struct ila_locator loc) 62 { 63 u32 *v = (u32 *)loc.v32; 64 65 return jhash_2words(v[0], v[1], hashrnd); 66 } 67 68 static inline spinlock_t *ila_get_lock(struct ila_net *ilan, 69 struct ila_locator loc) 70 { 71 return &ilan->locks[ila_locator_hash(loc) & ilan->locks_mask]; 72 } 73 74 static inline int ila_cmp_wildcards(struct ila_map *ila, 75 struct ila_addr *iaddr, int ifindex) 76 { 77 return (ila->xp.ifindex && ila->xp.ifindex != ifindex); 78 } 79 80 static inline int ila_cmp_params(struct ila_map *ila, 81 struct ila_xlat_params *xp) 82 { 83 return (ila->xp.ifindex != xp->ifindex); 84 } 85 86 static int ila_cmpfn(struct rhashtable_compare_arg *arg, 87 const void *obj) 88 { 89 const struct ila_map *ila = obj; 90 91 return (ila->xp.ip.locator_match.v64 != *(__be64 *)arg->key); 92 } 93 94 static inline int ila_order(struct ila_map *ila) 95 { 96 int score = 0; 97 98 if (ila->xp.ifindex) 99 score += 1 << 1; 100 101 return score; 102 } 103 104 static const struct rhashtable_params rht_params = { 105 .nelem_hint = 1024, 106 .head_offset = offsetof(struct ila_map, node), 107 .key_offset = offsetof(struct ila_map, xp.ip.locator_match), 108 .key_len = sizeof(u64), /* identifier */ 109 .max_size = 1048576, 110 .min_size = 256, 111 .automatic_shrinking = true, 112 .obj_cmpfn = ila_cmpfn, 113 }; 114 115 static struct genl_family ila_nl_family; 116 117 static const struct nla_policy ila_nl_policy[ILA_ATTR_MAX + 1] = { 118 [ILA_ATTR_LOCATOR] = { .type = NLA_U64, }, 119 [ILA_ATTR_LOCATOR_MATCH] = { .type = NLA_U64, }, 120 [ILA_ATTR_IFINDEX] = { .type = NLA_U32, }, 121 [ILA_ATTR_CSUM_MODE] = { .type = NLA_U8, }, 122 }; 123 124 static int parse_nl_config(struct genl_info *info, 125 struct ila_xlat_params *xp) 126 { 127 memset(xp, 0, sizeof(*xp)); 128 129 if (info->attrs[ILA_ATTR_LOCATOR]) 130 xp->ip.locator.v64 = (__force __be64)nla_get_u64( 131 info->attrs[ILA_ATTR_LOCATOR]); 132 133 if (info->attrs[ILA_ATTR_LOCATOR_MATCH]) 134 xp->ip.locator_match.v64 = (__force __be64)nla_get_u64( 135 info->attrs[ILA_ATTR_LOCATOR_MATCH]); 136 137 if (info->attrs[ILA_ATTR_CSUM_MODE]) 138 xp->ip.csum_mode = nla_get_u8(info->attrs[ILA_ATTR_CSUM_MODE]); 139 140 if (info->attrs[ILA_ATTR_IFINDEX]) 141 xp->ifindex = nla_get_s32(info->attrs[ILA_ATTR_IFINDEX]); 142 143 return 0; 144 } 145 146 /* Must be called with rcu readlock */ 147 static inline struct ila_map *ila_lookup_wildcards(struct ila_addr *iaddr, 148 int ifindex, 149 struct ila_net *ilan) 150 { 151 struct ila_map *ila; 152 153 ila = rhashtable_lookup_fast(&ilan->rhash_table, &iaddr->loc, 154 rht_params); 155 while (ila) { 156 if (!ila_cmp_wildcards(ila, iaddr, ifindex)) 157 return ila; 158 ila = rcu_access_pointer(ila->next); 159 } 160 161 return NULL; 162 } 163 164 /* Must be called with rcu readlock */ 165 static inline struct ila_map *ila_lookup_by_params(struct ila_xlat_params *xp, 166 struct ila_net *ilan) 167 { 168 struct ila_map *ila; 169 170 ila = rhashtable_lookup_fast(&ilan->rhash_table, 171 &xp->ip.locator_match, 172 rht_params); 173 while (ila) { 174 if (!ila_cmp_params(ila, xp)) 175 return ila; 176 ila = rcu_access_pointer(ila->next); 177 } 178 179 return NULL; 180 } 181 182 static inline void ila_release(struct ila_map *ila) 183 { 184 kfree_rcu(ila, rcu); 185 } 186 187 static void ila_free_cb(void *ptr, void *arg) 188 { 189 struct ila_map *ila = (struct ila_map *)ptr, *next; 190 191 /* Assume rcu_readlock held */ 192 while (ila) { 193 next = rcu_access_pointer(ila->next); 194 ila_release(ila); 195 ila = next; 196 } 197 } 198 199 static int ila_xlat_addr(struct sk_buff *skb, bool set_csum_neutral); 200 201 static unsigned int 202 ila_nf_input(void *priv, 203 struct sk_buff *skb, 204 const struct nf_hook_state *state) 205 { 206 ila_xlat_addr(skb, false); 207 return NF_ACCEPT; 208 } 209 210 static struct nf_hook_ops ila_nf_hook_ops[] __read_mostly = { 211 { 212 .hook = ila_nf_input, 213 .pf = NFPROTO_IPV6, 214 .hooknum = NF_INET_PRE_ROUTING, 215 .priority = -1, 216 }, 217 }; 218 219 static int ila_add_mapping(struct net *net, struct ila_xlat_params *xp) 220 { 221 struct ila_net *ilan = net_generic(net, ila_net_id); 222 struct ila_map *ila, *head; 223 spinlock_t *lock = ila_get_lock(ilan, xp->ip.locator_match); 224 int err = 0, order; 225 226 if (!ilan->hooks_registered) { 227 /* We defer registering net hooks in the namespace until the 228 * first mapping is added. 229 */ 230 err = nf_register_net_hooks(net, ila_nf_hook_ops, 231 ARRAY_SIZE(ila_nf_hook_ops)); 232 if (err) 233 return err; 234 235 ilan->hooks_registered = true; 236 } 237 238 ila = kzalloc(sizeof(*ila), GFP_KERNEL); 239 if (!ila) 240 return -ENOMEM; 241 242 ila_init_saved_csum(&xp->ip); 243 244 ila->xp = *xp; 245 246 order = ila_order(ila); 247 248 spin_lock(lock); 249 250 head = rhashtable_lookup_fast(&ilan->rhash_table, 251 &xp->ip.locator_match, 252 rht_params); 253 if (!head) { 254 /* New entry for the rhash_table */ 255 err = rhashtable_lookup_insert_fast(&ilan->rhash_table, 256 &ila->node, rht_params); 257 } else { 258 struct ila_map *tila = head, *prev = NULL; 259 260 do { 261 if (!ila_cmp_params(tila, xp)) { 262 err = -EEXIST; 263 goto out; 264 } 265 266 if (order > ila_order(tila)) 267 break; 268 269 prev = tila; 270 tila = rcu_dereference_protected(tila->next, 271 lockdep_is_held(lock)); 272 } while (tila); 273 274 if (prev) { 275 /* Insert in sub list of head */ 276 RCU_INIT_POINTER(ila->next, tila); 277 rcu_assign_pointer(prev->next, ila); 278 } else { 279 /* Make this ila new head */ 280 RCU_INIT_POINTER(ila->next, head); 281 err = rhashtable_replace_fast(&ilan->rhash_table, 282 &head->node, 283 &ila->node, rht_params); 284 if (err) 285 goto out; 286 } 287 } 288 289 out: 290 spin_unlock(lock); 291 292 if (err) 293 kfree(ila); 294 295 return err; 296 } 297 298 static int ila_del_mapping(struct net *net, struct ila_xlat_params *xp) 299 { 300 struct ila_net *ilan = net_generic(net, ila_net_id); 301 struct ila_map *ila, *head, *prev; 302 spinlock_t *lock = ila_get_lock(ilan, xp->ip.locator_match); 303 int err = -ENOENT; 304 305 spin_lock(lock); 306 307 head = rhashtable_lookup_fast(&ilan->rhash_table, 308 &xp->ip.locator_match, rht_params); 309 ila = head; 310 311 prev = NULL; 312 313 while (ila) { 314 if (ila_cmp_params(ila, xp)) { 315 prev = ila; 316 ila = rcu_dereference_protected(ila->next, 317 lockdep_is_held(lock)); 318 continue; 319 } 320 321 err = 0; 322 323 if (prev) { 324 /* Not head, just delete from list */ 325 rcu_assign_pointer(prev->next, ila->next); 326 } else { 327 /* It is the head. If there is something in the 328 * sublist we need to make a new head. 329 */ 330 head = rcu_dereference_protected(ila->next, 331 lockdep_is_held(lock)); 332 if (head) { 333 /* Put first entry in the sublist into the 334 * table 335 */ 336 err = rhashtable_replace_fast( 337 &ilan->rhash_table, &ila->node, 338 &head->node, rht_params); 339 if (err) 340 goto out; 341 } else { 342 /* Entry no longer used */ 343 err = rhashtable_remove_fast(&ilan->rhash_table, 344 &ila->node, 345 rht_params); 346 } 347 } 348 349 ila_release(ila); 350 351 break; 352 } 353 354 out: 355 spin_unlock(lock); 356 357 return err; 358 } 359 360 static int ila_nl_cmd_add_mapping(struct sk_buff *skb, struct genl_info *info) 361 { 362 struct net *net = genl_info_net(info); 363 struct ila_xlat_params p; 364 int err; 365 366 err = parse_nl_config(info, &p); 367 if (err) 368 return err; 369 370 return ila_add_mapping(net, &p); 371 } 372 373 static int ila_nl_cmd_del_mapping(struct sk_buff *skb, struct genl_info *info) 374 { 375 struct net *net = genl_info_net(info); 376 struct ila_xlat_params xp; 377 int err; 378 379 err = parse_nl_config(info, &xp); 380 if (err) 381 return err; 382 383 ila_del_mapping(net, &xp); 384 385 return 0; 386 } 387 388 static int ila_fill_info(struct ila_map *ila, struct sk_buff *msg) 389 { 390 if (nla_put_u64_64bit(msg, ILA_ATTR_LOCATOR, 391 (__force u64)ila->xp.ip.locator.v64, 392 ILA_ATTR_PAD) || 393 nla_put_u64_64bit(msg, ILA_ATTR_LOCATOR_MATCH, 394 (__force u64)ila->xp.ip.locator_match.v64, 395 ILA_ATTR_PAD) || 396 nla_put_s32(msg, ILA_ATTR_IFINDEX, ila->xp.ifindex) || 397 nla_put_u32(msg, ILA_ATTR_CSUM_MODE, ila->xp.ip.csum_mode)) 398 return -1; 399 400 return 0; 401 } 402 403 static int ila_dump_info(struct ila_map *ila, 404 u32 portid, u32 seq, u32 flags, 405 struct sk_buff *skb, u8 cmd) 406 { 407 void *hdr; 408 409 hdr = genlmsg_put(skb, portid, seq, &ila_nl_family, flags, cmd); 410 if (!hdr) 411 return -ENOMEM; 412 413 if (ila_fill_info(ila, skb) < 0) 414 goto nla_put_failure; 415 416 genlmsg_end(skb, hdr); 417 return 0; 418 419 nla_put_failure: 420 genlmsg_cancel(skb, hdr); 421 return -EMSGSIZE; 422 } 423 424 static int ila_nl_cmd_get_mapping(struct sk_buff *skb, struct genl_info *info) 425 { 426 struct net *net = genl_info_net(info); 427 struct ila_net *ilan = net_generic(net, ila_net_id); 428 struct sk_buff *msg; 429 struct ila_xlat_params xp; 430 struct ila_map *ila; 431 int ret; 432 433 ret = parse_nl_config(info, &xp); 434 if (ret) 435 return ret; 436 437 msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL); 438 if (!msg) 439 return -ENOMEM; 440 441 rcu_read_lock(); 442 443 ila = ila_lookup_by_params(&xp, ilan); 444 if (ila) { 445 ret = ila_dump_info(ila, 446 info->snd_portid, 447 info->snd_seq, 0, msg, 448 info->genlhdr->cmd); 449 } 450 451 rcu_read_unlock(); 452 453 if (ret < 0) 454 goto out_free; 455 456 return genlmsg_reply(msg, info); 457 458 out_free: 459 nlmsg_free(msg); 460 return ret; 461 } 462 463 struct ila_dump_iter { 464 struct rhashtable_iter rhiter; 465 }; 466 467 static int ila_nl_dump_start(struct netlink_callback *cb) 468 { 469 struct net *net = sock_net(cb->skb->sk); 470 struct ila_net *ilan = net_generic(net, ila_net_id); 471 struct ila_dump_iter *iter = (struct ila_dump_iter *)cb->args[0]; 472 473 if (!iter) { 474 iter = kmalloc(sizeof(*iter), GFP_KERNEL); 475 if (!iter) 476 return -ENOMEM; 477 478 cb->args[0] = (long)iter; 479 } 480 481 return rhashtable_walk_init(&ilan->rhash_table, &iter->rhiter, 482 GFP_KERNEL); 483 } 484 485 static int ila_nl_dump_done(struct netlink_callback *cb) 486 { 487 struct ila_dump_iter *iter = (struct ila_dump_iter *)cb->args[0]; 488 489 rhashtable_walk_exit(&iter->rhiter); 490 491 kfree(iter); 492 493 return 0; 494 } 495 496 static int ila_nl_dump(struct sk_buff *skb, struct netlink_callback *cb) 497 { 498 struct ila_dump_iter *iter = (struct ila_dump_iter *)cb->args[0]; 499 struct rhashtable_iter *rhiter = &iter->rhiter; 500 struct ila_map *ila; 501 int ret; 502 503 ret = rhashtable_walk_start(rhiter); 504 if (ret && ret != -EAGAIN) 505 goto done; 506 507 for (;;) { 508 ila = rhashtable_walk_next(rhiter); 509 510 if (IS_ERR(ila)) { 511 if (PTR_ERR(ila) == -EAGAIN) 512 continue; 513 ret = PTR_ERR(ila); 514 goto done; 515 } else if (!ila) { 516 break; 517 } 518 519 while (ila) { 520 ret = ila_dump_info(ila, NETLINK_CB(cb->skb).portid, 521 cb->nlh->nlmsg_seq, NLM_F_MULTI, 522 skb, ILA_CMD_GET); 523 if (ret) 524 goto done; 525 526 ila = rcu_access_pointer(ila->next); 527 } 528 } 529 530 ret = skb->len; 531 532 done: 533 rhashtable_walk_stop(rhiter); 534 return ret; 535 } 536 537 static const struct genl_ops ila_nl_ops[] = { 538 { 539 .cmd = ILA_CMD_ADD, 540 .doit = ila_nl_cmd_add_mapping, 541 .policy = ila_nl_policy, 542 .flags = GENL_ADMIN_PERM, 543 }, 544 { 545 .cmd = ILA_CMD_DEL, 546 .doit = ila_nl_cmd_del_mapping, 547 .policy = ila_nl_policy, 548 .flags = GENL_ADMIN_PERM, 549 }, 550 { 551 .cmd = ILA_CMD_GET, 552 .doit = ila_nl_cmd_get_mapping, 553 .start = ila_nl_dump_start, 554 .dumpit = ila_nl_dump, 555 .done = ila_nl_dump_done, 556 .policy = ila_nl_policy, 557 }, 558 }; 559 560 static struct genl_family ila_nl_family __ro_after_init = { 561 .hdrsize = 0, 562 .name = ILA_GENL_NAME, 563 .version = ILA_GENL_VERSION, 564 .maxattr = ILA_ATTR_MAX, 565 .netnsok = true, 566 .parallel_ops = true, 567 .module = THIS_MODULE, 568 .ops = ila_nl_ops, 569 .n_ops = ARRAY_SIZE(ila_nl_ops), 570 }; 571 572 #define ILA_HASH_TABLE_SIZE 1024 573 574 static __net_init int ila_init_net(struct net *net) 575 { 576 int err; 577 struct ila_net *ilan = net_generic(net, ila_net_id); 578 579 err = alloc_ila_locks(ilan); 580 if (err) 581 return err; 582 583 rhashtable_init(&ilan->rhash_table, &rht_params); 584 585 return 0; 586 } 587 588 static __net_exit void ila_exit_net(struct net *net) 589 { 590 struct ila_net *ilan = net_generic(net, ila_net_id); 591 592 rhashtable_free_and_destroy(&ilan->rhash_table, ila_free_cb, NULL); 593 594 kvfree(ilan->locks); 595 596 if (ilan->hooks_registered) 597 nf_unregister_net_hooks(net, ila_nf_hook_ops, 598 ARRAY_SIZE(ila_nf_hook_ops)); 599 } 600 601 static struct pernet_operations ila_net_ops = { 602 .init = ila_init_net, 603 .exit = ila_exit_net, 604 .id = &ila_net_id, 605 .size = sizeof(struct ila_net), 606 }; 607 608 static int ila_xlat_addr(struct sk_buff *skb, bool set_csum_neutral) 609 { 610 struct ila_map *ila; 611 struct ipv6hdr *ip6h = ipv6_hdr(skb); 612 struct net *net = dev_net(skb->dev); 613 struct ila_net *ilan = net_generic(net, ila_net_id); 614 struct ila_addr *iaddr = ila_a2i(&ip6h->daddr); 615 616 /* Assumes skb contains a valid IPv6 header that is pulled */ 617 618 if (!ila_addr_is_ila(iaddr)) { 619 /* Type indicates this is not an ILA address */ 620 return 0; 621 } 622 623 rcu_read_lock(); 624 625 ila = ila_lookup_wildcards(iaddr, skb->dev->ifindex, ilan); 626 if (ila) 627 ila_update_ipv6_locator(skb, &ila->xp.ip, set_csum_neutral); 628 629 rcu_read_unlock(); 630 631 return 0; 632 } 633 634 int __init ila_xlat_init(void) 635 { 636 int ret; 637 638 ret = register_pernet_device(&ila_net_ops); 639 if (ret) 640 goto exit; 641 642 ret = genl_register_family(&ila_nl_family); 643 if (ret < 0) 644 goto unregister; 645 646 return 0; 647 648 unregister: 649 unregister_pernet_device(&ila_net_ops); 650 exit: 651 return ret; 652 } 653 654 void ila_xlat_fini(void) 655 { 656 genl_unregister_family(&ila_nl_family); 657 unregister_pernet_device(&ila_net_ops); 658 } 659