1 #include <linux/jhash.h> 2 #include <linux/netfilter.h> 3 #include <linux/rcupdate.h> 4 #include <linux/rhashtable.h> 5 #include <linux/vmalloc.h> 6 #include <net/genetlink.h> 7 #include <net/ila.h> 8 #include <net/netns/generic.h> 9 #include <uapi/linux/genetlink.h> 10 #include "ila.h" 11 12 struct ila_xlat_params { 13 struct ila_params ip; 14 int ifindex; 15 }; 16 17 struct ila_map { 18 struct ila_xlat_params xp; 19 struct rhash_head node; 20 struct ila_map __rcu *next; 21 struct rcu_head rcu; 22 }; 23 24 static unsigned int ila_net_id; 25 26 struct ila_net { 27 struct rhashtable rhash_table; 28 spinlock_t *locks; /* Bucket locks for entry manipulation */ 29 unsigned int locks_mask; 30 bool hooks_registered; 31 }; 32 33 #define LOCKS_PER_CPU 10 34 35 static int alloc_ila_locks(struct ila_net *ilan) 36 { 37 unsigned int i, size; 38 unsigned int nr_pcpus = num_possible_cpus(); 39 40 nr_pcpus = min_t(unsigned int, nr_pcpus, 32UL); 41 size = roundup_pow_of_two(nr_pcpus * LOCKS_PER_CPU); 42 43 if (sizeof(spinlock_t) != 0) { 44 ilan->locks = kvmalloc(size * sizeof(spinlock_t), GFP_KERNEL); 45 if (!ilan->locks) 46 return -ENOMEM; 47 for (i = 0; i < size; i++) 48 spin_lock_init(&ilan->locks[i]); 49 } 50 ilan->locks_mask = size - 1; 51 52 return 0; 53 } 54 55 static u32 hashrnd __read_mostly; 56 static __always_inline void __ila_hash_secret_init(void) 57 { 58 net_get_random_once(&hashrnd, sizeof(hashrnd)); 59 } 60 61 static inline u32 ila_locator_hash(struct ila_locator loc) 62 { 63 u32 *v = (u32 *)loc.v32; 64 65 __ila_hash_secret_init(); 66 return jhash_2words(v[0], v[1], hashrnd); 67 } 68 69 static inline spinlock_t *ila_get_lock(struct ila_net *ilan, 70 struct ila_locator loc) 71 { 72 return &ilan->locks[ila_locator_hash(loc) & ilan->locks_mask]; 73 } 74 75 static inline int ila_cmp_wildcards(struct ila_map *ila, 76 struct ila_addr *iaddr, int ifindex) 77 { 78 return (ila->xp.ifindex && ila->xp.ifindex != ifindex); 79 } 80 81 static inline int ila_cmp_params(struct ila_map *ila, 82 struct ila_xlat_params *xp) 83 { 84 return (ila->xp.ifindex != xp->ifindex); 85 } 86 87 static int ila_cmpfn(struct rhashtable_compare_arg *arg, 88 const void *obj) 89 { 90 const struct ila_map *ila = obj; 91 92 return (ila->xp.ip.locator_match.v64 != *(__be64 *)arg->key); 93 } 94 95 static inline int ila_order(struct ila_map *ila) 96 { 97 int score = 0; 98 99 if (ila->xp.ifindex) 100 score += 1 << 1; 101 102 return score; 103 } 104 105 static const struct rhashtable_params rht_params = { 106 .nelem_hint = 1024, 107 .head_offset = offsetof(struct ila_map, node), 108 .key_offset = offsetof(struct ila_map, xp.ip.locator_match), 109 .key_len = sizeof(u64), /* identifier */ 110 .max_size = 1048576, 111 .min_size = 256, 112 .automatic_shrinking = true, 113 .obj_cmpfn = ila_cmpfn, 114 }; 115 116 static struct genl_family ila_nl_family; 117 118 static const struct nla_policy ila_nl_policy[ILA_ATTR_MAX + 1] = { 119 [ILA_ATTR_LOCATOR] = { .type = NLA_U64, }, 120 [ILA_ATTR_LOCATOR_MATCH] = { .type = NLA_U64, }, 121 [ILA_ATTR_IFINDEX] = { .type = NLA_U32, }, 122 [ILA_ATTR_CSUM_MODE] = { .type = NLA_U8, }, 123 }; 124 125 static int parse_nl_config(struct genl_info *info, 126 struct ila_xlat_params *xp) 127 { 128 memset(xp, 0, sizeof(*xp)); 129 130 if (info->attrs[ILA_ATTR_LOCATOR]) 131 xp->ip.locator.v64 = (__force __be64)nla_get_u64( 132 info->attrs[ILA_ATTR_LOCATOR]); 133 134 if (info->attrs[ILA_ATTR_LOCATOR_MATCH]) 135 xp->ip.locator_match.v64 = (__force __be64)nla_get_u64( 136 info->attrs[ILA_ATTR_LOCATOR_MATCH]); 137 138 if (info->attrs[ILA_ATTR_CSUM_MODE]) 139 xp->ip.csum_mode = nla_get_u8(info->attrs[ILA_ATTR_CSUM_MODE]); 140 141 if (info->attrs[ILA_ATTR_IFINDEX]) 142 xp->ifindex = nla_get_s32(info->attrs[ILA_ATTR_IFINDEX]); 143 144 return 0; 145 } 146 147 /* Must be called with rcu readlock */ 148 static inline struct ila_map *ila_lookup_wildcards(struct ila_addr *iaddr, 149 int ifindex, 150 struct ila_net *ilan) 151 { 152 struct ila_map *ila; 153 154 ila = rhashtable_lookup_fast(&ilan->rhash_table, &iaddr->loc, 155 rht_params); 156 while (ila) { 157 if (!ila_cmp_wildcards(ila, iaddr, ifindex)) 158 return ila; 159 ila = rcu_access_pointer(ila->next); 160 } 161 162 return NULL; 163 } 164 165 /* Must be called with rcu readlock */ 166 static inline struct ila_map *ila_lookup_by_params(struct ila_xlat_params *xp, 167 struct ila_net *ilan) 168 { 169 struct ila_map *ila; 170 171 ila = rhashtable_lookup_fast(&ilan->rhash_table, 172 &xp->ip.locator_match, 173 rht_params); 174 while (ila) { 175 if (!ila_cmp_params(ila, xp)) 176 return ila; 177 ila = rcu_access_pointer(ila->next); 178 } 179 180 return NULL; 181 } 182 183 static inline void ila_release(struct ila_map *ila) 184 { 185 kfree_rcu(ila, rcu); 186 } 187 188 static void ila_free_cb(void *ptr, void *arg) 189 { 190 struct ila_map *ila = (struct ila_map *)ptr, *next; 191 192 /* Assume rcu_readlock held */ 193 while (ila) { 194 next = rcu_access_pointer(ila->next); 195 ila_release(ila); 196 ila = next; 197 } 198 } 199 200 static int ila_xlat_addr(struct sk_buff *skb, bool set_csum_neutral); 201 202 static unsigned int 203 ila_nf_input(void *priv, 204 struct sk_buff *skb, 205 const struct nf_hook_state *state) 206 { 207 ila_xlat_addr(skb, false); 208 return NF_ACCEPT; 209 } 210 211 static struct nf_hook_ops ila_nf_hook_ops[] __read_mostly = { 212 { 213 .hook = ila_nf_input, 214 .pf = NFPROTO_IPV6, 215 .hooknum = NF_INET_PRE_ROUTING, 216 .priority = -1, 217 }, 218 }; 219 220 static int ila_add_mapping(struct net *net, struct ila_xlat_params *xp) 221 { 222 struct ila_net *ilan = net_generic(net, ila_net_id); 223 struct ila_map *ila, *head; 224 spinlock_t *lock = ila_get_lock(ilan, xp->ip.locator_match); 225 int err = 0, order; 226 227 if (!ilan->hooks_registered) { 228 /* We defer registering net hooks in the namespace until the 229 * first mapping is added. 230 */ 231 err = nf_register_net_hooks(net, ila_nf_hook_ops, 232 ARRAY_SIZE(ila_nf_hook_ops)); 233 if (err) 234 return err; 235 236 ilan->hooks_registered = true; 237 } 238 239 ila = kzalloc(sizeof(*ila), GFP_KERNEL); 240 if (!ila) 241 return -ENOMEM; 242 243 ila_init_saved_csum(&xp->ip); 244 245 ila->xp = *xp; 246 247 order = ila_order(ila); 248 249 spin_lock(lock); 250 251 head = rhashtable_lookup_fast(&ilan->rhash_table, 252 &xp->ip.locator_match, 253 rht_params); 254 if (!head) { 255 /* New entry for the rhash_table */ 256 err = rhashtable_lookup_insert_fast(&ilan->rhash_table, 257 &ila->node, rht_params); 258 } else { 259 struct ila_map *tila = head, *prev = NULL; 260 261 do { 262 if (!ila_cmp_params(tila, xp)) { 263 err = -EEXIST; 264 goto out; 265 } 266 267 if (order > ila_order(tila)) 268 break; 269 270 prev = tila; 271 tila = rcu_dereference_protected(tila->next, 272 lockdep_is_held(lock)); 273 } while (tila); 274 275 if (prev) { 276 /* Insert in sub list of head */ 277 RCU_INIT_POINTER(ila->next, tila); 278 rcu_assign_pointer(prev->next, ila); 279 } else { 280 /* Make this ila new head */ 281 RCU_INIT_POINTER(ila->next, head); 282 err = rhashtable_replace_fast(&ilan->rhash_table, 283 &head->node, 284 &ila->node, rht_params); 285 if (err) 286 goto out; 287 } 288 } 289 290 out: 291 spin_unlock(lock); 292 293 if (err) 294 kfree(ila); 295 296 return err; 297 } 298 299 static int ila_del_mapping(struct net *net, struct ila_xlat_params *xp) 300 { 301 struct ila_net *ilan = net_generic(net, ila_net_id); 302 struct ila_map *ila, *head, *prev; 303 spinlock_t *lock = ila_get_lock(ilan, xp->ip.locator_match); 304 int err = -ENOENT; 305 306 spin_lock(lock); 307 308 head = rhashtable_lookup_fast(&ilan->rhash_table, 309 &xp->ip.locator_match, rht_params); 310 ila = head; 311 312 prev = NULL; 313 314 while (ila) { 315 if (ila_cmp_params(ila, xp)) { 316 prev = ila; 317 ila = rcu_dereference_protected(ila->next, 318 lockdep_is_held(lock)); 319 continue; 320 } 321 322 err = 0; 323 324 if (prev) { 325 /* Not head, just delete from list */ 326 rcu_assign_pointer(prev->next, ila->next); 327 } else { 328 /* It is the head. If there is something in the 329 * sublist we need to make a new head. 330 */ 331 head = rcu_dereference_protected(ila->next, 332 lockdep_is_held(lock)); 333 if (head) { 334 /* Put first entry in the sublist into the 335 * table 336 */ 337 err = rhashtable_replace_fast( 338 &ilan->rhash_table, &ila->node, 339 &head->node, rht_params); 340 if (err) 341 goto out; 342 } else { 343 /* Entry no longer used */ 344 err = rhashtable_remove_fast(&ilan->rhash_table, 345 &ila->node, 346 rht_params); 347 } 348 } 349 350 ila_release(ila); 351 352 break; 353 } 354 355 out: 356 spin_unlock(lock); 357 358 return err; 359 } 360 361 static int ila_nl_cmd_add_mapping(struct sk_buff *skb, struct genl_info *info) 362 { 363 struct net *net = genl_info_net(info); 364 struct ila_xlat_params p; 365 int err; 366 367 err = parse_nl_config(info, &p); 368 if (err) 369 return err; 370 371 return ila_add_mapping(net, &p); 372 } 373 374 static int ila_nl_cmd_del_mapping(struct sk_buff *skb, struct genl_info *info) 375 { 376 struct net *net = genl_info_net(info); 377 struct ila_xlat_params xp; 378 int err; 379 380 err = parse_nl_config(info, &xp); 381 if (err) 382 return err; 383 384 ila_del_mapping(net, &xp); 385 386 return 0; 387 } 388 389 static int ila_fill_info(struct ila_map *ila, struct sk_buff *msg) 390 { 391 if (nla_put_u64_64bit(msg, ILA_ATTR_LOCATOR, 392 (__force u64)ila->xp.ip.locator.v64, 393 ILA_ATTR_PAD) || 394 nla_put_u64_64bit(msg, ILA_ATTR_LOCATOR_MATCH, 395 (__force u64)ila->xp.ip.locator_match.v64, 396 ILA_ATTR_PAD) || 397 nla_put_s32(msg, ILA_ATTR_IFINDEX, ila->xp.ifindex) || 398 nla_put_u32(msg, ILA_ATTR_CSUM_MODE, ila->xp.ip.csum_mode)) 399 return -1; 400 401 return 0; 402 } 403 404 static int ila_dump_info(struct ila_map *ila, 405 u32 portid, u32 seq, u32 flags, 406 struct sk_buff *skb, u8 cmd) 407 { 408 void *hdr; 409 410 hdr = genlmsg_put(skb, portid, seq, &ila_nl_family, flags, cmd); 411 if (!hdr) 412 return -ENOMEM; 413 414 if (ila_fill_info(ila, skb) < 0) 415 goto nla_put_failure; 416 417 genlmsg_end(skb, hdr); 418 return 0; 419 420 nla_put_failure: 421 genlmsg_cancel(skb, hdr); 422 return -EMSGSIZE; 423 } 424 425 static int ila_nl_cmd_get_mapping(struct sk_buff *skb, struct genl_info *info) 426 { 427 struct net *net = genl_info_net(info); 428 struct ila_net *ilan = net_generic(net, ila_net_id); 429 struct sk_buff *msg; 430 struct ila_xlat_params xp; 431 struct ila_map *ila; 432 int ret; 433 434 ret = parse_nl_config(info, &xp); 435 if (ret) 436 return ret; 437 438 msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL); 439 if (!msg) 440 return -ENOMEM; 441 442 rcu_read_lock(); 443 444 ila = ila_lookup_by_params(&xp, ilan); 445 if (ila) { 446 ret = ila_dump_info(ila, 447 info->snd_portid, 448 info->snd_seq, 0, msg, 449 info->genlhdr->cmd); 450 } 451 452 rcu_read_unlock(); 453 454 if (ret < 0) 455 goto out_free; 456 457 return genlmsg_reply(msg, info); 458 459 out_free: 460 nlmsg_free(msg); 461 return ret; 462 } 463 464 struct ila_dump_iter { 465 struct rhashtable_iter rhiter; 466 }; 467 468 static int ila_nl_dump_start(struct netlink_callback *cb) 469 { 470 struct net *net = sock_net(cb->skb->sk); 471 struct ila_net *ilan = net_generic(net, ila_net_id); 472 struct ila_dump_iter *iter = (struct ila_dump_iter *)cb->args[0]; 473 474 if (!iter) { 475 iter = kmalloc(sizeof(*iter), GFP_KERNEL); 476 if (!iter) 477 return -ENOMEM; 478 479 cb->args[0] = (long)iter; 480 } 481 482 return rhashtable_walk_init(&ilan->rhash_table, &iter->rhiter, 483 GFP_KERNEL); 484 } 485 486 static int ila_nl_dump_done(struct netlink_callback *cb) 487 { 488 struct ila_dump_iter *iter = (struct ila_dump_iter *)cb->args[0]; 489 490 rhashtable_walk_exit(&iter->rhiter); 491 492 kfree(iter); 493 494 return 0; 495 } 496 497 static int ila_nl_dump(struct sk_buff *skb, struct netlink_callback *cb) 498 { 499 struct ila_dump_iter *iter = (struct ila_dump_iter *)cb->args[0]; 500 struct rhashtable_iter *rhiter = &iter->rhiter; 501 struct ila_map *ila; 502 int ret; 503 504 ret = rhashtable_walk_start(rhiter); 505 if (ret && ret != -EAGAIN) 506 goto done; 507 508 for (;;) { 509 ila = rhashtable_walk_next(rhiter); 510 511 if (IS_ERR(ila)) { 512 if (PTR_ERR(ila) == -EAGAIN) 513 continue; 514 ret = PTR_ERR(ila); 515 goto done; 516 } else if (!ila) { 517 break; 518 } 519 520 while (ila) { 521 ret = ila_dump_info(ila, NETLINK_CB(cb->skb).portid, 522 cb->nlh->nlmsg_seq, NLM_F_MULTI, 523 skb, ILA_CMD_GET); 524 if (ret) 525 goto done; 526 527 ila = rcu_access_pointer(ila->next); 528 } 529 } 530 531 ret = skb->len; 532 533 done: 534 rhashtable_walk_stop(rhiter); 535 return ret; 536 } 537 538 static const struct genl_ops ila_nl_ops[] = { 539 { 540 .cmd = ILA_CMD_ADD, 541 .doit = ila_nl_cmd_add_mapping, 542 .policy = ila_nl_policy, 543 .flags = GENL_ADMIN_PERM, 544 }, 545 { 546 .cmd = ILA_CMD_DEL, 547 .doit = ila_nl_cmd_del_mapping, 548 .policy = ila_nl_policy, 549 .flags = GENL_ADMIN_PERM, 550 }, 551 { 552 .cmd = ILA_CMD_GET, 553 .doit = ila_nl_cmd_get_mapping, 554 .start = ila_nl_dump_start, 555 .dumpit = ila_nl_dump, 556 .done = ila_nl_dump_done, 557 .policy = ila_nl_policy, 558 }, 559 }; 560 561 static struct genl_family ila_nl_family __ro_after_init = { 562 .hdrsize = 0, 563 .name = ILA_GENL_NAME, 564 .version = ILA_GENL_VERSION, 565 .maxattr = ILA_ATTR_MAX, 566 .netnsok = true, 567 .parallel_ops = true, 568 .module = THIS_MODULE, 569 .ops = ila_nl_ops, 570 .n_ops = ARRAY_SIZE(ila_nl_ops), 571 }; 572 573 #define ILA_HASH_TABLE_SIZE 1024 574 575 static __net_init int ila_init_net(struct net *net) 576 { 577 int err; 578 struct ila_net *ilan = net_generic(net, ila_net_id); 579 580 err = alloc_ila_locks(ilan); 581 if (err) 582 return err; 583 584 rhashtable_init(&ilan->rhash_table, &rht_params); 585 586 return 0; 587 } 588 589 static __net_exit void ila_exit_net(struct net *net) 590 { 591 struct ila_net *ilan = net_generic(net, ila_net_id); 592 593 rhashtable_free_and_destroy(&ilan->rhash_table, ila_free_cb, NULL); 594 595 kvfree(ilan->locks); 596 597 if (ilan->hooks_registered) 598 nf_unregister_net_hooks(net, ila_nf_hook_ops, 599 ARRAY_SIZE(ila_nf_hook_ops)); 600 } 601 602 static struct pernet_operations ila_net_ops = { 603 .init = ila_init_net, 604 .exit = ila_exit_net, 605 .id = &ila_net_id, 606 .size = sizeof(struct ila_net), 607 }; 608 609 static int ila_xlat_addr(struct sk_buff *skb, bool set_csum_neutral) 610 { 611 struct ila_map *ila; 612 struct ipv6hdr *ip6h = ipv6_hdr(skb); 613 struct net *net = dev_net(skb->dev); 614 struct ila_net *ilan = net_generic(net, ila_net_id); 615 struct ila_addr *iaddr = ila_a2i(&ip6h->daddr); 616 617 /* Assumes skb contains a valid IPv6 header that is pulled */ 618 619 if (!ila_addr_is_ila(iaddr)) { 620 /* Type indicates this is not an ILA address */ 621 return 0; 622 } 623 624 rcu_read_lock(); 625 626 ila = ila_lookup_wildcards(iaddr, skb->dev->ifindex, ilan); 627 if (ila) 628 ila_update_ipv6_locator(skb, &ila->xp.ip, set_csum_neutral); 629 630 rcu_read_unlock(); 631 632 return 0; 633 } 634 635 int __init ila_xlat_init(void) 636 { 637 int ret; 638 639 ret = register_pernet_device(&ila_net_ops); 640 if (ret) 641 goto exit; 642 643 ret = genl_register_family(&ila_nl_family); 644 if (ret < 0) 645 goto unregister; 646 647 return 0; 648 649 unregister: 650 unregister_pernet_device(&ila_net_ops); 651 exit: 652 return ret; 653 } 654 655 void ila_xlat_fini(void) 656 { 657 genl_unregister_family(&ila_nl_family); 658 unregister_pernet_device(&ila_net_ops); 659 } 660