1 #include <linux/jhash.h> 2 #include <linux/netfilter.h> 3 #include <linux/rcupdate.h> 4 #include <linux/rhashtable.h> 5 #include <linux/vmalloc.h> 6 #include <net/genetlink.h> 7 #include <net/ila.h> 8 #include <net/netns/generic.h> 9 #include <uapi/linux/genetlink.h> 10 #include "ila.h" 11 12 struct ila_xlat_params { 13 struct ila_params ip; 14 int ifindex; 15 }; 16 17 struct ila_map { 18 struct ila_xlat_params xp; 19 struct rhash_head node; 20 struct ila_map __rcu *next; 21 struct rcu_head rcu; 22 }; 23 24 static unsigned int ila_net_id; 25 26 struct ila_net { 27 struct rhashtable rhash_table; 28 spinlock_t *locks; /* Bucket locks for entry manipulation */ 29 unsigned int locks_mask; 30 bool hooks_registered; 31 }; 32 33 #define LOCKS_PER_CPU 10 34 35 static int alloc_ila_locks(struct ila_net *ilan) 36 { 37 unsigned int i, size; 38 unsigned int nr_pcpus = num_possible_cpus(); 39 40 nr_pcpus = min_t(unsigned int, nr_pcpus, 32UL); 41 size = roundup_pow_of_two(nr_pcpus * LOCKS_PER_CPU); 42 43 if (sizeof(spinlock_t) != 0) { 44 #ifdef CONFIG_NUMA 45 if (size * sizeof(spinlock_t) > PAGE_SIZE) 46 ilan->locks = vmalloc(size * sizeof(spinlock_t)); 47 else 48 #endif 49 ilan->locks = kmalloc_array(size, sizeof(spinlock_t), 50 GFP_KERNEL); 51 if (!ilan->locks) 52 return -ENOMEM; 53 for (i = 0; i < size; i++) 54 spin_lock_init(&ilan->locks[i]); 55 } 56 ilan->locks_mask = size - 1; 57 58 return 0; 59 } 60 61 static u32 hashrnd __read_mostly; 62 static __always_inline void __ila_hash_secret_init(void) 63 { 64 net_get_random_once(&hashrnd, sizeof(hashrnd)); 65 } 66 67 static inline u32 ila_locator_hash(struct ila_locator loc) 68 { 69 u32 *v = (u32 *)loc.v32; 70 71 return jhash_2words(v[0], v[1], hashrnd); 72 } 73 74 static inline spinlock_t *ila_get_lock(struct ila_net *ilan, 75 struct ila_locator loc) 76 { 77 return &ilan->locks[ila_locator_hash(loc) & ilan->locks_mask]; 78 } 79 80 static inline int ila_cmp_wildcards(struct ila_map *ila, 81 struct ila_addr *iaddr, int ifindex) 82 { 83 return (ila->xp.ifindex && ila->xp.ifindex != ifindex); 84 } 85 86 static inline int ila_cmp_params(struct ila_map *ila, 87 struct ila_xlat_params *xp) 88 { 89 return (ila->xp.ifindex != xp->ifindex); 90 } 91 92 static int ila_cmpfn(struct rhashtable_compare_arg *arg, 93 const void *obj) 94 { 95 const struct ila_map *ila = obj; 96 97 return (ila->xp.ip.locator_match.v64 != *(__be64 *)arg->key); 98 } 99 100 static inline int ila_order(struct ila_map *ila) 101 { 102 int score = 0; 103 104 if (ila->xp.ifindex) 105 score += 1 << 1; 106 107 return score; 108 } 109 110 static const struct rhashtable_params rht_params = { 111 .nelem_hint = 1024, 112 .head_offset = offsetof(struct ila_map, node), 113 .key_offset = offsetof(struct ila_map, xp.ip.locator_match), 114 .key_len = sizeof(u64), /* identifier */ 115 .max_size = 1048576, 116 .min_size = 256, 117 .automatic_shrinking = true, 118 .obj_cmpfn = ila_cmpfn, 119 }; 120 121 static struct genl_family ila_nl_family = { 122 .id = GENL_ID_GENERATE, 123 .hdrsize = 0, 124 .name = ILA_GENL_NAME, 125 .version = ILA_GENL_VERSION, 126 .maxattr = ILA_ATTR_MAX, 127 .netnsok = true, 128 .parallel_ops = true, 129 }; 130 131 static struct nla_policy ila_nl_policy[ILA_ATTR_MAX + 1] = { 132 [ILA_ATTR_LOCATOR] = { .type = NLA_U64, }, 133 [ILA_ATTR_LOCATOR_MATCH] = { .type = NLA_U64, }, 134 [ILA_ATTR_IFINDEX] = { .type = NLA_U32, }, 135 [ILA_ATTR_CSUM_MODE] = { .type = NLA_U8, }, 136 }; 137 138 static int parse_nl_config(struct genl_info *info, 139 struct ila_xlat_params *xp) 140 { 141 memset(xp, 0, sizeof(*xp)); 142 143 if (info->attrs[ILA_ATTR_LOCATOR]) 144 xp->ip.locator.v64 = (__force __be64)nla_get_u64( 145 info->attrs[ILA_ATTR_LOCATOR]); 146 147 if (info->attrs[ILA_ATTR_LOCATOR_MATCH]) 148 xp->ip.locator_match.v64 = (__force __be64)nla_get_u64( 149 info->attrs[ILA_ATTR_LOCATOR_MATCH]); 150 151 if (info->attrs[ILA_ATTR_CSUM_MODE]) 152 xp->ip.csum_mode = nla_get_u8(info->attrs[ILA_ATTR_CSUM_MODE]); 153 154 if (info->attrs[ILA_ATTR_IFINDEX]) 155 xp->ifindex = nla_get_s32(info->attrs[ILA_ATTR_IFINDEX]); 156 157 return 0; 158 } 159 160 /* Must be called with rcu readlock */ 161 static inline struct ila_map *ila_lookup_wildcards(struct ila_addr *iaddr, 162 int ifindex, 163 struct ila_net *ilan) 164 { 165 struct ila_map *ila; 166 167 ila = rhashtable_lookup_fast(&ilan->rhash_table, &iaddr->loc, 168 rht_params); 169 while (ila) { 170 if (!ila_cmp_wildcards(ila, iaddr, ifindex)) 171 return ila; 172 ila = rcu_access_pointer(ila->next); 173 } 174 175 return NULL; 176 } 177 178 /* Must be called with rcu readlock */ 179 static inline struct ila_map *ila_lookup_by_params(struct ila_xlat_params *xp, 180 struct ila_net *ilan) 181 { 182 struct ila_map *ila; 183 184 ila = rhashtable_lookup_fast(&ilan->rhash_table, 185 &xp->ip.locator_match, 186 rht_params); 187 while (ila) { 188 if (!ila_cmp_params(ila, xp)) 189 return ila; 190 ila = rcu_access_pointer(ila->next); 191 } 192 193 return NULL; 194 } 195 196 static inline void ila_release(struct ila_map *ila) 197 { 198 kfree_rcu(ila, rcu); 199 } 200 201 static void ila_free_cb(void *ptr, void *arg) 202 { 203 struct ila_map *ila = (struct ila_map *)ptr, *next; 204 205 /* Assume rcu_readlock held */ 206 while (ila) { 207 next = rcu_access_pointer(ila->next); 208 ila_release(ila); 209 ila = next; 210 } 211 } 212 213 static int ila_xlat_addr(struct sk_buff *skb); 214 215 static unsigned int 216 ila_nf_input(void *priv, 217 struct sk_buff *skb, 218 const struct nf_hook_state *state) 219 { 220 ila_xlat_addr(skb); 221 return NF_ACCEPT; 222 } 223 224 static struct nf_hook_ops ila_nf_hook_ops[] __read_mostly = { 225 { 226 .hook = ila_nf_input, 227 .pf = NFPROTO_IPV6, 228 .hooknum = NF_INET_PRE_ROUTING, 229 .priority = -1, 230 }, 231 }; 232 233 static int ila_add_mapping(struct net *net, struct ila_xlat_params *xp) 234 { 235 struct ila_net *ilan = net_generic(net, ila_net_id); 236 struct ila_map *ila, *head; 237 spinlock_t *lock = ila_get_lock(ilan, xp->ip.locator_match); 238 int err = 0, order; 239 240 if (!ilan->hooks_registered) { 241 /* We defer registering net hooks in the namespace until the 242 * first mapping is added. 243 */ 244 err = nf_register_net_hooks(net, ila_nf_hook_ops, 245 ARRAY_SIZE(ila_nf_hook_ops)); 246 if (err) 247 return err; 248 249 ilan->hooks_registered = true; 250 } 251 252 ila = kzalloc(sizeof(*ila), GFP_KERNEL); 253 if (!ila) 254 return -ENOMEM; 255 256 ila_init_saved_csum(&xp->ip); 257 258 ila->xp = *xp; 259 260 order = ila_order(ila); 261 262 spin_lock(lock); 263 264 head = rhashtable_lookup_fast(&ilan->rhash_table, 265 &xp->ip.locator_match, 266 rht_params); 267 if (!head) { 268 /* New entry for the rhash_table */ 269 err = rhashtable_lookup_insert_fast(&ilan->rhash_table, 270 &ila->node, rht_params); 271 } else { 272 struct ila_map *tila = head, *prev = NULL; 273 274 do { 275 if (!ila_cmp_params(tila, xp)) { 276 err = -EEXIST; 277 goto out; 278 } 279 280 if (order > ila_order(tila)) 281 break; 282 283 prev = tila; 284 tila = rcu_dereference_protected(tila->next, 285 lockdep_is_held(lock)); 286 } while (tila); 287 288 if (prev) { 289 /* Insert in sub list of head */ 290 RCU_INIT_POINTER(ila->next, tila); 291 rcu_assign_pointer(prev->next, ila); 292 } else { 293 /* Make this ila new head */ 294 RCU_INIT_POINTER(ila->next, head); 295 err = rhashtable_replace_fast(&ilan->rhash_table, 296 &head->node, 297 &ila->node, rht_params); 298 if (err) 299 goto out; 300 } 301 } 302 303 out: 304 spin_unlock(lock); 305 306 if (err) 307 kfree(ila); 308 309 return err; 310 } 311 312 static int ila_del_mapping(struct net *net, struct ila_xlat_params *xp) 313 { 314 struct ila_net *ilan = net_generic(net, ila_net_id); 315 struct ila_map *ila, *head, *prev; 316 spinlock_t *lock = ila_get_lock(ilan, xp->ip.locator_match); 317 int err = -ENOENT; 318 319 spin_lock(lock); 320 321 head = rhashtable_lookup_fast(&ilan->rhash_table, 322 &xp->ip.locator_match, rht_params); 323 ila = head; 324 325 prev = NULL; 326 327 while (ila) { 328 if (ila_cmp_params(ila, xp)) { 329 prev = ila; 330 ila = rcu_dereference_protected(ila->next, 331 lockdep_is_held(lock)); 332 continue; 333 } 334 335 err = 0; 336 337 if (prev) { 338 /* Not head, just delete from list */ 339 rcu_assign_pointer(prev->next, ila->next); 340 } else { 341 /* It is the head. If there is something in the 342 * sublist we need to make a new head. 343 */ 344 head = rcu_dereference_protected(ila->next, 345 lockdep_is_held(lock)); 346 if (head) { 347 /* Put first entry in the sublist into the 348 * table 349 */ 350 err = rhashtable_replace_fast( 351 &ilan->rhash_table, &ila->node, 352 &head->node, rht_params); 353 if (err) 354 goto out; 355 } else { 356 /* Entry no longer used */ 357 err = rhashtable_remove_fast(&ilan->rhash_table, 358 &ila->node, 359 rht_params); 360 } 361 } 362 363 ila_release(ila); 364 365 break; 366 } 367 368 out: 369 spin_unlock(lock); 370 371 return err; 372 } 373 374 static int ila_nl_cmd_add_mapping(struct sk_buff *skb, struct genl_info *info) 375 { 376 struct net *net = genl_info_net(info); 377 struct ila_xlat_params p; 378 int err; 379 380 err = parse_nl_config(info, &p); 381 if (err) 382 return err; 383 384 return ila_add_mapping(net, &p); 385 } 386 387 static int ila_nl_cmd_del_mapping(struct sk_buff *skb, struct genl_info *info) 388 { 389 struct net *net = genl_info_net(info); 390 struct ila_xlat_params xp; 391 int err; 392 393 err = parse_nl_config(info, &xp); 394 if (err) 395 return err; 396 397 ila_del_mapping(net, &xp); 398 399 return 0; 400 } 401 402 static int ila_fill_info(struct ila_map *ila, struct sk_buff *msg) 403 { 404 if (nla_put_u64_64bit(msg, ILA_ATTR_LOCATOR, 405 (__force u64)ila->xp.ip.locator.v64, 406 ILA_ATTR_PAD) || 407 nla_put_u64_64bit(msg, ILA_ATTR_LOCATOR_MATCH, 408 (__force u64)ila->xp.ip.locator_match.v64, 409 ILA_ATTR_PAD) || 410 nla_put_s32(msg, ILA_ATTR_IFINDEX, ila->xp.ifindex) || 411 nla_put_u32(msg, ILA_ATTR_CSUM_MODE, ila->xp.ip.csum_mode)) 412 return -1; 413 414 return 0; 415 } 416 417 static int ila_dump_info(struct ila_map *ila, 418 u32 portid, u32 seq, u32 flags, 419 struct sk_buff *skb, u8 cmd) 420 { 421 void *hdr; 422 423 hdr = genlmsg_put(skb, portid, seq, &ila_nl_family, flags, cmd); 424 if (!hdr) 425 return -ENOMEM; 426 427 if (ila_fill_info(ila, skb) < 0) 428 goto nla_put_failure; 429 430 genlmsg_end(skb, hdr); 431 return 0; 432 433 nla_put_failure: 434 genlmsg_cancel(skb, hdr); 435 return -EMSGSIZE; 436 } 437 438 static int ila_nl_cmd_get_mapping(struct sk_buff *skb, struct genl_info *info) 439 { 440 struct net *net = genl_info_net(info); 441 struct ila_net *ilan = net_generic(net, ila_net_id); 442 struct sk_buff *msg; 443 struct ila_xlat_params xp; 444 struct ila_map *ila; 445 int ret; 446 447 ret = parse_nl_config(info, &xp); 448 if (ret) 449 return ret; 450 451 msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL); 452 if (!msg) 453 return -ENOMEM; 454 455 rcu_read_lock(); 456 457 ila = ila_lookup_by_params(&xp, ilan); 458 if (ila) { 459 ret = ila_dump_info(ila, 460 info->snd_portid, 461 info->snd_seq, 0, msg, 462 info->genlhdr->cmd); 463 } 464 465 rcu_read_unlock(); 466 467 if (ret < 0) 468 goto out_free; 469 470 return genlmsg_reply(msg, info); 471 472 out_free: 473 nlmsg_free(msg); 474 return ret; 475 } 476 477 struct ila_dump_iter { 478 struct rhashtable_iter rhiter; 479 }; 480 481 static int ila_nl_dump_start(struct netlink_callback *cb) 482 { 483 struct net *net = sock_net(cb->skb->sk); 484 struct ila_net *ilan = net_generic(net, ila_net_id); 485 struct ila_dump_iter *iter = (struct ila_dump_iter *)cb->args; 486 487 return rhashtable_walk_init(&ilan->rhash_table, &iter->rhiter, 488 GFP_KERNEL); 489 } 490 491 static int ila_nl_dump_done(struct netlink_callback *cb) 492 { 493 struct ila_dump_iter *iter = (struct ila_dump_iter *)cb->args; 494 495 rhashtable_walk_exit(&iter->rhiter); 496 497 return 0; 498 } 499 500 static int ila_nl_dump(struct sk_buff *skb, struct netlink_callback *cb) 501 { 502 struct ila_dump_iter *iter = (struct ila_dump_iter *)cb->args; 503 struct rhashtable_iter *rhiter = &iter->rhiter; 504 struct ila_map *ila; 505 int ret; 506 507 ret = rhashtable_walk_start(rhiter); 508 if (ret && ret != -EAGAIN) 509 goto done; 510 511 for (;;) { 512 ila = rhashtable_walk_next(rhiter); 513 514 if (IS_ERR(ila)) { 515 if (PTR_ERR(ila) == -EAGAIN) 516 continue; 517 ret = PTR_ERR(ila); 518 goto done; 519 } else if (!ila) { 520 break; 521 } 522 523 while (ila) { 524 ret = ila_dump_info(ila, NETLINK_CB(cb->skb).portid, 525 cb->nlh->nlmsg_seq, NLM_F_MULTI, 526 skb, ILA_CMD_GET); 527 if (ret) 528 goto done; 529 530 ila = rcu_access_pointer(ila->next); 531 } 532 } 533 534 ret = skb->len; 535 536 done: 537 rhashtable_walk_stop(rhiter); 538 return ret; 539 } 540 541 static const struct genl_ops ila_nl_ops[] = { 542 { 543 .cmd = ILA_CMD_ADD, 544 .doit = ila_nl_cmd_add_mapping, 545 .policy = ila_nl_policy, 546 .flags = GENL_ADMIN_PERM, 547 }, 548 { 549 .cmd = ILA_CMD_DEL, 550 .doit = ila_nl_cmd_del_mapping, 551 .policy = ila_nl_policy, 552 .flags = GENL_ADMIN_PERM, 553 }, 554 { 555 .cmd = ILA_CMD_GET, 556 .doit = ila_nl_cmd_get_mapping, 557 .start = ila_nl_dump_start, 558 .dumpit = ila_nl_dump, 559 .done = ila_nl_dump_done, 560 .policy = ila_nl_policy, 561 }, 562 }; 563 564 #define ILA_HASH_TABLE_SIZE 1024 565 566 static __net_init int ila_init_net(struct net *net) 567 { 568 int err; 569 struct ila_net *ilan = net_generic(net, ila_net_id); 570 571 err = alloc_ila_locks(ilan); 572 if (err) 573 return err; 574 575 rhashtable_init(&ilan->rhash_table, &rht_params); 576 577 return 0; 578 } 579 580 static __net_exit void ila_exit_net(struct net *net) 581 { 582 struct ila_net *ilan = net_generic(net, ila_net_id); 583 584 rhashtable_free_and_destroy(&ilan->rhash_table, ila_free_cb, NULL); 585 586 kvfree(ilan->locks); 587 588 if (ilan->hooks_registered) 589 nf_unregister_net_hooks(net, ila_nf_hook_ops, 590 ARRAY_SIZE(ila_nf_hook_ops)); 591 } 592 593 static struct pernet_operations ila_net_ops = { 594 .init = ila_init_net, 595 .exit = ila_exit_net, 596 .id = &ila_net_id, 597 .size = sizeof(struct ila_net), 598 }; 599 600 static int ila_xlat_addr(struct sk_buff *skb) 601 { 602 struct ila_map *ila; 603 struct ipv6hdr *ip6h = ipv6_hdr(skb); 604 struct net *net = dev_net(skb->dev); 605 struct ila_net *ilan = net_generic(net, ila_net_id); 606 struct ila_addr *iaddr = ila_a2i(&ip6h->daddr); 607 608 /* Assumes skb contains a valid IPv6 header that is pulled */ 609 610 if (!ila_addr_is_ila(iaddr)) { 611 /* Type indicates this is not an ILA address */ 612 return 0; 613 } 614 615 rcu_read_lock(); 616 617 ila = ila_lookup_wildcards(iaddr, skb->dev->ifindex, ilan); 618 if (ila) 619 ila_update_ipv6_locator(skb, &ila->xp.ip); 620 621 rcu_read_unlock(); 622 623 return 0; 624 } 625 626 int ila_xlat_init(void) 627 { 628 int ret; 629 630 ret = register_pernet_device(&ila_net_ops); 631 if (ret) 632 goto exit; 633 634 ret = genl_register_family_with_ops(&ila_nl_family, 635 ila_nl_ops); 636 if (ret < 0) 637 goto unregister; 638 639 return 0; 640 641 unregister: 642 unregister_pernet_device(&ila_net_ops); 643 exit: 644 return ret; 645 } 646 647 void ila_xlat_fini(void) 648 { 649 genl_unregister_family(&ila_nl_family); 650 unregister_pernet_device(&ila_net_ops); 651 } 652