1 /* 2 * drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c 3 * Copyright (c) 2016-2017 Mellanox Technologies. All rights reserved. 4 * Copyright (c) 2016 Jiri Pirko <jiri@mellanox.com> 5 * Copyright (c) 2016 Ido Schimmel <idosch@mellanox.com> 6 * Copyright (c) 2016 Yotam Gigi <yotamg@mellanox.com> 7 * Copyright (c) 2017 Petr Machata <petrm@mellanox.com> 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions are met: 11 * 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 3. Neither the names of the copyright holders nor the names of its 18 * contributors may be used to endorse or promote products derived from 19 * this software without specific prior written permission. 20 * 21 * Alternatively, this software may be distributed under the terms of the 22 * GNU General Public License ("GPL") version 2 as published by the Free 23 * Software Foundation. 24 * 25 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 26 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 27 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 28 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 29 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 30 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 31 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 32 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 33 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 34 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 35 * POSSIBILITY OF SUCH DAMAGE. 36 */ 37 38 #include <linux/kernel.h> 39 #include <linux/types.h> 40 #include <linux/rhashtable.h> 41 #include <linux/bitops.h> 42 #include <linux/in6.h> 43 #include <linux/notifier.h> 44 #include <linux/inetdevice.h> 45 #include <linux/netdevice.h> 46 #include <linux/if_bridge.h> 47 #include <linux/socket.h> 48 #include <linux/route.h> 49 #include <linux/gcd.h> 50 #include <linux/random.h> 51 #include <net/netevent.h> 52 #include <net/neighbour.h> 53 #include <net/arp.h> 54 #include <net/ip_fib.h> 55 #include <net/ip6_fib.h> 56 #include <net/fib_rules.h> 57 #include <net/ip_tunnels.h> 58 #include <net/l3mdev.h> 59 #include <net/addrconf.h> 60 #include <net/ndisc.h> 61 #include <net/ipv6.h> 62 #include <net/fib_notifier.h> 63 64 #include "spectrum.h" 65 #include "core.h" 66 #include "reg.h" 67 #include "spectrum_cnt.h" 68 #include "spectrum_dpipe.h" 69 #include "spectrum_ipip.h" 70 #include "spectrum_mr.h" 71 #include "spectrum_mr_tcam.h" 72 #include "spectrum_router.h" 73 74 struct mlxsw_sp_vr; 75 struct mlxsw_sp_lpm_tree; 76 struct mlxsw_sp_rif_ops; 77 78 struct mlxsw_sp_router { 79 struct mlxsw_sp *mlxsw_sp; 80 struct mlxsw_sp_rif **rifs; 81 struct mlxsw_sp_vr *vrs; 82 struct rhashtable neigh_ht; 83 struct rhashtable nexthop_group_ht; 84 struct rhashtable nexthop_ht; 85 struct list_head nexthop_list; 86 struct { 87 struct mlxsw_sp_lpm_tree *trees; 88 unsigned int tree_count; 89 } lpm; 90 struct { 91 struct delayed_work dw; 92 unsigned long interval; /* ms */ 93 } neighs_update; 94 struct delayed_work nexthop_probe_dw; 95 #define MLXSW_SP_UNRESOLVED_NH_PROBE_INTERVAL 5000 /* ms */ 96 struct list_head nexthop_neighs_list; 97 struct list_head ipip_list; 98 bool aborted; 99 struct notifier_block fib_nb; 100 struct notifier_block netevent_nb; 101 const struct mlxsw_sp_rif_ops **rif_ops_arr; 102 const struct mlxsw_sp_ipip_ops **ipip_ops_arr; 103 }; 104 105 struct mlxsw_sp_rif { 106 struct list_head nexthop_list; 107 struct list_head neigh_list; 108 struct net_device *dev; 109 struct mlxsw_sp_fid *fid; 110 unsigned char addr[ETH_ALEN]; 111 int mtu; 112 u16 rif_index; 113 u16 vr_id; 114 const struct mlxsw_sp_rif_ops *ops; 115 struct mlxsw_sp *mlxsw_sp; 116 117 unsigned int counter_ingress; 118 bool counter_ingress_valid; 119 unsigned int counter_egress; 120 bool counter_egress_valid; 121 }; 122 123 struct mlxsw_sp_rif_params { 124 struct net_device *dev; 125 union { 126 u16 system_port; 127 u16 lag_id; 128 }; 129 u16 vid; 130 bool lag; 131 }; 132 133 struct mlxsw_sp_rif_subport { 134 struct mlxsw_sp_rif common; 135 union { 136 u16 system_port; 137 u16 lag_id; 138 }; 139 u16 vid; 140 bool lag; 141 }; 142 143 struct mlxsw_sp_rif_ipip_lb { 144 struct mlxsw_sp_rif common; 145 struct mlxsw_sp_rif_ipip_lb_config lb_config; 146 u16 ul_vr_id; /* Reserved for Spectrum-2. */ 147 }; 148 149 struct mlxsw_sp_rif_params_ipip_lb { 150 struct mlxsw_sp_rif_params common; 151 struct mlxsw_sp_rif_ipip_lb_config lb_config; 152 }; 153 154 struct mlxsw_sp_rif_ops { 155 enum mlxsw_sp_rif_type type; 156 size_t rif_size; 157 158 void (*setup)(struct mlxsw_sp_rif *rif, 159 const struct mlxsw_sp_rif_params *params); 160 int (*configure)(struct mlxsw_sp_rif *rif); 161 void (*deconfigure)(struct mlxsw_sp_rif *rif); 162 struct mlxsw_sp_fid * (*fid_get)(struct mlxsw_sp_rif *rif); 163 }; 164 165 static unsigned int * 166 mlxsw_sp_rif_p_counter_get(struct mlxsw_sp_rif *rif, 167 enum mlxsw_sp_rif_counter_dir dir) 168 { 169 switch (dir) { 170 case MLXSW_SP_RIF_COUNTER_EGRESS: 171 return &rif->counter_egress; 172 case MLXSW_SP_RIF_COUNTER_INGRESS: 173 return &rif->counter_ingress; 174 } 175 return NULL; 176 } 177 178 static bool 179 mlxsw_sp_rif_counter_valid_get(struct mlxsw_sp_rif *rif, 180 enum mlxsw_sp_rif_counter_dir dir) 181 { 182 switch (dir) { 183 case MLXSW_SP_RIF_COUNTER_EGRESS: 184 return rif->counter_egress_valid; 185 case MLXSW_SP_RIF_COUNTER_INGRESS: 186 return rif->counter_ingress_valid; 187 } 188 return false; 189 } 190 191 static void 192 mlxsw_sp_rif_counter_valid_set(struct mlxsw_sp_rif *rif, 193 enum mlxsw_sp_rif_counter_dir dir, 194 bool valid) 195 { 196 switch (dir) { 197 case MLXSW_SP_RIF_COUNTER_EGRESS: 198 rif->counter_egress_valid = valid; 199 break; 200 case MLXSW_SP_RIF_COUNTER_INGRESS: 201 rif->counter_ingress_valid = valid; 202 break; 203 } 204 } 205 206 static int mlxsw_sp_rif_counter_edit(struct mlxsw_sp *mlxsw_sp, u16 rif_index, 207 unsigned int counter_index, bool enable, 208 enum mlxsw_sp_rif_counter_dir dir) 209 { 210 char ritr_pl[MLXSW_REG_RITR_LEN]; 211 bool is_egress = false; 212 int err; 213 214 if (dir == MLXSW_SP_RIF_COUNTER_EGRESS) 215 is_egress = true; 216 mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index); 217 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 218 if (err) 219 return err; 220 221 mlxsw_reg_ritr_counter_pack(ritr_pl, counter_index, enable, 222 is_egress); 223 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 224 } 225 226 int mlxsw_sp_rif_counter_value_get(struct mlxsw_sp *mlxsw_sp, 227 struct mlxsw_sp_rif *rif, 228 enum mlxsw_sp_rif_counter_dir dir, u64 *cnt) 229 { 230 char ricnt_pl[MLXSW_REG_RICNT_LEN]; 231 unsigned int *p_counter_index; 232 bool valid; 233 int err; 234 235 valid = mlxsw_sp_rif_counter_valid_get(rif, dir); 236 if (!valid) 237 return -EINVAL; 238 239 p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir); 240 if (!p_counter_index) 241 return -EINVAL; 242 mlxsw_reg_ricnt_pack(ricnt_pl, *p_counter_index, 243 MLXSW_REG_RICNT_OPCODE_NOP); 244 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl); 245 if (err) 246 return err; 247 *cnt = mlxsw_reg_ricnt_good_unicast_packets_get(ricnt_pl); 248 return 0; 249 } 250 251 static int mlxsw_sp_rif_counter_clear(struct mlxsw_sp *mlxsw_sp, 252 unsigned int counter_index) 253 { 254 char ricnt_pl[MLXSW_REG_RICNT_LEN]; 255 256 mlxsw_reg_ricnt_pack(ricnt_pl, counter_index, 257 MLXSW_REG_RICNT_OPCODE_CLEAR); 258 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl); 259 } 260 261 int mlxsw_sp_rif_counter_alloc(struct mlxsw_sp *mlxsw_sp, 262 struct mlxsw_sp_rif *rif, 263 enum mlxsw_sp_rif_counter_dir dir) 264 { 265 unsigned int *p_counter_index; 266 int err; 267 268 p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir); 269 if (!p_counter_index) 270 return -EINVAL; 271 err = mlxsw_sp_counter_alloc(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF, 272 p_counter_index); 273 if (err) 274 return err; 275 276 err = mlxsw_sp_rif_counter_clear(mlxsw_sp, *p_counter_index); 277 if (err) 278 goto err_counter_clear; 279 280 err = mlxsw_sp_rif_counter_edit(mlxsw_sp, rif->rif_index, 281 *p_counter_index, true, dir); 282 if (err) 283 goto err_counter_edit; 284 mlxsw_sp_rif_counter_valid_set(rif, dir, true); 285 return 0; 286 287 err_counter_edit: 288 err_counter_clear: 289 mlxsw_sp_counter_free(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF, 290 *p_counter_index); 291 return err; 292 } 293 294 void mlxsw_sp_rif_counter_free(struct mlxsw_sp *mlxsw_sp, 295 struct mlxsw_sp_rif *rif, 296 enum mlxsw_sp_rif_counter_dir dir) 297 { 298 unsigned int *p_counter_index; 299 300 if (!mlxsw_sp_rif_counter_valid_get(rif, dir)) 301 return; 302 303 p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir); 304 if (WARN_ON(!p_counter_index)) 305 return; 306 mlxsw_sp_rif_counter_edit(mlxsw_sp, rif->rif_index, 307 *p_counter_index, false, dir); 308 mlxsw_sp_counter_free(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF, 309 *p_counter_index); 310 mlxsw_sp_rif_counter_valid_set(rif, dir, false); 311 } 312 313 static void mlxsw_sp_rif_counters_alloc(struct mlxsw_sp_rif *rif) 314 { 315 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 316 struct devlink *devlink; 317 318 devlink = priv_to_devlink(mlxsw_sp->core); 319 if (!devlink_dpipe_table_counter_enabled(devlink, 320 MLXSW_SP_DPIPE_TABLE_NAME_ERIF)) 321 return; 322 mlxsw_sp_rif_counter_alloc(mlxsw_sp, rif, MLXSW_SP_RIF_COUNTER_EGRESS); 323 } 324 325 static void mlxsw_sp_rif_counters_free(struct mlxsw_sp_rif *rif) 326 { 327 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 328 329 mlxsw_sp_rif_counter_free(mlxsw_sp, rif, MLXSW_SP_RIF_COUNTER_EGRESS); 330 } 331 332 static struct mlxsw_sp_rif * 333 mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp, 334 const struct net_device *dev); 335 336 #define MLXSW_SP_PREFIX_COUNT (sizeof(struct in6_addr) * BITS_PER_BYTE + 1) 337 338 struct mlxsw_sp_prefix_usage { 339 DECLARE_BITMAP(b, MLXSW_SP_PREFIX_COUNT); 340 }; 341 342 #define mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) \ 343 for_each_set_bit(prefix, (prefix_usage)->b, MLXSW_SP_PREFIX_COUNT) 344 345 static bool 346 mlxsw_sp_prefix_usage_eq(struct mlxsw_sp_prefix_usage *prefix_usage1, 347 struct mlxsw_sp_prefix_usage *prefix_usage2) 348 { 349 return !memcmp(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1)); 350 } 351 352 static bool 353 mlxsw_sp_prefix_usage_none(struct mlxsw_sp_prefix_usage *prefix_usage) 354 { 355 struct mlxsw_sp_prefix_usage prefix_usage_none = {{ 0 } }; 356 357 return mlxsw_sp_prefix_usage_eq(prefix_usage, &prefix_usage_none); 358 } 359 360 static void 361 mlxsw_sp_prefix_usage_cpy(struct mlxsw_sp_prefix_usage *prefix_usage1, 362 struct mlxsw_sp_prefix_usage *prefix_usage2) 363 { 364 memcpy(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1)); 365 } 366 367 static void 368 mlxsw_sp_prefix_usage_set(struct mlxsw_sp_prefix_usage *prefix_usage, 369 unsigned char prefix_len) 370 { 371 set_bit(prefix_len, prefix_usage->b); 372 } 373 374 static void 375 mlxsw_sp_prefix_usage_clear(struct mlxsw_sp_prefix_usage *prefix_usage, 376 unsigned char prefix_len) 377 { 378 clear_bit(prefix_len, prefix_usage->b); 379 } 380 381 struct mlxsw_sp_fib_key { 382 unsigned char addr[sizeof(struct in6_addr)]; 383 unsigned char prefix_len; 384 }; 385 386 enum mlxsw_sp_fib_entry_type { 387 MLXSW_SP_FIB_ENTRY_TYPE_REMOTE, 388 MLXSW_SP_FIB_ENTRY_TYPE_LOCAL, 389 MLXSW_SP_FIB_ENTRY_TYPE_TRAP, 390 391 /* This is a special case of local delivery, where a packet should be 392 * decapsulated on reception. Note that there is no corresponding ENCAP, 393 * because that's a type of next hop, not of FIB entry. (There can be 394 * several next hops in a REMOTE entry, and some of them may be 395 * encapsulating entries.) 396 */ 397 MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP, 398 }; 399 400 struct mlxsw_sp_nexthop_group; 401 struct mlxsw_sp_fib; 402 403 struct mlxsw_sp_fib_node { 404 struct list_head entry_list; 405 struct list_head list; 406 struct rhash_head ht_node; 407 struct mlxsw_sp_fib *fib; 408 struct mlxsw_sp_fib_key key; 409 }; 410 411 struct mlxsw_sp_fib_entry_decap { 412 struct mlxsw_sp_ipip_entry *ipip_entry; 413 u32 tunnel_index; 414 }; 415 416 struct mlxsw_sp_fib_entry { 417 struct list_head list; 418 struct mlxsw_sp_fib_node *fib_node; 419 enum mlxsw_sp_fib_entry_type type; 420 struct list_head nexthop_group_node; 421 struct mlxsw_sp_nexthop_group *nh_group; 422 struct mlxsw_sp_fib_entry_decap decap; /* Valid for decap entries. */ 423 }; 424 425 struct mlxsw_sp_fib4_entry { 426 struct mlxsw_sp_fib_entry common; 427 u32 tb_id; 428 u32 prio; 429 u8 tos; 430 u8 type; 431 }; 432 433 struct mlxsw_sp_fib6_entry { 434 struct mlxsw_sp_fib_entry common; 435 struct list_head rt6_list; 436 unsigned int nrt6; 437 }; 438 439 struct mlxsw_sp_rt6 { 440 struct list_head list; 441 struct rt6_info *rt; 442 }; 443 444 struct mlxsw_sp_lpm_tree { 445 u8 id; /* tree ID */ 446 unsigned int ref_count; 447 enum mlxsw_sp_l3proto proto; 448 struct mlxsw_sp_prefix_usage prefix_usage; 449 }; 450 451 struct mlxsw_sp_fib { 452 struct rhashtable ht; 453 struct list_head node_list; 454 struct mlxsw_sp_vr *vr; 455 struct mlxsw_sp_lpm_tree *lpm_tree; 456 unsigned long prefix_ref_count[MLXSW_SP_PREFIX_COUNT]; 457 struct mlxsw_sp_prefix_usage prefix_usage; 458 enum mlxsw_sp_l3proto proto; 459 }; 460 461 struct mlxsw_sp_vr { 462 u16 id; /* virtual router ID */ 463 u32 tb_id; /* kernel fib table id */ 464 unsigned int rif_count; 465 struct mlxsw_sp_fib *fib4; 466 struct mlxsw_sp_fib *fib6; 467 struct mlxsw_sp_mr_table *mr4_table; 468 }; 469 470 static const struct rhashtable_params mlxsw_sp_fib_ht_params; 471 472 static struct mlxsw_sp_fib *mlxsw_sp_fib_create(struct mlxsw_sp_vr *vr, 473 enum mlxsw_sp_l3proto proto) 474 { 475 struct mlxsw_sp_fib *fib; 476 int err; 477 478 fib = kzalloc(sizeof(*fib), GFP_KERNEL); 479 if (!fib) 480 return ERR_PTR(-ENOMEM); 481 err = rhashtable_init(&fib->ht, &mlxsw_sp_fib_ht_params); 482 if (err) 483 goto err_rhashtable_init; 484 INIT_LIST_HEAD(&fib->node_list); 485 fib->proto = proto; 486 fib->vr = vr; 487 return fib; 488 489 err_rhashtable_init: 490 kfree(fib); 491 return ERR_PTR(err); 492 } 493 494 static void mlxsw_sp_fib_destroy(struct mlxsw_sp_fib *fib) 495 { 496 WARN_ON(!list_empty(&fib->node_list)); 497 WARN_ON(fib->lpm_tree); 498 rhashtable_destroy(&fib->ht); 499 kfree(fib); 500 } 501 502 static struct mlxsw_sp_lpm_tree * 503 mlxsw_sp_lpm_tree_find_unused(struct mlxsw_sp *mlxsw_sp) 504 { 505 static struct mlxsw_sp_lpm_tree *lpm_tree; 506 int i; 507 508 for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) { 509 lpm_tree = &mlxsw_sp->router->lpm.trees[i]; 510 if (lpm_tree->ref_count == 0) 511 return lpm_tree; 512 } 513 return NULL; 514 } 515 516 static int mlxsw_sp_lpm_tree_alloc(struct mlxsw_sp *mlxsw_sp, 517 struct mlxsw_sp_lpm_tree *lpm_tree) 518 { 519 char ralta_pl[MLXSW_REG_RALTA_LEN]; 520 521 mlxsw_reg_ralta_pack(ralta_pl, true, 522 (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto, 523 lpm_tree->id); 524 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl); 525 } 526 527 static void mlxsw_sp_lpm_tree_free(struct mlxsw_sp *mlxsw_sp, 528 struct mlxsw_sp_lpm_tree *lpm_tree) 529 { 530 char ralta_pl[MLXSW_REG_RALTA_LEN]; 531 532 mlxsw_reg_ralta_pack(ralta_pl, false, 533 (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto, 534 lpm_tree->id); 535 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl); 536 } 537 538 static int 539 mlxsw_sp_lpm_tree_left_struct_set(struct mlxsw_sp *mlxsw_sp, 540 struct mlxsw_sp_prefix_usage *prefix_usage, 541 struct mlxsw_sp_lpm_tree *lpm_tree) 542 { 543 char ralst_pl[MLXSW_REG_RALST_LEN]; 544 u8 root_bin = 0; 545 u8 prefix; 546 u8 last_prefix = MLXSW_REG_RALST_BIN_NO_CHILD; 547 548 mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) 549 root_bin = prefix; 550 551 mlxsw_reg_ralst_pack(ralst_pl, root_bin, lpm_tree->id); 552 mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) { 553 if (prefix == 0) 554 continue; 555 mlxsw_reg_ralst_bin_pack(ralst_pl, prefix, last_prefix, 556 MLXSW_REG_RALST_BIN_NO_CHILD); 557 last_prefix = prefix; 558 } 559 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralst), ralst_pl); 560 } 561 562 static struct mlxsw_sp_lpm_tree * 563 mlxsw_sp_lpm_tree_create(struct mlxsw_sp *mlxsw_sp, 564 struct mlxsw_sp_prefix_usage *prefix_usage, 565 enum mlxsw_sp_l3proto proto) 566 { 567 struct mlxsw_sp_lpm_tree *lpm_tree; 568 int err; 569 570 lpm_tree = mlxsw_sp_lpm_tree_find_unused(mlxsw_sp); 571 if (!lpm_tree) 572 return ERR_PTR(-EBUSY); 573 lpm_tree->proto = proto; 574 err = mlxsw_sp_lpm_tree_alloc(mlxsw_sp, lpm_tree); 575 if (err) 576 return ERR_PTR(err); 577 578 err = mlxsw_sp_lpm_tree_left_struct_set(mlxsw_sp, prefix_usage, 579 lpm_tree); 580 if (err) 581 goto err_left_struct_set; 582 memcpy(&lpm_tree->prefix_usage, prefix_usage, 583 sizeof(lpm_tree->prefix_usage)); 584 return lpm_tree; 585 586 err_left_struct_set: 587 mlxsw_sp_lpm_tree_free(mlxsw_sp, lpm_tree); 588 return ERR_PTR(err); 589 } 590 591 static void mlxsw_sp_lpm_tree_destroy(struct mlxsw_sp *mlxsw_sp, 592 struct mlxsw_sp_lpm_tree *lpm_tree) 593 { 594 mlxsw_sp_lpm_tree_free(mlxsw_sp, lpm_tree); 595 } 596 597 static struct mlxsw_sp_lpm_tree * 598 mlxsw_sp_lpm_tree_get(struct mlxsw_sp *mlxsw_sp, 599 struct mlxsw_sp_prefix_usage *prefix_usage, 600 enum mlxsw_sp_l3proto proto) 601 { 602 struct mlxsw_sp_lpm_tree *lpm_tree; 603 int i; 604 605 for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) { 606 lpm_tree = &mlxsw_sp->router->lpm.trees[i]; 607 if (lpm_tree->ref_count != 0 && 608 lpm_tree->proto == proto && 609 mlxsw_sp_prefix_usage_eq(&lpm_tree->prefix_usage, 610 prefix_usage)) 611 return lpm_tree; 612 } 613 return mlxsw_sp_lpm_tree_create(mlxsw_sp, prefix_usage, proto); 614 } 615 616 static void mlxsw_sp_lpm_tree_hold(struct mlxsw_sp_lpm_tree *lpm_tree) 617 { 618 lpm_tree->ref_count++; 619 } 620 621 static void mlxsw_sp_lpm_tree_put(struct mlxsw_sp *mlxsw_sp, 622 struct mlxsw_sp_lpm_tree *lpm_tree) 623 { 624 if (--lpm_tree->ref_count == 0) 625 mlxsw_sp_lpm_tree_destroy(mlxsw_sp, lpm_tree); 626 } 627 628 #define MLXSW_SP_LPM_TREE_MIN 1 /* tree 0 is reserved */ 629 630 static int mlxsw_sp_lpm_init(struct mlxsw_sp *mlxsw_sp) 631 { 632 struct mlxsw_sp_lpm_tree *lpm_tree; 633 u64 max_trees; 634 int i; 635 636 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_LPM_TREES)) 637 return -EIO; 638 639 max_trees = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_LPM_TREES); 640 mlxsw_sp->router->lpm.tree_count = max_trees - MLXSW_SP_LPM_TREE_MIN; 641 mlxsw_sp->router->lpm.trees = kcalloc(mlxsw_sp->router->lpm.tree_count, 642 sizeof(struct mlxsw_sp_lpm_tree), 643 GFP_KERNEL); 644 if (!mlxsw_sp->router->lpm.trees) 645 return -ENOMEM; 646 647 for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) { 648 lpm_tree = &mlxsw_sp->router->lpm.trees[i]; 649 lpm_tree->id = i + MLXSW_SP_LPM_TREE_MIN; 650 } 651 652 return 0; 653 } 654 655 static void mlxsw_sp_lpm_fini(struct mlxsw_sp *mlxsw_sp) 656 { 657 kfree(mlxsw_sp->router->lpm.trees); 658 } 659 660 static bool mlxsw_sp_vr_is_used(const struct mlxsw_sp_vr *vr) 661 { 662 return !!vr->fib4 || !!vr->fib6 || !!vr->mr4_table; 663 } 664 665 static struct mlxsw_sp_vr *mlxsw_sp_vr_find_unused(struct mlxsw_sp *mlxsw_sp) 666 { 667 struct mlxsw_sp_vr *vr; 668 int i; 669 670 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) { 671 vr = &mlxsw_sp->router->vrs[i]; 672 if (!mlxsw_sp_vr_is_used(vr)) 673 return vr; 674 } 675 return NULL; 676 } 677 678 static int mlxsw_sp_vr_lpm_tree_bind(struct mlxsw_sp *mlxsw_sp, 679 const struct mlxsw_sp_fib *fib, u8 tree_id) 680 { 681 char raltb_pl[MLXSW_REG_RALTB_LEN]; 682 683 mlxsw_reg_raltb_pack(raltb_pl, fib->vr->id, 684 (enum mlxsw_reg_ralxx_protocol) fib->proto, 685 tree_id); 686 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), raltb_pl); 687 } 688 689 static int mlxsw_sp_vr_lpm_tree_unbind(struct mlxsw_sp *mlxsw_sp, 690 const struct mlxsw_sp_fib *fib) 691 { 692 char raltb_pl[MLXSW_REG_RALTB_LEN]; 693 694 /* Bind to tree 0 which is default */ 695 mlxsw_reg_raltb_pack(raltb_pl, fib->vr->id, 696 (enum mlxsw_reg_ralxx_protocol) fib->proto, 0); 697 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), raltb_pl); 698 } 699 700 static u32 mlxsw_sp_fix_tb_id(u32 tb_id) 701 { 702 /* For our purpose, squash main, default and local tables into one */ 703 if (tb_id == RT_TABLE_LOCAL || tb_id == RT_TABLE_DEFAULT) 704 tb_id = RT_TABLE_MAIN; 705 return tb_id; 706 } 707 708 static struct mlxsw_sp_vr *mlxsw_sp_vr_find(struct mlxsw_sp *mlxsw_sp, 709 u32 tb_id) 710 { 711 struct mlxsw_sp_vr *vr; 712 int i; 713 714 tb_id = mlxsw_sp_fix_tb_id(tb_id); 715 716 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) { 717 vr = &mlxsw_sp->router->vrs[i]; 718 if (mlxsw_sp_vr_is_used(vr) && vr->tb_id == tb_id) 719 return vr; 720 } 721 return NULL; 722 } 723 724 static struct mlxsw_sp_fib *mlxsw_sp_vr_fib(const struct mlxsw_sp_vr *vr, 725 enum mlxsw_sp_l3proto proto) 726 { 727 switch (proto) { 728 case MLXSW_SP_L3_PROTO_IPV4: 729 return vr->fib4; 730 case MLXSW_SP_L3_PROTO_IPV6: 731 return vr->fib6; 732 } 733 return NULL; 734 } 735 736 static struct mlxsw_sp_vr *mlxsw_sp_vr_create(struct mlxsw_sp *mlxsw_sp, 737 u32 tb_id, 738 struct netlink_ext_ack *extack) 739 { 740 struct mlxsw_sp_vr *vr; 741 int err; 742 743 vr = mlxsw_sp_vr_find_unused(mlxsw_sp); 744 if (!vr) { 745 NL_SET_ERR_MSG(extack, "spectrum: Exceeded number of supported virtual routers"); 746 return ERR_PTR(-EBUSY); 747 } 748 vr->fib4 = mlxsw_sp_fib_create(vr, MLXSW_SP_L3_PROTO_IPV4); 749 if (IS_ERR(vr->fib4)) 750 return ERR_CAST(vr->fib4); 751 vr->fib6 = mlxsw_sp_fib_create(vr, MLXSW_SP_L3_PROTO_IPV6); 752 if (IS_ERR(vr->fib6)) { 753 err = PTR_ERR(vr->fib6); 754 goto err_fib6_create; 755 } 756 vr->mr4_table = mlxsw_sp_mr_table_create(mlxsw_sp, vr->id, 757 MLXSW_SP_L3_PROTO_IPV4); 758 if (IS_ERR(vr->mr4_table)) { 759 err = PTR_ERR(vr->mr4_table); 760 goto err_mr_table_create; 761 } 762 vr->tb_id = tb_id; 763 return vr; 764 765 err_mr_table_create: 766 mlxsw_sp_fib_destroy(vr->fib6); 767 vr->fib6 = NULL; 768 err_fib6_create: 769 mlxsw_sp_fib_destroy(vr->fib4); 770 vr->fib4 = NULL; 771 return ERR_PTR(err); 772 } 773 774 static void mlxsw_sp_vr_destroy(struct mlxsw_sp_vr *vr) 775 { 776 mlxsw_sp_mr_table_destroy(vr->mr4_table); 777 vr->mr4_table = NULL; 778 mlxsw_sp_fib_destroy(vr->fib6); 779 vr->fib6 = NULL; 780 mlxsw_sp_fib_destroy(vr->fib4); 781 vr->fib4 = NULL; 782 } 783 784 static struct mlxsw_sp_vr *mlxsw_sp_vr_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id, 785 struct netlink_ext_ack *extack) 786 { 787 struct mlxsw_sp_vr *vr; 788 789 tb_id = mlxsw_sp_fix_tb_id(tb_id); 790 vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id); 791 if (!vr) 792 vr = mlxsw_sp_vr_create(mlxsw_sp, tb_id, extack); 793 return vr; 794 } 795 796 static void mlxsw_sp_vr_put(struct mlxsw_sp_vr *vr) 797 { 798 if (!vr->rif_count && list_empty(&vr->fib4->node_list) && 799 list_empty(&vr->fib6->node_list) && 800 mlxsw_sp_mr_table_empty(vr->mr4_table)) 801 mlxsw_sp_vr_destroy(vr); 802 } 803 804 static bool 805 mlxsw_sp_vr_lpm_tree_should_replace(struct mlxsw_sp_vr *vr, 806 enum mlxsw_sp_l3proto proto, u8 tree_id) 807 { 808 struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto); 809 810 if (!mlxsw_sp_vr_is_used(vr)) 811 return false; 812 if (fib->lpm_tree && fib->lpm_tree->id == tree_id) 813 return true; 814 return false; 815 } 816 817 static int mlxsw_sp_vr_lpm_tree_replace(struct mlxsw_sp *mlxsw_sp, 818 struct mlxsw_sp_fib *fib, 819 struct mlxsw_sp_lpm_tree *new_tree) 820 { 821 struct mlxsw_sp_lpm_tree *old_tree = fib->lpm_tree; 822 int err; 823 824 err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib, new_tree->id); 825 if (err) 826 return err; 827 fib->lpm_tree = new_tree; 828 mlxsw_sp_lpm_tree_hold(new_tree); 829 mlxsw_sp_lpm_tree_put(mlxsw_sp, old_tree); 830 return 0; 831 } 832 833 static int mlxsw_sp_vrs_lpm_tree_replace(struct mlxsw_sp *mlxsw_sp, 834 struct mlxsw_sp_fib *fib, 835 struct mlxsw_sp_lpm_tree *new_tree) 836 { 837 struct mlxsw_sp_lpm_tree *old_tree = fib->lpm_tree; 838 enum mlxsw_sp_l3proto proto = fib->proto; 839 u8 old_id, new_id = new_tree->id; 840 struct mlxsw_sp_vr *vr; 841 int i, err; 842 843 if (!old_tree) 844 goto no_replace; 845 old_id = old_tree->id; 846 847 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) { 848 vr = &mlxsw_sp->router->vrs[i]; 849 if (!mlxsw_sp_vr_lpm_tree_should_replace(vr, proto, old_id)) 850 continue; 851 err = mlxsw_sp_vr_lpm_tree_replace(mlxsw_sp, 852 mlxsw_sp_vr_fib(vr, proto), 853 new_tree); 854 if (err) 855 goto err_tree_replace; 856 } 857 858 return 0; 859 860 err_tree_replace: 861 for (i--; i >= 0; i--) { 862 if (!mlxsw_sp_vr_lpm_tree_should_replace(vr, proto, new_id)) 863 continue; 864 mlxsw_sp_vr_lpm_tree_replace(mlxsw_sp, 865 mlxsw_sp_vr_fib(vr, proto), 866 old_tree); 867 } 868 return err; 869 870 no_replace: 871 err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib, new_tree->id); 872 if (err) 873 return err; 874 fib->lpm_tree = new_tree; 875 mlxsw_sp_lpm_tree_hold(new_tree); 876 return 0; 877 } 878 879 static void 880 mlxsw_sp_vrs_prefixes(struct mlxsw_sp *mlxsw_sp, 881 enum mlxsw_sp_l3proto proto, 882 struct mlxsw_sp_prefix_usage *req_prefix_usage) 883 { 884 int i; 885 886 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) { 887 struct mlxsw_sp_vr *vr = &mlxsw_sp->router->vrs[i]; 888 struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto); 889 unsigned char prefix; 890 891 if (!mlxsw_sp_vr_is_used(vr)) 892 continue; 893 mlxsw_sp_prefix_usage_for_each(prefix, &fib->prefix_usage) 894 mlxsw_sp_prefix_usage_set(req_prefix_usage, prefix); 895 } 896 } 897 898 static int mlxsw_sp_vrs_init(struct mlxsw_sp *mlxsw_sp) 899 { 900 struct mlxsw_sp_vr *vr; 901 u64 max_vrs; 902 int i; 903 904 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_VRS)) 905 return -EIO; 906 907 max_vrs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); 908 mlxsw_sp->router->vrs = kcalloc(max_vrs, sizeof(struct mlxsw_sp_vr), 909 GFP_KERNEL); 910 if (!mlxsw_sp->router->vrs) 911 return -ENOMEM; 912 913 for (i = 0; i < max_vrs; i++) { 914 vr = &mlxsw_sp->router->vrs[i]; 915 vr->id = i; 916 } 917 918 return 0; 919 } 920 921 static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp); 922 923 static void mlxsw_sp_vrs_fini(struct mlxsw_sp *mlxsw_sp) 924 { 925 /* At this stage we're guaranteed not to have new incoming 926 * FIB notifications and the work queue is free from FIBs 927 * sitting on top of mlxsw netdevs. However, we can still 928 * have other FIBs queued. Flush the queue before flushing 929 * the device's tables. No need for locks, as we're the only 930 * writer. 931 */ 932 mlxsw_core_flush_owq(); 933 mlxsw_sp_router_fib_flush(mlxsw_sp); 934 kfree(mlxsw_sp->router->vrs); 935 } 936 937 static struct net_device * 938 __mlxsw_sp_ipip_netdev_ul_dev_get(const struct net_device *ol_dev) 939 { 940 struct ip_tunnel *tun = netdev_priv(ol_dev); 941 struct net *net = dev_net(ol_dev); 942 943 return __dev_get_by_index(net, tun->parms.link); 944 } 945 946 u32 mlxsw_sp_ipip_dev_ul_tb_id(const struct net_device *ol_dev) 947 { 948 struct net_device *d = __mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev); 949 950 if (d) 951 return l3mdev_fib_table(d) ? : RT_TABLE_MAIN; 952 else 953 return l3mdev_fib_table(ol_dev) ? : RT_TABLE_MAIN; 954 } 955 956 static struct mlxsw_sp_rif * 957 mlxsw_sp_rif_create(struct mlxsw_sp *mlxsw_sp, 958 const struct mlxsw_sp_rif_params *params, 959 struct netlink_ext_ack *extack); 960 961 static struct mlxsw_sp_rif_ipip_lb * 962 mlxsw_sp_ipip_ol_ipip_lb_create(struct mlxsw_sp *mlxsw_sp, 963 enum mlxsw_sp_ipip_type ipipt, 964 struct net_device *ol_dev, 965 struct netlink_ext_ack *extack) 966 { 967 struct mlxsw_sp_rif_params_ipip_lb lb_params; 968 const struct mlxsw_sp_ipip_ops *ipip_ops; 969 struct mlxsw_sp_rif *rif; 970 971 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipipt]; 972 lb_params = (struct mlxsw_sp_rif_params_ipip_lb) { 973 .common.dev = ol_dev, 974 .common.lag = false, 975 .lb_config = ipip_ops->ol_loopback_config(mlxsw_sp, ol_dev), 976 }; 977 978 rif = mlxsw_sp_rif_create(mlxsw_sp, &lb_params.common, extack); 979 if (IS_ERR(rif)) 980 return ERR_CAST(rif); 981 return container_of(rif, struct mlxsw_sp_rif_ipip_lb, common); 982 } 983 984 static struct mlxsw_sp_ipip_entry * 985 mlxsw_sp_ipip_entry_alloc(struct mlxsw_sp *mlxsw_sp, 986 enum mlxsw_sp_ipip_type ipipt, 987 struct net_device *ol_dev) 988 { 989 struct mlxsw_sp_ipip_entry *ipip_entry; 990 struct mlxsw_sp_ipip_entry *ret = NULL; 991 992 ipip_entry = kzalloc(sizeof(*ipip_entry), GFP_KERNEL); 993 if (!ipip_entry) 994 return ERR_PTR(-ENOMEM); 995 996 ipip_entry->ol_lb = mlxsw_sp_ipip_ol_ipip_lb_create(mlxsw_sp, ipipt, 997 ol_dev, NULL); 998 if (IS_ERR(ipip_entry->ol_lb)) { 999 ret = ERR_CAST(ipip_entry->ol_lb); 1000 goto err_ol_ipip_lb_create; 1001 } 1002 1003 ipip_entry->ipipt = ipipt; 1004 ipip_entry->ol_dev = ol_dev; 1005 ipip_entry->parms = mlxsw_sp_ipip_netdev_parms(ol_dev); 1006 1007 return ipip_entry; 1008 1009 err_ol_ipip_lb_create: 1010 kfree(ipip_entry); 1011 return ret; 1012 } 1013 1014 static void 1015 mlxsw_sp_ipip_entry_dealloc(struct mlxsw_sp_ipip_entry *ipip_entry) 1016 { 1017 mlxsw_sp_rif_destroy(&ipip_entry->ol_lb->common); 1018 kfree(ipip_entry); 1019 } 1020 1021 static bool 1022 mlxsw_sp_ipip_entry_saddr_matches(struct mlxsw_sp *mlxsw_sp, 1023 const enum mlxsw_sp_l3proto ul_proto, 1024 union mlxsw_sp_l3addr saddr, 1025 u32 ul_tb_id, 1026 struct mlxsw_sp_ipip_entry *ipip_entry) 1027 { 1028 u32 tun_ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ipip_entry->ol_dev); 1029 enum mlxsw_sp_ipip_type ipipt = ipip_entry->ipipt; 1030 union mlxsw_sp_l3addr tun_saddr; 1031 1032 if (mlxsw_sp->router->ipip_ops_arr[ipipt]->ul_proto != ul_proto) 1033 return false; 1034 1035 tun_saddr = mlxsw_sp_ipip_netdev_saddr(ul_proto, ipip_entry->ol_dev); 1036 return tun_ul_tb_id == ul_tb_id && 1037 mlxsw_sp_l3addr_eq(&tun_saddr, &saddr); 1038 } 1039 1040 static int 1041 mlxsw_sp_fib_entry_decap_init(struct mlxsw_sp *mlxsw_sp, 1042 struct mlxsw_sp_fib_entry *fib_entry, 1043 struct mlxsw_sp_ipip_entry *ipip_entry) 1044 { 1045 u32 tunnel_index; 1046 int err; 1047 1048 err = mlxsw_sp_kvdl_alloc(mlxsw_sp, 1, &tunnel_index); 1049 if (err) 1050 return err; 1051 1052 ipip_entry->decap_fib_entry = fib_entry; 1053 fib_entry->decap.ipip_entry = ipip_entry; 1054 fib_entry->decap.tunnel_index = tunnel_index; 1055 return 0; 1056 } 1057 1058 static void mlxsw_sp_fib_entry_decap_fini(struct mlxsw_sp *mlxsw_sp, 1059 struct mlxsw_sp_fib_entry *fib_entry) 1060 { 1061 /* Unlink this node from the IPIP entry that it's the decap entry of. */ 1062 fib_entry->decap.ipip_entry->decap_fib_entry = NULL; 1063 fib_entry->decap.ipip_entry = NULL; 1064 mlxsw_sp_kvdl_free(mlxsw_sp, fib_entry->decap.tunnel_index); 1065 } 1066 1067 static struct mlxsw_sp_fib_node * 1068 mlxsw_sp_fib_node_lookup(struct mlxsw_sp_fib *fib, const void *addr, 1069 size_t addr_len, unsigned char prefix_len); 1070 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp, 1071 struct mlxsw_sp_fib_entry *fib_entry); 1072 1073 static void 1074 mlxsw_sp_ipip_entry_demote_decap(struct mlxsw_sp *mlxsw_sp, 1075 struct mlxsw_sp_ipip_entry *ipip_entry) 1076 { 1077 struct mlxsw_sp_fib_entry *fib_entry = ipip_entry->decap_fib_entry; 1078 1079 mlxsw_sp_fib_entry_decap_fini(mlxsw_sp, fib_entry); 1080 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP; 1081 1082 mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry); 1083 } 1084 1085 static void 1086 mlxsw_sp_ipip_entry_promote_decap(struct mlxsw_sp *mlxsw_sp, 1087 struct mlxsw_sp_ipip_entry *ipip_entry, 1088 struct mlxsw_sp_fib_entry *decap_fib_entry) 1089 { 1090 if (mlxsw_sp_fib_entry_decap_init(mlxsw_sp, decap_fib_entry, 1091 ipip_entry)) 1092 return; 1093 decap_fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP; 1094 1095 if (mlxsw_sp_fib_entry_update(mlxsw_sp, decap_fib_entry)) 1096 mlxsw_sp_ipip_entry_demote_decap(mlxsw_sp, ipip_entry); 1097 } 1098 1099 /* Given an IPIP entry, find the corresponding decap route. */ 1100 static struct mlxsw_sp_fib_entry * 1101 mlxsw_sp_ipip_entry_find_decap(struct mlxsw_sp *mlxsw_sp, 1102 struct mlxsw_sp_ipip_entry *ipip_entry) 1103 { 1104 static struct mlxsw_sp_fib_node *fib_node; 1105 const struct mlxsw_sp_ipip_ops *ipip_ops; 1106 struct mlxsw_sp_fib_entry *fib_entry; 1107 unsigned char saddr_prefix_len; 1108 union mlxsw_sp_l3addr saddr; 1109 struct mlxsw_sp_fib *ul_fib; 1110 struct mlxsw_sp_vr *ul_vr; 1111 const void *saddrp; 1112 size_t saddr_len; 1113 u32 ul_tb_id; 1114 u32 saddr4; 1115 1116 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt]; 1117 1118 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ipip_entry->ol_dev); 1119 ul_vr = mlxsw_sp_vr_find(mlxsw_sp, ul_tb_id); 1120 if (!ul_vr) 1121 return NULL; 1122 1123 ul_fib = mlxsw_sp_vr_fib(ul_vr, ipip_ops->ul_proto); 1124 saddr = mlxsw_sp_ipip_netdev_saddr(ipip_ops->ul_proto, 1125 ipip_entry->ol_dev); 1126 1127 switch (ipip_ops->ul_proto) { 1128 case MLXSW_SP_L3_PROTO_IPV4: 1129 saddr4 = be32_to_cpu(saddr.addr4); 1130 saddrp = &saddr4; 1131 saddr_len = 4; 1132 saddr_prefix_len = 32; 1133 break; 1134 case MLXSW_SP_L3_PROTO_IPV6: 1135 WARN_ON(1); 1136 return NULL; 1137 } 1138 1139 fib_node = mlxsw_sp_fib_node_lookup(ul_fib, saddrp, saddr_len, 1140 saddr_prefix_len); 1141 if (!fib_node || list_empty(&fib_node->entry_list)) 1142 return NULL; 1143 1144 fib_entry = list_first_entry(&fib_node->entry_list, 1145 struct mlxsw_sp_fib_entry, list); 1146 if (fib_entry->type != MLXSW_SP_FIB_ENTRY_TYPE_TRAP) 1147 return NULL; 1148 1149 return fib_entry; 1150 } 1151 1152 static struct mlxsw_sp_ipip_entry * 1153 mlxsw_sp_ipip_entry_create(struct mlxsw_sp *mlxsw_sp, 1154 enum mlxsw_sp_ipip_type ipipt, 1155 struct net_device *ol_dev) 1156 { 1157 struct mlxsw_sp_ipip_entry *ipip_entry; 1158 1159 ipip_entry = mlxsw_sp_ipip_entry_alloc(mlxsw_sp, ipipt, ol_dev); 1160 if (IS_ERR(ipip_entry)) 1161 return ipip_entry; 1162 1163 list_add_tail(&ipip_entry->ipip_list_node, 1164 &mlxsw_sp->router->ipip_list); 1165 1166 return ipip_entry; 1167 } 1168 1169 static void 1170 mlxsw_sp_ipip_entry_destroy(struct mlxsw_sp *mlxsw_sp, 1171 struct mlxsw_sp_ipip_entry *ipip_entry) 1172 { 1173 list_del(&ipip_entry->ipip_list_node); 1174 mlxsw_sp_ipip_entry_dealloc(ipip_entry); 1175 } 1176 1177 static bool 1178 mlxsw_sp_ipip_entry_matches_decap(struct mlxsw_sp *mlxsw_sp, 1179 const struct net_device *ul_dev, 1180 enum mlxsw_sp_l3proto ul_proto, 1181 union mlxsw_sp_l3addr ul_dip, 1182 struct mlxsw_sp_ipip_entry *ipip_entry) 1183 { 1184 u32 ul_tb_id = l3mdev_fib_table(ul_dev) ? : RT_TABLE_MAIN; 1185 enum mlxsw_sp_ipip_type ipipt = ipip_entry->ipipt; 1186 struct net_device *ipip_ul_dev; 1187 1188 if (mlxsw_sp->router->ipip_ops_arr[ipipt]->ul_proto != ul_proto) 1189 return false; 1190 1191 ipip_ul_dev = __mlxsw_sp_ipip_netdev_ul_dev_get(ipip_entry->ol_dev); 1192 return mlxsw_sp_ipip_entry_saddr_matches(mlxsw_sp, ul_proto, ul_dip, 1193 ul_tb_id, ipip_entry) && 1194 (!ipip_ul_dev || ipip_ul_dev == ul_dev); 1195 } 1196 1197 /* Given decap parameters, find the corresponding IPIP entry. */ 1198 static struct mlxsw_sp_ipip_entry * 1199 mlxsw_sp_ipip_entry_find_by_decap(struct mlxsw_sp *mlxsw_sp, 1200 const struct net_device *ul_dev, 1201 enum mlxsw_sp_l3proto ul_proto, 1202 union mlxsw_sp_l3addr ul_dip) 1203 { 1204 struct mlxsw_sp_ipip_entry *ipip_entry; 1205 1206 list_for_each_entry(ipip_entry, &mlxsw_sp->router->ipip_list, 1207 ipip_list_node) 1208 if (mlxsw_sp_ipip_entry_matches_decap(mlxsw_sp, ul_dev, 1209 ul_proto, ul_dip, 1210 ipip_entry)) 1211 return ipip_entry; 1212 1213 return NULL; 1214 } 1215 1216 static bool mlxsw_sp_netdev_ipip_type(const struct mlxsw_sp *mlxsw_sp, 1217 const struct net_device *dev, 1218 enum mlxsw_sp_ipip_type *p_type) 1219 { 1220 struct mlxsw_sp_router *router = mlxsw_sp->router; 1221 const struct mlxsw_sp_ipip_ops *ipip_ops; 1222 enum mlxsw_sp_ipip_type ipipt; 1223 1224 for (ipipt = 0; ipipt < MLXSW_SP_IPIP_TYPE_MAX; ++ipipt) { 1225 ipip_ops = router->ipip_ops_arr[ipipt]; 1226 if (dev->type == ipip_ops->dev_type) { 1227 if (p_type) 1228 *p_type = ipipt; 1229 return true; 1230 } 1231 } 1232 return false; 1233 } 1234 1235 bool mlxsw_sp_netdev_is_ipip_ol(const struct mlxsw_sp *mlxsw_sp, 1236 const struct net_device *dev) 1237 { 1238 return mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, NULL); 1239 } 1240 1241 static struct mlxsw_sp_ipip_entry * 1242 mlxsw_sp_ipip_entry_find_by_ol_dev(struct mlxsw_sp *mlxsw_sp, 1243 const struct net_device *ol_dev) 1244 { 1245 struct mlxsw_sp_ipip_entry *ipip_entry; 1246 1247 list_for_each_entry(ipip_entry, &mlxsw_sp->router->ipip_list, 1248 ipip_list_node) 1249 if (ipip_entry->ol_dev == ol_dev) 1250 return ipip_entry; 1251 1252 return NULL; 1253 } 1254 1255 static struct mlxsw_sp_ipip_entry * 1256 mlxsw_sp_ipip_entry_find_by_ul_dev(const struct mlxsw_sp *mlxsw_sp, 1257 const struct net_device *ul_dev, 1258 struct mlxsw_sp_ipip_entry *start) 1259 { 1260 struct mlxsw_sp_ipip_entry *ipip_entry; 1261 1262 ipip_entry = list_prepare_entry(start, &mlxsw_sp->router->ipip_list, 1263 ipip_list_node); 1264 list_for_each_entry_continue(ipip_entry, &mlxsw_sp->router->ipip_list, 1265 ipip_list_node) { 1266 struct net_device *ipip_ul_dev = 1267 __mlxsw_sp_ipip_netdev_ul_dev_get(ipip_entry->ol_dev); 1268 1269 if (ipip_ul_dev == ul_dev) 1270 return ipip_entry; 1271 } 1272 1273 return NULL; 1274 } 1275 1276 bool mlxsw_sp_netdev_is_ipip_ul(const struct mlxsw_sp *mlxsw_sp, 1277 const struct net_device *dev) 1278 { 1279 return mlxsw_sp_ipip_entry_find_by_ul_dev(mlxsw_sp, dev, NULL); 1280 } 1281 1282 static bool mlxsw_sp_netdevice_ipip_can_offload(struct mlxsw_sp *mlxsw_sp, 1283 const struct net_device *ol_dev, 1284 enum mlxsw_sp_ipip_type ipipt) 1285 { 1286 const struct mlxsw_sp_ipip_ops *ops 1287 = mlxsw_sp->router->ipip_ops_arr[ipipt]; 1288 1289 /* For deciding whether decap should be offloaded, we don't care about 1290 * overlay protocol, so ask whether either one is supported. 1291 */ 1292 return ops->can_offload(mlxsw_sp, ol_dev, MLXSW_SP_L3_PROTO_IPV4) || 1293 ops->can_offload(mlxsw_sp, ol_dev, MLXSW_SP_L3_PROTO_IPV6); 1294 } 1295 1296 static int mlxsw_sp_netdevice_ipip_ol_reg_event(struct mlxsw_sp *mlxsw_sp, 1297 struct net_device *ol_dev) 1298 { 1299 struct mlxsw_sp_ipip_entry *ipip_entry; 1300 enum mlxsw_sp_l3proto ul_proto; 1301 enum mlxsw_sp_ipip_type ipipt; 1302 union mlxsw_sp_l3addr saddr; 1303 u32 ul_tb_id; 1304 1305 mlxsw_sp_netdev_ipip_type(mlxsw_sp, ol_dev, &ipipt); 1306 if (mlxsw_sp_netdevice_ipip_can_offload(mlxsw_sp, ol_dev, ipipt)) { 1307 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ol_dev); 1308 ul_proto = mlxsw_sp->router->ipip_ops_arr[ipipt]->ul_proto; 1309 saddr = mlxsw_sp_ipip_netdev_saddr(ul_proto, ol_dev); 1310 if (!mlxsw_sp_ipip_demote_tunnel_by_saddr(mlxsw_sp, ul_proto, 1311 saddr, ul_tb_id, 1312 NULL)) { 1313 ipip_entry = mlxsw_sp_ipip_entry_create(mlxsw_sp, ipipt, 1314 ol_dev); 1315 if (IS_ERR(ipip_entry)) 1316 return PTR_ERR(ipip_entry); 1317 } 1318 } 1319 1320 return 0; 1321 } 1322 1323 static void mlxsw_sp_netdevice_ipip_ol_unreg_event(struct mlxsw_sp *mlxsw_sp, 1324 struct net_device *ol_dev) 1325 { 1326 struct mlxsw_sp_ipip_entry *ipip_entry; 1327 1328 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev); 1329 if (ipip_entry) 1330 mlxsw_sp_ipip_entry_destroy(mlxsw_sp, ipip_entry); 1331 } 1332 1333 static void 1334 mlxsw_sp_ipip_entry_ol_up_event(struct mlxsw_sp *mlxsw_sp, 1335 struct mlxsw_sp_ipip_entry *ipip_entry) 1336 { 1337 struct mlxsw_sp_fib_entry *decap_fib_entry; 1338 1339 decap_fib_entry = mlxsw_sp_ipip_entry_find_decap(mlxsw_sp, ipip_entry); 1340 if (decap_fib_entry) 1341 mlxsw_sp_ipip_entry_promote_decap(mlxsw_sp, ipip_entry, 1342 decap_fib_entry); 1343 } 1344 1345 static void mlxsw_sp_netdevice_ipip_ol_up_event(struct mlxsw_sp *mlxsw_sp, 1346 struct net_device *ol_dev) 1347 { 1348 struct mlxsw_sp_ipip_entry *ipip_entry; 1349 1350 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev); 1351 if (ipip_entry) 1352 mlxsw_sp_ipip_entry_ol_up_event(mlxsw_sp, ipip_entry); 1353 } 1354 1355 static void 1356 mlxsw_sp_ipip_entry_ol_down_event(struct mlxsw_sp *mlxsw_sp, 1357 struct mlxsw_sp_ipip_entry *ipip_entry) 1358 { 1359 if (ipip_entry->decap_fib_entry) 1360 mlxsw_sp_ipip_entry_demote_decap(mlxsw_sp, ipip_entry); 1361 } 1362 1363 static void mlxsw_sp_netdevice_ipip_ol_down_event(struct mlxsw_sp *mlxsw_sp, 1364 struct net_device *ol_dev) 1365 { 1366 struct mlxsw_sp_ipip_entry *ipip_entry; 1367 1368 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev); 1369 if (ipip_entry) 1370 mlxsw_sp_ipip_entry_ol_down_event(mlxsw_sp, ipip_entry); 1371 } 1372 1373 static void mlxsw_sp_nexthop_rif_update(struct mlxsw_sp *mlxsw_sp, 1374 struct mlxsw_sp_rif *rif); 1375 static int 1376 mlxsw_sp_ipip_entry_ol_lb_update(struct mlxsw_sp *mlxsw_sp, 1377 struct mlxsw_sp_ipip_entry *ipip_entry, 1378 bool keep_encap, 1379 struct netlink_ext_ack *extack) 1380 { 1381 struct mlxsw_sp_rif_ipip_lb *old_lb_rif = ipip_entry->ol_lb; 1382 struct mlxsw_sp_rif_ipip_lb *new_lb_rif; 1383 1384 new_lb_rif = mlxsw_sp_ipip_ol_ipip_lb_create(mlxsw_sp, 1385 ipip_entry->ipipt, 1386 ipip_entry->ol_dev, 1387 extack); 1388 if (IS_ERR(new_lb_rif)) 1389 return PTR_ERR(new_lb_rif); 1390 ipip_entry->ol_lb = new_lb_rif; 1391 1392 if (keep_encap) { 1393 list_splice_init(&old_lb_rif->common.nexthop_list, 1394 &new_lb_rif->common.nexthop_list); 1395 mlxsw_sp_nexthop_rif_update(mlxsw_sp, &new_lb_rif->common); 1396 } 1397 1398 mlxsw_sp_rif_destroy(&old_lb_rif->common); 1399 1400 return 0; 1401 } 1402 1403 /** 1404 * Update the offload related to an IPIP entry. This always updates decap, and 1405 * in addition to that it also: 1406 * @recreate_loopback: recreates the associated loopback RIF 1407 * @keep_encap: updates next hops that use the tunnel netdevice. This is only 1408 * relevant when recreate_loopback is true. 1409 * @update_nexthops: updates next hops, keeping the current loopback RIF. This 1410 * is only relevant when recreate_loopback is false. 1411 */ 1412 int __mlxsw_sp_ipip_entry_update_tunnel(struct mlxsw_sp *mlxsw_sp, 1413 struct mlxsw_sp_ipip_entry *ipip_entry, 1414 bool recreate_loopback, 1415 bool keep_encap, 1416 bool update_nexthops, 1417 struct netlink_ext_ack *extack) 1418 { 1419 int err; 1420 1421 /* RIFs can't be edited, so to update loopback, we need to destroy and 1422 * recreate it. That creates a window of opportunity where RALUE and 1423 * RATR registers end up referencing a RIF that's already gone. RATRs 1424 * are handled in mlxsw_sp_ipip_entry_ol_lb_update(), and to take care 1425 * of RALUE, demote the decap route back. 1426 */ 1427 if (ipip_entry->decap_fib_entry) 1428 mlxsw_sp_ipip_entry_demote_decap(mlxsw_sp, ipip_entry); 1429 1430 if (recreate_loopback) { 1431 err = mlxsw_sp_ipip_entry_ol_lb_update(mlxsw_sp, ipip_entry, 1432 keep_encap, extack); 1433 if (err) 1434 return err; 1435 } else if (update_nexthops) { 1436 mlxsw_sp_nexthop_rif_update(mlxsw_sp, 1437 &ipip_entry->ol_lb->common); 1438 } 1439 1440 if (ipip_entry->ol_dev->flags & IFF_UP) 1441 mlxsw_sp_ipip_entry_ol_up_event(mlxsw_sp, ipip_entry); 1442 1443 return 0; 1444 } 1445 1446 static int mlxsw_sp_netdevice_ipip_ol_vrf_event(struct mlxsw_sp *mlxsw_sp, 1447 struct net_device *ol_dev, 1448 struct netlink_ext_ack *extack) 1449 { 1450 struct mlxsw_sp_ipip_entry *ipip_entry = 1451 mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev); 1452 1453 if (!ipip_entry) 1454 return 0; 1455 return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry, 1456 true, false, false, extack); 1457 } 1458 1459 static int 1460 mlxsw_sp_netdevice_ipip_ul_vrf_event(struct mlxsw_sp *mlxsw_sp, 1461 struct mlxsw_sp_ipip_entry *ipip_entry, 1462 struct net_device *ul_dev, 1463 struct netlink_ext_ack *extack) 1464 { 1465 return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry, 1466 true, true, false, extack); 1467 } 1468 1469 static int 1470 mlxsw_sp_netdevice_ipip_ul_up_event(struct mlxsw_sp *mlxsw_sp, 1471 struct mlxsw_sp_ipip_entry *ipip_entry, 1472 struct net_device *ul_dev) 1473 { 1474 return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry, 1475 false, false, true, NULL); 1476 } 1477 1478 static int 1479 mlxsw_sp_netdevice_ipip_ul_down_event(struct mlxsw_sp *mlxsw_sp, 1480 struct mlxsw_sp_ipip_entry *ipip_entry, 1481 struct net_device *ul_dev) 1482 { 1483 /* A down underlay device causes encapsulated packets to not be 1484 * forwarded, but decap still works. So refresh next hops without 1485 * touching anything else. 1486 */ 1487 return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry, 1488 false, false, true, NULL); 1489 } 1490 1491 static int 1492 mlxsw_sp_netdevice_ipip_ol_change_event(struct mlxsw_sp *mlxsw_sp, 1493 struct net_device *ol_dev, 1494 struct netlink_ext_ack *extack) 1495 { 1496 const struct mlxsw_sp_ipip_ops *ipip_ops; 1497 struct mlxsw_sp_ipip_entry *ipip_entry; 1498 int err; 1499 1500 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev); 1501 if (!ipip_entry) 1502 /* A change might make a tunnel eligible for offloading, but 1503 * that is currently not implemented. What falls to slow path 1504 * stays there. 1505 */ 1506 return 0; 1507 1508 /* A change might make a tunnel not eligible for offloading. */ 1509 if (!mlxsw_sp_netdevice_ipip_can_offload(mlxsw_sp, ol_dev, 1510 ipip_entry->ipipt)) { 1511 mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry); 1512 return 0; 1513 } 1514 1515 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt]; 1516 err = ipip_ops->ol_netdev_change(mlxsw_sp, ipip_entry, extack); 1517 return err; 1518 } 1519 1520 void mlxsw_sp_ipip_entry_demote_tunnel(struct mlxsw_sp *mlxsw_sp, 1521 struct mlxsw_sp_ipip_entry *ipip_entry) 1522 { 1523 struct net_device *ol_dev = ipip_entry->ol_dev; 1524 1525 if (ol_dev->flags & IFF_UP) 1526 mlxsw_sp_ipip_entry_ol_down_event(mlxsw_sp, ipip_entry); 1527 mlxsw_sp_ipip_entry_destroy(mlxsw_sp, ipip_entry); 1528 } 1529 1530 /* The configuration where several tunnels have the same local address in the 1531 * same underlay table needs special treatment in the HW. That is currently not 1532 * implemented in the driver. This function finds and demotes the first tunnel 1533 * with a given source address, except the one passed in in the argument 1534 * `except'. 1535 */ 1536 bool 1537 mlxsw_sp_ipip_demote_tunnel_by_saddr(struct mlxsw_sp *mlxsw_sp, 1538 enum mlxsw_sp_l3proto ul_proto, 1539 union mlxsw_sp_l3addr saddr, 1540 u32 ul_tb_id, 1541 const struct mlxsw_sp_ipip_entry *except) 1542 { 1543 struct mlxsw_sp_ipip_entry *ipip_entry, *tmp; 1544 1545 list_for_each_entry_safe(ipip_entry, tmp, &mlxsw_sp->router->ipip_list, 1546 ipip_list_node) { 1547 if (ipip_entry != except && 1548 mlxsw_sp_ipip_entry_saddr_matches(mlxsw_sp, ul_proto, saddr, 1549 ul_tb_id, ipip_entry)) { 1550 mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry); 1551 return true; 1552 } 1553 } 1554 1555 return false; 1556 } 1557 1558 static void mlxsw_sp_ipip_demote_tunnel_by_ul_netdev(struct mlxsw_sp *mlxsw_sp, 1559 struct net_device *ul_dev) 1560 { 1561 struct mlxsw_sp_ipip_entry *ipip_entry, *tmp; 1562 1563 list_for_each_entry_safe(ipip_entry, tmp, &mlxsw_sp->router->ipip_list, 1564 ipip_list_node) { 1565 struct net_device *ipip_ul_dev = 1566 __mlxsw_sp_ipip_netdev_ul_dev_get(ipip_entry->ol_dev); 1567 1568 if (ipip_ul_dev == ul_dev) 1569 mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry); 1570 } 1571 } 1572 1573 int mlxsw_sp_netdevice_ipip_ol_event(struct mlxsw_sp *mlxsw_sp, 1574 struct net_device *ol_dev, 1575 unsigned long event, 1576 struct netdev_notifier_info *info) 1577 { 1578 struct netdev_notifier_changeupper_info *chup; 1579 struct netlink_ext_ack *extack; 1580 1581 switch (event) { 1582 case NETDEV_REGISTER: 1583 return mlxsw_sp_netdevice_ipip_ol_reg_event(mlxsw_sp, ol_dev); 1584 case NETDEV_UNREGISTER: 1585 mlxsw_sp_netdevice_ipip_ol_unreg_event(mlxsw_sp, ol_dev); 1586 return 0; 1587 case NETDEV_UP: 1588 mlxsw_sp_netdevice_ipip_ol_up_event(mlxsw_sp, ol_dev); 1589 return 0; 1590 case NETDEV_DOWN: 1591 mlxsw_sp_netdevice_ipip_ol_down_event(mlxsw_sp, ol_dev); 1592 return 0; 1593 case NETDEV_CHANGEUPPER: 1594 chup = container_of(info, typeof(*chup), info); 1595 extack = info->extack; 1596 if (netif_is_l3_master(chup->upper_dev)) 1597 return mlxsw_sp_netdevice_ipip_ol_vrf_event(mlxsw_sp, 1598 ol_dev, 1599 extack); 1600 return 0; 1601 case NETDEV_CHANGE: 1602 extack = info->extack; 1603 return mlxsw_sp_netdevice_ipip_ol_change_event(mlxsw_sp, 1604 ol_dev, extack); 1605 } 1606 return 0; 1607 } 1608 1609 static int 1610 __mlxsw_sp_netdevice_ipip_ul_event(struct mlxsw_sp *mlxsw_sp, 1611 struct mlxsw_sp_ipip_entry *ipip_entry, 1612 struct net_device *ul_dev, 1613 unsigned long event, 1614 struct netdev_notifier_info *info) 1615 { 1616 struct netdev_notifier_changeupper_info *chup; 1617 struct netlink_ext_ack *extack; 1618 1619 switch (event) { 1620 case NETDEV_CHANGEUPPER: 1621 chup = container_of(info, typeof(*chup), info); 1622 extack = info->extack; 1623 if (netif_is_l3_master(chup->upper_dev)) 1624 return mlxsw_sp_netdevice_ipip_ul_vrf_event(mlxsw_sp, 1625 ipip_entry, 1626 ul_dev, 1627 extack); 1628 break; 1629 1630 case NETDEV_UP: 1631 return mlxsw_sp_netdevice_ipip_ul_up_event(mlxsw_sp, ipip_entry, 1632 ul_dev); 1633 case NETDEV_DOWN: 1634 return mlxsw_sp_netdevice_ipip_ul_down_event(mlxsw_sp, 1635 ipip_entry, 1636 ul_dev); 1637 } 1638 return 0; 1639 } 1640 1641 int 1642 mlxsw_sp_netdevice_ipip_ul_event(struct mlxsw_sp *mlxsw_sp, 1643 struct net_device *ul_dev, 1644 unsigned long event, 1645 struct netdev_notifier_info *info) 1646 { 1647 struct mlxsw_sp_ipip_entry *ipip_entry = NULL; 1648 int err; 1649 1650 while ((ipip_entry = mlxsw_sp_ipip_entry_find_by_ul_dev(mlxsw_sp, 1651 ul_dev, 1652 ipip_entry))) { 1653 err = __mlxsw_sp_netdevice_ipip_ul_event(mlxsw_sp, ipip_entry, 1654 ul_dev, event, info); 1655 if (err) { 1656 mlxsw_sp_ipip_demote_tunnel_by_ul_netdev(mlxsw_sp, 1657 ul_dev); 1658 return err; 1659 } 1660 } 1661 1662 return 0; 1663 } 1664 1665 struct mlxsw_sp_neigh_key { 1666 struct neighbour *n; 1667 }; 1668 1669 struct mlxsw_sp_neigh_entry { 1670 struct list_head rif_list_node; 1671 struct rhash_head ht_node; 1672 struct mlxsw_sp_neigh_key key; 1673 u16 rif; 1674 bool connected; 1675 unsigned char ha[ETH_ALEN]; 1676 struct list_head nexthop_list; /* list of nexthops using 1677 * this neigh entry 1678 */ 1679 struct list_head nexthop_neighs_list_node; 1680 unsigned int counter_index; 1681 bool counter_valid; 1682 }; 1683 1684 static const struct rhashtable_params mlxsw_sp_neigh_ht_params = { 1685 .key_offset = offsetof(struct mlxsw_sp_neigh_entry, key), 1686 .head_offset = offsetof(struct mlxsw_sp_neigh_entry, ht_node), 1687 .key_len = sizeof(struct mlxsw_sp_neigh_key), 1688 }; 1689 1690 struct mlxsw_sp_neigh_entry * 1691 mlxsw_sp_rif_neigh_next(struct mlxsw_sp_rif *rif, 1692 struct mlxsw_sp_neigh_entry *neigh_entry) 1693 { 1694 if (!neigh_entry) { 1695 if (list_empty(&rif->neigh_list)) 1696 return NULL; 1697 else 1698 return list_first_entry(&rif->neigh_list, 1699 typeof(*neigh_entry), 1700 rif_list_node); 1701 } 1702 if (list_is_last(&neigh_entry->rif_list_node, &rif->neigh_list)) 1703 return NULL; 1704 return list_next_entry(neigh_entry, rif_list_node); 1705 } 1706 1707 int mlxsw_sp_neigh_entry_type(struct mlxsw_sp_neigh_entry *neigh_entry) 1708 { 1709 return neigh_entry->key.n->tbl->family; 1710 } 1711 1712 unsigned char * 1713 mlxsw_sp_neigh_entry_ha(struct mlxsw_sp_neigh_entry *neigh_entry) 1714 { 1715 return neigh_entry->ha; 1716 } 1717 1718 u32 mlxsw_sp_neigh4_entry_dip(struct mlxsw_sp_neigh_entry *neigh_entry) 1719 { 1720 struct neighbour *n; 1721 1722 n = neigh_entry->key.n; 1723 return ntohl(*((__be32 *) n->primary_key)); 1724 } 1725 1726 struct in6_addr * 1727 mlxsw_sp_neigh6_entry_dip(struct mlxsw_sp_neigh_entry *neigh_entry) 1728 { 1729 struct neighbour *n; 1730 1731 n = neigh_entry->key.n; 1732 return (struct in6_addr *) &n->primary_key; 1733 } 1734 1735 int mlxsw_sp_neigh_counter_get(struct mlxsw_sp *mlxsw_sp, 1736 struct mlxsw_sp_neigh_entry *neigh_entry, 1737 u64 *p_counter) 1738 { 1739 if (!neigh_entry->counter_valid) 1740 return -EINVAL; 1741 1742 return mlxsw_sp_flow_counter_get(mlxsw_sp, neigh_entry->counter_index, 1743 p_counter, NULL); 1744 } 1745 1746 static struct mlxsw_sp_neigh_entry * 1747 mlxsw_sp_neigh_entry_alloc(struct mlxsw_sp *mlxsw_sp, struct neighbour *n, 1748 u16 rif) 1749 { 1750 struct mlxsw_sp_neigh_entry *neigh_entry; 1751 1752 neigh_entry = kzalloc(sizeof(*neigh_entry), GFP_KERNEL); 1753 if (!neigh_entry) 1754 return NULL; 1755 1756 neigh_entry->key.n = n; 1757 neigh_entry->rif = rif; 1758 INIT_LIST_HEAD(&neigh_entry->nexthop_list); 1759 1760 return neigh_entry; 1761 } 1762 1763 static void mlxsw_sp_neigh_entry_free(struct mlxsw_sp_neigh_entry *neigh_entry) 1764 { 1765 kfree(neigh_entry); 1766 } 1767 1768 static int 1769 mlxsw_sp_neigh_entry_insert(struct mlxsw_sp *mlxsw_sp, 1770 struct mlxsw_sp_neigh_entry *neigh_entry) 1771 { 1772 return rhashtable_insert_fast(&mlxsw_sp->router->neigh_ht, 1773 &neigh_entry->ht_node, 1774 mlxsw_sp_neigh_ht_params); 1775 } 1776 1777 static void 1778 mlxsw_sp_neigh_entry_remove(struct mlxsw_sp *mlxsw_sp, 1779 struct mlxsw_sp_neigh_entry *neigh_entry) 1780 { 1781 rhashtable_remove_fast(&mlxsw_sp->router->neigh_ht, 1782 &neigh_entry->ht_node, 1783 mlxsw_sp_neigh_ht_params); 1784 } 1785 1786 static bool 1787 mlxsw_sp_neigh_counter_should_alloc(struct mlxsw_sp *mlxsw_sp, 1788 struct mlxsw_sp_neigh_entry *neigh_entry) 1789 { 1790 struct devlink *devlink; 1791 const char *table_name; 1792 1793 switch (mlxsw_sp_neigh_entry_type(neigh_entry)) { 1794 case AF_INET: 1795 table_name = MLXSW_SP_DPIPE_TABLE_NAME_HOST4; 1796 break; 1797 case AF_INET6: 1798 table_name = MLXSW_SP_DPIPE_TABLE_NAME_HOST6; 1799 break; 1800 default: 1801 WARN_ON(1); 1802 return false; 1803 } 1804 1805 devlink = priv_to_devlink(mlxsw_sp->core); 1806 return devlink_dpipe_table_counter_enabled(devlink, table_name); 1807 } 1808 1809 static void 1810 mlxsw_sp_neigh_counter_alloc(struct mlxsw_sp *mlxsw_sp, 1811 struct mlxsw_sp_neigh_entry *neigh_entry) 1812 { 1813 if (!mlxsw_sp_neigh_counter_should_alloc(mlxsw_sp, neigh_entry)) 1814 return; 1815 1816 if (mlxsw_sp_flow_counter_alloc(mlxsw_sp, &neigh_entry->counter_index)) 1817 return; 1818 1819 neigh_entry->counter_valid = true; 1820 } 1821 1822 static void 1823 mlxsw_sp_neigh_counter_free(struct mlxsw_sp *mlxsw_sp, 1824 struct mlxsw_sp_neigh_entry *neigh_entry) 1825 { 1826 if (!neigh_entry->counter_valid) 1827 return; 1828 mlxsw_sp_flow_counter_free(mlxsw_sp, 1829 neigh_entry->counter_index); 1830 neigh_entry->counter_valid = false; 1831 } 1832 1833 static struct mlxsw_sp_neigh_entry * 1834 mlxsw_sp_neigh_entry_create(struct mlxsw_sp *mlxsw_sp, struct neighbour *n) 1835 { 1836 struct mlxsw_sp_neigh_entry *neigh_entry; 1837 struct mlxsw_sp_rif *rif; 1838 int err; 1839 1840 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, n->dev); 1841 if (!rif) 1842 return ERR_PTR(-EINVAL); 1843 1844 neigh_entry = mlxsw_sp_neigh_entry_alloc(mlxsw_sp, n, rif->rif_index); 1845 if (!neigh_entry) 1846 return ERR_PTR(-ENOMEM); 1847 1848 err = mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry); 1849 if (err) 1850 goto err_neigh_entry_insert; 1851 1852 mlxsw_sp_neigh_counter_alloc(mlxsw_sp, neigh_entry); 1853 list_add(&neigh_entry->rif_list_node, &rif->neigh_list); 1854 1855 return neigh_entry; 1856 1857 err_neigh_entry_insert: 1858 mlxsw_sp_neigh_entry_free(neigh_entry); 1859 return ERR_PTR(err); 1860 } 1861 1862 static void 1863 mlxsw_sp_neigh_entry_destroy(struct mlxsw_sp *mlxsw_sp, 1864 struct mlxsw_sp_neigh_entry *neigh_entry) 1865 { 1866 list_del(&neigh_entry->rif_list_node); 1867 mlxsw_sp_neigh_counter_free(mlxsw_sp, neigh_entry); 1868 mlxsw_sp_neigh_entry_remove(mlxsw_sp, neigh_entry); 1869 mlxsw_sp_neigh_entry_free(neigh_entry); 1870 } 1871 1872 static struct mlxsw_sp_neigh_entry * 1873 mlxsw_sp_neigh_entry_lookup(struct mlxsw_sp *mlxsw_sp, struct neighbour *n) 1874 { 1875 struct mlxsw_sp_neigh_key key; 1876 1877 key.n = n; 1878 return rhashtable_lookup_fast(&mlxsw_sp->router->neigh_ht, 1879 &key, mlxsw_sp_neigh_ht_params); 1880 } 1881 1882 static void 1883 mlxsw_sp_router_neighs_update_interval_init(struct mlxsw_sp *mlxsw_sp) 1884 { 1885 unsigned long interval; 1886 1887 #if IS_ENABLED(CONFIG_IPV6) 1888 interval = min_t(unsigned long, 1889 NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME), 1890 NEIGH_VAR(&nd_tbl.parms, DELAY_PROBE_TIME)); 1891 #else 1892 interval = NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME); 1893 #endif 1894 mlxsw_sp->router->neighs_update.interval = jiffies_to_msecs(interval); 1895 } 1896 1897 static void mlxsw_sp_router_neigh_ent_ipv4_process(struct mlxsw_sp *mlxsw_sp, 1898 char *rauhtd_pl, 1899 int ent_index) 1900 { 1901 struct net_device *dev; 1902 struct neighbour *n; 1903 __be32 dipn; 1904 u32 dip; 1905 u16 rif; 1906 1907 mlxsw_reg_rauhtd_ent_ipv4_unpack(rauhtd_pl, ent_index, &rif, &dip); 1908 1909 if (!mlxsw_sp->router->rifs[rif]) { 1910 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Incorrect RIF in neighbour entry\n"); 1911 return; 1912 } 1913 1914 dipn = htonl(dip); 1915 dev = mlxsw_sp->router->rifs[rif]->dev; 1916 n = neigh_lookup(&arp_tbl, &dipn, dev); 1917 if (!n) { 1918 netdev_err(dev, "Failed to find matching neighbour for IP=%pI4h\n", 1919 &dip); 1920 return; 1921 } 1922 1923 netdev_dbg(dev, "Updating neighbour with IP=%pI4h\n", &dip); 1924 neigh_event_send(n, NULL); 1925 neigh_release(n); 1926 } 1927 1928 #if IS_ENABLED(CONFIG_IPV6) 1929 static void mlxsw_sp_router_neigh_ent_ipv6_process(struct mlxsw_sp *mlxsw_sp, 1930 char *rauhtd_pl, 1931 int rec_index) 1932 { 1933 struct net_device *dev; 1934 struct neighbour *n; 1935 struct in6_addr dip; 1936 u16 rif; 1937 1938 mlxsw_reg_rauhtd_ent_ipv6_unpack(rauhtd_pl, rec_index, &rif, 1939 (char *) &dip); 1940 1941 if (!mlxsw_sp->router->rifs[rif]) { 1942 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Incorrect RIF in neighbour entry\n"); 1943 return; 1944 } 1945 1946 dev = mlxsw_sp->router->rifs[rif]->dev; 1947 n = neigh_lookup(&nd_tbl, &dip, dev); 1948 if (!n) { 1949 netdev_err(dev, "Failed to find matching neighbour for IP=%pI6c\n", 1950 &dip); 1951 return; 1952 } 1953 1954 netdev_dbg(dev, "Updating neighbour with IP=%pI6c\n", &dip); 1955 neigh_event_send(n, NULL); 1956 neigh_release(n); 1957 } 1958 #else 1959 static void mlxsw_sp_router_neigh_ent_ipv6_process(struct mlxsw_sp *mlxsw_sp, 1960 char *rauhtd_pl, 1961 int rec_index) 1962 { 1963 } 1964 #endif 1965 1966 static void mlxsw_sp_router_neigh_rec_ipv4_process(struct mlxsw_sp *mlxsw_sp, 1967 char *rauhtd_pl, 1968 int rec_index) 1969 { 1970 u8 num_entries; 1971 int i; 1972 1973 num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl, 1974 rec_index); 1975 /* Hardware starts counting at 0, so add 1. */ 1976 num_entries++; 1977 1978 /* Each record consists of several neighbour entries. */ 1979 for (i = 0; i < num_entries; i++) { 1980 int ent_index; 1981 1982 ent_index = rec_index * MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC + i; 1983 mlxsw_sp_router_neigh_ent_ipv4_process(mlxsw_sp, rauhtd_pl, 1984 ent_index); 1985 } 1986 1987 } 1988 1989 static void mlxsw_sp_router_neigh_rec_ipv6_process(struct mlxsw_sp *mlxsw_sp, 1990 char *rauhtd_pl, 1991 int rec_index) 1992 { 1993 /* One record contains one entry. */ 1994 mlxsw_sp_router_neigh_ent_ipv6_process(mlxsw_sp, rauhtd_pl, 1995 rec_index); 1996 } 1997 1998 static void mlxsw_sp_router_neigh_rec_process(struct mlxsw_sp *mlxsw_sp, 1999 char *rauhtd_pl, int rec_index) 2000 { 2001 switch (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, rec_index)) { 2002 case MLXSW_REG_RAUHTD_TYPE_IPV4: 2003 mlxsw_sp_router_neigh_rec_ipv4_process(mlxsw_sp, rauhtd_pl, 2004 rec_index); 2005 break; 2006 case MLXSW_REG_RAUHTD_TYPE_IPV6: 2007 mlxsw_sp_router_neigh_rec_ipv6_process(mlxsw_sp, rauhtd_pl, 2008 rec_index); 2009 break; 2010 } 2011 } 2012 2013 static bool mlxsw_sp_router_rauhtd_is_full(char *rauhtd_pl) 2014 { 2015 u8 num_rec, last_rec_index, num_entries; 2016 2017 num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl); 2018 last_rec_index = num_rec - 1; 2019 2020 if (num_rec < MLXSW_REG_RAUHTD_REC_MAX_NUM) 2021 return false; 2022 if (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, last_rec_index) == 2023 MLXSW_REG_RAUHTD_TYPE_IPV6) 2024 return true; 2025 2026 num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl, 2027 last_rec_index); 2028 if (++num_entries == MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC) 2029 return true; 2030 return false; 2031 } 2032 2033 static int 2034 __mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp, 2035 char *rauhtd_pl, 2036 enum mlxsw_reg_rauhtd_type type) 2037 { 2038 int i, num_rec; 2039 int err; 2040 2041 /* Make sure the neighbour's netdev isn't removed in the 2042 * process. 2043 */ 2044 rtnl_lock(); 2045 do { 2046 mlxsw_reg_rauhtd_pack(rauhtd_pl, type); 2047 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(rauhtd), 2048 rauhtd_pl); 2049 if (err) { 2050 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Failed to dump neighbour table\n"); 2051 break; 2052 } 2053 num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl); 2054 for (i = 0; i < num_rec; i++) 2055 mlxsw_sp_router_neigh_rec_process(mlxsw_sp, rauhtd_pl, 2056 i); 2057 } while (mlxsw_sp_router_rauhtd_is_full(rauhtd_pl)); 2058 rtnl_unlock(); 2059 2060 return err; 2061 } 2062 2063 static int mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp) 2064 { 2065 enum mlxsw_reg_rauhtd_type type; 2066 char *rauhtd_pl; 2067 int err; 2068 2069 rauhtd_pl = kmalloc(MLXSW_REG_RAUHTD_LEN, GFP_KERNEL); 2070 if (!rauhtd_pl) 2071 return -ENOMEM; 2072 2073 type = MLXSW_REG_RAUHTD_TYPE_IPV4; 2074 err = __mlxsw_sp_router_neighs_update_rauhtd(mlxsw_sp, rauhtd_pl, type); 2075 if (err) 2076 goto out; 2077 2078 type = MLXSW_REG_RAUHTD_TYPE_IPV6; 2079 err = __mlxsw_sp_router_neighs_update_rauhtd(mlxsw_sp, rauhtd_pl, type); 2080 out: 2081 kfree(rauhtd_pl); 2082 return err; 2083 } 2084 2085 static void mlxsw_sp_router_neighs_update_nh(struct mlxsw_sp *mlxsw_sp) 2086 { 2087 struct mlxsw_sp_neigh_entry *neigh_entry; 2088 2089 /* Take RTNL mutex here to prevent lists from changes */ 2090 rtnl_lock(); 2091 list_for_each_entry(neigh_entry, &mlxsw_sp->router->nexthop_neighs_list, 2092 nexthop_neighs_list_node) 2093 /* If this neigh have nexthops, make the kernel think this neigh 2094 * is active regardless of the traffic. 2095 */ 2096 neigh_event_send(neigh_entry->key.n, NULL); 2097 rtnl_unlock(); 2098 } 2099 2100 static void 2101 mlxsw_sp_router_neighs_update_work_schedule(struct mlxsw_sp *mlxsw_sp) 2102 { 2103 unsigned long interval = mlxsw_sp->router->neighs_update.interval; 2104 2105 mlxsw_core_schedule_dw(&mlxsw_sp->router->neighs_update.dw, 2106 msecs_to_jiffies(interval)); 2107 } 2108 2109 static void mlxsw_sp_router_neighs_update_work(struct work_struct *work) 2110 { 2111 struct mlxsw_sp_router *router; 2112 int err; 2113 2114 router = container_of(work, struct mlxsw_sp_router, 2115 neighs_update.dw.work); 2116 err = mlxsw_sp_router_neighs_update_rauhtd(router->mlxsw_sp); 2117 if (err) 2118 dev_err(router->mlxsw_sp->bus_info->dev, "Could not update kernel for neigh activity"); 2119 2120 mlxsw_sp_router_neighs_update_nh(router->mlxsw_sp); 2121 2122 mlxsw_sp_router_neighs_update_work_schedule(router->mlxsw_sp); 2123 } 2124 2125 static void mlxsw_sp_router_probe_unresolved_nexthops(struct work_struct *work) 2126 { 2127 struct mlxsw_sp_neigh_entry *neigh_entry; 2128 struct mlxsw_sp_router *router; 2129 2130 router = container_of(work, struct mlxsw_sp_router, 2131 nexthop_probe_dw.work); 2132 /* Iterate over nexthop neighbours, find those who are unresolved and 2133 * send arp on them. This solves the chicken-egg problem when 2134 * the nexthop wouldn't get offloaded until the neighbor is resolved 2135 * but it wouldn't get resolved ever in case traffic is flowing in HW 2136 * using different nexthop. 2137 * 2138 * Take RTNL mutex here to prevent lists from changes. 2139 */ 2140 rtnl_lock(); 2141 list_for_each_entry(neigh_entry, &router->nexthop_neighs_list, 2142 nexthop_neighs_list_node) 2143 if (!neigh_entry->connected) 2144 neigh_event_send(neigh_entry->key.n, NULL); 2145 rtnl_unlock(); 2146 2147 mlxsw_core_schedule_dw(&router->nexthop_probe_dw, 2148 MLXSW_SP_UNRESOLVED_NH_PROBE_INTERVAL); 2149 } 2150 2151 static void 2152 mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp, 2153 struct mlxsw_sp_neigh_entry *neigh_entry, 2154 bool removing); 2155 2156 static enum mlxsw_reg_rauht_op mlxsw_sp_rauht_op(bool adding) 2157 { 2158 return adding ? MLXSW_REG_RAUHT_OP_WRITE_ADD : 2159 MLXSW_REG_RAUHT_OP_WRITE_DELETE; 2160 } 2161 2162 static void 2163 mlxsw_sp_router_neigh_entry_op4(struct mlxsw_sp *mlxsw_sp, 2164 struct mlxsw_sp_neigh_entry *neigh_entry, 2165 enum mlxsw_reg_rauht_op op) 2166 { 2167 struct neighbour *n = neigh_entry->key.n; 2168 u32 dip = ntohl(*((__be32 *) n->primary_key)); 2169 char rauht_pl[MLXSW_REG_RAUHT_LEN]; 2170 2171 mlxsw_reg_rauht_pack4(rauht_pl, op, neigh_entry->rif, neigh_entry->ha, 2172 dip); 2173 if (neigh_entry->counter_valid) 2174 mlxsw_reg_rauht_pack_counter(rauht_pl, 2175 neigh_entry->counter_index); 2176 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl); 2177 } 2178 2179 static void 2180 mlxsw_sp_router_neigh_entry_op6(struct mlxsw_sp *mlxsw_sp, 2181 struct mlxsw_sp_neigh_entry *neigh_entry, 2182 enum mlxsw_reg_rauht_op op) 2183 { 2184 struct neighbour *n = neigh_entry->key.n; 2185 char rauht_pl[MLXSW_REG_RAUHT_LEN]; 2186 const char *dip = n->primary_key; 2187 2188 mlxsw_reg_rauht_pack6(rauht_pl, op, neigh_entry->rif, neigh_entry->ha, 2189 dip); 2190 if (neigh_entry->counter_valid) 2191 mlxsw_reg_rauht_pack_counter(rauht_pl, 2192 neigh_entry->counter_index); 2193 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl); 2194 } 2195 2196 bool mlxsw_sp_neigh_ipv6_ignore(struct mlxsw_sp_neigh_entry *neigh_entry) 2197 { 2198 struct neighbour *n = neigh_entry->key.n; 2199 2200 /* Packets with a link-local destination address are trapped 2201 * after LPM lookup and never reach the neighbour table, so 2202 * there is no need to program such neighbours to the device. 2203 */ 2204 if (ipv6_addr_type((struct in6_addr *) &n->primary_key) & 2205 IPV6_ADDR_LINKLOCAL) 2206 return true; 2207 return false; 2208 } 2209 2210 static void 2211 mlxsw_sp_neigh_entry_update(struct mlxsw_sp *mlxsw_sp, 2212 struct mlxsw_sp_neigh_entry *neigh_entry, 2213 bool adding) 2214 { 2215 if (!adding && !neigh_entry->connected) 2216 return; 2217 neigh_entry->connected = adding; 2218 if (neigh_entry->key.n->tbl->family == AF_INET) { 2219 mlxsw_sp_router_neigh_entry_op4(mlxsw_sp, neigh_entry, 2220 mlxsw_sp_rauht_op(adding)); 2221 } else if (neigh_entry->key.n->tbl->family == AF_INET6) { 2222 if (mlxsw_sp_neigh_ipv6_ignore(neigh_entry)) 2223 return; 2224 mlxsw_sp_router_neigh_entry_op6(mlxsw_sp, neigh_entry, 2225 mlxsw_sp_rauht_op(adding)); 2226 } else { 2227 WARN_ON_ONCE(1); 2228 } 2229 } 2230 2231 void 2232 mlxsw_sp_neigh_entry_counter_update(struct mlxsw_sp *mlxsw_sp, 2233 struct mlxsw_sp_neigh_entry *neigh_entry, 2234 bool adding) 2235 { 2236 if (adding) 2237 mlxsw_sp_neigh_counter_alloc(mlxsw_sp, neigh_entry); 2238 else 2239 mlxsw_sp_neigh_counter_free(mlxsw_sp, neigh_entry); 2240 mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, true); 2241 } 2242 2243 struct mlxsw_sp_netevent_work { 2244 struct work_struct work; 2245 struct mlxsw_sp *mlxsw_sp; 2246 struct neighbour *n; 2247 }; 2248 2249 static void mlxsw_sp_router_neigh_event_work(struct work_struct *work) 2250 { 2251 struct mlxsw_sp_netevent_work *net_work = 2252 container_of(work, struct mlxsw_sp_netevent_work, work); 2253 struct mlxsw_sp *mlxsw_sp = net_work->mlxsw_sp; 2254 struct mlxsw_sp_neigh_entry *neigh_entry; 2255 struct neighbour *n = net_work->n; 2256 unsigned char ha[ETH_ALEN]; 2257 bool entry_connected; 2258 u8 nud_state, dead; 2259 2260 /* If these parameters are changed after we release the lock, 2261 * then we are guaranteed to receive another event letting us 2262 * know about it. 2263 */ 2264 read_lock_bh(&n->lock); 2265 memcpy(ha, n->ha, ETH_ALEN); 2266 nud_state = n->nud_state; 2267 dead = n->dead; 2268 read_unlock_bh(&n->lock); 2269 2270 rtnl_lock(); 2271 entry_connected = nud_state & NUD_VALID && !dead; 2272 neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n); 2273 if (!entry_connected && !neigh_entry) 2274 goto out; 2275 if (!neigh_entry) { 2276 neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n); 2277 if (IS_ERR(neigh_entry)) 2278 goto out; 2279 } 2280 2281 memcpy(neigh_entry->ha, ha, ETH_ALEN); 2282 mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, entry_connected); 2283 mlxsw_sp_nexthop_neigh_update(mlxsw_sp, neigh_entry, !entry_connected); 2284 2285 if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list)) 2286 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry); 2287 2288 out: 2289 rtnl_unlock(); 2290 neigh_release(n); 2291 kfree(net_work); 2292 } 2293 2294 static int mlxsw_sp_mp_hash_init(struct mlxsw_sp *mlxsw_sp); 2295 2296 static void mlxsw_sp_router_mp_hash_event_work(struct work_struct *work) 2297 { 2298 struct mlxsw_sp_netevent_work *net_work = 2299 container_of(work, struct mlxsw_sp_netevent_work, work); 2300 struct mlxsw_sp *mlxsw_sp = net_work->mlxsw_sp; 2301 2302 mlxsw_sp_mp_hash_init(mlxsw_sp); 2303 kfree(net_work); 2304 } 2305 2306 static int mlxsw_sp_router_netevent_event(struct notifier_block *nb, 2307 unsigned long event, void *ptr) 2308 { 2309 struct mlxsw_sp_netevent_work *net_work; 2310 struct mlxsw_sp_port *mlxsw_sp_port; 2311 struct mlxsw_sp_router *router; 2312 struct mlxsw_sp *mlxsw_sp; 2313 unsigned long interval; 2314 struct neigh_parms *p; 2315 struct neighbour *n; 2316 struct net *net; 2317 2318 switch (event) { 2319 case NETEVENT_DELAY_PROBE_TIME_UPDATE: 2320 p = ptr; 2321 2322 /* We don't care about changes in the default table. */ 2323 if (!p->dev || (p->tbl->family != AF_INET && 2324 p->tbl->family != AF_INET6)) 2325 return NOTIFY_DONE; 2326 2327 /* We are in atomic context and can't take RTNL mutex, 2328 * so use RCU variant to walk the device chain. 2329 */ 2330 mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(p->dev); 2331 if (!mlxsw_sp_port) 2332 return NOTIFY_DONE; 2333 2334 mlxsw_sp = mlxsw_sp_port->mlxsw_sp; 2335 interval = jiffies_to_msecs(NEIGH_VAR(p, DELAY_PROBE_TIME)); 2336 mlxsw_sp->router->neighs_update.interval = interval; 2337 2338 mlxsw_sp_port_dev_put(mlxsw_sp_port); 2339 break; 2340 case NETEVENT_NEIGH_UPDATE: 2341 n = ptr; 2342 2343 if (n->tbl->family != AF_INET && n->tbl->family != AF_INET6) 2344 return NOTIFY_DONE; 2345 2346 mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(n->dev); 2347 if (!mlxsw_sp_port) 2348 return NOTIFY_DONE; 2349 2350 net_work = kzalloc(sizeof(*net_work), GFP_ATOMIC); 2351 if (!net_work) { 2352 mlxsw_sp_port_dev_put(mlxsw_sp_port); 2353 return NOTIFY_BAD; 2354 } 2355 2356 INIT_WORK(&net_work->work, mlxsw_sp_router_neigh_event_work); 2357 net_work->mlxsw_sp = mlxsw_sp_port->mlxsw_sp; 2358 net_work->n = n; 2359 2360 /* Take a reference to ensure the neighbour won't be 2361 * destructed until we drop the reference in delayed 2362 * work. 2363 */ 2364 neigh_clone(n); 2365 mlxsw_core_schedule_work(&net_work->work); 2366 mlxsw_sp_port_dev_put(mlxsw_sp_port); 2367 break; 2368 case NETEVENT_MULTIPATH_HASH_UPDATE: 2369 net = ptr; 2370 2371 if (!net_eq(net, &init_net)) 2372 return NOTIFY_DONE; 2373 2374 net_work = kzalloc(sizeof(*net_work), GFP_ATOMIC); 2375 if (!net_work) 2376 return NOTIFY_BAD; 2377 2378 router = container_of(nb, struct mlxsw_sp_router, netevent_nb); 2379 INIT_WORK(&net_work->work, mlxsw_sp_router_mp_hash_event_work); 2380 net_work->mlxsw_sp = router->mlxsw_sp; 2381 mlxsw_core_schedule_work(&net_work->work); 2382 break; 2383 } 2384 2385 return NOTIFY_DONE; 2386 } 2387 2388 static int mlxsw_sp_neigh_init(struct mlxsw_sp *mlxsw_sp) 2389 { 2390 int err; 2391 2392 err = rhashtable_init(&mlxsw_sp->router->neigh_ht, 2393 &mlxsw_sp_neigh_ht_params); 2394 if (err) 2395 return err; 2396 2397 /* Initialize the polling interval according to the default 2398 * table. 2399 */ 2400 mlxsw_sp_router_neighs_update_interval_init(mlxsw_sp); 2401 2402 /* Create the delayed works for the activity_update */ 2403 INIT_DELAYED_WORK(&mlxsw_sp->router->neighs_update.dw, 2404 mlxsw_sp_router_neighs_update_work); 2405 INIT_DELAYED_WORK(&mlxsw_sp->router->nexthop_probe_dw, 2406 mlxsw_sp_router_probe_unresolved_nexthops); 2407 mlxsw_core_schedule_dw(&mlxsw_sp->router->neighs_update.dw, 0); 2408 mlxsw_core_schedule_dw(&mlxsw_sp->router->nexthop_probe_dw, 0); 2409 return 0; 2410 } 2411 2412 static void mlxsw_sp_neigh_fini(struct mlxsw_sp *mlxsw_sp) 2413 { 2414 cancel_delayed_work_sync(&mlxsw_sp->router->neighs_update.dw); 2415 cancel_delayed_work_sync(&mlxsw_sp->router->nexthop_probe_dw); 2416 rhashtable_destroy(&mlxsw_sp->router->neigh_ht); 2417 } 2418 2419 static int mlxsw_sp_neigh_rif_flush(struct mlxsw_sp *mlxsw_sp, 2420 const struct mlxsw_sp_rif *rif) 2421 { 2422 char rauht_pl[MLXSW_REG_RAUHT_LEN]; 2423 2424 mlxsw_reg_rauht_pack(rauht_pl, MLXSW_REG_RAUHT_OP_WRITE_DELETE_ALL, 2425 rif->rif_index, rif->addr); 2426 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl); 2427 } 2428 2429 static void mlxsw_sp_neigh_rif_gone_sync(struct mlxsw_sp *mlxsw_sp, 2430 struct mlxsw_sp_rif *rif) 2431 { 2432 struct mlxsw_sp_neigh_entry *neigh_entry, *tmp; 2433 2434 mlxsw_sp_neigh_rif_flush(mlxsw_sp, rif); 2435 list_for_each_entry_safe(neigh_entry, tmp, &rif->neigh_list, 2436 rif_list_node) 2437 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry); 2438 } 2439 2440 enum mlxsw_sp_nexthop_type { 2441 MLXSW_SP_NEXTHOP_TYPE_ETH, 2442 MLXSW_SP_NEXTHOP_TYPE_IPIP, 2443 }; 2444 2445 struct mlxsw_sp_nexthop_key { 2446 struct fib_nh *fib_nh; 2447 }; 2448 2449 struct mlxsw_sp_nexthop { 2450 struct list_head neigh_list_node; /* member of neigh entry list */ 2451 struct list_head rif_list_node; 2452 struct list_head router_list_node; 2453 struct mlxsw_sp_nexthop_group *nh_grp; /* pointer back to the group 2454 * this belongs to 2455 */ 2456 struct rhash_head ht_node; 2457 struct mlxsw_sp_nexthop_key key; 2458 unsigned char gw_addr[sizeof(struct in6_addr)]; 2459 int ifindex; 2460 int nh_weight; 2461 int norm_nh_weight; 2462 int num_adj_entries; 2463 struct mlxsw_sp_rif *rif; 2464 u8 should_offload:1, /* set indicates this neigh is connected and 2465 * should be put to KVD linear area of this group. 2466 */ 2467 offloaded:1, /* set in case the neigh is actually put into 2468 * KVD linear area of this group. 2469 */ 2470 update:1; /* set indicates that MAC of this neigh should be 2471 * updated in HW 2472 */ 2473 enum mlxsw_sp_nexthop_type type; 2474 union { 2475 struct mlxsw_sp_neigh_entry *neigh_entry; 2476 struct mlxsw_sp_ipip_entry *ipip_entry; 2477 }; 2478 unsigned int counter_index; 2479 bool counter_valid; 2480 }; 2481 2482 struct mlxsw_sp_nexthop_group { 2483 void *priv; 2484 struct rhash_head ht_node; 2485 struct list_head fib_list; /* list of fib entries that use this group */ 2486 struct neigh_table *neigh_tbl; 2487 u8 adj_index_valid:1, 2488 gateway:1; /* routes using the group use a gateway */ 2489 u32 adj_index; 2490 u16 ecmp_size; 2491 u16 count; 2492 int sum_norm_weight; 2493 struct mlxsw_sp_nexthop nexthops[0]; 2494 #define nh_rif nexthops[0].rif 2495 }; 2496 2497 void mlxsw_sp_nexthop_counter_alloc(struct mlxsw_sp *mlxsw_sp, 2498 struct mlxsw_sp_nexthop *nh) 2499 { 2500 struct devlink *devlink; 2501 2502 devlink = priv_to_devlink(mlxsw_sp->core); 2503 if (!devlink_dpipe_table_counter_enabled(devlink, 2504 MLXSW_SP_DPIPE_TABLE_NAME_ADJ)) 2505 return; 2506 2507 if (mlxsw_sp_flow_counter_alloc(mlxsw_sp, &nh->counter_index)) 2508 return; 2509 2510 nh->counter_valid = true; 2511 } 2512 2513 void mlxsw_sp_nexthop_counter_free(struct mlxsw_sp *mlxsw_sp, 2514 struct mlxsw_sp_nexthop *nh) 2515 { 2516 if (!nh->counter_valid) 2517 return; 2518 mlxsw_sp_flow_counter_free(mlxsw_sp, nh->counter_index); 2519 nh->counter_valid = false; 2520 } 2521 2522 int mlxsw_sp_nexthop_counter_get(struct mlxsw_sp *mlxsw_sp, 2523 struct mlxsw_sp_nexthop *nh, u64 *p_counter) 2524 { 2525 if (!nh->counter_valid) 2526 return -EINVAL; 2527 2528 return mlxsw_sp_flow_counter_get(mlxsw_sp, nh->counter_index, 2529 p_counter, NULL); 2530 } 2531 2532 struct mlxsw_sp_nexthop *mlxsw_sp_nexthop_next(struct mlxsw_sp_router *router, 2533 struct mlxsw_sp_nexthop *nh) 2534 { 2535 if (!nh) { 2536 if (list_empty(&router->nexthop_list)) 2537 return NULL; 2538 else 2539 return list_first_entry(&router->nexthop_list, 2540 typeof(*nh), router_list_node); 2541 } 2542 if (list_is_last(&nh->router_list_node, &router->nexthop_list)) 2543 return NULL; 2544 return list_next_entry(nh, router_list_node); 2545 } 2546 2547 bool mlxsw_sp_nexthop_offload(struct mlxsw_sp_nexthop *nh) 2548 { 2549 return nh->offloaded; 2550 } 2551 2552 unsigned char *mlxsw_sp_nexthop_ha(struct mlxsw_sp_nexthop *nh) 2553 { 2554 if (!nh->offloaded) 2555 return NULL; 2556 return nh->neigh_entry->ha; 2557 } 2558 2559 int mlxsw_sp_nexthop_indexes(struct mlxsw_sp_nexthop *nh, u32 *p_adj_index, 2560 u32 *p_adj_size, u32 *p_adj_hash_index) 2561 { 2562 struct mlxsw_sp_nexthop_group *nh_grp = nh->nh_grp; 2563 u32 adj_hash_index = 0; 2564 int i; 2565 2566 if (!nh->offloaded || !nh_grp->adj_index_valid) 2567 return -EINVAL; 2568 2569 *p_adj_index = nh_grp->adj_index; 2570 *p_adj_size = nh_grp->ecmp_size; 2571 2572 for (i = 0; i < nh_grp->count; i++) { 2573 struct mlxsw_sp_nexthop *nh_iter = &nh_grp->nexthops[i]; 2574 2575 if (nh_iter == nh) 2576 break; 2577 if (nh_iter->offloaded) 2578 adj_hash_index += nh_iter->num_adj_entries; 2579 } 2580 2581 *p_adj_hash_index = adj_hash_index; 2582 return 0; 2583 } 2584 2585 struct mlxsw_sp_rif *mlxsw_sp_nexthop_rif(struct mlxsw_sp_nexthop *nh) 2586 { 2587 return nh->rif; 2588 } 2589 2590 bool mlxsw_sp_nexthop_group_has_ipip(struct mlxsw_sp_nexthop *nh) 2591 { 2592 struct mlxsw_sp_nexthop_group *nh_grp = nh->nh_grp; 2593 int i; 2594 2595 for (i = 0; i < nh_grp->count; i++) { 2596 struct mlxsw_sp_nexthop *nh_iter = &nh_grp->nexthops[i]; 2597 2598 if (nh_iter->type == MLXSW_SP_NEXTHOP_TYPE_IPIP) 2599 return true; 2600 } 2601 return false; 2602 } 2603 2604 static struct fib_info * 2605 mlxsw_sp_nexthop4_group_fi(const struct mlxsw_sp_nexthop_group *nh_grp) 2606 { 2607 return nh_grp->priv; 2608 } 2609 2610 struct mlxsw_sp_nexthop_group_cmp_arg { 2611 enum mlxsw_sp_l3proto proto; 2612 union { 2613 struct fib_info *fi; 2614 struct mlxsw_sp_fib6_entry *fib6_entry; 2615 }; 2616 }; 2617 2618 static bool 2619 mlxsw_sp_nexthop6_group_has_nexthop(const struct mlxsw_sp_nexthop_group *nh_grp, 2620 const struct in6_addr *gw, int ifindex) 2621 { 2622 int i; 2623 2624 for (i = 0; i < nh_grp->count; i++) { 2625 const struct mlxsw_sp_nexthop *nh; 2626 2627 nh = &nh_grp->nexthops[i]; 2628 if (nh->ifindex == ifindex && 2629 ipv6_addr_equal(gw, (struct in6_addr *) nh->gw_addr)) 2630 return true; 2631 } 2632 2633 return false; 2634 } 2635 2636 static bool 2637 mlxsw_sp_nexthop6_group_cmp(const struct mlxsw_sp_nexthop_group *nh_grp, 2638 const struct mlxsw_sp_fib6_entry *fib6_entry) 2639 { 2640 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 2641 2642 if (nh_grp->count != fib6_entry->nrt6) 2643 return false; 2644 2645 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) { 2646 struct in6_addr *gw; 2647 int ifindex; 2648 2649 ifindex = mlxsw_sp_rt6->rt->dst.dev->ifindex; 2650 gw = &mlxsw_sp_rt6->rt->rt6i_gateway; 2651 if (!mlxsw_sp_nexthop6_group_has_nexthop(nh_grp, gw, ifindex)) 2652 return false; 2653 } 2654 2655 return true; 2656 } 2657 2658 static int 2659 mlxsw_sp_nexthop_group_cmp(struct rhashtable_compare_arg *arg, const void *ptr) 2660 { 2661 const struct mlxsw_sp_nexthop_group_cmp_arg *cmp_arg = arg->key; 2662 const struct mlxsw_sp_nexthop_group *nh_grp = ptr; 2663 2664 switch (cmp_arg->proto) { 2665 case MLXSW_SP_L3_PROTO_IPV4: 2666 return cmp_arg->fi != mlxsw_sp_nexthop4_group_fi(nh_grp); 2667 case MLXSW_SP_L3_PROTO_IPV6: 2668 return !mlxsw_sp_nexthop6_group_cmp(nh_grp, 2669 cmp_arg->fib6_entry); 2670 default: 2671 WARN_ON(1); 2672 return 1; 2673 } 2674 } 2675 2676 static int 2677 mlxsw_sp_nexthop_group_type(const struct mlxsw_sp_nexthop_group *nh_grp) 2678 { 2679 return nh_grp->neigh_tbl->family; 2680 } 2681 2682 static u32 mlxsw_sp_nexthop_group_hash_obj(const void *data, u32 len, u32 seed) 2683 { 2684 const struct mlxsw_sp_nexthop_group *nh_grp = data; 2685 const struct mlxsw_sp_nexthop *nh; 2686 struct fib_info *fi; 2687 unsigned int val; 2688 int i; 2689 2690 switch (mlxsw_sp_nexthop_group_type(nh_grp)) { 2691 case AF_INET: 2692 fi = mlxsw_sp_nexthop4_group_fi(nh_grp); 2693 return jhash(&fi, sizeof(fi), seed); 2694 case AF_INET6: 2695 val = nh_grp->count; 2696 for (i = 0; i < nh_grp->count; i++) { 2697 nh = &nh_grp->nexthops[i]; 2698 val ^= nh->ifindex; 2699 } 2700 return jhash(&val, sizeof(val), seed); 2701 default: 2702 WARN_ON(1); 2703 return 0; 2704 } 2705 } 2706 2707 static u32 2708 mlxsw_sp_nexthop6_group_hash(struct mlxsw_sp_fib6_entry *fib6_entry, u32 seed) 2709 { 2710 unsigned int val = fib6_entry->nrt6; 2711 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 2712 struct net_device *dev; 2713 2714 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) { 2715 dev = mlxsw_sp_rt6->rt->dst.dev; 2716 val ^= dev->ifindex; 2717 } 2718 2719 return jhash(&val, sizeof(val), seed); 2720 } 2721 2722 static u32 2723 mlxsw_sp_nexthop_group_hash(const void *data, u32 len, u32 seed) 2724 { 2725 const struct mlxsw_sp_nexthop_group_cmp_arg *cmp_arg = data; 2726 2727 switch (cmp_arg->proto) { 2728 case MLXSW_SP_L3_PROTO_IPV4: 2729 return jhash(&cmp_arg->fi, sizeof(cmp_arg->fi), seed); 2730 case MLXSW_SP_L3_PROTO_IPV6: 2731 return mlxsw_sp_nexthop6_group_hash(cmp_arg->fib6_entry, seed); 2732 default: 2733 WARN_ON(1); 2734 return 0; 2735 } 2736 } 2737 2738 static const struct rhashtable_params mlxsw_sp_nexthop_group_ht_params = { 2739 .head_offset = offsetof(struct mlxsw_sp_nexthop_group, ht_node), 2740 .hashfn = mlxsw_sp_nexthop_group_hash, 2741 .obj_hashfn = mlxsw_sp_nexthop_group_hash_obj, 2742 .obj_cmpfn = mlxsw_sp_nexthop_group_cmp, 2743 }; 2744 2745 static int mlxsw_sp_nexthop_group_insert(struct mlxsw_sp *mlxsw_sp, 2746 struct mlxsw_sp_nexthop_group *nh_grp) 2747 { 2748 if (mlxsw_sp_nexthop_group_type(nh_grp) == AF_INET6 && 2749 !nh_grp->gateway) 2750 return 0; 2751 2752 return rhashtable_insert_fast(&mlxsw_sp->router->nexthop_group_ht, 2753 &nh_grp->ht_node, 2754 mlxsw_sp_nexthop_group_ht_params); 2755 } 2756 2757 static void mlxsw_sp_nexthop_group_remove(struct mlxsw_sp *mlxsw_sp, 2758 struct mlxsw_sp_nexthop_group *nh_grp) 2759 { 2760 if (mlxsw_sp_nexthop_group_type(nh_grp) == AF_INET6 && 2761 !nh_grp->gateway) 2762 return; 2763 2764 rhashtable_remove_fast(&mlxsw_sp->router->nexthop_group_ht, 2765 &nh_grp->ht_node, 2766 mlxsw_sp_nexthop_group_ht_params); 2767 } 2768 2769 static struct mlxsw_sp_nexthop_group * 2770 mlxsw_sp_nexthop4_group_lookup(struct mlxsw_sp *mlxsw_sp, 2771 struct fib_info *fi) 2772 { 2773 struct mlxsw_sp_nexthop_group_cmp_arg cmp_arg; 2774 2775 cmp_arg.proto = MLXSW_SP_L3_PROTO_IPV4; 2776 cmp_arg.fi = fi; 2777 return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_group_ht, 2778 &cmp_arg, 2779 mlxsw_sp_nexthop_group_ht_params); 2780 } 2781 2782 static struct mlxsw_sp_nexthop_group * 2783 mlxsw_sp_nexthop6_group_lookup(struct mlxsw_sp *mlxsw_sp, 2784 struct mlxsw_sp_fib6_entry *fib6_entry) 2785 { 2786 struct mlxsw_sp_nexthop_group_cmp_arg cmp_arg; 2787 2788 cmp_arg.proto = MLXSW_SP_L3_PROTO_IPV6; 2789 cmp_arg.fib6_entry = fib6_entry; 2790 return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_group_ht, 2791 &cmp_arg, 2792 mlxsw_sp_nexthop_group_ht_params); 2793 } 2794 2795 static const struct rhashtable_params mlxsw_sp_nexthop_ht_params = { 2796 .key_offset = offsetof(struct mlxsw_sp_nexthop, key), 2797 .head_offset = offsetof(struct mlxsw_sp_nexthop, ht_node), 2798 .key_len = sizeof(struct mlxsw_sp_nexthop_key), 2799 }; 2800 2801 static int mlxsw_sp_nexthop_insert(struct mlxsw_sp *mlxsw_sp, 2802 struct mlxsw_sp_nexthop *nh) 2803 { 2804 return rhashtable_insert_fast(&mlxsw_sp->router->nexthop_ht, 2805 &nh->ht_node, mlxsw_sp_nexthop_ht_params); 2806 } 2807 2808 static void mlxsw_sp_nexthop_remove(struct mlxsw_sp *mlxsw_sp, 2809 struct mlxsw_sp_nexthop *nh) 2810 { 2811 rhashtable_remove_fast(&mlxsw_sp->router->nexthop_ht, &nh->ht_node, 2812 mlxsw_sp_nexthop_ht_params); 2813 } 2814 2815 static struct mlxsw_sp_nexthop * 2816 mlxsw_sp_nexthop_lookup(struct mlxsw_sp *mlxsw_sp, 2817 struct mlxsw_sp_nexthop_key key) 2818 { 2819 return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_ht, &key, 2820 mlxsw_sp_nexthop_ht_params); 2821 } 2822 2823 static int mlxsw_sp_adj_index_mass_update_vr(struct mlxsw_sp *mlxsw_sp, 2824 const struct mlxsw_sp_fib *fib, 2825 u32 adj_index, u16 ecmp_size, 2826 u32 new_adj_index, 2827 u16 new_ecmp_size) 2828 { 2829 char raleu_pl[MLXSW_REG_RALEU_LEN]; 2830 2831 mlxsw_reg_raleu_pack(raleu_pl, 2832 (enum mlxsw_reg_ralxx_protocol) fib->proto, 2833 fib->vr->id, adj_index, ecmp_size, new_adj_index, 2834 new_ecmp_size); 2835 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raleu), raleu_pl); 2836 } 2837 2838 static int mlxsw_sp_adj_index_mass_update(struct mlxsw_sp *mlxsw_sp, 2839 struct mlxsw_sp_nexthop_group *nh_grp, 2840 u32 old_adj_index, u16 old_ecmp_size) 2841 { 2842 struct mlxsw_sp_fib_entry *fib_entry; 2843 struct mlxsw_sp_fib *fib = NULL; 2844 int err; 2845 2846 list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) { 2847 if (fib == fib_entry->fib_node->fib) 2848 continue; 2849 fib = fib_entry->fib_node->fib; 2850 err = mlxsw_sp_adj_index_mass_update_vr(mlxsw_sp, fib, 2851 old_adj_index, 2852 old_ecmp_size, 2853 nh_grp->adj_index, 2854 nh_grp->ecmp_size); 2855 if (err) 2856 return err; 2857 } 2858 return 0; 2859 } 2860 2861 static int __mlxsw_sp_nexthop_update(struct mlxsw_sp *mlxsw_sp, u32 adj_index, 2862 struct mlxsw_sp_nexthop *nh) 2863 { 2864 struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry; 2865 char ratr_pl[MLXSW_REG_RATR_LEN]; 2866 2867 mlxsw_reg_ratr_pack(ratr_pl, MLXSW_REG_RATR_OP_WRITE_WRITE_ENTRY, 2868 true, MLXSW_REG_RATR_TYPE_ETHERNET, 2869 adj_index, neigh_entry->rif); 2870 mlxsw_reg_ratr_eth_entry_pack(ratr_pl, neigh_entry->ha); 2871 if (nh->counter_valid) 2872 mlxsw_reg_ratr_counter_pack(ratr_pl, nh->counter_index, true); 2873 else 2874 mlxsw_reg_ratr_counter_pack(ratr_pl, 0, false); 2875 2876 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ratr), ratr_pl); 2877 } 2878 2879 int mlxsw_sp_nexthop_update(struct mlxsw_sp *mlxsw_sp, u32 adj_index, 2880 struct mlxsw_sp_nexthop *nh) 2881 { 2882 int i; 2883 2884 for (i = 0; i < nh->num_adj_entries; i++) { 2885 int err; 2886 2887 err = __mlxsw_sp_nexthop_update(mlxsw_sp, adj_index + i, nh); 2888 if (err) 2889 return err; 2890 } 2891 2892 return 0; 2893 } 2894 2895 static int __mlxsw_sp_nexthop_ipip_update(struct mlxsw_sp *mlxsw_sp, 2896 u32 adj_index, 2897 struct mlxsw_sp_nexthop *nh) 2898 { 2899 const struct mlxsw_sp_ipip_ops *ipip_ops; 2900 2901 ipip_ops = mlxsw_sp->router->ipip_ops_arr[nh->ipip_entry->ipipt]; 2902 return ipip_ops->nexthop_update(mlxsw_sp, adj_index, nh->ipip_entry); 2903 } 2904 2905 static int mlxsw_sp_nexthop_ipip_update(struct mlxsw_sp *mlxsw_sp, 2906 u32 adj_index, 2907 struct mlxsw_sp_nexthop *nh) 2908 { 2909 int i; 2910 2911 for (i = 0; i < nh->num_adj_entries; i++) { 2912 int err; 2913 2914 err = __mlxsw_sp_nexthop_ipip_update(mlxsw_sp, adj_index + i, 2915 nh); 2916 if (err) 2917 return err; 2918 } 2919 2920 return 0; 2921 } 2922 2923 static int 2924 mlxsw_sp_nexthop_group_update(struct mlxsw_sp *mlxsw_sp, 2925 struct mlxsw_sp_nexthop_group *nh_grp, 2926 bool reallocate) 2927 { 2928 u32 adj_index = nh_grp->adj_index; /* base */ 2929 struct mlxsw_sp_nexthop *nh; 2930 int i; 2931 int err; 2932 2933 for (i = 0; i < nh_grp->count; i++) { 2934 nh = &nh_grp->nexthops[i]; 2935 2936 if (!nh->should_offload) { 2937 nh->offloaded = 0; 2938 continue; 2939 } 2940 2941 if (nh->update || reallocate) { 2942 switch (nh->type) { 2943 case MLXSW_SP_NEXTHOP_TYPE_ETH: 2944 err = mlxsw_sp_nexthop_update 2945 (mlxsw_sp, adj_index, nh); 2946 break; 2947 case MLXSW_SP_NEXTHOP_TYPE_IPIP: 2948 err = mlxsw_sp_nexthop_ipip_update 2949 (mlxsw_sp, adj_index, nh); 2950 break; 2951 } 2952 if (err) 2953 return err; 2954 nh->update = 0; 2955 nh->offloaded = 1; 2956 } 2957 adj_index += nh->num_adj_entries; 2958 } 2959 return 0; 2960 } 2961 2962 static bool 2963 mlxsw_sp_fib_node_entry_is_first(const struct mlxsw_sp_fib_node *fib_node, 2964 const struct mlxsw_sp_fib_entry *fib_entry); 2965 2966 static int 2967 mlxsw_sp_nexthop_fib_entries_update(struct mlxsw_sp *mlxsw_sp, 2968 struct mlxsw_sp_nexthop_group *nh_grp) 2969 { 2970 struct mlxsw_sp_fib_entry *fib_entry; 2971 int err; 2972 2973 list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) { 2974 if (!mlxsw_sp_fib_node_entry_is_first(fib_entry->fib_node, 2975 fib_entry)) 2976 continue; 2977 err = mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry); 2978 if (err) 2979 return err; 2980 } 2981 return 0; 2982 } 2983 2984 static void 2985 mlxsw_sp_fib_entry_offload_refresh(struct mlxsw_sp_fib_entry *fib_entry, 2986 enum mlxsw_reg_ralue_op op, int err); 2987 2988 static void 2989 mlxsw_sp_nexthop_fib_entries_refresh(struct mlxsw_sp_nexthop_group *nh_grp) 2990 { 2991 enum mlxsw_reg_ralue_op op = MLXSW_REG_RALUE_OP_WRITE_WRITE; 2992 struct mlxsw_sp_fib_entry *fib_entry; 2993 2994 list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) { 2995 if (!mlxsw_sp_fib_node_entry_is_first(fib_entry->fib_node, 2996 fib_entry)) 2997 continue; 2998 mlxsw_sp_fib_entry_offload_refresh(fib_entry, op, 0); 2999 } 3000 } 3001 3002 static void mlxsw_sp_adj_grp_size_round_up(u16 *p_adj_grp_size) 3003 { 3004 /* Valid sizes for an adjacency group are: 3005 * 1-64, 512, 1024, 2048 and 4096. 3006 */ 3007 if (*p_adj_grp_size <= 64) 3008 return; 3009 else if (*p_adj_grp_size <= 512) 3010 *p_adj_grp_size = 512; 3011 else if (*p_adj_grp_size <= 1024) 3012 *p_adj_grp_size = 1024; 3013 else if (*p_adj_grp_size <= 2048) 3014 *p_adj_grp_size = 2048; 3015 else 3016 *p_adj_grp_size = 4096; 3017 } 3018 3019 static void mlxsw_sp_adj_grp_size_round_down(u16 *p_adj_grp_size, 3020 unsigned int alloc_size) 3021 { 3022 if (alloc_size >= 4096) 3023 *p_adj_grp_size = 4096; 3024 else if (alloc_size >= 2048) 3025 *p_adj_grp_size = 2048; 3026 else if (alloc_size >= 1024) 3027 *p_adj_grp_size = 1024; 3028 else if (alloc_size >= 512) 3029 *p_adj_grp_size = 512; 3030 } 3031 3032 static int mlxsw_sp_fix_adj_grp_size(struct mlxsw_sp *mlxsw_sp, 3033 u16 *p_adj_grp_size) 3034 { 3035 unsigned int alloc_size; 3036 int err; 3037 3038 /* Round up the requested group size to the next size supported 3039 * by the device and make sure the request can be satisfied. 3040 */ 3041 mlxsw_sp_adj_grp_size_round_up(p_adj_grp_size); 3042 err = mlxsw_sp_kvdl_alloc_size_query(mlxsw_sp, *p_adj_grp_size, 3043 &alloc_size); 3044 if (err) 3045 return err; 3046 /* It is possible the allocation results in more allocated 3047 * entries than requested. Try to use as much of them as 3048 * possible. 3049 */ 3050 mlxsw_sp_adj_grp_size_round_down(p_adj_grp_size, alloc_size); 3051 3052 return 0; 3053 } 3054 3055 static void 3056 mlxsw_sp_nexthop_group_normalize(struct mlxsw_sp_nexthop_group *nh_grp) 3057 { 3058 int i, g = 0, sum_norm_weight = 0; 3059 struct mlxsw_sp_nexthop *nh; 3060 3061 for (i = 0; i < nh_grp->count; i++) { 3062 nh = &nh_grp->nexthops[i]; 3063 3064 if (!nh->should_offload) 3065 continue; 3066 if (g > 0) 3067 g = gcd(nh->nh_weight, g); 3068 else 3069 g = nh->nh_weight; 3070 } 3071 3072 for (i = 0; i < nh_grp->count; i++) { 3073 nh = &nh_grp->nexthops[i]; 3074 3075 if (!nh->should_offload) 3076 continue; 3077 nh->norm_nh_weight = nh->nh_weight / g; 3078 sum_norm_weight += nh->norm_nh_weight; 3079 } 3080 3081 nh_grp->sum_norm_weight = sum_norm_weight; 3082 } 3083 3084 static void 3085 mlxsw_sp_nexthop_group_rebalance(struct mlxsw_sp_nexthop_group *nh_grp) 3086 { 3087 int total = nh_grp->sum_norm_weight; 3088 u16 ecmp_size = nh_grp->ecmp_size; 3089 int i, weight = 0, lower_bound = 0; 3090 3091 for (i = 0; i < nh_grp->count; i++) { 3092 struct mlxsw_sp_nexthop *nh = &nh_grp->nexthops[i]; 3093 int upper_bound; 3094 3095 if (!nh->should_offload) 3096 continue; 3097 weight += nh->norm_nh_weight; 3098 upper_bound = DIV_ROUND_CLOSEST(ecmp_size * weight, total); 3099 nh->num_adj_entries = upper_bound - lower_bound; 3100 lower_bound = upper_bound; 3101 } 3102 } 3103 3104 static void 3105 mlxsw_sp_nexthop_group_refresh(struct mlxsw_sp *mlxsw_sp, 3106 struct mlxsw_sp_nexthop_group *nh_grp) 3107 { 3108 u16 ecmp_size, old_ecmp_size; 3109 struct mlxsw_sp_nexthop *nh; 3110 bool offload_change = false; 3111 u32 adj_index; 3112 bool old_adj_index_valid; 3113 u32 old_adj_index; 3114 int i; 3115 int err; 3116 3117 if (!nh_grp->gateway) { 3118 mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp); 3119 return; 3120 } 3121 3122 for (i = 0; i < nh_grp->count; i++) { 3123 nh = &nh_grp->nexthops[i]; 3124 3125 if (nh->should_offload != nh->offloaded) { 3126 offload_change = true; 3127 if (nh->should_offload) 3128 nh->update = 1; 3129 } 3130 } 3131 if (!offload_change) { 3132 /* Nothing was added or removed, so no need to reallocate. Just 3133 * update MAC on existing adjacency indexes. 3134 */ 3135 err = mlxsw_sp_nexthop_group_update(mlxsw_sp, nh_grp, false); 3136 if (err) { 3137 dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n"); 3138 goto set_trap; 3139 } 3140 return; 3141 } 3142 mlxsw_sp_nexthop_group_normalize(nh_grp); 3143 if (!nh_grp->sum_norm_weight) 3144 /* No neigh of this group is connected so we just set 3145 * the trap and let everthing flow through kernel. 3146 */ 3147 goto set_trap; 3148 3149 ecmp_size = nh_grp->sum_norm_weight; 3150 err = mlxsw_sp_fix_adj_grp_size(mlxsw_sp, &ecmp_size); 3151 if (err) 3152 /* No valid allocation size available. */ 3153 goto set_trap; 3154 3155 err = mlxsw_sp_kvdl_alloc(mlxsw_sp, ecmp_size, &adj_index); 3156 if (err) { 3157 /* We ran out of KVD linear space, just set the 3158 * trap and let everything flow through kernel. 3159 */ 3160 dev_warn(mlxsw_sp->bus_info->dev, "Failed to allocate KVD linear area for nexthop group.\n"); 3161 goto set_trap; 3162 } 3163 old_adj_index_valid = nh_grp->adj_index_valid; 3164 old_adj_index = nh_grp->adj_index; 3165 old_ecmp_size = nh_grp->ecmp_size; 3166 nh_grp->adj_index_valid = 1; 3167 nh_grp->adj_index = adj_index; 3168 nh_grp->ecmp_size = ecmp_size; 3169 mlxsw_sp_nexthop_group_rebalance(nh_grp); 3170 err = mlxsw_sp_nexthop_group_update(mlxsw_sp, nh_grp, true); 3171 if (err) { 3172 dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n"); 3173 goto set_trap; 3174 } 3175 3176 if (!old_adj_index_valid) { 3177 /* The trap was set for fib entries, so we have to call 3178 * fib entry update to unset it and use adjacency index. 3179 */ 3180 err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp); 3181 if (err) { 3182 dev_warn(mlxsw_sp->bus_info->dev, "Failed to add adjacency index to fib entries.\n"); 3183 goto set_trap; 3184 } 3185 return; 3186 } 3187 3188 err = mlxsw_sp_adj_index_mass_update(mlxsw_sp, nh_grp, 3189 old_adj_index, old_ecmp_size); 3190 mlxsw_sp_kvdl_free(mlxsw_sp, old_adj_index); 3191 if (err) { 3192 dev_warn(mlxsw_sp->bus_info->dev, "Failed to mass-update adjacency index for nexthop group.\n"); 3193 goto set_trap; 3194 } 3195 3196 /* Offload state within the group changed, so update the flags. */ 3197 mlxsw_sp_nexthop_fib_entries_refresh(nh_grp); 3198 3199 return; 3200 3201 set_trap: 3202 old_adj_index_valid = nh_grp->adj_index_valid; 3203 nh_grp->adj_index_valid = 0; 3204 for (i = 0; i < nh_grp->count; i++) { 3205 nh = &nh_grp->nexthops[i]; 3206 nh->offloaded = 0; 3207 } 3208 err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp); 3209 if (err) 3210 dev_warn(mlxsw_sp->bus_info->dev, "Failed to set traps for fib entries.\n"); 3211 if (old_adj_index_valid) 3212 mlxsw_sp_kvdl_free(mlxsw_sp, nh_grp->adj_index); 3213 } 3214 3215 static void __mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp_nexthop *nh, 3216 bool removing) 3217 { 3218 if (!removing) 3219 nh->should_offload = 1; 3220 else if (nh->offloaded) 3221 nh->should_offload = 0; 3222 nh->update = 1; 3223 } 3224 3225 static void 3226 mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp, 3227 struct mlxsw_sp_neigh_entry *neigh_entry, 3228 bool removing) 3229 { 3230 struct mlxsw_sp_nexthop *nh; 3231 3232 list_for_each_entry(nh, &neigh_entry->nexthop_list, 3233 neigh_list_node) { 3234 __mlxsw_sp_nexthop_neigh_update(nh, removing); 3235 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp); 3236 } 3237 } 3238 3239 static void mlxsw_sp_nexthop_rif_init(struct mlxsw_sp_nexthop *nh, 3240 struct mlxsw_sp_rif *rif) 3241 { 3242 if (nh->rif) 3243 return; 3244 3245 nh->rif = rif; 3246 list_add(&nh->rif_list_node, &rif->nexthop_list); 3247 } 3248 3249 static void mlxsw_sp_nexthop_rif_fini(struct mlxsw_sp_nexthop *nh) 3250 { 3251 if (!nh->rif) 3252 return; 3253 3254 list_del(&nh->rif_list_node); 3255 nh->rif = NULL; 3256 } 3257 3258 static int mlxsw_sp_nexthop_neigh_init(struct mlxsw_sp *mlxsw_sp, 3259 struct mlxsw_sp_nexthop *nh) 3260 { 3261 struct mlxsw_sp_neigh_entry *neigh_entry; 3262 struct neighbour *n; 3263 u8 nud_state, dead; 3264 int err; 3265 3266 if (!nh->nh_grp->gateway || nh->neigh_entry) 3267 return 0; 3268 3269 /* Take a reference of neigh here ensuring that neigh would 3270 * not be destructed before the nexthop entry is finished. 3271 * The reference is taken either in neigh_lookup() or 3272 * in neigh_create() in case n is not found. 3273 */ 3274 n = neigh_lookup(nh->nh_grp->neigh_tbl, &nh->gw_addr, nh->rif->dev); 3275 if (!n) { 3276 n = neigh_create(nh->nh_grp->neigh_tbl, &nh->gw_addr, 3277 nh->rif->dev); 3278 if (IS_ERR(n)) 3279 return PTR_ERR(n); 3280 neigh_event_send(n, NULL); 3281 } 3282 neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n); 3283 if (!neigh_entry) { 3284 neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n); 3285 if (IS_ERR(neigh_entry)) { 3286 err = -EINVAL; 3287 goto err_neigh_entry_create; 3288 } 3289 } 3290 3291 /* If that is the first nexthop connected to that neigh, add to 3292 * nexthop_neighs_list 3293 */ 3294 if (list_empty(&neigh_entry->nexthop_list)) 3295 list_add_tail(&neigh_entry->nexthop_neighs_list_node, 3296 &mlxsw_sp->router->nexthop_neighs_list); 3297 3298 nh->neigh_entry = neigh_entry; 3299 list_add_tail(&nh->neigh_list_node, &neigh_entry->nexthop_list); 3300 read_lock_bh(&n->lock); 3301 nud_state = n->nud_state; 3302 dead = n->dead; 3303 read_unlock_bh(&n->lock); 3304 __mlxsw_sp_nexthop_neigh_update(nh, !(nud_state & NUD_VALID && !dead)); 3305 3306 return 0; 3307 3308 err_neigh_entry_create: 3309 neigh_release(n); 3310 return err; 3311 } 3312 3313 static void mlxsw_sp_nexthop_neigh_fini(struct mlxsw_sp *mlxsw_sp, 3314 struct mlxsw_sp_nexthop *nh) 3315 { 3316 struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry; 3317 struct neighbour *n; 3318 3319 if (!neigh_entry) 3320 return; 3321 n = neigh_entry->key.n; 3322 3323 __mlxsw_sp_nexthop_neigh_update(nh, true); 3324 list_del(&nh->neigh_list_node); 3325 nh->neigh_entry = NULL; 3326 3327 /* If that is the last nexthop connected to that neigh, remove from 3328 * nexthop_neighs_list 3329 */ 3330 if (list_empty(&neigh_entry->nexthop_list)) 3331 list_del(&neigh_entry->nexthop_neighs_list_node); 3332 3333 if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list)) 3334 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry); 3335 3336 neigh_release(n); 3337 } 3338 3339 static bool mlxsw_sp_ipip_netdev_ul_up(struct net_device *ol_dev) 3340 { 3341 struct net_device *ul_dev = __mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev); 3342 3343 return ul_dev ? (ul_dev->flags & IFF_UP) : true; 3344 } 3345 3346 static int mlxsw_sp_nexthop_ipip_init(struct mlxsw_sp *mlxsw_sp, 3347 struct mlxsw_sp_nexthop *nh, 3348 struct net_device *ol_dev) 3349 { 3350 bool removing; 3351 3352 if (!nh->nh_grp->gateway || nh->ipip_entry) 3353 return 0; 3354 3355 nh->ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev); 3356 if (!nh->ipip_entry) 3357 return -ENOENT; 3358 3359 removing = !mlxsw_sp_ipip_netdev_ul_up(ol_dev); 3360 __mlxsw_sp_nexthop_neigh_update(nh, removing); 3361 return 0; 3362 } 3363 3364 static void mlxsw_sp_nexthop_ipip_fini(struct mlxsw_sp *mlxsw_sp, 3365 struct mlxsw_sp_nexthop *nh) 3366 { 3367 struct mlxsw_sp_ipip_entry *ipip_entry = nh->ipip_entry; 3368 3369 if (!ipip_entry) 3370 return; 3371 3372 __mlxsw_sp_nexthop_neigh_update(nh, true); 3373 nh->ipip_entry = NULL; 3374 } 3375 3376 static bool mlxsw_sp_nexthop4_ipip_type(const struct mlxsw_sp *mlxsw_sp, 3377 const struct fib_nh *fib_nh, 3378 enum mlxsw_sp_ipip_type *p_ipipt) 3379 { 3380 struct net_device *dev = fib_nh->nh_dev; 3381 3382 return dev && 3383 fib_nh->nh_parent->fib_type == RTN_UNICAST && 3384 mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, p_ipipt); 3385 } 3386 3387 static void mlxsw_sp_nexthop_type_fini(struct mlxsw_sp *mlxsw_sp, 3388 struct mlxsw_sp_nexthop *nh) 3389 { 3390 switch (nh->type) { 3391 case MLXSW_SP_NEXTHOP_TYPE_ETH: 3392 mlxsw_sp_nexthop_neigh_fini(mlxsw_sp, nh); 3393 mlxsw_sp_nexthop_rif_fini(nh); 3394 break; 3395 case MLXSW_SP_NEXTHOP_TYPE_IPIP: 3396 mlxsw_sp_nexthop_rif_fini(nh); 3397 mlxsw_sp_nexthop_ipip_fini(mlxsw_sp, nh); 3398 break; 3399 } 3400 } 3401 3402 static int mlxsw_sp_nexthop4_type_init(struct mlxsw_sp *mlxsw_sp, 3403 struct mlxsw_sp_nexthop *nh, 3404 struct fib_nh *fib_nh) 3405 { 3406 struct mlxsw_sp_router *router = mlxsw_sp->router; 3407 struct net_device *dev = fib_nh->nh_dev; 3408 enum mlxsw_sp_ipip_type ipipt; 3409 struct mlxsw_sp_rif *rif; 3410 int err; 3411 3412 if (mlxsw_sp_nexthop4_ipip_type(mlxsw_sp, fib_nh, &ipipt) && 3413 router->ipip_ops_arr[ipipt]->can_offload(mlxsw_sp, dev, 3414 MLXSW_SP_L3_PROTO_IPV4)) { 3415 nh->type = MLXSW_SP_NEXTHOP_TYPE_IPIP; 3416 err = mlxsw_sp_nexthop_ipip_init(mlxsw_sp, nh, dev); 3417 if (err) 3418 return err; 3419 mlxsw_sp_nexthop_rif_init(nh, &nh->ipip_entry->ol_lb->common); 3420 return 0; 3421 } 3422 3423 nh->type = MLXSW_SP_NEXTHOP_TYPE_ETH; 3424 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 3425 if (!rif) 3426 return 0; 3427 3428 mlxsw_sp_nexthop_rif_init(nh, rif); 3429 err = mlxsw_sp_nexthop_neigh_init(mlxsw_sp, nh); 3430 if (err) 3431 goto err_neigh_init; 3432 3433 return 0; 3434 3435 err_neigh_init: 3436 mlxsw_sp_nexthop_rif_fini(nh); 3437 return err; 3438 } 3439 3440 static void mlxsw_sp_nexthop4_type_fini(struct mlxsw_sp *mlxsw_sp, 3441 struct mlxsw_sp_nexthop *nh) 3442 { 3443 mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh); 3444 } 3445 3446 static int mlxsw_sp_nexthop4_init(struct mlxsw_sp *mlxsw_sp, 3447 struct mlxsw_sp_nexthop_group *nh_grp, 3448 struct mlxsw_sp_nexthop *nh, 3449 struct fib_nh *fib_nh) 3450 { 3451 struct net_device *dev = fib_nh->nh_dev; 3452 struct in_device *in_dev; 3453 int err; 3454 3455 nh->nh_grp = nh_grp; 3456 nh->key.fib_nh = fib_nh; 3457 #ifdef CONFIG_IP_ROUTE_MULTIPATH 3458 nh->nh_weight = fib_nh->nh_weight; 3459 #else 3460 nh->nh_weight = 1; 3461 #endif 3462 memcpy(&nh->gw_addr, &fib_nh->nh_gw, sizeof(fib_nh->nh_gw)); 3463 err = mlxsw_sp_nexthop_insert(mlxsw_sp, nh); 3464 if (err) 3465 return err; 3466 3467 mlxsw_sp_nexthop_counter_alloc(mlxsw_sp, nh); 3468 list_add_tail(&nh->router_list_node, &mlxsw_sp->router->nexthop_list); 3469 3470 if (!dev) 3471 return 0; 3472 3473 in_dev = __in_dev_get_rtnl(dev); 3474 if (in_dev && IN_DEV_IGNORE_ROUTES_WITH_LINKDOWN(in_dev) && 3475 fib_nh->nh_flags & RTNH_F_LINKDOWN) 3476 return 0; 3477 3478 err = mlxsw_sp_nexthop4_type_init(mlxsw_sp, nh, fib_nh); 3479 if (err) 3480 goto err_nexthop_neigh_init; 3481 3482 return 0; 3483 3484 err_nexthop_neigh_init: 3485 mlxsw_sp_nexthop_remove(mlxsw_sp, nh); 3486 return err; 3487 } 3488 3489 static void mlxsw_sp_nexthop4_fini(struct mlxsw_sp *mlxsw_sp, 3490 struct mlxsw_sp_nexthop *nh) 3491 { 3492 mlxsw_sp_nexthop4_type_fini(mlxsw_sp, nh); 3493 list_del(&nh->router_list_node); 3494 mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh); 3495 mlxsw_sp_nexthop_remove(mlxsw_sp, nh); 3496 } 3497 3498 static void mlxsw_sp_nexthop4_event(struct mlxsw_sp *mlxsw_sp, 3499 unsigned long event, struct fib_nh *fib_nh) 3500 { 3501 struct mlxsw_sp_nexthop_key key; 3502 struct mlxsw_sp_nexthop *nh; 3503 3504 if (mlxsw_sp->router->aborted) 3505 return; 3506 3507 key.fib_nh = fib_nh; 3508 nh = mlxsw_sp_nexthop_lookup(mlxsw_sp, key); 3509 if (WARN_ON_ONCE(!nh)) 3510 return; 3511 3512 switch (event) { 3513 case FIB_EVENT_NH_ADD: 3514 mlxsw_sp_nexthop4_type_init(mlxsw_sp, nh, fib_nh); 3515 break; 3516 case FIB_EVENT_NH_DEL: 3517 mlxsw_sp_nexthop4_type_fini(mlxsw_sp, nh); 3518 break; 3519 } 3520 3521 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp); 3522 } 3523 3524 static void mlxsw_sp_nexthop_rif_update(struct mlxsw_sp *mlxsw_sp, 3525 struct mlxsw_sp_rif *rif) 3526 { 3527 struct mlxsw_sp_nexthop *nh; 3528 bool removing; 3529 3530 list_for_each_entry(nh, &rif->nexthop_list, rif_list_node) { 3531 switch (nh->type) { 3532 case MLXSW_SP_NEXTHOP_TYPE_ETH: 3533 removing = false; 3534 break; 3535 case MLXSW_SP_NEXTHOP_TYPE_IPIP: 3536 removing = !mlxsw_sp_ipip_netdev_ul_up(rif->dev); 3537 break; 3538 default: 3539 WARN_ON(1); 3540 continue; 3541 } 3542 3543 __mlxsw_sp_nexthop_neigh_update(nh, removing); 3544 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp); 3545 } 3546 } 3547 3548 static void mlxsw_sp_nexthop_rif_gone_sync(struct mlxsw_sp *mlxsw_sp, 3549 struct mlxsw_sp_rif *rif) 3550 { 3551 struct mlxsw_sp_nexthop *nh, *tmp; 3552 3553 list_for_each_entry_safe(nh, tmp, &rif->nexthop_list, rif_list_node) { 3554 mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh); 3555 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp); 3556 } 3557 } 3558 3559 static bool mlxsw_sp_fi_is_gateway(const struct mlxsw_sp *mlxsw_sp, 3560 const struct fib_info *fi) 3561 { 3562 return fi->fib_nh->nh_scope == RT_SCOPE_LINK || 3563 mlxsw_sp_nexthop4_ipip_type(mlxsw_sp, fi->fib_nh, NULL); 3564 } 3565 3566 static struct mlxsw_sp_nexthop_group * 3567 mlxsw_sp_nexthop4_group_create(struct mlxsw_sp *mlxsw_sp, struct fib_info *fi) 3568 { 3569 struct mlxsw_sp_nexthop_group *nh_grp; 3570 struct mlxsw_sp_nexthop *nh; 3571 struct fib_nh *fib_nh; 3572 size_t alloc_size; 3573 int i; 3574 int err; 3575 3576 alloc_size = sizeof(*nh_grp) + 3577 fi->fib_nhs * sizeof(struct mlxsw_sp_nexthop); 3578 nh_grp = kzalloc(alloc_size, GFP_KERNEL); 3579 if (!nh_grp) 3580 return ERR_PTR(-ENOMEM); 3581 nh_grp->priv = fi; 3582 INIT_LIST_HEAD(&nh_grp->fib_list); 3583 nh_grp->neigh_tbl = &arp_tbl; 3584 3585 nh_grp->gateway = mlxsw_sp_fi_is_gateway(mlxsw_sp, fi); 3586 nh_grp->count = fi->fib_nhs; 3587 fib_info_hold(fi); 3588 for (i = 0; i < nh_grp->count; i++) { 3589 nh = &nh_grp->nexthops[i]; 3590 fib_nh = &fi->fib_nh[i]; 3591 err = mlxsw_sp_nexthop4_init(mlxsw_sp, nh_grp, nh, fib_nh); 3592 if (err) 3593 goto err_nexthop4_init; 3594 } 3595 err = mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp); 3596 if (err) 3597 goto err_nexthop_group_insert; 3598 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp); 3599 return nh_grp; 3600 3601 err_nexthop_group_insert: 3602 err_nexthop4_init: 3603 for (i--; i >= 0; i--) { 3604 nh = &nh_grp->nexthops[i]; 3605 mlxsw_sp_nexthop4_fini(mlxsw_sp, nh); 3606 } 3607 fib_info_put(fi); 3608 kfree(nh_grp); 3609 return ERR_PTR(err); 3610 } 3611 3612 static void 3613 mlxsw_sp_nexthop4_group_destroy(struct mlxsw_sp *mlxsw_sp, 3614 struct mlxsw_sp_nexthop_group *nh_grp) 3615 { 3616 struct mlxsw_sp_nexthop *nh; 3617 int i; 3618 3619 mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp); 3620 for (i = 0; i < nh_grp->count; i++) { 3621 nh = &nh_grp->nexthops[i]; 3622 mlxsw_sp_nexthop4_fini(mlxsw_sp, nh); 3623 } 3624 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp); 3625 WARN_ON_ONCE(nh_grp->adj_index_valid); 3626 fib_info_put(mlxsw_sp_nexthop4_group_fi(nh_grp)); 3627 kfree(nh_grp); 3628 } 3629 3630 static int mlxsw_sp_nexthop4_group_get(struct mlxsw_sp *mlxsw_sp, 3631 struct mlxsw_sp_fib_entry *fib_entry, 3632 struct fib_info *fi) 3633 { 3634 struct mlxsw_sp_nexthop_group *nh_grp; 3635 3636 nh_grp = mlxsw_sp_nexthop4_group_lookup(mlxsw_sp, fi); 3637 if (!nh_grp) { 3638 nh_grp = mlxsw_sp_nexthop4_group_create(mlxsw_sp, fi); 3639 if (IS_ERR(nh_grp)) 3640 return PTR_ERR(nh_grp); 3641 } 3642 list_add_tail(&fib_entry->nexthop_group_node, &nh_grp->fib_list); 3643 fib_entry->nh_group = nh_grp; 3644 return 0; 3645 } 3646 3647 static void mlxsw_sp_nexthop4_group_put(struct mlxsw_sp *mlxsw_sp, 3648 struct mlxsw_sp_fib_entry *fib_entry) 3649 { 3650 struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group; 3651 3652 list_del(&fib_entry->nexthop_group_node); 3653 if (!list_empty(&nh_grp->fib_list)) 3654 return; 3655 mlxsw_sp_nexthop4_group_destroy(mlxsw_sp, nh_grp); 3656 } 3657 3658 static bool 3659 mlxsw_sp_fib4_entry_should_offload(const struct mlxsw_sp_fib_entry *fib_entry) 3660 { 3661 struct mlxsw_sp_fib4_entry *fib4_entry; 3662 3663 fib4_entry = container_of(fib_entry, struct mlxsw_sp_fib4_entry, 3664 common); 3665 return !fib4_entry->tos; 3666 } 3667 3668 static bool 3669 mlxsw_sp_fib_entry_should_offload(const struct mlxsw_sp_fib_entry *fib_entry) 3670 { 3671 struct mlxsw_sp_nexthop_group *nh_group = fib_entry->nh_group; 3672 3673 switch (fib_entry->fib_node->fib->proto) { 3674 case MLXSW_SP_L3_PROTO_IPV4: 3675 if (!mlxsw_sp_fib4_entry_should_offload(fib_entry)) 3676 return false; 3677 break; 3678 case MLXSW_SP_L3_PROTO_IPV6: 3679 break; 3680 } 3681 3682 switch (fib_entry->type) { 3683 case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE: 3684 return !!nh_group->adj_index_valid; 3685 case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL: 3686 return !!nh_group->nh_rif; 3687 case MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP: 3688 return true; 3689 default: 3690 return false; 3691 } 3692 } 3693 3694 static struct mlxsw_sp_nexthop * 3695 mlxsw_sp_rt6_nexthop(struct mlxsw_sp_nexthop_group *nh_grp, 3696 const struct mlxsw_sp_rt6 *mlxsw_sp_rt6) 3697 { 3698 int i; 3699 3700 for (i = 0; i < nh_grp->count; i++) { 3701 struct mlxsw_sp_nexthop *nh = &nh_grp->nexthops[i]; 3702 struct rt6_info *rt = mlxsw_sp_rt6->rt; 3703 3704 if (nh->rif && nh->rif->dev == rt->dst.dev && 3705 ipv6_addr_equal((const struct in6_addr *) &nh->gw_addr, 3706 &rt->rt6i_gateway)) 3707 return nh; 3708 continue; 3709 } 3710 3711 return NULL; 3712 } 3713 3714 static void 3715 mlxsw_sp_fib4_entry_offload_set(struct mlxsw_sp_fib_entry *fib_entry) 3716 { 3717 struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group; 3718 int i; 3719 3720 if (fib_entry->type == MLXSW_SP_FIB_ENTRY_TYPE_LOCAL || 3721 fib_entry->type == MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP) { 3722 nh_grp->nexthops->key.fib_nh->nh_flags |= RTNH_F_OFFLOAD; 3723 return; 3724 } 3725 3726 for (i = 0; i < nh_grp->count; i++) { 3727 struct mlxsw_sp_nexthop *nh = &nh_grp->nexthops[i]; 3728 3729 if (nh->offloaded) 3730 nh->key.fib_nh->nh_flags |= RTNH_F_OFFLOAD; 3731 else 3732 nh->key.fib_nh->nh_flags &= ~RTNH_F_OFFLOAD; 3733 } 3734 } 3735 3736 static void 3737 mlxsw_sp_fib4_entry_offload_unset(struct mlxsw_sp_fib_entry *fib_entry) 3738 { 3739 struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group; 3740 int i; 3741 3742 for (i = 0; i < nh_grp->count; i++) { 3743 struct mlxsw_sp_nexthop *nh = &nh_grp->nexthops[i]; 3744 3745 nh->key.fib_nh->nh_flags &= ~RTNH_F_OFFLOAD; 3746 } 3747 } 3748 3749 static void 3750 mlxsw_sp_fib6_entry_offload_set(struct mlxsw_sp_fib_entry *fib_entry) 3751 { 3752 struct mlxsw_sp_fib6_entry *fib6_entry; 3753 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 3754 3755 fib6_entry = container_of(fib_entry, struct mlxsw_sp_fib6_entry, 3756 common); 3757 3758 if (fib_entry->type == MLXSW_SP_FIB_ENTRY_TYPE_LOCAL) { 3759 list_first_entry(&fib6_entry->rt6_list, struct mlxsw_sp_rt6, 3760 list)->rt->rt6i_nh_flags |= RTNH_F_OFFLOAD; 3761 return; 3762 } 3763 3764 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) { 3765 struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group; 3766 struct mlxsw_sp_nexthop *nh; 3767 3768 nh = mlxsw_sp_rt6_nexthop(nh_grp, mlxsw_sp_rt6); 3769 if (nh && nh->offloaded) 3770 mlxsw_sp_rt6->rt->rt6i_nh_flags |= RTNH_F_OFFLOAD; 3771 else 3772 mlxsw_sp_rt6->rt->rt6i_nh_flags &= ~RTNH_F_OFFLOAD; 3773 } 3774 } 3775 3776 static void 3777 mlxsw_sp_fib6_entry_offload_unset(struct mlxsw_sp_fib_entry *fib_entry) 3778 { 3779 struct mlxsw_sp_fib6_entry *fib6_entry; 3780 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 3781 3782 fib6_entry = container_of(fib_entry, struct mlxsw_sp_fib6_entry, 3783 common); 3784 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) { 3785 struct rt6_info *rt = mlxsw_sp_rt6->rt; 3786 3787 rt->rt6i_nh_flags &= ~RTNH_F_OFFLOAD; 3788 } 3789 } 3790 3791 static void mlxsw_sp_fib_entry_offload_set(struct mlxsw_sp_fib_entry *fib_entry) 3792 { 3793 switch (fib_entry->fib_node->fib->proto) { 3794 case MLXSW_SP_L3_PROTO_IPV4: 3795 mlxsw_sp_fib4_entry_offload_set(fib_entry); 3796 break; 3797 case MLXSW_SP_L3_PROTO_IPV6: 3798 mlxsw_sp_fib6_entry_offload_set(fib_entry); 3799 break; 3800 } 3801 } 3802 3803 static void 3804 mlxsw_sp_fib_entry_offload_unset(struct mlxsw_sp_fib_entry *fib_entry) 3805 { 3806 switch (fib_entry->fib_node->fib->proto) { 3807 case MLXSW_SP_L3_PROTO_IPV4: 3808 mlxsw_sp_fib4_entry_offload_unset(fib_entry); 3809 break; 3810 case MLXSW_SP_L3_PROTO_IPV6: 3811 mlxsw_sp_fib6_entry_offload_unset(fib_entry); 3812 break; 3813 } 3814 } 3815 3816 static void 3817 mlxsw_sp_fib_entry_offload_refresh(struct mlxsw_sp_fib_entry *fib_entry, 3818 enum mlxsw_reg_ralue_op op, int err) 3819 { 3820 switch (op) { 3821 case MLXSW_REG_RALUE_OP_WRITE_DELETE: 3822 return mlxsw_sp_fib_entry_offload_unset(fib_entry); 3823 case MLXSW_REG_RALUE_OP_WRITE_WRITE: 3824 if (err) 3825 return; 3826 if (mlxsw_sp_fib_entry_should_offload(fib_entry)) 3827 mlxsw_sp_fib_entry_offload_set(fib_entry); 3828 else 3829 mlxsw_sp_fib_entry_offload_unset(fib_entry); 3830 return; 3831 default: 3832 return; 3833 } 3834 } 3835 3836 static void 3837 mlxsw_sp_fib_entry_ralue_pack(char *ralue_pl, 3838 const struct mlxsw_sp_fib_entry *fib_entry, 3839 enum mlxsw_reg_ralue_op op) 3840 { 3841 struct mlxsw_sp_fib *fib = fib_entry->fib_node->fib; 3842 enum mlxsw_reg_ralxx_protocol proto; 3843 u32 *p_dip; 3844 3845 proto = (enum mlxsw_reg_ralxx_protocol) fib->proto; 3846 3847 switch (fib->proto) { 3848 case MLXSW_SP_L3_PROTO_IPV4: 3849 p_dip = (u32 *) fib_entry->fib_node->key.addr; 3850 mlxsw_reg_ralue_pack4(ralue_pl, proto, op, fib->vr->id, 3851 fib_entry->fib_node->key.prefix_len, 3852 *p_dip); 3853 break; 3854 case MLXSW_SP_L3_PROTO_IPV6: 3855 mlxsw_reg_ralue_pack6(ralue_pl, proto, op, fib->vr->id, 3856 fib_entry->fib_node->key.prefix_len, 3857 fib_entry->fib_node->key.addr); 3858 break; 3859 } 3860 } 3861 3862 static int mlxsw_sp_fib_entry_op_remote(struct mlxsw_sp *mlxsw_sp, 3863 struct mlxsw_sp_fib_entry *fib_entry, 3864 enum mlxsw_reg_ralue_op op) 3865 { 3866 char ralue_pl[MLXSW_REG_RALUE_LEN]; 3867 enum mlxsw_reg_ralue_trap_action trap_action; 3868 u16 trap_id = 0; 3869 u32 adjacency_index = 0; 3870 u16 ecmp_size = 0; 3871 3872 /* In case the nexthop group adjacency index is valid, use it 3873 * with provided ECMP size. Otherwise, setup trap and pass 3874 * traffic to kernel. 3875 */ 3876 if (mlxsw_sp_fib_entry_should_offload(fib_entry)) { 3877 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP; 3878 adjacency_index = fib_entry->nh_group->adj_index; 3879 ecmp_size = fib_entry->nh_group->ecmp_size; 3880 } else { 3881 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP; 3882 trap_id = MLXSW_TRAP_ID_RTR_INGRESS0; 3883 } 3884 3885 mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op); 3886 mlxsw_reg_ralue_act_remote_pack(ralue_pl, trap_action, trap_id, 3887 adjacency_index, ecmp_size); 3888 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl); 3889 } 3890 3891 static int mlxsw_sp_fib_entry_op_local(struct mlxsw_sp *mlxsw_sp, 3892 struct mlxsw_sp_fib_entry *fib_entry, 3893 enum mlxsw_reg_ralue_op op) 3894 { 3895 struct mlxsw_sp_rif *rif = fib_entry->nh_group->nh_rif; 3896 enum mlxsw_reg_ralue_trap_action trap_action; 3897 char ralue_pl[MLXSW_REG_RALUE_LEN]; 3898 u16 trap_id = 0; 3899 u16 rif_index = 0; 3900 3901 if (mlxsw_sp_fib_entry_should_offload(fib_entry)) { 3902 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP; 3903 rif_index = rif->rif_index; 3904 } else { 3905 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP; 3906 trap_id = MLXSW_TRAP_ID_RTR_INGRESS0; 3907 } 3908 3909 mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op); 3910 mlxsw_reg_ralue_act_local_pack(ralue_pl, trap_action, trap_id, 3911 rif_index); 3912 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl); 3913 } 3914 3915 static int mlxsw_sp_fib_entry_op_trap(struct mlxsw_sp *mlxsw_sp, 3916 struct mlxsw_sp_fib_entry *fib_entry, 3917 enum mlxsw_reg_ralue_op op) 3918 { 3919 char ralue_pl[MLXSW_REG_RALUE_LEN]; 3920 3921 mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op); 3922 mlxsw_reg_ralue_act_ip2me_pack(ralue_pl); 3923 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl); 3924 } 3925 3926 static int 3927 mlxsw_sp_fib_entry_op_ipip_decap(struct mlxsw_sp *mlxsw_sp, 3928 struct mlxsw_sp_fib_entry *fib_entry, 3929 enum mlxsw_reg_ralue_op op) 3930 { 3931 struct mlxsw_sp_ipip_entry *ipip_entry = fib_entry->decap.ipip_entry; 3932 const struct mlxsw_sp_ipip_ops *ipip_ops; 3933 3934 if (WARN_ON(!ipip_entry)) 3935 return -EINVAL; 3936 3937 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt]; 3938 return ipip_ops->fib_entry_op(mlxsw_sp, ipip_entry, op, 3939 fib_entry->decap.tunnel_index); 3940 } 3941 3942 static int __mlxsw_sp_fib_entry_op(struct mlxsw_sp *mlxsw_sp, 3943 struct mlxsw_sp_fib_entry *fib_entry, 3944 enum mlxsw_reg_ralue_op op) 3945 { 3946 switch (fib_entry->type) { 3947 case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE: 3948 return mlxsw_sp_fib_entry_op_remote(mlxsw_sp, fib_entry, op); 3949 case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL: 3950 return mlxsw_sp_fib_entry_op_local(mlxsw_sp, fib_entry, op); 3951 case MLXSW_SP_FIB_ENTRY_TYPE_TRAP: 3952 return mlxsw_sp_fib_entry_op_trap(mlxsw_sp, fib_entry, op); 3953 case MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP: 3954 return mlxsw_sp_fib_entry_op_ipip_decap(mlxsw_sp, 3955 fib_entry, op); 3956 } 3957 return -EINVAL; 3958 } 3959 3960 static int mlxsw_sp_fib_entry_op(struct mlxsw_sp *mlxsw_sp, 3961 struct mlxsw_sp_fib_entry *fib_entry, 3962 enum mlxsw_reg_ralue_op op) 3963 { 3964 int err = __mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry, op); 3965 3966 mlxsw_sp_fib_entry_offload_refresh(fib_entry, op, err); 3967 3968 return err; 3969 } 3970 3971 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp, 3972 struct mlxsw_sp_fib_entry *fib_entry) 3973 { 3974 return mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry, 3975 MLXSW_REG_RALUE_OP_WRITE_WRITE); 3976 } 3977 3978 static int mlxsw_sp_fib_entry_del(struct mlxsw_sp *mlxsw_sp, 3979 struct mlxsw_sp_fib_entry *fib_entry) 3980 { 3981 return mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry, 3982 MLXSW_REG_RALUE_OP_WRITE_DELETE); 3983 } 3984 3985 static int 3986 mlxsw_sp_fib4_entry_type_set(struct mlxsw_sp *mlxsw_sp, 3987 const struct fib_entry_notifier_info *fen_info, 3988 struct mlxsw_sp_fib_entry *fib_entry) 3989 { 3990 union mlxsw_sp_l3addr dip = { .addr4 = htonl(fen_info->dst) }; 3991 struct net_device *dev = fen_info->fi->fib_dev; 3992 struct mlxsw_sp_ipip_entry *ipip_entry; 3993 struct fib_info *fi = fen_info->fi; 3994 3995 switch (fen_info->type) { 3996 case RTN_LOCAL: 3997 ipip_entry = mlxsw_sp_ipip_entry_find_by_decap(mlxsw_sp, dev, 3998 MLXSW_SP_L3_PROTO_IPV4, dip); 3999 if (ipip_entry) { 4000 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP; 4001 return mlxsw_sp_fib_entry_decap_init(mlxsw_sp, 4002 fib_entry, 4003 ipip_entry); 4004 } 4005 /* fall through */ 4006 case RTN_BROADCAST: 4007 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP; 4008 return 0; 4009 case RTN_UNREACHABLE: /* fall through */ 4010 case RTN_BLACKHOLE: /* fall through */ 4011 case RTN_PROHIBIT: 4012 /* Packets hitting these routes need to be trapped, but 4013 * can do so with a lower priority than packets directed 4014 * at the host, so use action type local instead of trap. 4015 */ 4016 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL; 4017 return 0; 4018 case RTN_UNICAST: 4019 if (mlxsw_sp_fi_is_gateway(mlxsw_sp, fi)) 4020 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_REMOTE; 4021 else 4022 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL; 4023 return 0; 4024 default: 4025 return -EINVAL; 4026 } 4027 } 4028 4029 static struct mlxsw_sp_fib4_entry * 4030 mlxsw_sp_fib4_entry_create(struct mlxsw_sp *mlxsw_sp, 4031 struct mlxsw_sp_fib_node *fib_node, 4032 const struct fib_entry_notifier_info *fen_info) 4033 { 4034 struct mlxsw_sp_fib4_entry *fib4_entry; 4035 struct mlxsw_sp_fib_entry *fib_entry; 4036 int err; 4037 4038 fib4_entry = kzalloc(sizeof(*fib4_entry), GFP_KERNEL); 4039 if (!fib4_entry) 4040 return ERR_PTR(-ENOMEM); 4041 fib_entry = &fib4_entry->common; 4042 4043 err = mlxsw_sp_fib4_entry_type_set(mlxsw_sp, fen_info, fib_entry); 4044 if (err) 4045 goto err_fib4_entry_type_set; 4046 4047 err = mlxsw_sp_nexthop4_group_get(mlxsw_sp, fib_entry, fen_info->fi); 4048 if (err) 4049 goto err_nexthop4_group_get; 4050 4051 fib4_entry->prio = fen_info->fi->fib_priority; 4052 fib4_entry->tb_id = fen_info->tb_id; 4053 fib4_entry->type = fen_info->type; 4054 fib4_entry->tos = fen_info->tos; 4055 4056 fib_entry->fib_node = fib_node; 4057 4058 return fib4_entry; 4059 4060 err_nexthop4_group_get: 4061 err_fib4_entry_type_set: 4062 kfree(fib4_entry); 4063 return ERR_PTR(err); 4064 } 4065 4066 static void mlxsw_sp_fib4_entry_destroy(struct mlxsw_sp *mlxsw_sp, 4067 struct mlxsw_sp_fib4_entry *fib4_entry) 4068 { 4069 mlxsw_sp_nexthop4_group_put(mlxsw_sp, &fib4_entry->common); 4070 kfree(fib4_entry); 4071 } 4072 4073 static struct mlxsw_sp_fib4_entry * 4074 mlxsw_sp_fib4_entry_lookup(struct mlxsw_sp *mlxsw_sp, 4075 const struct fib_entry_notifier_info *fen_info) 4076 { 4077 struct mlxsw_sp_fib4_entry *fib4_entry; 4078 struct mlxsw_sp_fib_node *fib_node; 4079 struct mlxsw_sp_fib *fib; 4080 struct mlxsw_sp_vr *vr; 4081 4082 vr = mlxsw_sp_vr_find(mlxsw_sp, fen_info->tb_id); 4083 if (!vr) 4084 return NULL; 4085 fib = mlxsw_sp_vr_fib(vr, MLXSW_SP_L3_PROTO_IPV4); 4086 4087 fib_node = mlxsw_sp_fib_node_lookup(fib, &fen_info->dst, 4088 sizeof(fen_info->dst), 4089 fen_info->dst_len); 4090 if (!fib_node) 4091 return NULL; 4092 4093 list_for_each_entry(fib4_entry, &fib_node->entry_list, common.list) { 4094 if (fib4_entry->tb_id == fen_info->tb_id && 4095 fib4_entry->tos == fen_info->tos && 4096 fib4_entry->type == fen_info->type && 4097 mlxsw_sp_nexthop4_group_fi(fib4_entry->common.nh_group) == 4098 fen_info->fi) { 4099 return fib4_entry; 4100 } 4101 } 4102 4103 return NULL; 4104 } 4105 4106 static const struct rhashtable_params mlxsw_sp_fib_ht_params = { 4107 .key_offset = offsetof(struct mlxsw_sp_fib_node, key), 4108 .head_offset = offsetof(struct mlxsw_sp_fib_node, ht_node), 4109 .key_len = sizeof(struct mlxsw_sp_fib_key), 4110 .automatic_shrinking = true, 4111 }; 4112 4113 static int mlxsw_sp_fib_node_insert(struct mlxsw_sp_fib *fib, 4114 struct mlxsw_sp_fib_node *fib_node) 4115 { 4116 return rhashtable_insert_fast(&fib->ht, &fib_node->ht_node, 4117 mlxsw_sp_fib_ht_params); 4118 } 4119 4120 static void mlxsw_sp_fib_node_remove(struct mlxsw_sp_fib *fib, 4121 struct mlxsw_sp_fib_node *fib_node) 4122 { 4123 rhashtable_remove_fast(&fib->ht, &fib_node->ht_node, 4124 mlxsw_sp_fib_ht_params); 4125 } 4126 4127 static struct mlxsw_sp_fib_node * 4128 mlxsw_sp_fib_node_lookup(struct mlxsw_sp_fib *fib, const void *addr, 4129 size_t addr_len, unsigned char prefix_len) 4130 { 4131 struct mlxsw_sp_fib_key key; 4132 4133 memset(&key, 0, sizeof(key)); 4134 memcpy(key.addr, addr, addr_len); 4135 key.prefix_len = prefix_len; 4136 return rhashtable_lookup_fast(&fib->ht, &key, mlxsw_sp_fib_ht_params); 4137 } 4138 4139 static struct mlxsw_sp_fib_node * 4140 mlxsw_sp_fib_node_create(struct mlxsw_sp_fib *fib, const void *addr, 4141 size_t addr_len, unsigned char prefix_len) 4142 { 4143 struct mlxsw_sp_fib_node *fib_node; 4144 4145 fib_node = kzalloc(sizeof(*fib_node), GFP_KERNEL); 4146 if (!fib_node) 4147 return NULL; 4148 4149 INIT_LIST_HEAD(&fib_node->entry_list); 4150 list_add(&fib_node->list, &fib->node_list); 4151 memcpy(fib_node->key.addr, addr, addr_len); 4152 fib_node->key.prefix_len = prefix_len; 4153 4154 return fib_node; 4155 } 4156 4157 static void mlxsw_sp_fib_node_destroy(struct mlxsw_sp_fib_node *fib_node) 4158 { 4159 list_del(&fib_node->list); 4160 WARN_ON(!list_empty(&fib_node->entry_list)); 4161 kfree(fib_node); 4162 } 4163 4164 static bool 4165 mlxsw_sp_fib_node_entry_is_first(const struct mlxsw_sp_fib_node *fib_node, 4166 const struct mlxsw_sp_fib_entry *fib_entry) 4167 { 4168 return list_first_entry(&fib_node->entry_list, 4169 struct mlxsw_sp_fib_entry, list) == fib_entry; 4170 } 4171 4172 static int mlxsw_sp_fib_lpm_tree_link(struct mlxsw_sp *mlxsw_sp, 4173 struct mlxsw_sp_fib *fib, 4174 struct mlxsw_sp_fib_node *fib_node) 4175 { 4176 struct mlxsw_sp_prefix_usage req_prefix_usage = {{ 0 } }; 4177 struct mlxsw_sp_lpm_tree *lpm_tree; 4178 int err; 4179 4180 /* Since the tree is shared between all virtual routers we must 4181 * make sure it contains all the required prefix lengths. This 4182 * can be computed by either adding the new prefix length to the 4183 * existing prefix usage of a bound tree, or by aggregating the 4184 * prefix lengths across all virtual routers and adding the new 4185 * one as well. 4186 */ 4187 if (fib->lpm_tree) 4188 mlxsw_sp_prefix_usage_cpy(&req_prefix_usage, 4189 &fib->lpm_tree->prefix_usage); 4190 else 4191 mlxsw_sp_vrs_prefixes(mlxsw_sp, fib->proto, &req_prefix_usage); 4192 mlxsw_sp_prefix_usage_set(&req_prefix_usage, fib_node->key.prefix_len); 4193 4194 lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage, 4195 fib->proto); 4196 if (IS_ERR(lpm_tree)) 4197 return PTR_ERR(lpm_tree); 4198 4199 if (fib->lpm_tree && fib->lpm_tree->id == lpm_tree->id) 4200 return 0; 4201 4202 err = mlxsw_sp_vrs_lpm_tree_replace(mlxsw_sp, fib, lpm_tree); 4203 if (err) 4204 return err; 4205 4206 return 0; 4207 } 4208 4209 static void mlxsw_sp_fib_lpm_tree_unlink(struct mlxsw_sp *mlxsw_sp, 4210 struct mlxsw_sp_fib *fib) 4211 { 4212 if (!mlxsw_sp_prefix_usage_none(&fib->prefix_usage)) 4213 return; 4214 mlxsw_sp_vr_lpm_tree_unbind(mlxsw_sp, fib); 4215 mlxsw_sp_lpm_tree_put(mlxsw_sp, fib->lpm_tree); 4216 fib->lpm_tree = NULL; 4217 } 4218 4219 static void mlxsw_sp_fib_node_prefix_inc(struct mlxsw_sp_fib_node *fib_node) 4220 { 4221 unsigned char prefix_len = fib_node->key.prefix_len; 4222 struct mlxsw_sp_fib *fib = fib_node->fib; 4223 4224 if (fib->prefix_ref_count[prefix_len]++ == 0) 4225 mlxsw_sp_prefix_usage_set(&fib->prefix_usage, prefix_len); 4226 } 4227 4228 static void mlxsw_sp_fib_node_prefix_dec(struct mlxsw_sp_fib_node *fib_node) 4229 { 4230 unsigned char prefix_len = fib_node->key.prefix_len; 4231 struct mlxsw_sp_fib *fib = fib_node->fib; 4232 4233 if (--fib->prefix_ref_count[prefix_len] == 0) 4234 mlxsw_sp_prefix_usage_clear(&fib->prefix_usage, prefix_len); 4235 } 4236 4237 static int mlxsw_sp_fib_node_init(struct mlxsw_sp *mlxsw_sp, 4238 struct mlxsw_sp_fib_node *fib_node, 4239 struct mlxsw_sp_fib *fib) 4240 { 4241 int err; 4242 4243 err = mlxsw_sp_fib_node_insert(fib, fib_node); 4244 if (err) 4245 return err; 4246 fib_node->fib = fib; 4247 4248 err = mlxsw_sp_fib_lpm_tree_link(mlxsw_sp, fib, fib_node); 4249 if (err) 4250 goto err_fib_lpm_tree_link; 4251 4252 mlxsw_sp_fib_node_prefix_inc(fib_node); 4253 4254 return 0; 4255 4256 err_fib_lpm_tree_link: 4257 fib_node->fib = NULL; 4258 mlxsw_sp_fib_node_remove(fib, fib_node); 4259 return err; 4260 } 4261 4262 static void mlxsw_sp_fib_node_fini(struct mlxsw_sp *mlxsw_sp, 4263 struct mlxsw_sp_fib_node *fib_node) 4264 { 4265 struct mlxsw_sp_fib *fib = fib_node->fib; 4266 4267 mlxsw_sp_fib_node_prefix_dec(fib_node); 4268 mlxsw_sp_fib_lpm_tree_unlink(mlxsw_sp, fib); 4269 fib_node->fib = NULL; 4270 mlxsw_sp_fib_node_remove(fib, fib_node); 4271 } 4272 4273 static struct mlxsw_sp_fib_node * 4274 mlxsw_sp_fib_node_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id, const void *addr, 4275 size_t addr_len, unsigned char prefix_len, 4276 enum mlxsw_sp_l3proto proto) 4277 { 4278 struct mlxsw_sp_fib_node *fib_node; 4279 struct mlxsw_sp_fib *fib; 4280 struct mlxsw_sp_vr *vr; 4281 int err; 4282 4283 vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id, NULL); 4284 if (IS_ERR(vr)) 4285 return ERR_CAST(vr); 4286 fib = mlxsw_sp_vr_fib(vr, proto); 4287 4288 fib_node = mlxsw_sp_fib_node_lookup(fib, addr, addr_len, prefix_len); 4289 if (fib_node) 4290 return fib_node; 4291 4292 fib_node = mlxsw_sp_fib_node_create(fib, addr, addr_len, prefix_len); 4293 if (!fib_node) { 4294 err = -ENOMEM; 4295 goto err_fib_node_create; 4296 } 4297 4298 err = mlxsw_sp_fib_node_init(mlxsw_sp, fib_node, fib); 4299 if (err) 4300 goto err_fib_node_init; 4301 4302 return fib_node; 4303 4304 err_fib_node_init: 4305 mlxsw_sp_fib_node_destroy(fib_node); 4306 err_fib_node_create: 4307 mlxsw_sp_vr_put(vr); 4308 return ERR_PTR(err); 4309 } 4310 4311 static void mlxsw_sp_fib_node_put(struct mlxsw_sp *mlxsw_sp, 4312 struct mlxsw_sp_fib_node *fib_node) 4313 { 4314 struct mlxsw_sp_vr *vr = fib_node->fib->vr; 4315 4316 if (!list_empty(&fib_node->entry_list)) 4317 return; 4318 mlxsw_sp_fib_node_fini(mlxsw_sp, fib_node); 4319 mlxsw_sp_fib_node_destroy(fib_node); 4320 mlxsw_sp_vr_put(vr); 4321 } 4322 4323 static struct mlxsw_sp_fib4_entry * 4324 mlxsw_sp_fib4_node_entry_find(const struct mlxsw_sp_fib_node *fib_node, 4325 const struct mlxsw_sp_fib4_entry *new4_entry) 4326 { 4327 struct mlxsw_sp_fib4_entry *fib4_entry; 4328 4329 list_for_each_entry(fib4_entry, &fib_node->entry_list, common.list) { 4330 if (fib4_entry->tb_id > new4_entry->tb_id) 4331 continue; 4332 if (fib4_entry->tb_id != new4_entry->tb_id) 4333 break; 4334 if (fib4_entry->tos > new4_entry->tos) 4335 continue; 4336 if (fib4_entry->prio >= new4_entry->prio || 4337 fib4_entry->tos < new4_entry->tos) 4338 return fib4_entry; 4339 } 4340 4341 return NULL; 4342 } 4343 4344 static int 4345 mlxsw_sp_fib4_node_list_append(struct mlxsw_sp_fib4_entry *fib4_entry, 4346 struct mlxsw_sp_fib4_entry *new4_entry) 4347 { 4348 struct mlxsw_sp_fib_node *fib_node; 4349 4350 if (WARN_ON(!fib4_entry)) 4351 return -EINVAL; 4352 4353 fib_node = fib4_entry->common.fib_node; 4354 list_for_each_entry_from(fib4_entry, &fib_node->entry_list, 4355 common.list) { 4356 if (fib4_entry->tb_id != new4_entry->tb_id || 4357 fib4_entry->tos != new4_entry->tos || 4358 fib4_entry->prio != new4_entry->prio) 4359 break; 4360 } 4361 4362 list_add_tail(&new4_entry->common.list, &fib4_entry->common.list); 4363 return 0; 4364 } 4365 4366 static int 4367 mlxsw_sp_fib4_node_list_insert(struct mlxsw_sp_fib4_entry *new4_entry, 4368 bool replace, bool append) 4369 { 4370 struct mlxsw_sp_fib_node *fib_node = new4_entry->common.fib_node; 4371 struct mlxsw_sp_fib4_entry *fib4_entry; 4372 4373 fib4_entry = mlxsw_sp_fib4_node_entry_find(fib_node, new4_entry); 4374 4375 if (append) 4376 return mlxsw_sp_fib4_node_list_append(fib4_entry, new4_entry); 4377 if (replace && WARN_ON(!fib4_entry)) 4378 return -EINVAL; 4379 4380 /* Insert new entry before replaced one, so that we can later 4381 * remove the second. 4382 */ 4383 if (fib4_entry) { 4384 list_add_tail(&new4_entry->common.list, 4385 &fib4_entry->common.list); 4386 } else { 4387 struct mlxsw_sp_fib4_entry *last; 4388 4389 list_for_each_entry(last, &fib_node->entry_list, common.list) { 4390 if (new4_entry->tb_id > last->tb_id) 4391 break; 4392 fib4_entry = last; 4393 } 4394 4395 if (fib4_entry) 4396 list_add(&new4_entry->common.list, 4397 &fib4_entry->common.list); 4398 else 4399 list_add(&new4_entry->common.list, 4400 &fib_node->entry_list); 4401 } 4402 4403 return 0; 4404 } 4405 4406 static void 4407 mlxsw_sp_fib4_node_list_remove(struct mlxsw_sp_fib4_entry *fib4_entry) 4408 { 4409 list_del(&fib4_entry->common.list); 4410 } 4411 4412 static int mlxsw_sp_fib_node_entry_add(struct mlxsw_sp *mlxsw_sp, 4413 struct mlxsw_sp_fib_entry *fib_entry) 4414 { 4415 struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node; 4416 4417 if (!mlxsw_sp_fib_node_entry_is_first(fib_node, fib_entry)) 4418 return 0; 4419 4420 /* To prevent packet loss, overwrite the previously offloaded 4421 * entry. 4422 */ 4423 if (!list_is_singular(&fib_node->entry_list)) { 4424 enum mlxsw_reg_ralue_op op = MLXSW_REG_RALUE_OP_WRITE_DELETE; 4425 struct mlxsw_sp_fib_entry *n = list_next_entry(fib_entry, list); 4426 4427 mlxsw_sp_fib_entry_offload_refresh(n, op, 0); 4428 } 4429 4430 return mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry); 4431 } 4432 4433 static void mlxsw_sp_fib_node_entry_del(struct mlxsw_sp *mlxsw_sp, 4434 struct mlxsw_sp_fib_entry *fib_entry) 4435 { 4436 struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node; 4437 4438 if (!mlxsw_sp_fib_node_entry_is_first(fib_node, fib_entry)) 4439 return; 4440 4441 /* Promote the next entry by overwriting the deleted entry */ 4442 if (!list_is_singular(&fib_node->entry_list)) { 4443 struct mlxsw_sp_fib_entry *n = list_next_entry(fib_entry, list); 4444 enum mlxsw_reg_ralue_op op = MLXSW_REG_RALUE_OP_WRITE_DELETE; 4445 4446 mlxsw_sp_fib_entry_update(mlxsw_sp, n); 4447 mlxsw_sp_fib_entry_offload_refresh(fib_entry, op, 0); 4448 return; 4449 } 4450 4451 mlxsw_sp_fib_entry_del(mlxsw_sp, fib_entry); 4452 } 4453 4454 static int mlxsw_sp_fib4_node_entry_link(struct mlxsw_sp *mlxsw_sp, 4455 struct mlxsw_sp_fib4_entry *fib4_entry, 4456 bool replace, bool append) 4457 { 4458 int err; 4459 4460 err = mlxsw_sp_fib4_node_list_insert(fib4_entry, replace, append); 4461 if (err) 4462 return err; 4463 4464 err = mlxsw_sp_fib_node_entry_add(mlxsw_sp, &fib4_entry->common); 4465 if (err) 4466 goto err_fib_node_entry_add; 4467 4468 return 0; 4469 4470 err_fib_node_entry_add: 4471 mlxsw_sp_fib4_node_list_remove(fib4_entry); 4472 return err; 4473 } 4474 4475 static void 4476 mlxsw_sp_fib4_node_entry_unlink(struct mlxsw_sp *mlxsw_sp, 4477 struct mlxsw_sp_fib4_entry *fib4_entry) 4478 { 4479 mlxsw_sp_fib_node_entry_del(mlxsw_sp, &fib4_entry->common); 4480 mlxsw_sp_fib4_node_list_remove(fib4_entry); 4481 4482 if (fib4_entry->common.type == MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP) 4483 mlxsw_sp_fib_entry_decap_fini(mlxsw_sp, &fib4_entry->common); 4484 } 4485 4486 static void mlxsw_sp_fib4_entry_replace(struct mlxsw_sp *mlxsw_sp, 4487 struct mlxsw_sp_fib4_entry *fib4_entry, 4488 bool replace) 4489 { 4490 struct mlxsw_sp_fib_node *fib_node = fib4_entry->common.fib_node; 4491 struct mlxsw_sp_fib4_entry *replaced; 4492 4493 if (!replace) 4494 return; 4495 4496 /* We inserted the new entry before replaced one */ 4497 replaced = list_next_entry(fib4_entry, common.list); 4498 4499 mlxsw_sp_fib4_node_entry_unlink(mlxsw_sp, replaced); 4500 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, replaced); 4501 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 4502 } 4503 4504 static int 4505 mlxsw_sp_router_fib4_add(struct mlxsw_sp *mlxsw_sp, 4506 const struct fib_entry_notifier_info *fen_info, 4507 bool replace, bool append) 4508 { 4509 struct mlxsw_sp_fib4_entry *fib4_entry; 4510 struct mlxsw_sp_fib_node *fib_node; 4511 int err; 4512 4513 if (mlxsw_sp->router->aborted) 4514 return 0; 4515 4516 fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, fen_info->tb_id, 4517 &fen_info->dst, sizeof(fen_info->dst), 4518 fen_info->dst_len, 4519 MLXSW_SP_L3_PROTO_IPV4); 4520 if (IS_ERR(fib_node)) { 4521 dev_warn(mlxsw_sp->bus_info->dev, "Failed to get FIB node\n"); 4522 return PTR_ERR(fib_node); 4523 } 4524 4525 fib4_entry = mlxsw_sp_fib4_entry_create(mlxsw_sp, fib_node, fen_info); 4526 if (IS_ERR(fib4_entry)) { 4527 dev_warn(mlxsw_sp->bus_info->dev, "Failed to create FIB entry\n"); 4528 err = PTR_ERR(fib4_entry); 4529 goto err_fib4_entry_create; 4530 } 4531 4532 err = mlxsw_sp_fib4_node_entry_link(mlxsw_sp, fib4_entry, replace, 4533 append); 4534 if (err) { 4535 dev_warn(mlxsw_sp->bus_info->dev, "Failed to link FIB entry to node\n"); 4536 goto err_fib4_node_entry_link; 4537 } 4538 4539 mlxsw_sp_fib4_entry_replace(mlxsw_sp, fib4_entry, replace); 4540 4541 return 0; 4542 4543 err_fib4_node_entry_link: 4544 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry); 4545 err_fib4_entry_create: 4546 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 4547 return err; 4548 } 4549 4550 static void mlxsw_sp_router_fib4_del(struct mlxsw_sp *mlxsw_sp, 4551 struct fib_entry_notifier_info *fen_info) 4552 { 4553 struct mlxsw_sp_fib4_entry *fib4_entry; 4554 struct mlxsw_sp_fib_node *fib_node; 4555 4556 if (mlxsw_sp->router->aborted) 4557 return; 4558 4559 fib4_entry = mlxsw_sp_fib4_entry_lookup(mlxsw_sp, fen_info); 4560 if (WARN_ON(!fib4_entry)) 4561 return; 4562 fib_node = fib4_entry->common.fib_node; 4563 4564 mlxsw_sp_fib4_node_entry_unlink(mlxsw_sp, fib4_entry); 4565 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry); 4566 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 4567 } 4568 4569 static bool mlxsw_sp_fib6_rt_should_ignore(const struct rt6_info *rt) 4570 { 4571 /* Packets with link-local destination IP arriving to the router 4572 * are trapped to the CPU, so no need to program specific routes 4573 * for them. 4574 */ 4575 if (ipv6_addr_type(&rt->rt6i_dst.addr) & IPV6_ADDR_LINKLOCAL) 4576 return true; 4577 4578 /* Multicast routes aren't supported, so ignore them. Neighbour 4579 * Discovery packets are specifically trapped. 4580 */ 4581 if (ipv6_addr_type(&rt->rt6i_dst.addr) & IPV6_ADDR_MULTICAST) 4582 return true; 4583 4584 /* Cloned routes are irrelevant in the forwarding path. */ 4585 if (rt->rt6i_flags & RTF_CACHE) 4586 return true; 4587 4588 return false; 4589 } 4590 4591 static struct mlxsw_sp_rt6 *mlxsw_sp_rt6_create(struct rt6_info *rt) 4592 { 4593 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 4594 4595 mlxsw_sp_rt6 = kzalloc(sizeof(*mlxsw_sp_rt6), GFP_KERNEL); 4596 if (!mlxsw_sp_rt6) 4597 return ERR_PTR(-ENOMEM); 4598 4599 /* In case of route replace, replaced route is deleted with 4600 * no notification. Take reference to prevent accessing freed 4601 * memory. 4602 */ 4603 mlxsw_sp_rt6->rt = rt; 4604 rt6_hold(rt); 4605 4606 return mlxsw_sp_rt6; 4607 } 4608 4609 #if IS_ENABLED(CONFIG_IPV6) 4610 static void mlxsw_sp_rt6_release(struct rt6_info *rt) 4611 { 4612 rt6_release(rt); 4613 } 4614 #else 4615 static void mlxsw_sp_rt6_release(struct rt6_info *rt) 4616 { 4617 } 4618 #endif 4619 4620 static void mlxsw_sp_rt6_destroy(struct mlxsw_sp_rt6 *mlxsw_sp_rt6) 4621 { 4622 mlxsw_sp_rt6_release(mlxsw_sp_rt6->rt); 4623 kfree(mlxsw_sp_rt6); 4624 } 4625 4626 static bool mlxsw_sp_fib6_rt_can_mp(const struct rt6_info *rt) 4627 { 4628 /* RTF_CACHE routes are ignored */ 4629 return (rt->rt6i_flags & (RTF_GATEWAY | RTF_ADDRCONF)) == RTF_GATEWAY; 4630 } 4631 4632 static struct rt6_info * 4633 mlxsw_sp_fib6_entry_rt(const struct mlxsw_sp_fib6_entry *fib6_entry) 4634 { 4635 return list_first_entry(&fib6_entry->rt6_list, struct mlxsw_sp_rt6, 4636 list)->rt; 4637 } 4638 4639 static struct mlxsw_sp_fib6_entry * 4640 mlxsw_sp_fib6_node_mp_entry_find(const struct mlxsw_sp_fib_node *fib_node, 4641 const struct rt6_info *nrt, bool replace) 4642 { 4643 struct mlxsw_sp_fib6_entry *fib6_entry; 4644 4645 if (!mlxsw_sp_fib6_rt_can_mp(nrt) || replace) 4646 return NULL; 4647 4648 list_for_each_entry(fib6_entry, &fib_node->entry_list, common.list) { 4649 struct rt6_info *rt = mlxsw_sp_fib6_entry_rt(fib6_entry); 4650 4651 /* RT6_TABLE_LOCAL and RT6_TABLE_MAIN share the same 4652 * virtual router. 4653 */ 4654 if (rt->rt6i_table->tb6_id > nrt->rt6i_table->tb6_id) 4655 continue; 4656 if (rt->rt6i_table->tb6_id != nrt->rt6i_table->tb6_id) 4657 break; 4658 if (rt->rt6i_metric < nrt->rt6i_metric) 4659 continue; 4660 if (rt->rt6i_metric == nrt->rt6i_metric && 4661 mlxsw_sp_fib6_rt_can_mp(rt)) 4662 return fib6_entry; 4663 if (rt->rt6i_metric > nrt->rt6i_metric) 4664 break; 4665 } 4666 4667 return NULL; 4668 } 4669 4670 static struct mlxsw_sp_rt6 * 4671 mlxsw_sp_fib6_entry_rt_find(const struct mlxsw_sp_fib6_entry *fib6_entry, 4672 const struct rt6_info *rt) 4673 { 4674 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 4675 4676 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) { 4677 if (mlxsw_sp_rt6->rt == rt) 4678 return mlxsw_sp_rt6; 4679 } 4680 4681 return NULL; 4682 } 4683 4684 static bool mlxsw_sp_nexthop6_ipip_type(const struct mlxsw_sp *mlxsw_sp, 4685 const struct rt6_info *rt, 4686 enum mlxsw_sp_ipip_type *ret) 4687 { 4688 return rt->dst.dev && 4689 mlxsw_sp_netdev_ipip_type(mlxsw_sp, rt->dst.dev, ret); 4690 } 4691 4692 static int mlxsw_sp_nexthop6_type_init(struct mlxsw_sp *mlxsw_sp, 4693 struct mlxsw_sp_nexthop_group *nh_grp, 4694 struct mlxsw_sp_nexthop *nh, 4695 const struct rt6_info *rt) 4696 { 4697 struct mlxsw_sp_router *router = mlxsw_sp->router; 4698 struct net_device *dev = rt->dst.dev; 4699 enum mlxsw_sp_ipip_type ipipt; 4700 struct mlxsw_sp_rif *rif; 4701 int err; 4702 4703 if (mlxsw_sp_nexthop6_ipip_type(mlxsw_sp, rt, &ipipt) && 4704 router->ipip_ops_arr[ipipt]->can_offload(mlxsw_sp, dev, 4705 MLXSW_SP_L3_PROTO_IPV6)) { 4706 nh->type = MLXSW_SP_NEXTHOP_TYPE_IPIP; 4707 err = mlxsw_sp_nexthop_ipip_init(mlxsw_sp, nh, dev); 4708 if (err) 4709 return err; 4710 mlxsw_sp_nexthop_rif_init(nh, &nh->ipip_entry->ol_lb->common); 4711 return 0; 4712 } 4713 4714 nh->type = MLXSW_SP_NEXTHOP_TYPE_ETH; 4715 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 4716 if (!rif) 4717 return 0; 4718 mlxsw_sp_nexthop_rif_init(nh, rif); 4719 4720 err = mlxsw_sp_nexthop_neigh_init(mlxsw_sp, nh); 4721 if (err) 4722 goto err_nexthop_neigh_init; 4723 4724 return 0; 4725 4726 err_nexthop_neigh_init: 4727 mlxsw_sp_nexthop_rif_fini(nh); 4728 return err; 4729 } 4730 4731 static void mlxsw_sp_nexthop6_type_fini(struct mlxsw_sp *mlxsw_sp, 4732 struct mlxsw_sp_nexthop *nh) 4733 { 4734 mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh); 4735 } 4736 4737 static int mlxsw_sp_nexthop6_init(struct mlxsw_sp *mlxsw_sp, 4738 struct mlxsw_sp_nexthop_group *nh_grp, 4739 struct mlxsw_sp_nexthop *nh, 4740 const struct rt6_info *rt) 4741 { 4742 struct net_device *dev = rt->dst.dev; 4743 4744 nh->nh_grp = nh_grp; 4745 nh->nh_weight = 1; 4746 memcpy(&nh->gw_addr, &rt->rt6i_gateway, sizeof(nh->gw_addr)); 4747 mlxsw_sp_nexthop_counter_alloc(mlxsw_sp, nh); 4748 4749 list_add_tail(&nh->router_list_node, &mlxsw_sp->router->nexthop_list); 4750 4751 if (!dev) 4752 return 0; 4753 nh->ifindex = dev->ifindex; 4754 4755 return mlxsw_sp_nexthop6_type_init(mlxsw_sp, nh_grp, nh, rt); 4756 } 4757 4758 static void mlxsw_sp_nexthop6_fini(struct mlxsw_sp *mlxsw_sp, 4759 struct mlxsw_sp_nexthop *nh) 4760 { 4761 mlxsw_sp_nexthop6_type_fini(mlxsw_sp, nh); 4762 list_del(&nh->router_list_node); 4763 mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh); 4764 } 4765 4766 static bool mlxsw_sp_rt6_is_gateway(const struct mlxsw_sp *mlxsw_sp, 4767 const struct rt6_info *rt) 4768 { 4769 return rt->rt6i_flags & RTF_GATEWAY || 4770 mlxsw_sp_nexthop6_ipip_type(mlxsw_sp, rt, NULL); 4771 } 4772 4773 static struct mlxsw_sp_nexthop_group * 4774 mlxsw_sp_nexthop6_group_create(struct mlxsw_sp *mlxsw_sp, 4775 struct mlxsw_sp_fib6_entry *fib6_entry) 4776 { 4777 struct mlxsw_sp_nexthop_group *nh_grp; 4778 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 4779 struct mlxsw_sp_nexthop *nh; 4780 size_t alloc_size; 4781 int i = 0; 4782 int err; 4783 4784 alloc_size = sizeof(*nh_grp) + 4785 fib6_entry->nrt6 * sizeof(struct mlxsw_sp_nexthop); 4786 nh_grp = kzalloc(alloc_size, GFP_KERNEL); 4787 if (!nh_grp) 4788 return ERR_PTR(-ENOMEM); 4789 INIT_LIST_HEAD(&nh_grp->fib_list); 4790 #if IS_ENABLED(CONFIG_IPV6) 4791 nh_grp->neigh_tbl = &nd_tbl; 4792 #endif 4793 mlxsw_sp_rt6 = list_first_entry(&fib6_entry->rt6_list, 4794 struct mlxsw_sp_rt6, list); 4795 nh_grp->gateway = mlxsw_sp_rt6_is_gateway(mlxsw_sp, mlxsw_sp_rt6->rt); 4796 nh_grp->count = fib6_entry->nrt6; 4797 for (i = 0; i < nh_grp->count; i++) { 4798 struct rt6_info *rt = mlxsw_sp_rt6->rt; 4799 4800 nh = &nh_grp->nexthops[i]; 4801 err = mlxsw_sp_nexthop6_init(mlxsw_sp, nh_grp, nh, rt); 4802 if (err) 4803 goto err_nexthop6_init; 4804 mlxsw_sp_rt6 = list_next_entry(mlxsw_sp_rt6, list); 4805 } 4806 4807 err = mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp); 4808 if (err) 4809 goto err_nexthop_group_insert; 4810 4811 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp); 4812 return nh_grp; 4813 4814 err_nexthop_group_insert: 4815 err_nexthop6_init: 4816 for (i--; i >= 0; i--) { 4817 nh = &nh_grp->nexthops[i]; 4818 mlxsw_sp_nexthop6_fini(mlxsw_sp, nh); 4819 } 4820 kfree(nh_grp); 4821 return ERR_PTR(err); 4822 } 4823 4824 static void 4825 mlxsw_sp_nexthop6_group_destroy(struct mlxsw_sp *mlxsw_sp, 4826 struct mlxsw_sp_nexthop_group *nh_grp) 4827 { 4828 struct mlxsw_sp_nexthop *nh; 4829 int i = nh_grp->count; 4830 4831 mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp); 4832 for (i--; i >= 0; i--) { 4833 nh = &nh_grp->nexthops[i]; 4834 mlxsw_sp_nexthop6_fini(mlxsw_sp, nh); 4835 } 4836 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp); 4837 WARN_ON(nh_grp->adj_index_valid); 4838 kfree(nh_grp); 4839 } 4840 4841 static int mlxsw_sp_nexthop6_group_get(struct mlxsw_sp *mlxsw_sp, 4842 struct mlxsw_sp_fib6_entry *fib6_entry) 4843 { 4844 struct mlxsw_sp_nexthop_group *nh_grp; 4845 4846 nh_grp = mlxsw_sp_nexthop6_group_lookup(mlxsw_sp, fib6_entry); 4847 if (!nh_grp) { 4848 nh_grp = mlxsw_sp_nexthop6_group_create(mlxsw_sp, fib6_entry); 4849 if (IS_ERR(nh_grp)) 4850 return PTR_ERR(nh_grp); 4851 } 4852 4853 list_add_tail(&fib6_entry->common.nexthop_group_node, 4854 &nh_grp->fib_list); 4855 fib6_entry->common.nh_group = nh_grp; 4856 4857 return 0; 4858 } 4859 4860 static void mlxsw_sp_nexthop6_group_put(struct mlxsw_sp *mlxsw_sp, 4861 struct mlxsw_sp_fib_entry *fib_entry) 4862 { 4863 struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group; 4864 4865 list_del(&fib_entry->nexthop_group_node); 4866 if (!list_empty(&nh_grp->fib_list)) 4867 return; 4868 mlxsw_sp_nexthop6_group_destroy(mlxsw_sp, nh_grp); 4869 } 4870 4871 static int 4872 mlxsw_sp_nexthop6_group_update(struct mlxsw_sp *mlxsw_sp, 4873 struct mlxsw_sp_fib6_entry *fib6_entry) 4874 { 4875 struct mlxsw_sp_nexthop_group *old_nh_grp = fib6_entry->common.nh_group; 4876 int err; 4877 4878 fib6_entry->common.nh_group = NULL; 4879 list_del(&fib6_entry->common.nexthop_group_node); 4880 4881 err = mlxsw_sp_nexthop6_group_get(mlxsw_sp, fib6_entry); 4882 if (err) 4883 goto err_nexthop6_group_get; 4884 4885 /* In case this entry is offloaded, then the adjacency index 4886 * currently associated with it in the device's table is that 4887 * of the old group. Start using the new one instead. 4888 */ 4889 err = mlxsw_sp_fib_node_entry_add(mlxsw_sp, &fib6_entry->common); 4890 if (err) 4891 goto err_fib_node_entry_add; 4892 4893 if (list_empty(&old_nh_grp->fib_list)) 4894 mlxsw_sp_nexthop6_group_destroy(mlxsw_sp, old_nh_grp); 4895 4896 return 0; 4897 4898 err_fib_node_entry_add: 4899 mlxsw_sp_nexthop6_group_put(mlxsw_sp, &fib6_entry->common); 4900 err_nexthop6_group_get: 4901 list_add_tail(&fib6_entry->common.nexthop_group_node, 4902 &old_nh_grp->fib_list); 4903 fib6_entry->common.nh_group = old_nh_grp; 4904 return err; 4905 } 4906 4907 static int 4908 mlxsw_sp_fib6_entry_nexthop_add(struct mlxsw_sp *mlxsw_sp, 4909 struct mlxsw_sp_fib6_entry *fib6_entry, 4910 struct rt6_info *rt) 4911 { 4912 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 4913 int err; 4914 4915 mlxsw_sp_rt6 = mlxsw_sp_rt6_create(rt); 4916 if (IS_ERR(mlxsw_sp_rt6)) 4917 return PTR_ERR(mlxsw_sp_rt6); 4918 4919 list_add_tail(&mlxsw_sp_rt6->list, &fib6_entry->rt6_list); 4920 fib6_entry->nrt6++; 4921 4922 err = mlxsw_sp_nexthop6_group_update(mlxsw_sp, fib6_entry); 4923 if (err) 4924 goto err_nexthop6_group_update; 4925 4926 return 0; 4927 4928 err_nexthop6_group_update: 4929 fib6_entry->nrt6--; 4930 list_del(&mlxsw_sp_rt6->list); 4931 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6); 4932 return err; 4933 } 4934 4935 static void 4936 mlxsw_sp_fib6_entry_nexthop_del(struct mlxsw_sp *mlxsw_sp, 4937 struct mlxsw_sp_fib6_entry *fib6_entry, 4938 struct rt6_info *rt) 4939 { 4940 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 4941 4942 mlxsw_sp_rt6 = mlxsw_sp_fib6_entry_rt_find(fib6_entry, rt); 4943 if (WARN_ON(!mlxsw_sp_rt6)) 4944 return; 4945 4946 fib6_entry->nrt6--; 4947 list_del(&mlxsw_sp_rt6->list); 4948 mlxsw_sp_nexthop6_group_update(mlxsw_sp, fib6_entry); 4949 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6); 4950 } 4951 4952 static void mlxsw_sp_fib6_entry_type_set(struct mlxsw_sp *mlxsw_sp, 4953 struct mlxsw_sp_fib_entry *fib_entry, 4954 const struct rt6_info *rt) 4955 { 4956 /* Packets hitting RTF_REJECT routes need to be discarded by the 4957 * stack. We can rely on their destination device not having a 4958 * RIF (it's the loopback device) and can thus use action type 4959 * local, which will cause them to be trapped with a lower 4960 * priority than packets that need to be locally received. 4961 */ 4962 if (rt->rt6i_flags & (RTF_LOCAL | RTF_ANYCAST)) 4963 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP; 4964 else if (rt->rt6i_flags & RTF_REJECT) 4965 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL; 4966 else if (mlxsw_sp_rt6_is_gateway(mlxsw_sp, rt)) 4967 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_REMOTE; 4968 else 4969 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL; 4970 } 4971 4972 static void 4973 mlxsw_sp_fib6_entry_rt_destroy_all(struct mlxsw_sp_fib6_entry *fib6_entry) 4974 { 4975 struct mlxsw_sp_rt6 *mlxsw_sp_rt6, *tmp; 4976 4977 list_for_each_entry_safe(mlxsw_sp_rt6, tmp, &fib6_entry->rt6_list, 4978 list) { 4979 fib6_entry->nrt6--; 4980 list_del(&mlxsw_sp_rt6->list); 4981 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6); 4982 } 4983 } 4984 4985 static struct mlxsw_sp_fib6_entry * 4986 mlxsw_sp_fib6_entry_create(struct mlxsw_sp *mlxsw_sp, 4987 struct mlxsw_sp_fib_node *fib_node, 4988 struct rt6_info *rt) 4989 { 4990 struct mlxsw_sp_fib6_entry *fib6_entry; 4991 struct mlxsw_sp_fib_entry *fib_entry; 4992 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 4993 int err; 4994 4995 fib6_entry = kzalloc(sizeof(*fib6_entry), GFP_KERNEL); 4996 if (!fib6_entry) 4997 return ERR_PTR(-ENOMEM); 4998 fib_entry = &fib6_entry->common; 4999 5000 mlxsw_sp_rt6 = mlxsw_sp_rt6_create(rt); 5001 if (IS_ERR(mlxsw_sp_rt6)) { 5002 err = PTR_ERR(mlxsw_sp_rt6); 5003 goto err_rt6_create; 5004 } 5005 5006 mlxsw_sp_fib6_entry_type_set(mlxsw_sp, fib_entry, mlxsw_sp_rt6->rt); 5007 5008 INIT_LIST_HEAD(&fib6_entry->rt6_list); 5009 list_add_tail(&mlxsw_sp_rt6->list, &fib6_entry->rt6_list); 5010 fib6_entry->nrt6 = 1; 5011 err = mlxsw_sp_nexthop6_group_get(mlxsw_sp, fib6_entry); 5012 if (err) 5013 goto err_nexthop6_group_get; 5014 5015 fib_entry->fib_node = fib_node; 5016 5017 return fib6_entry; 5018 5019 err_nexthop6_group_get: 5020 list_del(&mlxsw_sp_rt6->list); 5021 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6); 5022 err_rt6_create: 5023 kfree(fib6_entry); 5024 return ERR_PTR(err); 5025 } 5026 5027 static void mlxsw_sp_fib6_entry_destroy(struct mlxsw_sp *mlxsw_sp, 5028 struct mlxsw_sp_fib6_entry *fib6_entry) 5029 { 5030 mlxsw_sp_nexthop6_group_put(mlxsw_sp, &fib6_entry->common); 5031 mlxsw_sp_fib6_entry_rt_destroy_all(fib6_entry); 5032 WARN_ON(fib6_entry->nrt6); 5033 kfree(fib6_entry); 5034 } 5035 5036 static struct mlxsw_sp_fib6_entry * 5037 mlxsw_sp_fib6_node_entry_find(const struct mlxsw_sp_fib_node *fib_node, 5038 const struct rt6_info *nrt, bool replace) 5039 { 5040 struct mlxsw_sp_fib6_entry *fib6_entry, *fallback = NULL; 5041 5042 list_for_each_entry(fib6_entry, &fib_node->entry_list, common.list) { 5043 struct rt6_info *rt = mlxsw_sp_fib6_entry_rt(fib6_entry); 5044 5045 if (rt->rt6i_table->tb6_id > nrt->rt6i_table->tb6_id) 5046 continue; 5047 if (rt->rt6i_table->tb6_id != nrt->rt6i_table->tb6_id) 5048 break; 5049 if (replace && rt->rt6i_metric == nrt->rt6i_metric) { 5050 if (mlxsw_sp_fib6_rt_can_mp(rt) == 5051 mlxsw_sp_fib6_rt_can_mp(nrt)) 5052 return fib6_entry; 5053 if (mlxsw_sp_fib6_rt_can_mp(nrt)) 5054 fallback = fallback ?: fib6_entry; 5055 } 5056 if (rt->rt6i_metric > nrt->rt6i_metric) 5057 return fallback ?: fib6_entry; 5058 } 5059 5060 return fallback; 5061 } 5062 5063 static int 5064 mlxsw_sp_fib6_node_list_insert(struct mlxsw_sp_fib6_entry *new6_entry, 5065 bool replace) 5066 { 5067 struct mlxsw_sp_fib_node *fib_node = new6_entry->common.fib_node; 5068 struct rt6_info *nrt = mlxsw_sp_fib6_entry_rt(new6_entry); 5069 struct mlxsw_sp_fib6_entry *fib6_entry; 5070 5071 fib6_entry = mlxsw_sp_fib6_node_entry_find(fib_node, nrt, replace); 5072 5073 if (replace && WARN_ON(!fib6_entry)) 5074 return -EINVAL; 5075 5076 if (fib6_entry) { 5077 list_add_tail(&new6_entry->common.list, 5078 &fib6_entry->common.list); 5079 } else { 5080 struct mlxsw_sp_fib6_entry *last; 5081 5082 list_for_each_entry(last, &fib_node->entry_list, common.list) { 5083 struct rt6_info *rt = mlxsw_sp_fib6_entry_rt(last); 5084 5085 if (nrt->rt6i_table->tb6_id > rt->rt6i_table->tb6_id) 5086 break; 5087 fib6_entry = last; 5088 } 5089 5090 if (fib6_entry) 5091 list_add(&new6_entry->common.list, 5092 &fib6_entry->common.list); 5093 else 5094 list_add(&new6_entry->common.list, 5095 &fib_node->entry_list); 5096 } 5097 5098 return 0; 5099 } 5100 5101 static void 5102 mlxsw_sp_fib6_node_list_remove(struct mlxsw_sp_fib6_entry *fib6_entry) 5103 { 5104 list_del(&fib6_entry->common.list); 5105 } 5106 5107 static int mlxsw_sp_fib6_node_entry_link(struct mlxsw_sp *mlxsw_sp, 5108 struct mlxsw_sp_fib6_entry *fib6_entry, 5109 bool replace) 5110 { 5111 int err; 5112 5113 err = mlxsw_sp_fib6_node_list_insert(fib6_entry, replace); 5114 if (err) 5115 return err; 5116 5117 err = mlxsw_sp_fib_node_entry_add(mlxsw_sp, &fib6_entry->common); 5118 if (err) 5119 goto err_fib_node_entry_add; 5120 5121 return 0; 5122 5123 err_fib_node_entry_add: 5124 mlxsw_sp_fib6_node_list_remove(fib6_entry); 5125 return err; 5126 } 5127 5128 static void 5129 mlxsw_sp_fib6_node_entry_unlink(struct mlxsw_sp *mlxsw_sp, 5130 struct mlxsw_sp_fib6_entry *fib6_entry) 5131 { 5132 mlxsw_sp_fib_node_entry_del(mlxsw_sp, &fib6_entry->common); 5133 mlxsw_sp_fib6_node_list_remove(fib6_entry); 5134 } 5135 5136 static struct mlxsw_sp_fib6_entry * 5137 mlxsw_sp_fib6_entry_lookup(struct mlxsw_sp *mlxsw_sp, 5138 const struct rt6_info *rt) 5139 { 5140 struct mlxsw_sp_fib6_entry *fib6_entry; 5141 struct mlxsw_sp_fib_node *fib_node; 5142 struct mlxsw_sp_fib *fib; 5143 struct mlxsw_sp_vr *vr; 5144 5145 vr = mlxsw_sp_vr_find(mlxsw_sp, rt->rt6i_table->tb6_id); 5146 if (!vr) 5147 return NULL; 5148 fib = mlxsw_sp_vr_fib(vr, MLXSW_SP_L3_PROTO_IPV6); 5149 5150 fib_node = mlxsw_sp_fib_node_lookup(fib, &rt->rt6i_dst.addr, 5151 sizeof(rt->rt6i_dst.addr), 5152 rt->rt6i_dst.plen); 5153 if (!fib_node) 5154 return NULL; 5155 5156 list_for_each_entry(fib6_entry, &fib_node->entry_list, common.list) { 5157 struct rt6_info *iter_rt = mlxsw_sp_fib6_entry_rt(fib6_entry); 5158 5159 if (rt->rt6i_table->tb6_id == iter_rt->rt6i_table->tb6_id && 5160 rt->rt6i_metric == iter_rt->rt6i_metric && 5161 mlxsw_sp_fib6_entry_rt_find(fib6_entry, rt)) 5162 return fib6_entry; 5163 } 5164 5165 return NULL; 5166 } 5167 5168 static void mlxsw_sp_fib6_entry_replace(struct mlxsw_sp *mlxsw_sp, 5169 struct mlxsw_sp_fib6_entry *fib6_entry, 5170 bool replace) 5171 { 5172 struct mlxsw_sp_fib_node *fib_node = fib6_entry->common.fib_node; 5173 struct mlxsw_sp_fib6_entry *replaced; 5174 5175 if (!replace) 5176 return; 5177 5178 replaced = list_next_entry(fib6_entry, common.list); 5179 5180 mlxsw_sp_fib6_node_entry_unlink(mlxsw_sp, replaced); 5181 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, replaced); 5182 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 5183 } 5184 5185 static int mlxsw_sp_router_fib6_add(struct mlxsw_sp *mlxsw_sp, 5186 struct rt6_info *rt, bool replace) 5187 { 5188 struct mlxsw_sp_fib6_entry *fib6_entry; 5189 struct mlxsw_sp_fib_node *fib_node; 5190 int err; 5191 5192 if (mlxsw_sp->router->aborted) 5193 return 0; 5194 5195 if (rt->rt6i_src.plen) 5196 return -EINVAL; 5197 5198 if (mlxsw_sp_fib6_rt_should_ignore(rt)) 5199 return 0; 5200 5201 fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, rt->rt6i_table->tb6_id, 5202 &rt->rt6i_dst.addr, 5203 sizeof(rt->rt6i_dst.addr), 5204 rt->rt6i_dst.plen, 5205 MLXSW_SP_L3_PROTO_IPV6); 5206 if (IS_ERR(fib_node)) 5207 return PTR_ERR(fib_node); 5208 5209 /* Before creating a new entry, try to append route to an existing 5210 * multipath entry. 5211 */ 5212 fib6_entry = mlxsw_sp_fib6_node_mp_entry_find(fib_node, rt, replace); 5213 if (fib6_entry) { 5214 err = mlxsw_sp_fib6_entry_nexthop_add(mlxsw_sp, fib6_entry, rt); 5215 if (err) 5216 goto err_fib6_entry_nexthop_add; 5217 return 0; 5218 } 5219 5220 fib6_entry = mlxsw_sp_fib6_entry_create(mlxsw_sp, fib_node, rt); 5221 if (IS_ERR(fib6_entry)) { 5222 err = PTR_ERR(fib6_entry); 5223 goto err_fib6_entry_create; 5224 } 5225 5226 err = mlxsw_sp_fib6_node_entry_link(mlxsw_sp, fib6_entry, replace); 5227 if (err) 5228 goto err_fib6_node_entry_link; 5229 5230 mlxsw_sp_fib6_entry_replace(mlxsw_sp, fib6_entry, replace); 5231 5232 return 0; 5233 5234 err_fib6_node_entry_link: 5235 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry); 5236 err_fib6_entry_create: 5237 err_fib6_entry_nexthop_add: 5238 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 5239 return err; 5240 } 5241 5242 static void mlxsw_sp_router_fib6_del(struct mlxsw_sp *mlxsw_sp, 5243 struct rt6_info *rt) 5244 { 5245 struct mlxsw_sp_fib6_entry *fib6_entry; 5246 struct mlxsw_sp_fib_node *fib_node; 5247 5248 if (mlxsw_sp->router->aborted) 5249 return; 5250 5251 if (mlxsw_sp_fib6_rt_should_ignore(rt)) 5252 return; 5253 5254 fib6_entry = mlxsw_sp_fib6_entry_lookup(mlxsw_sp, rt); 5255 if (WARN_ON(!fib6_entry)) 5256 return; 5257 5258 /* If route is part of a multipath entry, but not the last one 5259 * removed, then only reduce its nexthop group. 5260 */ 5261 if (!list_is_singular(&fib6_entry->rt6_list)) { 5262 mlxsw_sp_fib6_entry_nexthop_del(mlxsw_sp, fib6_entry, rt); 5263 return; 5264 } 5265 5266 fib_node = fib6_entry->common.fib_node; 5267 5268 mlxsw_sp_fib6_node_entry_unlink(mlxsw_sp, fib6_entry); 5269 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry); 5270 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 5271 } 5272 5273 static int __mlxsw_sp_router_set_abort_trap(struct mlxsw_sp *mlxsw_sp, 5274 enum mlxsw_reg_ralxx_protocol proto, 5275 u8 tree_id) 5276 { 5277 char ralta_pl[MLXSW_REG_RALTA_LEN]; 5278 char ralst_pl[MLXSW_REG_RALST_LEN]; 5279 int i, err; 5280 5281 mlxsw_reg_ralta_pack(ralta_pl, true, proto, tree_id); 5282 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl); 5283 if (err) 5284 return err; 5285 5286 mlxsw_reg_ralst_pack(ralst_pl, 0xff, tree_id); 5287 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralst), ralst_pl); 5288 if (err) 5289 return err; 5290 5291 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) { 5292 struct mlxsw_sp_vr *vr = &mlxsw_sp->router->vrs[i]; 5293 char raltb_pl[MLXSW_REG_RALTB_LEN]; 5294 char ralue_pl[MLXSW_REG_RALUE_LEN]; 5295 5296 mlxsw_reg_raltb_pack(raltb_pl, vr->id, proto, tree_id); 5297 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), 5298 raltb_pl); 5299 if (err) 5300 return err; 5301 5302 mlxsw_reg_ralue_pack(ralue_pl, proto, 5303 MLXSW_REG_RALUE_OP_WRITE_WRITE, vr->id, 0); 5304 mlxsw_reg_ralue_act_ip2me_pack(ralue_pl); 5305 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), 5306 ralue_pl); 5307 if (err) 5308 return err; 5309 } 5310 5311 return 0; 5312 } 5313 5314 static int mlxsw_sp_router_fibmr_add(struct mlxsw_sp *mlxsw_sp, 5315 struct mfc_entry_notifier_info *men_info, 5316 bool replace) 5317 { 5318 struct mlxsw_sp_vr *vr; 5319 5320 if (mlxsw_sp->router->aborted) 5321 return 0; 5322 5323 vr = mlxsw_sp_vr_get(mlxsw_sp, men_info->tb_id, NULL); 5324 if (IS_ERR(vr)) 5325 return PTR_ERR(vr); 5326 5327 return mlxsw_sp_mr_route4_add(vr->mr4_table, men_info->mfc, replace); 5328 } 5329 5330 static void mlxsw_sp_router_fibmr_del(struct mlxsw_sp *mlxsw_sp, 5331 struct mfc_entry_notifier_info *men_info) 5332 { 5333 struct mlxsw_sp_vr *vr; 5334 5335 if (mlxsw_sp->router->aborted) 5336 return; 5337 5338 vr = mlxsw_sp_vr_find(mlxsw_sp, men_info->tb_id); 5339 if (WARN_ON(!vr)) 5340 return; 5341 5342 mlxsw_sp_mr_route4_del(vr->mr4_table, men_info->mfc); 5343 mlxsw_sp_vr_put(vr); 5344 } 5345 5346 static int 5347 mlxsw_sp_router_fibmr_vif_add(struct mlxsw_sp *mlxsw_sp, 5348 struct vif_entry_notifier_info *ven_info) 5349 { 5350 struct mlxsw_sp_rif *rif; 5351 struct mlxsw_sp_vr *vr; 5352 5353 if (mlxsw_sp->router->aborted) 5354 return 0; 5355 5356 vr = mlxsw_sp_vr_get(mlxsw_sp, ven_info->tb_id, NULL); 5357 if (IS_ERR(vr)) 5358 return PTR_ERR(vr); 5359 5360 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, ven_info->dev); 5361 return mlxsw_sp_mr_vif_add(vr->mr4_table, ven_info->dev, 5362 ven_info->vif_index, 5363 ven_info->vif_flags, rif); 5364 } 5365 5366 static void 5367 mlxsw_sp_router_fibmr_vif_del(struct mlxsw_sp *mlxsw_sp, 5368 struct vif_entry_notifier_info *ven_info) 5369 { 5370 struct mlxsw_sp_vr *vr; 5371 5372 if (mlxsw_sp->router->aborted) 5373 return; 5374 5375 vr = mlxsw_sp_vr_find(mlxsw_sp, ven_info->tb_id); 5376 if (WARN_ON(!vr)) 5377 return; 5378 5379 mlxsw_sp_mr_vif_del(vr->mr4_table, ven_info->vif_index); 5380 mlxsw_sp_vr_put(vr); 5381 } 5382 5383 static int mlxsw_sp_router_set_abort_trap(struct mlxsw_sp *mlxsw_sp) 5384 { 5385 enum mlxsw_reg_ralxx_protocol proto = MLXSW_REG_RALXX_PROTOCOL_IPV4; 5386 int err; 5387 5388 err = __mlxsw_sp_router_set_abort_trap(mlxsw_sp, proto, 5389 MLXSW_SP_LPM_TREE_MIN); 5390 if (err) 5391 return err; 5392 5393 /* The multicast router code does not need an abort trap as by default, 5394 * packets that don't match any routes are trapped to the CPU. 5395 */ 5396 5397 proto = MLXSW_REG_RALXX_PROTOCOL_IPV6; 5398 return __mlxsw_sp_router_set_abort_trap(mlxsw_sp, proto, 5399 MLXSW_SP_LPM_TREE_MIN + 1); 5400 } 5401 5402 static void mlxsw_sp_fib4_node_flush(struct mlxsw_sp *mlxsw_sp, 5403 struct mlxsw_sp_fib_node *fib_node) 5404 { 5405 struct mlxsw_sp_fib4_entry *fib4_entry, *tmp; 5406 5407 list_for_each_entry_safe(fib4_entry, tmp, &fib_node->entry_list, 5408 common.list) { 5409 bool do_break = &tmp->common.list == &fib_node->entry_list; 5410 5411 mlxsw_sp_fib4_node_entry_unlink(mlxsw_sp, fib4_entry); 5412 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry); 5413 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 5414 /* Break when entry list is empty and node was freed. 5415 * Otherwise, we'll access freed memory in the next 5416 * iteration. 5417 */ 5418 if (do_break) 5419 break; 5420 } 5421 } 5422 5423 static void mlxsw_sp_fib6_node_flush(struct mlxsw_sp *mlxsw_sp, 5424 struct mlxsw_sp_fib_node *fib_node) 5425 { 5426 struct mlxsw_sp_fib6_entry *fib6_entry, *tmp; 5427 5428 list_for_each_entry_safe(fib6_entry, tmp, &fib_node->entry_list, 5429 common.list) { 5430 bool do_break = &tmp->common.list == &fib_node->entry_list; 5431 5432 mlxsw_sp_fib6_node_entry_unlink(mlxsw_sp, fib6_entry); 5433 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry); 5434 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 5435 if (do_break) 5436 break; 5437 } 5438 } 5439 5440 static void mlxsw_sp_fib_node_flush(struct mlxsw_sp *mlxsw_sp, 5441 struct mlxsw_sp_fib_node *fib_node) 5442 { 5443 switch (fib_node->fib->proto) { 5444 case MLXSW_SP_L3_PROTO_IPV4: 5445 mlxsw_sp_fib4_node_flush(mlxsw_sp, fib_node); 5446 break; 5447 case MLXSW_SP_L3_PROTO_IPV6: 5448 mlxsw_sp_fib6_node_flush(mlxsw_sp, fib_node); 5449 break; 5450 } 5451 } 5452 5453 static void mlxsw_sp_vr_fib_flush(struct mlxsw_sp *mlxsw_sp, 5454 struct mlxsw_sp_vr *vr, 5455 enum mlxsw_sp_l3proto proto) 5456 { 5457 struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto); 5458 struct mlxsw_sp_fib_node *fib_node, *tmp; 5459 5460 list_for_each_entry_safe(fib_node, tmp, &fib->node_list, list) { 5461 bool do_break = &tmp->list == &fib->node_list; 5462 5463 mlxsw_sp_fib_node_flush(mlxsw_sp, fib_node); 5464 if (do_break) 5465 break; 5466 } 5467 } 5468 5469 static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp) 5470 { 5471 int i; 5472 5473 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) { 5474 struct mlxsw_sp_vr *vr = &mlxsw_sp->router->vrs[i]; 5475 5476 if (!mlxsw_sp_vr_is_used(vr)) 5477 continue; 5478 5479 mlxsw_sp_mr_table_flush(vr->mr4_table); 5480 mlxsw_sp_vr_fib_flush(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV4); 5481 5482 /* If virtual router was only used for IPv4, then it's no 5483 * longer used. 5484 */ 5485 if (!mlxsw_sp_vr_is_used(vr)) 5486 continue; 5487 mlxsw_sp_vr_fib_flush(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV6); 5488 } 5489 } 5490 5491 static void mlxsw_sp_router_fib_abort(struct mlxsw_sp *mlxsw_sp) 5492 { 5493 int err; 5494 5495 if (mlxsw_sp->router->aborted) 5496 return; 5497 dev_warn(mlxsw_sp->bus_info->dev, "FIB abort triggered. Note that FIB entries are no longer being offloaded to this device.\n"); 5498 mlxsw_sp_router_fib_flush(mlxsw_sp); 5499 mlxsw_sp->router->aborted = true; 5500 err = mlxsw_sp_router_set_abort_trap(mlxsw_sp); 5501 if (err) 5502 dev_warn(mlxsw_sp->bus_info->dev, "Failed to set abort trap.\n"); 5503 } 5504 5505 struct mlxsw_sp_fib_event_work { 5506 struct work_struct work; 5507 union { 5508 struct fib6_entry_notifier_info fen6_info; 5509 struct fib_entry_notifier_info fen_info; 5510 struct fib_rule_notifier_info fr_info; 5511 struct fib_nh_notifier_info fnh_info; 5512 struct mfc_entry_notifier_info men_info; 5513 struct vif_entry_notifier_info ven_info; 5514 }; 5515 struct mlxsw_sp *mlxsw_sp; 5516 unsigned long event; 5517 }; 5518 5519 static void mlxsw_sp_router_fib4_event_work(struct work_struct *work) 5520 { 5521 struct mlxsw_sp_fib_event_work *fib_work = 5522 container_of(work, struct mlxsw_sp_fib_event_work, work); 5523 struct mlxsw_sp *mlxsw_sp = fib_work->mlxsw_sp; 5524 bool replace, append; 5525 int err; 5526 5527 /* Protect internal structures from changes */ 5528 rtnl_lock(); 5529 switch (fib_work->event) { 5530 case FIB_EVENT_ENTRY_REPLACE: /* fall through */ 5531 case FIB_EVENT_ENTRY_APPEND: /* fall through */ 5532 case FIB_EVENT_ENTRY_ADD: 5533 replace = fib_work->event == FIB_EVENT_ENTRY_REPLACE; 5534 append = fib_work->event == FIB_EVENT_ENTRY_APPEND; 5535 err = mlxsw_sp_router_fib4_add(mlxsw_sp, &fib_work->fen_info, 5536 replace, append); 5537 if (err) 5538 mlxsw_sp_router_fib_abort(mlxsw_sp); 5539 fib_info_put(fib_work->fen_info.fi); 5540 break; 5541 case FIB_EVENT_ENTRY_DEL: 5542 mlxsw_sp_router_fib4_del(mlxsw_sp, &fib_work->fen_info); 5543 fib_info_put(fib_work->fen_info.fi); 5544 break; 5545 case FIB_EVENT_RULE_ADD: 5546 /* if we get here, a rule was added that we do not support. 5547 * just do the fib_abort 5548 */ 5549 mlxsw_sp_router_fib_abort(mlxsw_sp); 5550 break; 5551 case FIB_EVENT_NH_ADD: /* fall through */ 5552 case FIB_EVENT_NH_DEL: 5553 mlxsw_sp_nexthop4_event(mlxsw_sp, fib_work->event, 5554 fib_work->fnh_info.fib_nh); 5555 fib_info_put(fib_work->fnh_info.fib_nh->nh_parent); 5556 break; 5557 } 5558 rtnl_unlock(); 5559 kfree(fib_work); 5560 } 5561 5562 static void mlxsw_sp_router_fib6_event_work(struct work_struct *work) 5563 { 5564 struct mlxsw_sp_fib_event_work *fib_work = 5565 container_of(work, struct mlxsw_sp_fib_event_work, work); 5566 struct mlxsw_sp *mlxsw_sp = fib_work->mlxsw_sp; 5567 bool replace; 5568 int err; 5569 5570 rtnl_lock(); 5571 switch (fib_work->event) { 5572 case FIB_EVENT_ENTRY_REPLACE: /* fall through */ 5573 case FIB_EVENT_ENTRY_ADD: 5574 replace = fib_work->event == FIB_EVENT_ENTRY_REPLACE; 5575 err = mlxsw_sp_router_fib6_add(mlxsw_sp, 5576 fib_work->fen6_info.rt, replace); 5577 if (err) 5578 mlxsw_sp_router_fib_abort(mlxsw_sp); 5579 mlxsw_sp_rt6_release(fib_work->fen6_info.rt); 5580 break; 5581 case FIB_EVENT_ENTRY_DEL: 5582 mlxsw_sp_router_fib6_del(mlxsw_sp, fib_work->fen6_info.rt); 5583 mlxsw_sp_rt6_release(fib_work->fen6_info.rt); 5584 break; 5585 case FIB_EVENT_RULE_ADD: 5586 /* if we get here, a rule was added that we do not support. 5587 * just do the fib_abort 5588 */ 5589 mlxsw_sp_router_fib_abort(mlxsw_sp); 5590 break; 5591 } 5592 rtnl_unlock(); 5593 kfree(fib_work); 5594 } 5595 5596 static void mlxsw_sp_router_fibmr_event_work(struct work_struct *work) 5597 { 5598 struct mlxsw_sp_fib_event_work *fib_work = 5599 container_of(work, struct mlxsw_sp_fib_event_work, work); 5600 struct mlxsw_sp *mlxsw_sp = fib_work->mlxsw_sp; 5601 bool replace; 5602 int err; 5603 5604 rtnl_lock(); 5605 switch (fib_work->event) { 5606 case FIB_EVENT_ENTRY_REPLACE: /* fall through */ 5607 case FIB_EVENT_ENTRY_ADD: 5608 replace = fib_work->event == FIB_EVENT_ENTRY_REPLACE; 5609 5610 err = mlxsw_sp_router_fibmr_add(mlxsw_sp, &fib_work->men_info, 5611 replace); 5612 if (err) 5613 mlxsw_sp_router_fib_abort(mlxsw_sp); 5614 ipmr_cache_put(fib_work->men_info.mfc); 5615 break; 5616 case FIB_EVENT_ENTRY_DEL: 5617 mlxsw_sp_router_fibmr_del(mlxsw_sp, &fib_work->men_info); 5618 ipmr_cache_put(fib_work->men_info.mfc); 5619 break; 5620 case FIB_EVENT_VIF_ADD: 5621 err = mlxsw_sp_router_fibmr_vif_add(mlxsw_sp, 5622 &fib_work->ven_info); 5623 if (err) 5624 mlxsw_sp_router_fib_abort(mlxsw_sp); 5625 dev_put(fib_work->ven_info.dev); 5626 break; 5627 case FIB_EVENT_VIF_DEL: 5628 mlxsw_sp_router_fibmr_vif_del(mlxsw_sp, 5629 &fib_work->ven_info); 5630 dev_put(fib_work->ven_info.dev); 5631 break; 5632 case FIB_EVENT_RULE_ADD: 5633 /* if we get here, a rule was added that we do not support. 5634 * just do the fib_abort 5635 */ 5636 mlxsw_sp_router_fib_abort(mlxsw_sp); 5637 break; 5638 } 5639 rtnl_unlock(); 5640 kfree(fib_work); 5641 } 5642 5643 static void mlxsw_sp_router_fib4_event(struct mlxsw_sp_fib_event_work *fib_work, 5644 struct fib_notifier_info *info) 5645 { 5646 struct fib_entry_notifier_info *fen_info; 5647 struct fib_nh_notifier_info *fnh_info; 5648 5649 switch (fib_work->event) { 5650 case FIB_EVENT_ENTRY_REPLACE: /* fall through */ 5651 case FIB_EVENT_ENTRY_APPEND: /* fall through */ 5652 case FIB_EVENT_ENTRY_ADD: /* fall through */ 5653 case FIB_EVENT_ENTRY_DEL: 5654 fen_info = container_of(info, struct fib_entry_notifier_info, 5655 info); 5656 fib_work->fen_info = *fen_info; 5657 /* Take reference on fib_info to prevent it from being 5658 * freed while work is queued. Release it afterwards. 5659 */ 5660 fib_info_hold(fib_work->fen_info.fi); 5661 break; 5662 case FIB_EVENT_NH_ADD: /* fall through */ 5663 case FIB_EVENT_NH_DEL: 5664 fnh_info = container_of(info, struct fib_nh_notifier_info, 5665 info); 5666 fib_work->fnh_info = *fnh_info; 5667 fib_info_hold(fib_work->fnh_info.fib_nh->nh_parent); 5668 break; 5669 } 5670 } 5671 5672 static void mlxsw_sp_router_fib6_event(struct mlxsw_sp_fib_event_work *fib_work, 5673 struct fib_notifier_info *info) 5674 { 5675 struct fib6_entry_notifier_info *fen6_info; 5676 5677 switch (fib_work->event) { 5678 case FIB_EVENT_ENTRY_REPLACE: /* fall through */ 5679 case FIB_EVENT_ENTRY_ADD: /* fall through */ 5680 case FIB_EVENT_ENTRY_DEL: 5681 fen6_info = container_of(info, struct fib6_entry_notifier_info, 5682 info); 5683 fib_work->fen6_info = *fen6_info; 5684 rt6_hold(fib_work->fen6_info.rt); 5685 break; 5686 } 5687 } 5688 5689 static void 5690 mlxsw_sp_router_fibmr_event(struct mlxsw_sp_fib_event_work *fib_work, 5691 struct fib_notifier_info *info) 5692 { 5693 switch (fib_work->event) { 5694 case FIB_EVENT_ENTRY_REPLACE: /* fall through */ 5695 case FIB_EVENT_ENTRY_ADD: /* fall through */ 5696 case FIB_EVENT_ENTRY_DEL: 5697 memcpy(&fib_work->men_info, info, sizeof(fib_work->men_info)); 5698 ipmr_cache_hold(fib_work->men_info.mfc); 5699 break; 5700 case FIB_EVENT_VIF_ADD: /* fall through */ 5701 case FIB_EVENT_VIF_DEL: 5702 memcpy(&fib_work->ven_info, info, sizeof(fib_work->ven_info)); 5703 dev_hold(fib_work->ven_info.dev); 5704 break; 5705 } 5706 } 5707 5708 static int mlxsw_sp_router_fib_rule_event(unsigned long event, 5709 struct fib_notifier_info *info, 5710 struct mlxsw_sp *mlxsw_sp) 5711 { 5712 struct netlink_ext_ack *extack = info->extack; 5713 struct fib_rule_notifier_info *fr_info; 5714 struct fib_rule *rule; 5715 int err = 0; 5716 5717 /* nothing to do at the moment */ 5718 if (event == FIB_EVENT_RULE_DEL) 5719 return 0; 5720 5721 if (mlxsw_sp->router->aborted) 5722 return 0; 5723 5724 fr_info = container_of(info, struct fib_rule_notifier_info, info); 5725 rule = fr_info->rule; 5726 5727 switch (info->family) { 5728 case AF_INET: 5729 if (!fib4_rule_default(rule) && !rule->l3mdev) 5730 err = -1; 5731 break; 5732 case AF_INET6: 5733 if (!fib6_rule_default(rule) && !rule->l3mdev) 5734 err = -1; 5735 break; 5736 case RTNL_FAMILY_IPMR: 5737 if (!ipmr_rule_default(rule) && !rule->l3mdev) 5738 err = -1; 5739 break; 5740 } 5741 5742 if (err < 0) 5743 NL_SET_ERR_MSG(extack, "spectrum: FIB rules not supported. Aborting offload"); 5744 5745 return err; 5746 } 5747 5748 /* Called with rcu_read_lock() */ 5749 static int mlxsw_sp_router_fib_event(struct notifier_block *nb, 5750 unsigned long event, void *ptr) 5751 { 5752 struct mlxsw_sp_fib_event_work *fib_work; 5753 struct fib_notifier_info *info = ptr; 5754 struct mlxsw_sp_router *router; 5755 int err; 5756 5757 if (!net_eq(info->net, &init_net) || 5758 (info->family != AF_INET && info->family != AF_INET6 && 5759 info->family != RTNL_FAMILY_IPMR)) 5760 return NOTIFY_DONE; 5761 5762 router = container_of(nb, struct mlxsw_sp_router, fib_nb); 5763 5764 switch (event) { 5765 case FIB_EVENT_RULE_ADD: /* fall through */ 5766 case FIB_EVENT_RULE_DEL: 5767 err = mlxsw_sp_router_fib_rule_event(event, info, 5768 router->mlxsw_sp); 5769 if (!err) 5770 return NOTIFY_DONE; 5771 } 5772 5773 fib_work = kzalloc(sizeof(*fib_work), GFP_ATOMIC); 5774 if (WARN_ON(!fib_work)) 5775 return NOTIFY_BAD; 5776 5777 fib_work->mlxsw_sp = router->mlxsw_sp; 5778 fib_work->event = event; 5779 5780 switch (info->family) { 5781 case AF_INET: 5782 INIT_WORK(&fib_work->work, mlxsw_sp_router_fib4_event_work); 5783 mlxsw_sp_router_fib4_event(fib_work, info); 5784 break; 5785 case AF_INET6: 5786 INIT_WORK(&fib_work->work, mlxsw_sp_router_fib6_event_work); 5787 mlxsw_sp_router_fib6_event(fib_work, info); 5788 break; 5789 case RTNL_FAMILY_IPMR: 5790 INIT_WORK(&fib_work->work, mlxsw_sp_router_fibmr_event_work); 5791 mlxsw_sp_router_fibmr_event(fib_work, info); 5792 break; 5793 } 5794 5795 mlxsw_core_schedule_work(&fib_work->work); 5796 5797 return NOTIFY_DONE; 5798 } 5799 5800 static struct mlxsw_sp_rif * 5801 mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp, 5802 const struct net_device *dev) 5803 { 5804 int i; 5805 5806 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++) 5807 if (mlxsw_sp->router->rifs[i] && 5808 mlxsw_sp->router->rifs[i]->dev == dev) 5809 return mlxsw_sp->router->rifs[i]; 5810 5811 return NULL; 5812 } 5813 5814 static int mlxsw_sp_router_rif_disable(struct mlxsw_sp *mlxsw_sp, u16 rif) 5815 { 5816 char ritr_pl[MLXSW_REG_RITR_LEN]; 5817 int err; 5818 5819 mlxsw_reg_ritr_rif_pack(ritr_pl, rif); 5820 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 5821 if (WARN_ON_ONCE(err)) 5822 return err; 5823 5824 mlxsw_reg_ritr_enable_set(ritr_pl, false); 5825 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 5826 } 5827 5828 static void mlxsw_sp_router_rif_gone_sync(struct mlxsw_sp *mlxsw_sp, 5829 struct mlxsw_sp_rif *rif) 5830 { 5831 mlxsw_sp_router_rif_disable(mlxsw_sp, rif->rif_index); 5832 mlxsw_sp_nexthop_rif_gone_sync(mlxsw_sp, rif); 5833 mlxsw_sp_neigh_rif_gone_sync(mlxsw_sp, rif); 5834 } 5835 5836 static bool 5837 mlxsw_sp_rif_should_config(struct mlxsw_sp_rif *rif, struct net_device *dev, 5838 unsigned long event) 5839 { 5840 struct inet6_dev *inet6_dev; 5841 bool addr_list_empty = true; 5842 struct in_device *idev; 5843 5844 switch (event) { 5845 case NETDEV_UP: 5846 return rif == NULL; 5847 case NETDEV_DOWN: 5848 idev = __in_dev_get_rtnl(dev); 5849 if (idev && idev->ifa_list) 5850 addr_list_empty = false; 5851 5852 inet6_dev = __in6_dev_get(dev); 5853 if (addr_list_empty && inet6_dev && 5854 !list_empty(&inet6_dev->addr_list)) 5855 addr_list_empty = false; 5856 5857 if (rif && addr_list_empty && 5858 !netif_is_l3_slave(rif->dev)) 5859 return true; 5860 /* It is possible we already removed the RIF ourselves 5861 * if it was assigned to a netdev that is now a bridge 5862 * or LAG slave. 5863 */ 5864 return false; 5865 } 5866 5867 return false; 5868 } 5869 5870 static enum mlxsw_sp_rif_type 5871 mlxsw_sp_dev_rif_type(const struct mlxsw_sp *mlxsw_sp, 5872 const struct net_device *dev) 5873 { 5874 enum mlxsw_sp_fid_type type; 5875 5876 if (mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, NULL)) 5877 return MLXSW_SP_RIF_TYPE_IPIP_LB; 5878 5879 /* Otherwise RIF type is derived from the type of the underlying FID. */ 5880 if (is_vlan_dev(dev) && netif_is_bridge_master(vlan_dev_real_dev(dev))) 5881 type = MLXSW_SP_FID_TYPE_8021Q; 5882 else if (netif_is_bridge_master(dev) && br_vlan_enabled(dev)) 5883 type = MLXSW_SP_FID_TYPE_8021Q; 5884 else if (netif_is_bridge_master(dev)) 5885 type = MLXSW_SP_FID_TYPE_8021D; 5886 else 5887 type = MLXSW_SP_FID_TYPE_RFID; 5888 5889 return mlxsw_sp_fid_type_rif_type(mlxsw_sp, type); 5890 } 5891 5892 static int mlxsw_sp_rif_index_alloc(struct mlxsw_sp *mlxsw_sp, u16 *p_rif_index) 5893 { 5894 int i; 5895 5896 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++) { 5897 if (!mlxsw_sp->router->rifs[i]) { 5898 *p_rif_index = i; 5899 return 0; 5900 } 5901 } 5902 5903 return -ENOBUFS; 5904 } 5905 5906 static struct mlxsw_sp_rif *mlxsw_sp_rif_alloc(size_t rif_size, u16 rif_index, 5907 u16 vr_id, 5908 struct net_device *l3_dev) 5909 { 5910 struct mlxsw_sp_rif *rif; 5911 5912 rif = kzalloc(rif_size, GFP_KERNEL); 5913 if (!rif) 5914 return NULL; 5915 5916 INIT_LIST_HEAD(&rif->nexthop_list); 5917 INIT_LIST_HEAD(&rif->neigh_list); 5918 ether_addr_copy(rif->addr, l3_dev->dev_addr); 5919 rif->mtu = l3_dev->mtu; 5920 rif->vr_id = vr_id; 5921 rif->dev = l3_dev; 5922 rif->rif_index = rif_index; 5923 5924 return rif; 5925 } 5926 5927 struct mlxsw_sp_rif *mlxsw_sp_rif_by_index(const struct mlxsw_sp *mlxsw_sp, 5928 u16 rif_index) 5929 { 5930 return mlxsw_sp->router->rifs[rif_index]; 5931 } 5932 5933 u16 mlxsw_sp_rif_index(const struct mlxsw_sp_rif *rif) 5934 { 5935 return rif->rif_index; 5936 } 5937 5938 u16 mlxsw_sp_ipip_lb_rif_index(const struct mlxsw_sp_rif_ipip_lb *lb_rif) 5939 { 5940 return lb_rif->common.rif_index; 5941 } 5942 5943 u16 mlxsw_sp_ipip_lb_ul_vr_id(const struct mlxsw_sp_rif_ipip_lb *lb_rif) 5944 { 5945 return lb_rif->ul_vr_id; 5946 } 5947 5948 int mlxsw_sp_rif_dev_ifindex(const struct mlxsw_sp_rif *rif) 5949 { 5950 return rif->dev->ifindex; 5951 } 5952 5953 const struct net_device *mlxsw_sp_rif_dev(const struct mlxsw_sp_rif *rif) 5954 { 5955 return rif->dev; 5956 } 5957 5958 static struct mlxsw_sp_rif * 5959 mlxsw_sp_rif_create(struct mlxsw_sp *mlxsw_sp, 5960 const struct mlxsw_sp_rif_params *params, 5961 struct netlink_ext_ack *extack) 5962 { 5963 u32 tb_id = l3mdev_fib_table(params->dev); 5964 const struct mlxsw_sp_rif_ops *ops; 5965 struct mlxsw_sp_fid *fid = NULL; 5966 enum mlxsw_sp_rif_type type; 5967 struct mlxsw_sp_rif *rif; 5968 struct mlxsw_sp_vr *vr; 5969 u16 rif_index; 5970 int err; 5971 5972 type = mlxsw_sp_dev_rif_type(mlxsw_sp, params->dev); 5973 ops = mlxsw_sp->router->rif_ops_arr[type]; 5974 5975 vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id ? : RT_TABLE_MAIN, extack); 5976 if (IS_ERR(vr)) 5977 return ERR_CAST(vr); 5978 vr->rif_count++; 5979 5980 err = mlxsw_sp_rif_index_alloc(mlxsw_sp, &rif_index); 5981 if (err) { 5982 NL_SET_ERR_MSG(extack, "spectrum: Exceeded number of supported router interfaces"); 5983 goto err_rif_index_alloc; 5984 } 5985 5986 rif = mlxsw_sp_rif_alloc(ops->rif_size, rif_index, vr->id, params->dev); 5987 if (!rif) { 5988 err = -ENOMEM; 5989 goto err_rif_alloc; 5990 } 5991 rif->mlxsw_sp = mlxsw_sp; 5992 rif->ops = ops; 5993 5994 if (ops->fid_get) { 5995 fid = ops->fid_get(rif); 5996 if (IS_ERR(fid)) { 5997 err = PTR_ERR(fid); 5998 goto err_fid_get; 5999 } 6000 rif->fid = fid; 6001 } 6002 6003 if (ops->setup) 6004 ops->setup(rif, params); 6005 6006 err = ops->configure(rif); 6007 if (err) 6008 goto err_configure; 6009 6010 err = mlxsw_sp_mr_rif_add(vr->mr4_table, rif); 6011 if (err) 6012 goto err_mr_rif_add; 6013 6014 mlxsw_sp_rif_counters_alloc(rif); 6015 mlxsw_sp->router->rifs[rif_index] = rif; 6016 6017 return rif; 6018 6019 err_mr_rif_add: 6020 ops->deconfigure(rif); 6021 err_configure: 6022 if (fid) 6023 mlxsw_sp_fid_put(fid); 6024 err_fid_get: 6025 kfree(rif); 6026 err_rif_alloc: 6027 err_rif_index_alloc: 6028 vr->rif_count--; 6029 mlxsw_sp_vr_put(vr); 6030 return ERR_PTR(err); 6031 } 6032 6033 void mlxsw_sp_rif_destroy(struct mlxsw_sp_rif *rif) 6034 { 6035 const struct mlxsw_sp_rif_ops *ops = rif->ops; 6036 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 6037 struct mlxsw_sp_fid *fid = rif->fid; 6038 struct mlxsw_sp_vr *vr; 6039 6040 mlxsw_sp_router_rif_gone_sync(mlxsw_sp, rif); 6041 vr = &mlxsw_sp->router->vrs[rif->vr_id]; 6042 6043 mlxsw_sp->router->rifs[rif->rif_index] = NULL; 6044 mlxsw_sp_rif_counters_free(rif); 6045 mlxsw_sp_mr_rif_del(vr->mr4_table, rif); 6046 ops->deconfigure(rif); 6047 if (fid) 6048 /* Loopback RIFs are not associated with a FID. */ 6049 mlxsw_sp_fid_put(fid); 6050 kfree(rif); 6051 vr->rif_count--; 6052 mlxsw_sp_vr_put(vr); 6053 } 6054 6055 static void 6056 mlxsw_sp_rif_subport_params_init(struct mlxsw_sp_rif_params *params, 6057 struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan) 6058 { 6059 struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port; 6060 6061 params->vid = mlxsw_sp_port_vlan->vid; 6062 params->lag = mlxsw_sp_port->lagged; 6063 if (params->lag) 6064 params->lag_id = mlxsw_sp_port->lag_id; 6065 else 6066 params->system_port = mlxsw_sp_port->local_port; 6067 } 6068 6069 static int 6070 mlxsw_sp_port_vlan_router_join(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan, 6071 struct net_device *l3_dev, 6072 struct netlink_ext_ack *extack) 6073 { 6074 struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port; 6075 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp; 6076 u16 vid = mlxsw_sp_port_vlan->vid; 6077 struct mlxsw_sp_rif *rif; 6078 struct mlxsw_sp_fid *fid; 6079 int err; 6080 6081 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev); 6082 if (!rif) { 6083 struct mlxsw_sp_rif_params params = { 6084 .dev = l3_dev, 6085 }; 6086 6087 mlxsw_sp_rif_subport_params_init(¶ms, mlxsw_sp_port_vlan); 6088 rif = mlxsw_sp_rif_create(mlxsw_sp, ¶ms, extack); 6089 if (IS_ERR(rif)) 6090 return PTR_ERR(rif); 6091 } 6092 6093 /* FID was already created, just take a reference */ 6094 fid = rif->ops->fid_get(rif); 6095 err = mlxsw_sp_fid_port_vid_map(fid, mlxsw_sp_port, vid); 6096 if (err) 6097 goto err_fid_port_vid_map; 6098 6099 err = mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, false); 6100 if (err) 6101 goto err_port_vid_learning_set; 6102 6103 err = mlxsw_sp_port_vid_stp_set(mlxsw_sp_port, vid, 6104 BR_STATE_FORWARDING); 6105 if (err) 6106 goto err_port_vid_stp_set; 6107 6108 mlxsw_sp_port_vlan->fid = fid; 6109 6110 return 0; 6111 6112 err_port_vid_stp_set: 6113 mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, true); 6114 err_port_vid_learning_set: 6115 mlxsw_sp_fid_port_vid_unmap(fid, mlxsw_sp_port, vid); 6116 err_fid_port_vid_map: 6117 mlxsw_sp_fid_put(fid); 6118 return err; 6119 } 6120 6121 void 6122 mlxsw_sp_port_vlan_router_leave(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan) 6123 { 6124 struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port; 6125 struct mlxsw_sp_fid *fid = mlxsw_sp_port_vlan->fid; 6126 u16 vid = mlxsw_sp_port_vlan->vid; 6127 6128 if (WARN_ON(mlxsw_sp_fid_type(fid) != MLXSW_SP_FID_TYPE_RFID)) 6129 return; 6130 6131 mlxsw_sp_port_vlan->fid = NULL; 6132 mlxsw_sp_port_vid_stp_set(mlxsw_sp_port, vid, BR_STATE_BLOCKING); 6133 mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, true); 6134 mlxsw_sp_fid_port_vid_unmap(fid, mlxsw_sp_port, vid); 6135 /* If router port holds the last reference on the rFID, then the 6136 * associated Sub-port RIF will be destroyed. 6137 */ 6138 mlxsw_sp_fid_put(fid); 6139 } 6140 6141 static int mlxsw_sp_inetaddr_port_vlan_event(struct net_device *l3_dev, 6142 struct net_device *port_dev, 6143 unsigned long event, u16 vid, 6144 struct netlink_ext_ack *extack) 6145 { 6146 struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(port_dev); 6147 struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan; 6148 6149 mlxsw_sp_port_vlan = mlxsw_sp_port_vlan_find_by_vid(mlxsw_sp_port, vid); 6150 if (WARN_ON(!mlxsw_sp_port_vlan)) 6151 return -EINVAL; 6152 6153 switch (event) { 6154 case NETDEV_UP: 6155 return mlxsw_sp_port_vlan_router_join(mlxsw_sp_port_vlan, 6156 l3_dev, extack); 6157 case NETDEV_DOWN: 6158 mlxsw_sp_port_vlan_router_leave(mlxsw_sp_port_vlan); 6159 break; 6160 } 6161 6162 return 0; 6163 } 6164 6165 static int mlxsw_sp_inetaddr_port_event(struct net_device *port_dev, 6166 unsigned long event, 6167 struct netlink_ext_ack *extack) 6168 { 6169 if (netif_is_bridge_port(port_dev) || 6170 netif_is_lag_port(port_dev) || 6171 netif_is_ovs_port(port_dev)) 6172 return 0; 6173 6174 return mlxsw_sp_inetaddr_port_vlan_event(port_dev, port_dev, event, 1, 6175 extack); 6176 } 6177 6178 static int __mlxsw_sp_inetaddr_lag_event(struct net_device *l3_dev, 6179 struct net_device *lag_dev, 6180 unsigned long event, u16 vid, 6181 struct netlink_ext_ack *extack) 6182 { 6183 struct net_device *port_dev; 6184 struct list_head *iter; 6185 int err; 6186 6187 netdev_for_each_lower_dev(lag_dev, port_dev, iter) { 6188 if (mlxsw_sp_port_dev_check(port_dev)) { 6189 err = mlxsw_sp_inetaddr_port_vlan_event(l3_dev, 6190 port_dev, 6191 event, vid, 6192 extack); 6193 if (err) 6194 return err; 6195 } 6196 } 6197 6198 return 0; 6199 } 6200 6201 static int mlxsw_sp_inetaddr_lag_event(struct net_device *lag_dev, 6202 unsigned long event, 6203 struct netlink_ext_ack *extack) 6204 { 6205 if (netif_is_bridge_port(lag_dev)) 6206 return 0; 6207 6208 return __mlxsw_sp_inetaddr_lag_event(lag_dev, lag_dev, event, 1, 6209 extack); 6210 } 6211 6212 static int mlxsw_sp_inetaddr_bridge_event(struct net_device *l3_dev, 6213 unsigned long event, 6214 struct netlink_ext_ack *extack) 6215 { 6216 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_lower_get(l3_dev); 6217 struct mlxsw_sp_rif_params params = { 6218 .dev = l3_dev, 6219 }; 6220 struct mlxsw_sp_rif *rif; 6221 6222 switch (event) { 6223 case NETDEV_UP: 6224 rif = mlxsw_sp_rif_create(mlxsw_sp, ¶ms, extack); 6225 if (IS_ERR(rif)) 6226 return PTR_ERR(rif); 6227 break; 6228 case NETDEV_DOWN: 6229 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev); 6230 mlxsw_sp_rif_destroy(rif); 6231 break; 6232 } 6233 6234 return 0; 6235 } 6236 6237 static int mlxsw_sp_inetaddr_vlan_event(struct net_device *vlan_dev, 6238 unsigned long event, 6239 struct netlink_ext_ack *extack) 6240 { 6241 struct net_device *real_dev = vlan_dev_real_dev(vlan_dev); 6242 u16 vid = vlan_dev_vlan_id(vlan_dev); 6243 6244 if (netif_is_bridge_port(vlan_dev)) 6245 return 0; 6246 6247 if (mlxsw_sp_port_dev_check(real_dev)) 6248 return mlxsw_sp_inetaddr_port_vlan_event(vlan_dev, real_dev, 6249 event, vid, extack); 6250 else if (netif_is_lag_master(real_dev)) 6251 return __mlxsw_sp_inetaddr_lag_event(vlan_dev, real_dev, event, 6252 vid, extack); 6253 else if (netif_is_bridge_master(real_dev) && br_vlan_enabled(real_dev)) 6254 return mlxsw_sp_inetaddr_bridge_event(vlan_dev, event, extack); 6255 6256 return 0; 6257 } 6258 6259 static int __mlxsw_sp_inetaddr_event(struct net_device *dev, 6260 unsigned long event, 6261 struct netlink_ext_ack *extack) 6262 { 6263 if (mlxsw_sp_port_dev_check(dev)) 6264 return mlxsw_sp_inetaddr_port_event(dev, event, extack); 6265 else if (netif_is_lag_master(dev)) 6266 return mlxsw_sp_inetaddr_lag_event(dev, event, extack); 6267 else if (netif_is_bridge_master(dev)) 6268 return mlxsw_sp_inetaddr_bridge_event(dev, event, extack); 6269 else if (is_vlan_dev(dev)) 6270 return mlxsw_sp_inetaddr_vlan_event(dev, event, extack); 6271 else 6272 return 0; 6273 } 6274 6275 int mlxsw_sp_inetaddr_event(struct notifier_block *unused, 6276 unsigned long event, void *ptr) 6277 { 6278 struct in_ifaddr *ifa = (struct in_ifaddr *) ptr; 6279 struct net_device *dev = ifa->ifa_dev->dev; 6280 struct mlxsw_sp *mlxsw_sp; 6281 struct mlxsw_sp_rif *rif; 6282 int err = 0; 6283 6284 /* NETDEV_UP event is handled by mlxsw_sp_inetaddr_valid_event */ 6285 if (event == NETDEV_UP) 6286 goto out; 6287 6288 mlxsw_sp = mlxsw_sp_lower_get(dev); 6289 if (!mlxsw_sp) 6290 goto out; 6291 6292 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 6293 if (!mlxsw_sp_rif_should_config(rif, dev, event)) 6294 goto out; 6295 6296 err = __mlxsw_sp_inetaddr_event(dev, event, NULL); 6297 out: 6298 return notifier_from_errno(err); 6299 } 6300 6301 int mlxsw_sp_inetaddr_valid_event(struct notifier_block *unused, 6302 unsigned long event, void *ptr) 6303 { 6304 struct in_validator_info *ivi = (struct in_validator_info *) ptr; 6305 struct net_device *dev = ivi->ivi_dev->dev; 6306 struct mlxsw_sp *mlxsw_sp; 6307 struct mlxsw_sp_rif *rif; 6308 int err = 0; 6309 6310 mlxsw_sp = mlxsw_sp_lower_get(dev); 6311 if (!mlxsw_sp) 6312 goto out; 6313 6314 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 6315 if (!mlxsw_sp_rif_should_config(rif, dev, event)) 6316 goto out; 6317 6318 err = __mlxsw_sp_inetaddr_event(dev, event, ivi->extack); 6319 out: 6320 return notifier_from_errno(err); 6321 } 6322 6323 struct mlxsw_sp_inet6addr_event_work { 6324 struct work_struct work; 6325 struct net_device *dev; 6326 unsigned long event; 6327 }; 6328 6329 static void mlxsw_sp_inet6addr_event_work(struct work_struct *work) 6330 { 6331 struct mlxsw_sp_inet6addr_event_work *inet6addr_work = 6332 container_of(work, struct mlxsw_sp_inet6addr_event_work, work); 6333 struct net_device *dev = inet6addr_work->dev; 6334 unsigned long event = inet6addr_work->event; 6335 struct mlxsw_sp *mlxsw_sp; 6336 struct mlxsw_sp_rif *rif; 6337 6338 rtnl_lock(); 6339 mlxsw_sp = mlxsw_sp_lower_get(dev); 6340 if (!mlxsw_sp) 6341 goto out; 6342 6343 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 6344 if (!mlxsw_sp_rif_should_config(rif, dev, event)) 6345 goto out; 6346 6347 __mlxsw_sp_inetaddr_event(dev, event, NULL); 6348 out: 6349 rtnl_unlock(); 6350 dev_put(dev); 6351 kfree(inet6addr_work); 6352 } 6353 6354 /* Called with rcu_read_lock() */ 6355 int mlxsw_sp_inet6addr_event(struct notifier_block *unused, 6356 unsigned long event, void *ptr) 6357 { 6358 struct inet6_ifaddr *if6 = (struct inet6_ifaddr *) ptr; 6359 struct mlxsw_sp_inet6addr_event_work *inet6addr_work; 6360 struct net_device *dev = if6->idev->dev; 6361 6362 /* NETDEV_UP event is handled by mlxsw_sp_inet6addr_valid_event */ 6363 if (event == NETDEV_UP) 6364 return NOTIFY_DONE; 6365 6366 if (!mlxsw_sp_port_dev_lower_find_rcu(dev)) 6367 return NOTIFY_DONE; 6368 6369 inet6addr_work = kzalloc(sizeof(*inet6addr_work), GFP_ATOMIC); 6370 if (!inet6addr_work) 6371 return NOTIFY_BAD; 6372 6373 INIT_WORK(&inet6addr_work->work, mlxsw_sp_inet6addr_event_work); 6374 inet6addr_work->dev = dev; 6375 inet6addr_work->event = event; 6376 dev_hold(dev); 6377 mlxsw_core_schedule_work(&inet6addr_work->work); 6378 6379 return NOTIFY_DONE; 6380 } 6381 6382 int mlxsw_sp_inet6addr_valid_event(struct notifier_block *unused, 6383 unsigned long event, void *ptr) 6384 { 6385 struct in6_validator_info *i6vi = (struct in6_validator_info *) ptr; 6386 struct net_device *dev = i6vi->i6vi_dev->dev; 6387 struct mlxsw_sp *mlxsw_sp; 6388 struct mlxsw_sp_rif *rif; 6389 int err = 0; 6390 6391 mlxsw_sp = mlxsw_sp_lower_get(dev); 6392 if (!mlxsw_sp) 6393 goto out; 6394 6395 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 6396 if (!mlxsw_sp_rif_should_config(rif, dev, event)) 6397 goto out; 6398 6399 err = __mlxsw_sp_inetaddr_event(dev, event, i6vi->extack); 6400 out: 6401 return notifier_from_errno(err); 6402 } 6403 6404 static int mlxsw_sp_rif_edit(struct mlxsw_sp *mlxsw_sp, u16 rif_index, 6405 const char *mac, int mtu) 6406 { 6407 char ritr_pl[MLXSW_REG_RITR_LEN]; 6408 int err; 6409 6410 mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index); 6411 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 6412 if (err) 6413 return err; 6414 6415 mlxsw_reg_ritr_mtu_set(ritr_pl, mtu); 6416 mlxsw_reg_ritr_if_mac_memcpy_to(ritr_pl, mac); 6417 mlxsw_reg_ritr_op_set(ritr_pl, MLXSW_REG_RITR_RIF_CREATE); 6418 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 6419 } 6420 6421 int mlxsw_sp_netdevice_router_port_event(struct net_device *dev) 6422 { 6423 struct mlxsw_sp *mlxsw_sp; 6424 struct mlxsw_sp_rif *rif; 6425 u16 fid_index; 6426 int err; 6427 6428 mlxsw_sp = mlxsw_sp_lower_get(dev); 6429 if (!mlxsw_sp) 6430 return 0; 6431 6432 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 6433 if (!rif) 6434 return 0; 6435 fid_index = mlxsw_sp_fid_index(rif->fid); 6436 6437 err = mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->addr, fid_index, false); 6438 if (err) 6439 return err; 6440 6441 err = mlxsw_sp_rif_edit(mlxsw_sp, rif->rif_index, dev->dev_addr, 6442 dev->mtu); 6443 if (err) 6444 goto err_rif_edit; 6445 6446 err = mlxsw_sp_rif_fdb_op(mlxsw_sp, dev->dev_addr, fid_index, true); 6447 if (err) 6448 goto err_rif_fdb_op; 6449 6450 if (rif->mtu != dev->mtu) { 6451 struct mlxsw_sp_vr *vr; 6452 6453 /* The RIF is relevant only to its mr_table instance, as unlike 6454 * unicast routing, in multicast routing a RIF cannot be shared 6455 * between several multicast routing tables. 6456 */ 6457 vr = &mlxsw_sp->router->vrs[rif->vr_id]; 6458 mlxsw_sp_mr_rif_mtu_update(vr->mr4_table, rif, dev->mtu); 6459 } 6460 6461 ether_addr_copy(rif->addr, dev->dev_addr); 6462 rif->mtu = dev->mtu; 6463 6464 netdev_dbg(dev, "Updated RIF=%d\n", rif->rif_index); 6465 6466 return 0; 6467 6468 err_rif_fdb_op: 6469 mlxsw_sp_rif_edit(mlxsw_sp, rif->rif_index, rif->addr, rif->mtu); 6470 err_rif_edit: 6471 mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->addr, fid_index, true); 6472 return err; 6473 } 6474 6475 static int mlxsw_sp_port_vrf_join(struct mlxsw_sp *mlxsw_sp, 6476 struct net_device *l3_dev, 6477 struct netlink_ext_ack *extack) 6478 { 6479 struct mlxsw_sp_rif *rif; 6480 6481 /* If netdev is already associated with a RIF, then we need to 6482 * destroy it and create a new one with the new virtual router ID. 6483 */ 6484 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev); 6485 if (rif) 6486 __mlxsw_sp_inetaddr_event(l3_dev, NETDEV_DOWN, extack); 6487 6488 return __mlxsw_sp_inetaddr_event(l3_dev, NETDEV_UP, extack); 6489 } 6490 6491 static void mlxsw_sp_port_vrf_leave(struct mlxsw_sp *mlxsw_sp, 6492 struct net_device *l3_dev) 6493 { 6494 struct mlxsw_sp_rif *rif; 6495 6496 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev); 6497 if (!rif) 6498 return; 6499 __mlxsw_sp_inetaddr_event(l3_dev, NETDEV_DOWN, NULL); 6500 } 6501 6502 int mlxsw_sp_netdevice_vrf_event(struct net_device *l3_dev, unsigned long event, 6503 struct netdev_notifier_changeupper_info *info) 6504 { 6505 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_lower_get(l3_dev); 6506 int err = 0; 6507 6508 if (!mlxsw_sp) 6509 return 0; 6510 6511 switch (event) { 6512 case NETDEV_PRECHANGEUPPER: 6513 return 0; 6514 case NETDEV_CHANGEUPPER: 6515 if (info->linking) { 6516 struct netlink_ext_ack *extack; 6517 6518 extack = netdev_notifier_info_to_extack(&info->info); 6519 err = mlxsw_sp_port_vrf_join(mlxsw_sp, l3_dev, extack); 6520 } else { 6521 mlxsw_sp_port_vrf_leave(mlxsw_sp, l3_dev); 6522 } 6523 break; 6524 } 6525 6526 return err; 6527 } 6528 6529 static struct mlxsw_sp_rif_subport * 6530 mlxsw_sp_rif_subport_rif(const struct mlxsw_sp_rif *rif) 6531 { 6532 return container_of(rif, struct mlxsw_sp_rif_subport, common); 6533 } 6534 6535 static void mlxsw_sp_rif_subport_setup(struct mlxsw_sp_rif *rif, 6536 const struct mlxsw_sp_rif_params *params) 6537 { 6538 struct mlxsw_sp_rif_subport *rif_subport; 6539 6540 rif_subport = mlxsw_sp_rif_subport_rif(rif); 6541 rif_subport->vid = params->vid; 6542 rif_subport->lag = params->lag; 6543 if (params->lag) 6544 rif_subport->lag_id = params->lag_id; 6545 else 6546 rif_subport->system_port = params->system_port; 6547 } 6548 6549 static int mlxsw_sp_rif_subport_op(struct mlxsw_sp_rif *rif, bool enable) 6550 { 6551 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 6552 struct mlxsw_sp_rif_subport *rif_subport; 6553 char ritr_pl[MLXSW_REG_RITR_LEN]; 6554 6555 rif_subport = mlxsw_sp_rif_subport_rif(rif); 6556 mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_SP_IF, 6557 rif->rif_index, rif->vr_id, rif->dev->mtu); 6558 mlxsw_reg_ritr_mac_pack(ritr_pl, rif->dev->dev_addr); 6559 mlxsw_reg_ritr_sp_if_pack(ritr_pl, rif_subport->lag, 6560 rif_subport->lag ? rif_subport->lag_id : 6561 rif_subport->system_port, 6562 rif_subport->vid); 6563 6564 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 6565 } 6566 6567 static int mlxsw_sp_rif_subport_configure(struct mlxsw_sp_rif *rif) 6568 { 6569 int err; 6570 6571 err = mlxsw_sp_rif_subport_op(rif, true); 6572 if (err) 6573 return err; 6574 6575 err = mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr, 6576 mlxsw_sp_fid_index(rif->fid), true); 6577 if (err) 6578 goto err_rif_fdb_op; 6579 6580 mlxsw_sp_fid_rif_set(rif->fid, rif); 6581 return 0; 6582 6583 err_rif_fdb_op: 6584 mlxsw_sp_rif_subport_op(rif, false); 6585 return err; 6586 } 6587 6588 static void mlxsw_sp_rif_subport_deconfigure(struct mlxsw_sp_rif *rif) 6589 { 6590 struct mlxsw_sp_fid *fid = rif->fid; 6591 6592 mlxsw_sp_fid_rif_set(fid, NULL); 6593 mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr, 6594 mlxsw_sp_fid_index(fid), false); 6595 mlxsw_sp_rif_subport_op(rif, false); 6596 } 6597 6598 static struct mlxsw_sp_fid * 6599 mlxsw_sp_rif_subport_fid_get(struct mlxsw_sp_rif *rif) 6600 { 6601 return mlxsw_sp_fid_rfid_get(rif->mlxsw_sp, rif->rif_index); 6602 } 6603 6604 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_subport_ops = { 6605 .type = MLXSW_SP_RIF_TYPE_SUBPORT, 6606 .rif_size = sizeof(struct mlxsw_sp_rif_subport), 6607 .setup = mlxsw_sp_rif_subport_setup, 6608 .configure = mlxsw_sp_rif_subport_configure, 6609 .deconfigure = mlxsw_sp_rif_subport_deconfigure, 6610 .fid_get = mlxsw_sp_rif_subport_fid_get, 6611 }; 6612 6613 static int mlxsw_sp_rif_vlan_fid_op(struct mlxsw_sp_rif *rif, 6614 enum mlxsw_reg_ritr_if_type type, 6615 u16 vid_fid, bool enable) 6616 { 6617 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 6618 char ritr_pl[MLXSW_REG_RITR_LEN]; 6619 6620 mlxsw_reg_ritr_pack(ritr_pl, enable, type, rif->rif_index, rif->vr_id, 6621 rif->dev->mtu); 6622 mlxsw_reg_ritr_mac_pack(ritr_pl, rif->dev->dev_addr); 6623 mlxsw_reg_ritr_fid_set(ritr_pl, type, vid_fid); 6624 6625 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 6626 } 6627 6628 u8 mlxsw_sp_router_port(const struct mlxsw_sp *mlxsw_sp) 6629 { 6630 return mlxsw_core_max_ports(mlxsw_sp->core) + 1; 6631 } 6632 6633 static int mlxsw_sp_rif_vlan_configure(struct mlxsw_sp_rif *rif) 6634 { 6635 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 6636 u16 vid = mlxsw_sp_fid_8021q_vid(rif->fid); 6637 int err; 6638 6639 err = mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_VLAN_IF, vid, true); 6640 if (err) 6641 return err; 6642 6643 err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC, 6644 mlxsw_sp_router_port(mlxsw_sp), true); 6645 if (err) 6646 goto err_fid_mc_flood_set; 6647 6648 err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC, 6649 mlxsw_sp_router_port(mlxsw_sp), true); 6650 if (err) 6651 goto err_fid_bc_flood_set; 6652 6653 err = mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr, 6654 mlxsw_sp_fid_index(rif->fid), true); 6655 if (err) 6656 goto err_rif_fdb_op; 6657 6658 mlxsw_sp_fid_rif_set(rif->fid, rif); 6659 return 0; 6660 6661 err_rif_fdb_op: 6662 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC, 6663 mlxsw_sp_router_port(mlxsw_sp), false); 6664 err_fid_bc_flood_set: 6665 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC, 6666 mlxsw_sp_router_port(mlxsw_sp), false); 6667 err_fid_mc_flood_set: 6668 mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_VLAN_IF, vid, false); 6669 return err; 6670 } 6671 6672 static void mlxsw_sp_rif_vlan_deconfigure(struct mlxsw_sp_rif *rif) 6673 { 6674 u16 vid = mlxsw_sp_fid_8021q_vid(rif->fid); 6675 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 6676 struct mlxsw_sp_fid *fid = rif->fid; 6677 6678 mlxsw_sp_fid_rif_set(fid, NULL); 6679 mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr, 6680 mlxsw_sp_fid_index(fid), false); 6681 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC, 6682 mlxsw_sp_router_port(mlxsw_sp), false); 6683 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC, 6684 mlxsw_sp_router_port(mlxsw_sp), false); 6685 mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_VLAN_IF, vid, false); 6686 } 6687 6688 static struct mlxsw_sp_fid * 6689 mlxsw_sp_rif_vlan_fid_get(struct mlxsw_sp_rif *rif) 6690 { 6691 u16 vid = is_vlan_dev(rif->dev) ? vlan_dev_vlan_id(rif->dev) : 1; 6692 6693 return mlxsw_sp_fid_8021q_get(rif->mlxsw_sp, vid); 6694 } 6695 6696 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_vlan_ops = { 6697 .type = MLXSW_SP_RIF_TYPE_VLAN, 6698 .rif_size = sizeof(struct mlxsw_sp_rif), 6699 .configure = mlxsw_sp_rif_vlan_configure, 6700 .deconfigure = mlxsw_sp_rif_vlan_deconfigure, 6701 .fid_get = mlxsw_sp_rif_vlan_fid_get, 6702 }; 6703 6704 static int mlxsw_sp_rif_fid_configure(struct mlxsw_sp_rif *rif) 6705 { 6706 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 6707 u16 fid_index = mlxsw_sp_fid_index(rif->fid); 6708 int err; 6709 6710 err = mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index, 6711 true); 6712 if (err) 6713 return err; 6714 6715 err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC, 6716 mlxsw_sp_router_port(mlxsw_sp), true); 6717 if (err) 6718 goto err_fid_mc_flood_set; 6719 6720 err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC, 6721 mlxsw_sp_router_port(mlxsw_sp), true); 6722 if (err) 6723 goto err_fid_bc_flood_set; 6724 6725 err = mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr, 6726 mlxsw_sp_fid_index(rif->fid), true); 6727 if (err) 6728 goto err_rif_fdb_op; 6729 6730 mlxsw_sp_fid_rif_set(rif->fid, rif); 6731 return 0; 6732 6733 err_rif_fdb_op: 6734 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC, 6735 mlxsw_sp_router_port(mlxsw_sp), false); 6736 err_fid_bc_flood_set: 6737 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC, 6738 mlxsw_sp_router_port(mlxsw_sp), false); 6739 err_fid_mc_flood_set: 6740 mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index, false); 6741 return err; 6742 } 6743 6744 static void mlxsw_sp_rif_fid_deconfigure(struct mlxsw_sp_rif *rif) 6745 { 6746 u16 fid_index = mlxsw_sp_fid_index(rif->fid); 6747 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 6748 struct mlxsw_sp_fid *fid = rif->fid; 6749 6750 mlxsw_sp_fid_rif_set(fid, NULL); 6751 mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr, 6752 mlxsw_sp_fid_index(fid), false); 6753 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC, 6754 mlxsw_sp_router_port(mlxsw_sp), false); 6755 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC, 6756 mlxsw_sp_router_port(mlxsw_sp), false); 6757 mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index, false); 6758 } 6759 6760 static struct mlxsw_sp_fid * 6761 mlxsw_sp_rif_fid_fid_get(struct mlxsw_sp_rif *rif) 6762 { 6763 return mlxsw_sp_fid_8021d_get(rif->mlxsw_sp, rif->dev->ifindex); 6764 } 6765 6766 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_fid_ops = { 6767 .type = MLXSW_SP_RIF_TYPE_FID, 6768 .rif_size = sizeof(struct mlxsw_sp_rif), 6769 .configure = mlxsw_sp_rif_fid_configure, 6770 .deconfigure = mlxsw_sp_rif_fid_deconfigure, 6771 .fid_get = mlxsw_sp_rif_fid_fid_get, 6772 }; 6773 6774 static struct mlxsw_sp_rif_ipip_lb * 6775 mlxsw_sp_rif_ipip_lb_rif(struct mlxsw_sp_rif *rif) 6776 { 6777 return container_of(rif, struct mlxsw_sp_rif_ipip_lb, common); 6778 } 6779 6780 static void 6781 mlxsw_sp_rif_ipip_lb_setup(struct mlxsw_sp_rif *rif, 6782 const struct mlxsw_sp_rif_params *params) 6783 { 6784 struct mlxsw_sp_rif_params_ipip_lb *params_lb; 6785 struct mlxsw_sp_rif_ipip_lb *rif_lb; 6786 6787 params_lb = container_of(params, struct mlxsw_sp_rif_params_ipip_lb, 6788 common); 6789 rif_lb = mlxsw_sp_rif_ipip_lb_rif(rif); 6790 rif_lb->lb_config = params_lb->lb_config; 6791 } 6792 6793 static int 6794 mlxsw_sp_rif_ipip_lb_op(struct mlxsw_sp_rif_ipip_lb *lb_rif, 6795 struct mlxsw_sp_vr *ul_vr, bool enable) 6796 { 6797 struct mlxsw_sp_rif_ipip_lb_config lb_cf = lb_rif->lb_config; 6798 struct mlxsw_sp_rif *rif = &lb_rif->common; 6799 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 6800 char ritr_pl[MLXSW_REG_RITR_LEN]; 6801 u32 saddr4; 6802 6803 switch (lb_cf.ul_protocol) { 6804 case MLXSW_SP_L3_PROTO_IPV4: 6805 saddr4 = be32_to_cpu(lb_cf.saddr.addr4); 6806 mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_LOOPBACK_IF, 6807 rif->rif_index, rif->vr_id, rif->dev->mtu); 6808 mlxsw_reg_ritr_loopback_ipip4_pack(ritr_pl, lb_cf.lb_ipipt, 6809 MLXSW_REG_RITR_LOOPBACK_IPIP_OPTIONS_GRE_KEY_PRESET, 6810 ul_vr->id, saddr4, lb_cf.okey); 6811 break; 6812 6813 case MLXSW_SP_L3_PROTO_IPV6: 6814 return -EAFNOSUPPORT; 6815 } 6816 6817 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 6818 } 6819 6820 static int 6821 mlxsw_sp_rif_ipip_lb_configure(struct mlxsw_sp_rif *rif) 6822 { 6823 struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif); 6824 u32 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(rif->dev); 6825 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 6826 struct mlxsw_sp_vr *ul_vr; 6827 int err; 6828 6829 ul_vr = mlxsw_sp_vr_get(mlxsw_sp, ul_tb_id, NULL); 6830 if (IS_ERR(ul_vr)) 6831 return PTR_ERR(ul_vr); 6832 6833 err = mlxsw_sp_rif_ipip_lb_op(lb_rif, ul_vr, true); 6834 if (err) 6835 goto err_loopback_op; 6836 6837 lb_rif->ul_vr_id = ul_vr->id; 6838 ++ul_vr->rif_count; 6839 return 0; 6840 6841 err_loopback_op: 6842 mlxsw_sp_vr_put(ul_vr); 6843 return err; 6844 } 6845 6846 static void mlxsw_sp_rif_ipip_lb_deconfigure(struct mlxsw_sp_rif *rif) 6847 { 6848 struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif); 6849 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 6850 struct mlxsw_sp_vr *ul_vr; 6851 6852 ul_vr = &mlxsw_sp->router->vrs[lb_rif->ul_vr_id]; 6853 mlxsw_sp_rif_ipip_lb_op(lb_rif, ul_vr, false); 6854 6855 --ul_vr->rif_count; 6856 mlxsw_sp_vr_put(ul_vr); 6857 } 6858 6859 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_ipip_lb_ops = { 6860 .type = MLXSW_SP_RIF_TYPE_IPIP_LB, 6861 .rif_size = sizeof(struct mlxsw_sp_rif_ipip_lb), 6862 .setup = mlxsw_sp_rif_ipip_lb_setup, 6863 .configure = mlxsw_sp_rif_ipip_lb_configure, 6864 .deconfigure = mlxsw_sp_rif_ipip_lb_deconfigure, 6865 }; 6866 6867 static const struct mlxsw_sp_rif_ops *mlxsw_sp_rif_ops_arr[] = { 6868 [MLXSW_SP_RIF_TYPE_SUBPORT] = &mlxsw_sp_rif_subport_ops, 6869 [MLXSW_SP_RIF_TYPE_VLAN] = &mlxsw_sp_rif_vlan_ops, 6870 [MLXSW_SP_RIF_TYPE_FID] = &mlxsw_sp_rif_fid_ops, 6871 [MLXSW_SP_RIF_TYPE_IPIP_LB] = &mlxsw_sp_rif_ipip_lb_ops, 6872 }; 6873 6874 static int mlxsw_sp_rifs_init(struct mlxsw_sp *mlxsw_sp) 6875 { 6876 u64 max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); 6877 6878 mlxsw_sp->router->rifs = kcalloc(max_rifs, 6879 sizeof(struct mlxsw_sp_rif *), 6880 GFP_KERNEL); 6881 if (!mlxsw_sp->router->rifs) 6882 return -ENOMEM; 6883 6884 mlxsw_sp->router->rif_ops_arr = mlxsw_sp_rif_ops_arr; 6885 6886 return 0; 6887 } 6888 6889 static void mlxsw_sp_rifs_fini(struct mlxsw_sp *mlxsw_sp) 6890 { 6891 int i; 6892 6893 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++) 6894 WARN_ON_ONCE(mlxsw_sp->router->rifs[i]); 6895 6896 kfree(mlxsw_sp->router->rifs); 6897 } 6898 6899 static int 6900 mlxsw_sp_ipip_config_tigcr(struct mlxsw_sp *mlxsw_sp) 6901 { 6902 char tigcr_pl[MLXSW_REG_TIGCR_LEN]; 6903 6904 mlxsw_reg_tigcr_pack(tigcr_pl, true, 0); 6905 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(tigcr), tigcr_pl); 6906 } 6907 6908 static int mlxsw_sp_ipips_init(struct mlxsw_sp *mlxsw_sp) 6909 { 6910 mlxsw_sp->router->ipip_ops_arr = mlxsw_sp_ipip_ops_arr; 6911 INIT_LIST_HEAD(&mlxsw_sp->router->ipip_list); 6912 return mlxsw_sp_ipip_config_tigcr(mlxsw_sp); 6913 } 6914 6915 static void mlxsw_sp_ipips_fini(struct mlxsw_sp *mlxsw_sp) 6916 { 6917 WARN_ON(!list_empty(&mlxsw_sp->router->ipip_list)); 6918 } 6919 6920 static void mlxsw_sp_router_fib_dump_flush(struct notifier_block *nb) 6921 { 6922 struct mlxsw_sp_router *router; 6923 6924 /* Flush pending FIB notifications and then flush the device's 6925 * table before requesting another dump. The FIB notification 6926 * block is unregistered, so no need to take RTNL. 6927 */ 6928 mlxsw_core_flush_owq(); 6929 router = container_of(nb, struct mlxsw_sp_router, fib_nb); 6930 mlxsw_sp_router_fib_flush(router->mlxsw_sp); 6931 } 6932 6933 #ifdef CONFIG_IP_ROUTE_MULTIPATH 6934 static void mlxsw_sp_mp_hash_header_set(char *recr2_pl, int header) 6935 { 6936 mlxsw_reg_recr2_outer_header_enables_set(recr2_pl, header, true); 6937 } 6938 6939 static void mlxsw_sp_mp_hash_field_set(char *recr2_pl, int field) 6940 { 6941 mlxsw_reg_recr2_outer_header_fields_enable_set(recr2_pl, field, true); 6942 } 6943 6944 static void mlxsw_sp_mp4_hash_init(char *recr2_pl) 6945 { 6946 bool only_l3 = !init_net.ipv4.sysctl_fib_multipath_hash_policy; 6947 6948 mlxsw_sp_mp_hash_header_set(recr2_pl, 6949 MLXSW_REG_RECR2_IPV4_EN_NOT_TCP_NOT_UDP); 6950 mlxsw_sp_mp_hash_header_set(recr2_pl, MLXSW_REG_RECR2_IPV4_EN_TCP_UDP); 6951 mlxsw_reg_recr2_ipv4_sip_enable(recr2_pl); 6952 mlxsw_reg_recr2_ipv4_dip_enable(recr2_pl); 6953 if (only_l3) 6954 return; 6955 mlxsw_sp_mp_hash_header_set(recr2_pl, MLXSW_REG_RECR2_TCP_UDP_EN_IPV4); 6956 mlxsw_sp_mp_hash_field_set(recr2_pl, MLXSW_REG_RECR2_IPV4_PROTOCOL); 6957 mlxsw_sp_mp_hash_field_set(recr2_pl, MLXSW_REG_RECR2_TCP_UDP_SPORT); 6958 mlxsw_sp_mp_hash_field_set(recr2_pl, MLXSW_REG_RECR2_TCP_UDP_DPORT); 6959 } 6960 6961 static void mlxsw_sp_mp6_hash_init(char *recr2_pl) 6962 { 6963 mlxsw_sp_mp_hash_header_set(recr2_pl, 6964 MLXSW_REG_RECR2_IPV6_EN_NOT_TCP_NOT_UDP); 6965 mlxsw_sp_mp_hash_header_set(recr2_pl, MLXSW_REG_RECR2_IPV6_EN_TCP_UDP); 6966 mlxsw_reg_recr2_ipv6_sip_enable(recr2_pl); 6967 mlxsw_reg_recr2_ipv6_dip_enable(recr2_pl); 6968 mlxsw_sp_mp_hash_field_set(recr2_pl, MLXSW_REG_RECR2_IPV6_FLOW_LABEL); 6969 mlxsw_sp_mp_hash_field_set(recr2_pl, MLXSW_REG_RECR2_IPV6_NEXT_HEADER); 6970 } 6971 6972 static int mlxsw_sp_mp_hash_init(struct mlxsw_sp *mlxsw_sp) 6973 { 6974 char recr2_pl[MLXSW_REG_RECR2_LEN]; 6975 u32 seed; 6976 6977 get_random_bytes(&seed, sizeof(seed)); 6978 mlxsw_reg_recr2_pack(recr2_pl, seed); 6979 mlxsw_sp_mp4_hash_init(recr2_pl); 6980 mlxsw_sp_mp6_hash_init(recr2_pl); 6981 6982 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(recr2), recr2_pl); 6983 } 6984 #else 6985 static int mlxsw_sp_mp_hash_init(struct mlxsw_sp *mlxsw_sp) 6986 { 6987 return 0; 6988 } 6989 #endif 6990 6991 static int __mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp) 6992 { 6993 char rgcr_pl[MLXSW_REG_RGCR_LEN]; 6994 u64 max_rifs; 6995 int err; 6996 6997 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_RIFS)) 6998 return -EIO; 6999 max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); 7000 7001 mlxsw_reg_rgcr_pack(rgcr_pl, true, true); 7002 mlxsw_reg_rgcr_max_router_interfaces_set(rgcr_pl, max_rifs); 7003 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl); 7004 if (err) 7005 return err; 7006 return 0; 7007 } 7008 7009 static void __mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp) 7010 { 7011 char rgcr_pl[MLXSW_REG_RGCR_LEN]; 7012 7013 mlxsw_reg_rgcr_pack(rgcr_pl, false, false); 7014 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl); 7015 } 7016 7017 int mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp) 7018 { 7019 struct mlxsw_sp_router *router; 7020 int err; 7021 7022 router = kzalloc(sizeof(*mlxsw_sp->router), GFP_KERNEL); 7023 if (!router) 7024 return -ENOMEM; 7025 mlxsw_sp->router = router; 7026 router->mlxsw_sp = mlxsw_sp; 7027 7028 INIT_LIST_HEAD(&mlxsw_sp->router->nexthop_neighs_list); 7029 err = __mlxsw_sp_router_init(mlxsw_sp); 7030 if (err) 7031 goto err_router_init; 7032 7033 err = mlxsw_sp_rifs_init(mlxsw_sp); 7034 if (err) 7035 goto err_rifs_init; 7036 7037 err = mlxsw_sp_ipips_init(mlxsw_sp); 7038 if (err) 7039 goto err_ipips_init; 7040 7041 err = rhashtable_init(&mlxsw_sp->router->nexthop_ht, 7042 &mlxsw_sp_nexthop_ht_params); 7043 if (err) 7044 goto err_nexthop_ht_init; 7045 7046 err = rhashtable_init(&mlxsw_sp->router->nexthop_group_ht, 7047 &mlxsw_sp_nexthop_group_ht_params); 7048 if (err) 7049 goto err_nexthop_group_ht_init; 7050 7051 INIT_LIST_HEAD(&mlxsw_sp->router->nexthop_list); 7052 err = mlxsw_sp_lpm_init(mlxsw_sp); 7053 if (err) 7054 goto err_lpm_init; 7055 7056 err = mlxsw_sp_mr_init(mlxsw_sp, &mlxsw_sp_mr_tcam_ops); 7057 if (err) 7058 goto err_mr_init; 7059 7060 err = mlxsw_sp_vrs_init(mlxsw_sp); 7061 if (err) 7062 goto err_vrs_init; 7063 7064 err = mlxsw_sp_neigh_init(mlxsw_sp); 7065 if (err) 7066 goto err_neigh_init; 7067 7068 mlxsw_sp->router->netevent_nb.notifier_call = 7069 mlxsw_sp_router_netevent_event; 7070 err = register_netevent_notifier(&mlxsw_sp->router->netevent_nb); 7071 if (err) 7072 goto err_register_netevent_notifier; 7073 7074 err = mlxsw_sp_mp_hash_init(mlxsw_sp); 7075 if (err) 7076 goto err_mp_hash_init; 7077 7078 mlxsw_sp->router->fib_nb.notifier_call = mlxsw_sp_router_fib_event; 7079 err = register_fib_notifier(&mlxsw_sp->router->fib_nb, 7080 mlxsw_sp_router_fib_dump_flush); 7081 if (err) 7082 goto err_register_fib_notifier; 7083 7084 return 0; 7085 7086 err_register_fib_notifier: 7087 err_mp_hash_init: 7088 unregister_netevent_notifier(&mlxsw_sp->router->netevent_nb); 7089 err_register_netevent_notifier: 7090 mlxsw_sp_neigh_fini(mlxsw_sp); 7091 err_neigh_init: 7092 mlxsw_sp_vrs_fini(mlxsw_sp); 7093 err_vrs_init: 7094 mlxsw_sp_mr_fini(mlxsw_sp); 7095 err_mr_init: 7096 mlxsw_sp_lpm_fini(mlxsw_sp); 7097 err_lpm_init: 7098 rhashtable_destroy(&mlxsw_sp->router->nexthop_group_ht); 7099 err_nexthop_group_ht_init: 7100 rhashtable_destroy(&mlxsw_sp->router->nexthop_ht); 7101 err_nexthop_ht_init: 7102 mlxsw_sp_ipips_fini(mlxsw_sp); 7103 err_ipips_init: 7104 mlxsw_sp_rifs_fini(mlxsw_sp); 7105 err_rifs_init: 7106 __mlxsw_sp_router_fini(mlxsw_sp); 7107 err_router_init: 7108 kfree(mlxsw_sp->router); 7109 return err; 7110 } 7111 7112 void mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp) 7113 { 7114 unregister_fib_notifier(&mlxsw_sp->router->fib_nb); 7115 unregister_netevent_notifier(&mlxsw_sp->router->netevent_nb); 7116 mlxsw_sp_neigh_fini(mlxsw_sp); 7117 mlxsw_sp_vrs_fini(mlxsw_sp); 7118 mlxsw_sp_mr_fini(mlxsw_sp); 7119 mlxsw_sp_lpm_fini(mlxsw_sp); 7120 rhashtable_destroy(&mlxsw_sp->router->nexthop_group_ht); 7121 rhashtable_destroy(&mlxsw_sp->router->nexthop_ht); 7122 mlxsw_sp_ipips_fini(mlxsw_sp); 7123 mlxsw_sp_rifs_fini(mlxsw_sp); 7124 __mlxsw_sp_router_fini(mlxsw_sp); 7125 kfree(mlxsw_sp->router); 7126 } 7127