1 /* 2 * drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c 3 * Copyright (c) 2016-2017 Mellanox Technologies. All rights reserved. 4 * Copyright (c) 2016 Jiri Pirko <jiri@mellanox.com> 5 * Copyright (c) 2016 Ido Schimmel <idosch@mellanox.com> 6 * Copyright (c) 2016 Yotam Gigi <yotamg@mellanox.com> 7 * Copyright (c) 2017 Petr Machata <petrm@mellanox.com> 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions are met: 11 * 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 3. Neither the names of the copyright holders nor the names of its 18 * contributors may be used to endorse or promote products derived from 19 * this software without specific prior written permission. 20 * 21 * Alternatively, this software may be distributed under the terms of the 22 * GNU General Public License ("GPL") version 2 as published by the Free 23 * Software Foundation. 24 * 25 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 26 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 27 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 28 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 29 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 30 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 31 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 32 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 33 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 34 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 35 * POSSIBILITY OF SUCH DAMAGE. 36 */ 37 38 #include <linux/kernel.h> 39 #include <linux/types.h> 40 #include <linux/rhashtable.h> 41 #include <linux/bitops.h> 42 #include <linux/in6.h> 43 #include <linux/notifier.h> 44 #include <linux/inetdevice.h> 45 #include <linux/netdevice.h> 46 #include <linux/if_bridge.h> 47 #include <linux/socket.h> 48 #include <linux/route.h> 49 #include <linux/gcd.h> 50 #include <linux/random.h> 51 #include <net/netevent.h> 52 #include <net/neighbour.h> 53 #include <net/arp.h> 54 #include <net/ip_fib.h> 55 #include <net/ip6_fib.h> 56 #include <net/fib_rules.h> 57 #include <net/ip_tunnels.h> 58 #include <net/l3mdev.h> 59 #include <net/addrconf.h> 60 #include <net/ndisc.h> 61 #include <net/ipv6.h> 62 #include <net/fib_notifier.h> 63 64 #include "spectrum.h" 65 #include "core.h" 66 #include "reg.h" 67 #include "spectrum_cnt.h" 68 #include "spectrum_dpipe.h" 69 #include "spectrum_ipip.h" 70 #include "spectrum_mr.h" 71 #include "spectrum_mr_tcam.h" 72 #include "spectrum_router.h" 73 74 struct mlxsw_sp_vr; 75 struct mlxsw_sp_lpm_tree; 76 struct mlxsw_sp_rif_ops; 77 78 struct mlxsw_sp_router { 79 struct mlxsw_sp *mlxsw_sp; 80 struct mlxsw_sp_rif **rifs; 81 struct mlxsw_sp_vr *vrs; 82 struct rhashtable neigh_ht; 83 struct rhashtable nexthop_group_ht; 84 struct rhashtable nexthop_ht; 85 struct list_head nexthop_list; 86 struct { 87 struct mlxsw_sp_lpm_tree *trees; 88 unsigned int tree_count; 89 } lpm; 90 struct { 91 struct delayed_work dw; 92 unsigned long interval; /* ms */ 93 } neighs_update; 94 struct delayed_work nexthop_probe_dw; 95 #define MLXSW_SP_UNRESOLVED_NH_PROBE_INTERVAL 5000 /* ms */ 96 struct list_head nexthop_neighs_list; 97 struct list_head ipip_list; 98 bool aborted; 99 struct notifier_block fib_nb; 100 struct notifier_block netevent_nb; 101 const struct mlxsw_sp_rif_ops **rif_ops_arr; 102 const struct mlxsw_sp_ipip_ops **ipip_ops_arr; 103 }; 104 105 struct mlxsw_sp_rif { 106 struct list_head nexthop_list; 107 struct list_head neigh_list; 108 struct net_device *dev; 109 struct mlxsw_sp_fid *fid; 110 unsigned char addr[ETH_ALEN]; 111 int mtu; 112 u16 rif_index; 113 u16 vr_id; 114 const struct mlxsw_sp_rif_ops *ops; 115 struct mlxsw_sp *mlxsw_sp; 116 117 unsigned int counter_ingress; 118 bool counter_ingress_valid; 119 unsigned int counter_egress; 120 bool counter_egress_valid; 121 }; 122 123 struct mlxsw_sp_rif_params { 124 struct net_device *dev; 125 union { 126 u16 system_port; 127 u16 lag_id; 128 }; 129 u16 vid; 130 bool lag; 131 }; 132 133 struct mlxsw_sp_rif_subport { 134 struct mlxsw_sp_rif common; 135 union { 136 u16 system_port; 137 u16 lag_id; 138 }; 139 u16 vid; 140 bool lag; 141 }; 142 143 struct mlxsw_sp_rif_ipip_lb { 144 struct mlxsw_sp_rif common; 145 struct mlxsw_sp_rif_ipip_lb_config lb_config; 146 u16 ul_vr_id; /* Reserved for Spectrum-2. */ 147 }; 148 149 struct mlxsw_sp_rif_params_ipip_lb { 150 struct mlxsw_sp_rif_params common; 151 struct mlxsw_sp_rif_ipip_lb_config lb_config; 152 }; 153 154 struct mlxsw_sp_rif_ops { 155 enum mlxsw_sp_rif_type type; 156 size_t rif_size; 157 158 void (*setup)(struct mlxsw_sp_rif *rif, 159 const struct mlxsw_sp_rif_params *params); 160 int (*configure)(struct mlxsw_sp_rif *rif); 161 void (*deconfigure)(struct mlxsw_sp_rif *rif); 162 struct mlxsw_sp_fid * (*fid_get)(struct mlxsw_sp_rif *rif); 163 }; 164 165 static unsigned int * 166 mlxsw_sp_rif_p_counter_get(struct mlxsw_sp_rif *rif, 167 enum mlxsw_sp_rif_counter_dir dir) 168 { 169 switch (dir) { 170 case MLXSW_SP_RIF_COUNTER_EGRESS: 171 return &rif->counter_egress; 172 case MLXSW_SP_RIF_COUNTER_INGRESS: 173 return &rif->counter_ingress; 174 } 175 return NULL; 176 } 177 178 static bool 179 mlxsw_sp_rif_counter_valid_get(struct mlxsw_sp_rif *rif, 180 enum mlxsw_sp_rif_counter_dir dir) 181 { 182 switch (dir) { 183 case MLXSW_SP_RIF_COUNTER_EGRESS: 184 return rif->counter_egress_valid; 185 case MLXSW_SP_RIF_COUNTER_INGRESS: 186 return rif->counter_ingress_valid; 187 } 188 return false; 189 } 190 191 static void 192 mlxsw_sp_rif_counter_valid_set(struct mlxsw_sp_rif *rif, 193 enum mlxsw_sp_rif_counter_dir dir, 194 bool valid) 195 { 196 switch (dir) { 197 case MLXSW_SP_RIF_COUNTER_EGRESS: 198 rif->counter_egress_valid = valid; 199 break; 200 case MLXSW_SP_RIF_COUNTER_INGRESS: 201 rif->counter_ingress_valid = valid; 202 break; 203 } 204 } 205 206 static int mlxsw_sp_rif_counter_edit(struct mlxsw_sp *mlxsw_sp, u16 rif_index, 207 unsigned int counter_index, bool enable, 208 enum mlxsw_sp_rif_counter_dir dir) 209 { 210 char ritr_pl[MLXSW_REG_RITR_LEN]; 211 bool is_egress = false; 212 int err; 213 214 if (dir == MLXSW_SP_RIF_COUNTER_EGRESS) 215 is_egress = true; 216 mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index); 217 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 218 if (err) 219 return err; 220 221 mlxsw_reg_ritr_counter_pack(ritr_pl, counter_index, enable, 222 is_egress); 223 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 224 } 225 226 int mlxsw_sp_rif_counter_value_get(struct mlxsw_sp *mlxsw_sp, 227 struct mlxsw_sp_rif *rif, 228 enum mlxsw_sp_rif_counter_dir dir, u64 *cnt) 229 { 230 char ricnt_pl[MLXSW_REG_RICNT_LEN]; 231 unsigned int *p_counter_index; 232 bool valid; 233 int err; 234 235 valid = mlxsw_sp_rif_counter_valid_get(rif, dir); 236 if (!valid) 237 return -EINVAL; 238 239 p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir); 240 if (!p_counter_index) 241 return -EINVAL; 242 mlxsw_reg_ricnt_pack(ricnt_pl, *p_counter_index, 243 MLXSW_REG_RICNT_OPCODE_NOP); 244 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl); 245 if (err) 246 return err; 247 *cnt = mlxsw_reg_ricnt_good_unicast_packets_get(ricnt_pl); 248 return 0; 249 } 250 251 static int mlxsw_sp_rif_counter_clear(struct mlxsw_sp *mlxsw_sp, 252 unsigned int counter_index) 253 { 254 char ricnt_pl[MLXSW_REG_RICNT_LEN]; 255 256 mlxsw_reg_ricnt_pack(ricnt_pl, counter_index, 257 MLXSW_REG_RICNT_OPCODE_CLEAR); 258 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl); 259 } 260 261 int mlxsw_sp_rif_counter_alloc(struct mlxsw_sp *mlxsw_sp, 262 struct mlxsw_sp_rif *rif, 263 enum mlxsw_sp_rif_counter_dir dir) 264 { 265 unsigned int *p_counter_index; 266 int err; 267 268 p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir); 269 if (!p_counter_index) 270 return -EINVAL; 271 err = mlxsw_sp_counter_alloc(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF, 272 p_counter_index); 273 if (err) 274 return err; 275 276 err = mlxsw_sp_rif_counter_clear(mlxsw_sp, *p_counter_index); 277 if (err) 278 goto err_counter_clear; 279 280 err = mlxsw_sp_rif_counter_edit(mlxsw_sp, rif->rif_index, 281 *p_counter_index, true, dir); 282 if (err) 283 goto err_counter_edit; 284 mlxsw_sp_rif_counter_valid_set(rif, dir, true); 285 return 0; 286 287 err_counter_edit: 288 err_counter_clear: 289 mlxsw_sp_counter_free(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF, 290 *p_counter_index); 291 return err; 292 } 293 294 void mlxsw_sp_rif_counter_free(struct mlxsw_sp *mlxsw_sp, 295 struct mlxsw_sp_rif *rif, 296 enum mlxsw_sp_rif_counter_dir dir) 297 { 298 unsigned int *p_counter_index; 299 300 if (!mlxsw_sp_rif_counter_valid_get(rif, dir)) 301 return; 302 303 p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir); 304 if (WARN_ON(!p_counter_index)) 305 return; 306 mlxsw_sp_rif_counter_edit(mlxsw_sp, rif->rif_index, 307 *p_counter_index, false, dir); 308 mlxsw_sp_counter_free(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF, 309 *p_counter_index); 310 mlxsw_sp_rif_counter_valid_set(rif, dir, false); 311 } 312 313 static void mlxsw_sp_rif_counters_alloc(struct mlxsw_sp_rif *rif) 314 { 315 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 316 struct devlink *devlink; 317 318 devlink = priv_to_devlink(mlxsw_sp->core); 319 if (!devlink_dpipe_table_counter_enabled(devlink, 320 MLXSW_SP_DPIPE_TABLE_NAME_ERIF)) 321 return; 322 mlxsw_sp_rif_counter_alloc(mlxsw_sp, rif, MLXSW_SP_RIF_COUNTER_EGRESS); 323 } 324 325 static void mlxsw_sp_rif_counters_free(struct mlxsw_sp_rif *rif) 326 { 327 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 328 329 mlxsw_sp_rif_counter_free(mlxsw_sp, rif, MLXSW_SP_RIF_COUNTER_EGRESS); 330 } 331 332 static struct mlxsw_sp_rif * 333 mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp, 334 const struct net_device *dev); 335 336 #define MLXSW_SP_PREFIX_COUNT (sizeof(struct in6_addr) * BITS_PER_BYTE + 1) 337 338 struct mlxsw_sp_prefix_usage { 339 DECLARE_BITMAP(b, MLXSW_SP_PREFIX_COUNT); 340 }; 341 342 #define mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) \ 343 for_each_set_bit(prefix, (prefix_usage)->b, MLXSW_SP_PREFIX_COUNT) 344 345 static bool 346 mlxsw_sp_prefix_usage_eq(struct mlxsw_sp_prefix_usage *prefix_usage1, 347 struct mlxsw_sp_prefix_usage *prefix_usage2) 348 { 349 return !memcmp(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1)); 350 } 351 352 static bool 353 mlxsw_sp_prefix_usage_none(struct mlxsw_sp_prefix_usage *prefix_usage) 354 { 355 struct mlxsw_sp_prefix_usage prefix_usage_none = {{ 0 } }; 356 357 return mlxsw_sp_prefix_usage_eq(prefix_usage, &prefix_usage_none); 358 } 359 360 static void 361 mlxsw_sp_prefix_usage_cpy(struct mlxsw_sp_prefix_usage *prefix_usage1, 362 struct mlxsw_sp_prefix_usage *prefix_usage2) 363 { 364 memcpy(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1)); 365 } 366 367 static void 368 mlxsw_sp_prefix_usage_set(struct mlxsw_sp_prefix_usage *prefix_usage, 369 unsigned char prefix_len) 370 { 371 set_bit(prefix_len, prefix_usage->b); 372 } 373 374 static void 375 mlxsw_sp_prefix_usage_clear(struct mlxsw_sp_prefix_usage *prefix_usage, 376 unsigned char prefix_len) 377 { 378 clear_bit(prefix_len, prefix_usage->b); 379 } 380 381 struct mlxsw_sp_fib_key { 382 unsigned char addr[sizeof(struct in6_addr)]; 383 unsigned char prefix_len; 384 }; 385 386 enum mlxsw_sp_fib_entry_type { 387 MLXSW_SP_FIB_ENTRY_TYPE_REMOTE, 388 MLXSW_SP_FIB_ENTRY_TYPE_LOCAL, 389 MLXSW_SP_FIB_ENTRY_TYPE_TRAP, 390 391 /* This is a special case of local delivery, where a packet should be 392 * decapsulated on reception. Note that there is no corresponding ENCAP, 393 * because that's a type of next hop, not of FIB entry. (There can be 394 * several next hops in a REMOTE entry, and some of them may be 395 * encapsulating entries.) 396 */ 397 MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP, 398 }; 399 400 struct mlxsw_sp_nexthop_group; 401 struct mlxsw_sp_fib; 402 403 struct mlxsw_sp_fib_node { 404 struct list_head entry_list; 405 struct list_head list; 406 struct rhash_head ht_node; 407 struct mlxsw_sp_fib *fib; 408 struct mlxsw_sp_fib_key key; 409 }; 410 411 struct mlxsw_sp_fib_entry_decap { 412 struct mlxsw_sp_ipip_entry *ipip_entry; 413 u32 tunnel_index; 414 }; 415 416 struct mlxsw_sp_fib_entry { 417 struct list_head list; 418 struct mlxsw_sp_fib_node *fib_node; 419 enum mlxsw_sp_fib_entry_type type; 420 struct list_head nexthop_group_node; 421 struct mlxsw_sp_nexthop_group *nh_group; 422 struct mlxsw_sp_fib_entry_decap decap; /* Valid for decap entries. */ 423 }; 424 425 struct mlxsw_sp_fib4_entry { 426 struct mlxsw_sp_fib_entry common; 427 u32 tb_id; 428 u32 prio; 429 u8 tos; 430 u8 type; 431 }; 432 433 struct mlxsw_sp_fib6_entry { 434 struct mlxsw_sp_fib_entry common; 435 struct list_head rt6_list; 436 unsigned int nrt6; 437 }; 438 439 struct mlxsw_sp_rt6 { 440 struct list_head list; 441 struct rt6_info *rt; 442 }; 443 444 struct mlxsw_sp_lpm_tree { 445 u8 id; /* tree ID */ 446 unsigned int ref_count; 447 enum mlxsw_sp_l3proto proto; 448 struct mlxsw_sp_prefix_usage prefix_usage; 449 }; 450 451 struct mlxsw_sp_fib { 452 struct rhashtable ht; 453 struct list_head node_list; 454 struct mlxsw_sp_vr *vr; 455 struct mlxsw_sp_lpm_tree *lpm_tree; 456 unsigned long prefix_ref_count[MLXSW_SP_PREFIX_COUNT]; 457 struct mlxsw_sp_prefix_usage prefix_usage; 458 enum mlxsw_sp_l3proto proto; 459 }; 460 461 struct mlxsw_sp_vr { 462 u16 id; /* virtual router ID */ 463 u32 tb_id; /* kernel fib table id */ 464 unsigned int rif_count; 465 struct mlxsw_sp_fib *fib4; 466 struct mlxsw_sp_fib *fib6; 467 struct mlxsw_sp_mr_table *mr4_table; 468 }; 469 470 static const struct rhashtable_params mlxsw_sp_fib_ht_params; 471 472 static struct mlxsw_sp_fib *mlxsw_sp_fib_create(struct mlxsw_sp_vr *vr, 473 enum mlxsw_sp_l3proto proto) 474 { 475 struct mlxsw_sp_fib *fib; 476 int err; 477 478 fib = kzalloc(sizeof(*fib), GFP_KERNEL); 479 if (!fib) 480 return ERR_PTR(-ENOMEM); 481 err = rhashtable_init(&fib->ht, &mlxsw_sp_fib_ht_params); 482 if (err) 483 goto err_rhashtable_init; 484 INIT_LIST_HEAD(&fib->node_list); 485 fib->proto = proto; 486 fib->vr = vr; 487 return fib; 488 489 err_rhashtable_init: 490 kfree(fib); 491 return ERR_PTR(err); 492 } 493 494 static void mlxsw_sp_fib_destroy(struct mlxsw_sp_fib *fib) 495 { 496 WARN_ON(!list_empty(&fib->node_list)); 497 WARN_ON(fib->lpm_tree); 498 rhashtable_destroy(&fib->ht); 499 kfree(fib); 500 } 501 502 static struct mlxsw_sp_lpm_tree * 503 mlxsw_sp_lpm_tree_find_unused(struct mlxsw_sp *mlxsw_sp) 504 { 505 static struct mlxsw_sp_lpm_tree *lpm_tree; 506 int i; 507 508 for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) { 509 lpm_tree = &mlxsw_sp->router->lpm.trees[i]; 510 if (lpm_tree->ref_count == 0) 511 return lpm_tree; 512 } 513 return NULL; 514 } 515 516 static int mlxsw_sp_lpm_tree_alloc(struct mlxsw_sp *mlxsw_sp, 517 struct mlxsw_sp_lpm_tree *lpm_tree) 518 { 519 char ralta_pl[MLXSW_REG_RALTA_LEN]; 520 521 mlxsw_reg_ralta_pack(ralta_pl, true, 522 (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto, 523 lpm_tree->id); 524 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl); 525 } 526 527 static void mlxsw_sp_lpm_tree_free(struct mlxsw_sp *mlxsw_sp, 528 struct mlxsw_sp_lpm_tree *lpm_tree) 529 { 530 char ralta_pl[MLXSW_REG_RALTA_LEN]; 531 532 mlxsw_reg_ralta_pack(ralta_pl, false, 533 (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto, 534 lpm_tree->id); 535 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl); 536 } 537 538 static int 539 mlxsw_sp_lpm_tree_left_struct_set(struct mlxsw_sp *mlxsw_sp, 540 struct mlxsw_sp_prefix_usage *prefix_usage, 541 struct mlxsw_sp_lpm_tree *lpm_tree) 542 { 543 char ralst_pl[MLXSW_REG_RALST_LEN]; 544 u8 root_bin = 0; 545 u8 prefix; 546 u8 last_prefix = MLXSW_REG_RALST_BIN_NO_CHILD; 547 548 mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) 549 root_bin = prefix; 550 551 mlxsw_reg_ralst_pack(ralst_pl, root_bin, lpm_tree->id); 552 mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) { 553 if (prefix == 0) 554 continue; 555 mlxsw_reg_ralst_bin_pack(ralst_pl, prefix, last_prefix, 556 MLXSW_REG_RALST_BIN_NO_CHILD); 557 last_prefix = prefix; 558 } 559 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralst), ralst_pl); 560 } 561 562 static struct mlxsw_sp_lpm_tree * 563 mlxsw_sp_lpm_tree_create(struct mlxsw_sp *mlxsw_sp, 564 struct mlxsw_sp_prefix_usage *prefix_usage, 565 enum mlxsw_sp_l3proto proto) 566 { 567 struct mlxsw_sp_lpm_tree *lpm_tree; 568 int err; 569 570 lpm_tree = mlxsw_sp_lpm_tree_find_unused(mlxsw_sp); 571 if (!lpm_tree) 572 return ERR_PTR(-EBUSY); 573 lpm_tree->proto = proto; 574 err = mlxsw_sp_lpm_tree_alloc(mlxsw_sp, lpm_tree); 575 if (err) 576 return ERR_PTR(err); 577 578 err = mlxsw_sp_lpm_tree_left_struct_set(mlxsw_sp, prefix_usage, 579 lpm_tree); 580 if (err) 581 goto err_left_struct_set; 582 memcpy(&lpm_tree->prefix_usage, prefix_usage, 583 sizeof(lpm_tree->prefix_usage)); 584 return lpm_tree; 585 586 err_left_struct_set: 587 mlxsw_sp_lpm_tree_free(mlxsw_sp, lpm_tree); 588 return ERR_PTR(err); 589 } 590 591 static void mlxsw_sp_lpm_tree_destroy(struct mlxsw_sp *mlxsw_sp, 592 struct mlxsw_sp_lpm_tree *lpm_tree) 593 { 594 mlxsw_sp_lpm_tree_free(mlxsw_sp, lpm_tree); 595 } 596 597 static struct mlxsw_sp_lpm_tree * 598 mlxsw_sp_lpm_tree_get(struct mlxsw_sp *mlxsw_sp, 599 struct mlxsw_sp_prefix_usage *prefix_usage, 600 enum mlxsw_sp_l3proto proto) 601 { 602 struct mlxsw_sp_lpm_tree *lpm_tree; 603 int i; 604 605 for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) { 606 lpm_tree = &mlxsw_sp->router->lpm.trees[i]; 607 if (lpm_tree->ref_count != 0 && 608 lpm_tree->proto == proto && 609 mlxsw_sp_prefix_usage_eq(&lpm_tree->prefix_usage, 610 prefix_usage)) 611 return lpm_tree; 612 } 613 return mlxsw_sp_lpm_tree_create(mlxsw_sp, prefix_usage, proto); 614 } 615 616 static void mlxsw_sp_lpm_tree_hold(struct mlxsw_sp_lpm_tree *lpm_tree) 617 { 618 lpm_tree->ref_count++; 619 } 620 621 static void mlxsw_sp_lpm_tree_put(struct mlxsw_sp *mlxsw_sp, 622 struct mlxsw_sp_lpm_tree *lpm_tree) 623 { 624 if (--lpm_tree->ref_count == 0) 625 mlxsw_sp_lpm_tree_destroy(mlxsw_sp, lpm_tree); 626 } 627 628 #define MLXSW_SP_LPM_TREE_MIN 1 /* tree 0 is reserved */ 629 630 static int mlxsw_sp_lpm_init(struct mlxsw_sp *mlxsw_sp) 631 { 632 struct mlxsw_sp_lpm_tree *lpm_tree; 633 u64 max_trees; 634 int i; 635 636 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_LPM_TREES)) 637 return -EIO; 638 639 max_trees = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_LPM_TREES); 640 mlxsw_sp->router->lpm.tree_count = max_trees - MLXSW_SP_LPM_TREE_MIN; 641 mlxsw_sp->router->lpm.trees = kcalloc(mlxsw_sp->router->lpm.tree_count, 642 sizeof(struct mlxsw_sp_lpm_tree), 643 GFP_KERNEL); 644 if (!mlxsw_sp->router->lpm.trees) 645 return -ENOMEM; 646 647 for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) { 648 lpm_tree = &mlxsw_sp->router->lpm.trees[i]; 649 lpm_tree->id = i + MLXSW_SP_LPM_TREE_MIN; 650 } 651 652 return 0; 653 } 654 655 static void mlxsw_sp_lpm_fini(struct mlxsw_sp *mlxsw_sp) 656 { 657 kfree(mlxsw_sp->router->lpm.trees); 658 } 659 660 static bool mlxsw_sp_vr_is_used(const struct mlxsw_sp_vr *vr) 661 { 662 return !!vr->fib4 || !!vr->fib6 || !!vr->mr4_table; 663 } 664 665 static struct mlxsw_sp_vr *mlxsw_sp_vr_find_unused(struct mlxsw_sp *mlxsw_sp) 666 { 667 struct mlxsw_sp_vr *vr; 668 int i; 669 670 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) { 671 vr = &mlxsw_sp->router->vrs[i]; 672 if (!mlxsw_sp_vr_is_used(vr)) 673 return vr; 674 } 675 return NULL; 676 } 677 678 static int mlxsw_sp_vr_lpm_tree_bind(struct mlxsw_sp *mlxsw_sp, 679 const struct mlxsw_sp_fib *fib, u8 tree_id) 680 { 681 char raltb_pl[MLXSW_REG_RALTB_LEN]; 682 683 mlxsw_reg_raltb_pack(raltb_pl, fib->vr->id, 684 (enum mlxsw_reg_ralxx_protocol) fib->proto, 685 tree_id); 686 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), raltb_pl); 687 } 688 689 static int mlxsw_sp_vr_lpm_tree_unbind(struct mlxsw_sp *mlxsw_sp, 690 const struct mlxsw_sp_fib *fib) 691 { 692 char raltb_pl[MLXSW_REG_RALTB_LEN]; 693 694 /* Bind to tree 0 which is default */ 695 mlxsw_reg_raltb_pack(raltb_pl, fib->vr->id, 696 (enum mlxsw_reg_ralxx_protocol) fib->proto, 0); 697 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), raltb_pl); 698 } 699 700 static u32 mlxsw_sp_fix_tb_id(u32 tb_id) 701 { 702 /* For our purpose, squash main, default and local tables into one */ 703 if (tb_id == RT_TABLE_LOCAL || tb_id == RT_TABLE_DEFAULT) 704 tb_id = RT_TABLE_MAIN; 705 return tb_id; 706 } 707 708 static struct mlxsw_sp_vr *mlxsw_sp_vr_find(struct mlxsw_sp *mlxsw_sp, 709 u32 tb_id) 710 { 711 struct mlxsw_sp_vr *vr; 712 int i; 713 714 tb_id = mlxsw_sp_fix_tb_id(tb_id); 715 716 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) { 717 vr = &mlxsw_sp->router->vrs[i]; 718 if (mlxsw_sp_vr_is_used(vr) && vr->tb_id == tb_id) 719 return vr; 720 } 721 return NULL; 722 } 723 724 static struct mlxsw_sp_fib *mlxsw_sp_vr_fib(const struct mlxsw_sp_vr *vr, 725 enum mlxsw_sp_l3proto proto) 726 { 727 switch (proto) { 728 case MLXSW_SP_L3_PROTO_IPV4: 729 return vr->fib4; 730 case MLXSW_SP_L3_PROTO_IPV6: 731 return vr->fib6; 732 } 733 return NULL; 734 } 735 736 static struct mlxsw_sp_vr *mlxsw_sp_vr_create(struct mlxsw_sp *mlxsw_sp, 737 u32 tb_id, 738 struct netlink_ext_ack *extack) 739 { 740 struct mlxsw_sp_vr *vr; 741 int err; 742 743 vr = mlxsw_sp_vr_find_unused(mlxsw_sp); 744 if (!vr) { 745 NL_SET_ERR_MSG(extack, "spectrum: Exceeded number of supported virtual routers"); 746 return ERR_PTR(-EBUSY); 747 } 748 vr->fib4 = mlxsw_sp_fib_create(vr, MLXSW_SP_L3_PROTO_IPV4); 749 if (IS_ERR(vr->fib4)) 750 return ERR_CAST(vr->fib4); 751 vr->fib6 = mlxsw_sp_fib_create(vr, MLXSW_SP_L3_PROTO_IPV6); 752 if (IS_ERR(vr->fib6)) { 753 err = PTR_ERR(vr->fib6); 754 goto err_fib6_create; 755 } 756 vr->mr4_table = mlxsw_sp_mr_table_create(mlxsw_sp, vr->id, 757 MLXSW_SP_L3_PROTO_IPV4); 758 if (IS_ERR(vr->mr4_table)) { 759 err = PTR_ERR(vr->mr4_table); 760 goto err_mr_table_create; 761 } 762 vr->tb_id = tb_id; 763 return vr; 764 765 err_mr_table_create: 766 mlxsw_sp_fib_destroy(vr->fib6); 767 vr->fib6 = NULL; 768 err_fib6_create: 769 mlxsw_sp_fib_destroy(vr->fib4); 770 vr->fib4 = NULL; 771 return ERR_PTR(err); 772 } 773 774 static void mlxsw_sp_vr_destroy(struct mlxsw_sp_vr *vr) 775 { 776 mlxsw_sp_mr_table_destroy(vr->mr4_table); 777 vr->mr4_table = NULL; 778 mlxsw_sp_fib_destroy(vr->fib6); 779 vr->fib6 = NULL; 780 mlxsw_sp_fib_destroy(vr->fib4); 781 vr->fib4 = NULL; 782 } 783 784 static struct mlxsw_sp_vr *mlxsw_sp_vr_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id, 785 struct netlink_ext_ack *extack) 786 { 787 struct mlxsw_sp_vr *vr; 788 789 tb_id = mlxsw_sp_fix_tb_id(tb_id); 790 vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id); 791 if (!vr) 792 vr = mlxsw_sp_vr_create(mlxsw_sp, tb_id, extack); 793 return vr; 794 } 795 796 static void mlxsw_sp_vr_put(struct mlxsw_sp_vr *vr) 797 { 798 if (!vr->rif_count && list_empty(&vr->fib4->node_list) && 799 list_empty(&vr->fib6->node_list) && 800 mlxsw_sp_mr_table_empty(vr->mr4_table)) 801 mlxsw_sp_vr_destroy(vr); 802 } 803 804 static bool 805 mlxsw_sp_vr_lpm_tree_should_replace(struct mlxsw_sp_vr *vr, 806 enum mlxsw_sp_l3proto proto, u8 tree_id) 807 { 808 struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto); 809 810 if (!mlxsw_sp_vr_is_used(vr)) 811 return false; 812 if (fib->lpm_tree && fib->lpm_tree->id == tree_id) 813 return true; 814 return false; 815 } 816 817 static int mlxsw_sp_vr_lpm_tree_replace(struct mlxsw_sp *mlxsw_sp, 818 struct mlxsw_sp_fib *fib, 819 struct mlxsw_sp_lpm_tree *new_tree) 820 { 821 struct mlxsw_sp_lpm_tree *old_tree = fib->lpm_tree; 822 int err; 823 824 err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib, new_tree->id); 825 if (err) 826 return err; 827 fib->lpm_tree = new_tree; 828 mlxsw_sp_lpm_tree_hold(new_tree); 829 mlxsw_sp_lpm_tree_put(mlxsw_sp, old_tree); 830 return 0; 831 } 832 833 static int mlxsw_sp_vrs_lpm_tree_replace(struct mlxsw_sp *mlxsw_sp, 834 struct mlxsw_sp_fib *fib, 835 struct mlxsw_sp_lpm_tree *new_tree) 836 { 837 struct mlxsw_sp_lpm_tree *old_tree = fib->lpm_tree; 838 enum mlxsw_sp_l3proto proto = fib->proto; 839 u8 old_id, new_id = new_tree->id; 840 struct mlxsw_sp_vr *vr; 841 int i, err; 842 843 if (!old_tree) 844 goto no_replace; 845 old_id = old_tree->id; 846 847 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) { 848 vr = &mlxsw_sp->router->vrs[i]; 849 if (!mlxsw_sp_vr_lpm_tree_should_replace(vr, proto, old_id)) 850 continue; 851 err = mlxsw_sp_vr_lpm_tree_replace(mlxsw_sp, 852 mlxsw_sp_vr_fib(vr, proto), 853 new_tree); 854 if (err) 855 goto err_tree_replace; 856 } 857 858 return 0; 859 860 err_tree_replace: 861 for (i--; i >= 0; i--) { 862 if (!mlxsw_sp_vr_lpm_tree_should_replace(vr, proto, new_id)) 863 continue; 864 mlxsw_sp_vr_lpm_tree_replace(mlxsw_sp, 865 mlxsw_sp_vr_fib(vr, proto), 866 old_tree); 867 } 868 return err; 869 870 no_replace: 871 err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib, new_tree->id); 872 if (err) 873 return err; 874 fib->lpm_tree = new_tree; 875 mlxsw_sp_lpm_tree_hold(new_tree); 876 return 0; 877 } 878 879 static void 880 mlxsw_sp_vrs_prefixes(struct mlxsw_sp *mlxsw_sp, 881 enum mlxsw_sp_l3proto proto, 882 struct mlxsw_sp_prefix_usage *req_prefix_usage) 883 { 884 int i; 885 886 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) { 887 struct mlxsw_sp_vr *vr = &mlxsw_sp->router->vrs[i]; 888 struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto); 889 unsigned char prefix; 890 891 if (!mlxsw_sp_vr_is_used(vr)) 892 continue; 893 mlxsw_sp_prefix_usage_for_each(prefix, &fib->prefix_usage) 894 mlxsw_sp_prefix_usage_set(req_prefix_usage, prefix); 895 } 896 } 897 898 static int mlxsw_sp_vrs_init(struct mlxsw_sp *mlxsw_sp) 899 { 900 struct mlxsw_sp_vr *vr; 901 u64 max_vrs; 902 int i; 903 904 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_VRS)) 905 return -EIO; 906 907 max_vrs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); 908 mlxsw_sp->router->vrs = kcalloc(max_vrs, sizeof(struct mlxsw_sp_vr), 909 GFP_KERNEL); 910 if (!mlxsw_sp->router->vrs) 911 return -ENOMEM; 912 913 for (i = 0; i < max_vrs; i++) { 914 vr = &mlxsw_sp->router->vrs[i]; 915 vr->id = i; 916 } 917 918 return 0; 919 } 920 921 static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp); 922 923 static void mlxsw_sp_vrs_fini(struct mlxsw_sp *mlxsw_sp) 924 { 925 /* At this stage we're guaranteed not to have new incoming 926 * FIB notifications and the work queue is free from FIBs 927 * sitting on top of mlxsw netdevs. However, we can still 928 * have other FIBs queued. Flush the queue before flushing 929 * the device's tables. No need for locks, as we're the only 930 * writer. 931 */ 932 mlxsw_core_flush_owq(); 933 mlxsw_sp_router_fib_flush(mlxsw_sp); 934 kfree(mlxsw_sp->router->vrs); 935 } 936 937 static struct net_device * 938 __mlxsw_sp_ipip_netdev_ul_dev_get(const struct net_device *ol_dev) 939 { 940 struct ip_tunnel *tun = netdev_priv(ol_dev); 941 struct net *net = dev_net(ol_dev); 942 943 return __dev_get_by_index(net, tun->parms.link); 944 } 945 946 u32 mlxsw_sp_ipip_dev_ul_tb_id(const struct net_device *ol_dev) 947 { 948 struct net_device *d = __mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev); 949 950 if (d) 951 return l3mdev_fib_table(d) ? : RT_TABLE_MAIN; 952 else 953 return l3mdev_fib_table(ol_dev) ? : RT_TABLE_MAIN; 954 } 955 956 static struct mlxsw_sp_rif * 957 mlxsw_sp_rif_create(struct mlxsw_sp *mlxsw_sp, 958 const struct mlxsw_sp_rif_params *params, 959 struct netlink_ext_ack *extack); 960 961 static struct mlxsw_sp_rif_ipip_lb * 962 mlxsw_sp_ipip_ol_ipip_lb_create(struct mlxsw_sp *mlxsw_sp, 963 enum mlxsw_sp_ipip_type ipipt, 964 struct net_device *ol_dev, 965 struct netlink_ext_ack *extack) 966 { 967 struct mlxsw_sp_rif_params_ipip_lb lb_params; 968 const struct mlxsw_sp_ipip_ops *ipip_ops; 969 struct mlxsw_sp_rif *rif; 970 971 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipipt]; 972 lb_params = (struct mlxsw_sp_rif_params_ipip_lb) { 973 .common.dev = ol_dev, 974 .common.lag = false, 975 .lb_config = ipip_ops->ol_loopback_config(mlxsw_sp, ol_dev), 976 }; 977 978 rif = mlxsw_sp_rif_create(mlxsw_sp, &lb_params.common, extack); 979 if (IS_ERR(rif)) 980 return ERR_CAST(rif); 981 return container_of(rif, struct mlxsw_sp_rif_ipip_lb, common); 982 } 983 984 static struct mlxsw_sp_ipip_entry * 985 mlxsw_sp_ipip_entry_alloc(struct mlxsw_sp *mlxsw_sp, 986 enum mlxsw_sp_ipip_type ipipt, 987 struct net_device *ol_dev) 988 { 989 struct mlxsw_sp_ipip_entry *ipip_entry; 990 struct mlxsw_sp_ipip_entry *ret = NULL; 991 992 ipip_entry = kzalloc(sizeof(*ipip_entry), GFP_KERNEL); 993 if (!ipip_entry) 994 return ERR_PTR(-ENOMEM); 995 996 ipip_entry->ol_lb = mlxsw_sp_ipip_ol_ipip_lb_create(mlxsw_sp, ipipt, 997 ol_dev, NULL); 998 if (IS_ERR(ipip_entry->ol_lb)) { 999 ret = ERR_CAST(ipip_entry->ol_lb); 1000 goto err_ol_ipip_lb_create; 1001 } 1002 1003 ipip_entry->ipipt = ipipt; 1004 ipip_entry->ol_dev = ol_dev; 1005 ipip_entry->parms = mlxsw_sp_ipip_netdev_parms(ol_dev); 1006 1007 return ipip_entry; 1008 1009 err_ol_ipip_lb_create: 1010 kfree(ipip_entry); 1011 return ret; 1012 } 1013 1014 static void 1015 mlxsw_sp_ipip_entry_dealloc(struct mlxsw_sp_ipip_entry *ipip_entry) 1016 { 1017 mlxsw_sp_rif_destroy(&ipip_entry->ol_lb->common); 1018 kfree(ipip_entry); 1019 } 1020 1021 static bool 1022 mlxsw_sp_ipip_entry_saddr_matches(struct mlxsw_sp *mlxsw_sp, 1023 const enum mlxsw_sp_l3proto ul_proto, 1024 union mlxsw_sp_l3addr saddr, 1025 u32 ul_tb_id, 1026 struct mlxsw_sp_ipip_entry *ipip_entry) 1027 { 1028 u32 tun_ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ipip_entry->ol_dev); 1029 enum mlxsw_sp_ipip_type ipipt = ipip_entry->ipipt; 1030 union mlxsw_sp_l3addr tun_saddr; 1031 1032 if (mlxsw_sp->router->ipip_ops_arr[ipipt]->ul_proto != ul_proto) 1033 return false; 1034 1035 tun_saddr = mlxsw_sp_ipip_netdev_saddr(ul_proto, ipip_entry->ol_dev); 1036 return tun_ul_tb_id == ul_tb_id && 1037 mlxsw_sp_l3addr_eq(&tun_saddr, &saddr); 1038 } 1039 1040 static int 1041 mlxsw_sp_fib_entry_decap_init(struct mlxsw_sp *mlxsw_sp, 1042 struct mlxsw_sp_fib_entry *fib_entry, 1043 struct mlxsw_sp_ipip_entry *ipip_entry) 1044 { 1045 u32 tunnel_index; 1046 int err; 1047 1048 err = mlxsw_sp_kvdl_alloc(mlxsw_sp, 1, &tunnel_index); 1049 if (err) 1050 return err; 1051 1052 ipip_entry->decap_fib_entry = fib_entry; 1053 fib_entry->decap.ipip_entry = ipip_entry; 1054 fib_entry->decap.tunnel_index = tunnel_index; 1055 return 0; 1056 } 1057 1058 static void mlxsw_sp_fib_entry_decap_fini(struct mlxsw_sp *mlxsw_sp, 1059 struct mlxsw_sp_fib_entry *fib_entry) 1060 { 1061 /* Unlink this node from the IPIP entry that it's the decap entry of. */ 1062 fib_entry->decap.ipip_entry->decap_fib_entry = NULL; 1063 fib_entry->decap.ipip_entry = NULL; 1064 mlxsw_sp_kvdl_free(mlxsw_sp, fib_entry->decap.tunnel_index); 1065 } 1066 1067 static struct mlxsw_sp_fib_node * 1068 mlxsw_sp_fib_node_lookup(struct mlxsw_sp_fib *fib, const void *addr, 1069 size_t addr_len, unsigned char prefix_len); 1070 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp, 1071 struct mlxsw_sp_fib_entry *fib_entry); 1072 1073 static void 1074 mlxsw_sp_ipip_entry_demote_decap(struct mlxsw_sp *mlxsw_sp, 1075 struct mlxsw_sp_ipip_entry *ipip_entry) 1076 { 1077 struct mlxsw_sp_fib_entry *fib_entry = ipip_entry->decap_fib_entry; 1078 1079 mlxsw_sp_fib_entry_decap_fini(mlxsw_sp, fib_entry); 1080 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP; 1081 1082 mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry); 1083 } 1084 1085 static void 1086 mlxsw_sp_ipip_entry_promote_decap(struct mlxsw_sp *mlxsw_sp, 1087 struct mlxsw_sp_ipip_entry *ipip_entry, 1088 struct mlxsw_sp_fib_entry *decap_fib_entry) 1089 { 1090 if (mlxsw_sp_fib_entry_decap_init(mlxsw_sp, decap_fib_entry, 1091 ipip_entry)) 1092 return; 1093 decap_fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP; 1094 1095 if (mlxsw_sp_fib_entry_update(mlxsw_sp, decap_fib_entry)) 1096 mlxsw_sp_ipip_entry_demote_decap(mlxsw_sp, ipip_entry); 1097 } 1098 1099 /* Given an IPIP entry, find the corresponding decap route. */ 1100 static struct mlxsw_sp_fib_entry * 1101 mlxsw_sp_ipip_entry_find_decap(struct mlxsw_sp *mlxsw_sp, 1102 struct mlxsw_sp_ipip_entry *ipip_entry) 1103 { 1104 static struct mlxsw_sp_fib_node *fib_node; 1105 const struct mlxsw_sp_ipip_ops *ipip_ops; 1106 struct mlxsw_sp_fib_entry *fib_entry; 1107 unsigned char saddr_prefix_len; 1108 union mlxsw_sp_l3addr saddr; 1109 struct mlxsw_sp_fib *ul_fib; 1110 struct mlxsw_sp_vr *ul_vr; 1111 const void *saddrp; 1112 size_t saddr_len; 1113 u32 ul_tb_id; 1114 u32 saddr4; 1115 1116 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt]; 1117 1118 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ipip_entry->ol_dev); 1119 ul_vr = mlxsw_sp_vr_find(mlxsw_sp, ul_tb_id); 1120 if (!ul_vr) 1121 return NULL; 1122 1123 ul_fib = mlxsw_sp_vr_fib(ul_vr, ipip_ops->ul_proto); 1124 saddr = mlxsw_sp_ipip_netdev_saddr(ipip_ops->ul_proto, 1125 ipip_entry->ol_dev); 1126 1127 switch (ipip_ops->ul_proto) { 1128 case MLXSW_SP_L3_PROTO_IPV4: 1129 saddr4 = be32_to_cpu(saddr.addr4); 1130 saddrp = &saddr4; 1131 saddr_len = 4; 1132 saddr_prefix_len = 32; 1133 break; 1134 case MLXSW_SP_L3_PROTO_IPV6: 1135 WARN_ON(1); 1136 return NULL; 1137 } 1138 1139 fib_node = mlxsw_sp_fib_node_lookup(ul_fib, saddrp, saddr_len, 1140 saddr_prefix_len); 1141 if (!fib_node || list_empty(&fib_node->entry_list)) 1142 return NULL; 1143 1144 fib_entry = list_first_entry(&fib_node->entry_list, 1145 struct mlxsw_sp_fib_entry, list); 1146 if (fib_entry->type != MLXSW_SP_FIB_ENTRY_TYPE_TRAP) 1147 return NULL; 1148 1149 return fib_entry; 1150 } 1151 1152 static struct mlxsw_sp_ipip_entry * 1153 mlxsw_sp_ipip_entry_create(struct mlxsw_sp *mlxsw_sp, 1154 enum mlxsw_sp_ipip_type ipipt, 1155 struct net_device *ol_dev) 1156 { 1157 struct mlxsw_sp_ipip_entry *ipip_entry; 1158 1159 ipip_entry = mlxsw_sp_ipip_entry_alloc(mlxsw_sp, ipipt, ol_dev); 1160 if (IS_ERR(ipip_entry)) 1161 return ipip_entry; 1162 1163 list_add_tail(&ipip_entry->ipip_list_node, 1164 &mlxsw_sp->router->ipip_list); 1165 1166 return ipip_entry; 1167 } 1168 1169 static void 1170 mlxsw_sp_ipip_entry_destroy(struct mlxsw_sp *mlxsw_sp, 1171 struct mlxsw_sp_ipip_entry *ipip_entry) 1172 { 1173 list_del(&ipip_entry->ipip_list_node); 1174 mlxsw_sp_ipip_entry_dealloc(ipip_entry); 1175 } 1176 1177 static bool 1178 mlxsw_sp_ipip_entry_matches_decap(struct mlxsw_sp *mlxsw_sp, 1179 const struct net_device *ul_dev, 1180 enum mlxsw_sp_l3proto ul_proto, 1181 union mlxsw_sp_l3addr ul_dip, 1182 struct mlxsw_sp_ipip_entry *ipip_entry) 1183 { 1184 u32 ul_tb_id = l3mdev_fib_table(ul_dev) ? : RT_TABLE_MAIN; 1185 enum mlxsw_sp_ipip_type ipipt = ipip_entry->ipipt; 1186 struct net_device *ipip_ul_dev; 1187 1188 if (mlxsw_sp->router->ipip_ops_arr[ipipt]->ul_proto != ul_proto) 1189 return false; 1190 1191 ipip_ul_dev = __mlxsw_sp_ipip_netdev_ul_dev_get(ipip_entry->ol_dev); 1192 return mlxsw_sp_ipip_entry_saddr_matches(mlxsw_sp, ul_proto, ul_dip, 1193 ul_tb_id, ipip_entry) && 1194 (!ipip_ul_dev || ipip_ul_dev == ul_dev); 1195 } 1196 1197 /* Given decap parameters, find the corresponding IPIP entry. */ 1198 static struct mlxsw_sp_ipip_entry * 1199 mlxsw_sp_ipip_entry_find_by_decap(struct mlxsw_sp *mlxsw_sp, 1200 const struct net_device *ul_dev, 1201 enum mlxsw_sp_l3proto ul_proto, 1202 union mlxsw_sp_l3addr ul_dip) 1203 { 1204 struct mlxsw_sp_ipip_entry *ipip_entry; 1205 1206 list_for_each_entry(ipip_entry, &mlxsw_sp->router->ipip_list, 1207 ipip_list_node) 1208 if (mlxsw_sp_ipip_entry_matches_decap(mlxsw_sp, ul_dev, 1209 ul_proto, ul_dip, 1210 ipip_entry)) 1211 return ipip_entry; 1212 1213 return NULL; 1214 } 1215 1216 static bool mlxsw_sp_netdev_ipip_type(const struct mlxsw_sp *mlxsw_sp, 1217 const struct net_device *dev, 1218 enum mlxsw_sp_ipip_type *p_type) 1219 { 1220 struct mlxsw_sp_router *router = mlxsw_sp->router; 1221 const struct mlxsw_sp_ipip_ops *ipip_ops; 1222 enum mlxsw_sp_ipip_type ipipt; 1223 1224 for (ipipt = 0; ipipt < MLXSW_SP_IPIP_TYPE_MAX; ++ipipt) { 1225 ipip_ops = router->ipip_ops_arr[ipipt]; 1226 if (dev->type == ipip_ops->dev_type) { 1227 if (p_type) 1228 *p_type = ipipt; 1229 return true; 1230 } 1231 } 1232 return false; 1233 } 1234 1235 bool mlxsw_sp_netdev_is_ipip_ol(const struct mlxsw_sp *mlxsw_sp, 1236 const struct net_device *dev) 1237 { 1238 return mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, NULL); 1239 } 1240 1241 static struct mlxsw_sp_ipip_entry * 1242 mlxsw_sp_ipip_entry_find_by_ol_dev(struct mlxsw_sp *mlxsw_sp, 1243 const struct net_device *ol_dev) 1244 { 1245 struct mlxsw_sp_ipip_entry *ipip_entry; 1246 1247 list_for_each_entry(ipip_entry, &mlxsw_sp->router->ipip_list, 1248 ipip_list_node) 1249 if (ipip_entry->ol_dev == ol_dev) 1250 return ipip_entry; 1251 1252 return NULL; 1253 } 1254 1255 static struct mlxsw_sp_ipip_entry * 1256 mlxsw_sp_ipip_entry_find_by_ul_dev(const struct mlxsw_sp *mlxsw_sp, 1257 const struct net_device *ul_dev, 1258 struct mlxsw_sp_ipip_entry *start) 1259 { 1260 struct mlxsw_sp_ipip_entry *ipip_entry; 1261 1262 ipip_entry = list_prepare_entry(start, &mlxsw_sp->router->ipip_list, 1263 ipip_list_node); 1264 list_for_each_entry_continue(ipip_entry, &mlxsw_sp->router->ipip_list, 1265 ipip_list_node) { 1266 struct net_device *ipip_ul_dev = 1267 __mlxsw_sp_ipip_netdev_ul_dev_get(ipip_entry->ol_dev); 1268 1269 if (ipip_ul_dev == ul_dev) 1270 return ipip_entry; 1271 } 1272 1273 return NULL; 1274 } 1275 1276 bool mlxsw_sp_netdev_is_ipip_ul(const struct mlxsw_sp *mlxsw_sp, 1277 const struct net_device *dev) 1278 { 1279 return mlxsw_sp_ipip_entry_find_by_ul_dev(mlxsw_sp, dev, NULL); 1280 } 1281 1282 static bool mlxsw_sp_netdevice_ipip_can_offload(struct mlxsw_sp *mlxsw_sp, 1283 const struct net_device *ol_dev, 1284 enum mlxsw_sp_ipip_type ipipt) 1285 { 1286 const struct mlxsw_sp_ipip_ops *ops 1287 = mlxsw_sp->router->ipip_ops_arr[ipipt]; 1288 1289 /* For deciding whether decap should be offloaded, we don't care about 1290 * overlay protocol, so ask whether either one is supported. 1291 */ 1292 return ops->can_offload(mlxsw_sp, ol_dev, MLXSW_SP_L3_PROTO_IPV4) || 1293 ops->can_offload(mlxsw_sp, ol_dev, MLXSW_SP_L3_PROTO_IPV6); 1294 } 1295 1296 static int mlxsw_sp_netdevice_ipip_ol_reg_event(struct mlxsw_sp *mlxsw_sp, 1297 struct net_device *ol_dev) 1298 { 1299 struct mlxsw_sp_ipip_entry *ipip_entry; 1300 enum mlxsw_sp_l3proto ul_proto; 1301 enum mlxsw_sp_ipip_type ipipt; 1302 union mlxsw_sp_l3addr saddr; 1303 u32 ul_tb_id; 1304 1305 mlxsw_sp_netdev_ipip_type(mlxsw_sp, ol_dev, &ipipt); 1306 if (mlxsw_sp_netdevice_ipip_can_offload(mlxsw_sp, ol_dev, ipipt)) { 1307 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ol_dev); 1308 ul_proto = mlxsw_sp->router->ipip_ops_arr[ipipt]->ul_proto; 1309 saddr = mlxsw_sp_ipip_netdev_saddr(ul_proto, ol_dev); 1310 if (!mlxsw_sp_ipip_demote_tunnel_by_saddr(mlxsw_sp, ul_proto, 1311 saddr, ul_tb_id, 1312 NULL)) { 1313 ipip_entry = mlxsw_sp_ipip_entry_create(mlxsw_sp, ipipt, 1314 ol_dev); 1315 if (IS_ERR(ipip_entry)) 1316 return PTR_ERR(ipip_entry); 1317 } 1318 } 1319 1320 return 0; 1321 } 1322 1323 static void mlxsw_sp_netdevice_ipip_ol_unreg_event(struct mlxsw_sp *mlxsw_sp, 1324 struct net_device *ol_dev) 1325 { 1326 struct mlxsw_sp_ipip_entry *ipip_entry; 1327 1328 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev); 1329 if (ipip_entry) 1330 mlxsw_sp_ipip_entry_destroy(mlxsw_sp, ipip_entry); 1331 } 1332 1333 static void 1334 mlxsw_sp_ipip_entry_ol_up_event(struct mlxsw_sp *mlxsw_sp, 1335 struct mlxsw_sp_ipip_entry *ipip_entry) 1336 { 1337 struct mlxsw_sp_fib_entry *decap_fib_entry; 1338 1339 decap_fib_entry = mlxsw_sp_ipip_entry_find_decap(mlxsw_sp, ipip_entry); 1340 if (decap_fib_entry) 1341 mlxsw_sp_ipip_entry_promote_decap(mlxsw_sp, ipip_entry, 1342 decap_fib_entry); 1343 } 1344 1345 static void mlxsw_sp_netdevice_ipip_ol_up_event(struct mlxsw_sp *mlxsw_sp, 1346 struct net_device *ol_dev) 1347 { 1348 struct mlxsw_sp_ipip_entry *ipip_entry; 1349 1350 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev); 1351 if (ipip_entry) 1352 mlxsw_sp_ipip_entry_ol_up_event(mlxsw_sp, ipip_entry); 1353 } 1354 1355 static void 1356 mlxsw_sp_ipip_entry_ol_down_event(struct mlxsw_sp *mlxsw_sp, 1357 struct mlxsw_sp_ipip_entry *ipip_entry) 1358 { 1359 if (ipip_entry->decap_fib_entry) 1360 mlxsw_sp_ipip_entry_demote_decap(mlxsw_sp, ipip_entry); 1361 } 1362 1363 static void mlxsw_sp_netdevice_ipip_ol_down_event(struct mlxsw_sp *mlxsw_sp, 1364 struct net_device *ol_dev) 1365 { 1366 struct mlxsw_sp_ipip_entry *ipip_entry; 1367 1368 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev); 1369 if (ipip_entry) 1370 mlxsw_sp_ipip_entry_ol_down_event(mlxsw_sp, ipip_entry); 1371 } 1372 1373 static void mlxsw_sp_nexthop_rif_migrate(struct mlxsw_sp *mlxsw_sp, 1374 struct mlxsw_sp_rif *old_rif, 1375 struct mlxsw_sp_rif *new_rif); 1376 static int 1377 mlxsw_sp_ipip_entry_ol_lb_update(struct mlxsw_sp *mlxsw_sp, 1378 struct mlxsw_sp_ipip_entry *ipip_entry, 1379 bool keep_encap, 1380 struct netlink_ext_ack *extack) 1381 { 1382 struct mlxsw_sp_rif_ipip_lb *old_lb_rif = ipip_entry->ol_lb; 1383 struct mlxsw_sp_rif_ipip_lb *new_lb_rif; 1384 1385 new_lb_rif = mlxsw_sp_ipip_ol_ipip_lb_create(mlxsw_sp, 1386 ipip_entry->ipipt, 1387 ipip_entry->ol_dev, 1388 extack); 1389 if (IS_ERR(new_lb_rif)) 1390 return PTR_ERR(new_lb_rif); 1391 ipip_entry->ol_lb = new_lb_rif; 1392 1393 if (keep_encap) 1394 mlxsw_sp_nexthop_rif_migrate(mlxsw_sp, &old_lb_rif->common, 1395 &new_lb_rif->common); 1396 1397 mlxsw_sp_rif_destroy(&old_lb_rif->common); 1398 1399 return 0; 1400 } 1401 1402 static void mlxsw_sp_nexthop_rif_update(struct mlxsw_sp *mlxsw_sp, 1403 struct mlxsw_sp_rif *rif); 1404 1405 /** 1406 * Update the offload related to an IPIP entry. This always updates decap, and 1407 * in addition to that it also: 1408 * @recreate_loopback: recreates the associated loopback RIF 1409 * @keep_encap: updates next hops that use the tunnel netdevice. This is only 1410 * relevant when recreate_loopback is true. 1411 * @update_nexthops: updates next hops, keeping the current loopback RIF. This 1412 * is only relevant when recreate_loopback is false. 1413 */ 1414 int __mlxsw_sp_ipip_entry_update_tunnel(struct mlxsw_sp *mlxsw_sp, 1415 struct mlxsw_sp_ipip_entry *ipip_entry, 1416 bool recreate_loopback, 1417 bool keep_encap, 1418 bool update_nexthops, 1419 struct netlink_ext_ack *extack) 1420 { 1421 int err; 1422 1423 /* RIFs can't be edited, so to update loopback, we need to destroy and 1424 * recreate it. That creates a window of opportunity where RALUE and 1425 * RATR registers end up referencing a RIF that's already gone. RATRs 1426 * are handled in mlxsw_sp_ipip_entry_ol_lb_update(), and to take care 1427 * of RALUE, demote the decap route back. 1428 */ 1429 if (ipip_entry->decap_fib_entry) 1430 mlxsw_sp_ipip_entry_demote_decap(mlxsw_sp, ipip_entry); 1431 1432 if (recreate_loopback) { 1433 err = mlxsw_sp_ipip_entry_ol_lb_update(mlxsw_sp, ipip_entry, 1434 keep_encap, extack); 1435 if (err) 1436 return err; 1437 } else if (update_nexthops) { 1438 mlxsw_sp_nexthop_rif_update(mlxsw_sp, 1439 &ipip_entry->ol_lb->common); 1440 } 1441 1442 if (ipip_entry->ol_dev->flags & IFF_UP) 1443 mlxsw_sp_ipip_entry_ol_up_event(mlxsw_sp, ipip_entry); 1444 1445 return 0; 1446 } 1447 1448 static int mlxsw_sp_netdevice_ipip_ol_vrf_event(struct mlxsw_sp *mlxsw_sp, 1449 struct net_device *ol_dev, 1450 struct netlink_ext_ack *extack) 1451 { 1452 struct mlxsw_sp_ipip_entry *ipip_entry = 1453 mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev); 1454 enum mlxsw_sp_l3proto ul_proto; 1455 union mlxsw_sp_l3addr saddr; 1456 u32 ul_tb_id; 1457 1458 if (!ipip_entry) 1459 return 0; 1460 1461 /* For flat configuration cases, moving overlay to a different VRF might 1462 * cause local address conflict, and the conflicting tunnels need to be 1463 * demoted. 1464 */ 1465 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ol_dev); 1466 ul_proto = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt]->ul_proto; 1467 saddr = mlxsw_sp_ipip_netdev_saddr(ul_proto, ol_dev); 1468 if (mlxsw_sp_ipip_demote_tunnel_by_saddr(mlxsw_sp, ul_proto, 1469 saddr, ul_tb_id, 1470 ipip_entry)) { 1471 mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry); 1472 return 0; 1473 } 1474 1475 return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry, 1476 true, false, false, extack); 1477 } 1478 1479 static int 1480 mlxsw_sp_netdevice_ipip_ul_vrf_event(struct mlxsw_sp *mlxsw_sp, 1481 struct mlxsw_sp_ipip_entry *ipip_entry, 1482 struct net_device *ul_dev, 1483 struct netlink_ext_ack *extack) 1484 { 1485 return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry, 1486 true, true, false, extack); 1487 } 1488 1489 static int 1490 mlxsw_sp_netdevice_ipip_ul_up_event(struct mlxsw_sp *mlxsw_sp, 1491 struct mlxsw_sp_ipip_entry *ipip_entry, 1492 struct net_device *ul_dev) 1493 { 1494 return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry, 1495 false, false, true, NULL); 1496 } 1497 1498 static int 1499 mlxsw_sp_netdevice_ipip_ul_down_event(struct mlxsw_sp *mlxsw_sp, 1500 struct mlxsw_sp_ipip_entry *ipip_entry, 1501 struct net_device *ul_dev) 1502 { 1503 /* A down underlay device causes encapsulated packets to not be 1504 * forwarded, but decap still works. So refresh next hops without 1505 * touching anything else. 1506 */ 1507 return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry, 1508 false, false, true, NULL); 1509 } 1510 1511 static int 1512 mlxsw_sp_netdevice_ipip_ol_change_event(struct mlxsw_sp *mlxsw_sp, 1513 struct net_device *ol_dev, 1514 struct netlink_ext_ack *extack) 1515 { 1516 const struct mlxsw_sp_ipip_ops *ipip_ops; 1517 struct mlxsw_sp_ipip_entry *ipip_entry; 1518 int err; 1519 1520 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev); 1521 if (!ipip_entry) 1522 /* A change might make a tunnel eligible for offloading, but 1523 * that is currently not implemented. What falls to slow path 1524 * stays there. 1525 */ 1526 return 0; 1527 1528 /* A change might make a tunnel not eligible for offloading. */ 1529 if (!mlxsw_sp_netdevice_ipip_can_offload(mlxsw_sp, ol_dev, 1530 ipip_entry->ipipt)) { 1531 mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry); 1532 return 0; 1533 } 1534 1535 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt]; 1536 err = ipip_ops->ol_netdev_change(mlxsw_sp, ipip_entry, extack); 1537 return err; 1538 } 1539 1540 void mlxsw_sp_ipip_entry_demote_tunnel(struct mlxsw_sp *mlxsw_sp, 1541 struct mlxsw_sp_ipip_entry *ipip_entry) 1542 { 1543 struct net_device *ol_dev = ipip_entry->ol_dev; 1544 1545 if (ol_dev->flags & IFF_UP) 1546 mlxsw_sp_ipip_entry_ol_down_event(mlxsw_sp, ipip_entry); 1547 mlxsw_sp_ipip_entry_destroy(mlxsw_sp, ipip_entry); 1548 } 1549 1550 /* The configuration where several tunnels have the same local address in the 1551 * same underlay table needs special treatment in the HW. That is currently not 1552 * implemented in the driver. This function finds and demotes the first tunnel 1553 * with a given source address, except the one passed in in the argument 1554 * `except'. 1555 */ 1556 bool 1557 mlxsw_sp_ipip_demote_tunnel_by_saddr(struct mlxsw_sp *mlxsw_sp, 1558 enum mlxsw_sp_l3proto ul_proto, 1559 union mlxsw_sp_l3addr saddr, 1560 u32 ul_tb_id, 1561 const struct mlxsw_sp_ipip_entry *except) 1562 { 1563 struct mlxsw_sp_ipip_entry *ipip_entry, *tmp; 1564 1565 list_for_each_entry_safe(ipip_entry, tmp, &mlxsw_sp->router->ipip_list, 1566 ipip_list_node) { 1567 if (ipip_entry != except && 1568 mlxsw_sp_ipip_entry_saddr_matches(mlxsw_sp, ul_proto, saddr, 1569 ul_tb_id, ipip_entry)) { 1570 mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry); 1571 return true; 1572 } 1573 } 1574 1575 return false; 1576 } 1577 1578 static void mlxsw_sp_ipip_demote_tunnel_by_ul_netdev(struct mlxsw_sp *mlxsw_sp, 1579 struct net_device *ul_dev) 1580 { 1581 struct mlxsw_sp_ipip_entry *ipip_entry, *tmp; 1582 1583 list_for_each_entry_safe(ipip_entry, tmp, &mlxsw_sp->router->ipip_list, 1584 ipip_list_node) { 1585 struct net_device *ipip_ul_dev = 1586 __mlxsw_sp_ipip_netdev_ul_dev_get(ipip_entry->ol_dev); 1587 1588 if (ipip_ul_dev == ul_dev) 1589 mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry); 1590 } 1591 } 1592 1593 int mlxsw_sp_netdevice_ipip_ol_event(struct mlxsw_sp *mlxsw_sp, 1594 struct net_device *ol_dev, 1595 unsigned long event, 1596 struct netdev_notifier_info *info) 1597 { 1598 struct netdev_notifier_changeupper_info *chup; 1599 struct netlink_ext_ack *extack; 1600 1601 switch (event) { 1602 case NETDEV_REGISTER: 1603 return mlxsw_sp_netdevice_ipip_ol_reg_event(mlxsw_sp, ol_dev); 1604 case NETDEV_UNREGISTER: 1605 mlxsw_sp_netdevice_ipip_ol_unreg_event(mlxsw_sp, ol_dev); 1606 return 0; 1607 case NETDEV_UP: 1608 mlxsw_sp_netdevice_ipip_ol_up_event(mlxsw_sp, ol_dev); 1609 return 0; 1610 case NETDEV_DOWN: 1611 mlxsw_sp_netdevice_ipip_ol_down_event(mlxsw_sp, ol_dev); 1612 return 0; 1613 case NETDEV_CHANGEUPPER: 1614 chup = container_of(info, typeof(*chup), info); 1615 extack = info->extack; 1616 if (netif_is_l3_master(chup->upper_dev)) 1617 return mlxsw_sp_netdevice_ipip_ol_vrf_event(mlxsw_sp, 1618 ol_dev, 1619 extack); 1620 return 0; 1621 case NETDEV_CHANGE: 1622 extack = info->extack; 1623 return mlxsw_sp_netdevice_ipip_ol_change_event(mlxsw_sp, 1624 ol_dev, extack); 1625 } 1626 return 0; 1627 } 1628 1629 static int 1630 __mlxsw_sp_netdevice_ipip_ul_event(struct mlxsw_sp *mlxsw_sp, 1631 struct mlxsw_sp_ipip_entry *ipip_entry, 1632 struct net_device *ul_dev, 1633 unsigned long event, 1634 struct netdev_notifier_info *info) 1635 { 1636 struct netdev_notifier_changeupper_info *chup; 1637 struct netlink_ext_ack *extack; 1638 1639 switch (event) { 1640 case NETDEV_CHANGEUPPER: 1641 chup = container_of(info, typeof(*chup), info); 1642 extack = info->extack; 1643 if (netif_is_l3_master(chup->upper_dev)) 1644 return mlxsw_sp_netdevice_ipip_ul_vrf_event(mlxsw_sp, 1645 ipip_entry, 1646 ul_dev, 1647 extack); 1648 break; 1649 1650 case NETDEV_UP: 1651 return mlxsw_sp_netdevice_ipip_ul_up_event(mlxsw_sp, ipip_entry, 1652 ul_dev); 1653 case NETDEV_DOWN: 1654 return mlxsw_sp_netdevice_ipip_ul_down_event(mlxsw_sp, 1655 ipip_entry, 1656 ul_dev); 1657 } 1658 return 0; 1659 } 1660 1661 int 1662 mlxsw_sp_netdevice_ipip_ul_event(struct mlxsw_sp *mlxsw_sp, 1663 struct net_device *ul_dev, 1664 unsigned long event, 1665 struct netdev_notifier_info *info) 1666 { 1667 struct mlxsw_sp_ipip_entry *ipip_entry = NULL; 1668 int err; 1669 1670 while ((ipip_entry = mlxsw_sp_ipip_entry_find_by_ul_dev(mlxsw_sp, 1671 ul_dev, 1672 ipip_entry))) { 1673 err = __mlxsw_sp_netdevice_ipip_ul_event(mlxsw_sp, ipip_entry, 1674 ul_dev, event, info); 1675 if (err) { 1676 mlxsw_sp_ipip_demote_tunnel_by_ul_netdev(mlxsw_sp, 1677 ul_dev); 1678 return err; 1679 } 1680 } 1681 1682 return 0; 1683 } 1684 1685 struct mlxsw_sp_neigh_key { 1686 struct neighbour *n; 1687 }; 1688 1689 struct mlxsw_sp_neigh_entry { 1690 struct list_head rif_list_node; 1691 struct rhash_head ht_node; 1692 struct mlxsw_sp_neigh_key key; 1693 u16 rif; 1694 bool connected; 1695 unsigned char ha[ETH_ALEN]; 1696 struct list_head nexthop_list; /* list of nexthops using 1697 * this neigh entry 1698 */ 1699 struct list_head nexthop_neighs_list_node; 1700 unsigned int counter_index; 1701 bool counter_valid; 1702 }; 1703 1704 static const struct rhashtable_params mlxsw_sp_neigh_ht_params = { 1705 .key_offset = offsetof(struct mlxsw_sp_neigh_entry, key), 1706 .head_offset = offsetof(struct mlxsw_sp_neigh_entry, ht_node), 1707 .key_len = sizeof(struct mlxsw_sp_neigh_key), 1708 }; 1709 1710 struct mlxsw_sp_neigh_entry * 1711 mlxsw_sp_rif_neigh_next(struct mlxsw_sp_rif *rif, 1712 struct mlxsw_sp_neigh_entry *neigh_entry) 1713 { 1714 if (!neigh_entry) { 1715 if (list_empty(&rif->neigh_list)) 1716 return NULL; 1717 else 1718 return list_first_entry(&rif->neigh_list, 1719 typeof(*neigh_entry), 1720 rif_list_node); 1721 } 1722 if (list_is_last(&neigh_entry->rif_list_node, &rif->neigh_list)) 1723 return NULL; 1724 return list_next_entry(neigh_entry, rif_list_node); 1725 } 1726 1727 int mlxsw_sp_neigh_entry_type(struct mlxsw_sp_neigh_entry *neigh_entry) 1728 { 1729 return neigh_entry->key.n->tbl->family; 1730 } 1731 1732 unsigned char * 1733 mlxsw_sp_neigh_entry_ha(struct mlxsw_sp_neigh_entry *neigh_entry) 1734 { 1735 return neigh_entry->ha; 1736 } 1737 1738 u32 mlxsw_sp_neigh4_entry_dip(struct mlxsw_sp_neigh_entry *neigh_entry) 1739 { 1740 struct neighbour *n; 1741 1742 n = neigh_entry->key.n; 1743 return ntohl(*((__be32 *) n->primary_key)); 1744 } 1745 1746 struct in6_addr * 1747 mlxsw_sp_neigh6_entry_dip(struct mlxsw_sp_neigh_entry *neigh_entry) 1748 { 1749 struct neighbour *n; 1750 1751 n = neigh_entry->key.n; 1752 return (struct in6_addr *) &n->primary_key; 1753 } 1754 1755 int mlxsw_sp_neigh_counter_get(struct mlxsw_sp *mlxsw_sp, 1756 struct mlxsw_sp_neigh_entry *neigh_entry, 1757 u64 *p_counter) 1758 { 1759 if (!neigh_entry->counter_valid) 1760 return -EINVAL; 1761 1762 return mlxsw_sp_flow_counter_get(mlxsw_sp, neigh_entry->counter_index, 1763 p_counter, NULL); 1764 } 1765 1766 static struct mlxsw_sp_neigh_entry * 1767 mlxsw_sp_neigh_entry_alloc(struct mlxsw_sp *mlxsw_sp, struct neighbour *n, 1768 u16 rif) 1769 { 1770 struct mlxsw_sp_neigh_entry *neigh_entry; 1771 1772 neigh_entry = kzalloc(sizeof(*neigh_entry), GFP_KERNEL); 1773 if (!neigh_entry) 1774 return NULL; 1775 1776 neigh_entry->key.n = n; 1777 neigh_entry->rif = rif; 1778 INIT_LIST_HEAD(&neigh_entry->nexthop_list); 1779 1780 return neigh_entry; 1781 } 1782 1783 static void mlxsw_sp_neigh_entry_free(struct mlxsw_sp_neigh_entry *neigh_entry) 1784 { 1785 kfree(neigh_entry); 1786 } 1787 1788 static int 1789 mlxsw_sp_neigh_entry_insert(struct mlxsw_sp *mlxsw_sp, 1790 struct mlxsw_sp_neigh_entry *neigh_entry) 1791 { 1792 return rhashtable_insert_fast(&mlxsw_sp->router->neigh_ht, 1793 &neigh_entry->ht_node, 1794 mlxsw_sp_neigh_ht_params); 1795 } 1796 1797 static void 1798 mlxsw_sp_neigh_entry_remove(struct mlxsw_sp *mlxsw_sp, 1799 struct mlxsw_sp_neigh_entry *neigh_entry) 1800 { 1801 rhashtable_remove_fast(&mlxsw_sp->router->neigh_ht, 1802 &neigh_entry->ht_node, 1803 mlxsw_sp_neigh_ht_params); 1804 } 1805 1806 static bool 1807 mlxsw_sp_neigh_counter_should_alloc(struct mlxsw_sp *mlxsw_sp, 1808 struct mlxsw_sp_neigh_entry *neigh_entry) 1809 { 1810 struct devlink *devlink; 1811 const char *table_name; 1812 1813 switch (mlxsw_sp_neigh_entry_type(neigh_entry)) { 1814 case AF_INET: 1815 table_name = MLXSW_SP_DPIPE_TABLE_NAME_HOST4; 1816 break; 1817 case AF_INET6: 1818 table_name = MLXSW_SP_DPIPE_TABLE_NAME_HOST6; 1819 break; 1820 default: 1821 WARN_ON(1); 1822 return false; 1823 } 1824 1825 devlink = priv_to_devlink(mlxsw_sp->core); 1826 return devlink_dpipe_table_counter_enabled(devlink, table_name); 1827 } 1828 1829 static void 1830 mlxsw_sp_neigh_counter_alloc(struct mlxsw_sp *mlxsw_sp, 1831 struct mlxsw_sp_neigh_entry *neigh_entry) 1832 { 1833 if (!mlxsw_sp_neigh_counter_should_alloc(mlxsw_sp, neigh_entry)) 1834 return; 1835 1836 if (mlxsw_sp_flow_counter_alloc(mlxsw_sp, &neigh_entry->counter_index)) 1837 return; 1838 1839 neigh_entry->counter_valid = true; 1840 } 1841 1842 static void 1843 mlxsw_sp_neigh_counter_free(struct mlxsw_sp *mlxsw_sp, 1844 struct mlxsw_sp_neigh_entry *neigh_entry) 1845 { 1846 if (!neigh_entry->counter_valid) 1847 return; 1848 mlxsw_sp_flow_counter_free(mlxsw_sp, 1849 neigh_entry->counter_index); 1850 neigh_entry->counter_valid = false; 1851 } 1852 1853 static struct mlxsw_sp_neigh_entry * 1854 mlxsw_sp_neigh_entry_create(struct mlxsw_sp *mlxsw_sp, struct neighbour *n) 1855 { 1856 struct mlxsw_sp_neigh_entry *neigh_entry; 1857 struct mlxsw_sp_rif *rif; 1858 int err; 1859 1860 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, n->dev); 1861 if (!rif) 1862 return ERR_PTR(-EINVAL); 1863 1864 neigh_entry = mlxsw_sp_neigh_entry_alloc(mlxsw_sp, n, rif->rif_index); 1865 if (!neigh_entry) 1866 return ERR_PTR(-ENOMEM); 1867 1868 err = mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry); 1869 if (err) 1870 goto err_neigh_entry_insert; 1871 1872 mlxsw_sp_neigh_counter_alloc(mlxsw_sp, neigh_entry); 1873 list_add(&neigh_entry->rif_list_node, &rif->neigh_list); 1874 1875 return neigh_entry; 1876 1877 err_neigh_entry_insert: 1878 mlxsw_sp_neigh_entry_free(neigh_entry); 1879 return ERR_PTR(err); 1880 } 1881 1882 static void 1883 mlxsw_sp_neigh_entry_destroy(struct mlxsw_sp *mlxsw_sp, 1884 struct mlxsw_sp_neigh_entry *neigh_entry) 1885 { 1886 list_del(&neigh_entry->rif_list_node); 1887 mlxsw_sp_neigh_counter_free(mlxsw_sp, neigh_entry); 1888 mlxsw_sp_neigh_entry_remove(mlxsw_sp, neigh_entry); 1889 mlxsw_sp_neigh_entry_free(neigh_entry); 1890 } 1891 1892 static struct mlxsw_sp_neigh_entry * 1893 mlxsw_sp_neigh_entry_lookup(struct mlxsw_sp *mlxsw_sp, struct neighbour *n) 1894 { 1895 struct mlxsw_sp_neigh_key key; 1896 1897 key.n = n; 1898 return rhashtable_lookup_fast(&mlxsw_sp->router->neigh_ht, 1899 &key, mlxsw_sp_neigh_ht_params); 1900 } 1901 1902 static void 1903 mlxsw_sp_router_neighs_update_interval_init(struct mlxsw_sp *mlxsw_sp) 1904 { 1905 unsigned long interval; 1906 1907 #if IS_ENABLED(CONFIG_IPV6) 1908 interval = min_t(unsigned long, 1909 NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME), 1910 NEIGH_VAR(&nd_tbl.parms, DELAY_PROBE_TIME)); 1911 #else 1912 interval = NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME); 1913 #endif 1914 mlxsw_sp->router->neighs_update.interval = jiffies_to_msecs(interval); 1915 } 1916 1917 static void mlxsw_sp_router_neigh_ent_ipv4_process(struct mlxsw_sp *mlxsw_sp, 1918 char *rauhtd_pl, 1919 int ent_index) 1920 { 1921 struct net_device *dev; 1922 struct neighbour *n; 1923 __be32 dipn; 1924 u32 dip; 1925 u16 rif; 1926 1927 mlxsw_reg_rauhtd_ent_ipv4_unpack(rauhtd_pl, ent_index, &rif, &dip); 1928 1929 if (!mlxsw_sp->router->rifs[rif]) { 1930 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Incorrect RIF in neighbour entry\n"); 1931 return; 1932 } 1933 1934 dipn = htonl(dip); 1935 dev = mlxsw_sp->router->rifs[rif]->dev; 1936 n = neigh_lookup(&arp_tbl, &dipn, dev); 1937 if (!n) { 1938 netdev_err(dev, "Failed to find matching neighbour for IP=%pI4h\n", 1939 &dip); 1940 return; 1941 } 1942 1943 netdev_dbg(dev, "Updating neighbour with IP=%pI4h\n", &dip); 1944 neigh_event_send(n, NULL); 1945 neigh_release(n); 1946 } 1947 1948 #if IS_ENABLED(CONFIG_IPV6) 1949 static void mlxsw_sp_router_neigh_ent_ipv6_process(struct mlxsw_sp *mlxsw_sp, 1950 char *rauhtd_pl, 1951 int rec_index) 1952 { 1953 struct net_device *dev; 1954 struct neighbour *n; 1955 struct in6_addr dip; 1956 u16 rif; 1957 1958 mlxsw_reg_rauhtd_ent_ipv6_unpack(rauhtd_pl, rec_index, &rif, 1959 (char *) &dip); 1960 1961 if (!mlxsw_sp->router->rifs[rif]) { 1962 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Incorrect RIF in neighbour entry\n"); 1963 return; 1964 } 1965 1966 dev = mlxsw_sp->router->rifs[rif]->dev; 1967 n = neigh_lookup(&nd_tbl, &dip, dev); 1968 if (!n) { 1969 netdev_err(dev, "Failed to find matching neighbour for IP=%pI6c\n", 1970 &dip); 1971 return; 1972 } 1973 1974 netdev_dbg(dev, "Updating neighbour with IP=%pI6c\n", &dip); 1975 neigh_event_send(n, NULL); 1976 neigh_release(n); 1977 } 1978 #else 1979 static void mlxsw_sp_router_neigh_ent_ipv6_process(struct mlxsw_sp *mlxsw_sp, 1980 char *rauhtd_pl, 1981 int rec_index) 1982 { 1983 } 1984 #endif 1985 1986 static void mlxsw_sp_router_neigh_rec_ipv4_process(struct mlxsw_sp *mlxsw_sp, 1987 char *rauhtd_pl, 1988 int rec_index) 1989 { 1990 u8 num_entries; 1991 int i; 1992 1993 num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl, 1994 rec_index); 1995 /* Hardware starts counting at 0, so add 1. */ 1996 num_entries++; 1997 1998 /* Each record consists of several neighbour entries. */ 1999 for (i = 0; i < num_entries; i++) { 2000 int ent_index; 2001 2002 ent_index = rec_index * MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC + i; 2003 mlxsw_sp_router_neigh_ent_ipv4_process(mlxsw_sp, rauhtd_pl, 2004 ent_index); 2005 } 2006 2007 } 2008 2009 static void mlxsw_sp_router_neigh_rec_ipv6_process(struct mlxsw_sp *mlxsw_sp, 2010 char *rauhtd_pl, 2011 int rec_index) 2012 { 2013 /* One record contains one entry. */ 2014 mlxsw_sp_router_neigh_ent_ipv6_process(mlxsw_sp, rauhtd_pl, 2015 rec_index); 2016 } 2017 2018 static void mlxsw_sp_router_neigh_rec_process(struct mlxsw_sp *mlxsw_sp, 2019 char *rauhtd_pl, int rec_index) 2020 { 2021 switch (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, rec_index)) { 2022 case MLXSW_REG_RAUHTD_TYPE_IPV4: 2023 mlxsw_sp_router_neigh_rec_ipv4_process(mlxsw_sp, rauhtd_pl, 2024 rec_index); 2025 break; 2026 case MLXSW_REG_RAUHTD_TYPE_IPV6: 2027 mlxsw_sp_router_neigh_rec_ipv6_process(mlxsw_sp, rauhtd_pl, 2028 rec_index); 2029 break; 2030 } 2031 } 2032 2033 static bool mlxsw_sp_router_rauhtd_is_full(char *rauhtd_pl) 2034 { 2035 u8 num_rec, last_rec_index, num_entries; 2036 2037 num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl); 2038 last_rec_index = num_rec - 1; 2039 2040 if (num_rec < MLXSW_REG_RAUHTD_REC_MAX_NUM) 2041 return false; 2042 if (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, last_rec_index) == 2043 MLXSW_REG_RAUHTD_TYPE_IPV6) 2044 return true; 2045 2046 num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl, 2047 last_rec_index); 2048 if (++num_entries == MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC) 2049 return true; 2050 return false; 2051 } 2052 2053 static int 2054 __mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp, 2055 char *rauhtd_pl, 2056 enum mlxsw_reg_rauhtd_type type) 2057 { 2058 int i, num_rec; 2059 int err; 2060 2061 /* Make sure the neighbour's netdev isn't removed in the 2062 * process. 2063 */ 2064 rtnl_lock(); 2065 do { 2066 mlxsw_reg_rauhtd_pack(rauhtd_pl, type); 2067 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(rauhtd), 2068 rauhtd_pl); 2069 if (err) { 2070 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Failed to dump neighbour table\n"); 2071 break; 2072 } 2073 num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl); 2074 for (i = 0; i < num_rec; i++) 2075 mlxsw_sp_router_neigh_rec_process(mlxsw_sp, rauhtd_pl, 2076 i); 2077 } while (mlxsw_sp_router_rauhtd_is_full(rauhtd_pl)); 2078 rtnl_unlock(); 2079 2080 return err; 2081 } 2082 2083 static int mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp) 2084 { 2085 enum mlxsw_reg_rauhtd_type type; 2086 char *rauhtd_pl; 2087 int err; 2088 2089 rauhtd_pl = kmalloc(MLXSW_REG_RAUHTD_LEN, GFP_KERNEL); 2090 if (!rauhtd_pl) 2091 return -ENOMEM; 2092 2093 type = MLXSW_REG_RAUHTD_TYPE_IPV4; 2094 err = __mlxsw_sp_router_neighs_update_rauhtd(mlxsw_sp, rauhtd_pl, type); 2095 if (err) 2096 goto out; 2097 2098 type = MLXSW_REG_RAUHTD_TYPE_IPV6; 2099 err = __mlxsw_sp_router_neighs_update_rauhtd(mlxsw_sp, rauhtd_pl, type); 2100 out: 2101 kfree(rauhtd_pl); 2102 return err; 2103 } 2104 2105 static void mlxsw_sp_router_neighs_update_nh(struct mlxsw_sp *mlxsw_sp) 2106 { 2107 struct mlxsw_sp_neigh_entry *neigh_entry; 2108 2109 /* Take RTNL mutex here to prevent lists from changes */ 2110 rtnl_lock(); 2111 list_for_each_entry(neigh_entry, &mlxsw_sp->router->nexthop_neighs_list, 2112 nexthop_neighs_list_node) 2113 /* If this neigh have nexthops, make the kernel think this neigh 2114 * is active regardless of the traffic. 2115 */ 2116 neigh_event_send(neigh_entry->key.n, NULL); 2117 rtnl_unlock(); 2118 } 2119 2120 static void 2121 mlxsw_sp_router_neighs_update_work_schedule(struct mlxsw_sp *mlxsw_sp) 2122 { 2123 unsigned long interval = mlxsw_sp->router->neighs_update.interval; 2124 2125 mlxsw_core_schedule_dw(&mlxsw_sp->router->neighs_update.dw, 2126 msecs_to_jiffies(interval)); 2127 } 2128 2129 static void mlxsw_sp_router_neighs_update_work(struct work_struct *work) 2130 { 2131 struct mlxsw_sp_router *router; 2132 int err; 2133 2134 router = container_of(work, struct mlxsw_sp_router, 2135 neighs_update.dw.work); 2136 err = mlxsw_sp_router_neighs_update_rauhtd(router->mlxsw_sp); 2137 if (err) 2138 dev_err(router->mlxsw_sp->bus_info->dev, "Could not update kernel for neigh activity"); 2139 2140 mlxsw_sp_router_neighs_update_nh(router->mlxsw_sp); 2141 2142 mlxsw_sp_router_neighs_update_work_schedule(router->mlxsw_sp); 2143 } 2144 2145 static void mlxsw_sp_router_probe_unresolved_nexthops(struct work_struct *work) 2146 { 2147 struct mlxsw_sp_neigh_entry *neigh_entry; 2148 struct mlxsw_sp_router *router; 2149 2150 router = container_of(work, struct mlxsw_sp_router, 2151 nexthop_probe_dw.work); 2152 /* Iterate over nexthop neighbours, find those who are unresolved and 2153 * send arp on them. This solves the chicken-egg problem when 2154 * the nexthop wouldn't get offloaded until the neighbor is resolved 2155 * but it wouldn't get resolved ever in case traffic is flowing in HW 2156 * using different nexthop. 2157 * 2158 * Take RTNL mutex here to prevent lists from changes. 2159 */ 2160 rtnl_lock(); 2161 list_for_each_entry(neigh_entry, &router->nexthop_neighs_list, 2162 nexthop_neighs_list_node) 2163 if (!neigh_entry->connected) 2164 neigh_event_send(neigh_entry->key.n, NULL); 2165 rtnl_unlock(); 2166 2167 mlxsw_core_schedule_dw(&router->nexthop_probe_dw, 2168 MLXSW_SP_UNRESOLVED_NH_PROBE_INTERVAL); 2169 } 2170 2171 static void 2172 mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp, 2173 struct mlxsw_sp_neigh_entry *neigh_entry, 2174 bool removing); 2175 2176 static enum mlxsw_reg_rauht_op mlxsw_sp_rauht_op(bool adding) 2177 { 2178 return adding ? MLXSW_REG_RAUHT_OP_WRITE_ADD : 2179 MLXSW_REG_RAUHT_OP_WRITE_DELETE; 2180 } 2181 2182 static void 2183 mlxsw_sp_router_neigh_entry_op4(struct mlxsw_sp *mlxsw_sp, 2184 struct mlxsw_sp_neigh_entry *neigh_entry, 2185 enum mlxsw_reg_rauht_op op) 2186 { 2187 struct neighbour *n = neigh_entry->key.n; 2188 u32 dip = ntohl(*((__be32 *) n->primary_key)); 2189 char rauht_pl[MLXSW_REG_RAUHT_LEN]; 2190 2191 mlxsw_reg_rauht_pack4(rauht_pl, op, neigh_entry->rif, neigh_entry->ha, 2192 dip); 2193 if (neigh_entry->counter_valid) 2194 mlxsw_reg_rauht_pack_counter(rauht_pl, 2195 neigh_entry->counter_index); 2196 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl); 2197 } 2198 2199 static void 2200 mlxsw_sp_router_neigh_entry_op6(struct mlxsw_sp *mlxsw_sp, 2201 struct mlxsw_sp_neigh_entry *neigh_entry, 2202 enum mlxsw_reg_rauht_op op) 2203 { 2204 struct neighbour *n = neigh_entry->key.n; 2205 char rauht_pl[MLXSW_REG_RAUHT_LEN]; 2206 const char *dip = n->primary_key; 2207 2208 mlxsw_reg_rauht_pack6(rauht_pl, op, neigh_entry->rif, neigh_entry->ha, 2209 dip); 2210 if (neigh_entry->counter_valid) 2211 mlxsw_reg_rauht_pack_counter(rauht_pl, 2212 neigh_entry->counter_index); 2213 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl); 2214 } 2215 2216 bool mlxsw_sp_neigh_ipv6_ignore(struct mlxsw_sp_neigh_entry *neigh_entry) 2217 { 2218 struct neighbour *n = neigh_entry->key.n; 2219 2220 /* Packets with a link-local destination address are trapped 2221 * after LPM lookup and never reach the neighbour table, so 2222 * there is no need to program such neighbours to the device. 2223 */ 2224 if (ipv6_addr_type((struct in6_addr *) &n->primary_key) & 2225 IPV6_ADDR_LINKLOCAL) 2226 return true; 2227 return false; 2228 } 2229 2230 static void 2231 mlxsw_sp_neigh_entry_update(struct mlxsw_sp *mlxsw_sp, 2232 struct mlxsw_sp_neigh_entry *neigh_entry, 2233 bool adding) 2234 { 2235 if (!adding && !neigh_entry->connected) 2236 return; 2237 neigh_entry->connected = adding; 2238 if (neigh_entry->key.n->tbl->family == AF_INET) { 2239 mlxsw_sp_router_neigh_entry_op4(mlxsw_sp, neigh_entry, 2240 mlxsw_sp_rauht_op(adding)); 2241 } else if (neigh_entry->key.n->tbl->family == AF_INET6) { 2242 if (mlxsw_sp_neigh_ipv6_ignore(neigh_entry)) 2243 return; 2244 mlxsw_sp_router_neigh_entry_op6(mlxsw_sp, neigh_entry, 2245 mlxsw_sp_rauht_op(adding)); 2246 } else { 2247 WARN_ON_ONCE(1); 2248 } 2249 } 2250 2251 void 2252 mlxsw_sp_neigh_entry_counter_update(struct mlxsw_sp *mlxsw_sp, 2253 struct mlxsw_sp_neigh_entry *neigh_entry, 2254 bool adding) 2255 { 2256 if (adding) 2257 mlxsw_sp_neigh_counter_alloc(mlxsw_sp, neigh_entry); 2258 else 2259 mlxsw_sp_neigh_counter_free(mlxsw_sp, neigh_entry); 2260 mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, true); 2261 } 2262 2263 struct mlxsw_sp_netevent_work { 2264 struct work_struct work; 2265 struct mlxsw_sp *mlxsw_sp; 2266 struct neighbour *n; 2267 }; 2268 2269 static void mlxsw_sp_router_neigh_event_work(struct work_struct *work) 2270 { 2271 struct mlxsw_sp_netevent_work *net_work = 2272 container_of(work, struct mlxsw_sp_netevent_work, work); 2273 struct mlxsw_sp *mlxsw_sp = net_work->mlxsw_sp; 2274 struct mlxsw_sp_neigh_entry *neigh_entry; 2275 struct neighbour *n = net_work->n; 2276 unsigned char ha[ETH_ALEN]; 2277 bool entry_connected; 2278 u8 nud_state, dead; 2279 2280 /* If these parameters are changed after we release the lock, 2281 * then we are guaranteed to receive another event letting us 2282 * know about it. 2283 */ 2284 read_lock_bh(&n->lock); 2285 memcpy(ha, n->ha, ETH_ALEN); 2286 nud_state = n->nud_state; 2287 dead = n->dead; 2288 read_unlock_bh(&n->lock); 2289 2290 rtnl_lock(); 2291 entry_connected = nud_state & NUD_VALID && !dead; 2292 neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n); 2293 if (!entry_connected && !neigh_entry) 2294 goto out; 2295 if (!neigh_entry) { 2296 neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n); 2297 if (IS_ERR(neigh_entry)) 2298 goto out; 2299 } 2300 2301 memcpy(neigh_entry->ha, ha, ETH_ALEN); 2302 mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, entry_connected); 2303 mlxsw_sp_nexthop_neigh_update(mlxsw_sp, neigh_entry, !entry_connected); 2304 2305 if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list)) 2306 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry); 2307 2308 out: 2309 rtnl_unlock(); 2310 neigh_release(n); 2311 kfree(net_work); 2312 } 2313 2314 static int mlxsw_sp_mp_hash_init(struct mlxsw_sp *mlxsw_sp); 2315 2316 static void mlxsw_sp_router_mp_hash_event_work(struct work_struct *work) 2317 { 2318 struct mlxsw_sp_netevent_work *net_work = 2319 container_of(work, struct mlxsw_sp_netevent_work, work); 2320 struct mlxsw_sp *mlxsw_sp = net_work->mlxsw_sp; 2321 2322 mlxsw_sp_mp_hash_init(mlxsw_sp); 2323 kfree(net_work); 2324 } 2325 2326 static int mlxsw_sp_router_netevent_event(struct notifier_block *nb, 2327 unsigned long event, void *ptr) 2328 { 2329 struct mlxsw_sp_netevent_work *net_work; 2330 struct mlxsw_sp_port *mlxsw_sp_port; 2331 struct mlxsw_sp_router *router; 2332 struct mlxsw_sp *mlxsw_sp; 2333 unsigned long interval; 2334 struct neigh_parms *p; 2335 struct neighbour *n; 2336 struct net *net; 2337 2338 switch (event) { 2339 case NETEVENT_DELAY_PROBE_TIME_UPDATE: 2340 p = ptr; 2341 2342 /* We don't care about changes in the default table. */ 2343 if (!p->dev || (p->tbl->family != AF_INET && 2344 p->tbl->family != AF_INET6)) 2345 return NOTIFY_DONE; 2346 2347 /* We are in atomic context and can't take RTNL mutex, 2348 * so use RCU variant to walk the device chain. 2349 */ 2350 mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(p->dev); 2351 if (!mlxsw_sp_port) 2352 return NOTIFY_DONE; 2353 2354 mlxsw_sp = mlxsw_sp_port->mlxsw_sp; 2355 interval = jiffies_to_msecs(NEIGH_VAR(p, DELAY_PROBE_TIME)); 2356 mlxsw_sp->router->neighs_update.interval = interval; 2357 2358 mlxsw_sp_port_dev_put(mlxsw_sp_port); 2359 break; 2360 case NETEVENT_NEIGH_UPDATE: 2361 n = ptr; 2362 2363 if (n->tbl->family != AF_INET && n->tbl->family != AF_INET6) 2364 return NOTIFY_DONE; 2365 2366 mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(n->dev); 2367 if (!mlxsw_sp_port) 2368 return NOTIFY_DONE; 2369 2370 net_work = kzalloc(sizeof(*net_work), GFP_ATOMIC); 2371 if (!net_work) { 2372 mlxsw_sp_port_dev_put(mlxsw_sp_port); 2373 return NOTIFY_BAD; 2374 } 2375 2376 INIT_WORK(&net_work->work, mlxsw_sp_router_neigh_event_work); 2377 net_work->mlxsw_sp = mlxsw_sp_port->mlxsw_sp; 2378 net_work->n = n; 2379 2380 /* Take a reference to ensure the neighbour won't be 2381 * destructed until we drop the reference in delayed 2382 * work. 2383 */ 2384 neigh_clone(n); 2385 mlxsw_core_schedule_work(&net_work->work); 2386 mlxsw_sp_port_dev_put(mlxsw_sp_port); 2387 break; 2388 case NETEVENT_MULTIPATH_HASH_UPDATE: 2389 net = ptr; 2390 2391 if (!net_eq(net, &init_net)) 2392 return NOTIFY_DONE; 2393 2394 net_work = kzalloc(sizeof(*net_work), GFP_ATOMIC); 2395 if (!net_work) 2396 return NOTIFY_BAD; 2397 2398 router = container_of(nb, struct mlxsw_sp_router, netevent_nb); 2399 INIT_WORK(&net_work->work, mlxsw_sp_router_mp_hash_event_work); 2400 net_work->mlxsw_sp = router->mlxsw_sp; 2401 mlxsw_core_schedule_work(&net_work->work); 2402 break; 2403 } 2404 2405 return NOTIFY_DONE; 2406 } 2407 2408 static int mlxsw_sp_neigh_init(struct mlxsw_sp *mlxsw_sp) 2409 { 2410 int err; 2411 2412 err = rhashtable_init(&mlxsw_sp->router->neigh_ht, 2413 &mlxsw_sp_neigh_ht_params); 2414 if (err) 2415 return err; 2416 2417 /* Initialize the polling interval according to the default 2418 * table. 2419 */ 2420 mlxsw_sp_router_neighs_update_interval_init(mlxsw_sp); 2421 2422 /* Create the delayed works for the activity_update */ 2423 INIT_DELAYED_WORK(&mlxsw_sp->router->neighs_update.dw, 2424 mlxsw_sp_router_neighs_update_work); 2425 INIT_DELAYED_WORK(&mlxsw_sp->router->nexthop_probe_dw, 2426 mlxsw_sp_router_probe_unresolved_nexthops); 2427 mlxsw_core_schedule_dw(&mlxsw_sp->router->neighs_update.dw, 0); 2428 mlxsw_core_schedule_dw(&mlxsw_sp->router->nexthop_probe_dw, 0); 2429 return 0; 2430 } 2431 2432 static void mlxsw_sp_neigh_fini(struct mlxsw_sp *mlxsw_sp) 2433 { 2434 cancel_delayed_work_sync(&mlxsw_sp->router->neighs_update.dw); 2435 cancel_delayed_work_sync(&mlxsw_sp->router->nexthop_probe_dw); 2436 rhashtable_destroy(&mlxsw_sp->router->neigh_ht); 2437 } 2438 2439 static int mlxsw_sp_neigh_rif_flush(struct mlxsw_sp *mlxsw_sp, 2440 const struct mlxsw_sp_rif *rif) 2441 { 2442 char rauht_pl[MLXSW_REG_RAUHT_LEN]; 2443 2444 mlxsw_reg_rauht_pack(rauht_pl, MLXSW_REG_RAUHT_OP_WRITE_DELETE_ALL, 2445 rif->rif_index, rif->addr); 2446 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl); 2447 } 2448 2449 static void mlxsw_sp_neigh_rif_gone_sync(struct mlxsw_sp *mlxsw_sp, 2450 struct mlxsw_sp_rif *rif) 2451 { 2452 struct mlxsw_sp_neigh_entry *neigh_entry, *tmp; 2453 2454 mlxsw_sp_neigh_rif_flush(mlxsw_sp, rif); 2455 list_for_each_entry_safe(neigh_entry, tmp, &rif->neigh_list, 2456 rif_list_node) 2457 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry); 2458 } 2459 2460 enum mlxsw_sp_nexthop_type { 2461 MLXSW_SP_NEXTHOP_TYPE_ETH, 2462 MLXSW_SP_NEXTHOP_TYPE_IPIP, 2463 }; 2464 2465 struct mlxsw_sp_nexthop_key { 2466 struct fib_nh *fib_nh; 2467 }; 2468 2469 struct mlxsw_sp_nexthop { 2470 struct list_head neigh_list_node; /* member of neigh entry list */ 2471 struct list_head rif_list_node; 2472 struct list_head router_list_node; 2473 struct mlxsw_sp_nexthop_group *nh_grp; /* pointer back to the group 2474 * this belongs to 2475 */ 2476 struct rhash_head ht_node; 2477 struct mlxsw_sp_nexthop_key key; 2478 unsigned char gw_addr[sizeof(struct in6_addr)]; 2479 int ifindex; 2480 int nh_weight; 2481 int norm_nh_weight; 2482 int num_adj_entries; 2483 struct mlxsw_sp_rif *rif; 2484 u8 should_offload:1, /* set indicates this neigh is connected and 2485 * should be put to KVD linear area of this group. 2486 */ 2487 offloaded:1, /* set in case the neigh is actually put into 2488 * KVD linear area of this group. 2489 */ 2490 update:1; /* set indicates that MAC of this neigh should be 2491 * updated in HW 2492 */ 2493 enum mlxsw_sp_nexthop_type type; 2494 union { 2495 struct mlxsw_sp_neigh_entry *neigh_entry; 2496 struct mlxsw_sp_ipip_entry *ipip_entry; 2497 }; 2498 unsigned int counter_index; 2499 bool counter_valid; 2500 }; 2501 2502 struct mlxsw_sp_nexthop_group { 2503 void *priv; 2504 struct rhash_head ht_node; 2505 struct list_head fib_list; /* list of fib entries that use this group */ 2506 struct neigh_table *neigh_tbl; 2507 u8 adj_index_valid:1, 2508 gateway:1; /* routes using the group use a gateway */ 2509 u32 adj_index; 2510 u16 ecmp_size; 2511 u16 count; 2512 int sum_norm_weight; 2513 struct mlxsw_sp_nexthop nexthops[0]; 2514 #define nh_rif nexthops[0].rif 2515 }; 2516 2517 void mlxsw_sp_nexthop_counter_alloc(struct mlxsw_sp *mlxsw_sp, 2518 struct mlxsw_sp_nexthop *nh) 2519 { 2520 struct devlink *devlink; 2521 2522 devlink = priv_to_devlink(mlxsw_sp->core); 2523 if (!devlink_dpipe_table_counter_enabled(devlink, 2524 MLXSW_SP_DPIPE_TABLE_NAME_ADJ)) 2525 return; 2526 2527 if (mlxsw_sp_flow_counter_alloc(mlxsw_sp, &nh->counter_index)) 2528 return; 2529 2530 nh->counter_valid = true; 2531 } 2532 2533 void mlxsw_sp_nexthop_counter_free(struct mlxsw_sp *mlxsw_sp, 2534 struct mlxsw_sp_nexthop *nh) 2535 { 2536 if (!nh->counter_valid) 2537 return; 2538 mlxsw_sp_flow_counter_free(mlxsw_sp, nh->counter_index); 2539 nh->counter_valid = false; 2540 } 2541 2542 int mlxsw_sp_nexthop_counter_get(struct mlxsw_sp *mlxsw_sp, 2543 struct mlxsw_sp_nexthop *nh, u64 *p_counter) 2544 { 2545 if (!nh->counter_valid) 2546 return -EINVAL; 2547 2548 return mlxsw_sp_flow_counter_get(mlxsw_sp, nh->counter_index, 2549 p_counter, NULL); 2550 } 2551 2552 struct mlxsw_sp_nexthop *mlxsw_sp_nexthop_next(struct mlxsw_sp_router *router, 2553 struct mlxsw_sp_nexthop *nh) 2554 { 2555 if (!nh) { 2556 if (list_empty(&router->nexthop_list)) 2557 return NULL; 2558 else 2559 return list_first_entry(&router->nexthop_list, 2560 typeof(*nh), router_list_node); 2561 } 2562 if (list_is_last(&nh->router_list_node, &router->nexthop_list)) 2563 return NULL; 2564 return list_next_entry(nh, router_list_node); 2565 } 2566 2567 bool mlxsw_sp_nexthop_offload(struct mlxsw_sp_nexthop *nh) 2568 { 2569 return nh->offloaded; 2570 } 2571 2572 unsigned char *mlxsw_sp_nexthop_ha(struct mlxsw_sp_nexthop *nh) 2573 { 2574 if (!nh->offloaded) 2575 return NULL; 2576 return nh->neigh_entry->ha; 2577 } 2578 2579 int mlxsw_sp_nexthop_indexes(struct mlxsw_sp_nexthop *nh, u32 *p_adj_index, 2580 u32 *p_adj_size, u32 *p_adj_hash_index) 2581 { 2582 struct mlxsw_sp_nexthop_group *nh_grp = nh->nh_grp; 2583 u32 adj_hash_index = 0; 2584 int i; 2585 2586 if (!nh->offloaded || !nh_grp->adj_index_valid) 2587 return -EINVAL; 2588 2589 *p_adj_index = nh_grp->adj_index; 2590 *p_adj_size = nh_grp->ecmp_size; 2591 2592 for (i = 0; i < nh_grp->count; i++) { 2593 struct mlxsw_sp_nexthop *nh_iter = &nh_grp->nexthops[i]; 2594 2595 if (nh_iter == nh) 2596 break; 2597 if (nh_iter->offloaded) 2598 adj_hash_index += nh_iter->num_adj_entries; 2599 } 2600 2601 *p_adj_hash_index = adj_hash_index; 2602 return 0; 2603 } 2604 2605 struct mlxsw_sp_rif *mlxsw_sp_nexthop_rif(struct mlxsw_sp_nexthop *nh) 2606 { 2607 return nh->rif; 2608 } 2609 2610 bool mlxsw_sp_nexthop_group_has_ipip(struct mlxsw_sp_nexthop *nh) 2611 { 2612 struct mlxsw_sp_nexthop_group *nh_grp = nh->nh_grp; 2613 int i; 2614 2615 for (i = 0; i < nh_grp->count; i++) { 2616 struct mlxsw_sp_nexthop *nh_iter = &nh_grp->nexthops[i]; 2617 2618 if (nh_iter->type == MLXSW_SP_NEXTHOP_TYPE_IPIP) 2619 return true; 2620 } 2621 return false; 2622 } 2623 2624 static struct fib_info * 2625 mlxsw_sp_nexthop4_group_fi(const struct mlxsw_sp_nexthop_group *nh_grp) 2626 { 2627 return nh_grp->priv; 2628 } 2629 2630 struct mlxsw_sp_nexthop_group_cmp_arg { 2631 enum mlxsw_sp_l3proto proto; 2632 union { 2633 struct fib_info *fi; 2634 struct mlxsw_sp_fib6_entry *fib6_entry; 2635 }; 2636 }; 2637 2638 static bool 2639 mlxsw_sp_nexthop6_group_has_nexthop(const struct mlxsw_sp_nexthop_group *nh_grp, 2640 const struct in6_addr *gw, int ifindex) 2641 { 2642 int i; 2643 2644 for (i = 0; i < nh_grp->count; i++) { 2645 const struct mlxsw_sp_nexthop *nh; 2646 2647 nh = &nh_grp->nexthops[i]; 2648 if (nh->ifindex == ifindex && 2649 ipv6_addr_equal(gw, (struct in6_addr *) nh->gw_addr)) 2650 return true; 2651 } 2652 2653 return false; 2654 } 2655 2656 static bool 2657 mlxsw_sp_nexthop6_group_cmp(const struct mlxsw_sp_nexthop_group *nh_grp, 2658 const struct mlxsw_sp_fib6_entry *fib6_entry) 2659 { 2660 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 2661 2662 if (nh_grp->count != fib6_entry->nrt6) 2663 return false; 2664 2665 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) { 2666 struct in6_addr *gw; 2667 int ifindex; 2668 2669 ifindex = mlxsw_sp_rt6->rt->dst.dev->ifindex; 2670 gw = &mlxsw_sp_rt6->rt->rt6i_gateway; 2671 if (!mlxsw_sp_nexthop6_group_has_nexthop(nh_grp, gw, ifindex)) 2672 return false; 2673 } 2674 2675 return true; 2676 } 2677 2678 static int 2679 mlxsw_sp_nexthop_group_cmp(struct rhashtable_compare_arg *arg, const void *ptr) 2680 { 2681 const struct mlxsw_sp_nexthop_group_cmp_arg *cmp_arg = arg->key; 2682 const struct mlxsw_sp_nexthop_group *nh_grp = ptr; 2683 2684 switch (cmp_arg->proto) { 2685 case MLXSW_SP_L3_PROTO_IPV4: 2686 return cmp_arg->fi != mlxsw_sp_nexthop4_group_fi(nh_grp); 2687 case MLXSW_SP_L3_PROTO_IPV6: 2688 return !mlxsw_sp_nexthop6_group_cmp(nh_grp, 2689 cmp_arg->fib6_entry); 2690 default: 2691 WARN_ON(1); 2692 return 1; 2693 } 2694 } 2695 2696 static int 2697 mlxsw_sp_nexthop_group_type(const struct mlxsw_sp_nexthop_group *nh_grp) 2698 { 2699 return nh_grp->neigh_tbl->family; 2700 } 2701 2702 static u32 mlxsw_sp_nexthop_group_hash_obj(const void *data, u32 len, u32 seed) 2703 { 2704 const struct mlxsw_sp_nexthop_group *nh_grp = data; 2705 const struct mlxsw_sp_nexthop *nh; 2706 struct fib_info *fi; 2707 unsigned int val; 2708 int i; 2709 2710 switch (mlxsw_sp_nexthop_group_type(nh_grp)) { 2711 case AF_INET: 2712 fi = mlxsw_sp_nexthop4_group_fi(nh_grp); 2713 return jhash(&fi, sizeof(fi), seed); 2714 case AF_INET6: 2715 val = nh_grp->count; 2716 for (i = 0; i < nh_grp->count; i++) { 2717 nh = &nh_grp->nexthops[i]; 2718 val ^= nh->ifindex; 2719 } 2720 return jhash(&val, sizeof(val), seed); 2721 default: 2722 WARN_ON(1); 2723 return 0; 2724 } 2725 } 2726 2727 static u32 2728 mlxsw_sp_nexthop6_group_hash(struct mlxsw_sp_fib6_entry *fib6_entry, u32 seed) 2729 { 2730 unsigned int val = fib6_entry->nrt6; 2731 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 2732 struct net_device *dev; 2733 2734 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) { 2735 dev = mlxsw_sp_rt6->rt->dst.dev; 2736 val ^= dev->ifindex; 2737 } 2738 2739 return jhash(&val, sizeof(val), seed); 2740 } 2741 2742 static u32 2743 mlxsw_sp_nexthop_group_hash(const void *data, u32 len, u32 seed) 2744 { 2745 const struct mlxsw_sp_nexthop_group_cmp_arg *cmp_arg = data; 2746 2747 switch (cmp_arg->proto) { 2748 case MLXSW_SP_L3_PROTO_IPV4: 2749 return jhash(&cmp_arg->fi, sizeof(cmp_arg->fi), seed); 2750 case MLXSW_SP_L3_PROTO_IPV6: 2751 return mlxsw_sp_nexthop6_group_hash(cmp_arg->fib6_entry, seed); 2752 default: 2753 WARN_ON(1); 2754 return 0; 2755 } 2756 } 2757 2758 static const struct rhashtable_params mlxsw_sp_nexthop_group_ht_params = { 2759 .head_offset = offsetof(struct mlxsw_sp_nexthop_group, ht_node), 2760 .hashfn = mlxsw_sp_nexthop_group_hash, 2761 .obj_hashfn = mlxsw_sp_nexthop_group_hash_obj, 2762 .obj_cmpfn = mlxsw_sp_nexthop_group_cmp, 2763 }; 2764 2765 static int mlxsw_sp_nexthop_group_insert(struct mlxsw_sp *mlxsw_sp, 2766 struct mlxsw_sp_nexthop_group *nh_grp) 2767 { 2768 if (mlxsw_sp_nexthop_group_type(nh_grp) == AF_INET6 && 2769 !nh_grp->gateway) 2770 return 0; 2771 2772 return rhashtable_insert_fast(&mlxsw_sp->router->nexthop_group_ht, 2773 &nh_grp->ht_node, 2774 mlxsw_sp_nexthop_group_ht_params); 2775 } 2776 2777 static void mlxsw_sp_nexthop_group_remove(struct mlxsw_sp *mlxsw_sp, 2778 struct mlxsw_sp_nexthop_group *nh_grp) 2779 { 2780 if (mlxsw_sp_nexthop_group_type(nh_grp) == AF_INET6 && 2781 !nh_grp->gateway) 2782 return; 2783 2784 rhashtable_remove_fast(&mlxsw_sp->router->nexthop_group_ht, 2785 &nh_grp->ht_node, 2786 mlxsw_sp_nexthop_group_ht_params); 2787 } 2788 2789 static struct mlxsw_sp_nexthop_group * 2790 mlxsw_sp_nexthop4_group_lookup(struct mlxsw_sp *mlxsw_sp, 2791 struct fib_info *fi) 2792 { 2793 struct mlxsw_sp_nexthop_group_cmp_arg cmp_arg; 2794 2795 cmp_arg.proto = MLXSW_SP_L3_PROTO_IPV4; 2796 cmp_arg.fi = fi; 2797 return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_group_ht, 2798 &cmp_arg, 2799 mlxsw_sp_nexthop_group_ht_params); 2800 } 2801 2802 static struct mlxsw_sp_nexthop_group * 2803 mlxsw_sp_nexthop6_group_lookup(struct mlxsw_sp *mlxsw_sp, 2804 struct mlxsw_sp_fib6_entry *fib6_entry) 2805 { 2806 struct mlxsw_sp_nexthop_group_cmp_arg cmp_arg; 2807 2808 cmp_arg.proto = MLXSW_SP_L3_PROTO_IPV6; 2809 cmp_arg.fib6_entry = fib6_entry; 2810 return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_group_ht, 2811 &cmp_arg, 2812 mlxsw_sp_nexthop_group_ht_params); 2813 } 2814 2815 static const struct rhashtable_params mlxsw_sp_nexthop_ht_params = { 2816 .key_offset = offsetof(struct mlxsw_sp_nexthop, key), 2817 .head_offset = offsetof(struct mlxsw_sp_nexthop, ht_node), 2818 .key_len = sizeof(struct mlxsw_sp_nexthop_key), 2819 }; 2820 2821 static int mlxsw_sp_nexthop_insert(struct mlxsw_sp *mlxsw_sp, 2822 struct mlxsw_sp_nexthop *nh) 2823 { 2824 return rhashtable_insert_fast(&mlxsw_sp->router->nexthop_ht, 2825 &nh->ht_node, mlxsw_sp_nexthop_ht_params); 2826 } 2827 2828 static void mlxsw_sp_nexthop_remove(struct mlxsw_sp *mlxsw_sp, 2829 struct mlxsw_sp_nexthop *nh) 2830 { 2831 rhashtable_remove_fast(&mlxsw_sp->router->nexthop_ht, &nh->ht_node, 2832 mlxsw_sp_nexthop_ht_params); 2833 } 2834 2835 static struct mlxsw_sp_nexthop * 2836 mlxsw_sp_nexthop_lookup(struct mlxsw_sp *mlxsw_sp, 2837 struct mlxsw_sp_nexthop_key key) 2838 { 2839 return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_ht, &key, 2840 mlxsw_sp_nexthop_ht_params); 2841 } 2842 2843 static int mlxsw_sp_adj_index_mass_update_vr(struct mlxsw_sp *mlxsw_sp, 2844 const struct mlxsw_sp_fib *fib, 2845 u32 adj_index, u16 ecmp_size, 2846 u32 new_adj_index, 2847 u16 new_ecmp_size) 2848 { 2849 char raleu_pl[MLXSW_REG_RALEU_LEN]; 2850 2851 mlxsw_reg_raleu_pack(raleu_pl, 2852 (enum mlxsw_reg_ralxx_protocol) fib->proto, 2853 fib->vr->id, adj_index, ecmp_size, new_adj_index, 2854 new_ecmp_size); 2855 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raleu), raleu_pl); 2856 } 2857 2858 static int mlxsw_sp_adj_index_mass_update(struct mlxsw_sp *mlxsw_sp, 2859 struct mlxsw_sp_nexthop_group *nh_grp, 2860 u32 old_adj_index, u16 old_ecmp_size) 2861 { 2862 struct mlxsw_sp_fib_entry *fib_entry; 2863 struct mlxsw_sp_fib *fib = NULL; 2864 int err; 2865 2866 list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) { 2867 if (fib == fib_entry->fib_node->fib) 2868 continue; 2869 fib = fib_entry->fib_node->fib; 2870 err = mlxsw_sp_adj_index_mass_update_vr(mlxsw_sp, fib, 2871 old_adj_index, 2872 old_ecmp_size, 2873 nh_grp->adj_index, 2874 nh_grp->ecmp_size); 2875 if (err) 2876 return err; 2877 } 2878 return 0; 2879 } 2880 2881 static int __mlxsw_sp_nexthop_update(struct mlxsw_sp *mlxsw_sp, u32 adj_index, 2882 struct mlxsw_sp_nexthop *nh) 2883 { 2884 struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry; 2885 char ratr_pl[MLXSW_REG_RATR_LEN]; 2886 2887 mlxsw_reg_ratr_pack(ratr_pl, MLXSW_REG_RATR_OP_WRITE_WRITE_ENTRY, 2888 true, MLXSW_REG_RATR_TYPE_ETHERNET, 2889 adj_index, neigh_entry->rif); 2890 mlxsw_reg_ratr_eth_entry_pack(ratr_pl, neigh_entry->ha); 2891 if (nh->counter_valid) 2892 mlxsw_reg_ratr_counter_pack(ratr_pl, nh->counter_index, true); 2893 else 2894 mlxsw_reg_ratr_counter_pack(ratr_pl, 0, false); 2895 2896 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ratr), ratr_pl); 2897 } 2898 2899 int mlxsw_sp_nexthop_update(struct mlxsw_sp *mlxsw_sp, u32 adj_index, 2900 struct mlxsw_sp_nexthop *nh) 2901 { 2902 int i; 2903 2904 for (i = 0; i < nh->num_adj_entries; i++) { 2905 int err; 2906 2907 err = __mlxsw_sp_nexthop_update(mlxsw_sp, adj_index + i, nh); 2908 if (err) 2909 return err; 2910 } 2911 2912 return 0; 2913 } 2914 2915 static int __mlxsw_sp_nexthop_ipip_update(struct mlxsw_sp *mlxsw_sp, 2916 u32 adj_index, 2917 struct mlxsw_sp_nexthop *nh) 2918 { 2919 const struct mlxsw_sp_ipip_ops *ipip_ops; 2920 2921 ipip_ops = mlxsw_sp->router->ipip_ops_arr[nh->ipip_entry->ipipt]; 2922 return ipip_ops->nexthop_update(mlxsw_sp, adj_index, nh->ipip_entry); 2923 } 2924 2925 static int mlxsw_sp_nexthop_ipip_update(struct mlxsw_sp *mlxsw_sp, 2926 u32 adj_index, 2927 struct mlxsw_sp_nexthop *nh) 2928 { 2929 int i; 2930 2931 for (i = 0; i < nh->num_adj_entries; i++) { 2932 int err; 2933 2934 err = __mlxsw_sp_nexthop_ipip_update(mlxsw_sp, adj_index + i, 2935 nh); 2936 if (err) 2937 return err; 2938 } 2939 2940 return 0; 2941 } 2942 2943 static int 2944 mlxsw_sp_nexthop_group_update(struct mlxsw_sp *mlxsw_sp, 2945 struct mlxsw_sp_nexthop_group *nh_grp, 2946 bool reallocate) 2947 { 2948 u32 adj_index = nh_grp->adj_index; /* base */ 2949 struct mlxsw_sp_nexthop *nh; 2950 int i; 2951 int err; 2952 2953 for (i = 0; i < nh_grp->count; i++) { 2954 nh = &nh_grp->nexthops[i]; 2955 2956 if (!nh->should_offload) { 2957 nh->offloaded = 0; 2958 continue; 2959 } 2960 2961 if (nh->update || reallocate) { 2962 switch (nh->type) { 2963 case MLXSW_SP_NEXTHOP_TYPE_ETH: 2964 err = mlxsw_sp_nexthop_update 2965 (mlxsw_sp, adj_index, nh); 2966 break; 2967 case MLXSW_SP_NEXTHOP_TYPE_IPIP: 2968 err = mlxsw_sp_nexthop_ipip_update 2969 (mlxsw_sp, adj_index, nh); 2970 break; 2971 } 2972 if (err) 2973 return err; 2974 nh->update = 0; 2975 nh->offloaded = 1; 2976 } 2977 adj_index += nh->num_adj_entries; 2978 } 2979 return 0; 2980 } 2981 2982 static bool 2983 mlxsw_sp_fib_node_entry_is_first(const struct mlxsw_sp_fib_node *fib_node, 2984 const struct mlxsw_sp_fib_entry *fib_entry); 2985 2986 static int 2987 mlxsw_sp_nexthop_fib_entries_update(struct mlxsw_sp *mlxsw_sp, 2988 struct mlxsw_sp_nexthop_group *nh_grp) 2989 { 2990 struct mlxsw_sp_fib_entry *fib_entry; 2991 int err; 2992 2993 list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) { 2994 if (!mlxsw_sp_fib_node_entry_is_first(fib_entry->fib_node, 2995 fib_entry)) 2996 continue; 2997 err = mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry); 2998 if (err) 2999 return err; 3000 } 3001 return 0; 3002 } 3003 3004 static void 3005 mlxsw_sp_fib_entry_offload_refresh(struct mlxsw_sp_fib_entry *fib_entry, 3006 enum mlxsw_reg_ralue_op op, int err); 3007 3008 static void 3009 mlxsw_sp_nexthop_fib_entries_refresh(struct mlxsw_sp_nexthop_group *nh_grp) 3010 { 3011 enum mlxsw_reg_ralue_op op = MLXSW_REG_RALUE_OP_WRITE_WRITE; 3012 struct mlxsw_sp_fib_entry *fib_entry; 3013 3014 list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) { 3015 if (!mlxsw_sp_fib_node_entry_is_first(fib_entry->fib_node, 3016 fib_entry)) 3017 continue; 3018 mlxsw_sp_fib_entry_offload_refresh(fib_entry, op, 0); 3019 } 3020 } 3021 3022 static void mlxsw_sp_adj_grp_size_round_up(u16 *p_adj_grp_size) 3023 { 3024 /* Valid sizes for an adjacency group are: 3025 * 1-64, 512, 1024, 2048 and 4096. 3026 */ 3027 if (*p_adj_grp_size <= 64) 3028 return; 3029 else if (*p_adj_grp_size <= 512) 3030 *p_adj_grp_size = 512; 3031 else if (*p_adj_grp_size <= 1024) 3032 *p_adj_grp_size = 1024; 3033 else if (*p_adj_grp_size <= 2048) 3034 *p_adj_grp_size = 2048; 3035 else 3036 *p_adj_grp_size = 4096; 3037 } 3038 3039 static void mlxsw_sp_adj_grp_size_round_down(u16 *p_adj_grp_size, 3040 unsigned int alloc_size) 3041 { 3042 if (alloc_size >= 4096) 3043 *p_adj_grp_size = 4096; 3044 else if (alloc_size >= 2048) 3045 *p_adj_grp_size = 2048; 3046 else if (alloc_size >= 1024) 3047 *p_adj_grp_size = 1024; 3048 else if (alloc_size >= 512) 3049 *p_adj_grp_size = 512; 3050 } 3051 3052 static int mlxsw_sp_fix_adj_grp_size(struct mlxsw_sp *mlxsw_sp, 3053 u16 *p_adj_grp_size) 3054 { 3055 unsigned int alloc_size; 3056 int err; 3057 3058 /* Round up the requested group size to the next size supported 3059 * by the device and make sure the request can be satisfied. 3060 */ 3061 mlxsw_sp_adj_grp_size_round_up(p_adj_grp_size); 3062 err = mlxsw_sp_kvdl_alloc_size_query(mlxsw_sp, *p_adj_grp_size, 3063 &alloc_size); 3064 if (err) 3065 return err; 3066 /* It is possible the allocation results in more allocated 3067 * entries than requested. Try to use as much of them as 3068 * possible. 3069 */ 3070 mlxsw_sp_adj_grp_size_round_down(p_adj_grp_size, alloc_size); 3071 3072 return 0; 3073 } 3074 3075 static void 3076 mlxsw_sp_nexthop_group_normalize(struct mlxsw_sp_nexthop_group *nh_grp) 3077 { 3078 int i, g = 0, sum_norm_weight = 0; 3079 struct mlxsw_sp_nexthop *nh; 3080 3081 for (i = 0; i < nh_grp->count; i++) { 3082 nh = &nh_grp->nexthops[i]; 3083 3084 if (!nh->should_offload) 3085 continue; 3086 if (g > 0) 3087 g = gcd(nh->nh_weight, g); 3088 else 3089 g = nh->nh_weight; 3090 } 3091 3092 for (i = 0; i < nh_grp->count; i++) { 3093 nh = &nh_grp->nexthops[i]; 3094 3095 if (!nh->should_offload) 3096 continue; 3097 nh->norm_nh_weight = nh->nh_weight / g; 3098 sum_norm_weight += nh->norm_nh_weight; 3099 } 3100 3101 nh_grp->sum_norm_weight = sum_norm_weight; 3102 } 3103 3104 static void 3105 mlxsw_sp_nexthop_group_rebalance(struct mlxsw_sp_nexthop_group *nh_grp) 3106 { 3107 int total = nh_grp->sum_norm_weight; 3108 u16 ecmp_size = nh_grp->ecmp_size; 3109 int i, weight = 0, lower_bound = 0; 3110 3111 for (i = 0; i < nh_grp->count; i++) { 3112 struct mlxsw_sp_nexthop *nh = &nh_grp->nexthops[i]; 3113 int upper_bound; 3114 3115 if (!nh->should_offload) 3116 continue; 3117 weight += nh->norm_nh_weight; 3118 upper_bound = DIV_ROUND_CLOSEST(ecmp_size * weight, total); 3119 nh->num_adj_entries = upper_bound - lower_bound; 3120 lower_bound = upper_bound; 3121 } 3122 } 3123 3124 static void 3125 mlxsw_sp_nexthop_group_refresh(struct mlxsw_sp *mlxsw_sp, 3126 struct mlxsw_sp_nexthop_group *nh_grp) 3127 { 3128 u16 ecmp_size, old_ecmp_size; 3129 struct mlxsw_sp_nexthop *nh; 3130 bool offload_change = false; 3131 u32 adj_index; 3132 bool old_adj_index_valid; 3133 u32 old_adj_index; 3134 int i; 3135 int err; 3136 3137 if (!nh_grp->gateway) { 3138 mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp); 3139 return; 3140 } 3141 3142 for (i = 0; i < nh_grp->count; i++) { 3143 nh = &nh_grp->nexthops[i]; 3144 3145 if (nh->should_offload != nh->offloaded) { 3146 offload_change = true; 3147 if (nh->should_offload) 3148 nh->update = 1; 3149 } 3150 } 3151 if (!offload_change) { 3152 /* Nothing was added or removed, so no need to reallocate. Just 3153 * update MAC on existing adjacency indexes. 3154 */ 3155 err = mlxsw_sp_nexthop_group_update(mlxsw_sp, nh_grp, false); 3156 if (err) { 3157 dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n"); 3158 goto set_trap; 3159 } 3160 return; 3161 } 3162 mlxsw_sp_nexthop_group_normalize(nh_grp); 3163 if (!nh_grp->sum_norm_weight) 3164 /* No neigh of this group is connected so we just set 3165 * the trap and let everthing flow through kernel. 3166 */ 3167 goto set_trap; 3168 3169 ecmp_size = nh_grp->sum_norm_weight; 3170 err = mlxsw_sp_fix_adj_grp_size(mlxsw_sp, &ecmp_size); 3171 if (err) 3172 /* No valid allocation size available. */ 3173 goto set_trap; 3174 3175 err = mlxsw_sp_kvdl_alloc(mlxsw_sp, ecmp_size, &adj_index); 3176 if (err) { 3177 /* We ran out of KVD linear space, just set the 3178 * trap and let everything flow through kernel. 3179 */ 3180 dev_warn(mlxsw_sp->bus_info->dev, "Failed to allocate KVD linear area for nexthop group.\n"); 3181 goto set_trap; 3182 } 3183 old_adj_index_valid = nh_grp->adj_index_valid; 3184 old_adj_index = nh_grp->adj_index; 3185 old_ecmp_size = nh_grp->ecmp_size; 3186 nh_grp->adj_index_valid = 1; 3187 nh_grp->adj_index = adj_index; 3188 nh_grp->ecmp_size = ecmp_size; 3189 mlxsw_sp_nexthop_group_rebalance(nh_grp); 3190 err = mlxsw_sp_nexthop_group_update(mlxsw_sp, nh_grp, true); 3191 if (err) { 3192 dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n"); 3193 goto set_trap; 3194 } 3195 3196 if (!old_adj_index_valid) { 3197 /* The trap was set for fib entries, so we have to call 3198 * fib entry update to unset it and use adjacency index. 3199 */ 3200 err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp); 3201 if (err) { 3202 dev_warn(mlxsw_sp->bus_info->dev, "Failed to add adjacency index to fib entries.\n"); 3203 goto set_trap; 3204 } 3205 return; 3206 } 3207 3208 err = mlxsw_sp_adj_index_mass_update(mlxsw_sp, nh_grp, 3209 old_adj_index, old_ecmp_size); 3210 mlxsw_sp_kvdl_free(mlxsw_sp, old_adj_index); 3211 if (err) { 3212 dev_warn(mlxsw_sp->bus_info->dev, "Failed to mass-update adjacency index for nexthop group.\n"); 3213 goto set_trap; 3214 } 3215 3216 /* Offload state within the group changed, so update the flags. */ 3217 mlxsw_sp_nexthop_fib_entries_refresh(nh_grp); 3218 3219 return; 3220 3221 set_trap: 3222 old_adj_index_valid = nh_grp->adj_index_valid; 3223 nh_grp->adj_index_valid = 0; 3224 for (i = 0; i < nh_grp->count; i++) { 3225 nh = &nh_grp->nexthops[i]; 3226 nh->offloaded = 0; 3227 } 3228 err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp); 3229 if (err) 3230 dev_warn(mlxsw_sp->bus_info->dev, "Failed to set traps for fib entries.\n"); 3231 if (old_adj_index_valid) 3232 mlxsw_sp_kvdl_free(mlxsw_sp, nh_grp->adj_index); 3233 } 3234 3235 static void __mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp_nexthop *nh, 3236 bool removing) 3237 { 3238 if (!removing) 3239 nh->should_offload = 1; 3240 else if (nh->offloaded) 3241 nh->should_offload = 0; 3242 nh->update = 1; 3243 } 3244 3245 static void 3246 mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp, 3247 struct mlxsw_sp_neigh_entry *neigh_entry, 3248 bool removing) 3249 { 3250 struct mlxsw_sp_nexthop *nh; 3251 3252 list_for_each_entry(nh, &neigh_entry->nexthop_list, 3253 neigh_list_node) { 3254 __mlxsw_sp_nexthop_neigh_update(nh, removing); 3255 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp); 3256 } 3257 } 3258 3259 static void mlxsw_sp_nexthop_rif_init(struct mlxsw_sp_nexthop *nh, 3260 struct mlxsw_sp_rif *rif) 3261 { 3262 if (nh->rif) 3263 return; 3264 3265 nh->rif = rif; 3266 list_add(&nh->rif_list_node, &rif->nexthop_list); 3267 } 3268 3269 static void mlxsw_sp_nexthop_rif_fini(struct mlxsw_sp_nexthop *nh) 3270 { 3271 if (!nh->rif) 3272 return; 3273 3274 list_del(&nh->rif_list_node); 3275 nh->rif = NULL; 3276 } 3277 3278 static int mlxsw_sp_nexthop_neigh_init(struct mlxsw_sp *mlxsw_sp, 3279 struct mlxsw_sp_nexthop *nh) 3280 { 3281 struct mlxsw_sp_neigh_entry *neigh_entry; 3282 struct neighbour *n; 3283 u8 nud_state, dead; 3284 int err; 3285 3286 if (!nh->nh_grp->gateway || nh->neigh_entry) 3287 return 0; 3288 3289 /* Take a reference of neigh here ensuring that neigh would 3290 * not be destructed before the nexthop entry is finished. 3291 * The reference is taken either in neigh_lookup() or 3292 * in neigh_create() in case n is not found. 3293 */ 3294 n = neigh_lookup(nh->nh_grp->neigh_tbl, &nh->gw_addr, nh->rif->dev); 3295 if (!n) { 3296 n = neigh_create(nh->nh_grp->neigh_tbl, &nh->gw_addr, 3297 nh->rif->dev); 3298 if (IS_ERR(n)) 3299 return PTR_ERR(n); 3300 neigh_event_send(n, NULL); 3301 } 3302 neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n); 3303 if (!neigh_entry) { 3304 neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n); 3305 if (IS_ERR(neigh_entry)) { 3306 err = -EINVAL; 3307 goto err_neigh_entry_create; 3308 } 3309 } 3310 3311 /* If that is the first nexthop connected to that neigh, add to 3312 * nexthop_neighs_list 3313 */ 3314 if (list_empty(&neigh_entry->nexthop_list)) 3315 list_add_tail(&neigh_entry->nexthop_neighs_list_node, 3316 &mlxsw_sp->router->nexthop_neighs_list); 3317 3318 nh->neigh_entry = neigh_entry; 3319 list_add_tail(&nh->neigh_list_node, &neigh_entry->nexthop_list); 3320 read_lock_bh(&n->lock); 3321 nud_state = n->nud_state; 3322 dead = n->dead; 3323 read_unlock_bh(&n->lock); 3324 __mlxsw_sp_nexthop_neigh_update(nh, !(nud_state & NUD_VALID && !dead)); 3325 3326 return 0; 3327 3328 err_neigh_entry_create: 3329 neigh_release(n); 3330 return err; 3331 } 3332 3333 static void mlxsw_sp_nexthop_neigh_fini(struct mlxsw_sp *mlxsw_sp, 3334 struct mlxsw_sp_nexthop *nh) 3335 { 3336 struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry; 3337 struct neighbour *n; 3338 3339 if (!neigh_entry) 3340 return; 3341 n = neigh_entry->key.n; 3342 3343 __mlxsw_sp_nexthop_neigh_update(nh, true); 3344 list_del(&nh->neigh_list_node); 3345 nh->neigh_entry = NULL; 3346 3347 /* If that is the last nexthop connected to that neigh, remove from 3348 * nexthop_neighs_list 3349 */ 3350 if (list_empty(&neigh_entry->nexthop_list)) 3351 list_del(&neigh_entry->nexthop_neighs_list_node); 3352 3353 if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list)) 3354 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry); 3355 3356 neigh_release(n); 3357 } 3358 3359 static bool mlxsw_sp_ipip_netdev_ul_up(struct net_device *ol_dev) 3360 { 3361 struct net_device *ul_dev = __mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev); 3362 3363 return ul_dev ? (ul_dev->flags & IFF_UP) : true; 3364 } 3365 3366 static void mlxsw_sp_nexthop_ipip_init(struct mlxsw_sp *mlxsw_sp, 3367 struct mlxsw_sp_nexthop *nh, 3368 struct mlxsw_sp_ipip_entry *ipip_entry) 3369 { 3370 bool removing; 3371 3372 if (!nh->nh_grp->gateway || nh->ipip_entry) 3373 return; 3374 3375 nh->ipip_entry = ipip_entry; 3376 removing = !mlxsw_sp_ipip_netdev_ul_up(ipip_entry->ol_dev); 3377 __mlxsw_sp_nexthop_neigh_update(nh, removing); 3378 mlxsw_sp_nexthop_rif_init(nh, &ipip_entry->ol_lb->common); 3379 } 3380 3381 static void mlxsw_sp_nexthop_ipip_fini(struct mlxsw_sp *mlxsw_sp, 3382 struct mlxsw_sp_nexthop *nh) 3383 { 3384 struct mlxsw_sp_ipip_entry *ipip_entry = nh->ipip_entry; 3385 3386 if (!ipip_entry) 3387 return; 3388 3389 __mlxsw_sp_nexthop_neigh_update(nh, true); 3390 nh->ipip_entry = NULL; 3391 } 3392 3393 static bool mlxsw_sp_nexthop4_ipip_type(const struct mlxsw_sp *mlxsw_sp, 3394 const struct fib_nh *fib_nh, 3395 enum mlxsw_sp_ipip_type *p_ipipt) 3396 { 3397 struct net_device *dev = fib_nh->nh_dev; 3398 3399 return dev && 3400 fib_nh->nh_parent->fib_type == RTN_UNICAST && 3401 mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, p_ipipt); 3402 } 3403 3404 static void mlxsw_sp_nexthop_type_fini(struct mlxsw_sp *mlxsw_sp, 3405 struct mlxsw_sp_nexthop *nh) 3406 { 3407 switch (nh->type) { 3408 case MLXSW_SP_NEXTHOP_TYPE_ETH: 3409 mlxsw_sp_nexthop_neigh_fini(mlxsw_sp, nh); 3410 mlxsw_sp_nexthop_rif_fini(nh); 3411 break; 3412 case MLXSW_SP_NEXTHOP_TYPE_IPIP: 3413 mlxsw_sp_nexthop_rif_fini(nh); 3414 mlxsw_sp_nexthop_ipip_fini(mlxsw_sp, nh); 3415 break; 3416 } 3417 } 3418 3419 static int mlxsw_sp_nexthop4_type_init(struct mlxsw_sp *mlxsw_sp, 3420 struct mlxsw_sp_nexthop *nh, 3421 struct fib_nh *fib_nh) 3422 { 3423 const struct mlxsw_sp_ipip_ops *ipip_ops; 3424 struct net_device *dev = fib_nh->nh_dev; 3425 struct mlxsw_sp_ipip_entry *ipip_entry; 3426 struct mlxsw_sp_rif *rif; 3427 int err; 3428 3429 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, dev); 3430 if (ipip_entry) { 3431 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt]; 3432 if (ipip_ops->can_offload(mlxsw_sp, dev, 3433 MLXSW_SP_L3_PROTO_IPV4)) { 3434 nh->type = MLXSW_SP_NEXTHOP_TYPE_IPIP; 3435 mlxsw_sp_nexthop_ipip_init(mlxsw_sp, nh, ipip_entry); 3436 return 0; 3437 } 3438 } 3439 3440 nh->type = MLXSW_SP_NEXTHOP_TYPE_ETH; 3441 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 3442 if (!rif) 3443 return 0; 3444 3445 mlxsw_sp_nexthop_rif_init(nh, rif); 3446 err = mlxsw_sp_nexthop_neigh_init(mlxsw_sp, nh); 3447 if (err) 3448 goto err_neigh_init; 3449 3450 return 0; 3451 3452 err_neigh_init: 3453 mlxsw_sp_nexthop_rif_fini(nh); 3454 return err; 3455 } 3456 3457 static void mlxsw_sp_nexthop4_type_fini(struct mlxsw_sp *mlxsw_sp, 3458 struct mlxsw_sp_nexthop *nh) 3459 { 3460 mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh); 3461 } 3462 3463 static int mlxsw_sp_nexthop4_init(struct mlxsw_sp *mlxsw_sp, 3464 struct mlxsw_sp_nexthop_group *nh_grp, 3465 struct mlxsw_sp_nexthop *nh, 3466 struct fib_nh *fib_nh) 3467 { 3468 struct net_device *dev = fib_nh->nh_dev; 3469 struct in_device *in_dev; 3470 int err; 3471 3472 nh->nh_grp = nh_grp; 3473 nh->key.fib_nh = fib_nh; 3474 #ifdef CONFIG_IP_ROUTE_MULTIPATH 3475 nh->nh_weight = fib_nh->nh_weight; 3476 #else 3477 nh->nh_weight = 1; 3478 #endif 3479 memcpy(&nh->gw_addr, &fib_nh->nh_gw, sizeof(fib_nh->nh_gw)); 3480 err = mlxsw_sp_nexthop_insert(mlxsw_sp, nh); 3481 if (err) 3482 return err; 3483 3484 mlxsw_sp_nexthop_counter_alloc(mlxsw_sp, nh); 3485 list_add_tail(&nh->router_list_node, &mlxsw_sp->router->nexthop_list); 3486 3487 if (!dev) 3488 return 0; 3489 3490 in_dev = __in_dev_get_rtnl(dev); 3491 if (in_dev && IN_DEV_IGNORE_ROUTES_WITH_LINKDOWN(in_dev) && 3492 fib_nh->nh_flags & RTNH_F_LINKDOWN) 3493 return 0; 3494 3495 err = mlxsw_sp_nexthop4_type_init(mlxsw_sp, nh, fib_nh); 3496 if (err) 3497 goto err_nexthop_neigh_init; 3498 3499 return 0; 3500 3501 err_nexthop_neigh_init: 3502 mlxsw_sp_nexthop_remove(mlxsw_sp, nh); 3503 return err; 3504 } 3505 3506 static void mlxsw_sp_nexthop4_fini(struct mlxsw_sp *mlxsw_sp, 3507 struct mlxsw_sp_nexthop *nh) 3508 { 3509 mlxsw_sp_nexthop4_type_fini(mlxsw_sp, nh); 3510 list_del(&nh->router_list_node); 3511 mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh); 3512 mlxsw_sp_nexthop_remove(mlxsw_sp, nh); 3513 } 3514 3515 static void mlxsw_sp_nexthop4_event(struct mlxsw_sp *mlxsw_sp, 3516 unsigned long event, struct fib_nh *fib_nh) 3517 { 3518 struct mlxsw_sp_nexthop_key key; 3519 struct mlxsw_sp_nexthop *nh; 3520 3521 if (mlxsw_sp->router->aborted) 3522 return; 3523 3524 key.fib_nh = fib_nh; 3525 nh = mlxsw_sp_nexthop_lookup(mlxsw_sp, key); 3526 if (WARN_ON_ONCE(!nh)) 3527 return; 3528 3529 switch (event) { 3530 case FIB_EVENT_NH_ADD: 3531 mlxsw_sp_nexthop4_type_init(mlxsw_sp, nh, fib_nh); 3532 break; 3533 case FIB_EVENT_NH_DEL: 3534 mlxsw_sp_nexthop4_type_fini(mlxsw_sp, nh); 3535 break; 3536 } 3537 3538 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp); 3539 } 3540 3541 static void mlxsw_sp_nexthop_rif_update(struct mlxsw_sp *mlxsw_sp, 3542 struct mlxsw_sp_rif *rif) 3543 { 3544 struct mlxsw_sp_nexthop *nh; 3545 bool removing; 3546 3547 list_for_each_entry(nh, &rif->nexthop_list, rif_list_node) { 3548 switch (nh->type) { 3549 case MLXSW_SP_NEXTHOP_TYPE_ETH: 3550 removing = false; 3551 break; 3552 case MLXSW_SP_NEXTHOP_TYPE_IPIP: 3553 removing = !mlxsw_sp_ipip_netdev_ul_up(rif->dev); 3554 break; 3555 default: 3556 WARN_ON(1); 3557 continue; 3558 } 3559 3560 __mlxsw_sp_nexthop_neigh_update(nh, removing); 3561 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp); 3562 } 3563 } 3564 3565 static void mlxsw_sp_nexthop_rif_migrate(struct mlxsw_sp *mlxsw_sp, 3566 struct mlxsw_sp_rif *old_rif, 3567 struct mlxsw_sp_rif *new_rif) 3568 { 3569 struct mlxsw_sp_nexthop *nh; 3570 3571 list_splice_init(&old_rif->nexthop_list, &new_rif->nexthop_list); 3572 list_for_each_entry(nh, &new_rif->nexthop_list, rif_list_node) 3573 nh->rif = new_rif; 3574 mlxsw_sp_nexthop_rif_update(mlxsw_sp, new_rif); 3575 } 3576 3577 static void mlxsw_sp_nexthop_rif_gone_sync(struct mlxsw_sp *mlxsw_sp, 3578 struct mlxsw_sp_rif *rif) 3579 { 3580 struct mlxsw_sp_nexthop *nh, *tmp; 3581 3582 list_for_each_entry_safe(nh, tmp, &rif->nexthop_list, rif_list_node) { 3583 mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh); 3584 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp); 3585 } 3586 } 3587 3588 static bool mlxsw_sp_fi_is_gateway(const struct mlxsw_sp *mlxsw_sp, 3589 const struct fib_info *fi) 3590 { 3591 return fi->fib_nh->nh_scope == RT_SCOPE_LINK || 3592 mlxsw_sp_nexthop4_ipip_type(mlxsw_sp, fi->fib_nh, NULL); 3593 } 3594 3595 static struct mlxsw_sp_nexthop_group * 3596 mlxsw_sp_nexthop4_group_create(struct mlxsw_sp *mlxsw_sp, struct fib_info *fi) 3597 { 3598 struct mlxsw_sp_nexthop_group *nh_grp; 3599 struct mlxsw_sp_nexthop *nh; 3600 struct fib_nh *fib_nh; 3601 size_t alloc_size; 3602 int i; 3603 int err; 3604 3605 alloc_size = sizeof(*nh_grp) + 3606 fi->fib_nhs * sizeof(struct mlxsw_sp_nexthop); 3607 nh_grp = kzalloc(alloc_size, GFP_KERNEL); 3608 if (!nh_grp) 3609 return ERR_PTR(-ENOMEM); 3610 nh_grp->priv = fi; 3611 INIT_LIST_HEAD(&nh_grp->fib_list); 3612 nh_grp->neigh_tbl = &arp_tbl; 3613 3614 nh_grp->gateway = mlxsw_sp_fi_is_gateway(mlxsw_sp, fi); 3615 nh_grp->count = fi->fib_nhs; 3616 fib_info_hold(fi); 3617 for (i = 0; i < nh_grp->count; i++) { 3618 nh = &nh_grp->nexthops[i]; 3619 fib_nh = &fi->fib_nh[i]; 3620 err = mlxsw_sp_nexthop4_init(mlxsw_sp, nh_grp, nh, fib_nh); 3621 if (err) 3622 goto err_nexthop4_init; 3623 } 3624 err = mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp); 3625 if (err) 3626 goto err_nexthop_group_insert; 3627 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp); 3628 return nh_grp; 3629 3630 err_nexthop_group_insert: 3631 err_nexthop4_init: 3632 for (i--; i >= 0; i--) { 3633 nh = &nh_grp->nexthops[i]; 3634 mlxsw_sp_nexthop4_fini(mlxsw_sp, nh); 3635 } 3636 fib_info_put(fi); 3637 kfree(nh_grp); 3638 return ERR_PTR(err); 3639 } 3640 3641 static void 3642 mlxsw_sp_nexthop4_group_destroy(struct mlxsw_sp *mlxsw_sp, 3643 struct mlxsw_sp_nexthop_group *nh_grp) 3644 { 3645 struct mlxsw_sp_nexthop *nh; 3646 int i; 3647 3648 mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp); 3649 for (i = 0; i < nh_grp->count; i++) { 3650 nh = &nh_grp->nexthops[i]; 3651 mlxsw_sp_nexthop4_fini(mlxsw_sp, nh); 3652 } 3653 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp); 3654 WARN_ON_ONCE(nh_grp->adj_index_valid); 3655 fib_info_put(mlxsw_sp_nexthop4_group_fi(nh_grp)); 3656 kfree(nh_grp); 3657 } 3658 3659 static int mlxsw_sp_nexthop4_group_get(struct mlxsw_sp *mlxsw_sp, 3660 struct mlxsw_sp_fib_entry *fib_entry, 3661 struct fib_info *fi) 3662 { 3663 struct mlxsw_sp_nexthop_group *nh_grp; 3664 3665 nh_grp = mlxsw_sp_nexthop4_group_lookup(mlxsw_sp, fi); 3666 if (!nh_grp) { 3667 nh_grp = mlxsw_sp_nexthop4_group_create(mlxsw_sp, fi); 3668 if (IS_ERR(nh_grp)) 3669 return PTR_ERR(nh_grp); 3670 } 3671 list_add_tail(&fib_entry->nexthop_group_node, &nh_grp->fib_list); 3672 fib_entry->nh_group = nh_grp; 3673 return 0; 3674 } 3675 3676 static void mlxsw_sp_nexthop4_group_put(struct mlxsw_sp *mlxsw_sp, 3677 struct mlxsw_sp_fib_entry *fib_entry) 3678 { 3679 struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group; 3680 3681 list_del(&fib_entry->nexthop_group_node); 3682 if (!list_empty(&nh_grp->fib_list)) 3683 return; 3684 mlxsw_sp_nexthop4_group_destroy(mlxsw_sp, nh_grp); 3685 } 3686 3687 static bool 3688 mlxsw_sp_fib4_entry_should_offload(const struct mlxsw_sp_fib_entry *fib_entry) 3689 { 3690 struct mlxsw_sp_fib4_entry *fib4_entry; 3691 3692 fib4_entry = container_of(fib_entry, struct mlxsw_sp_fib4_entry, 3693 common); 3694 return !fib4_entry->tos; 3695 } 3696 3697 static bool 3698 mlxsw_sp_fib_entry_should_offload(const struct mlxsw_sp_fib_entry *fib_entry) 3699 { 3700 struct mlxsw_sp_nexthop_group *nh_group = fib_entry->nh_group; 3701 3702 switch (fib_entry->fib_node->fib->proto) { 3703 case MLXSW_SP_L3_PROTO_IPV4: 3704 if (!mlxsw_sp_fib4_entry_should_offload(fib_entry)) 3705 return false; 3706 break; 3707 case MLXSW_SP_L3_PROTO_IPV6: 3708 break; 3709 } 3710 3711 switch (fib_entry->type) { 3712 case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE: 3713 return !!nh_group->adj_index_valid; 3714 case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL: 3715 return !!nh_group->nh_rif; 3716 case MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP: 3717 return true; 3718 default: 3719 return false; 3720 } 3721 } 3722 3723 static struct mlxsw_sp_nexthop * 3724 mlxsw_sp_rt6_nexthop(struct mlxsw_sp_nexthop_group *nh_grp, 3725 const struct mlxsw_sp_rt6 *mlxsw_sp_rt6) 3726 { 3727 int i; 3728 3729 for (i = 0; i < nh_grp->count; i++) { 3730 struct mlxsw_sp_nexthop *nh = &nh_grp->nexthops[i]; 3731 struct rt6_info *rt = mlxsw_sp_rt6->rt; 3732 3733 if (nh->rif && nh->rif->dev == rt->dst.dev && 3734 ipv6_addr_equal((const struct in6_addr *) &nh->gw_addr, 3735 &rt->rt6i_gateway)) 3736 return nh; 3737 continue; 3738 } 3739 3740 return NULL; 3741 } 3742 3743 static void 3744 mlxsw_sp_fib4_entry_offload_set(struct mlxsw_sp_fib_entry *fib_entry) 3745 { 3746 struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group; 3747 int i; 3748 3749 if (fib_entry->type == MLXSW_SP_FIB_ENTRY_TYPE_LOCAL || 3750 fib_entry->type == MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP) { 3751 nh_grp->nexthops->key.fib_nh->nh_flags |= RTNH_F_OFFLOAD; 3752 return; 3753 } 3754 3755 for (i = 0; i < nh_grp->count; i++) { 3756 struct mlxsw_sp_nexthop *nh = &nh_grp->nexthops[i]; 3757 3758 if (nh->offloaded) 3759 nh->key.fib_nh->nh_flags |= RTNH_F_OFFLOAD; 3760 else 3761 nh->key.fib_nh->nh_flags &= ~RTNH_F_OFFLOAD; 3762 } 3763 } 3764 3765 static void 3766 mlxsw_sp_fib4_entry_offload_unset(struct mlxsw_sp_fib_entry *fib_entry) 3767 { 3768 struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group; 3769 int i; 3770 3771 for (i = 0; i < nh_grp->count; i++) { 3772 struct mlxsw_sp_nexthop *nh = &nh_grp->nexthops[i]; 3773 3774 nh->key.fib_nh->nh_flags &= ~RTNH_F_OFFLOAD; 3775 } 3776 } 3777 3778 static void 3779 mlxsw_sp_fib6_entry_offload_set(struct mlxsw_sp_fib_entry *fib_entry) 3780 { 3781 struct mlxsw_sp_fib6_entry *fib6_entry; 3782 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 3783 3784 fib6_entry = container_of(fib_entry, struct mlxsw_sp_fib6_entry, 3785 common); 3786 3787 if (fib_entry->type == MLXSW_SP_FIB_ENTRY_TYPE_LOCAL) { 3788 list_first_entry(&fib6_entry->rt6_list, struct mlxsw_sp_rt6, 3789 list)->rt->rt6i_nh_flags |= RTNH_F_OFFLOAD; 3790 return; 3791 } 3792 3793 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) { 3794 struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group; 3795 struct mlxsw_sp_nexthop *nh; 3796 3797 nh = mlxsw_sp_rt6_nexthop(nh_grp, mlxsw_sp_rt6); 3798 if (nh && nh->offloaded) 3799 mlxsw_sp_rt6->rt->rt6i_nh_flags |= RTNH_F_OFFLOAD; 3800 else 3801 mlxsw_sp_rt6->rt->rt6i_nh_flags &= ~RTNH_F_OFFLOAD; 3802 } 3803 } 3804 3805 static void 3806 mlxsw_sp_fib6_entry_offload_unset(struct mlxsw_sp_fib_entry *fib_entry) 3807 { 3808 struct mlxsw_sp_fib6_entry *fib6_entry; 3809 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 3810 3811 fib6_entry = container_of(fib_entry, struct mlxsw_sp_fib6_entry, 3812 common); 3813 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) { 3814 struct rt6_info *rt = mlxsw_sp_rt6->rt; 3815 3816 rt->rt6i_nh_flags &= ~RTNH_F_OFFLOAD; 3817 } 3818 } 3819 3820 static void mlxsw_sp_fib_entry_offload_set(struct mlxsw_sp_fib_entry *fib_entry) 3821 { 3822 switch (fib_entry->fib_node->fib->proto) { 3823 case MLXSW_SP_L3_PROTO_IPV4: 3824 mlxsw_sp_fib4_entry_offload_set(fib_entry); 3825 break; 3826 case MLXSW_SP_L3_PROTO_IPV6: 3827 mlxsw_sp_fib6_entry_offload_set(fib_entry); 3828 break; 3829 } 3830 } 3831 3832 static void 3833 mlxsw_sp_fib_entry_offload_unset(struct mlxsw_sp_fib_entry *fib_entry) 3834 { 3835 switch (fib_entry->fib_node->fib->proto) { 3836 case MLXSW_SP_L3_PROTO_IPV4: 3837 mlxsw_sp_fib4_entry_offload_unset(fib_entry); 3838 break; 3839 case MLXSW_SP_L3_PROTO_IPV6: 3840 mlxsw_sp_fib6_entry_offload_unset(fib_entry); 3841 break; 3842 } 3843 } 3844 3845 static void 3846 mlxsw_sp_fib_entry_offload_refresh(struct mlxsw_sp_fib_entry *fib_entry, 3847 enum mlxsw_reg_ralue_op op, int err) 3848 { 3849 switch (op) { 3850 case MLXSW_REG_RALUE_OP_WRITE_DELETE: 3851 return mlxsw_sp_fib_entry_offload_unset(fib_entry); 3852 case MLXSW_REG_RALUE_OP_WRITE_WRITE: 3853 if (err) 3854 return; 3855 if (mlxsw_sp_fib_entry_should_offload(fib_entry)) 3856 mlxsw_sp_fib_entry_offload_set(fib_entry); 3857 else 3858 mlxsw_sp_fib_entry_offload_unset(fib_entry); 3859 return; 3860 default: 3861 return; 3862 } 3863 } 3864 3865 static void 3866 mlxsw_sp_fib_entry_ralue_pack(char *ralue_pl, 3867 const struct mlxsw_sp_fib_entry *fib_entry, 3868 enum mlxsw_reg_ralue_op op) 3869 { 3870 struct mlxsw_sp_fib *fib = fib_entry->fib_node->fib; 3871 enum mlxsw_reg_ralxx_protocol proto; 3872 u32 *p_dip; 3873 3874 proto = (enum mlxsw_reg_ralxx_protocol) fib->proto; 3875 3876 switch (fib->proto) { 3877 case MLXSW_SP_L3_PROTO_IPV4: 3878 p_dip = (u32 *) fib_entry->fib_node->key.addr; 3879 mlxsw_reg_ralue_pack4(ralue_pl, proto, op, fib->vr->id, 3880 fib_entry->fib_node->key.prefix_len, 3881 *p_dip); 3882 break; 3883 case MLXSW_SP_L3_PROTO_IPV6: 3884 mlxsw_reg_ralue_pack6(ralue_pl, proto, op, fib->vr->id, 3885 fib_entry->fib_node->key.prefix_len, 3886 fib_entry->fib_node->key.addr); 3887 break; 3888 } 3889 } 3890 3891 static int mlxsw_sp_fib_entry_op_remote(struct mlxsw_sp *mlxsw_sp, 3892 struct mlxsw_sp_fib_entry *fib_entry, 3893 enum mlxsw_reg_ralue_op op) 3894 { 3895 char ralue_pl[MLXSW_REG_RALUE_LEN]; 3896 enum mlxsw_reg_ralue_trap_action trap_action; 3897 u16 trap_id = 0; 3898 u32 adjacency_index = 0; 3899 u16 ecmp_size = 0; 3900 3901 /* In case the nexthop group adjacency index is valid, use it 3902 * with provided ECMP size. Otherwise, setup trap and pass 3903 * traffic to kernel. 3904 */ 3905 if (mlxsw_sp_fib_entry_should_offload(fib_entry)) { 3906 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP; 3907 adjacency_index = fib_entry->nh_group->adj_index; 3908 ecmp_size = fib_entry->nh_group->ecmp_size; 3909 } else { 3910 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP; 3911 trap_id = MLXSW_TRAP_ID_RTR_INGRESS0; 3912 } 3913 3914 mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op); 3915 mlxsw_reg_ralue_act_remote_pack(ralue_pl, trap_action, trap_id, 3916 adjacency_index, ecmp_size); 3917 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl); 3918 } 3919 3920 static int mlxsw_sp_fib_entry_op_local(struct mlxsw_sp *mlxsw_sp, 3921 struct mlxsw_sp_fib_entry *fib_entry, 3922 enum mlxsw_reg_ralue_op op) 3923 { 3924 struct mlxsw_sp_rif *rif = fib_entry->nh_group->nh_rif; 3925 enum mlxsw_reg_ralue_trap_action trap_action; 3926 char ralue_pl[MLXSW_REG_RALUE_LEN]; 3927 u16 trap_id = 0; 3928 u16 rif_index = 0; 3929 3930 if (mlxsw_sp_fib_entry_should_offload(fib_entry)) { 3931 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP; 3932 rif_index = rif->rif_index; 3933 } else { 3934 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP; 3935 trap_id = MLXSW_TRAP_ID_RTR_INGRESS0; 3936 } 3937 3938 mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op); 3939 mlxsw_reg_ralue_act_local_pack(ralue_pl, trap_action, trap_id, 3940 rif_index); 3941 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl); 3942 } 3943 3944 static int mlxsw_sp_fib_entry_op_trap(struct mlxsw_sp *mlxsw_sp, 3945 struct mlxsw_sp_fib_entry *fib_entry, 3946 enum mlxsw_reg_ralue_op op) 3947 { 3948 char ralue_pl[MLXSW_REG_RALUE_LEN]; 3949 3950 mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op); 3951 mlxsw_reg_ralue_act_ip2me_pack(ralue_pl); 3952 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl); 3953 } 3954 3955 static int 3956 mlxsw_sp_fib_entry_op_ipip_decap(struct mlxsw_sp *mlxsw_sp, 3957 struct mlxsw_sp_fib_entry *fib_entry, 3958 enum mlxsw_reg_ralue_op op) 3959 { 3960 struct mlxsw_sp_ipip_entry *ipip_entry = fib_entry->decap.ipip_entry; 3961 const struct mlxsw_sp_ipip_ops *ipip_ops; 3962 3963 if (WARN_ON(!ipip_entry)) 3964 return -EINVAL; 3965 3966 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt]; 3967 return ipip_ops->fib_entry_op(mlxsw_sp, ipip_entry, op, 3968 fib_entry->decap.tunnel_index); 3969 } 3970 3971 static int __mlxsw_sp_fib_entry_op(struct mlxsw_sp *mlxsw_sp, 3972 struct mlxsw_sp_fib_entry *fib_entry, 3973 enum mlxsw_reg_ralue_op op) 3974 { 3975 switch (fib_entry->type) { 3976 case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE: 3977 return mlxsw_sp_fib_entry_op_remote(mlxsw_sp, fib_entry, op); 3978 case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL: 3979 return mlxsw_sp_fib_entry_op_local(mlxsw_sp, fib_entry, op); 3980 case MLXSW_SP_FIB_ENTRY_TYPE_TRAP: 3981 return mlxsw_sp_fib_entry_op_trap(mlxsw_sp, fib_entry, op); 3982 case MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP: 3983 return mlxsw_sp_fib_entry_op_ipip_decap(mlxsw_sp, 3984 fib_entry, op); 3985 } 3986 return -EINVAL; 3987 } 3988 3989 static int mlxsw_sp_fib_entry_op(struct mlxsw_sp *mlxsw_sp, 3990 struct mlxsw_sp_fib_entry *fib_entry, 3991 enum mlxsw_reg_ralue_op op) 3992 { 3993 int err = __mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry, op); 3994 3995 mlxsw_sp_fib_entry_offload_refresh(fib_entry, op, err); 3996 3997 return err; 3998 } 3999 4000 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp, 4001 struct mlxsw_sp_fib_entry *fib_entry) 4002 { 4003 return mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry, 4004 MLXSW_REG_RALUE_OP_WRITE_WRITE); 4005 } 4006 4007 static int mlxsw_sp_fib_entry_del(struct mlxsw_sp *mlxsw_sp, 4008 struct mlxsw_sp_fib_entry *fib_entry) 4009 { 4010 return mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry, 4011 MLXSW_REG_RALUE_OP_WRITE_DELETE); 4012 } 4013 4014 static int 4015 mlxsw_sp_fib4_entry_type_set(struct mlxsw_sp *mlxsw_sp, 4016 const struct fib_entry_notifier_info *fen_info, 4017 struct mlxsw_sp_fib_entry *fib_entry) 4018 { 4019 union mlxsw_sp_l3addr dip = { .addr4 = htonl(fen_info->dst) }; 4020 struct net_device *dev = fen_info->fi->fib_dev; 4021 struct mlxsw_sp_ipip_entry *ipip_entry; 4022 struct fib_info *fi = fen_info->fi; 4023 4024 switch (fen_info->type) { 4025 case RTN_LOCAL: 4026 ipip_entry = mlxsw_sp_ipip_entry_find_by_decap(mlxsw_sp, dev, 4027 MLXSW_SP_L3_PROTO_IPV4, dip); 4028 if (ipip_entry && ipip_entry->ol_dev->flags & IFF_UP) { 4029 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP; 4030 return mlxsw_sp_fib_entry_decap_init(mlxsw_sp, 4031 fib_entry, 4032 ipip_entry); 4033 } 4034 /* fall through */ 4035 case RTN_BROADCAST: 4036 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP; 4037 return 0; 4038 case RTN_UNREACHABLE: /* fall through */ 4039 case RTN_BLACKHOLE: /* fall through */ 4040 case RTN_PROHIBIT: 4041 /* Packets hitting these routes need to be trapped, but 4042 * can do so with a lower priority than packets directed 4043 * at the host, so use action type local instead of trap. 4044 */ 4045 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL; 4046 return 0; 4047 case RTN_UNICAST: 4048 if (mlxsw_sp_fi_is_gateway(mlxsw_sp, fi)) 4049 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_REMOTE; 4050 else 4051 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL; 4052 return 0; 4053 default: 4054 return -EINVAL; 4055 } 4056 } 4057 4058 static struct mlxsw_sp_fib4_entry * 4059 mlxsw_sp_fib4_entry_create(struct mlxsw_sp *mlxsw_sp, 4060 struct mlxsw_sp_fib_node *fib_node, 4061 const struct fib_entry_notifier_info *fen_info) 4062 { 4063 struct mlxsw_sp_fib4_entry *fib4_entry; 4064 struct mlxsw_sp_fib_entry *fib_entry; 4065 int err; 4066 4067 fib4_entry = kzalloc(sizeof(*fib4_entry), GFP_KERNEL); 4068 if (!fib4_entry) 4069 return ERR_PTR(-ENOMEM); 4070 fib_entry = &fib4_entry->common; 4071 4072 err = mlxsw_sp_fib4_entry_type_set(mlxsw_sp, fen_info, fib_entry); 4073 if (err) 4074 goto err_fib4_entry_type_set; 4075 4076 err = mlxsw_sp_nexthop4_group_get(mlxsw_sp, fib_entry, fen_info->fi); 4077 if (err) 4078 goto err_nexthop4_group_get; 4079 4080 fib4_entry->prio = fen_info->fi->fib_priority; 4081 fib4_entry->tb_id = fen_info->tb_id; 4082 fib4_entry->type = fen_info->type; 4083 fib4_entry->tos = fen_info->tos; 4084 4085 fib_entry->fib_node = fib_node; 4086 4087 return fib4_entry; 4088 4089 err_nexthop4_group_get: 4090 err_fib4_entry_type_set: 4091 kfree(fib4_entry); 4092 return ERR_PTR(err); 4093 } 4094 4095 static void mlxsw_sp_fib4_entry_destroy(struct mlxsw_sp *mlxsw_sp, 4096 struct mlxsw_sp_fib4_entry *fib4_entry) 4097 { 4098 mlxsw_sp_nexthop4_group_put(mlxsw_sp, &fib4_entry->common); 4099 kfree(fib4_entry); 4100 } 4101 4102 static struct mlxsw_sp_fib4_entry * 4103 mlxsw_sp_fib4_entry_lookup(struct mlxsw_sp *mlxsw_sp, 4104 const struct fib_entry_notifier_info *fen_info) 4105 { 4106 struct mlxsw_sp_fib4_entry *fib4_entry; 4107 struct mlxsw_sp_fib_node *fib_node; 4108 struct mlxsw_sp_fib *fib; 4109 struct mlxsw_sp_vr *vr; 4110 4111 vr = mlxsw_sp_vr_find(mlxsw_sp, fen_info->tb_id); 4112 if (!vr) 4113 return NULL; 4114 fib = mlxsw_sp_vr_fib(vr, MLXSW_SP_L3_PROTO_IPV4); 4115 4116 fib_node = mlxsw_sp_fib_node_lookup(fib, &fen_info->dst, 4117 sizeof(fen_info->dst), 4118 fen_info->dst_len); 4119 if (!fib_node) 4120 return NULL; 4121 4122 list_for_each_entry(fib4_entry, &fib_node->entry_list, common.list) { 4123 if (fib4_entry->tb_id == fen_info->tb_id && 4124 fib4_entry->tos == fen_info->tos && 4125 fib4_entry->type == fen_info->type && 4126 mlxsw_sp_nexthop4_group_fi(fib4_entry->common.nh_group) == 4127 fen_info->fi) { 4128 return fib4_entry; 4129 } 4130 } 4131 4132 return NULL; 4133 } 4134 4135 static const struct rhashtable_params mlxsw_sp_fib_ht_params = { 4136 .key_offset = offsetof(struct mlxsw_sp_fib_node, key), 4137 .head_offset = offsetof(struct mlxsw_sp_fib_node, ht_node), 4138 .key_len = sizeof(struct mlxsw_sp_fib_key), 4139 .automatic_shrinking = true, 4140 }; 4141 4142 static int mlxsw_sp_fib_node_insert(struct mlxsw_sp_fib *fib, 4143 struct mlxsw_sp_fib_node *fib_node) 4144 { 4145 return rhashtable_insert_fast(&fib->ht, &fib_node->ht_node, 4146 mlxsw_sp_fib_ht_params); 4147 } 4148 4149 static void mlxsw_sp_fib_node_remove(struct mlxsw_sp_fib *fib, 4150 struct mlxsw_sp_fib_node *fib_node) 4151 { 4152 rhashtable_remove_fast(&fib->ht, &fib_node->ht_node, 4153 mlxsw_sp_fib_ht_params); 4154 } 4155 4156 static struct mlxsw_sp_fib_node * 4157 mlxsw_sp_fib_node_lookup(struct mlxsw_sp_fib *fib, const void *addr, 4158 size_t addr_len, unsigned char prefix_len) 4159 { 4160 struct mlxsw_sp_fib_key key; 4161 4162 memset(&key, 0, sizeof(key)); 4163 memcpy(key.addr, addr, addr_len); 4164 key.prefix_len = prefix_len; 4165 return rhashtable_lookup_fast(&fib->ht, &key, mlxsw_sp_fib_ht_params); 4166 } 4167 4168 static struct mlxsw_sp_fib_node * 4169 mlxsw_sp_fib_node_create(struct mlxsw_sp_fib *fib, const void *addr, 4170 size_t addr_len, unsigned char prefix_len) 4171 { 4172 struct mlxsw_sp_fib_node *fib_node; 4173 4174 fib_node = kzalloc(sizeof(*fib_node), GFP_KERNEL); 4175 if (!fib_node) 4176 return NULL; 4177 4178 INIT_LIST_HEAD(&fib_node->entry_list); 4179 list_add(&fib_node->list, &fib->node_list); 4180 memcpy(fib_node->key.addr, addr, addr_len); 4181 fib_node->key.prefix_len = prefix_len; 4182 4183 return fib_node; 4184 } 4185 4186 static void mlxsw_sp_fib_node_destroy(struct mlxsw_sp_fib_node *fib_node) 4187 { 4188 list_del(&fib_node->list); 4189 WARN_ON(!list_empty(&fib_node->entry_list)); 4190 kfree(fib_node); 4191 } 4192 4193 static bool 4194 mlxsw_sp_fib_node_entry_is_first(const struct mlxsw_sp_fib_node *fib_node, 4195 const struct mlxsw_sp_fib_entry *fib_entry) 4196 { 4197 return list_first_entry(&fib_node->entry_list, 4198 struct mlxsw_sp_fib_entry, list) == fib_entry; 4199 } 4200 4201 static int mlxsw_sp_fib_lpm_tree_link(struct mlxsw_sp *mlxsw_sp, 4202 struct mlxsw_sp_fib *fib, 4203 struct mlxsw_sp_fib_node *fib_node) 4204 { 4205 struct mlxsw_sp_prefix_usage req_prefix_usage = {{ 0 } }; 4206 struct mlxsw_sp_lpm_tree *lpm_tree; 4207 int err; 4208 4209 /* Since the tree is shared between all virtual routers we must 4210 * make sure it contains all the required prefix lengths. This 4211 * can be computed by either adding the new prefix length to the 4212 * existing prefix usage of a bound tree, or by aggregating the 4213 * prefix lengths across all virtual routers and adding the new 4214 * one as well. 4215 */ 4216 if (fib->lpm_tree) 4217 mlxsw_sp_prefix_usage_cpy(&req_prefix_usage, 4218 &fib->lpm_tree->prefix_usage); 4219 else 4220 mlxsw_sp_vrs_prefixes(mlxsw_sp, fib->proto, &req_prefix_usage); 4221 mlxsw_sp_prefix_usage_set(&req_prefix_usage, fib_node->key.prefix_len); 4222 4223 lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage, 4224 fib->proto); 4225 if (IS_ERR(lpm_tree)) 4226 return PTR_ERR(lpm_tree); 4227 4228 if (fib->lpm_tree && fib->lpm_tree->id == lpm_tree->id) 4229 return 0; 4230 4231 err = mlxsw_sp_vrs_lpm_tree_replace(mlxsw_sp, fib, lpm_tree); 4232 if (err) 4233 return err; 4234 4235 return 0; 4236 } 4237 4238 static void mlxsw_sp_fib_lpm_tree_unlink(struct mlxsw_sp *mlxsw_sp, 4239 struct mlxsw_sp_fib *fib) 4240 { 4241 if (!mlxsw_sp_prefix_usage_none(&fib->prefix_usage)) 4242 return; 4243 mlxsw_sp_vr_lpm_tree_unbind(mlxsw_sp, fib); 4244 mlxsw_sp_lpm_tree_put(mlxsw_sp, fib->lpm_tree); 4245 fib->lpm_tree = NULL; 4246 } 4247 4248 static void mlxsw_sp_fib_node_prefix_inc(struct mlxsw_sp_fib_node *fib_node) 4249 { 4250 unsigned char prefix_len = fib_node->key.prefix_len; 4251 struct mlxsw_sp_fib *fib = fib_node->fib; 4252 4253 if (fib->prefix_ref_count[prefix_len]++ == 0) 4254 mlxsw_sp_prefix_usage_set(&fib->prefix_usage, prefix_len); 4255 } 4256 4257 static void mlxsw_sp_fib_node_prefix_dec(struct mlxsw_sp_fib_node *fib_node) 4258 { 4259 unsigned char prefix_len = fib_node->key.prefix_len; 4260 struct mlxsw_sp_fib *fib = fib_node->fib; 4261 4262 if (--fib->prefix_ref_count[prefix_len] == 0) 4263 mlxsw_sp_prefix_usage_clear(&fib->prefix_usage, prefix_len); 4264 } 4265 4266 static int mlxsw_sp_fib_node_init(struct mlxsw_sp *mlxsw_sp, 4267 struct mlxsw_sp_fib_node *fib_node, 4268 struct mlxsw_sp_fib *fib) 4269 { 4270 int err; 4271 4272 err = mlxsw_sp_fib_node_insert(fib, fib_node); 4273 if (err) 4274 return err; 4275 fib_node->fib = fib; 4276 4277 err = mlxsw_sp_fib_lpm_tree_link(mlxsw_sp, fib, fib_node); 4278 if (err) 4279 goto err_fib_lpm_tree_link; 4280 4281 mlxsw_sp_fib_node_prefix_inc(fib_node); 4282 4283 return 0; 4284 4285 err_fib_lpm_tree_link: 4286 fib_node->fib = NULL; 4287 mlxsw_sp_fib_node_remove(fib, fib_node); 4288 return err; 4289 } 4290 4291 static void mlxsw_sp_fib_node_fini(struct mlxsw_sp *mlxsw_sp, 4292 struct mlxsw_sp_fib_node *fib_node) 4293 { 4294 struct mlxsw_sp_fib *fib = fib_node->fib; 4295 4296 mlxsw_sp_fib_node_prefix_dec(fib_node); 4297 mlxsw_sp_fib_lpm_tree_unlink(mlxsw_sp, fib); 4298 fib_node->fib = NULL; 4299 mlxsw_sp_fib_node_remove(fib, fib_node); 4300 } 4301 4302 static struct mlxsw_sp_fib_node * 4303 mlxsw_sp_fib_node_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id, const void *addr, 4304 size_t addr_len, unsigned char prefix_len, 4305 enum mlxsw_sp_l3proto proto) 4306 { 4307 struct mlxsw_sp_fib_node *fib_node; 4308 struct mlxsw_sp_fib *fib; 4309 struct mlxsw_sp_vr *vr; 4310 int err; 4311 4312 vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id, NULL); 4313 if (IS_ERR(vr)) 4314 return ERR_CAST(vr); 4315 fib = mlxsw_sp_vr_fib(vr, proto); 4316 4317 fib_node = mlxsw_sp_fib_node_lookup(fib, addr, addr_len, prefix_len); 4318 if (fib_node) 4319 return fib_node; 4320 4321 fib_node = mlxsw_sp_fib_node_create(fib, addr, addr_len, prefix_len); 4322 if (!fib_node) { 4323 err = -ENOMEM; 4324 goto err_fib_node_create; 4325 } 4326 4327 err = mlxsw_sp_fib_node_init(mlxsw_sp, fib_node, fib); 4328 if (err) 4329 goto err_fib_node_init; 4330 4331 return fib_node; 4332 4333 err_fib_node_init: 4334 mlxsw_sp_fib_node_destroy(fib_node); 4335 err_fib_node_create: 4336 mlxsw_sp_vr_put(vr); 4337 return ERR_PTR(err); 4338 } 4339 4340 static void mlxsw_sp_fib_node_put(struct mlxsw_sp *mlxsw_sp, 4341 struct mlxsw_sp_fib_node *fib_node) 4342 { 4343 struct mlxsw_sp_vr *vr = fib_node->fib->vr; 4344 4345 if (!list_empty(&fib_node->entry_list)) 4346 return; 4347 mlxsw_sp_fib_node_fini(mlxsw_sp, fib_node); 4348 mlxsw_sp_fib_node_destroy(fib_node); 4349 mlxsw_sp_vr_put(vr); 4350 } 4351 4352 static struct mlxsw_sp_fib4_entry * 4353 mlxsw_sp_fib4_node_entry_find(const struct mlxsw_sp_fib_node *fib_node, 4354 const struct mlxsw_sp_fib4_entry *new4_entry) 4355 { 4356 struct mlxsw_sp_fib4_entry *fib4_entry; 4357 4358 list_for_each_entry(fib4_entry, &fib_node->entry_list, common.list) { 4359 if (fib4_entry->tb_id > new4_entry->tb_id) 4360 continue; 4361 if (fib4_entry->tb_id != new4_entry->tb_id) 4362 break; 4363 if (fib4_entry->tos > new4_entry->tos) 4364 continue; 4365 if (fib4_entry->prio >= new4_entry->prio || 4366 fib4_entry->tos < new4_entry->tos) 4367 return fib4_entry; 4368 } 4369 4370 return NULL; 4371 } 4372 4373 static int 4374 mlxsw_sp_fib4_node_list_append(struct mlxsw_sp_fib4_entry *fib4_entry, 4375 struct mlxsw_sp_fib4_entry *new4_entry) 4376 { 4377 struct mlxsw_sp_fib_node *fib_node; 4378 4379 if (WARN_ON(!fib4_entry)) 4380 return -EINVAL; 4381 4382 fib_node = fib4_entry->common.fib_node; 4383 list_for_each_entry_from(fib4_entry, &fib_node->entry_list, 4384 common.list) { 4385 if (fib4_entry->tb_id != new4_entry->tb_id || 4386 fib4_entry->tos != new4_entry->tos || 4387 fib4_entry->prio != new4_entry->prio) 4388 break; 4389 } 4390 4391 list_add_tail(&new4_entry->common.list, &fib4_entry->common.list); 4392 return 0; 4393 } 4394 4395 static int 4396 mlxsw_sp_fib4_node_list_insert(struct mlxsw_sp_fib4_entry *new4_entry, 4397 bool replace, bool append) 4398 { 4399 struct mlxsw_sp_fib_node *fib_node = new4_entry->common.fib_node; 4400 struct mlxsw_sp_fib4_entry *fib4_entry; 4401 4402 fib4_entry = mlxsw_sp_fib4_node_entry_find(fib_node, new4_entry); 4403 4404 if (append) 4405 return mlxsw_sp_fib4_node_list_append(fib4_entry, new4_entry); 4406 if (replace && WARN_ON(!fib4_entry)) 4407 return -EINVAL; 4408 4409 /* Insert new entry before replaced one, so that we can later 4410 * remove the second. 4411 */ 4412 if (fib4_entry) { 4413 list_add_tail(&new4_entry->common.list, 4414 &fib4_entry->common.list); 4415 } else { 4416 struct mlxsw_sp_fib4_entry *last; 4417 4418 list_for_each_entry(last, &fib_node->entry_list, common.list) { 4419 if (new4_entry->tb_id > last->tb_id) 4420 break; 4421 fib4_entry = last; 4422 } 4423 4424 if (fib4_entry) 4425 list_add(&new4_entry->common.list, 4426 &fib4_entry->common.list); 4427 else 4428 list_add(&new4_entry->common.list, 4429 &fib_node->entry_list); 4430 } 4431 4432 return 0; 4433 } 4434 4435 static void 4436 mlxsw_sp_fib4_node_list_remove(struct mlxsw_sp_fib4_entry *fib4_entry) 4437 { 4438 list_del(&fib4_entry->common.list); 4439 } 4440 4441 static int mlxsw_sp_fib_node_entry_add(struct mlxsw_sp *mlxsw_sp, 4442 struct mlxsw_sp_fib_entry *fib_entry) 4443 { 4444 struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node; 4445 4446 if (!mlxsw_sp_fib_node_entry_is_first(fib_node, fib_entry)) 4447 return 0; 4448 4449 /* To prevent packet loss, overwrite the previously offloaded 4450 * entry. 4451 */ 4452 if (!list_is_singular(&fib_node->entry_list)) { 4453 enum mlxsw_reg_ralue_op op = MLXSW_REG_RALUE_OP_WRITE_DELETE; 4454 struct mlxsw_sp_fib_entry *n = list_next_entry(fib_entry, list); 4455 4456 mlxsw_sp_fib_entry_offload_refresh(n, op, 0); 4457 } 4458 4459 return mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry); 4460 } 4461 4462 static void mlxsw_sp_fib_node_entry_del(struct mlxsw_sp *mlxsw_sp, 4463 struct mlxsw_sp_fib_entry *fib_entry) 4464 { 4465 struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node; 4466 4467 if (!mlxsw_sp_fib_node_entry_is_first(fib_node, fib_entry)) 4468 return; 4469 4470 /* Promote the next entry by overwriting the deleted entry */ 4471 if (!list_is_singular(&fib_node->entry_list)) { 4472 struct mlxsw_sp_fib_entry *n = list_next_entry(fib_entry, list); 4473 enum mlxsw_reg_ralue_op op = MLXSW_REG_RALUE_OP_WRITE_DELETE; 4474 4475 mlxsw_sp_fib_entry_update(mlxsw_sp, n); 4476 mlxsw_sp_fib_entry_offload_refresh(fib_entry, op, 0); 4477 return; 4478 } 4479 4480 mlxsw_sp_fib_entry_del(mlxsw_sp, fib_entry); 4481 } 4482 4483 static int mlxsw_sp_fib4_node_entry_link(struct mlxsw_sp *mlxsw_sp, 4484 struct mlxsw_sp_fib4_entry *fib4_entry, 4485 bool replace, bool append) 4486 { 4487 int err; 4488 4489 err = mlxsw_sp_fib4_node_list_insert(fib4_entry, replace, append); 4490 if (err) 4491 return err; 4492 4493 err = mlxsw_sp_fib_node_entry_add(mlxsw_sp, &fib4_entry->common); 4494 if (err) 4495 goto err_fib_node_entry_add; 4496 4497 return 0; 4498 4499 err_fib_node_entry_add: 4500 mlxsw_sp_fib4_node_list_remove(fib4_entry); 4501 return err; 4502 } 4503 4504 static void 4505 mlxsw_sp_fib4_node_entry_unlink(struct mlxsw_sp *mlxsw_sp, 4506 struct mlxsw_sp_fib4_entry *fib4_entry) 4507 { 4508 mlxsw_sp_fib_node_entry_del(mlxsw_sp, &fib4_entry->common); 4509 mlxsw_sp_fib4_node_list_remove(fib4_entry); 4510 4511 if (fib4_entry->common.type == MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP) 4512 mlxsw_sp_fib_entry_decap_fini(mlxsw_sp, &fib4_entry->common); 4513 } 4514 4515 static void mlxsw_sp_fib4_entry_replace(struct mlxsw_sp *mlxsw_sp, 4516 struct mlxsw_sp_fib4_entry *fib4_entry, 4517 bool replace) 4518 { 4519 struct mlxsw_sp_fib_node *fib_node = fib4_entry->common.fib_node; 4520 struct mlxsw_sp_fib4_entry *replaced; 4521 4522 if (!replace) 4523 return; 4524 4525 /* We inserted the new entry before replaced one */ 4526 replaced = list_next_entry(fib4_entry, common.list); 4527 4528 mlxsw_sp_fib4_node_entry_unlink(mlxsw_sp, replaced); 4529 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, replaced); 4530 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 4531 } 4532 4533 static int 4534 mlxsw_sp_router_fib4_add(struct mlxsw_sp *mlxsw_sp, 4535 const struct fib_entry_notifier_info *fen_info, 4536 bool replace, bool append) 4537 { 4538 struct mlxsw_sp_fib4_entry *fib4_entry; 4539 struct mlxsw_sp_fib_node *fib_node; 4540 int err; 4541 4542 if (mlxsw_sp->router->aborted) 4543 return 0; 4544 4545 fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, fen_info->tb_id, 4546 &fen_info->dst, sizeof(fen_info->dst), 4547 fen_info->dst_len, 4548 MLXSW_SP_L3_PROTO_IPV4); 4549 if (IS_ERR(fib_node)) { 4550 dev_warn(mlxsw_sp->bus_info->dev, "Failed to get FIB node\n"); 4551 return PTR_ERR(fib_node); 4552 } 4553 4554 fib4_entry = mlxsw_sp_fib4_entry_create(mlxsw_sp, fib_node, fen_info); 4555 if (IS_ERR(fib4_entry)) { 4556 dev_warn(mlxsw_sp->bus_info->dev, "Failed to create FIB entry\n"); 4557 err = PTR_ERR(fib4_entry); 4558 goto err_fib4_entry_create; 4559 } 4560 4561 err = mlxsw_sp_fib4_node_entry_link(mlxsw_sp, fib4_entry, replace, 4562 append); 4563 if (err) { 4564 dev_warn(mlxsw_sp->bus_info->dev, "Failed to link FIB entry to node\n"); 4565 goto err_fib4_node_entry_link; 4566 } 4567 4568 mlxsw_sp_fib4_entry_replace(mlxsw_sp, fib4_entry, replace); 4569 4570 return 0; 4571 4572 err_fib4_node_entry_link: 4573 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry); 4574 err_fib4_entry_create: 4575 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 4576 return err; 4577 } 4578 4579 static void mlxsw_sp_router_fib4_del(struct mlxsw_sp *mlxsw_sp, 4580 struct fib_entry_notifier_info *fen_info) 4581 { 4582 struct mlxsw_sp_fib4_entry *fib4_entry; 4583 struct mlxsw_sp_fib_node *fib_node; 4584 4585 if (mlxsw_sp->router->aborted) 4586 return; 4587 4588 fib4_entry = mlxsw_sp_fib4_entry_lookup(mlxsw_sp, fen_info); 4589 if (WARN_ON(!fib4_entry)) 4590 return; 4591 fib_node = fib4_entry->common.fib_node; 4592 4593 mlxsw_sp_fib4_node_entry_unlink(mlxsw_sp, fib4_entry); 4594 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry); 4595 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 4596 } 4597 4598 static bool mlxsw_sp_fib6_rt_should_ignore(const struct rt6_info *rt) 4599 { 4600 /* Packets with link-local destination IP arriving to the router 4601 * are trapped to the CPU, so no need to program specific routes 4602 * for them. 4603 */ 4604 if (ipv6_addr_type(&rt->rt6i_dst.addr) & IPV6_ADDR_LINKLOCAL) 4605 return true; 4606 4607 /* Multicast routes aren't supported, so ignore them. Neighbour 4608 * Discovery packets are specifically trapped. 4609 */ 4610 if (ipv6_addr_type(&rt->rt6i_dst.addr) & IPV6_ADDR_MULTICAST) 4611 return true; 4612 4613 /* Cloned routes are irrelevant in the forwarding path. */ 4614 if (rt->rt6i_flags & RTF_CACHE) 4615 return true; 4616 4617 return false; 4618 } 4619 4620 static struct mlxsw_sp_rt6 *mlxsw_sp_rt6_create(struct rt6_info *rt) 4621 { 4622 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 4623 4624 mlxsw_sp_rt6 = kzalloc(sizeof(*mlxsw_sp_rt6), GFP_KERNEL); 4625 if (!mlxsw_sp_rt6) 4626 return ERR_PTR(-ENOMEM); 4627 4628 /* In case of route replace, replaced route is deleted with 4629 * no notification. Take reference to prevent accessing freed 4630 * memory. 4631 */ 4632 mlxsw_sp_rt6->rt = rt; 4633 rt6_hold(rt); 4634 4635 return mlxsw_sp_rt6; 4636 } 4637 4638 #if IS_ENABLED(CONFIG_IPV6) 4639 static void mlxsw_sp_rt6_release(struct rt6_info *rt) 4640 { 4641 rt6_release(rt); 4642 } 4643 #else 4644 static void mlxsw_sp_rt6_release(struct rt6_info *rt) 4645 { 4646 } 4647 #endif 4648 4649 static void mlxsw_sp_rt6_destroy(struct mlxsw_sp_rt6 *mlxsw_sp_rt6) 4650 { 4651 mlxsw_sp_rt6_release(mlxsw_sp_rt6->rt); 4652 kfree(mlxsw_sp_rt6); 4653 } 4654 4655 static bool mlxsw_sp_fib6_rt_can_mp(const struct rt6_info *rt) 4656 { 4657 /* RTF_CACHE routes are ignored */ 4658 return (rt->rt6i_flags & (RTF_GATEWAY | RTF_ADDRCONF)) == RTF_GATEWAY; 4659 } 4660 4661 static struct rt6_info * 4662 mlxsw_sp_fib6_entry_rt(const struct mlxsw_sp_fib6_entry *fib6_entry) 4663 { 4664 return list_first_entry(&fib6_entry->rt6_list, struct mlxsw_sp_rt6, 4665 list)->rt; 4666 } 4667 4668 static struct mlxsw_sp_fib6_entry * 4669 mlxsw_sp_fib6_node_mp_entry_find(const struct mlxsw_sp_fib_node *fib_node, 4670 const struct rt6_info *nrt, bool replace) 4671 { 4672 struct mlxsw_sp_fib6_entry *fib6_entry; 4673 4674 if (!mlxsw_sp_fib6_rt_can_mp(nrt) || replace) 4675 return NULL; 4676 4677 list_for_each_entry(fib6_entry, &fib_node->entry_list, common.list) { 4678 struct rt6_info *rt = mlxsw_sp_fib6_entry_rt(fib6_entry); 4679 4680 /* RT6_TABLE_LOCAL and RT6_TABLE_MAIN share the same 4681 * virtual router. 4682 */ 4683 if (rt->rt6i_table->tb6_id > nrt->rt6i_table->tb6_id) 4684 continue; 4685 if (rt->rt6i_table->tb6_id != nrt->rt6i_table->tb6_id) 4686 break; 4687 if (rt->rt6i_metric < nrt->rt6i_metric) 4688 continue; 4689 if (rt->rt6i_metric == nrt->rt6i_metric && 4690 mlxsw_sp_fib6_rt_can_mp(rt)) 4691 return fib6_entry; 4692 if (rt->rt6i_metric > nrt->rt6i_metric) 4693 break; 4694 } 4695 4696 return NULL; 4697 } 4698 4699 static struct mlxsw_sp_rt6 * 4700 mlxsw_sp_fib6_entry_rt_find(const struct mlxsw_sp_fib6_entry *fib6_entry, 4701 const struct rt6_info *rt) 4702 { 4703 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 4704 4705 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) { 4706 if (mlxsw_sp_rt6->rt == rt) 4707 return mlxsw_sp_rt6; 4708 } 4709 4710 return NULL; 4711 } 4712 4713 static bool mlxsw_sp_nexthop6_ipip_type(const struct mlxsw_sp *mlxsw_sp, 4714 const struct rt6_info *rt, 4715 enum mlxsw_sp_ipip_type *ret) 4716 { 4717 return rt->dst.dev && 4718 mlxsw_sp_netdev_ipip_type(mlxsw_sp, rt->dst.dev, ret); 4719 } 4720 4721 static int mlxsw_sp_nexthop6_type_init(struct mlxsw_sp *mlxsw_sp, 4722 struct mlxsw_sp_nexthop_group *nh_grp, 4723 struct mlxsw_sp_nexthop *nh, 4724 const struct rt6_info *rt) 4725 { 4726 const struct mlxsw_sp_ipip_ops *ipip_ops; 4727 struct mlxsw_sp_ipip_entry *ipip_entry; 4728 struct net_device *dev = rt->dst.dev; 4729 struct mlxsw_sp_rif *rif; 4730 int err; 4731 4732 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, dev); 4733 if (ipip_entry) { 4734 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt]; 4735 if (ipip_ops->can_offload(mlxsw_sp, dev, 4736 MLXSW_SP_L3_PROTO_IPV6)) { 4737 nh->type = MLXSW_SP_NEXTHOP_TYPE_IPIP; 4738 mlxsw_sp_nexthop_ipip_init(mlxsw_sp, nh, ipip_entry); 4739 return 0; 4740 } 4741 } 4742 4743 nh->type = MLXSW_SP_NEXTHOP_TYPE_ETH; 4744 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 4745 if (!rif) 4746 return 0; 4747 mlxsw_sp_nexthop_rif_init(nh, rif); 4748 4749 err = mlxsw_sp_nexthop_neigh_init(mlxsw_sp, nh); 4750 if (err) 4751 goto err_nexthop_neigh_init; 4752 4753 return 0; 4754 4755 err_nexthop_neigh_init: 4756 mlxsw_sp_nexthop_rif_fini(nh); 4757 return err; 4758 } 4759 4760 static void mlxsw_sp_nexthop6_type_fini(struct mlxsw_sp *mlxsw_sp, 4761 struct mlxsw_sp_nexthop *nh) 4762 { 4763 mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh); 4764 } 4765 4766 static int mlxsw_sp_nexthop6_init(struct mlxsw_sp *mlxsw_sp, 4767 struct mlxsw_sp_nexthop_group *nh_grp, 4768 struct mlxsw_sp_nexthop *nh, 4769 const struct rt6_info *rt) 4770 { 4771 struct net_device *dev = rt->dst.dev; 4772 4773 nh->nh_grp = nh_grp; 4774 nh->nh_weight = 1; 4775 memcpy(&nh->gw_addr, &rt->rt6i_gateway, sizeof(nh->gw_addr)); 4776 mlxsw_sp_nexthop_counter_alloc(mlxsw_sp, nh); 4777 4778 list_add_tail(&nh->router_list_node, &mlxsw_sp->router->nexthop_list); 4779 4780 if (!dev) 4781 return 0; 4782 nh->ifindex = dev->ifindex; 4783 4784 return mlxsw_sp_nexthop6_type_init(mlxsw_sp, nh_grp, nh, rt); 4785 } 4786 4787 static void mlxsw_sp_nexthop6_fini(struct mlxsw_sp *mlxsw_sp, 4788 struct mlxsw_sp_nexthop *nh) 4789 { 4790 mlxsw_sp_nexthop6_type_fini(mlxsw_sp, nh); 4791 list_del(&nh->router_list_node); 4792 mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh); 4793 } 4794 4795 static bool mlxsw_sp_rt6_is_gateway(const struct mlxsw_sp *mlxsw_sp, 4796 const struct rt6_info *rt) 4797 { 4798 return rt->rt6i_flags & RTF_GATEWAY || 4799 mlxsw_sp_nexthop6_ipip_type(mlxsw_sp, rt, NULL); 4800 } 4801 4802 static struct mlxsw_sp_nexthop_group * 4803 mlxsw_sp_nexthop6_group_create(struct mlxsw_sp *mlxsw_sp, 4804 struct mlxsw_sp_fib6_entry *fib6_entry) 4805 { 4806 struct mlxsw_sp_nexthop_group *nh_grp; 4807 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 4808 struct mlxsw_sp_nexthop *nh; 4809 size_t alloc_size; 4810 int i = 0; 4811 int err; 4812 4813 alloc_size = sizeof(*nh_grp) + 4814 fib6_entry->nrt6 * sizeof(struct mlxsw_sp_nexthop); 4815 nh_grp = kzalloc(alloc_size, GFP_KERNEL); 4816 if (!nh_grp) 4817 return ERR_PTR(-ENOMEM); 4818 INIT_LIST_HEAD(&nh_grp->fib_list); 4819 #if IS_ENABLED(CONFIG_IPV6) 4820 nh_grp->neigh_tbl = &nd_tbl; 4821 #endif 4822 mlxsw_sp_rt6 = list_first_entry(&fib6_entry->rt6_list, 4823 struct mlxsw_sp_rt6, list); 4824 nh_grp->gateway = mlxsw_sp_rt6_is_gateway(mlxsw_sp, mlxsw_sp_rt6->rt); 4825 nh_grp->count = fib6_entry->nrt6; 4826 for (i = 0; i < nh_grp->count; i++) { 4827 struct rt6_info *rt = mlxsw_sp_rt6->rt; 4828 4829 nh = &nh_grp->nexthops[i]; 4830 err = mlxsw_sp_nexthop6_init(mlxsw_sp, nh_grp, nh, rt); 4831 if (err) 4832 goto err_nexthop6_init; 4833 mlxsw_sp_rt6 = list_next_entry(mlxsw_sp_rt6, list); 4834 } 4835 4836 err = mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp); 4837 if (err) 4838 goto err_nexthop_group_insert; 4839 4840 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp); 4841 return nh_grp; 4842 4843 err_nexthop_group_insert: 4844 err_nexthop6_init: 4845 for (i--; i >= 0; i--) { 4846 nh = &nh_grp->nexthops[i]; 4847 mlxsw_sp_nexthop6_fini(mlxsw_sp, nh); 4848 } 4849 kfree(nh_grp); 4850 return ERR_PTR(err); 4851 } 4852 4853 static void 4854 mlxsw_sp_nexthop6_group_destroy(struct mlxsw_sp *mlxsw_sp, 4855 struct mlxsw_sp_nexthop_group *nh_grp) 4856 { 4857 struct mlxsw_sp_nexthop *nh; 4858 int i = nh_grp->count; 4859 4860 mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp); 4861 for (i--; i >= 0; i--) { 4862 nh = &nh_grp->nexthops[i]; 4863 mlxsw_sp_nexthop6_fini(mlxsw_sp, nh); 4864 } 4865 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp); 4866 WARN_ON(nh_grp->adj_index_valid); 4867 kfree(nh_grp); 4868 } 4869 4870 static int mlxsw_sp_nexthop6_group_get(struct mlxsw_sp *mlxsw_sp, 4871 struct mlxsw_sp_fib6_entry *fib6_entry) 4872 { 4873 struct mlxsw_sp_nexthop_group *nh_grp; 4874 4875 nh_grp = mlxsw_sp_nexthop6_group_lookup(mlxsw_sp, fib6_entry); 4876 if (!nh_grp) { 4877 nh_grp = mlxsw_sp_nexthop6_group_create(mlxsw_sp, fib6_entry); 4878 if (IS_ERR(nh_grp)) 4879 return PTR_ERR(nh_grp); 4880 } 4881 4882 list_add_tail(&fib6_entry->common.nexthop_group_node, 4883 &nh_grp->fib_list); 4884 fib6_entry->common.nh_group = nh_grp; 4885 4886 return 0; 4887 } 4888 4889 static void mlxsw_sp_nexthop6_group_put(struct mlxsw_sp *mlxsw_sp, 4890 struct mlxsw_sp_fib_entry *fib_entry) 4891 { 4892 struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group; 4893 4894 list_del(&fib_entry->nexthop_group_node); 4895 if (!list_empty(&nh_grp->fib_list)) 4896 return; 4897 mlxsw_sp_nexthop6_group_destroy(mlxsw_sp, nh_grp); 4898 } 4899 4900 static int 4901 mlxsw_sp_nexthop6_group_update(struct mlxsw_sp *mlxsw_sp, 4902 struct mlxsw_sp_fib6_entry *fib6_entry) 4903 { 4904 struct mlxsw_sp_nexthop_group *old_nh_grp = fib6_entry->common.nh_group; 4905 int err; 4906 4907 fib6_entry->common.nh_group = NULL; 4908 list_del(&fib6_entry->common.nexthop_group_node); 4909 4910 err = mlxsw_sp_nexthop6_group_get(mlxsw_sp, fib6_entry); 4911 if (err) 4912 goto err_nexthop6_group_get; 4913 4914 /* In case this entry is offloaded, then the adjacency index 4915 * currently associated with it in the device's table is that 4916 * of the old group. Start using the new one instead. 4917 */ 4918 err = mlxsw_sp_fib_node_entry_add(mlxsw_sp, &fib6_entry->common); 4919 if (err) 4920 goto err_fib_node_entry_add; 4921 4922 if (list_empty(&old_nh_grp->fib_list)) 4923 mlxsw_sp_nexthop6_group_destroy(mlxsw_sp, old_nh_grp); 4924 4925 return 0; 4926 4927 err_fib_node_entry_add: 4928 mlxsw_sp_nexthop6_group_put(mlxsw_sp, &fib6_entry->common); 4929 err_nexthop6_group_get: 4930 list_add_tail(&fib6_entry->common.nexthop_group_node, 4931 &old_nh_grp->fib_list); 4932 fib6_entry->common.nh_group = old_nh_grp; 4933 return err; 4934 } 4935 4936 static int 4937 mlxsw_sp_fib6_entry_nexthop_add(struct mlxsw_sp *mlxsw_sp, 4938 struct mlxsw_sp_fib6_entry *fib6_entry, 4939 struct rt6_info *rt) 4940 { 4941 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 4942 int err; 4943 4944 mlxsw_sp_rt6 = mlxsw_sp_rt6_create(rt); 4945 if (IS_ERR(mlxsw_sp_rt6)) 4946 return PTR_ERR(mlxsw_sp_rt6); 4947 4948 list_add_tail(&mlxsw_sp_rt6->list, &fib6_entry->rt6_list); 4949 fib6_entry->nrt6++; 4950 4951 err = mlxsw_sp_nexthop6_group_update(mlxsw_sp, fib6_entry); 4952 if (err) 4953 goto err_nexthop6_group_update; 4954 4955 return 0; 4956 4957 err_nexthop6_group_update: 4958 fib6_entry->nrt6--; 4959 list_del(&mlxsw_sp_rt6->list); 4960 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6); 4961 return err; 4962 } 4963 4964 static void 4965 mlxsw_sp_fib6_entry_nexthop_del(struct mlxsw_sp *mlxsw_sp, 4966 struct mlxsw_sp_fib6_entry *fib6_entry, 4967 struct rt6_info *rt) 4968 { 4969 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 4970 4971 mlxsw_sp_rt6 = mlxsw_sp_fib6_entry_rt_find(fib6_entry, rt); 4972 if (WARN_ON(!mlxsw_sp_rt6)) 4973 return; 4974 4975 fib6_entry->nrt6--; 4976 list_del(&mlxsw_sp_rt6->list); 4977 mlxsw_sp_nexthop6_group_update(mlxsw_sp, fib6_entry); 4978 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6); 4979 } 4980 4981 static void mlxsw_sp_fib6_entry_type_set(struct mlxsw_sp *mlxsw_sp, 4982 struct mlxsw_sp_fib_entry *fib_entry, 4983 const struct rt6_info *rt) 4984 { 4985 /* Packets hitting RTF_REJECT routes need to be discarded by the 4986 * stack. We can rely on their destination device not having a 4987 * RIF (it's the loopback device) and can thus use action type 4988 * local, which will cause them to be trapped with a lower 4989 * priority than packets that need to be locally received. 4990 */ 4991 if (rt->rt6i_flags & (RTF_LOCAL | RTF_ANYCAST)) 4992 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP; 4993 else if (rt->rt6i_flags & RTF_REJECT) 4994 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL; 4995 else if (mlxsw_sp_rt6_is_gateway(mlxsw_sp, rt)) 4996 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_REMOTE; 4997 else 4998 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL; 4999 } 5000 5001 static void 5002 mlxsw_sp_fib6_entry_rt_destroy_all(struct mlxsw_sp_fib6_entry *fib6_entry) 5003 { 5004 struct mlxsw_sp_rt6 *mlxsw_sp_rt6, *tmp; 5005 5006 list_for_each_entry_safe(mlxsw_sp_rt6, tmp, &fib6_entry->rt6_list, 5007 list) { 5008 fib6_entry->nrt6--; 5009 list_del(&mlxsw_sp_rt6->list); 5010 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6); 5011 } 5012 } 5013 5014 static struct mlxsw_sp_fib6_entry * 5015 mlxsw_sp_fib6_entry_create(struct mlxsw_sp *mlxsw_sp, 5016 struct mlxsw_sp_fib_node *fib_node, 5017 struct rt6_info *rt) 5018 { 5019 struct mlxsw_sp_fib6_entry *fib6_entry; 5020 struct mlxsw_sp_fib_entry *fib_entry; 5021 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 5022 int err; 5023 5024 fib6_entry = kzalloc(sizeof(*fib6_entry), GFP_KERNEL); 5025 if (!fib6_entry) 5026 return ERR_PTR(-ENOMEM); 5027 fib_entry = &fib6_entry->common; 5028 5029 mlxsw_sp_rt6 = mlxsw_sp_rt6_create(rt); 5030 if (IS_ERR(mlxsw_sp_rt6)) { 5031 err = PTR_ERR(mlxsw_sp_rt6); 5032 goto err_rt6_create; 5033 } 5034 5035 mlxsw_sp_fib6_entry_type_set(mlxsw_sp, fib_entry, mlxsw_sp_rt6->rt); 5036 5037 INIT_LIST_HEAD(&fib6_entry->rt6_list); 5038 list_add_tail(&mlxsw_sp_rt6->list, &fib6_entry->rt6_list); 5039 fib6_entry->nrt6 = 1; 5040 err = mlxsw_sp_nexthop6_group_get(mlxsw_sp, fib6_entry); 5041 if (err) 5042 goto err_nexthop6_group_get; 5043 5044 fib_entry->fib_node = fib_node; 5045 5046 return fib6_entry; 5047 5048 err_nexthop6_group_get: 5049 list_del(&mlxsw_sp_rt6->list); 5050 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6); 5051 err_rt6_create: 5052 kfree(fib6_entry); 5053 return ERR_PTR(err); 5054 } 5055 5056 static void mlxsw_sp_fib6_entry_destroy(struct mlxsw_sp *mlxsw_sp, 5057 struct mlxsw_sp_fib6_entry *fib6_entry) 5058 { 5059 mlxsw_sp_nexthop6_group_put(mlxsw_sp, &fib6_entry->common); 5060 mlxsw_sp_fib6_entry_rt_destroy_all(fib6_entry); 5061 WARN_ON(fib6_entry->nrt6); 5062 kfree(fib6_entry); 5063 } 5064 5065 static struct mlxsw_sp_fib6_entry * 5066 mlxsw_sp_fib6_node_entry_find(const struct mlxsw_sp_fib_node *fib_node, 5067 const struct rt6_info *nrt, bool replace) 5068 { 5069 struct mlxsw_sp_fib6_entry *fib6_entry, *fallback = NULL; 5070 5071 list_for_each_entry(fib6_entry, &fib_node->entry_list, common.list) { 5072 struct rt6_info *rt = mlxsw_sp_fib6_entry_rt(fib6_entry); 5073 5074 if (rt->rt6i_table->tb6_id > nrt->rt6i_table->tb6_id) 5075 continue; 5076 if (rt->rt6i_table->tb6_id != nrt->rt6i_table->tb6_id) 5077 break; 5078 if (replace && rt->rt6i_metric == nrt->rt6i_metric) { 5079 if (mlxsw_sp_fib6_rt_can_mp(rt) == 5080 mlxsw_sp_fib6_rt_can_mp(nrt)) 5081 return fib6_entry; 5082 if (mlxsw_sp_fib6_rt_can_mp(nrt)) 5083 fallback = fallback ?: fib6_entry; 5084 } 5085 if (rt->rt6i_metric > nrt->rt6i_metric) 5086 return fallback ?: fib6_entry; 5087 } 5088 5089 return fallback; 5090 } 5091 5092 static int 5093 mlxsw_sp_fib6_node_list_insert(struct mlxsw_sp_fib6_entry *new6_entry, 5094 bool replace) 5095 { 5096 struct mlxsw_sp_fib_node *fib_node = new6_entry->common.fib_node; 5097 struct rt6_info *nrt = mlxsw_sp_fib6_entry_rt(new6_entry); 5098 struct mlxsw_sp_fib6_entry *fib6_entry; 5099 5100 fib6_entry = mlxsw_sp_fib6_node_entry_find(fib_node, nrt, replace); 5101 5102 if (replace && WARN_ON(!fib6_entry)) 5103 return -EINVAL; 5104 5105 if (fib6_entry) { 5106 list_add_tail(&new6_entry->common.list, 5107 &fib6_entry->common.list); 5108 } else { 5109 struct mlxsw_sp_fib6_entry *last; 5110 5111 list_for_each_entry(last, &fib_node->entry_list, common.list) { 5112 struct rt6_info *rt = mlxsw_sp_fib6_entry_rt(last); 5113 5114 if (nrt->rt6i_table->tb6_id > rt->rt6i_table->tb6_id) 5115 break; 5116 fib6_entry = last; 5117 } 5118 5119 if (fib6_entry) 5120 list_add(&new6_entry->common.list, 5121 &fib6_entry->common.list); 5122 else 5123 list_add(&new6_entry->common.list, 5124 &fib_node->entry_list); 5125 } 5126 5127 return 0; 5128 } 5129 5130 static void 5131 mlxsw_sp_fib6_node_list_remove(struct mlxsw_sp_fib6_entry *fib6_entry) 5132 { 5133 list_del(&fib6_entry->common.list); 5134 } 5135 5136 static int mlxsw_sp_fib6_node_entry_link(struct mlxsw_sp *mlxsw_sp, 5137 struct mlxsw_sp_fib6_entry *fib6_entry, 5138 bool replace) 5139 { 5140 int err; 5141 5142 err = mlxsw_sp_fib6_node_list_insert(fib6_entry, replace); 5143 if (err) 5144 return err; 5145 5146 err = mlxsw_sp_fib_node_entry_add(mlxsw_sp, &fib6_entry->common); 5147 if (err) 5148 goto err_fib_node_entry_add; 5149 5150 return 0; 5151 5152 err_fib_node_entry_add: 5153 mlxsw_sp_fib6_node_list_remove(fib6_entry); 5154 return err; 5155 } 5156 5157 static void 5158 mlxsw_sp_fib6_node_entry_unlink(struct mlxsw_sp *mlxsw_sp, 5159 struct mlxsw_sp_fib6_entry *fib6_entry) 5160 { 5161 mlxsw_sp_fib_node_entry_del(mlxsw_sp, &fib6_entry->common); 5162 mlxsw_sp_fib6_node_list_remove(fib6_entry); 5163 } 5164 5165 static struct mlxsw_sp_fib6_entry * 5166 mlxsw_sp_fib6_entry_lookup(struct mlxsw_sp *mlxsw_sp, 5167 const struct rt6_info *rt) 5168 { 5169 struct mlxsw_sp_fib6_entry *fib6_entry; 5170 struct mlxsw_sp_fib_node *fib_node; 5171 struct mlxsw_sp_fib *fib; 5172 struct mlxsw_sp_vr *vr; 5173 5174 vr = mlxsw_sp_vr_find(mlxsw_sp, rt->rt6i_table->tb6_id); 5175 if (!vr) 5176 return NULL; 5177 fib = mlxsw_sp_vr_fib(vr, MLXSW_SP_L3_PROTO_IPV6); 5178 5179 fib_node = mlxsw_sp_fib_node_lookup(fib, &rt->rt6i_dst.addr, 5180 sizeof(rt->rt6i_dst.addr), 5181 rt->rt6i_dst.plen); 5182 if (!fib_node) 5183 return NULL; 5184 5185 list_for_each_entry(fib6_entry, &fib_node->entry_list, common.list) { 5186 struct rt6_info *iter_rt = mlxsw_sp_fib6_entry_rt(fib6_entry); 5187 5188 if (rt->rt6i_table->tb6_id == iter_rt->rt6i_table->tb6_id && 5189 rt->rt6i_metric == iter_rt->rt6i_metric && 5190 mlxsw_sp_fib6_entry_rt_find(fib6_entry, rt)) 5191 return fib6_entry; 5192 } 5193 5194 return NULL; 5195 } 5196 5197 static void mlxsw_sp_fib6_entry_replace(struct mlxsw_sp *mlxsw_sp, 5198 struct mlxsw_sp_fib6_entry *fib6_entry, 5199 bool replace) 5200 { 5201 struct mlxsw_sp_fib_node *fib_node = fib6_entry->common.fib_node; 5202 struct mlxsw_sp_fib6_entry *replaced; 5203 5204 if (!replace) 5205 return; 5206 5207 replaced = list_next_entry(fib6_entry, common.list); 5208 5209 mlxsw_sp_fib6_node_entry_unlink(mlxsw_sp, replaced); 5210 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, replaced); 5211 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 5212 } 5213 5214 static int mlxsw_sp_router_fib6_add(struct mlxsw_sp *mlxsw_sp, 5215 struct rt6_info *rt, bool replace) 5216 { 5217 struct mlxsw_sp_fib6_entry *fib6_entry; 5218 struct mlxsw_sp_fib_node *fib_node; 5219 int err; 5220 5221 if (mlxsw_sp->router->aborted) 5222 return 0; 5223 5224 if (rt->rt6i_src.plen) 5225 return -EINVAL; 5226 5227 if (mlxsw_sp_fib6_rt_should_ignore(rt)) 5228 return 0; 5229 5230 fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, rt->rt6i_table->tb6_id, 5231 &rt->rt6i_dst.addr, 5232 sizeof(rt->rt6i_dst.addr), 5233 rt->rt6i_dst.plen, 5234 MLXSW_SP_L3_PROTO_IPV6); 5235 if (IS_ERR(fib_node)) 5236 return PTR_ERR(fib_node); 5237 5238 /* Before creating a new entry, try to append route to an existing 5239 * multipath entry. 5240 */ 5241 fib6_entry = mlxsw_sp_fib6_node_mp_entry_find(fib_node, rt, replace); 5242 if (fib6_entry) { 5243 err = mlxsw_sp_fib6_entry_nexthop_add(mlxsw_sp, fib6_entry, rt); 5244 if (err) 5245 goto err_fib6_entry_nexthop_add; 5246 return 0; 5247 } 5248 5249 fib6_entry = mlxsw_sp_fib6_entry_create(mlxsw_sp, fib_node, rt); 5250 if (IS_ERR(fib6_entry)) { 5251 err = PTR_ERR(fib6_entry); 5252 goto err_fib6_entry_create; 5253 } 5254 5255 err = mlxsw_sp_fib6_node_entry_link(mlxsw_sp, fib6_entry, replace); 5256 if (err) 5257 goto err_fib6_node_entry_link; 5258 5259 mlxsw_sp_fib6_entry_replace(mlxsw_sp, fib6_entry, replace); 5260 5261 return 0; 5262 5263 err_fib6_node_entry_link: 5264 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry); 5265 err_fib6_entry_create: 5266 err_fib6_entry_nexthop_add: 5267 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 5268 return err; 5269 } 5270 5271 static void mlxsw_sp_router_fib6_del(struct mlxsw_sp *mlxsw_sp, 5272 struct rt6_info *rt) 5273 { 5274 struct mlxsw_sp_fib6_entry *fib6_entry; 5275 struct mlxsw_sp_fib_node *fib_node; 5276 5277 if (mlxsw_sp->router->aborted) 5278 return; 5279 5280 if (mlxsw_sp_fib6_rt_should_ignore(rt)) 5281 return; 5282 5283 fib6_entry = mlxsw_sp_fib6_entry_lookup(mlxsw_sp, rt); 5284 if (WARN_ON(!fib6_entry)) 5285 return; 5286 5287 /* If route is part of a multipath entry, but not the last one 5288 * removed, then only reduce its nexthop group. 5289 */ 5290 if (!list_is_singular(&fib6_entry->rt6_list)) { 5291 mlxsw_sp_fib6_entry_nexthop_del(mlxsw_sp, fib6_entry, rt); 5292 return; 5293 } 5294 5295 fib_node = fib6_entry->common.fib_node; 5296 5297 mlxsw_sp_fib6_node_entry_unlink(mlxsw_sp, fib6_entry); 5298 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry); 5299 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 5300 } 5301 5302 static int __mlxsw_sp_router_set_abort_trap(struct mlxsw_sp *mlxsw_sp, 5303 enum mlxsw_reg_ralxx_protocol proto, 5304 u8 tree_id) 5305 { 5306 char ralta_pl[MLXSW_REG_RALTA_LEN]; 5307 char ralst_pl[MLXSW_REG_RALST_LEN]; 5308 int i, err; 5309 5310 mlxsw_reg_ralta_pack(ralta_pl, true, proto, tree_id); 5311 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl); 5312 if (err) 5313 return err; 5314 5315 mlxsw_reg_ralst_pack(ralst_pl, 0xff, tree_id); 5316 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralst), ralst_pl); 5317 if (err) 5318 return err; 5319 5320 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) { 5321 struct mlxsw_sp_vr *vr = &mlxsw_sp->router->vrs[i]; 5322 char raltb_pl[MLXSW_REG_RALTB_LEN]; 5323 char ralue_pl[MLXSW_REG_RALUE_LEN]; 5324 5325 mlxsw_reg_raltb_pack(raltb_pl, vr->id, proto, tree_id); 5326 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), 5327 raltb_pl); 5328 if (err) 5329 return err; 5330 5331 mlxsw_reg_ralue_pack(ralue_pl, proto, 5332 MLXSW_REG_RALUE_OP_WRITE_WRITE, vr->id, 0); 5333 mlxsw_reg_ralue_act_ip2me_pack(ralue_pl); 5334 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), 5335 ralue_pl); 5336 if (err) 5337 return err; 5338 } 5339 5340 return 0; 5341 } 5342 5343 static int mlxsw_sp_router_fibmr_add(struct mlxsw_sp *mlxsw_sp, 5344 struct mfc_entry_notifier_info *men_info, 5345 bool replace) 5346 { 5347 struct mlxsw_sp_vr *vr; 5348 5349 if (mlxsw_sp->router->aborted) 5350 return 0; 5351 5352 vr = mlxsw_sp_vr_get(mlxsw_sp, men_info->tb_id, NULL); 5353 if (IS_ERR(vr)) 5354 return PTR_ERR(vr); 5355 5356 return mlxsw_sp_mr_route4_add(vr->mr4_table, men_info->mfc, replace); 5357 } 5358 5359 static void mlxsw_sp_router_fibmr_del(struct mlxsw_sp *mlxsw_sp, 5360 struct mfc_entry_notifier_info *men_info) 5361 { 5362 struct mlxsw_sp_vr *vr; 5363 5364 if (mlxsw_sp->router->aborted) 5365 return; 5366 5367 vr = mlxsw_sp_vr_find(mlxsw_sp, men_info->tb_id); 5368 if (WARN_ON(!vr)) 5369 return; 5370 5371 mlxsw_sp_mr_route4_del(vr->mr4_table, men_info->mfc); 5372 mlxsw_sp_vr_put(vr); 5373 } 5374 5375 static int 5376 mlxsw_sp_router_fibmr_vif_add(struct mlxsw_sp *mlxsw_sp, 5377 struct vif_entry_notifier_info *ven_info) 5378 { 5379 struct mlxsw_sp_rif *rif; 5380 struct mlxsw_sp_vr *vr; 5381 5382 if (mlxsw_sp->router->aborted) 5383 return 0; 5384 5385 vr = mlxsw_sp_vr_get(mlxsw_sp, ven_info->tb_id, NULL); 5386 if (IS_ERR(vr)) 5387 return PTR_ERR(vr); 5388 5389 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, ven_info->dev); 5390 return mlxsw_sp_mr_vif_add(vr->mr4_table, ven_info->dev, 5391 ven_info->vif_index, 5392 ven_info->vif_flags, rif); 5393 } 5394 5395 static void 5396 mlxsw_sp_router_fibmr_vif_del(struct mlxsw_sp *mlxsw_sp, 5397 struct vif_entry_notifier_info *ven_info) 5398 { 5399 struct mlxsw_sp_vr *vr; 5400 5401 if (mlxsw_sp->router->aborted) 5402 return; 5403 5404 vr = mlxsw_sp_vr_find(mlxsw_sp, ven_info->tb_id); 5405 if (WARN_ON(!vr)) 5406 return; 5407 5408 mlxsw_sp_mr_vif_del(vr->mr4_table, ven_info->vif_index); 5409 mlxsw_sp_vr_put(vr); 5410 } 5411 5412 static int mlxsw_sp_router_set_abort_trap(struct mlxsw_sp *mlxsw_sp) 5413 { 5414 enum mlxsw_reg_ralxx_protocol proto = MLXSW_REG_RALXX_PROTOCOL_IPV4; 5415 int err; 5416 5417 err = __mlxsw_sp_router_set_abort_trap(mlxsw_sp, proto, 5418 MLXSW_SP_LPM_TREE_MIN); 5419 if (err) 5420 return err; 5421 5422 /* The multicast router code does not need an abort trap as by default, 5423 * packets that don't match any routes are trapped to the CPU. 5424 */ 5425 5426 proto = MLXSW_REG_RALXX_PROTOCOL_IPV6; 5427 return __mlxsw_sp_router_set_abort_trap(mlxsw_sp, proto, 5428 MLXSW_SP_LPM_TREE_MIN + 1); 5429 } 5430 5431 static void mlxsw_sp_fib4_node_flush(struct mlxsw_sp *mlxsw_sp, 5432 struct mlxsw_sp_fib_node *fib_node) 5433 { 5434 struct mlxsw_sp_fib4_entry *fib4_entry, *tmp; 5435 5436 list_for_each_entry_safe(fib4_entry, tmp, &fib_node->entry_list, 5437 common.list) { 5438 bool do_break = &tmp->common.list == &fib_node->entry_list; 5439 5440 mlxsw_sp_fib4_node_entry_unlink(mlxsw_sp, fib4_entry); 5441 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry); 5442 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 5443 /* Break when entry list is empty and node was freed. 5444 * Otherwise, we'll access freed memory in the next 5445 * iteration. 5446 */ 5447 if (do_break) 5448 break; 5449 } 5450 } 5451 5452 static void mlxsw_sp_fib6_node_flush(struct mlxsw_sp *mlxsw_sp, 5453 struct mlxsw_sp_fib_node *fib_node) 5454 { 5455 struct mlxsw_sp_fib6_entry *fib6_entry, *tmp; 5456 5457 list_for_each_entry_safe(fib6_entry, tmp, &fib_node->entry_list, 5458 common.list) { 5459 bool do_break = &tmp->common.list == &fib_node->entry_list; 5460 5461 mlxsw_sp_fib6_node_entry_unlink(mlxsw_sp, fib6_entry); 5462 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry); 5463 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 5464 if (do_break) 5465 break; 5466 } 5467 } 5468 5469 static void mlxsw_sp_fib_node_flush(struct mlxsw_sp *mlxsw_sp, 5470 struct mlxsw_sp_fib_node *fib_node) 5471 { 5472 switch (fib_node->fib->proto) { 5473 case MLXSW_SP_L3_PROTO_IPV4: 5474 mlxsw_sp_fib4_node_flush(mlxsw_sp, fib_node); 5475 break; 5476 case MLXSW_SP_L3_PROTO_IPV6: 5477 mlxsw_sp_fib6_node_flush(mlxsw_sp, fib_node); 5478 break; 5479 } 5480 } 5481 5482 static void mlxsw_sp_vr_fib_flush(struct mlxsw_sp *mlxsw_sp, 5483 struct mlxsw_sp_vr *vr, 5484 enum mlxsw_sp_l3proto proto) 5485 { 5486 struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto); 5487 struct mlxsw_sp_fib_node *fib_node, *tmp; 5488 5489 list_for_each_entry_safe(fib_node, tmp, &fib->node_list, list) { 5490 bool do_break = &tmp->list == &fib->node_list; 5491 5492 mlxsw_sp_fib_node_flush(mlxsw_sp, fib_node); 5493 if (do_break) 5494 break; 5495 } 5496 } 5497 5498 static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp) 5499 { 5500 int i; 5501 5502 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) { 5503 struct mlxsw_sp_vr *vr = &mlxsw_sp->router->vrs[i]; 5504 5505 if (!mlxsw_sp_vr_is_used(vr)) 5506 continue; 5507 5508 mlxsw_sp_mr_table_flush(vr->mr4_table); 5509 mlxsw_sp_vr_fib_flush(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV4); 5510 5511 /* If virtual router was only used for IPv4, then it's no 5512 * longer used. 5513 */ 5514 if (!mlxsw_sp_vr_is_used(vr)) 5515 continue; 5516 mlxsw_sp_vr_fib_flush(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV6); 5517 } 5518 } 5519 5520 static void mlxsw_sp_router_fib_abort(struct mlxsw_sp *mlxsw_sp) 5521 { 5522 int err; 5523 5524 if (mlxsw_sp->router->aborted) 5525 return; 5526 dev_warn(mlxsw_sp->bus_info->dev, "FIB abort triggered. Note that FIB entries are no longer being offloaded to this device.\n"); 5527 mlxsw_sp_router_fib_flush(mlxsw_sp); 5528 mlxsw_sp->router->aborted = true; 5529 err = mlxsw_sp_router_set_abort_trap(mlxsw_sp); 5530 if (err) 5531 dev_warn(mlxsw_sp->bus_info->dev, "Failed to set abort trap.\n"); 5532 } 5533 5534 struct mlxsw_sp_fib_event_work { 5535 struct work_struct work; 5536 union { 5537 struct fib6_entry_notifier_info fen6_info; 5538 struct fib_entry_notifier_info fen_info; 5539 struct fib_rule_notifier_info fr_info; 5540 struct fib_nh_notifier_info fnh_info; 5541 struct mfc_entry_notifier_info men_info; 5542 struct vif_entry_notifier_info ven_info; 5543 }; 5544 struct mlxsw_sp *mlxsw_sp; 5545 unsigned long event; 5546 }; 5547 5548 static void mlxsw_sp_router_fib4_event_work(struct work_struct *work) 5549 { 5550 struct mlxsw_sp_fib_event_work *fib_work = 5551 container_of(work, struct mlxsw_sp_fib_event_work, work); 5552 struct mlxsw_sp *mlxsw_sp = fib_work->mlxsw_sp; 5553 bool replace, append; 5554 int err; 5555 5556 /* Protect internal structures from changes */ 5557 rtnl_lock(); 5558 switch (fib_work->event) { 5559 case FIB_EVENT_ENTRY_REPLACE: /* fall through */ 5560 case FIB_EVENT_ENTRY_APPEND: /* fall through */ 5561 case FIB_EVENT_ENTRY_ADD: 5562 replace = fib_work->event == FIB_EVENT_ENTRY_REPLACE; 5563 append = fib_work->event == FIB_EVENT_ENTRY_APPEND; 5564 err = mlxsw_sp_router_fib4_add(mlxsw_sp, &fib_work->fen_info, 5565 replace, append); 5566 if (err) 5567 mlxsw_sp_router_fib_abort(mlxsw_sp); 5568 fib_info_put(fib_work->fen_info.fi); 5569 break; 5570 case FIB_EVENT_ENTRY_DEL: 5571 mlxsw_sp_router_fib4_del(mlxsw_sp, &fib_work->fen_info); 5572 fib_info_put(fib_work->fen_info.fi); 5573 break; 5574 case FIB_EVENT_RULE_ADD: 5575 /* if we get here, a rule was added that we do not support. 5576 * just do the fib_abort 5577 */ 5578 mlxsw_sp_router_fib_abort(mlxsw_sp); 5579 break; 5580 case FIB_EVENT_NH_ADD: /* fall through */ 5581 case FIB_EVENT_NH_DEL: 5582 mlxsw_sp_nexthop4_event(mlxsw_sp, fib_work->event, 5583 fib_work->fnh_info.fib_nh); 5584 fib_info_put(fib_work->fnh_info.fib_nh->nh_parent); 5585 break; 5586 } 5587 rtnl_unlock(); 5588 kfree(fib_work); 5589 } 5590 5591 static void mlxsw_sp_router_fib6_event_work(struct work_struct *work) 5592 { 5593 struct mlxsw_sp_fib_event_work *fib_work = 5594 container_of(work, struct mlxsw_sp_fib_event_work, work); 5595 struct mlxsw_sp *mlxsw_sp = fib_work->mlxsw_sp; 5596 bool replace; 5597 int err; 5598 5599 rtnl_lock(); 5600 switch (fib_work->event) { 5601 case FIB_EVENT_ENTRY_REPLACE: /* fall through */ 5602 case FIB_EVENT_ENTRY_ADD: 5603 replace = fib_work->event == FIB_EVENT_ENTRY_REPLACE; 5604 err = mlxsw_sp_router_fib6_add(mlxsw_sp, 5605 fib_work->fen6_info.rt, replace); 5606 if (err) 5607 mlxsw_sp_router_fib_abort(mlxsw_sp); 5608 mlxsw_sp_rt6_release(fib_work->fen6_info.rt); 5609 break; 5610 case FIB_EVENT_ENTRY_DEL: 5611 mlxsw_sp_router_fib6_del(mlxsw_sp, fib_work->fen6_info.rt); 5612 mlxsw_sp_rt6_release(fib_work->fen6_info.rt); 5613 break; 5614 case FIB_EVENT_RULE_ADD: 5615 /* if we get here, a rule was added that we do not support. 5616 * just do the fib_abort 5617 */ 5618 mlxsw_sp_router_fib_abort(mlxsw_sp); 5619 break; 5620 } 5621 rtnl_unlock(); 5622 kfree(fib_work); 5623 } 5624 5625 static void mlxsw_sp_router_fibmr_event_work(struct work_struct *work) 5626 { 5627 struct mlxsw_sp_fib_event_work *fib_work = 5628 container_of(work, struct mlxsw_sp_fib_event_work, work); 5629 struct mlxsw_sp *mlxsw_sp = fib_work->mlxsw_sp; 5630 bool replace; 5631 int err; 5632 5633 rtnl_lock(); 5634 switch (fib_work->event) { 5635 case FIB_EVENT_ENTRY_REPLACE: /* fall through */ 5636 case FIB_EVENT_ENTRY_ADD: 5637 replace = fib_work->event == FIB_EVENT_ENTRY_REPLACE; 5638 5639 err = mlxsw_sp_router_fibmr_add(mlxsw_sp, &fib_work->men_info, 5640 replace); 5641 if (err) 5642 mlxsw_sp_router_fib_abort(mlxsw_sp); 5643 ipmr_cache_put(fib_work->men_info.mfc); 5644 break; 5645 case FIB_EVENT_ENTRY_DEL: 5646 mlxsw_sp_router_fibmr_del(mlxsw_sp, &fib_work->men_info); 5647 ipmr_cache_put(fib_work->men_info.mfc); 5648 break; 5649 case FIB_EVENT_VIF_ADD: 5650 err = mlxsw_sp_router_fibmr_vif_add(mlxsw_sp, 5651 &fib_work->ven_info); 5652 if (err) 5653 mlxsw_sp_router_fib_abort(mlxsw_sp); 5654 dev_put(fib_work->ven_info.dev); 5655 break; 5656 case FIB_EVENT_VIF_DEL: 5657 mlxsw_sp_router_fibmr_vif_del(mlxsw_sp, 5658 &fib_work->ven_info); 5659 dev_put(fib_work->ven_info.dev); 5660 break; 5661 case FIB_EVENT_RULE_ADD: 5662 /* if we get here, a rule was added that we do not support. 5663 * just do the fib_abort 5664 */ 5665 mlxsw_sp_router_fib_abort(mlxsw_sp); 5666 break; 5667 } 5668 rtnl_unlock(); 5669 kfree(fib_work); 5670 } 5671 5672 static void mlxsw_sp_router_fib4_event(struct mlxsw_sp_fib_event_work *fib_work, 5673 struct fib_notifier_info *info) 5674 { 5675 struct fib_entry_notifier_info *fen_info; 5676 struct fib_nh_notifier_info *fnh_info; 5677 5678 switch (fib_work->event) { 5679 case FIB_EVENT_ENTRY_REPLACE: /* fall through */ 5680 case FIB_EVENT_ENTRY_APPEND: /* fall through */ 5681 case FIB_EVENT_ENTRY_ADD: /* fall through */ 5682 case FIB_EVENT_ENTRY_DEL: 5683 fen_info = container_of(info, struct fib_entry_notifier_info, 5684 info); 5685 fib_work->fen_info = *fen_info; 5686 /* Take reference on fib_info to prevent it from being 5687 * freed while work is queued. Release it afterwards. 5688 */ 5689 fib_info_hold(fib_work->fen_info.fi); 5690 break; 5691 case FIB_EVENT_NH_ADD: /* fall through */ 5692 case FIB_EVENT_NH_DEL: 5693 fnh_info = container_of(info, struct fib_nh_notifier_info, 5694 info); 5695 fib_work->fnh_info = *fnh_info; 5696 fib_info_hold(fib_work->fnh_info.fib_nh->nh_parent); 5697 break; 5698 } 5699 } 5700 5701 static void mlxsw_sp_router_fib6_event(struct mlxsw_sp_fib_event_work *fib_work, 5702 struct fib_notifier_info *info) 5703 { 5704 struct fib6_entry_notifier_info *fen6_info; 5705 5706 switch (fib_work->event) { 5707 case FIB_EVENT_ENTRY_REPLACE: /* fall through */ 5708 case FIB_EVENT_ENTRY_ADD: /* fall through */ 5709 case FIB_EVENT_ENTRY_DEL: 5710 fen6_info = container_of(info, struct fib6_entry_notifier_info, 5711 info); 5712 fib_work->fen6_info = *fen6_info; 5713 rt6_hold(fib_work->fen6_info.rt); 5714 break; 5715 } 5716 } 5717 5718 static void 5719 mlxsw_sp_router_fibmr_event(struct mlxsw_sp_fib_event_work *fib_work, 5720 struct fib_notifier_info *info) 5721 { 5722 switch (fib_work->event) { 5723 case FIB_EVENT_ENTRY_REPLACE: /* fall through */ 5724 case FIB_EVENT_ENTRY_ADD: /* fall through */ 5725 case FIB_EVENT_ENTRY_DEL: 5726 memcpy(&fib_work->men_info, info, sizeof(fib_work->men_info)); 5727 ipmr_cache_hold(fib_work->men_info.mfc); 5728 break; 5729 case FIB_EVENT_VIF_ADD: /* fall through */ 5730 case FIB_EVENT_VIF_DEL: 5731 memcpy(&fib_work->ven_info, info, sizeof(fib_work->ven_info)); 5732 dev_hold(fib_work->ven_info.dev); 5733 break; 5734 } 5735 } 5736 5737 static int mlxsw_sp_router_fib_rule_event(unsigned long event, 5738 struct fib_notifier_info *info, 5739 struct mlxsw_sp *mlxsw_sp) 5740 { 5741 struct netlink_ext_ack *extack = info->extack; 5742 struct fib_rule_notifier_info *fr_info; 5743 struct fib_rule *rule; 5744 int err = 0; 5745 5746 /* nothing to do at the moment */ 5747 if (event == FIB_EVENT_RULE_DEL) 5748 return 0; 5749 5750 if (mlxsw_sp->router->aborted) 5751 return 0; 5752 5753 fr_info = container_of(info, struct fib_rule_notifier_info, info); 5754 rule = fr_info->rule; 5755 5756 switch (info->family) { 5757 case AF_INET: 5758 if (!fib4_rule_default(rule) && !rule->l3mdev) 5759 err = -1; 5760 break; 5761 case AF_INET6: 5762 if (!fib6_rule_default(rule) && !rule->l3mdev) 5763 err = -1; 5764 break; 5765 case RTNL_FAMILY_IPMR: 5766 if (!ipmr_rule_default(rule) && !rule->l3mdev) 5767 err = -1; 5768 break; 5769 } 5770 5771 if (err < 0) 5772 NL_SET_ERR_MSG(extack, "spectrum: FIB rules not supported. Aborting offload"); 5773 5774 return err; 5775 } 5776 5777 /* Called with rcu_read_lock() */ 5778 static int mlxsw_sp_router_fib_event(struct notifier_block *nb, 5779 unsigned long event, void *ptr) 5780 { 5781 struct mlxsw_sp_fib_event_work *fib_work; 5782 struct fib_notifier_info *info = ptr; 5783 struct mlxsw_sp_router *router; 5784 int err; 5785 5786 if (!net_eq(info->net, &init_net) || 5787 (info->family != AF_INET && info->family != AF_INET6 && 5788 info->family != RTNL_FAMILY_IPMR)) 5789 return NOTIFY_DONE; 5790 5791 router = container_of(nb, struct mlxsw_sp_router, fib_nb); 5792 5793 switch (event) { 5794 case FIB_EVENT_RULE_ADD: /* fall through */ 5795 case FIB_EVENT_RULE_DEL: 5796 err = mlxsw_sp_router_fib_rule_event(event, info, 5797 router->mlxsw_sp); 5798 if (!err) 5799 return NOTIFY_DONE; 5800 } 5801 5802 fib_work = kzalloc(sizeof(*fib_work), GFP_ATOMIC); 5803 if (WARN_ON(!fib_work)) 5804 return NOTIFY_BAD; 5805 5806 fib_work->mlxsw_sp = router->mlxsw_sp; 5807 fib_work->event = event; 5808 5809 switch (info->family) { 5810 case AF_INET: 5811 INIT_WORK(&fib_work->work, mlxsw_sp_router_fib4_event_work); 5812 mlxsw_sp_router_fib4_event(fib_work, info); 5813 break; 5814 case AF_INET6: 5815 INIT_WORK(&fib_work->work, mlxsw_sp_router_fib6_event_work); 5816 mlxsw_sp_router_fib6_event(fib_work, info); 5817 break; 5818 case RTNL_FAMILY_IPMR: 5819 INIT_WORK(&fib_work->work, mlxsw_sp_router_fibmr_event_work); 5820 mlxsw_sp_router_fibmr_event(fib_work, info); 5821 break; 5822 } 5823 5824 mlxsw_core_schedule_work(&fib_work->work); 5825 5826 return NOTIFY_DONE; 5827 } 5828 5829 static struct mlxsw_sp_rif * 5830 mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp, 5831 const struct net_device *dev) 5832 { 5833 int i; 5834 5835 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++) 5836 if (mlxsw_sp->router->rifs[i] && 5837 mlxsw_sp->router->rifs[i]->dev == dev) 5838 return mlxsw_sp->router->rifs[i]; 5839 5840 return NULL; 5841 } 5842 5843 static int mlxsw_sp_router_rif_disable(struct mlxsw_sp *mlxsw_sp, u16 rif) 5844 { 5845 char ritr_pl[MLXSW_REG_RITR_LEN]; 5846 int err; 5847 5848 mlxsw_reg_ritr_rif_pack(ritr_pl, rif); 5849 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 5850 if (WARN_ON_ONCE(err)) 5851 return err; 5852 5853 mlxsw_reg_ritr_enable_set(ritr_pl, false); 5854 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 5855 } 5856 5857 static void mlxsw_sp_router_rif_gone_sync(struct mlxsw_sp *mlxsw_sp, 5858 struct mlxsw_sp_rif *rif) 5859 { 5860 mlxsw_sp_router_rif_disable(mlxsw_sp, rif->rif_index); 5861 mlxsw_sp_nexthop_rif_gone_sync(mlxsw_sp, rif); 5862 mlxsw_sp_neigh_rif_gone_sync(mlxsw_sp, rif); 5863 } 5864 5865 static bool 5866 mlxsw_sp_rif_should_config(struct mlxsw_sp_rif *rif, struct net_device *dev, 5867 unsigned long event) 5868 { 5869 struct inet6_dev *inet6_dev; 5870 bool addr_list_empty = true; 5871 struct in_device *idev; 5872 5873 switch (event) { 5874 case NETDEV_UP: 5875 return rif == NULL; 5876 case NETDEV_DOWN: 5877 idev = __in_dev_get_rtnl(dev); 5878 if (idev && idev->ifa_list) 5879 addr_list_empty = false; 5880 5881 inet6_dev = __in6_dev_get(dev); 5882 if (addr_list_empty && inet6_dev && 5883 !list_empty(&inet6_dev->addr_list)) 5884 addr_list_empty = false; 5885 5886 if (rif && addr_list_empty && 5887 !netif_is_l3_slave(rif->dev)) 5888 return true; 5889 /* It is possible we already removed the RIF ourselves 5890 * if it was assigned to a netdev that is now a bridge 5891 * or LAG slave. 5892 */ 5893 return false; 5894 } 5895 5896 return false; 5897 } 5898 5899 static enum mlxsw_sp_rif_type 5900 mlxsw_sp_dev_rif_type(const struct mlxsw_sp *mlxsw_sp, 5901 const struct net_device *dev) 5902 { 5903 enum mlxsw_sp_fid_type type; 5904 5905 if (mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, NULL)) 5906 return MLXSW_SP_RIF_TYPE_IPIP_LB; 5907 5908 /* Otherwise RIF type is derived from the type of the underlying FID. */ 5909 if (is_vlan_dev(dev) && netif_is_bridge_master(vlan_dev_real_dev(dev))) 5910 type = MLXSW_SP_FID_TYPE_8021Q; 5911 else if (netif_is_bridge_master(dev) && br_vlan_enabled(dev)) 5912 type = MLXSW_SP_FID_TYPE_8021Q; 5913 else if (netif_is_bridge_master(dev)) 5914 type = MLXSW_SP_FID_TYPE_8021D; 5915 else 5916 type = MLXSW_SP_FID_TYPE_RFID; 5917 5918 return mlxsw_sp_fid_type_rif_type(mlxsw_sp, type); 5919 } 5920 5921 static int mlxsw_sp_rif_index_alloc(struct mlxsw_sp *mlxsw_sp, u16 *p_rif_index) 5922 { 5923 int i; 5924 5925 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++) { 5926 if (!mlxsw_sp->router->rifs[i]) { 5927 *p_rif_index = i; 5928 return 0; 5929 } 5930 } 5931 5932 return -ENOBUFS; 5933 } 5934 5935 static struct mlxsw_sp_rif *mlxsw_sp_rif_alloc(size_t rif_size, u16 rif_index, 5936 u16 vr_id, 5937 struct net_device *l3_dev) 5938 { 5939 struct mlxsw_sp_rif *rif; 5940 5941 rif = kzalloc(rif_size, GFP_KERNEL); 5942 if (!rif) 5943 return NULL; 5944 5945 INIT_LIST_HEAD(&rif->nexthop_list); 5946 INIT_LIST_HEAD(&rif->neigh_list); 5947 ether_addr_copy(rif->addr, l3_dev->dev_addr); 5948 rif->mtu = l3_dev->mtu; 5949 rif->vr_id = vr_id; 5950 rif->dev = l3_dev; 5951 rif->rif_index = rif_index; 5952 5953 return rif; 5954 } 5955 5956 struct mlxsw_sp_rif *mlxsw_sp_rif_by_index(const struct mlxsw_sp *mlxsw_sp, 5957 u16 rif_index) 5958 { 5959 return mlxsw_sp->router->rifs[rif_index]; 5960 } 5961 5962 u16 mlxsw_sp_rif_index(const struct mlxsw_sp_rif *rif) 5963 { 5964 return rif->rif_index; 5965 } 5966 5967 u16 mlxsw_sp_ipip_lb_rif_index(const struct mlxsw_sp_rif_ipip_lb *lb_rif) 5968 { 5969 return lb_rif->common.rif_index; 5970 } 5971 5972 u16 mlxsw_sp_ipip_lb_ul_vr_id(const struct mlxsw_sp_rif_ipip_lb *lb_rif) 5973 { 5974 return lb_rif->ul_vr_id; 5975 } 5976 5977 int mlxsw_sp_rif_dev_ifindex(const struct mlxsw_sp_rif *rif) 5978 { 5979 return rif->dev->ifindex; 5980 } 5981 5982 const struct net_device *mlxsw_sp_rif_dev(const struct mlxsw_sp_rif *rif) 5983 { 5984 return rif->dev; 5985 } 5986 5987 static struct mlxsw_sp_rif * 5988 mlxsw_sp_rif_create(struct mlxsw_sp *mlxsw_sp, 5989 const struct mlxsw_sp_rif_params *params, 5990 struct netlink_ext_ack *extack) 5991 { 5992 u32 tb_id = l3mdev_fib_table(params->dev); 5993 const struct mlxsw_sp_rif_ops *ops; 5994 struct mlxsw_sp_fid *fid = NULL; 5995 enum mlxsw_sp_rif_type type; 5996 struct mlxsw_sp_rif *rif; 5997 struct mlxsw_sp_vr *vr; 5998 u16 rif_index; 5999 int err; 6000 6001 type = mlxsw_sp_dev_rif_type(mlxsw_sp, params->dev); 6002 ops = mlxsw_sp->router->rif_ops_arr[type]; 6003 6004 vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id ? : RT_TABLE_MAIN, extack); 6005 if (IS_ERR(vr)) 6006 return ERR_CAST(vr); 6007 vr->rif_count++; 6008 6009 err = mlxsw_sp_rif_index_alloc(mlxsw_sp, &rif_index); 6010 if (err) { 6011 NL_SET_ERR_MSG(extack, "spectrum: Exceeded number of supported router interfaces"); 6012 goto err_rif_index_alloc; 6013 } 6014 6015 rif = mlxsw_sp_rif_alloc(ops->rif_size, rif_index, vr->id, params->dev); 6016 if (!rif) { 6017 err = -ENOMEM; 6018 goto err_rif_alloc; 6019 } 6020 rif->mlxsw_sp = mlxsw_sp; 6021 rif->ops = ops; 6022 6023 if (ops->fid_get) { 6024 fid = ops->fid_get(rif); 6025 if (IS_ERR(fid)) { 6026 err = PTR_ERR(fid); 6027 goto err_fid_get; 6028 } 6029 rif->fid = fid; 6030 } 6031 6032 if (ops->setup) 6033 ops->setup(rif, params); 6034 6035 err = ops->configure(rif); 6036 if (err) 6037 goto err_configure; 6038 6039 err = mlxsw_sp_mr_rif_add(vr->mr4_table, rif); 6040 if (err) 6041 goto err_mr_rif_add; 6042 6043 mlxsw_sp_rif_counters_alloc(rif); 6044 mlxsw_sp->router->rifs[rif_index] = rif; 6045 6046 return rif; 6047 6048 err_mr_rif_add: 6049 ops->deconfigure(rif); 6050 err_configure: 6051 if (fid) 6052 mlxsw_sp_fid_put(fid); 6053 err_fid_get: 6054 kfree(rif); 6055 err_rif_alloc: 6056 err_rif_index_alloc: 6057 vr->rif_count--; 6058 mlxsw_sp_vr_put(vr); 6059 return ERR_PTR(err); 6060 } 6061 6062 void mlxsw_sp_rif_destroy(struct mlxsw_sp_rif *rif) 6063 { 6064 const struct mlxsw_sp_rif_ops *ops = rif->ops; 6065 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 6066 struct mlxsw_sp_fid *fid = rif->fid; 6067 struct mlxsw_sp_vr *vr; 6068 6069 mlxsw_sp_router_rif_gone_sync(mlxsw_sp, rif); 6070 vr = &mlxsw_sp->router->vrs[rif->vr_id]; 6071 6072 mlxsw_sp->router->rifs[rif->rif_index] = NULL; 6073 mlxsw_sp_rif_counters_free(rif); 6074 mlxsw_sp_mr_rif_del(vr->mr4_table, rif); 6075 ops->deconfigure(rif); 6076 if (fid) 6077 /* Loopback RIFs are not associated with a FID. */ 6078 mlxsw_sp_fid_put(fid); 6079 kfree(rif); 6080 vr->rif_count--; 6081 mlxsw_sp_vr_put(vr); 6082 } 6083 6084 static void 6085 mlxsw_sp_rif_subport_params_init(struct mlxsw_sp_rif_params *params, 6086 struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan) 6087 { 6088 struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port; 6089 6090 params->vid = mlxsw_sp_port_vlan->vid; 6091 params->lag = mlxsw_sp_port->lagged; 6092 if (params->lag) 6093 params->lag_id = mlxsw_sp_port->lag_id; 6094 else 6095 params->system_port = mlxsw_sp_port->local_port; 6096 } 6097 6098 static int 6099 mlxsw_sp_port_vlan_router_join(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan, 6100 struct net_device *l3_dev, 6101 struct netlink_ext_ack *extack) 6102 { 6103 struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port; 6104 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp; 6105 u16 vid = mlxsw_sp_port_vlan->vid; 6106 struct mlxsw_sp_rif *rif; 6107 struct mlxsw_sp_fid *fid; 6108 int err; 6109 6110 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev); 6111 if (!rif) { 6112 struct mlxsw_sp_rif_params params = { 6113 .dev = l3_dev, 6114 }; 6115 6116 mlxsw_sp_rif_subport_params_init(¶ms, mlxsw_sp_port_vlan); 6117 rif = mlxsw_sp_rif_create(mlxsw_sp, ¶ms, extack); 6118 if (IS_ERR(rif)) 6119 return PTR_ERR(rif); 6120 } 6121 6122 /* FID was already created, just take a reference */ 6123 fid = rif->ops->fid_get(rif); 6124 err = mlxsw_sp_fid_port_vid_map(fid, mlxsw_sp_port, vid); 6125 if (err) 6126 goto err_fid_port_vid_map; 6127 6128 err = mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, false); 6129 if (err) 6130 goto err_port_vid_learning_set; 6131 6132 err = mlxsw_sp_port_vid_stp_set(mlxsw_sp_port, vid, 6133 BR_STATE_FORWARDING); 6134 if (err) 6135 goto err_port_vid_stp_set; 6136 6137 mlxsw_sp_port_vlan->fid = fid; 6138 6139 return 0; 6140 6141 err_port_vid_stp_set: 6142 mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, true); 6143 err_port_vid_learning_set: 6144 mlxsw_sp_fid_port_vid_unmap(fid, mlxsw_sp_port, vid); 6145 err_fid_port_vid_map: 6146 mlxsw_sp_fid_put(fid); 6147 return err; 6148 } 6149 6150 void 6151 mlxsw_sp_port_vlan_router_leave(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan) 6152 { 6153 struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port; 6154 struct mlxsw_sp_fid *fid = mlxsw_sp_port_vlan->fid; 6155 u16 vid = mlxsw_sp_port_vlan->vid; 6156 6157 if (WARN_ON(mlxsw_sp_fid_type(fid) != MLXSW_SP_FID_TYPE_RFID)) 6158 return; 6159 6160 mlxsw_sp_port_vlan->fid = NULL; 6161 mlxsw_sp_port_vid_stp_set(mlxsw_sp_port, vid, BR_STATE_BLOCKING); 6162 mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, true); 6163 mlxsw_sp_fid_port_vid_unmap(fid, mlxsw_sp_port, vid); 6164 /* If router port holds the last reference on the rFID, then the 6165 * associated Sub-port RIF will be destroyed. 6166 */ 6167 mlxsw_sp_fid_put(fid); 6168 } 6169 6170 static int mlxsw_sp_inetaddr_port_vlan_event(struct net_device *l3_dev, 6171 struct net_device *port_dev, 6172 unsigned long event, u16 vid, 6173 struct netlink_ext_ack *extack) 6174 { 6175 struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(port_dev); 6176 struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan; 6177 6178 mlxsw_sp_port_vlan = mlxsw_sp_port_vlan_find_by_vid(mlxsw_sp_port, vid); 6179 if (WARN_ON(!mlxsw_sp_port_vlan)) 6180 return -EINVAL; 6181 6182 switch (event) { 6183 case NETDEV_UP: 6184 return mlxsw_sp_port_vlan_router_join(mlxsw_sp_port_vlan, 6185 l3_dev, extack); 6186 case NETDEV_DOWN: 6187 mlxsw_sp_port_vlan_router_leave(mlxsw_sp_port_vlan); 6188 break; 6189 } 6190 6191 return 0; 6192 } 6193 6194 static int mlxsw_sp_inetaddr_port_event(struct net_device *port_dev, 6195 unsigned long event, 6196 struct netlink_ext_ack *extack) 6197 { 6198 if (netif_is_bridge_port(port_dev) || 6199 netif_is_lag_port(port_dev) || 6200 netif_is_ovs_port(port_dev)) 6201 return 0; 6202 6203 return mlxsw_sp_inetaddr_port_vlan_event(port_dev, port_dev, event, 1, 6204 extack); 6205 } 6206 6207 static int __mlxsw_sp_inetaddr_lag_event(struct net_device *l3_dev, 6208 struct net_device *lag_dev, 6209 unsigned long event, u16 vid, 6210 struct netlink_ext_ack *extack) 6211 { 6212 struct net_device *port_dev; 6213 struct list_head *iter; 6214 int err; 6215 6216 netdev_for_each_lower_dev(lag_dev, port_dev, iter) { 6217 if (mlxsw_sp_port_dev_check(port_dev)) { 6218 err = mlxsw_sp_inetaddr_port_vlan_event(l3_dev, 6219 port_dev, 6220 event, vid, 6221 extack); 6222 if (err) 6223 return err; 6224 } 6225 } 6226 6227 return 0; 6228 } 6229 6230 static int mlxsw_sp_inetaddr_lag_event(struct net_device *lag_dev, 6231 unsigned long event, 6232 struct netlink_ext_ack *extack) 6233 { 6234 if (netif_is_bridge_port(lag_dev)) 6235 return 0; 6236 6237 return __mlxsw_sp_inetaddr_lag_event(lag_dev, lag_dev, event, 1, 6238 extack); 6239 } 6240 6241 static int mlxsw_sp_inetaddr_bridge_event(struct net_device *l3_dev, 6242 unsigned long event, 6243 struct netlink_ext_ack *extack) 6244 { 6245 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_lower_get(l3_dev); 6246 struct mlxsw_sp_rif_params params = { 6247 .dev = l3_dev, 6248 }; 6249 struct mlxsw_sp_rif *rif; 6250 6251 switch (event) { 6252 case NETDEV_UP: 6253 rif = mlxsw_sp_rif_create(mlxsw_sp, ¶ms, extack); 6254 if (IS_ERR(rif)) 6255 return PTR_ERR(rif); 6256 break; 6257 case NETDEV_DOWN: 6258 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev); 6259 mlxsw_sp_rif_destroy(rif); 6260 break; 6261 } 6262 6263 return 0; 6264 } 6265 6266 static int mlxsw_sp_inetaddr_vlan_event(struct net_device *vlan_dev, 6267 unsigned long event, 6268 struct netlink_ext_ack *extack) 6269 { 6270 struct net_device *real_dev = vlan_dev_real_dev(vlan_dev); 6271 u16 vid = vlan_dev_vlan_id(vlan_dev); 6272 6273 if (netif_is_bridge_port(vlan_dev)) 6274 return 0; 6275 6276 if (mlxsw_sp_port_dev_check(real_dev)) 6277 return mlxsw_sp_inetaddr_port_vlan_event(vlan_dev, real_dev, 6278 event, vid, extack); 6279 else if (netif_is_lag_master(real_dev)) 6280 return __mlxsw_sp_inetaddr_lag_event(vlan_dev, real_dev, event, 6281 vid, extack); 6282 else if (netif_is_bridge_master(real_dev) && br_vlan_enabled(real_dev)) 6283 return mlxsw_sp_inetaddr_bridge_event(vlan_dev, event, extack); 6284 6285 return 0; 6286 } 6287 6288 static int __mlxsw_sp_inetaddr_event(struct net_device *dev, 6289 unsigned long event, 6290 struct netlink_ext_ack *extack) 6291 { 6292 if (mlxsw_sp_port_dev_check(dev)) 6293 return mlxsw_sp_inetaddr_port_event(dev, event, extack); 6294 else if (netif_is_lag_master(dev)) 6295 return mlxsw_sp_inetaddr_lag_event(dev, event, extack); 6296 else if (netif_is_bridge_master(dev)) 6297 return mlxsw_sp_inetaddr_bridge_event(dev, event, extack); 6298 else if (is_vlan_dev(dev)) 6299 return mlxsw_sp_inetaddr_vlan_event(dev, event, extack); 6300 else 6301 return 0; 6302 } 6303 6304 int mlxsw_sp_inetaddr_event(struct notifier_block *unused, 6305 unsigned long event, void *ptr) 6306 { 6307 struct in_ifaddr *ifa = (struct in_ifaddr *) ptr; 6308 struct net_device *dev = ifa->ifa_dev->dev; 6309 struct mlxsw_sp *mlxsw_sp; 6310 struct mlxsw_sp_rif *rif; 6311 int err = 0; 6312 6313 /* NETDEV_UP event is handled by mlxsw_sp_inetaddr_valid_event */ 6314 if (event == NETDEV_UP) 6315 goto out; 6316 6317 mlxsw_sp = mlxsw_sp_lower_get(dev); 6318 if (!mlxsw_sp) 6319 goto out; 6320 6321 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 6322 if (!mlxsw_sp_rif_should_config(rif, dev, event)) 6323 goto out; 6324 6325 err = __mlxsw_sp_inetaddr_event(dev, event, NULL); 6326 out: 6327 return notifier_from_errno(err); 6328 } 6329 6330 int mlxsw_sp_inetaddr_valid_event(struct notifier_block *unused, 6331 unsigned long event, void *ptr) 6332 { 6333 struct in_validator_info *ivi = (struct in_validator_info *) ptr; 6334 struct net_device *dev = ivi->ivi_dev->dev; 6335 struct mlxsw_sp *mlxsw_sp; 6336 struct mlxsw_sp_rif *rif; 6337 int err = 0; 6338 6339 mlxsw_sp = mlxsw_sp_lower_get(dev); 6340 if (!mlxsw_sp) 6341 goto out; 6342 6343 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 6344 if (!mlxsw_sp_rif_should_config(rif, dev, event)) 6345 goto out; 6346 6347 err = __mlxsw_sp_inetaddr_event(dev, event, ivi->extack); 6348 out: 6349 return notifier_from_errno(err); 6350 } 6351 6352 struct mlxsw_sp_inet6addr_event_work { 6353 struct work_struct work; 6354 struct net_device *dev; 6355 unsigned long event; 6356 }; 6357 6358 static void mlxsw_sp_inet6addr_event_work(struct work_struct *work) 6359 { 6360 struct mlxsw_sp_inet6addr_event_work *inet6addr_work = 6361 container_of(work, struct mlxsw_sp_inet6addr_event_work, work); 6362 struct net_device *dev = inet6addr_work->dev; 6363 unsigned long event = inet6addr_work->event; 6364 struct mlxsw_sp *mlxsw_sp; 6365 struct mlxsw_sp_rif *rif; 6366 6367 rtnl_lock(); 6368 mlxsw_sp = mlxsw_sp_lower_get(dev); 6369 if (!mlxsw_sp) 6370 goto out; 6371 6372 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 6373 if (!mlxsw_sp_rif_should_config(rif, dev, event)) 6374 goto out; 6375 6376 __mlxsw_sp_inetaddr_event(dev, event, NULL); 6377 out: 6378 rtnl_unlock(); 6379 dev_put(dev); 6380 kfree(inet6addr_work); 6381 } 6382 6383 /* Called with rcu_read_lock() */ 6384 int mlxsw_sp_inet6addr_event(struct notifier_block *unused, 6385 unsigned long event, void *ptr) 6386 { 6387 struct inet6_ifaddr *if6 = (struct inet6_ifaddr *) ptr; 6388 struct mlxsw_sp_inet6addr_event_work *inet6addr_work; 6389 struct net_device *dev = if6->idev->dev; 6390 6391 /* NETDEV_UP event is handled by mlxsw_sp_inet6addr_valid_event */ 6392 if (event == NETDEV_UP) 6393 return NOTIFY_DONE; 6394 6395 if (!mlxsw_sp_port_dev_lower_find_rcu(dev)) 6396 return NOTIFY_DONE; 6397 6398 inet6addr_work = kzalloc(sizeof(*inet6addr_work), GFP_ATOMIC); 6399 if (!inet6addr_work) 6400 return NOTIFY_BAD; 6401 6402 INIT_WORK(&inet6addr_work->work, mlxsw_sp_inet6addr_event_work); 6403 inet6addr_work->dev = dev; 6404 inet6addr_work->event = event; 6405 dev_hold(dev); 6406 mlxsw_core_schedule_work(&inet6addr_work->work); 6407 6408 return NOTIFY_DONE; 6409 } 6410 6411 int mlxsw_sp_inet6addr_valid_event(struct notifier_block *unused, 6412 unsigned long event, void *ptr) 6413 { 6414 struct in6_validator_info *i6vi = (struct in6_validator_info *) ptr; 6415 struct net_device *dev = i6vi->i6vi_dev->dev; 6416 struct mlxsw_sp *mlxsw_sp; 6417 struct mlxsw_sp_rif *rif; 6418 int err = 0; 6419 6420 mlxsw_sp = mlxsw_sp_lower_get(dev); 6421 if (!mlxsw_sp) 6422 goto out; 6423 6424 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 6425 if (!mlxsw_sp_rif_should_config(rif, dev, event)) 6426 goto out; 6427 6428 err = __mlxsw_sp_inetaddr_event(dev, event, i6vi->extack); 6429 out: 6430 return notifier_from_errno(err); 6431 } 6432 6433 static int mlxsw_sp_rif_edit(struct mlxsw_sp *mlxsw_sp, u16 rif_index, 6434 const char *mac, int mtu) 6435 { 6436 char ritr_pl[MLXSW_REG_RITR_LEN]; 6437 int err; 6438 6439 mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index); 6440 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 6441 if (err) 6442 return err; 6443 6444 mlxsw_reg_ritr_mtu_set(ritr_pl, mtu); 6445 mlxsw_reg_ritr_if_mac_memcpy_to(ritr_pl, mac); 6446 mlxsw_reg_ritr_op_set(ritr_pl, MLXSW_REG_RITR_RIF_CREATE); 6447 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 6448 } 6449 6450 int mlxsw_sp_netdevice_router_port_event(struct net_device *dev) 6451 { 6452 struct mlxsw_sp *mlxsw_sp; 6453 struct mlxsw_sp_rif *rif; 6454 u16 fid_index; 6455 int err; 6456 6457 mlxsw_sp = mlxsw_sp_lower_get(dev); 6458 if (!mlxsw_sp) 6459 return 0; 6460 6461 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 6462 if (!rif) 6463 return 0; 6464 fid_index = mlxsw_sp_fid_index(rif->fid); 6465 6466 err = mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->addr, fid_index, false); 6467 if (err) 6468 return err; 6469 6470 err = mlxsw_sp_rif_edit(mlxsw_sp, rif->rif_index, dev->dev_addr, 6471 dev->mtu); 6472 if (err) 6473 goto err_rif_edit; 6474 6475 err = mlxsw_sp_rif_fdb_op(mlxsw_sp, dev->dev_addr, fid_index, true); 6476 if (err) 6477 goto err_rif_fdb_op; 6478 6479 if (rif->mtu != dev->mtu) { 6480 struct mlxsw_sp_vr *vr; 6481 6482 /* The RIF is relevant only to its mr_table instance, as unlike 6483 * unicast routing, in multicast routing a RIF cannot be shared 6484 * between several multicast routing tables. 6485 */ 6486 vr = &mlxsw_sp->router->vrs[rif->vr_id]; 6487 mlxsw_sp_mr_rif_mtu_update(vr->mr4_table, rif, dev->mtu); 6488 } 6489 6490 ether_addr_copy(rif->addr, dev->dev_addr); 6491 rif->mtu = dev->mtu; 6492 6493 netdev_dbg(dev, "Updated RIF=%d\n", rif->rif_index); 6494 6495 return 0; 6496 6497 err_rif_fdb_op: 6498 mlxsw_sp_rif_edit(mlxsw_sp, rif->rif_index, rif->addr, rif->mtu); 6499 err_rif_edit: 6500 mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->addr, fid_index, true); 6501 return err; 6502 } 6503 6504 static int mlxsw_sp_port_vrf_join(struct mlxsw_sp *mlxsw_sp, 6505 struct net_device *l3_dev, 6506 struct netlink_ext_ack *extack) 6507 { 6508 struct mlxsw_sp_rif *rif; 6509 6510 /* If netdev is already associated with a RIF, then we need to 6511 * destroy it and create a new one with the new virtual router ID. 6512 */ 6513 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev); 6514 if (rif) 6515 __mlxsw_sp_inetaddr_event(l3_dev, NETDEV_DOWN, extack); 6516 6517 return __mlxsw_sp_inetaddr_event(l3_dev, NETDEV_UP, extack); 6518 } 6519 6520 static void mlxsw_sp_port_vrf_leave(struct mlxsw_sp *mlxsw_sp, 6521 struct net_device *l3_dev) 6522 { 6523 struct mlxsw_sp_rif *rif; 6524 6525 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev); 6526 if (!rif) 6527 return; 6528 __mlxsw_sp_inetaddr_event(l3_dev, NETDEV_DOWN, NULL); 6529 } 6530 6531 int mlxsw_sp_netdevice_vrf_event(struct net_device *l3_dev, unsigned long event, 6532 struct netdev_notifier_changeupper_info *info) 6533 { 6534 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_lower_get(l3_dev); 6535 int err = 0; 6536 6537 if (!mlxsw_sp) 6538 return 0; 6539 6540 switch (event) { 6541 case NETDEV_PRECHANGEUPPER: 6542 return 0; 6543 case NETDEV_CHANGEUPPER: 6544 if (info->linking) { 6545 struct netlink_ext_ack *extack; 6546 6547 extack = netdev_notifier_info_to_extack(&info->info); 6548 err = mlxsw_sp_port_vrf_join(mlxsw_sp, l3_dev, extack); 6549 } else { 6550 mlxsw_sp_port_vrf_leave(mlxsw_sp, l3_dev); 6551 } 6552 break; 6553 } 6554 6555 return err; 6556 } 6557 6558 static struct mlxsw_sp_rif_subport * 6559 mlxsw_sp_rif_subport_rif(const struct mlxsw_sp_rif *rif) 6560 { 6561 return container_of(rif, struct mlxsw_sp_rif_subport, common); 6562 } 6563 6564 static void mlxsw_sp_rif_subport_setup(struct mlxsw_sp_rif *rif, 6565 const struct mlxsw_sp_rif_params *params) 6566 { 6567 struct mlxsw_sp_rif_subport *rif_subport; 6568 6569 rif_subport = mlxsw_sp_rif_subport_rif(rif); 6570 rif_subport->vid = params->vid; 6571 rif_subport->lag = params->lag; 6572 if (params->lag) 6573 rif_subport->lag_id = params->lag_id; 6574 else 6575 rif_subport->system_port = params->system_port; 6576 } 6577 6578 static int mlxsw_sp_rif_subport_op(struct mlxsw_sp_rif *rif, bool enable) 6579 { 6580 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 6581 struct mlxsw_sp_rif_subport *rif_subport; 6582 char ritr_pl[MLXSW_REG_RITR_LEN]; 6583 6584 rif_subport = mlxsw_sp_rif_subport_rif(rif); 6585 mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_SP_IF, 6586 rif->rif_index, rif->vr_id, rif->dev->mtu); 6587 mlxsw_reg_ritr_mac_pack(ritr_pl, rif->dev->dev_addr); 6588 mlxsw_reg_ritr_sp_if_pack(ritr_pl, rif_subport->lag, 6589 rif_subport->lag ? rif_subport->lag_id : 6590 rif_subport->system_port, 6591 rif_subport->vid); 6592 6593 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 6594 } 6595 6596 static int mlxsw_sp_rif_subport_configure(struct mlxsw_sp_rif *rif) 6597 { 6598 int err; 6599 6600 err = mlxsw_sp_rif_subport_op(rif, true); 6601 if (err) 6602 return err; 6603 6604 err = mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr, 6605 mlxsw_sp_fid_index(rif->fid), true); 6606 if (err) 6607 goto err_rif_fdb_op; 6608 6609 mlxsw_sp_fid_rif_set(rif->fid, rif); 6610 return 0; 6611 6612 err_rif_fdb_op: 6613 mlxsw_sp_rif_subport_op(rif, false); 6614 return err; 6615 } 6616 6617 static void mlxsw_sp_rif_subport_deconfigure(struct mlxsw_sp_rif *rif) 6618 { 6619 struct mlxsw_sp_fid *fid = rif->fid; 6620 6621 mlxsw_sp_fid_rif_set(fid, NULL); 6622 mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr, 6623 mlxsw_sp_fid_index(fid), false); 6624 mlxsw_sp_rif_subport_op(rif, false); 6625 } 6626 6627 static struct mlxsw_sp_fid * 6628 mlxsw_sp_rif_subport_fid_get(struct mlxsw_sp_rif *rif) 6629 { 6630 return mlxsw_sp_fid_rfid_get(rif->mlxsw_sp, rif->rif_index); 6631 } 6632 6633 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_subport_ops = { 6634 .type = MLXSW_SP_RIF_TYPE_SUBPORT, 6635 .rif_size = sizeof(struct mlxsw_sp_rif_subport), 6636 .setup = mlxsw_sp_rif_subport_setup, 6637 .configure = mlxsw_sp_rif_subport_configure, 6638 .deconfigure = mlxsw_sp_rif_subport_deconfigure, 6639 .fid_get = mlxsw_sp_rif_subport_fid_get, 6640 }; 6641 6642 static int mlxsw_sp_rif_vlan_fid_op(struct mlxsw_sp_rif *rif, 6643 enum mlxsw_reg_ritr_if_type type, 6644 u16 vid_fid, bool enable) 6645 { 6646 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 6647 char ritr_pl[MLXSW_REG_RITR_LEN]; 6648 6649 mlxsw_reg_ritr_pack(ritr_pl, enable, type, rif->rif_index, rif->vr_id, 6650 rif->dev->mtu); 6651 mlxsw_reg_ritr_mac_pack(ritr_pl, rif->dev->dev_addr); 6652 mlxsw_reg_ritr_fid_set(ritr_pl, type, vid_fid); 6653 6654 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 6655 } 6656 6657 u8 mlxsw_sp_router_port(const struct mlxsw_sp *mlxsw_sp) 6658 { 6659 return mlxsw_core_max_ports(mlxsw_sp->core) + 1; 6660 } 6661 6662 static int mlxsw_sp_rif_vlan_configure(struct mlxsw_sp_rif *rif) 6663 { 6664 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 6665 u16 vid = mlxsw_sp_fid_8021q_vid(rif->fid); 6666 int err; 6667 6668 err = mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_VLAN_IF, vid, true); 6669 if (err) 6670 return err; 6671 6672 err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC, 6673 mlxsw_sp_router_port(mlxsw_sp), true); 6674 if (err) 6675 goto err_fid_mc_flood_set; 6676 6677 err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC, 6678 mlxsw_sp_router_port(mlxsw_sp), true); 6679 if (err) 6680 goto err_fid_bc_flood_set; 6681 6682 err = mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr, 6683 mlxsw_sp_fid_index(rif->fid), true); 6684 if (err) 6685 goto err_rif_fdb_op; 6686 6687 mlxsw_sp_fid_rif_set(rif->fid, rif); 6688 return 0; 6689 6690 err_rif_fdb_op: 6691 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC, 6692 mlxsw_sp_router_port(mlxsw_sp), false); 6693 err_fid_bc_flood_set: 6694 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC, 6695 mlxsw_sp_router_port(mlxsw_sp), false); 6696 err_fid_mc_flood_set: 6697 mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_VLAN_IF, vid, false); 6698 return err; 6699 } 6700 6701 static void mlxsw_sp_rif_vlan_deconfigure(struct mlxsw_sp_rif *rif) 6702 { 6703 u16 vid = mlxsw_sp_fid_8021q_vid(rif->fid); 6704 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 6705 struct mlxsw_sp_fid *fid = rif->fid; 6706 6707 mlxsw_sp_fid_rif_set(fid, NULL); 6708 mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr, 6709 mlxsw_sp_fid_index(fid), false); 6710 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC, 6711 mlxsw_sp_router_port(mlxsw_sp), false); 6712 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC, 6713 mlxsw_sp_router_port(mlxsw_sp), false); 6714 mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_VLAN_IF, vid, false); 6715 } 6716 6717 static struct mlxsw_sp_fid * 6718 mlxsw_sp_rif_vlan_fid_get(struct mlxsw_sp_rif *rif) 6719 { 6720 u16 vid = is_vlan_dev(rif->dev) ? vlan_dev_vlan_id(rif->dev) : 1; 6721 6722 return mlxsw_sp_fid_8021q_get(rif->mlxsw_sp, vid); 6723 } 6724 6725 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_vlan_ops = { 6726 .type = MLXSW_SP_RIF_TYPE_VLAN, 6727 .rif_size = sizeof(struct mlxsw_sp_rif), 6728 .configure = mlxsw_sp_rif_vlan_configure, 6729 .deconfigure = mlxsw_sp_rif_vlan_deconfigure, 6730 .fid_get = mlxsw_sp_rif_vlan_fid_get, 6731 }; 6732 6733 static int mlxsw_sp_rif_fid_configure(struct mlxsw_sp_rif *rif) 6734 { 6735 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 6736 u16 fid_index = mlxsw_sp_fid_index(rif->fid); 6737 int err; 6738 6739 err = mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index, 6740 true); 6741 if (err) 6742 return err; 6743 6744 err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC, 6745 mlxsw_sp_router_port(mlxsw_sp), true); 6746 if (err) 6747 goto err_fid_mc_flood_set; 6748 6749 err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC, 6750 mlxsw_sp_router_port(mlxsw_sp), true); 6751 if (err) 6752 goto err_fid_bc_flood_set; 6753 6754 err = mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr, 6755 mlxsw_sp_fid_index(rif->fid), true); 6756 if (err) 6757 goto err_rif_fdb_op; 6758 6759 mlxsw_sp_fid_rif_set(rif->fid, rif); 6760 return 0; 6761 6762 err_rif_fdb_op: 6763 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC, 6764 mlxsw_sp_router_port(mlxsw_sp), false); 6765 err_fid_bc_flood_set: 6766 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC, 6767 mlxsw_sp_router_port(mlxsw_sp), false); 6768 err_fid_mc_flood_set: 6769 mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index, false); 6770 return err; 6771 } 6772 6773 static void mlxsw_sp_rif_fid_deconfigure(struct mlxsw_sp_rif *rif) 6774 { 6775 u16 fid_index = mlxsw_sp_fid_index(rif->fid); 6776 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 6777 struct mlxsw_sp_fid *fid = rif->fid; 6778 6779 mlxsw_sp_fid_rif_set(fid, NULL); 6780 mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr, 6781 mlxsw_sp_fid_index(fid), false); 6782 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC, 6783 mlxsw_sp_router_port(mlxsw_sp), false); 6784 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC, 6785 mlxsw_sp_router_port(mlxsw_sp), false); 6786 mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index, false); 6787 } 6788 6789 static struct mlxsw_sp_fid * 6790 mlxsw_sp_rif_fid_fid_get(struct mlxsw_sp_rif *rif) 6791 { 6792 return mlxsw_sp_fid_8021d_get(rif->mlxsw_sp, rif->dev->ifindex); 6793 } 6794 6795 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_fid_ops = { 6796 .type = MLXSW_SP_RIF_TYPE_FID, 6797 .rif_size = sizeof(struct mlxsw_sp_rif), 6798 .configure = mlxsw_sp_rif_fid_configure, 6799 .deconfigure = mlxsw_sp_rif_fid_deconfigure, 6800 .fid_get = mlxsw_sp_rif_fid_fid_get, 6801 }; 6802 6803 static struct mlxsw_sp_rif_ipip_lb * 6804 mlxsw_sp_rif_ipip_lb_rif(struct mlxsw_sp_rif *rif) 6805 { 6806 return container_of(rif, struct mlxsw_sp_rif_ipip_lb, common); 6807 } 6808 6809 static void 6810 mlxsw_sp_rif_ipip_lb_setup(struct mlxsw_sp_rif *rif, 6811 const struct mlxsw_sp_rif_params *params) 6812 { 6813 struct mlxsw_sp_rif_params_ipip_lb *params_lb; 6814 struct mlxsw_sp_rif_ipip_lb *rif_lb; 6815 6816 params_lb = container_of(params, struct mlxsw_sp_rif_params_ipip_lb, 6817 common); 6818 rif_lb = mlxsw_sp_rif_ipip_lb_rif(rif); 6819 rif_lb->lb_config = params_lb->lb_config; 6820 } 6821 6822 static int 6823 mlxsw_sp_rif_ipip_lb_op(struct mlxsw_sp_rif_ipip_lb *lb_rif, 6824 struct mlxsw_sp_vr *ul_vr, bool enable) 6825 { 6826 struct mlxsw_sp_rif_ipip_lb_config lb_cf = lb_rif->lb_config; 6827 struct mlxsw_sp_rif *rif = &lb_rif->common; 6828 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 6829 char ritr_pl[MLXSW_REG_RITR_LEN]; 6830 u32 saddr4; 6831 6832 switch (lb_cf.ul_protocol) { 6833 case MLXSW_SP_L3_PROTO_IPV4: 6834 saddr4 = be32_to_cpu(lb_cf.saddr.addr4); 6835 mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_LOOPBACK_IF, 6836 rif->rif_index, rif->vr_id, rif->dev->mtu); 6837 mlxsw_reg_ritr_loopback_ipip4_pack(ritr_pl, lb_cf.lb_ipipt, 6838 MLXSW_REG_RITR_LOOPBACK_IPIP_OPTIONS_GRE_KEY_PRESET, 6839 ul_vr->id, saddr4, lb_cf.okey); 6840 break; 6841 6842 case MLXSW_SP_L3_PROTO_IPV6: 6843 return -EAFNOSUPPORT; 6844 } 6845 6846 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 6847 } 6848 6849 static int 6850 mlxsw_sp_rif_ipip_lb_configure(struct mlxsw_sp_rif *rif) 6851 { 6852 struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif); 6853 u32 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(rif->dev); 6854 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 6855 struct mlxsw_sp_vr *ul_vr; 6856 int err; 6857 6858 ul_vr = mlxsw_sp_vr_get(mlxsw_sp, ul_tb_id, NULL); 6859 if (IS_ERR(ul_vr)) 6860 return PTR_ERR(ul_vr); 6861 6862 err = mlxsw_sp_rif_ipip_lb_op(lb_rif, ul_vr, true); 6863 if (err) 6864 goto err_loopback_op; 6865 6866 lb_rif->ul_vr_id = ul_vr->id; 6867 ++ul_vr->rif_count; 6868 return 0; 6869 6870 err_loopback_op: 6871 mlxsw_sp_vr_put(ul_vr); 6872 return err; 6873 } 6874 6875 static void mlxsw_sp_rif_ipip_lb_deconfigure(struct mlxsw_sp_rif *rif) 6876 { 6877 struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif); 6878 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 6879 struct mlxsw_sp_vr *ul_vr; 6880 6881 ul_vr = &mlxsw_sp->router->vrs[lb_rif->ul_vr_id]; 6882 mlxsw_sp_rif_ipip_lb_op(lb_rif, ul_vr, false); 6883 6884 --ul_vr->rif_count; 6885 mlxsw_sp_vr_put(ul_vr); 6886 } 6887 6888 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_ipip_lb_ops = { 6889 .type = MLXSW_SP_RIF_TYPE_IPIP_LB, 6890 .rif_size = sizeof(struct mlxsw_sp_rif_ipip_lb), 6891 .setup = mlxsw_sp_rif_ipip_lb_setup, 6892 .configure = mlxsw_sp_rif_ipip_lb_configure, 6893 .deconfigure = mlxsw_sp_rif_ipip_lb_deconfigure, 6894 }; 6895 6896 static const struct mlxsw_sp_rif_ops *mlxsw_sp_rif_ops_arr[] = { 6897 [MLXSW_SP_RIF_TYPE_SUBPORT] = &mlxsw_sp_rif_subport_ops, 6898 [MLXSW_SP_RIF_TYPE_VLAN] = &mlxsw_sp_rif_vlan_ops, 6899 [MLXSW_SP_RIF_TYPE_FID] = &mlxsw_sp_rif_fid_ops, 6900 [MLXSW_SP_RIF_TYPE_IPIP_LB] = &mlxsw_sp_rif_ipip_lb_ops, 6901 }; 6902 6903 static int mlxsw_sp_rifs_init(struct mlxsw_sp *mlxsw_sp) 6904 { 6905 u64 max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); 6906 6907 mlxsw_sp->router->rifs = kcalloc(max_rifs, 6908 sizeof(struct mlxsw_sp_rif *), 6909 GFP_KERNEL); 6910 if (!mlxsw_sp->router->rifs) 6911 return -ENOMEM; 6912 6913 mlxsw_sp->router->rif_ops_arr = mlxsw_sp_rif_ops_arr; 6914 6915 return 0; 6916 } 6917 6918 static void mlxsw_sp_rifs_fini(struct mlxsw_sp *mlxsw_sp) 6919 { 6920 int i; 6921 6922 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++) 6923 WARN_ON_ONCE(mlxsw_sp->router->rifs[i]); 6924 6925 kfree(mlxsw_sp->router->rifs); 6926 } 6927 6928 static int 6929 mlxsw_sp_ipip_config_tigcr(struct mlxsw_sp *mlxsw_sp) 6930 { 6931 char tigcr_pl[MLXSW_REG_TIGCR_LEN]; 6932 6933 mlxsw_reg_tigcr_pack(tigcr_pl, true, 0); 6934 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(tigcr), tigcr_pl); 6935 } 6936 6937 static int mlxsw_sp_ipips_init(struct mlxsw_sp *mlxsw_sp) 6938 { 6939 mlxsw_sp->router->ipip_ops_arr = mlxsw_sp_ipip_ops_arr; 6940 INIT_LIST_HEAD(&mlxsw_sp->router->ipip_list); 6941 return mlxsw_sp_ipip_config_tigcr(mlxsw_sp); 6942 } 6943 6944 static void mlxsw_sp_ipips_fini(struct mlxsw_sp *mlxsw_sp) 6945 { 6946 WARN_ON(!list_empty(&mlxsw_sp->router->ipip_list)); 6947 } 6948 6949 static void mlxsw_sp_router_fib_dump_flush(struct notifier_block *nb) 6950 { 6951 struct mlxsw_sp_router *router; 6952 6953 /* Flush pending FIB notifications and then flush the device's 6954 * table before requesting another dump. The FIB notification 6955 * block is unregistered, so no need to take RTNL. 6956 */ 6957 mlxsw_core_flush_owq(); 6958 router = container_of(nb, struct mlxsw_sp_router, fib_nb); 6959 mlxsw_sp_router_fib_flush(router->mlxsw_sp); 6960 } 6961 6962 #ifdef CONFIG_IP_ROUTE_MULTIPATH 6963 static void mlxsw_sp_mp_hash_header_set(char *recr2_pl, int header) 6964 { 6965 mlxsw_reg_recr2_outer_header_enables_set(recr2_pl, header, true); 6966 } 6967 6968 static void mlxsw_sp_mp_hash_field_set(char *recr2_pl, int field) 6969 { 6970 mlxsw_reg_recr2_outer_header_fields_enable_set(recr2_pl, field, true); 6971 } 6972 6973 static void mlxsw_sp_mp4_hash_init(char *recr2_pl) 6974 { 6975 bool only_l3 = !init_net.ipv4.sysctl_fib_multipath_hash_policy; 6976 6977 mlxsw_sp_mp_hash_header_set(recr2_pl, 6978 MLXSW_REG_RECR2_IPV4_EN_NOT_TCP_NOT_UDP); 6979 mlxsw_sp_mp_hash_header_set(recr2_pl, MLXSW_REG_RECR2_IPV4_EN_TCP_UDP); 6980 mlxsw_reg_recr2_ipv4_sip_enable(recr2_pl); 6981 mlxsw_reg_recr2_ipv4_dip_enable(recr2_pl); 6982 if (only_l3) 6983 return; 6984 mlxsw_sp_mp_hash_header_set(recr2_pl, MLXSW_REG_RECR2_TCP_UDP_EN_IPV4); 6985 mlxsw_sp_mp_hash_field_set(recr2_pl, MLXSW_REG_RECR2_IPV4_PROTOCOL); 6986 mlxsw_sp_mp_hash_field_set(recr2_pl, MLXSW_REG_RECR2_TCP_UDP_SPORT); 6987 mlxsw_sp_mp_hash_field_set(recr2_pl, MLXSW_REG_RECR2_TCP_UDP_DPORT); 6988 } 6989 6990 static void mlxsw_sp_mp6_hash_init(char *recr2_pl) 6991 { 6992 mlxsw_sp_mp_hash_header_set(recr2_pl, 6993 MLXSW_REG_RECR2_IPV6_EN_NOT_TCP_NOT_UDP); 6994 mlxsw_sp_mp_hash_header_set(recr2_pl, MLXSW_REG_RECR2_IPV6_EN_TCP_UDP); 6995 mlxsw_reg_recr2_ipv6_sip_enable(recr2_pl); 6996 mlxsw_reg_recr2_ipv6_dip_enable(recr2_pl); 6997 mlxsw_sp_mp_hash_field_set(recr2_pl, MLXSW_REG_RECR2_IPV6_FLOW_LABEL); 6998 mlxsw_sp_mp_hash_field_set(recr2_pl, MLXSW_REG_RECR2_IPV6_NEXT_HEADER); 6999 } 7000 7001 static int mlxsw_sp_mp_hash_init(struct mlxsw_sp *mlxsw_sp) 7002 { 7003 char recr2_pl[MLXSW_REG_RECR2_LEN]; 7004 u32 seed; 7005 7006 get_random_bytes(&seed, sizeof(seed)); 7007 mlxsw_reg_recr2_pack(recr2_pl, seed); 7008 mlxsw_sp_mp4_hash_init(recr2_pl); 7009 mlxsw_sp_mp6_hash_init(recr2_pl); 7010 7011 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(recr2), recr2_pl); 7012 } 7013 #else 7014 static int mlxsw_sp_mp_hash_init(struct mlxsw_sp *mlxsw_sp) 7015 { 7016 return 0; 7017 } 7018 #endif 7019 7020 static int __mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp) 7021 { 7022 char rgcr_pl[MLXSW_REG_RGCR_LEN]; 7023 u64 max_rifs; 7024 int err; 7025 7026 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_RIFS)) 7027 return -EIO; 7028 max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); 7029 7030 mlxsw_reg_rgcr_pack(rgcr_pl, true, true); 7031 mlxsw_reg_rgcr_max_router_interfaces_set(rgcr_pl, max_rifs); 7032 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl); 7033 if (err) 7034 return err; 7035 return 0; 7036 } 7037 7038 static void __mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp) 7039 { 7040 char rgcr_pl[MLXSW_REG_RGCR_LEN]; 7041 7042 mlxsw_reg_rgcr_pack(rgcr_pl, false, false); 7043 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl); 7044 } 7045 7046 int mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp) 7047 { 7048 struct mlxsw_sp_router *router; 7049 int err; 7050 7051 router = kzalloc(sizeof(*mlxsw_sp->router), GFP_KERNEL); 7052 if (!router) 7053 return -ENOMEM; 7054 mlxsw_sp->router = router; 7055 router->mlxsw_sp = mlxsw_sp; 7056 7057 INIT_LIST_HEAD(&mlxsw_sp->router->nexthop_neighs_list); 7058 err = __mlxsw_sp_router_init(mlxsw_sp); 7059 if (err) 7060 goto err_router_init; 7061 7062 err = mlxsw_sp_rifs_init(mlxsw_sp); 7063 if (err) 7064 goto err_rifs_init; 7065 7066 err = mlxsw_sp_ipips_init(mlxsw_sp); 7067 if (err) 7068 goto err_ipips_init; 7069 7070 err = rhashtable_init(&mlxsw_sp->router->nexthop_ht, 7071 &mlxsw_sp_nexthop_ht_params); 7072 if (err) 7073 goto err_nexthop_ht_init; 7074 7075 err = rhashtable_init(&mlxsw_sp->router->nexthop_group_ht, 7076 &mlxsw_sp_nexthop_group_ht_params); 7077 if (err) 7078 goto err_nexthop_group_ht_init; 7079 7080 INIT_LIST_HEAD(&mlxsw_sp->router->nexthop_list); 7081 err = mlxsw_sp_lpm_init(mlxsw_sp); 7082 if (err) 7083 goto err_lpm_init; 7084 7085 err = mlxsw_sp_mr_init(mlxsw_sp, &mlxsw_sp_mr_tcam_ops); 7086 if (err) 7087 goto err_mr_init; 7088 7089 err = mlxsw_sp_vrs_init(mlxsw_sp); 7090 if (err) 7091 goto err_vrs_init; 7092 7093 err = mlxsw_sp_neigh_init(mlxsw_sp); 7094 if (err) 7095 goto err_neigh_init; 7096 7097 mlxsw_sp->router->netevent_nb.notifier_call = 7098 mlxsw_sp_router_netevent_event; 7099 err = register_netevent_notifier(&mlxsw_sp->router->netevent_nb); 7100 if (err) 7101 goto err_register_netevent_notifier; 7102 7103 err = mlxsw_sp_mp_hash_init(mlxsw_sp); 7104 if (err) 7105 goto err_mp_hash_init; 7106 7107 mlxsw_sp->router->fib_nb.notifier_call = mlxsw_sp_router_fib_event; 7108 err = register_fib_notifier(&mlxsw_sp->router->fib_nb, 7109 mlxsw_sp_router_fib_dump_flush); 7110 if (err) 7111 goto err_register_fib_notifier; 7112 7113 return 0; 7114 7115 err_register_fib_notifier: 7116 err_mp_hash_init: 7117 unregister_netevent_notifier(&mlxsw_sp->router->netevent_nb); 7118 err_register_netevent_notifier: 7119 mlxsw_sp_neigh_fini(mlxsw_sp); 7120 err_neigh_init: 7121 mlxsw_sp_vrs_fini(mlxsw_sp); 7122 err_vrs_init: 7123 mlxsw_sp_mr_fini(mlxsw_sp); 7124 err_mr_init: 7125 mlxsw_sp_lpm_fini(mlxsw_sp); 7126 err_lpm_init: 7127 rhashtable_destroy(&mlxsw_sp->router->nexthop_group_ht); 7128 err_nexthop_group_ht_init: 7129 rhashtable_destroy(&mlxsw_sp->router->nexthop_ht); 7130 err_nexthop_ht_init: 7131 mlxsw_sp_ipips_fini(mlxsw_sp); 7132 err_ipips_init: 7133 mlxsw_sp_rifs_fini(mlxsw_sp); 7134 err_rifs_init: 7135 __mlxsw_sp_router_fini(mlxsw_sp); 7136 err_router_init: 7137 kfree(mlxsw_sp->router); 7138 return err; 7139 } 7140 7141 void mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp) 7142 { 7143 unregister_fib_notifier(&mlxsw_sp->router->fib_nb); 7144 unregister_netevent_notifier(&mlxsw_sp->router->netevent_nb); 7145 mlxsw_sp_neigh_fini(mlxsw_sp); 7146 mlxsw_sp_vrs_fini(mlxsw_sp); 7147 mlxsw_sp_mr_fini(mlxsw_sp); 7148 mlxsw_sp_lpm_fini(mlxsw_sp); 7149 rhashtable_destroy(&mlxsw_sp->router->nexthop_group_ht); 7150 rhashtable_destroy(&mlxsw_sp->router->nexthop_ht); 7151 mlxsw_sp_ipips_fini(mlxsw_sp); 7152 mlxsw_sp_rifs_fini(mlxsw_sp); 7153 __mlxsw_sp_router_fini(mlxsw_sp); 7154 kfree(mlxsw_sp->router); 7155 } 7156