1 /* 2 * drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c 3 * Copyright (c) 2016 Mellanox Technologies. All rights reserved. 4 * Copyright (c) 2016 Jiri Pirko <jiri@mellanox.com> 5 * Copyright (c) 2016 Ido Schimmel <idosch@mellanox.com> 6 * Copyright (c) 2016 Yotam Gigi <yotamg@mellanox.com> 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions are met: 10 * 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 3. Neither the names of the copyright holders nor the names of its 17 * contributors may be used to endorse or promote products derived from 18 * this software without specific prior written permission. 19 * 20 * Alternatively, this software may be distributed under the terms of the 21 * GNU General Public License ("GPL") version 2 as published by the Free 22 * Software Foundation. 23 * 24 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 25 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 26 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 27 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 28 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 29 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 30 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 31 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 32 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 33 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 34 * POSSIBILITY OF SUCH DAMAGE. 35 */ 36 37 #include <linux/kernel.h> 38 #include <linux/types.h> 39 #include <linux/rhashtable.h> 40 #include <linux/bitops.h> 41 #include <linux/in6.h> 42 #include <linux/notifier.h> 43 #include <linux/inetdevice.h> 44 #include <linux/netdevice.h> 45 #include <net/netevent.h> 46 #include <net/neighbour.h> 47 #include <net/arp.h> 48 #include <net/ip_fib.h> 49 #include <net/fib_rules.h> 50 #include <net/l3mdev.h> 51 52 #include "spectrum.h" 53 #include "core.h" 54 #include "reg.h" 55 #include "spectrum_cnt.h" 56 #include "spectrum_dpipe.h" 57 #include "spectrum_router.h" 58 59 struct mlxsw_sp_rif { 60 struct list_head nexthop_list; 61 struct list_head neigh_list; 62 struct net_device *dev; 63 struct mlxsw_sp_fid *f; 64 unsigned char addr[ETH_ALEN]; 65 int mtu; 66 u16 rif_index; 67 u16 vr_id; 68 unsigned int counter_ingress; 69 bool counter_ingress_valid; 70 unsigned int counter_egress; 71 bool counter_egress_valid; 72 }; 73 74 static unsigned int * 75 mlxsw_sp_rif_p_counter_get(struct mlxsw_sp_rif *rif, 76 enum mlxsw_sp_rif_counter_dir dir) 77 { 78 switch (dir) { 79 case MLXSW_SP_RIF_COUNTER_EGRESS: 80 return &rif->counter_egress; 81 case MLXSW_SP_RIF_COUNTER_INGRESS: 82 return &rif->counter_ingress; 83 } 84 return NULL; 85 } 86 87 static bool 88 mlxsw_sp_rif_counter_valid_get(struct mlxsw_sp_rif *rif, 89 enum mlxsw_sp_rif_counter_dir dir) 90 { 91 switch (dir) { 92 case MLXSW_SP_RIF_COUNTER_EGRESS: 93 return rif->counter_egress_valid; 94 case MLXSW_SP_RIF_COUNTER_INGRESS: 95 return rif->counter_ingress_valid; 96 } 97 return false; 98 } 99 100 static void 101 mlxsw_sp_rif_counter_valid_set(struct mlxsw_sp_rif *rif, 102 enum mlxsw_sp_rif_counter_dir dir, 103 bool valid) 104 { 105 switch (dir) { 106 case MLXSW_SP_RIF_COUNTER_EGRESS: 107 rif->counter_egress_valid = valid; 108 break; 109 case MLXSW_SP_RIF_COUNTER_INGRESS: 110 rif->counter_ingress_valid = valid; 111 break; 112 } 113 } 114 115 static int mlxsw_sp_rif_counter_edit(struct mlxsw_sp *mlxsw_sp, u16 rif_index, 116 unsigned int counter_index, bool enable, 117 enum mlxsw_sp_rif_counter_dir dir) 118 { 119 char ritr_pl[MLXSW_REG_RITR_LEN]; 120 bool is_egress = false; 121 int err; 122 123 if (dir == MLXSW_SP_RIF_COUNTER_EGRESS) 124 is_egress = true; 125 mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index); 126 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 127 if (err) 128 return err; 129 130 mlxsw_reg_ritr_counter_pack(ritr_pl, counter_index, enable, 131 is_egress); 132 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 133 } 134 135 int mlxsw_sp_rif_counter_value_get(struct mlxsw_sp *mlxsw_sp, 136 struct mlxsw_sp_rif *rif, 137 enum mlxsw_sp_rif_counter_dir dir, u64 *cnt) 138 { 139 char ricnt_pl[MLXSW_REG_RICNT_LEN]; 140 unsigned int *p_counter_index; 141 bool valid; 142 int err; 143 144 valid = mlxsw_sp_rif_counter_valid_get(rif, dir); 145 if (!valid) 146 return -EINVAL; 147 148 p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir); 149 if (!p_counter_index) 150 return -EINVAL; 151 mlxsw_reg_ricnt_pack(ricnt_pl, *p_counter_index, 152 MLXSW_REG_RICNT_OPCODE_NOP); 153 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl); 154 if (err) 155 return err; 156 *cnt = mlxsw_reg_ricnt_good_unicast_packets_get(ricnt_pl); 157 return 0; 158 } 159 160 static int mlxsw_sp_rif_counter_clear(struct mlxsw_sp *mlxsw_sp, 161 unsigned int counter_index) 162 { 163 char ricnt_pl[MLXSW_REG_RICNT_LEN]; 164 165 mlxsw_reg_ricnt_pack(ricnt_pl, counter_index, 166 MLXSW_REG_RICNT_OPCODE_CLEAR); 167 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl); 168 } 169 170 int mlxsw_sp_rif_counter_alloc(struct mlxsw_sp *mlxsw_sp, 171 struct mlxsw_sp_rif *rif, 172 enum mlxsw_sp_rif_counter_dir dir) 173 { 174 unsigned int *p_counter_index; 175 int err; 176 177 p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir); 178 if (!p_counter_index) 179 return -EINVAL; 180 err = mlxsw_sp_counter_alloc(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF, 181 p_counter_index); 182 if (err) 183 return err; 184 185 err = mlxsw_sp_rif_counter_clear(mlxsw_sp, *p_counter_index); 186 if (err) 187 goto err_counter_clear; 188 189 err = mlxsw_sp_rif_counter_edit(mlxsw_sp, rif->rif_index, 190 *p_counter_index, true, dir); 191 if (err) 192 goto err_counter_edit; 193 mlxsw_sp_rif_counter_valid_set(rif, dir, true); 194 return 0; 195 196 err_counter_edit: 197 err_counter_clear: 198 mlxsw_sp_counter_free(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF, 199 *p_counter_index); 200 return err; 201 } 202 203 void mlxsw_sp_rif_counter_free(struct mlxsw_sp *mlxsw_sp, 204 struct mlxsw_sp_rif *rif, 205 enum mlxsw_sp_rif_counter_dir dir) 206 { 207 unsigned int *p_counter_index; 208 209 p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir); 210 if (WARN_ON(!p_counter_index)) 211 return; 212 mlxsw_sp_rif_counter_edit(mlxsw_sp, rif->rif_index, 213 *p_counter_index, false, dir); 214 mlxsw_sp_counter_free(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF, 215 *p_counter_index); 216 mlxsw_sp_rif_counter_valid_set(rif, dir, false); 217 } 218 219 static struct mlxsw_sp_rif * 220 mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp, 221 const struct net_device *dev); 222 223 #define mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) \ 224 for_each_set_bit(prefix, (prefix_usage)->b, MLXSW_SP_PREFIX_COUNT) 225 226 static bool 227 mlxsw_sp_prefix_usage_subset(struct mlxsw_sp_prefix_usage *prefix_usage1, 228 struct mlxsw_sp_prefix_usage *prefix_usage2) 229 { 230 unsigned char prefix; 231 232 mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage1) { 233 if (!test_bit(prefix, prefix_usage2->b)) 234 return false; 235 } 236 return true; 237 } 238 239 static bool 240 mlxsw_sp_prefix_usage_eq(struct mlxsw_sp_prefix_usage *prefix_usage1, 241 struct mlxsw_sp_prefix_usage *prefix_usage2) 242 { 243 return !memcmp(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1)); 244 } 245 246 static bool 247 mlxsw_sp_prefix_usage_none(struct mlxsw_sp_prefix_usage *prefix_usage) 248 { 249 struct mlxsw_sp_prefix_usage prefix_usage_none = {{ 0 } }; 250 251 return mlxsw_sp_prefix_usage_eq(prefix_usage, &prefix_usage_none); 252 } 253 254 static void 255 mlxsw_sp_prefix_usage_cpy(struct mlxsw_sp_prefix_usage *prefix_usage1, 256 struct mlxsw_sp_prefix_usage *prefix_usage2) 257 { 258 memcpy(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1)); 259 } 260 261 static void 262 mlxsw_sp_prefix_usage_set(struct mlxsw_sp_prefix_usage *prefix_usage, 263 unsigned char prefix_len) 264 { 265 set_bit(prefix_len, prefix_usage->b); 266 } 267 268 static void 269 mlxsw_sp_prefix_usage_clear(struct mlxsw_sp_prefix_usage *prefix_usage, 270 unsigned char prefix_len) 271 { 272 clear_bit(prefix_len, prefix_usage->b); 273 } 274 275 struct mlxsw_sp_fib_key { 276 unsigned char addr[sizeof(struct in6_addr)]; 277 unsigned char prefix_len; 278 }; 279 280 enum mlxsw_sp_fib_entry_type { 281 MLXSW_SP_FIB_ENTRY_TYPE_REMOTE, 282 MLXSW_SP_FIB_ENTRY_TYPE_LOCAL, 283 MLXSW_SP_FIB_ENTRY_TYPE_TRAP, 284 }; 285 286 struct mlxsw_sp_nexthop_group; 287 288 struct mlxsw_sp_fib_node { 289 struct list_head entry_list; 290 struct list_head list; 291 struct rhash_head ht_node; 292 struct mlxsw_sp_fib *fib; 293 struct mlxsw_sp_fib_key key; 294 }; 295 296 struct mlxsw_sp_fib_entry_params { 297 u32 tb_id; 298 u32 prio; 299 u8 tos; 300 u8 type; 301 }; 302 303 struct mlxsw_sp_fib_entry { 304 struct list_head list; 305 struct mlxsw_sp_fib_node *fib_node; 306 enum mlxsw_sp_fib_entry_type type; 307 struct list_head nexthop_group_node; 308 struct mlxsw_sp_nexthop_group *nh_group; 309 struct mlxsw_sp_fib_entry_params params; 310 bool offloaded; 311 }; 312 313 struct mlxsw_sp_fib { 314 struct rhashtable ht; 315 struct list_head node_list; 316 struct mlxsw_sp_vr *vr; 317 struct mlxsw_sp_lpm_tree *lpm_tree; 318 unsigned long prefix_ref_count[MLXSW_SP_PREFIX_COUNT]; 319 struct mlxsw_sp_prefix_usage prefix_usage; 320 enum mlxsw_sp_l3proto proto; 321 }; 322 323 static const struct rhashtable_params mlxsw_sp_fib_ht_params; 324 325 static struct mlxsw_sp_fib *mlxsw_sp_fib_create(struct mlxsw_sp_vr *vr, 326 enum mlxsw_sp_l3proto proto) 327 { 328 struct mlxsw_sp_fib *fib; 329 int err; 330 331 fib = kzalloc(sizeof(*fib), GFP_KERNEL); 332 if (!fib) 333 return ERR_PTR(-ENOMEM); 334 err = rhashtable_init(&fib->ht, &mlxsw_sp_fib_ht_params); 335 if (err) 336 goto err_rhashtable_init; 337 INIT_LIST_HEAD(&fib->node_list); 338 fib->proto = proto; 339 fib->vr = vr; 340 return fib; 341 342 err_rhashtable_init: 343 kfree(fib); 344 return ERR_PTR(err); 345 } 346 347 static void mlxsw_sp_fib_destroy(struct mlxsw_sp_fib *fib) 348 { 349 WARN_ON(!list_empty(&fib->node_list)); 350 WARN_ON(fib->lpm_tree); 351 rhashtable_destroy(&fib->ht); 352 kfree(fib); 353 } 354 355 static struct mlxsw_sp_lpm_tree * 356 mlxsw_sp_lpm_tree_find_unused(struct mlxsw_sp *mlxsw_sp) 357 { 358 static struct mlxsw_sp_lpm_tree *lpm_tree; 359 int i; 360 361 for (i = 0; i < mlxsw_sp->router.lpm.tree_count; i++) { 362 lpm_tree = &mlxsw_sp->router.lpm.trees[i]; 363 if (lpm_tree->ref_count == 0) 364 return lpm_tree; 365 } 366 return NULL; 367 } 368 369 static int mlxsw_sp_lpm_tree_alloc(struct mlxsw_sp *mlxsw_sp, 370 struct mlxsw_sp_lpm_tree *lpm_tree) 371 { 372 char ralta_pl[MLXSW_REG_RALTA_LEN]; 373 374 mlxsw_reg_ralta_pack(ralta_pl, true, 375 (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto, 376 lpm_tree->id); 377 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl); 378 } 379 380 static int mlxsw_sp_lpm_tree_free(struct mlxsw_sp *mlxsw_sp, 381 struct mlxsw_sp_lpm_tree *lpm_tree) 382 { 383 char ralta_pl[MLXSW_REG_RALTA_LEN]; 384 385 mlxsw_reg_ralta_pack(ralta_pl, false, 386 (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto, 387 lpm_tree->id); 388 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl); 389 } 390 391 static int 392 mlxsw_sp_lpm_tree_left_struct_set(struct mlxsw_sp *mlxsw_sp, 393 struct mlxsw_sp_prefix_usage *prefix_usage, 394 struct mlxsw_sp_lpm_tree *lpm_tree) 395 { 396 char ralst_pl[MLXSW_REG_RALST_LEN]; 397 u8 root_bin = 0; 398 u8 prefix; 399 u8 last_prefix = MLXSW_REG_RALST_BIN_NO_CHILD; 400 401 mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) 402 root_bin = prefix; 403 404 mlxsw_reg_ralst_pack(ralst_pl, root_bin, lpm_tree->id); 405 mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) { 406 if (prefix == 0) 407 continue; 408 mlxsw_reg_ralst_bin_pack(ralst_pl, prefix, last_prefix, 409 MLXSW_REG_RALST_BIN_NO_CHILD); 410 last_prefix = prefix; 411 } 412 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralst), ralst_pl); 413 } 414 415 static struct mlxsw_sp_lpm_tree * 416 mlxsw_sp_lpm_tree_create(struct mlxsw_sp *mlxsw_sp, 417 struct mlxsw_sp_prefix_usage *prefix_usage, 418 enum mlxsw_sp_l3proto proto) 419 { 420 struct mlxsw_sp_lpm_tree *lpm_tree; 421 int err; 422 423 lpm_tree = mlxsw_sp_lpm_tree_find_unused(mlxsw_sp); 424 if (!lpm_tree) 425 return ERR_PTR(-EBUSY); 426 lpm_tree->proto = proto; 427 err = mlxsw_sp_lpm_tree_alloc(mlxsw_sp, lpm_tree); 428 if (err) 429 return ERR_PTR(err); 430 431 err = mlxsw_sp_lpm_tree_left_struct_set(mlxsw_sp, prefix_usage, 432 lpm_tree); 433 if (err) 434 goto err_left_struct_set; 435 memcpy(&lpm_tree->prefix_usage, prefix_usage, 436 sizeof(lpm_tree->prefix_usage)); 437 return lpm_tree; 438 439 err_left_struct_set: 440 mlxsw_sp_lpm_tree_free(mlxsw_sp, lpm_tree); 441 return ERR_PTR(err); 442 } 443 444 static int mlxsw_sp_lpm_tree_destroy(struct mlxsw_sp *mlxsw_sp, 445 struct mlxsw_sp_lpm_tree *lpm_tree) 446 { 447 return mlxsw_sp_lpm_tree_free(mlxsw_sp, lpm_tree); 448 } 449 450 static struct mlxsw_sp_lpm_tree * 451 mlxsw_sp_lpm_tree_get(struct mlxsw_sp *mlxsw_sp, 452 struct mlxsw_sp_prefix_usage *prefix_usage, 453 enum mlxsw_sp_l3proto proto) 454 { 455 struct mlxsw_sp_lpm_tree *lpm_tree; 456 int i; 457 458 for (i = 0; i < mlxsw_sp->router.lpm.tree_count; i++) { 459 lpm_tree = &mlxsw_sp->router.lpm.trees[i]; 460 if (lpm_tree->ref_count != 0 && 461 lpm_tree->proto == proto && 462 mlxsw_sp_prefix_usage_eq(&lpm_tree->prefix_usage, 463 prefix_usage)) 464 goto inc_ref_count; 465 } 466 lpm_tree = mlxsw_sp_lpm_tree_create(mlxsw_sp, prefix_usage, 467 proto); 468 if (IS_ERR(lpm_tree)) 469 return lpm_tree; 470 471 inc_ref_count: 472 lpm_tree->ref_count++; 473 return lpm_tree; 474 } 475 476 static int mlxsw_sp_lpm_tree_put(struct mlxsw_sp *mlxsw_sp, 477 struct mlxsw_sp_lpm_tree *lpm_tree) 478 { 479 if (--lpm_tree->ref_count == 0) 480 return mlxsw_sp_lpm_tree_destroy(mlxsw_sp, lpm_tree); 481 return 0; 482 } 483 484 #define MLXSW_SP_LPM_TREE_MIN 2 /* trees 0 and 1 are reserved */ 485 486 static int mlxsw_sp_lpm_init(struct mlxsw_sp *mlxsw_sp) 487 { 488 struct mlxsw_sp_lpm_tree *lpm_tree; 489 u64 max_trees; 490 int i; 491 492 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_LPM_TREES)) 493 return -EIO; 494 495 max_trees = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_LPM_TREES); 496 mlxsw_sp->router.lpm.tree_count = max_trees - MLXSW_SP_LPM_TREE_MIN; 497 mlxsw_sp->router.lpm.trees = kcalloc(mlxsw_sp->router.lpm.tree_count, 498 sizeof(struct mlxsw_sp_lpm_tree), 499 GFP_KERNEL); 500 if (!mlxsw_sp->router.lpm.trees) 501 return -ENOMEM; 502 503 for (i = 0; i < mlxsw_sp->router.lpm.tree_count; i++) { 504 lpm_tree = &mlxsw_sp->router.lpm.trees[i]; 505 lpm_tree->id = i + MLXSW_SP_LPM_TREE_MIN; 506 } 507 508 return 0; 509 } 510 511 static void mlxsw_sp_lpm_fini(struct mlxsw_sp *mlxsw_sp) 512 { 513 kfree(mlxsw_sp->router.lpm.trees); 514 } 515 516 static bool mlxsw_sp_vr_is_used(const struct mlxsw_sp_vr *vr) 517 { 518 return !!vr->fib4; 519 } 520 521 static struct mlxsw_sp_vr *mlxsw_sp_vr_find_unused(struct mlxsw_sp *mlxsw_sp) 522 { 523 struct mlxsw_sp_vr *vr; 524 int i; 525 526 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) { 527 vr = &mlxsw_sp->router.vrs[i]; 528 if (!mlxsw_sp_vr_is_used(vr)) 529 return vr; 530 } 531 return NULL; 532 } 533 534 static int mlxsw_sp_vr_lpm_tree_bind(struct mlxsw_sp *mlxsw_sp, 535 const struct mlxsw_sp_fib *fib) 536 { 537 char raltb_pl[MLXSW_REG_RALTB_LEN]; 538 539 mlxsw_reg_raltb_pack(raltb_pl, fib->vr->id, 540 (enum mlxsw_reg_ralxx_protocol) fib->proto, 541 fib->lpm_tree->id); 542 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), raltb_pl); 543 } 544 545 static int mlxsw_sp_vr_lpm_tree_unbind(struct mlxsw_sp *mlxsw_sp, 546 const struct mlxsw_sp_fib *fib) 547 { 548 char raltb_pl[MLXSW_REG_RALTB_LEN]; 549 550 /* Bind to tree 0 which is default */ 551 mlxsw_reg_raltb_pack(raltb_pl, fib->vr->id, 552 (enum mlxsw_reg_ralxx_protocol) fib->proto, 0); 553 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), raltb_pl); 554 } 555 556 static u32 mlxsw_sp_fix_tb_id(u32 tb_id) 557 { 558 /* For our purpose, squash main and local table into one */ 559 if (tb_id == RT_TABLE_LOCAL) 560 tb_id = RT_TABLE_MAIN; 561 return tb_id; 562 } 563 564 static struct mlxsw_sp_vr *mlxsw_sp_vr_find(struct mlxsw_sp *mlxsw_sp, 565 u32 tb_id) 566 { 567 struct mlxsw_sp_vr *vr; 568 int i; 569 570 tb_id = mlxsw_sp_fix_tb_id(tb_id); 571 572 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) { 573 vr = &mlxsw_sp->router.vrs[i]; 574 if (mlxsw_sp_vr_is_used(vr) && vr->tb_id == tb_id) 575 return vr; 576 } 577 return NULL; 578 } 579 580 static struct mlxsw_sp_fib *mlxsw_sp_vr_fib(const struct mlxsw_sp_vr *vr, 581 enum mlxsw_sp_l3proto proto) 582 { 583 switch (proto) { 584 case MLXSW_SP_L3_PROTO_IPV4: 585 return vr->fib4; 586 case MLXSW_SP_L3_PROTO_IPV6: 587 BUG_ON(1); 588 } 589 return NULL; 590 } 591 592 static struct mlxsw_sp_vr *mlxsw_sp_vr_create(struct mlxsw_sp *mlxsw_sp, 593 u32 tb_id) 594 { 595 struct mlxsw_sp_vr *vr; 596 597 vr = mlxsw_sp_vr_find_unused(mlxsw_sp); 598 if (!vr) 599 return ERR_PTR(-EBUSY); 600 vr->fib4 = mlxsw_sp_fib_create(vr, MLXSW_SP_L3_PROTO_IPV4); 601 if (IS_ERR(vr->fib4)) 602 return ERR_CAST(vr->fib4); 603 vr->tb_id = tb_id; 604 return vr; 605 } 606 607 static void mlxsw_sp_vr_destroy(struct mlxsw_sp_vr *vr) 608 { 609 mlxsw_sp_fib_destroy(vr->fib4); 610 vr->fib4 = NULL; 611 } 612 613 static int 614 mlxsw_sp_vr_lpm_tree_check(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_fib *fib, 615 struct mlxsw_sp_prefix_usage *req_prefix_usage) 616 { 617 struct mlxsw_sp_lpm_tree *lpm_tree = fib->lpm_tree; 618 struct mlxsw_sp_lpm_tree *new_tree; 619 int err; 620 621 if (mlxsw_sp_prefix_usage_eq(req_prefix_usage, &lpm_tree->prefix_usage)) 622 return 0; 623 624 new_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, req_prefix_usage, 625 fib->proto); 626 if (IS_ERR(new_tree)) { 627 /* We failed to get a tree according to the required 628 * prefix usage. However, the current tree might be still good 629 * for us if our requirement is subset of the prefixes used 630 * in the tree. 631 */ 632 if (mlxsw_sp_prefix_usage_subset(req_prefix_usage, 633 &lpm_tree->prefix_usage)) 634 return 0; 635 return PTR_ERR(new_tree); 636 } 637 638 /* Prevent packet loss by overwriting existing binding */ 639 fib->lpm_tree = new_tree; 640 err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib); 641 if (err) 642 goto err_tree_bind; 643 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree); 644 645 return 0; 646 647 err_tree_bind: 648 fib->lpm_tree = lpm_tree; 649 mlxsw_sp_lpm_tree_put(mlxsw_sp, new_tree); 650 return err; 651 } 652 653 static struct mlxsw_sp_vr *mlxsw_sp_vr_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id) 654 { 655 struct mlxsw_sp_vr *vr; 656 657 tb_id = mlxsw_sp_fix_tb_id(tb_id); 658 vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id); 659 if (!vr) 660 vr = mlxsw_sp_vr_create(mlxsw_sp, tb_id); 661 return vr; 662 } 663 664 static void mlxsw_sp_vr_put(struct mlxsw_sp_vr *vr) 665 { 666 if (!vr->rif_count && list_empty(&vr->fib4->node_list)) 667 mlxsw_sp_vr_destroy(vr); 668 } 669 670 static int mlxsw_sp_vrs_init(struct mlxsw_sp *mlxsw_sp) 671 { 672 struct mlxsw_sp_vr *vr; 673 u64 max_vrs; 674 int i; 675 676 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_VRS)) 677 return -EIO; 678 679 max_vrs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); 680 mlxsw_sp->router.vrs = kcalloc(max_vrs, sizeof(struct mlxsw_sp_vr), 681 GFP_KERNEL); 682 if (!mlxsw_sp->router.vrs) 683 return -ENOMEM; 684 685 for (i = 0; i < max_vrs; i++) { 686 vr = &mlxsw_sp->router.vrs[i]; 687 vr->id = i; 688 } 689 690 return 0; 691 } 692 693 static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp); 694 695 static void mlxsw_sp_vrs_fini(struct mlxsw_sp *mlxsw_sp) 696 { 697 /* At this stage we're guaranteed not to have new incoming 698 * FIB notifications and the work queue is free from FIBs 699 * sitting on top of mlxsw netdevs. However, we can still 700 * have other FIBs queued. Flush the queue before flushing 701 * the device's tables. No need for locks, as we're the only 702 * writer. 703 */ 704 mlxsw_core_flush_owq(); 705 mlxsw_sp_router_fib_flush(mlxsw_sp); 706 kfree(mlxsw_sp->router.vrs); 707 } 708 709 struct mlxsw_sp_neigh_key { 710 struct neighbour *n; 711 }; 712 713 struct mlxsw_sp_neigh_entry { 714 struct list_head rif_list_node; 715 struct rhash_head ht_node; 716 struct mlxsw_sp_neigh_key key; 717 u16 rif; 718 bool connected; 719 unsigned char ha[ETH_ALEN]; 720 struct list_head nexthop_list; /* list of nexthops using 721 * this neigh entry 722 */ 723 struct list_head nexthop_neighs_list_node; 724 }; 725 726 static const struct rhashtable_params mlxsw_sp_neigh_ht_params = { 727 .key_offset = offsetof(struct mlxsw_sp_neigh_entry, key), 728 .head_offset = offsetof(struct mlxsw_sp_neigh_entry, ht_node), 729 .key_len = sizeof(struct mlxsw_sp_neigh_key), 730 }; 731 732 static struct mlxsw_sp_neigh_entry * 733 mlxsw_sp_neigh_entry_alloc(struct mlxsw_sp *mlxsw_sp, struct neighbour *n, 734 u16 rif) 735 { 736 struct mlxsw_sp_neigh_entry *neigh_entry; 737 738 neigh_entry = kzalloc(sizeof(*neigh_entry), GFP_KERNEL); 739 if (!neigh_entry) 740 return NULL; 741 742 neigh_entry->key.n = n; 743 neigh_entry->rif = rif; 744 INIT_LIST_HEAD(&neigh_entry->nexthop_list); 745 746 return neigh_entry; 747 } 748 749 static void mlxsw_sp_neigh_entry_free(struct mlxsw_sp_neigh_entry *neigh_entry) 750 { 751 kfree(neigh_entry); 752 } 753 754 static int 755 mlxsw_sp_neigh_entry_insert(struct mlxsw_sp *mlxsw_sp, 756 struct mlxsw_sp_neigh_entry *neigh_entry) 757 { 758 return rhashtable_insert_fast(&mlxsw_sp->router.neigh_ht, 759 &neigh_entry->ht_node, 760 mlxsw_sp_neigh_ht_params); 761 } 762 763 static void 764 mlxsw_sp_neigh_entry_remove(struct mlxsw_sp *mlxsw_sp, 765 struct mlxsw_sp_neigh_entry *neigh_entry) 766 { 767 rhashtable_remove_fast(&mlxsw_sp->router.neigh_ht, 768 &neigh_entry->ht_node, 769 mlxsw_sp_neigh_ht_params); 770 } 771 772 static struct mlxsw_sp_neigh_entry * 773 mlxsw_sp_neigh_entry_create(struct mlxsw_sp *mlxsw_sp, struct neighbour *n) 774 { 775 struct mlxsw_sp_neigh_entry *neigh_entry; 776 struct mlxsw_sp_rif *rif; 777 int err; 778 779 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, n->dev); 780 if (!rif) 781 return ERR_PTR(-EINVAL); 782 783 neigh_entry = mlxsw_sp_neigh_entry_alloc(mlxsw_sp, n, rif->rif_index); 784 if (!neigh_entry) 785 return ERR_PTR(-ENOMEM); 786 787 err = mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry); 788 if (err) 789 goto err_neigh_entry_insert; 790 791 list_add(&neigh_entry->rif_list_node, &rif->neigh_list); 792 793 return neigh_entry; 794 795 err_neigh_entry_insert: 796 mlxsw_sp_neigh_entry_free(neigh_entry); 797 return ERR_PTR(err); 798 } 799 800 static void 801 mlxsw_sp_neigh_entry_destroy(struct mlxsw_sp *mlxsw_sp, 802 struct mlxsw_sp_neigh_entry *neigh_entry) 803 { 804 list_del(&neigh_entry->rif_list_node); 805 mlxsw_sp_neigh_entry_remove(mlxsw_sp, neigh_entry); 806 mlxsw_sp_neigh_entry_free(neigh_entry); 807 } 808 809 static struct mlxsw_sp_neigh_entry * 810 mlxsw_sp_neigh_entry_lookup(struct mlxsw_sp *mlxsw_sp, struct neighbour *n) 811 { 812 struct mlxsw_sp_neigh_key key; 813 814 key.n = n; 815 return rhashtable_lookup_fast(&mlxsw_sp->router.neigh_ht, 816 &key, mlxsw_sp_neigh_ht_params); 817 } 818 819 static void 820 mlxsw_sp_router_neighs_update_interval_init(struct mlxsw_sp *mlxsw_sp) 821 { 822 unsigned long interval = NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME); 823 824 mlxsw_sp->router.neighs_update.interval = jiffies_to_msecs(interval); 825 } 826 827 static void mlxsw_sp_router_neigh_ent_ipv4_process(struct mlxsw_sp *mlxsw_sp, 828 char *rauhtd_pl, 829 int ent_index) 830 { 831 struct net_device *dev; 832 struct neighbour *n; 833 __be32 dipn; 834 u32 dip; 835 u16 rif; 836 837 mlxsw_reg_rauhtd_ent_ipv4_unpack(rauhtd_pl, ent_index, &rif, &dip); 838 839 if (!mlxsw_sp->rifs[rif]) { 840 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Incorrect RIF in neighbour entry\n"); 841 return; 842 } 843 844 dipn = htonl(dip); 845 dev = mlxsw_sp->rifs[rif]->dev; 846 n = neigh_lookup(&arp_tbl, &dipn, dev); 847 if (!n) { 848 netdev_err(dev, "Failed to find matching neighbour for IP=%pI4h\n", 849 &dip); 850 return; 851 } 852 853 netdev_dbg(dev, "Updating neighbour with IP=%pI4h\n", &dip); 854 neigh_event_send(n, NULL); 855 neigh_release(n); 856 } 857 858 static void mlxsw_sp_router_neigh_rec_ipv4_process(struct mlxsw_sp *mlxsw_sp, 859 char *rauhtd_pl, 860 int rec_index) 861 { 862 u8 num_entries; 863 int i; 864 865 num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl, 866 rec_index); 867 /* Hardware starts counting at 0, so add 1. */ 868 num_entries++; 869 870 /* Each record consists of several neighbour entries. */ 871 for (i = 0; i < num_entries; i++) { 872 int ent_index; 873 874 ent_index = rec_index * MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC + i; 875 mlxsw_sp_router_neigh_ent_ipv4_process(mlxsw_sp, rauhtd_pl, 876 ent_index); 877 } 878 879 } 880 881 static void mlxsw_sp_router_neigh_rec_process(struct mlxsw_sp *mlxsw_sp, 882 char *rauhtd_pl, int rec_index) 883 { 884 switch (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, rec_index)) { 885 case MLXSW_REG_RAUHTD_TYPE_IPV4: 886 mlxsw_sp_router_neigh_rec_ipv4_process(mlxsw_sp, rauhtd_pl, 887 rec_index); 888 break; 889 case MLXSW_REG_RAUHTD_TYPE_IPV6: 890 WARN_ON_ONCE(1); 891 break; 892 } 893 } 894 895 static bool mlxsw_sp_router_rauhtd_is_full(char *rauhtd_pl) 896 { 897 u8 num_rec, last_rec_index, num_entries; 898 899 num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl); 900 last_rec_index = num_rec - 1; 901 902 if (num_rec < MLXSW_REG_RAUHTD_REC_MAX_NUM) 903 return false; 904 if (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, last_rec_index) == 905 MLXSW_REG_RAUHTD_TYPE_IPV6) 906 return true; 907 908 num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl, 909 last_rec_index); 910 if (++num_entries == MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC) 911 return true; 912 return false; 913 } 914 915 static int mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp) 916 { 917 char *rauhtd_pl; 918 u8 num_rec; 919 int i, err; 920 921 rauhtd_pl = kmalloc(MLXSW_REG_RAUHTD_LEN, GFP_KERNEL); 922 if (!rauhtd_pl) 923 return -ENOMEM; 924 925 /* Make sure the neighbour's netdev isn't removed in the 926 * process. 927 */ 928 rtnl_lock(); 929 do { 930 mlxsw_reg_rauhtd_pack(rauhtd_pl, MLXSW_REG_RAUHTD_TYPE_IPV4); 931 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(rauhtd), 932 rauhtd_pl); 933 if (err) { 934 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Failed to dump neighbour talbe\n"); 935 break; 936 } 937 num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl); 938 for (i = 0; i < num_rec; i++) 939 mlxsw_sp_router_neigh_rec_process(mlxsw_sp, rauhtd_pl, 940 i); 941 } while (mlxsw_sp_router_rauhtd_is_full(rauhtd_pl)); 942 rtnl_unlock(); 943 944 kfree(rauhtd_pl); 945 return err; 946 } 947 948 static void mlxsw_sp_router_neighs_update_nh(struct mlxsw_sp *mlxsw_sp) 949 { 950 struct mlxsw_sp_neigh_entry *neigh_entry; 951 952 /* Take RTNL mutex here to prevent lists from changes */ 953 rtnl_lock(); 954 list_for_each_entry(neigh_entry, &mlxsw_sp->router.nexthop_neighs_list, 955 nexthop_neighs_list_node) 956 /* If this neigh have nexthops, make the kernel think this neigh 957 * is active regardless of the traffic. 958 */ 959 neigh_event_send(neigh_entry->key.n, NULL); 960 rtnl_unlock(); 961 } 962 963 static void 964 mlxsw_sp_router_neighs_update_work_schedule(struct mlxsw_sp *mlxsw_sp) 965 { 966 unsigned long interval = mlxsw_sp->router.neighs_update.interval; 967 968 mlxsw_core_schedule_dw(&mlxsw_sp->router.neighs_update.dw, 969 msecs_to_jiffies(interval)); 970 } 971 972 static void mlxsw_sp_router_neighs_update_work(struct work_struct *work) 973 { 974 struct mlxsw_sp *mlxsw_sp = container_of(work, struct mlxsw_sp, 975 router.neighs_update.dw.work); 976 int err; 977 978 err = mlxsw_sp_router_neighs_update_rauhtd(mlxsw_sp); 979 if (err) 980 dev_err(mlxsw_sp->bus_info->dev, "Could not update kernel for neigh activity"); 981 982 mlxsw_sp_router_neighs_update_nh(mlxsw_sp); 983 984 mlxsw_sp_router_neighs_update_work_schedule(mlxsw_sp); 985 } 986 987 static void mlxsw_sp_router_probe_unresolved_nexthops(struct work_struct *work) 988 { 989 struct mlxsw_sp_neigh_entry *neigh_entry; 990 struct mlxsw_sp *mlxsw_sp = container_of(work, struct mlxsw_sp, 991 router.nexthop_probe_dw.work); 992 993 /* Iterate over nexthop neighbours, find those who are unresolved and 994 * send arp on them. This solves the chicken-egg problem when 995 * the nexthop wouldn't get offloaded until the neighbor is resolved 996 * but it wouldn't get resolved ever in case traffic is flowing in HW 997 * using different nexthop. 998 * 999 * Take RTNL mutex here to prevent lists from changes. 1000 */ 1001 rtnl_lock(); 1002 list_for_each_entry(neigh_entry, &mlxsw_sp->router.nexthop_neighs_list, 1003 nexthop_neighs_list_node) 1004 if (!neigh_entry->connected) 1005 neigh_event_send(neigh_entry->key.n, NULL); 1006 rtnl_unlock(); 1007 1008 mlxsw_core_schedule_dw(&mlxsw_sp->router.nexthop_probe_dw, 1009 MLXSW_SP_UNRESOLVED_NH_PROBE_INTERVAL); 1010 } 1011 1012 static void 1013 mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp, 1014 struct mlxsw_sp_neigh_entry *neigh_entry, 1015 bool removing); 1016 1017 static enum mlxsw_reg_rauht_op mlxsw_sp_rauht_op(bool adding) 1018 { 1019 return adding ? MLXSW_REG_RAUHT_OP_WRITE_ADD : 1020 MLXSW_REG_RAUHT_OP_WRITE_DELETE; 1021 } 1022 1023 static void 1024 mlxsw_sp_router_neigh_entry_op4(struct mlxsw_sp *mlxsw_sp, 1025 struct mlxsw_sp_neigh_entry *neigh_entry, 1026 enum mlxsw_reg_rauht_op op) 1027 { 1028 struct neighbour *n = neigh_entry->key.n; 1029 u32 dip = ntohl(*((__be32 *) n->primary_key)); 1030 char rauht_pl[MLXSW_REG_RAUHT_LEN]; 1031 1032 mlxsw_reg_rauht_pack4(rauht_pl, op, neigh_entry->rif, neigh_entry->ha, 1033 dip); 1034 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl); 1035 } 1036 1037 static void 1038 mlxsw_sp_neigh_entry_update(struct mlxsw_sp *mlxsw_sp, 1039 struct mlxsw_sp_neigh_entry *neigh_entry, 1040 bool adding) 1041 { 1042 if (!adding && !neigh_entry->connected) 1043 return; 1044 neigh_entry->connected = adding; 1045 if (neigh_entry->key.n->tbl == &arp_tbl) 1046 mlxsw_sp_router_neigh_entry_op4(mlxsw_sp, neigh_entry, 1047 mlxsw_sp_rauht_op(adding)); 1048 else 1049 WARN_ON_ONCE(1); 1050 } 1051 1052 struct mlxsw_sp_neigh_event_work { 1053 struct work_struct work; 1054 struct mlxsw_sp *mlxsw_sp; 1055 struct neighbour *n; 1056 }; 1057 1058 static void mlxsw_sp_router_neigh_event_work(struct work_struct *work) 1059 { 1060 struct mlxsw_sp_neigh_event_work *neigh_work = 1061 container_of(work, struct mlxsw_sp_neigh_event_work, work); 1062 struct mlxsw_sp *mlxsw_sp = neigh_work->mlxsw_sp; 1063 struct mlxsw_sp_neigh_entry *neigh_entry; 1064 struct neighbour *n = neigh_work->n; 1065 unsigned char ha[ETH_ALEN]; 1066 bool entry_connected; 1067 u8 nud_state, dead; 1068 1069 /* If these parameters are changed after we release the lock, 1070 * then we are guaranteed to receive another event letting us 1071 * know about it. 1072 */ 1073 read_lock_bh(&n->lock); 1074 memcpy(ha, n->ha, ETH_ALEN); 1075 nud_state = n->nud_state; 1076 dead = n->dead; 1077 read_unlock_bh(&n->lock); 1078 1079 rtnl_lock(); 1080 entry_connected = nud_state & NUD_VALID && !dead; 1081 neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n); 1082 if (!entry_connected && !neigh_entry) 1083 goto out; 1084 if (!neigh_entry) { 1085 neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n); 1086 if (IS_ERR(neigh_entry)) 1087 goto out; 1088 } 1089 1090 memcpy(neigh_entry->ha, ha, ETH_ALEN); 1091 mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, entry_connected); 1092 mlxsw_sp_nexthop_neigh_update(mlxsw_sp, neigh_entry, !entry_connected); 1093 1094 if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list)) 1095 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry); 1096 1097 out: 1098 rtnl_unlock(); 1099 neigh_release(n); 1100 kfree(neigh_work); 1101 } 1102 1103 int mlxsw_sp_router_netevent_event(struct notifier_block *unused, 1104 unsigned long event, void *ptr) 1105 { 1106 struct mlxsw_sp_neigh_event_work *neigh_work; 1107 struct mlxsw_sp_port *mlxsw_sp_port; 1108 struct mlxsw_sp *mlxsw_sp; 1109 unsigned long interval; 1110 struct neigh_parms *p; 1111 struct neighbour *n; 1112 1113 switch (event) { 1114 case NETEVENT_DELAY_PROBE_TIME_UPDATE: 1115 p = ptr; 1116 1117 /* We don't care about changes in the default table. */ 1118 if (!p->dev || p->tbl != &arp_tbl) 1119 return NOTIFY_DONE; 1120 1121 /* We are in atomic context and can't take RTNL mutex, 1122 * so use RCU variant to walk the device chain. 1123 */ 1124 mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(p->dev); 1125 if (!mlxsw_sp_port) 1126 return NOTIFY_DONE; 1127 1128 mlxsw_sp = mlxsw_sp_port->mlxsw_sp; 1129 interval = jiffies_to_msecs(NEIGH_VAR(p, DELAY_PROBE_TIME)); 1130 mlxsw_sp->router.neighs_update.interval = interval; 1131 1132 mlxsw_sp_port_dev_put(mlxsw_sp_port); 1133 break; 1134 case NETEVENT_NEIGH_UPDATE: 1135 n = ptr; 1136 1137 if (n->tbl != &arp_tbl) 1138 return NOTIFY_DONE; 1139 1140 mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(n->dev); 1141 if (!mlxsw_sp_port) 1142 return NOTIFY_DONE; 1143 1144 neigh_work = kzalloc(sizeof(*neigh_work), GFP_ATOMIC); 1145 if (!neigh_work) { 1146 mlxsw_sp_port_dev_put(mlxsw_sp_port); 1147 return NOTIFY_BAD; 1148 } 1149 1150 INIT_WORK(&neigh_work->work, mlxsw_sp_router_neigh_event_work); 1151 neigh_work->mlxsw_sp = mlxsw_sp_port->mlxsw_sp; 1152 neigh_work->n = n; 1153 1154 /* Take a reference to ensure the neighbour won't be 1155 * destructed until we drop the reference in delayed 1156 * work. 1157 */ 1158 neigh_clone(n); 1159 mlxsw_core_schedule_work(&neigh_work->work); 1160 mlxsw_sp_port_dev_put(mlxsw_sp_port); 1161 break; 1162 } 1163 1164 return NOTIFY_DONE; 1165 } 1166 1167 static int mlxsw_sp_neigh_init(struct mlxsw_sp *mlxsw_sp) 1168 { 1169 int err; 1170 1171 err = rhashtable_init(&mlxsw_sp->router.neigh_ht, 1172 &mlxsw_sp_neigh_ht_params); 1173 if (err) 1174 return err; 1175 1176 /* Initialize the polling interval according to the default 1177 * table. 1178 */ 1179 mlxsw_sp_router_neighs_update_interval_init(mlxsw_sp); 1180 1181 /* Create the delayed works for the activity_update */ 1182 INIT_DELAYED_WORK(&mlxsw_sp->router.neighs_update.dw, 1183 mlxsw_sp_router_neighs_update_work); 1184 INIT_DELAYED_WORK(&mlxsw_sp->router.nexthop_probe_dw, 1185 mlxsw_sp_router_probe_unresolved_nexthops); 1186 mlxsw_core_schedule_dw(&mlxsw_sp->router.neighs_update.dw, 0); 1187 mlxsw_core_schedule_dw(&mlxsw_sp->router.nexthop_probe_dw, 0); 1188 return 0; 1189 } 1190 1191 static void mlxsw_sp_neigh_fini(struct mlxsw_sp *mlxsw_sp) 1192 { 1193 cancel_delayed_work_sync(&mlxsw_sp->router.neighs_update.dw); 1194 cancel_delayed_work_sync(&mlxsw_sp->router.nexthop_probe_dw); 1195 rhashtable_destroy(&mlxsw_sp->router.neigh_ht); 1196 } 1197 1198 static int mlxsw_sp_neigh_rif_flush(struct mlxsw_sp *mlxsw_sp, 1199 const struct mlxsw_sp_rif *rif) 1200 { 1201 char rauht_pl[MLXSW_REG_RAUHT_LEN]; 1202 1203 mlxsw_reg_rauht_pack(rauht_pl, MLXSW_REG_RAUHT_OP_WRITE_DELETE_ALL, 1204 rif->rif_index, rif->addr); 1205 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl); 1206 } 1207 1208 static void mlxsw_sp_neigh_rif_gone_sync(struct mlxsw_sp *mlxsw_sp, 1209 struct mlxsw_sp_rif *rif) 1210 { 1211 struct mlxsw_sp_neigh_entry *neigh_entry, *tmp; 1212 1213 mlxsw_sp_neigh_rif_flush(mlxsw_sp, rif); 1214 list_for_each_entry_safe(neigh_entry, tmp, &rif->neigh_list, 1215 rif_list_node) 1216 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry); 1217 } 1218 1219 struct mlxsw_sp_nexthop_key { 1220 struct fib_nh *fib_nh; 1221 }; 1222 1223 struct mlxsw_sp_nexthop { 1224 struct list_head neigh_list_node; /* member of neigh entry list */ 1225 struct list_head rif_list_node; 1226 struct mlxsw_sp_nexthop_group *nh_grp; /* pointer back to the group 1227 * this belongs to 1228 */ 1229 struct rhash_head ht_node; 1230 struct mlxsw_sp_nexthop_key key; 1231 struct mlxsw_sp_rif *rif; 1232 u8 should_offload:1, /* set indicates this neigh is connected and 1233 * should be put to KVD linear area of this group. 1234 */ 1235 offloaded:1, /* set in case the neigh is actually put into 1236 * KVD linear area of this group. 1237 */ 1238 update:1; /* set indicates that MAC of this neigh should be 1239 * updated in HW 1240 */ 1241 struct mlxsw_sp_neigh_entry *neigh_entry; 1242 }; 1243 1244 struct mlxsw_sp_nexthop_group_key { 1245 struct fib_info *fi; 1246 }; 1247 1248 struct mlxsw_sp_nexthop_group { 1249 struct rhash_head ht_node; 1250 struct list_head fib_list; /* list of fib entries that use this group */ 1251 struct mlxsw_sp_nexthop_group_key key; 1252 u8 adj_index_valid:1, 1253 gateway:1; /* routes using the group use a gateway */ 1254 u32 adj_index; 1255 u16 ecmp_size; 1256 u16 count; 1257 struct mlxsw_sp_nexthop nexthops[0]; 1258 #define nh_rif nexthops[0].rif 1259 }; 1260 1261 static const struct rhashtable_params mlxsw_sp_nexthop_group_ht_params = { 1262 .key_offset = offsetof(struct mlxsw_sp_nexthop_group, key), 1263 .head_offset = offsetof(struct mlxsw_sp_nexthop_group, ht_node), 1264 .key_len = sizeof(struct mlxsw_sp_nexthop_group_key), 1265 }; 1266 1267 static int mlxsw_sp_nexthop_group_insert(struct mlxsw_sp *mlxsw_sp, 1268 struct mlxsw_sp_nexthop_group *nh_grp) 1269 { 1270 return rhashtable_insert_fast(&mlxsw_sp->router.nexthop_group_ht, 1271 &nh_grp->ht_node, 1272 mlxsw_sp_nexthop_group_ht_params); 1273 } 1274 1275 static void mlxsw_sp_nexthop_group_remove(struct mlxsw_sp *mlxsw_sp, 1276 struct mlxsw_sp_nexthop_group *nh_grp) 1277 { 1278 rhashtable_remove_fast(&mlxsw_sp->router.nexthop_group_ht, 1279 &nh_grp->ht_node, 1280 mlxsw_sp_nexthop_group_ht_params); 1281 } 1282 1283 static struct mlxsw_sp_nexthop_group * 1284 mlxsw_sp_nexthop_group_lookup(struct mlxsw_sp *mlxsw_sp, 1285 struct mlxsw_sp_nexthop_group_key key) 1286 { 1287 return rhashtable_lookup_fast(&mlxsw_sp->router.nexthop_group_ht, &key, 1288 mlxsw_sp_nexthop_group_ht_params); 1289 } 1290 1291 static const struct rhashtable_params mlxsw_sp_nexthop_ht_params = { 1292 .key_offset = offsetof(struct mlxsw_sp_nexthop, key), 1293 .head_offset = offsetof(struct mlxsw_sp_nexthop, ht_node), 1294 .key_len = sizeof(struct mlxsw_sp_nexthop_key), 1295 }; 1296 1297 static int mlxsw_sp_nexthop_insert(struct mlxsw_sp *mlxsw_sp, 1298 struct mlxsw_sp_nexthop *nh) 1299 { 1300 return rhashtable_insert_fast(&mlxsw_sp->router.nexthop_ht, 1301 &nh->ht_node, mlxsw_sp_nexthop_ht_params); 1302 } 1303 1304 static void mlxsw_sp_nexthop_remove(struct mlxsw_sp *mlxsw_sp, 1305 struct mlxsw_sp_nexthop *nh) 1306 { 1307 rhashtable_remove_fast(&mlxsw_sp->router.nexthop_ht, &nh->ht_node, 1308 mlxsw_sp_nexthop_ht_params); 1309 } 1310 1311 static struct mlxsw_sp_nexthop * 1312 mlxsw_sp_nexthop_lookup(struct mlxsw_sp *mlxsw_sp, 1313 struct mlxsw_sp_nexthop_key key) 1314 { 1315 return rhashtable_lookup_fast(&mlxsw_sp->router.nexthop_ht, &key, 1316 mlxsw_sp_nexthop_ht_params); 1317 } 1318 1319 static int mlxsw_sp_adj_index_mass_update_vr(struct mlxsw_sp *mlxsw_sp, 1320 const struct mlxsw_sp_fib *fib, 1321 u32 adj_index, u16 ecmp_size, 1322 u32 new_adj_index, 1323 u16 new_ecmp_size) 1324 { 1325 char raleu_pl[MLXSW_REG_RALEU_LEN]; 1326 1327 mlxsw_reg_raleu_pack(raleu_pl, 1328 (enum mlxsw_reg_ralxx_protocol) fib->proto, 1329 fib->vr->id, adj_index, ecmp_size, new_adj_index, 1330 new_ecmp_size); 1331 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raleu), raleu_pl); 1332 } 1333 1334 static int mlxsw_sp_adj_index_mass_update(struct mlxsw_sp *mlxsw_sp, 1335 struct mlxsw_sp_nexthop_group *nh_grp, 1336 u32 old_adj_index, u16 old_ecmp_size) 1337 { 1338 struct mlxsw_sp_fib_entry *fib_entry; 1339 struct mlxsw_sp_fib *fib = NULL; 1340 int err; 1341 1342 list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) { 1343 if (fib == fib_entry->fib_node->fib) 1344 continue; 1345 fib = fib_entry->fib_node->fib; 1346 err = mlxsw_sp_adj_index_mass_update_vr(mlxsw_sp, fib, 1347 old_adj_index, 1348 old_ecmp_size, 1349 nh_grp->adj_index, 1350 nh_grp->ecmp_size); 1351 if (err) 1352 return err; 1353 } 1354 return 0; 1355 } 1356 1357 static int mlxsw_sp_nexthop_mac_update(struct mlxsw_sp *mlxsw_sp, u32 adj_index, 1358 struct mlxsw_sp_nexthop *nh) 1359 { 1360 struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry; 1361 char ratr_pl[MLXSW_REG_RATR_LEN]; 1362 1363 mlxsw_reg_ratr_pack(ratr_pl, MLXSW_REG_RATR_OP_WRITE_WRITE_ENTRY, 1364 true, adj_index, neigh_entry->rif); 1365 mlxsw_reg_ratr_eth_entry_pack(ratr_pl, neigh_entry->ha); 1366 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ratr), ratr_pl); 1367 } 1368 1369 static int 1370 mlxsw_sp_nexthop_group_mac_update(struct mlxsw_sp *mlxsw_sp, 1371 struct mlxsw_sp_nexthop_group *nh_grp, 1372 bool reallocate) 1373 { 1374 u32 adj_index = nh_grp->adj_index; /* base */ 1375 struct mlxsw_sp_nexthop *nh; 1376 int i; 1377 int err; 1378 1379 for (i = 0; i < nh_grp->count; i++) { 1380 nh = &nh_grp->nexthops[i]; 1381 1382 if (!nh->should_offload) { 1383 nh->offloaded = 0; 1384 continue; 1385 } 1386 1387 if (nh->update || reallocate) { 1388 err = mlxsw_sp_nexthop_mac_update(mlxsw_sp, 1389 adj_index, nh); 1390 if (err) 1391 return err; 1392 nh->update = 0; 1393 nh->offloaded = 1; 1394 } 1395 adj_index++; 1396 } 1397 return 0; 1398 } 1399 1400 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp, 1401 struct mlxsw_sp_fib_entry *fib_entry); 1402 1403 static int 1404 mlxsw_sp_nexthop_fib_entries_update(struct mlxsw_sp *mlxsw_sp, 1405 struct mlxsw_sp_nexthop_group *nh_grp) 1406 { 1407 struct mlxsw_sp_fib_entry *fib_entry; 1408 int err; 1409 1410 list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) { 1411 err = mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry); 1412 if (err) 1413 return err; 1414 } 1415 return 0; 1416 } 1417 1418 static void 1419 mlxsw_sp_nexthop_group_refresh(struct mlxsw_sp *mlxsw_sp, 1420 struct mlxsw_sp_nexthop_group *nh_grp) 1421 { 1422 struct mlxsw_sp_nexthop *nh; 1423 bool offload_change = false; 1424 u32 adj_index; 1425 u16 ecmp_size = 0; 1426 bool old_adj_index_valid; 1427 u32 old_adj_index; 1428 u16 old_ecmp_size; 1429 int i; 1430 int err; 1431 1432 if (!nh_grp->gateway) { 1433 mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp); 1434 return; 1435 } 1436 1437 for (i = 0; i < nh_grp->count; i++) { 1438 nh = &nh_grp->nexthops[i]; 1439 1440 if (nh->should_offload ^ nh->offloaded) { 1441 offload_change = true; 1442 if (nh->should_offload) 1443 nh->update = 1; 1444 } 1445 if (nh->should_offload) 1446 ecmp_size++; 1447 } 1448 if (!offload_change) { 1449 /* Nothing was added or removed, so no need to reallocate. Just 1450 * update MAC on existing adjacency indexes. 1451 */ 1452 err = mlxsw_sp_nexthop_group_mac_update(mlxsw_sp, nh_grp, 1453 false); 1454 if (err) { 1455 dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n"); 1456 goto set_trap; 1457 } 1458 return; 1459 } 1460 if (!ecmp_size) 1461 /* No neigh of this group is connected so we just set 1462 * the trap and let everthing flow through kernel. 1463 */ 1464 goto set_trap; 1465 1466 err = mlxsw_sp_kvdl_alloc(mlxsw_sp, ecmp_size, &adj_index); 1467 if (err) { 1468 /* We ran out of KVD linear space, just set the 1469 * trap and let everything flow through kernel. 1470 */ 1471 dev_warn(mlxsw_sp->bus_info->dev, "Failed to allocate KVD linear area for nexthop group.\n"); 1472 goto set_trap; 1473 } 1474 old_adj_index_valid = nh_grp->adj_index_valid; 1475 old_adj_index = nh_grp->adj_index; 1476 old_ecmp_size = nh_grp->ecmp_size; 1477 nh_grp->adj_index_valid = 1; 1478 nh_grp->adj_index = adj_index; 1479 nh_grp->ecmp_size = ecmp_size; 1480 err = mlxsw_sp_nexthop_group_mac_update(mlxsw_sp, nh_grp, true); 1481 if (err) { 1482 dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n"); 1483 goto set_trap; 1484 } 1485 1486 if (!old_adj_index_valid) { 1487 /* The trap was set for fib entries, so we have to call 1488 * fib entry update to unset it and use adjacency index. 1489 */ 1490 err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp); 1491 if (err) { 1492 dev_warn(mlxsw_sp->bus_info->dev, "Failed to add adjacency index to fib entries.\n"); 1493 goto set_trap; 1494 } 1495 return; 1496 } 1497 1498 err = mlxsw_sp_adj_index_mass_update(mlxsw_sp, nh_grp, 1499 old_adj_index, old_ecmp_size); 1500 mlxsw_sp_kvdl_free(mlxsw_sp, old_adj_index); 1501 if (err) { 1502 dev_warn(mlxsw_sp->bus_info->dev, "Failed to mass-update adjacency index for nexthop group.\n"); 1503 goto set_trap; 1504 } 1505 return; 1506 1507 set_trap: 1508 old_adj_index_valid = nh_grp->adj_index_valid; 1509 nh_grp->adj_index_valid = 0; 1510 for (i = 0; i < nh_grp->count; i++) { 1511 nh = &nh_grp->nexthops[i]; 1512 nh->offloaded = 0; 1513 } 1514 err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp); 1515 if (err) 1516 dev_warn(mlxsw_sp->bus_info->dev, "Failed to set traps for fib entries.\n"); 1517 if (old_adj_index_valid) 1518 mlxsw_sp_kvdl_free(mlxsw_sp, nh_grp->adj_index); 1519 } 1520 1521 static void __mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp_nexthop *nh, 1522 bool removing) 1523 { 1524 if (!removing && !nh->should_offload) 1525 nh->should_offload = 1; 1526 else if (removing && nh->offloaded) 1527 nh->should_offload = 0; 1528 nh->update = 1; 1529 } 1530 1531 static void 1532 mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp, 1533 struct mlxsw_sp_neigh_entry *neigh_entry, 1534 bool removing) 1535 { 1536 struct mlxsw_sp_nexthop *nh; 1537 1538 list_for_each_entry(nh, &neigh_entry->nexthop_list, 1539 neigh_list_node) { 1540 __mlxsw_sp_nexthop_neigh_update(nh, removing); 1541 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp); 1542 } 1543 } 1544 1545 static void mlxsw_sp_nexthop_rif_init(struct mlxsw_sp_nexthop *nh, 1546 struct mlxsw_sp_rif *rif) 1547 { 1548 if (nh->rif) 1549 return; 1550 1551 nh->rif = rif; 1552 list_add(&nh->rif_list_node, &rif->nexthop_list); 1553 } 1554 1555 static void mlxsw_sp_nexthop_rif_fini(struct mlxsw_sp_nexthop *nh) 1556 { 1557 if (!nh->rif) 1558 return; 1559 1560 list_del(&nh->rif_list_node); 1561 nh->rif = NULL; 1562 } 1563 1564 static int mlxsw_sp_nexthop_neigh_init(struct mlxsw_sp *mlxsw_sp, 1565 struct mlxsw_sp_nexthop *nh) 1566 { 1567 struct mlxsw_sp_neigh_entry *neigh_entry; 1568 struct fib_nh *fib_nh = nh->key.fib_nh; 1569 struct neighbour *n; 1570 u8 nud_state, dead; 1571 int err; 1572 1573 if (!nh->nh_grp->gateway || nh->neigh_entry) 1574 return 0; 1575 1576 /* Take a reference of neigh here ensuring that neigh would 1577 * not be detructed before the nexthop entry is finished. 1578 * The reference is taken either in neigh_lookup() or 1579 * in neigh_create() in case n is not found. 1580 */ 1581 n = neigh_lookup(&arp_tbl, &fib_nh->nh_gw, fib_nh->nh_dev); 1582 if (!n) { 1583 n = neigh_create(&arp_tbl, &fib_nh->nh_gw, fib_nh->nh_dev); 1584 if (IS_ERR(n)) 1585 return PTR_ERR(n); 1586 neigh_event_send(n, NULL); 1587 } 1588 neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n); 1589 if (!neigh_entry) { 1590 neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n); 1591 if (IS_ERR(neigh_entry)) { 1592 err = -EINVAL; 1593 goto err_neigh_entry_create; 1594 } 1595 } 1596 1597 /* If that is the first nexthop connected to that neigh, add to 1598 * nexthop_neighs_list 1599 */ 1600 if (list_empty(&neigh_entry->nexthop_list)) 1601 list_add_tail(&neigh_entry->nexthop_neighs_list_node, 1602 &mlxsw_sp->router.nexthop_neighs_list); 1603 1604 nh->neigh_entry = neigh_entry; 1605 list_add_tail(&nh->neigh_list_node, &neigh_entry->nexthop_list); 1606 read_lock_bh(&n->lock); 1607 nud_state = n->nud_state; 1608 dead = n->dead; 1609 read_unlock_bh(&n->lock); 1610 __mlxsw_sp_nexthop_neigh_update(nh, !(nud_state & NUD_VALID && !dead)); 1611 1612 return 0; 1613 1614 err_neigh_entry_create: 1615 neigh_release(n); 1616 return err; 1617 } 1618 1619 static void mlxsw_sp_nexthop_neigh_fini(struct mlxsw_sp *mlxsw_sp, 1620 struct mlxsw_sp_nexthop *nh) 1621 { 1622 struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry; 1623 struct neighbour *n; 1624 1625 if (!neigh_entry) 1626 return; 1627 n = neigh_entry->key.n; 1628 1629 __mlxsw_sp_nexthop_neigh_update(nh, true); 1630 list_del(&nh->neigh_list_node); 1631 nh->neigh_entry = NULL; 1632 1633 /* If that is the last nexthop connected to that neigh, remove from 1634 * nexthop_neighs_list 1635 */ 1636 if (list_empty(&neigh_entry->nexthop_list)) 1637 list_del(&neigh_entry->nexthop_neighs_list_node); 1638 1639 if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list)) 1640 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry); 1641 1642 neigh_release(n); 1643 } 1644 1645 static int mlxsw_sp_nexthop_init(struct mlxsw_sp *mlxsw_sp, 1646 struct mlxsw_sp_nexthop_group *nh_grp, 1647 struct mlxsw_sp_nexthop *nh, 1648 struct fib_nh *fib_nh) 1649 { 1650 struct net_device *dev = fib_nh->nh_dev; 1651 struct in_device *in_dev; 1652 struct mlxsw_sp_rif *rif; 1653 int err; 1654 1655 nh->nh_grp = nh_grp; 1656 nh->key.fib_nh = fib_nh; 1657 err = mlxsw_sp_nexthop_insert(mlxsw_sp, nh); 1658 if (err) 1659 return err; 1660 1661 if (!dev) 1662 return 0; 1663 1664 in_dev = __in_dev_get_rtnl(dev); 1665 if (in_dev && IN_DEV_IGNORE_ROUTES_WITH_LINKDOWN(in_dev) && 1666 fib_nh->nh_flags & RTNH_F_LINKDOWN) 1667 return 0; 1668 1669 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 1670 if (!rif) 1671 return 0; 1672 mlxsw_sp_nexthop_rif_init(nh, rif); 1673 1674 err = mlxsw_sp_nexthop_neigh_init(mlxsw_sp, nh); 1675 if (err) 1676 goto err_nexthop_neigh_init; 1677 1678 return 0; 1679 1680 err_nexthop_neigh_init: 1681 mlxsw_sp_nexthop_remove(mlxsw_sp, nh); 1682 return err; 1683 } 1684 1685 static void mlxsw_sp_nexthop_fini(struct mlxsw_sp *mlxsw_sp, 1686 struct mlxsw_sp_nexthop *nh) 1687 { 1688 mlxsw_sp_nexthop_neigh_fini(mlxsw_sp, nh); 1689 mlxsw_sp_nexthop_rif_fini(nh); 1690 mlxsw_sp_nexthop_remove(mlxsw_sp, nh); 1691 } 1692 1693 static void mlxsw_sp_nexthop_event(struct mlxsw_sp *mlxsw_sp, 1694 unsigned long event, struct fib_nh *fib_nh) 1695 { 1696 struct mlxsw_sp_nexthop_key key; 1697 struct mlxsw_sp_nexthop *nh; 1698 struct mlxsw_sp_rif *rif; 1699 1700 if (mlxsw_sp->router.aborted) 1701 return; 1702 1703 key.fib_nh = fib_nh; 1704 nh = mlxsw_sp_nexthop_lookup(mlxsw_sp, key); 1705 if (WARN_ON_ONCE(!nh)) 1706 return; 1707 1708 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, fib_nh->nh_dev); 1709 if (!rif) 1710 return; 1711 1712 switch (event) { 1713 case FIB_EVENT_NH_ADD: 1714 mlxsw_sp_nexthop_rif_init(nh, rif); 1715 mlxsw_sp_nexthop_neigh_init(mlxsw_sp, nh); 1716 break; 1717 case FIB_EVENT_NH_DEL: 1718 mlxsw_sp_nexthop_neigh_fini(mlxsw_sp, nh); 1719 mlxsw_sp_nexthop_rif_fini(nh); 1720 break; 1721 } 1722 1723 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp); 1724 } 1725 1726 static void mlxsw_sp_nexthop_rif_gone_sync(struct mlxsw_sp *mlxsw_sp, 1727 struct mlxsw_sp_rif *rif) 1728 { 1729 struct mlxsw_sp_nexthop *nh, *tmp; 1730 1731 list_for_each_entry_safe(nh, tmp, &rif->nexthop_list, rif_list_node) { 1732 mlxsw_sp_nexthop_neigh_fini(mlxsw_sp, nh); 1733 mlxsw_sp_nexthop_rif_fini(nh); 1734 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp); 1735 } 1736 } 1737 1738 static struct mlxsw_sp_nexthop_group * 1739 mlxsw_sp_nexthop_group_create(struct mlxsw_sp *mlxsw_sp, struct fib_info *fi) 1740 { 1741 struct mlxsw_sp_nexthop_group *nh_grp; 1742 struct mlxsw_sp_nexthop *nh; 1743 struct fib_nh *fib_nh; 1744 size_t alloc_size; 1745 int i; 1746 int err; 1747 1748 alloc_size = sizeof(*nh_grp) + 1749 fi->fib_nhs * sizeof(struct mlxsw_sp_nexthop); 1750 nh_grp = kzalloc(alloc_size, GFP_KERNEL); 1751 if (!nh_grp) 1752 return ERR_PTR(-ENOMEM); 1753 INIT_LIST_HEAD(&nh_grp->fib_list); 1754 nh_grp->gateway = fi->fib_nh->nh_scope == RT_SCOPE_LINK; 1755 nh_grp->count = fi->fib_nhs; 1756 nh_grp->key.fi = fi; 1757 for (i = 0; i < nh_grp->count; i++) { 1758 nh = &nh_grp->nexthops[i]; 1759 fib_nh = &fi->fib_nh[i]; 1760 err = mlxsw_sp_nexthop_init(mlxsw_sp, nh_grp, nh, fib_nh); 1761 if (err) 1762 goto err_nexthop_init; 1763 } 1764 err = mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp); 1765 if (err) 1766 goto err_nexthop_group_insert; 1767 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp); 1768 return nh_grp; 1769 1770 err_nexthop_group_insert: 1771 err_nexthop_init: 1772 for (i--; i >= 0; i--) { 1773 nh = &nh_grp->nexthops[i]; 1774 mlxsw_sp_nexthop_fini(mlxsw_sp, nh); 1775 } 1776 kfree(nh_grp); 1777 return ERR_PTR(err); 1778 } 1779 1780 static void 1781 mlxsw_sp_nexthop_group_destroy(struct mlxsw_sp *mlxsw_sp, 1782 struct mlxsw_sp_nexthop_group *nh_grp) 1783 { 1784 struct mlxsw_sp_nexthop *nh; 1785 int i; 1786 1787 mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp); 1788 for (i = 0; i < nh_grp->count; i++) { 1789 nh = &nh_grp->nexthops[i]; 1790 mlxsw_sp_nexthop_fini(mlxsw_sp, nh); 1791 } 1792 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp); 1793 WARN_ON_ONCE(nh_grp->adj_index_valid); 1794 kfree(nh_grp); 1795 } 1796 1797 static int mlxsw_sp_nexthop_group_get(struct mlxsw_sp *mlxsw_sp, 1798 struct mlxsw_sp_fib_entry *fib_entry, 1799 struct fib_info *fi) 1800 { 1801 struct mlxsw_sp_nexthop_group_key key; 1802 struct mlxsw_sp_nexthop_group *nh_grp; 1803 1804 key.fi = fi; 1805 nh_grp = mlxsw_sp_nexthop_group_lookup(mlxsw_sp, key); 1806 if (!nh_grp) { 1807 nh_grp = mlxsw_sp_nexthop_group_create(mlxsw_sp, fi); 1808 if (IS_ERR(nh_grp)) 1809 return PTR_ERR(nh_grp); 1810 } 1811 list_add_tail(&fib_entry->nexthop_group_node, &nh_grp->fib_list); 1812 fib_entry->nh_group = nh_grp; 1813 return 0; 1814 } 1815 1816 static void mlxsw_sp_nexthop_group_put(struct mlxsw_sp *mlxsw_sp, 1817 struct mlxsw_sp_fib_entry *fib_entry) 1818 { 1819 struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group; 1820 1821 list_del(&fib_entry->nexthop_group_node); 1822 if (!list_empty(&nh_grp->fib_list)) 1823 return; 1824 mlxsw_sp_nexthop_group_destroy(mlxsw_sp, nh_grp); 1825 } 1826 1827 static bool 1828 mlxsw_sp_fib_entry_should_offload(const struct mlxsw_sp_fib_entry *fib_entry) 1829 { 1830 struct mlxsw_sp_nexthop_group *nh_group = fib_entry->nh_group; 1831 1832 if (fib_entry->params.tos) 1833 return false; 1834 1835 switch (fib_entry->type) { 1836 case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE: 1837 return !!nh_group->adj_index_valid; 1838 case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL: 1839 return !!nh_group->nh_rif; 1840 default: 1841 return false; 1842 } 1843 } 1844 1845 static void mlxsw_sp_fib_entry_offload_set(struct mlxsw_sp_fib_entry *fib_entry) 1846 { 1847 fib_entry->offloaded = true; 1848 1849 switch (fib_entry->fib_node->fib->proto) { 1850 case MLXSW_SP_L3_PROTO_IPV4: 1851 fib_info_offload_inc(fib_entry->nh_group->key.fi); 1852 break; 1853 case MLXSW_SP_L3_PROTO_IPV6: 1854 WARN_ON_ONCE(1); 1855 } 1856 } 1857 1858 static void 1859 mlxsw_sp_fib_entry_offload_unset(struct mlxsw_sp_fib_entry *fib_entry) 1860 { 1861 switch (fib_entry->fib_node->fib->proto) { 1862 case MLXSW_SP_L3_PROTO_IPV4: 1863 fib_info_offload_dec(fib_entry->nh_group->key.fi); 1864 break; 1865 case MLXSW_SP_L3_PROTO_IPV6: 1866 WARN_ON_ONCE(1); 1867 } 1868 1869 fib_entry->offloaded = false; 1870 } 1871 1872 static void 1873 mlxsw_sp_fib_entry_offload_refresh(struct mlxsw_sp_fib_entry *fib_entry, 1874 enum mlxsw_reg_ralue_op op, int err) 1875 { 1876 switch (op) { 1877 case MLXSW_REG_RALUE_OP_WRITE_DELETE: 1878 if (!fib_entry->offloaded) 1879 return; 1880 return mlxsw_sp_fib_entry_offload_unset(fib_entry); 1881 case MLXSW_REG_RALUE_OP_WRITE_WRITE: 1882 if (err) 1883 return; 1884 if (mlxsw_sp_fib_entry_should_offload(fib_entry) && 1885 !fib_entry->offloaded) 1886 mlxsw_sp_fib_entry_offload_set(fib_entry); 1887 else if (!mlxsw_sp_fib_entry_should_offload(fib_entry) && 1888 fib_entry->offloaded) 1889 mlxsw_sp_fib_entry_offload_unset(fib_entry); 1890 return; 1891 default: 1892 return; 1893 } 1894 } 1895 1896 static int mlxsw_sp_fib_entry_op4_remote(struct mlxsw_sp *mlxsw_sp, 1897 struct mlxsw_sp_fib_entry *fib_entry, 1898 enum mlxsw_reg_ralue_op op) 1899 { 1900 char ralue_pl[MLXSW_REG_RALUE_LEN]; 1901 struct mlxsw_sp_fib *fib = fib_entry->fib_node->fib; 1902 u32 *p_dip = (u32 *) fib_entry->fib_node->key.addr; 1903 enum mlxsw_reg_ralue_trap_action trap_action; 1904 u16 trap_id = 0; 1905 u32 adjacency_index = 0; 1906 u16 ecmp_size = 0; 1907 1908 /* In case the nexthop group adjacency index is valid, use it 1909 * with provided ECMP size. Otherwise, setup trap and pass 1910 * traffic to kernel. 1911 */ 1912 if (mlxsw_sp_fib_entry_should_offload(fib_entry)) { 1913 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP; 1914 adjacency_index = fib_entry->nh_group->adj_index; 1915 ecmp_size = fib_entry->nh_group->ecmp_size; 1916 } else { 1917 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP; 1918 trap_id = MLXSW_TRAP_ID_RTR_INGRESS0; 1919 } 1920 1921 mlxsw_reg_ralue_pack4(ralue_pl, 1922 (enum mlxsw_reg_ralxx_protocol) fib->proto, op, 1923 fib->vr->id, fib_entry->fib_node->key.prefix_len, 1924 *p_dip); 1925 mlxsw_reg_ralue_act_remote_pack(ralue_pl, trap_action, trap_id, 1926 adjacency_index, ecmp_size); 1927 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl); 1928 } 1929 1930 static int mlxsw_sp_fib_entry_op4_local(struct mlxsw_sp *mlxsw_sp, 1931 struct mlxsw_sp_fib_entry *fib_entry, 1932 enum mlxsw_reg_ralue_op op) 1933 { 1934 struct mlxsw_sp_rif *rif = fib_entry->nh_group->nh_rif; 1935 struct mlxsw_sp_fib *fib = fib_entry->fib_node->fib; 1936 enum mlxsw_reg_ralue_trap_action trap_action; 1937 char ralue_pl[MLXSW_REG_RALUE_LEN]; 1938 u32 *p_dip = (u32 *) fib_entry->fib_node->key.addr; 1939 u16 trap_id = 0; 1940 u16 rif_index = 0; 1941 1942 if (mlxsw_sp_fib_entry_should_offload(fib_entry)) { 1943 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP; 1944 rif_index = rif->rif_index; 1945 } else { 1946 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP; 1947 trap_id = MLXSW_TRAP_ID_RTR_INGRESS0; 1948 } 1949 1950 mlxsw_reg_ralue_pack4(ralue_pl, 1951 (enum mlxsw_reg_ralxx_protocol) fib->proto, op, 1952 fib->vr->id, fib_entry->fib_node->key.prefix_len, 1953 *p_dip); 1954 mlxsw_reg_ralue_act_local_pack(ralue_pl, trap_action, trap_id, 1955 rif_index); 1956 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl); 1957 } 1958 1959 static int mlxsw_sp_fib_entry_op4_trap(struct mlxsw_sp *mlxsw_sp, 1960 struct mlxsw_sp_fib_entry *fib_entry, 1961 enum mlxsw_reg_ralue_op op) 1962 { 1963 struct mlxsw_sp_fib *fib = fib_entry->fib_node->fib; 1964 char ralue_pl[MLXSW_REG_RALUE_LEN]; 1965 u32 *p_dip = (u32 *) fib_entry->fib_node->key.addr; 1966 1967 mlxsw_reg_ralue_pack4(ralue_pl, 1968 (enum mlxsw_reg_ralxx_protocol) fib->proto, op, 1969 fib->vr->id, fib_entry->fib_node->key.prefix_len, 1970 *p_dip); 1971 mlxsw_reg_ralue_act_ip2me_pack(ralue_pl); 1972 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl); 1973 } 1974 1975 static int mlxsw_sp_fib_entry_op4(struct mlxsw_sp *mlxsw_sp, 1976 struct mlxsw_sp_fib_entry *fib_entry, 1977 enum mlxsw_reg_ralue_op op) 1978 { 1979 switch (fib_entry->type) { 1980 case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE: 1981 return mlxsw_sp_fib_entry_op4_remote(mlxsw_sp, fib_entry, op); 1982 case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL: 1983 return mlxsw_sp_fib_entry_op4_local(mlxsw_sp, fib_entry, op); 1984 case MLXSW_SP_FIB_ENTRY_TYPE_TRAP: 1985 return mlxsw_sp_fib_entry_op4_trap(mlxsw_sp, fib_entry, op); 1986 } 1987 return -EINVAL; 1988 } 1989 1990 static int mlxsw_sp_fib_entry_op(struct mlxsw_sp *mlxsw_sp, 1991 struct mlxsw_sp_fib_entry *fib_entry, 1992 enum mlxsw_reg_ralue_op op) 1993 { 1994 int err = -EINVAL; 1995 1996 switch (fib_entry->fib_node->fib->proto) { 1997 case MLXSW_SP_L3_PROTO_IPV4: 1998 err = mlxsw_sp_fib_entry_op4(mlxsw_sp, fib_entry, op); 1999 break; 2000 case MLXSW_SP_L3_PROTO_IPV6: 2001 return err; 2002 } 2003 mlxsw_sp_fib_entry_offload_refresh(fib_entry, op, err); 2004 return err; 2005 } 2006 2007 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp, 2008 struct mlxsw_sp_fib_entry *fib_entry) 2009 { 2010 return mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry, 2011 MLXSW_REG_RALUE_OP_WRITE_WRITE); 2012 } 2013 2014 static int mlxsw_sp_fib_entry_del(struct mlxsw_sp *mlxsw_sp, 2015 struct mlxsw_sp_fib_entry *fib_entry) 2016 { 2017 return mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry, 2018 MLXSW_REG_RALUE_OP_WRITE_DELETE); 2019 } 2020 2021 static int 2022 mlxsw_sp_fib4_entry_type_set(struct mlxsw_sp *mlxsw_sp, 2023 const struct fib_entry_notifier_info *fen_info, 2024 struct mlxsw_sp_fib_entry *fib_entry) 2025 { 2026 struct fib_info *fi = fen_info->fi; 2027 2028 switch (fen_info->type) { 2029 case RTN_BROADCAST: /* fall through */ 2030 case RTN_LOCAL: 2031 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP; 2032 return 0; 2033 case RTN_UNREACHABLE: /* fall through */ 2034 case RTN_BLACKHOLE: /* fall through */ 2035 case RTN_PROHIBIT: 2036 /* Packets hitting these routes need to be trapped, but 2037 * can do so with a lower priority than packets directed 2038 * at the host, so use action type local instead of trap. 2039 */ 2040 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL; 2041 return 0; 2042 case RTN_UNICAST: 2043 if (fi->fib_nh->nh_scope != RT_SCOPE_LINK) 2044 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL; 2045 else 2046 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_REMOTE; 2047 return 0; 2048 default: 2049 return -EINVAL; 2050 } 2051 } 2052 2053 static struct mlxsw_sp_fib_entry * 2054 mlxsw_sp_fib4_entry_create(struct mlxsw_sp *mlxsw_sp, 2055 struct mlxsw_sp_fib_node *fib_node, 2056 const struct fib_entry_notifier_info *fen_info) 2057 { 2058 struct mlxsw_sp_fib_entry *fib_entry; 2059 int err; 2060 2061 fib_entry = kzalloc(sizeof(*fib_entry), GFP_KERNEL); 2062 if (!fib_entry) { 2063 err = -ENOMEM; 2064 goto err_fib_entry_alloc; 2065 } 2066 2067 err = mlxsw_sp_fib4_entry_type_set(mlxsw_sp, fen_info, fib_entry); 2068 if (err) 2069 goto err_fib4_entry_type_set; 2070 2071 err = mlxsw_sp_nexthop_group_get(mlxsw_sp, fib_entry, fen_info->fi); 2072 if (err) 2073 goto err_nexthop_group_get; 2074 2075 fib_entry->params.prio = fen_info->fi->fib_priority; 2076 fib_entry->params.tb_id = fen_info->tb_id; 2077 fib_entry->params.type = fen_info->type; 2078 fib_entry->params.tos = fen_info->tos; 2079 2080 fib_entry->fib_node = fib_node; 2081 2082 return fib_entry; 2083 2084 err_nexthop_group_get: 2085 err_fib4_entry_type_set: 2086 kfree(fib_entry); 2087 err_fib_entry_alloc: 2088 return ERR_PTR(err); 2089 } 2090 2091 static void mlxsw_sp_fib4_entry_destroy(struct mlxsw_sp *mlxsw_sp, 2092 struct mlxsw_sp_fib_entry *fib_entry) 2093 { 2094 mlxsw_sp_nexthop_group_put(mlxsw_sp, fib_entry); 2095 kfree(fib_entry); 2096 } 2097 2098 static struct mlxsw_sp_fib_node * 2099 mlxsw_sp_fib4_node_get(struct mlxsw_sp *mlxsw_sp, 2100 const struct fib_entry_notifier_info *fen_info); 2101 2102 static struct mlxsw_sp_fib_entry * 2103 mlxsw_sp_fib4_entry_lookup(struct mlxsw_sp *mlxsw_sp, 2104 const struct fib_entry_notifier_info *fen_info) 2105 { 2106 struct mlxsw_sp_fib_entry *fib_entry; 2107 struct mlxsw_sp_fib_node *fib_node; 2108 2109 fib_node = mlxsw_sp_fib4_node_get(mlxsw_sp, fen_info); 2110 if (IS_ERR(fib_node)) 2111 return NULL; 2112 2113 list_for_each_entry(fib_entry, &fib_node->entry_list, list) { 2114 if (fib_entry->params.tb_id == fen_info->tb_id && 2115 fib_entry->params.tos == fen_info->tos && 2116 fib_entry->params.type == fen_info->type && 2117 fib_entry->nh_group->key.fi == fen_info->fi) { 2118 return fib_entry; 2119 } 2120 } 2121 2122 return NULL; 2123 } 2124 2125 static const struct rhashtable_params mlxsw_sp_fib_ht_params = { 2126 .key_offset = offsetof(struct mlxsw_sp_fib_node, key), 2127 .head_offset = offsetof(struct mlxsw_sp_fib_node, ht_node), 2128 .key_len = sizeof(struct mlxsw_sp_fib_key), 2129 .automatic_shrinking = true, 2130 }; 2131 2132 static int mlxsw_sp_fib_node_insert(struct mlxsw_sp_fib *fib, 2133 struct mlxsw_sp_fib_node *fib_node) 2134 { 2135 return rhashtable_insert_fast(&fib->ht, &fib_node->ht_node, 2136 mlxsw_sp_fib_ht_params); 2137 } 2138 2139 static void mlxsw_sp_fib_node_remove(struct mlxsw_sp_fib *fib, 2140 struct mlxsw_sp_fib_node *fib_node) 2141 { 2142 rhashtable_remove_fast(&fib->ht, &fib_node->ht_node, 2143 mlxsw_sp_fib_ht_params); 2144 } 2145 2146 static struct mlxsw_sp_fib_node * 2147 mlxsw_sp_fib_node_lookup(struct mlxsw_sp_fib *fib, const void *addr, 2148 size_t addr_len, unsigned char prefix_len) 2149 { 2150 struct mlxsw_sp_fib_key key; 2151 2152 memset(&key, 0, sizeof(key)); 2153 memcpy(key.addr, addr, addr_len); 2154 key.prefix_len = prefix_len; 2155 return rhashtable_lookup_fast(&fib->ht, &key, mlxsw_sp_fib_ht_params); 2156 } 2157 2158 static struct mlxsw_sp_fib_node * 2159 mlxsw_sp_fib_node_create(struct mlxsw_sp_fib *fib, const void *addr, 2160 size_t addr_len, unsigned char prefix_len) 2161 { 2162 struct mlxsw_sp_fib_node *fib_node; 2163 2164 fib_node = kzalloc(sizeof(*fib_node), GFP_KERNEL); 2165 if (!fib_node) 2166 return NULL; 2167 2168 INIT_LIST_HEAD(&fib_node->entry_list); 2169 list_add(&fib_node->list, &fib->node_list); 2170 memcpy(fib_node->key.addr, addr, addr_len); 2171 fib_node->key.prefix_len = prefix_len; 2172 2173 return fib_node; 2174 } 2175 2176 static void mlxsw_sp_fib_node_destroy(struct mlxsw_sp_fib_node *fib_node) 2177 { 2178 list_del(&fib_node->list); 2179 WARN_ON(!list_empty(&fib_node->entry_list)); 2180 kfree(fib_node); 2181 } 2182 2183 static bool 2184 mlxsw_sp_fib_node_entry_is_first(const struct mlxsw_sp_fib_node *fib_node, 2185 const struct mlxsw_sp_fib_entry *fib_entry) 2186 { 2187 return list_first_entry(&fib_node->entry_list, 2188 struct mlxsw_sp_fib_entry, list) == fib_entry; 2189 } 2190 2191 static void mlxsw_sp_fib_node_prefix_inc(struct mlxsw_sp_fib_node *fib_node) 2192 { 2193 unsigned char prefix_len = fib_node->key.prefix_len; 2194 struct mlxsw_sp_fib *fib = fib_node->fib; 2195 2196 if (fib->prefix_ref_count[prefix_len]++ == 0) 2197 mlxsw_sp_prefix_usage_set(&fib->prefix_usage, prefix_len); 2198 } 2199 2200 static void mlxsw_sp_fib_node_prefix_dec(struct mlxsw_sp_fib_node *fib_node) 2201 { 2202 unsigned char prefix_len = fib_node->key.prefix_len; 2203 struct mlxsw_sp_fib *fib = fib_node->fib; 2204 2205 if (--fib->prefix_ref_count[prefix_len] == 0) 2206 mlxsw_sp_prefix_usage_clear(&fib->prefix_usage, prefix_len); 2207 } 2208 2209 static int mlxsw_sp_fib_node_init(struct mlxsw_sp *mlxsw_sp, 2210 struct mlxsw_sp_fib_node *fib_node, 2211 struct mlxsw_sp_fib *fib) 2212 { 2213 struct mlxsw_sp_prefix_usage req_prefix_usage; 2214 struct mlxsw_sp_lpm_tree *lpm_tree; 2215 int err; 2216 2217 err = mlxsw_sp_fib_node_insert(fib, fib_node); 2218 if (err) 2219 return err; 2220 fib_node->fib = fib; 2221 2222 mlxsw_sp_prefix_usage_cpy(&req_prefix_usage, &fib->prefix_usage); 2223 mlxsw_sp_prefix_usage_set(&req_prefix_usage, fib_node->key.prefix_len); 2224 2225 if (!mlxsw_sp_prefix_usage_none(&fib->prefix_usage)) { 2226 err = mlxsw_sp_vr_lpm_tree_check(mlxsw_sp, fib, 2227 &req_prefix_usage); 2228 if (err) 2229 goto err_tree_check; 2230 } else { 2231 lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage, 2232 fib->proto); 2233 if (IS_ERR(lpm_tree)) 2234 return PTR_ERR(lpm_tree); 2235 fib->lpm_tree = lpm_tree; 2236 err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib); 2237 if (err) 2238 goto err_tree_bind; 2239 } 2240 2241 mlxsw_sp_fib_node_prefix_inc(fib_node); 2242 2243 return 0; 2244 2245 err_tree_bind: 2246 fib->lpm_tree = NULL; 2247 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree); 2248 err_tree_check: 2249 fib_node->fib = NULL; 2250 mlxsw_sp_fib_node_remove(fib, fib_node); 2251 return err; 2252 } 2253 2254 static void mlxsw_sp_fib_node_fini(struct mlxsw_sp *mlxsw_sp, 2255 struct mlxsw_sp_fib_node *fib_node) 2256 { 2257 struct mlxsw_sp_lpm_tree *lpm_tree = fib_node->fib->lpm_tree; 2258 struct mlxsw_sp_fib *fib = fib_node->fib; 2259 2260 mlxsw_sp_fib_node_prefix_dec(fib_node); 2261 2262 if (mlxsw_sp_prefix_usage_none(&fib->prefix_usage)) { 2263 mlxsw_sp_vr_lpm_tree_unbind(mlxsw_sp, fib); 2264 fib->lpm_tree = NULL; 2265 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree); 2266 } else { 2267 mlxsw_sp_vr_lpm_tree_check(mlxsw_sp, fib, &fib->prefix_usage); 2268 } 2269 2270 fib_node->fib = NULL; 2271 mlxsw_sp_fib_node_remove(fib, fib_node); 2272 } 2273 2274 static struct mlxsw_sp_fib_node * 2275 mlxsw_sp_fib4_node_get(struct mlxsw_sp *mlxsw_sp, 2276 const struct fib_entry_notifier_info *fen_info) 2277 { 2278 struct mlxsw_sp_fib_node *fib_node; 2279 struct mlxsw_sp_fib *fib; 2280 struct mlxsw_sp_vr *vr; 2281 int err; 2282 2283 vr = mlxsw_sp_vr_get(mlxsw_sp, fen_info->tb_id); 2284 if (IS_ERR(vr)) 2285 return ERR_CAST(vr); 2286 fib = mlxsw_sp_vr_fib(vr, MLXSW_SP_L3_PROTO_IPV4); 2287 2288 fib_node = mlxsw_sp_fib_node_lookup(fib, &fen_info->dst, 2289 sizeof(fen_info->dst), 2290 fen_info->dst_len); 2291 if (fib_node) 2292 return fib_node; 2293 2294 fib_node = mlxsw_sp_fib_node_create(fib, &fen_info->dst, 2295 sizeof(fen_info->dst), 2296 fen_info->dst_len); 2297 if (!fib_node) { 2298 err = -ENOMEM; 2299 goto err_fib_node_create; 2300 } 2301 2302 err = mlxsw_sp_fib_node_init(mlxsw_sp, fib_node, fib); 2303 if (err) 2304 goto err_fib_node_init; 2305 2306 return fib_node; 2307 2308 err_fib_node_init: 2309 mlxsw_sp_fib_node_destroy(fib_node); 2310 err_fib_node_create: 2311 mlxsw_sp_vr_put(vr); 2312 return ERR_PTR(err); 2313 } 2314 2315 static void mlxsw_sp_fib4_node_put(struct mlxsw_sp *mlxsw_sp, 2316 struct mlxsw_sp_fib_node *fib_node) 2317 { 2318 struct mlxsw_sp_vr *vr = fib_node->fib->vr; 2319 2320 if (!list_empty(&fib_node->entry_list)) 2321 return; 2322 mlxsw_sp_fib_node_fini(mlxsw_sp, fib_node); 2323 mlxsw_sp_fib_node_destroy(fib_node); 2324 mlxsw_sp_vr_put(vr); 2325 } 2326 2327 static struct mlxsw_sp_fib_entry * 2328 mlxsw_sp_fib4_node_entry_find(const struct mlxsw_sp_fib_node *fib_node, 2329 const struct mlxsw_sp_fib_entry_params *params) 2330 { 2331 struct mlxsw_sp_fib_entry *fib_entry; 2332 2333 list_for_each_entry(fib_entry, &fib_node->entry_list, list) { 2334 if (fib_entry->params.tb_id > params->tb_id) 2335 continue; 2336 if (fib_entry->params.tb_id != params->tb_id) 2337 break; 2338 if (fib_entry->params.tos > params->tos) 2339 continue; 2340 if (fib_entry->params.prio >= params->prio || 2341 fib_entry->params.tos < params->tos) 2342 return fib_entry; 2343 } 2344 2345 return NULL; 2346 } 2347 2348 static int mlxsw_sp_fib4_node_list_append(struct mlxsw_sp_fib_entry *fib_entry, 2349 struct mlxsw_sp_fib_entry *new_entry) 2350 { 2351 struct mlxsw_sp_fib_node *fib_node; 2352 2353 if (WARN_ON(!fib_entry)) 2354 return -EINVAL; 2355 2356 fib_node = fib_entry->fib_node; 2357 list_for_each_entry_from(fib_entry, &fib_node->entry_list, list) { 2358 if (fib_entry->params.tb_id != new_entry->params.tb_id || 2359 fib_entry->params.tos != new_entry->params.tos || 2360 fib_entry->params.prio != new_entry->params.prio) 2361 break; 2362 } 2363 2364 list_add_tail(&new_entry->list, &fib_entry->list); 2365 return 0; 2366 } 2367 2368 static int 2369 mlxsw_sp_fib4_node_list_insert(struct mlxsw_sp_fib_node *fib_node, 2370 struct mlxsw_sp_fib_entry *new_entry, 2371 bool replace, bool append) 2372 { 2373 struct mlxsw_sp_fib_entry *fib_entry; 2374 2375 fib_entry = mlxsw_sp_fib4_node_entry_find(fib_node, &new_entry->params); 2376 2377 if (append) 2378 return mlxsw_sp_fib4_node_list_append(fib_entry, new_entry); 2379 if (replace && WARN_ON(!fib_entry)) 2380 return -EINVAL; 2381 2382 /* Insert new entry before replaced one, so that we can later 2383 * remove the second. 2384 */ 2385 if (fib_entry) { 2386 list_add_tail(&new_entry->list, &fib_entry->list); 2387 } else { 2388 struct mlxsw_sp_fib_entry *last; 2389 2390 list_for_each_entry(last, &fib_node->entry_list, list) { 2391 if (new_entry->params.tb_id > last->params.tb_id) 2392 break; 2393 fib_entry = last; 2394 } 2395 2396 if (fib_entry) 2397 list_add(&new_entry->list, &fib_entry->list); 2398 else 2399 list_add(&new_entry->list, &fib_node->entry_list); 2400 } 2401 2402 return 0; 2403 } 2404 2405 static void 2406 mlxsw_sp_fib4_node_list_remove(struct mlxsw_sp_fib_entry *fib_entry) 2407 { 2408 list_del(&fib_entry->list); 2409 } 2410 2411 static int 2412 mlxsw_sp_fib4_node_entry_add(struct mlxsw_sp *mlxsw_sp, 2413 const struct mlxsw_sp_fib_node *fib_node, 2414 struct mlxsw_sp_fib_entry *fib_entry) 2415 { 2416 if (!mlxsw_sp_fib_node_entry_is_first(fib_node, fib_entry)) 2417 return 0; 2418 2419 /* To prevent packet loss, overwrite the previously offloaded 2420 * entry. 2421 */ 2422 if (!list_is_singular(&fib_node->entry_list)) { 2423 enum mlxsw_reg_ralue_op op = MLXSW_REG_RALUE_OP_WRITE_DELETE; 2424 struct mlxsw_sp_fib_entry *n = list_next_entry(fib_entry, list); 2425 2426 mlxsw_sp_fib_entry_offload_refresh(n, op, 0); 2427 } 2428 2429 return mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry); 2430 } 2431 2432 static void 2433 mlxsw_sp_fib4_node_entry_del(struct mlxsw_sp *mlxsw_sp, 2434 const struct mlxsw_sp_fib_node *fib_node, 2435 struct mlxsw_sp_fib_entry *fib_entry) 2436 { 2437 if (!mlxsw_sp_fib_node_entry_is_first(fib_node, fib_entry)) 2438 return; 2439 2440 /* Promote the next entry by overwriting the deleted entry */ 2441 if (!list_is_singular(&fib_node->entry_list)) { 2442 struct mlxsw_sp_fib_entry *n = list_next_entry(fib_entry, list); 2443 enum mlxsw_reg_ralue_op op = MLXSW_REG_RALUE_OP_WRITE_DELETE; 2444 2445 mlxsw_sp_fib_entry_update(mlxsw_sp, n); 2446 mlxsw_sp_fib_entry_offload_refresh(fib_entry, op, 0); 2447 return; 2448 } 2449 2450 mlxsw_sp_fib_entry_del(mlxsw_sp, fib_entry); 2451 } 2452 2453 static int mlxsw_sp_fib4_node_entry_link(struct mlxsw_sp *mlxsw_sp, 2454 struct mlxsw_sp_fib_entry *fib_entry, 2455 bool replace, bool append) 2456 { 2457 struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node; 2458 int err; 2459 2460 err = mlxsw_sp_fib4_node_list_insert(fib_node, fib_entry, replace, 2461 append); 2462 if (err) 2463 return err; 2464 2465 err = mlxsw_sp_fib4_node_entry_add(mlxsw_sp, fib_node, fib_entry); 2466 if (err) 2467 goto err_fib4_node_entry_add; 2468 2469 return 0; 2470 2471 err_fib4_node_entry_add: 2472 mlxsw_sp_fib4_node_list_remove(fib_entry); 2473 return err; 2474 } 2475 2476 static void 2477 mlxsw_sp_fib4_node_entry_unlink(struct mlxsw_sp *mlxsw_sp, 2478 struct mlxsw_sp_fib_entry *fib_entry) 2479 { 2480 struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node; 2481 2482 mlxsw_sp_fib4_node_entry_del(mlxsw_sp, fib_node, fib_entry); 2483 mlxsw_sp_fib4_node_list_remove(fib_entry); 2484 } 2485 2486 static void mlxsw_sp_fib4_entry_replace(struct mlxsw_sp *mlxsw_sp, 2487 struct mlxsw_sp_fib_entry *fib_entry, 2488 bool replace) 2489 { 2490 struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node; 2491 struct mlxsw_sp_fib_entry *replaced; 2492 2493 if (!replace) 2494 return; 2495 2496 /* We inserted the new entry before replaced one */ 2497 replaced = list_next_entry(fib_entry, list); 2498 2499 mlxsw_sp_fib4_node_entry_unlink(mlxsw_sp, replaced); 2500 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, replaced); 2501 mlxsw_sp_fib4_node_put(mlxsw_sp, fib_node); 2502 } 2503 2504 static int 2505 mlxsw_sp_router_fib4_add(struct mlxsw_sp *mlxsw_sp, 2506 const struct fib_entry_notifier_info *fen_info, 2507 bool replace, bool append) 2508 { 2509 struct mlxsw_sp_fib_entry *fib_entry; 2510 struct mlxsw_sp_fib_node *fib_node; 2511 int err; 2512 2513 if (mlxsw_sp->router.aborted) 2514 return 0; 2515 2516 fib_node = mlxsw_sp_fib4_node_get(mlxsw_sp, fen_info); 2517 if (IS_ERR(fib_node)) { 2518 dev_warn(mlxsw_sp->bus_info->dev, "Failed to get FIB node\n"); 2519 return PTR_ERR(fib_node); 2520 } 2521 2522 fib_entry = mlxsw_sp_fib4_entry_create(mlxsw_sp, fib_node, fen_info); 2523 if (IS_ERR(fib_entry)) { 2524 dev_warn(mlxsw_sp->bus_info->dev, "Failed to create FIB entry\n"); 2525 err = PTR_ERR(fib_entry); 2526 goto err_fib4_entry_create; 2527 } 2528 2529 err = mlxsw_sp_fib4_node_entry_link(mlxsw_sp, fib_entry, replace, 2530 append); 2531 if (err) { 2532 dev_warn(mlxsw_sp->bus_info->dev, "Failed to link FIB entry to node\n"); 2533 goto err_fib4_node_entry_link; 2534 } 2535 2536 mlxsw_sp_fib4_entry_replace(mlxsw_sp, fib_entry, replace); 2537 2538 return 0; 2539 2540 err_fib4_node_entry_link: 2541 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib_entry); 2542 err_fib4_entry_create: 2543 mlxsw_sp_fib4_node_put(mlxsw_sp, fib_node); 2544 return err; 2545 } 2546 2547 static void mlxsw_sp_router_fib4_del(struct mlxsw_sp *mlxsw_sp, 2548 struct fib_entry_notifier_info *fen_info) 2549 { 2550 struct mlxsw_sp_fib_entry *fib_entry; 2551 struct mlxsw_sp_fib_node *fib_node; 2552 2553 if (mlxsw_sp->router.aborted) 2554 return; 2555 2556 fib_entry = mlxsw_sp_fib4_entry_lookup(mlxsw_sp, fen_info); 2557 if (WARN_ON(!fib_entry)) 2558 return; 2559 fib_node = fib_entry->fib_node; 2560 2561 mlxsw_sp_fib4_node_entry_unlink(mlxsw_sp, fib_entry); 2562 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib_entry); 2563 mlxsw_sp_fib4_node_put(mlxsw_sp, fib_node); 2564 } 2565 2566 static int mlxsw_sp_router_set_abort_trap(struct mlxsw_sp *mlxsw_sp) 2567 { 2568 char ralta_pl[MLXSW_REG_RALTA_LEN]; 2569 char ralst_pl[MLXSW_REG_RALST_LEN]; 2570 int i, err; 2571 2572 mlxsw_reg_ralta_pack(ralta_pl, true, MLXSW_REG_RALXX_PROTOCOL_IPV4, 2573 MLXSW_SP_LPM_TREE_MIN); 2574 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl); 2575 if (err) 2576 return err; 2577 2578 mlxsw_reg_ralst_pack(ralst_pl, 0xff, MLXSW_SP_LPM_TREE_MIN); 2579 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralst), ralst_pl); 2580 if (err) 2581 return err; 2582 2583 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) { 2584 struct mlxsw_sp_vr *vr = &mlxsw_sp->router.vrs[i]; 2585 char raltb_pl[MLXSW_REG_RALTB_LEN]; 2586 char ralue_pl[MLXSW_REG_RALUE_LEN]; 2587 2588 if (!mlxsw_sp_vr_is_used(vr)) 2589 continue; 2590 2591 mlxsw_reg_raltb_pack(raltb_pl, vr->id, 2592 MLXSW_REG_RALXX_PROTOCOL_IPV4, 2593 MLXSW_SP_LPM_TREE_MIN); 2594 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), 2595 raltb_pl); 2596 if (err) 2597 return err; 2598 2599 mlxsw_reg_ralue_pack4(ralue_pl, MLXSW_SP_L3_PROTO_IPV4, 2600 MLXSW_REG_RALUE_OP_WRITE_WRITE, vr->id, 0, 2601 0); 2602 mlxsw_reg_ralue_act_ip2me_pack(ralue_pl); 2603 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), 2604 ralue_pl); 2605 if (err) 2606 return err; 2607 } 2608 2609 return 0; 2610 } 2611 2612 static void mlxsw_sp_fib4_node_flush(struct mlxsw_sp *mlxsw_sp, 2613 struct mlxsw_sp_fib_node *fib_node) 2614 { 2615 struct mlxsw_sp_fib_entry *fib_entry, *tmp; 2616 2617 list_for_each_entry_safe(fib_entry, tmp, &fib_node->entry_list, list) { 2618 bool do_break = &tmp->list == &fib_node->entry_list; 2619 2620 mlxsw_sp_fib4_node_entry_unlink(mlxsw_sp, fib_entry); 2621 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib_entry); 2622 mlxsw_sp_fib4_node_put(mlxsw_sp, fib_node); 2623 /* Break when entry list is empty and node was freed. 2624 * Otherwise, we'll access freed memory in the next 2625 * iteration. 2626 */ 2627 if (do_break) 2628 break; 2629 } 2630 } 2631 2632 static void mlxsw_sp_fib_node_flush(struct mlxsw_sp *mlxsw_sp, 2633 struct mlxsw_sp_fib_node *fib_node) 2634 { 2635 switch (fib_node->fib->proto) { 2636 case MLXSW_SP_L3_PROTO_IPV4: 2637 mlxsw_sp_fib4_node_flush(mlxsw_sp, fib_node); 2638 break; 2639 case MLXSW_SP_L3_PROTO_IPV6: 2640 WARN_ON_ONCE(1); 2641 break; 2642 } 2643 } 2644 2645 static void mlxsw_sp_vr_fib_flush(struct mlxsw_sp *mlxsw_sp, 2646 struct mlxsw_sp_vr *vr, 2647 enum mlxsw_sp_l3proto proto) 2648 { 2649 struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto); 2650 struct mlxsw_sp_fib_node *fib_node, *tmp; 2651 2652 list_for_each_entry_safe(fib_node, tmp, &fib->node_list, list) { 2653 bool do_break = &tmp->list == &fib->node_list; 2654 2655 mlxsw_sp_fib_node_flush(mlxsw_sp, fib_node); 2656 if (do_break) 2657 break; 2658 } 2659 } 2660 2661 static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp) 2662 { 2663 int i; 2664 2665 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) { 2666 struct mlxsw_sp_vr *vr = &mlxsw_sp->router.vrs[i]; 2667 2668 if (!mlxsw_sp_vr_is_used(vr)) 2669 continue; 2670 mlxsw_sp_vr_fib_flush(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV4); 2671 } 2672 } 2673 2674 static void mlxsw_sp_router_fib4_abort(struct mlxsw_sp *mlxsw_sp) 2675 { 2676 int err; 2677 2678 if (mlxsw_sp->router.aborted) 2679 return; 2680 dev_warn(mlxsw_sp->bus_info->dev, "FIB abort triggered. Note that FIB entries are no longer being offloaded to this device.\n"); 2681 mlxsw_sp_router_fib_flush(mlxsw_sp); 2682 mlxsw_sp->router.aborted = true; 2683 err = mlxsw_sp_router_set_abort_trap(mlxsw_sp); 2684 if (err) 2685 dev_warn(mlxsw_sp->bus_info->dev, "Failed to set abort trap.\n"); 2686 } 2687 2688 struct mlxsw_sp_fib_event_work { 2689 struct work_struct work; 2690 union { 2691 struct fib_entry_notifier_info fen_info; 2692 struct fib_rule_notifier_info fr_info; 2693 struct fib_nh_notifier_info fnh_info; 2694 }; 2695 struct mlxsw_sp *mlxsw_sp; 2696 unsigned long event; 2697 }; 2698 2699 static void mlxsw_sp_router_fib_event_work(struct work_struct *work) 2700 { 2701 struct mlxsw_sp_fib_event_work *fib_work = 2702 container_of(work, struct mlxsw_sp_fib_event_work, work); 2703 struct mlxsw_sp *mlxsw_sp = fib_work->mlxsw_sp; 2704 struct fib_rule *rule; 2705 bool replace, append; 2706 int err; 2707 2708 /* Protect internal structures from changes */ 2709 rtnl_lock(); 2710 switch (fib_work->event) { 2711 case FIB_EVENT_ENTRY_REPLACE: /* fall through */ 2712 case FIB_EVENT_ENTRY_APPEND: /* fall through */ 2713 case FIB_EVENT_ENTRY_ADD: 2714 replace = fib_work->event == FIB_EVENT_ENTRY_REPLACE; 2715 append = fib_work->event == FIB_EVENT_ENTRY_APPEND; 2716 err = mlxsw_sp_router_fib4_add(mlxsw_sp, &fib_work->fen_info, 2717 replace, append); 2718 if (err) 2719 mlxsw_sp_router_fib4_abort(mlxsw_sp); 2720 fib_info_put(fib_work->fen_info.fi); 2721 break; 2722 case FIB_EVENT_ENTRY_DEL: 2723 mlxsw_sp_router_fib4_del(mlxsw_sp, &fib_work->fen_info); 2724 fib_info_put(fib_work->fen_info.fi); 2725 break; 2726 case FIB_EVENT_RULE_ADD: /* fall through */ 2727 case FIB_EVENT_RULE_DEL: 2728 rule = fib_work->fr_info.rule; 2729 if (!fib4_rule_default(rule) && !rule->l3mdev) 2730 mlxsw_sp_router_fib4_abort(mlxsw_sp); 2731 fib_rule_put(rule); 2732 break; 2733 case FIB_EVENT_NH_ADD: /* fall through */ 2734 case FIB_EVENT_NH_DEL: 2735 mlxsw_sp_nexthop_event(mlxsw_sp, fib_work->event, 2736 fib_work->fnh_info.fib_nh); 2737 fib_info_put(fib_work->fnh_info.fib_nh->nh_parent); 2738 break; 2739 } 2740 rtnl_unlock(); 2741 kfree(fib_work); 2742 } 2743 2744 /* Called with rcu_read_lock() */ 2745 static int mlxsw_sp_router_fib_event(struct notifier_block *nb, 2746 unsigned long event, void *ptr) 2747 { 2748 struct mlxsw_sp *mlxsw_sp = container_of(nb, struct mlxsw_sp, fib_nb); 2749 struct mlxsw_sp_fib_event_work *fib_work; 2750 struct fib_notifier_info *info = ptr; 2751 2752 if (!net_eq(info->net, &init_net)) 2753 return NOTIFY_DONE; 2754 2755 fib_work = kzalloc(sizeof(*fib_work), GFP_ATOMIC); 2756 if (WARN_ON(!fib_work)) 2757 return NOTIFY_BAD; 2758 2759 INIT_WORK(&fib_work->work, mlxsw_sp_router_fib_event_work); 2760 fib_work->mlxsw_sp = mlxsw_sp; 2761 fib_work->event = event; 2762 2763 switch (event) { 2764 case FIB_EVENT_ENTRY_REPLACE: /* fall through */ 2765 case FIB_EVENT_ENTRY_APPEND: /* fall through */ 2766 case FIB_EVENT_ENTRY_ADD: /* fall through */ 2767 case FIB_EVENT_ENTRY_DEL: 2768 memcpy(&fib_work->fen_info, ptr, sizeof(fib_work->fen_info)); 2769 /* Take referece on fib_info to prevent it from being 2770 * freed while work is queued. Release it afterwards. 2771 */ 2772 fib_info_hold(fib_work->fen_info.fi); 2773 break; 2774 case FIB_EVENT_RULE_ADD: /* fall through */ 2775 case FIB_EVENT_RULE_DEL: 2776 memcpy(&fib_work->fr_info, ptr, sizeof(fib_work->fr_info)); 2777 fib_rule_get(fib_work->fr_info.rule); 2778 break; 2779 case FIB_EVENT_NH_ADD: /* fall through */ 2780 case FIB_EVENT_NH_DEL: 2781 memcpy(&fib_work->fnh_info, ptr, sizeof(fib_work->fnh_info)); 2782 fib_info_hold(fib_work->fnh_info.fib_nh->nh_parent); 2783 break; 2784 } 2785 2786 mlxsw_core_schedule_work(&fib_work->work); 2787 2788 return NOTIFY_DONE; 2789 } 2790 2791 static struct mlxsw_sp_rif * 2792 mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp, 2793 const struct net_device *dev) 2794 { 2795 int i; 2796 2797 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++) 2798 if (mlxsw_sp->rifs[i] && mlxsw_sp->rifs[i]->dev == dev) 2799 return mlxsw_sp->rifs[i]; 2800 2801 return NULL; 2802 } 2803 2804 static int mlxsw_sp_router_rif_disable(struct mlxsw_sp *mlxsw_sp, u16 rif) 2805 { 2806 char ritr_pl[MLXSW_REG_RITR_LEN]; 2807 int err; 2808 2809 mlxsw_reg_ritr_rif_pack(ritr_pl, rif); 2810 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 2811 if (WARN_ON_ONCE(err)) 2812 return err; 2813 2814 mlxsw_reg_ritr_enable_set(ritr_pl, false); 2815 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 2816 } 2817 2818 static void mlxsw_sp_router_rif_gone_sync(struct mlxsw_sp *mlxsw_sp, 2819 struct mlxsw_sp_rif *rif) 2820 { 2821 mlxsw_sp_router_rif_disable(mlxsw_sp, rif->rif_index); 2822 mlxsw_sp_nexthop_rif_gone_sync(mlxsw_sp, rif); 2823 mlxsw_sp_neigh_rif_gone_sync(mlxsw_sp, rif); 2824 } 2825 2826 static bool mlxsw_sp_rif_should_config(struct mlxsw_sp_rif *rif, 2827 const struct in_device *in_dev, 2828 unsigned long event) 2829 { 2830 switch (event) { 2831 case NETDEV_UP: 2832 if (!rif) 2833 return true; 2834 return false; 2835 case NETDEV_DOWN: 2836 if (rif && !in_dev->ifa_list && 2837 !netif_is_l3_slave(rif->dev)) 2838 return true; 2839 /* It is possible we already removed the RIF ourselves 2840 * if it was assigned to a netdev that is now a bridge 2841 * or LAG slave. 2842 */ 2843 return false; 2844 } 2845 2846 return false; 2847 } 2848 2849 #define MLXSW_SP_INVALID_INDEX_RIF 0xffff 2850 static int mlxsw_sp_avail_rif_get(struct mlxsw_sp *mlxsw_sp) 2851 { 2852 int i; 2853 2854 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++) 2855 if (!mlxsw_sp->rifs[i]) 2856 return i; 2857 2858 return MLXSW_SP_INVALID_INDEX_RIF; 2859 } 2860 2861 static void mlxsw_sp_vport_rif_sp_attr_get(struct mlxsw_sp_port *mlxsw_sp_vport, 2862 bool *p_lagged, u16 *p_system_port) 2863 { 2864 u8 local_port = mlxsw_sp_vport->local_port; 2865 2866 *p_lagged = mlxsw_sp_vport->lagged; 2867 *p_system_port = *p_lagged ? mlxsw_sp_vport->lag_id : local_port; 2868 } 2869 2870 static int mlxsw_sp_vport_rif_sp_op(struct mlxsw_sp_port *mlxsw_sp_vport, 2871 u16 vr_id, struct net_device *l3_dev, 2872 u16 rif_index, bool create) 2873 { 2874 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_vport->mlxsw_sp; 2875 bool lagged = mlxsw_sp_vport->lagged; 2876 char ritr_pl[MLXSW_REG_RITR_LEN]; 2877 u16 system_port; 2878 2879 mlxsw_reg_ritr_pack(ritr_pl, create, MLXSW_REG_RITR_SP_IF, rif_index, 2880 vr_id, l3_dev->mtu, l3_dev->dev_addr); 2881 2882 mlxsw_sp_vport_rif_sp_attr_get(mlxsw_sp_vport, &lagged, &system_port); 2883 mlxsw_reg_ritr_sp_if_pack(ritr_pl, lagged, system_port, 2884 mlxsw_sp_vport_vid_get(mlxsw_sp_vport)); 2885 2886 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 2887 } 2888 2889 static void mlxsw_sp_vport_rif_sp_leave(struct mlxsw_sp_port *mlxsw_sp_vport); 2890 2891 static u16 mlxsw_sp_rif_sp_to_fid(u16 rif_index) 2892 { 2893 return MLXSW_SP_RFID_BASE + rif_index; 2894 } 2895 2896 static struct mlxsw_sp_fid * 2897 mlxsw_sp_rfid_alloc(u16 fid, struct net_device *l3_dev) 2898 { 2899 struct mlxsw_sp_fid *f; 2900 2901 f = kzalloc(sizeof(*f), GFP_KERNEL); 2902 if (!f) 2903 return NULL; 2904 2905 f->leave = mlxsw_sp_vport_rif_sp_leave; 2906 f->ref_count = 0; 2907 f->dev = l3_dev; 2908 f->fid = fid; 2909 2910 return f; 2911 } 2912 2913 static struct mlxsw_sp_rif * 2914 mlxsw_sp_rif_alloc(u16 rif_index, u16 vr_id, struct net_device *l3_dev, 2915 struct mlxsw_sp_fid *f) 2916 { 2917 struct mlxsw_sp_rif *rif; 2918 2919 rif = kzalloc(sizeof(*rif), GFP_KERNEL); 2920 if (!rif) 2921 return NULL; 2922 2923 INIT_LIST_HEAD(&rif->nexthop_list); 2924 INIT_LIST_HEAD(&rif->neigh_list); 2925 ether_addr_copy(rif->addr, l3_dev->dev_addr); 2926 rif->mtu = l3_dev->mtu; 2927 rif->vr_id = vr_id; 2928 rif->dev = l3_dev; 2929 rif->rif_index = rif_index; 2930 rif->f = f; 2931 2932 return rif; 2933 } 2934 2935 u16 mlxsw_sp_rif_index(const struct mlxsw_sp_rif *rif) 2936 { 2937 return rif->rif_index; 2938 } 2939 2940 int mlxsw_sp_rif_dev_ifindex(const struct mlxsw_sp_rif *rif) 2941 { 2942 return rif->dev->ifindex; 2943 } 2944 2945 static struct mlxsw_sp_rif * 2946 mlxsw_sp_vport_rif_sp_create(struct mlxsw_sp_port *mlxsw_sp_vport, 2947 struct net_device *l3_dev) 2948 { 2949 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_vport->mlxsw_sp; 2950 u32 tb_id = l3mdev_fib_table(l3_dev); 2951 struct mlxsw_sp_vr *vr; 2952 struct mlxsw_sp_fid *f; 2953 struct mlxsw_sp_rif *rif; 2954 u16 fid, rif_index; 2955 int err; 2956 2957 rif_index = mlxsw_sp_avail_rif_get(mlxsw_sp); 2958 if (rif_index == MLXSW_SP_INVALID_INDEX_RIF) 2959 return ERR_PTR(-ERANGE); 2960 2961 vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id ? : RT_TABLE_MAIN); 2962 if (IS_ERR(vr)) 2963 return ERR_CAST(vr); 2964 2965 err = mlxsw_sp_vport_rif_sp_op(mlxsw_sp_vport, vr->id, l3_dev, 2966 rif_index, true); 2967 if (err) 2968 goto err_vport_rif_sp_op; 2969 2970 fid = mlxsw_sp_rif_sp_to_fid(rif_index); 2971 err = mlxsw_sp_rif_fdb_op(mlxsw_sp, l3_dev->dev_addr, fid, true); 2972 if (err) 2973 goto err_rif_fdb_op; 2974 2975 f = mlxsw_sp_rfid_alloc(fid, l3_dev); 2976 if (!f) { 2977 err = -ENOMEM; 2978 goto err_rfid_alloc; 2979 } 2980 2981 rif = mlxsw_sp_rif_alloc(rif_index, vr->id, l3_dev, f); 2982 if (!rif) { 2983 err = -ENOMEM; 2984 goto err_rif_alloc; 2985 } 2986 2987 if (devlink_dpipe_table_counter_enabled(priv_to_devlink(mlxsw_sp->core), 2988 MLXSW_SP_DPIPE_TABLE_NAME_ERIF)) { 2989 err = mlxsw_sp_rif_counter_alloc(mlxsw_sp, rif, 2990 MLXSW_SP_RIF_COUNTER_EGRESS); 2991 if (err) 2992 netdev_dbg(mlxsw_sp_vport->dev, 2993 "Counter alloc Failed err=%d\n", err); 2994 } 2995 2996 f->rif = rif; 2997 mlxsw_sp->rifs[rif_index] = rif; 2998 vr->rif_count++; 2999 3000 return rif; 3001 3002 err_rif_alloc: 3003 kfree(f); 3004 err_rfid_alloc: 3005 mlxsw_sp_rif_fdb_op(mlxsw_sp, l3_dev->dev_addr, fid, false); 3006 err_rif_fdb_op: 3007 mlxsw_sp_vport_rif_sp_op(mlxsw_sp_vport, vr->id, l3_dev, rif_index, 3008 false); 3009 err_vport_rif_sp_op: 3010 mlxsw_sp_vr_put(vr); 3011 return ERR_PTR(err); 3012 } 3013 3014 static void mlxsw_sp_vport_rif_sp_destroy(struct mlxsw_sp_port *mlxsw_sp_vport, 3015 struct mlxsw_sp_rif *rif) 3016 { 3017 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_vport->mlxsw_sp; 3018 struct mlxsw_sp_vr *vr = &mlxsw_sp->router.vrs[rif->vr_id]; 3019 struct net_device *l3_dev = rif->dev; 3020 struct mlxsw_sp_fid *f = rif->f; 3021 u16 rif_index = rif->rif_index; 3022 u16 fid = f->fid; 3023 3024 mlxsw_sp_router_rif_gone_sync(mlxsw_sp, rif); 3025 3026 mlxsw_sp_rif_counter_free(mlxsw_sp, rif, MLXSW_SP_RIF_COUNTER_EGRESS); 3027 mlxsw_sp_rif_counter_free(mlxsw_sp, rif, MLXSW_SP_RIF_COUNTER_INGRESS); 3028 3029 vr->rif_count--; 3030 mlxsw_sp->rifs[rif_index] = NULL; 3031 f->rif = NULL; 3032 3033 kfree(rif); 3034 3035 kfree(f); 3036 3037 mlxsw_sp_rif_fdb_op(mlxsw_sp, l3_dev->dev_addr, fid, false); 3038 3039 mlxsw_sp_vport_rif_sp_op(mlxsw_sp_vport, vr->id, l3_dev, rif_index, 3040 false); 3041 mlxsw_sp_vr_put(vr); 3042 } 3043 3044 static int mlxsw_sp_vport_rif_sp_join(struct mlxsw_sp_port *mlxsw_sp_vport, 3045 struct net_device *l3_dev) 3046 { 3047 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_vport->mlxsw_sp; 3048 struct mlxsw_sp_rif *rif; 3049 3050 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev); 3051 if (!rif) { 3052 rif = mlxsw_sp_vport_rif_sp_create(mlxsw_sp_vport, l3_dev); 3053 if (IS_ERR(rif)) 3054 return PTR_ERR(rif); 3055 } 3056 3057 mlxsw_sp_vport_fid_set(mlxsw_sp_vport, rif->f); 3058 rif->f->ref_count++; 3059 3060 netdev_dbg(mlxsw_sp_vport->dev, "Joined FID=%d\n", rif->f->fid); 3061 3062 return 0; 3063 } 3064 3065 static void mlxsw_sp_vport_rif_sp_leave(struct mlxsw_sp_port *mlxsw_sp_vport) 3066 { 3067 struct mlxsw_sp_fid *f = mlxsw_sp_vport_fid_get(mlxsw_sp_vport); 3068 3069 netdev_dbg(mlxsw_sp_vport->dev, "Left FID=%d\n", f->fid); 3070 3071 mlxsw_sp_vport_fid_set(mlxsw_sp_vport, NULL); 3072 if (--f->ref_count == 0) 3073 mlxsw_sp_vport_rif_sp_destroy(mlxsw_sp_vport, f->rif); 3074 } 3075 3076 static int mlxsw_sp_inetaddr_vport_event(struct net_device *l3_dev, 3077 struct net_device *port_dev, 3078 unsigned long event, u16 vid) 3079 { 3080 struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(port_dev); 3081 struct mlxsw_sp_port *mlxsw_sp_vport; 3082 3083 mlxsw_sp_vport = mlxsw_sp_port_vport_find(mlxsw_sp_port, vid); 3084 if (WARN_ON(!mlxsw_sp_vport)) 3085 return -EINVAL; 3086 3087 switch (event) { 3088 case NETDEV_UP: 3089 return mlxsw_sp_vport_rif_sp_join(mlxsw_sp_vport, l3_dev); 3090 case NETDEV_DOWN: 3091 mlxsw_sp_vport_rif_sp_leave(mlxsw_sp_vport); 3092 break; 3093 } 3094 3095 return 0; 3096 } 3097 3098 static int mlxsw_sp_inetaddr_port_event(struct net_device *port_dev, 3099 unsigned long event) 3100 { 3101 if (netif_is_bridge_port(port_dev) || 3102 netif_is_lag_port(port_dev) || 3103 netif_is_ovs_port(port_dev)) 3104 return 0; 3105 3106 return mlxsw_sp_inetaddr_vport_event(port_dev, port_dev, event, 1); 3107 } 3108 3109 static int __mlxsw_sp_inetaddr_lag_event(struct net_device *l3_dev, 3110 struct net_device *lag_dev, 3111 unsigned long event, u16 vid) 3112 { 3113 struct net_device *port_dev; 3114 struct list_head *iter; 3115 int err; 3116 3117 netdev_for_each_lower_dev(lag_dev, port_dev, iter) { 3118 if (mlxsw_sp_port_dev_check(port_dev)) { 3119 err = mlxsw_sp_inetaddr_vport_event(l3_dev, port_dev, 3120 event, vid); 3121 if (err) 3122 return err; 3123 } 3124 } 3125 3126 return 0; 3127 } 3128 3129 static int mlxsw_sp_inetaddr_lag_event(struct net_device *lag_dev, 3130 unsigned long event) 3131 { 3132 if (netif_is_bridge_port(lag_dev)) 3133 return 0; 3134 3135 return __mlxsw_sp_inetaddr_lag_event(lag_dev, lag_dev, event, 1); 3136 } 3137 3138 static struct mlxsw_sp_fid *mlxsw_sp_bridge_fid_get(struct mlxsw_sp *mlxsw_sp, 3139 struct net_device *l3_dev) 3140 { 3141 u16 fid; 3142 3143 if (is_vlan_dev(l3_dev)) 3144 fid = vlan_dev_vlan_id(l3_dev); 3145 else if (mlxsw_sp->master_bridge.dev == l3_dev) 3146 fid = 1; 3147 else 3148 return mlxsw_sp_vfid_find(mlxsw_sp, l3_dev); 3149 3150 return mlxsw_sp_fid_find(mlxsw_sp, fid); 3151 } 3152 3153 static u8 mlxsw_sp_router_port(const struct mlxsw_sp *mlxsw_sp) 3154 { 3155 return mlxsw_core_max_ports(mlxsw_sp->core) + 1; 3156 } 3157 3158 static enum mlxsw_flood_table_type mlxsw_sp_flood_table_type_get(u16 fid) 3159 { 3160 return mlxsw_sp_fid_is_vfid(fid) ? MLXSW_REG_SFGC_TABLE_TYPE_FID : 3161 MLXSW_REG_SFGC_TABLE_TYPE_FID_OFFEST; 3162 } 3163 3164 static u16 mlxsw_sp_flood_table_index_get(u16 fid) 3165 { 3166 return mlxsw_sp_fid_is_vfid(fid) ? mlxsw_sp_fid_to_vfid(fid) : fid; 3167 } 3168 3169 static int mlxsw_sp_router_port_flood_set(struct mlxsw_sp *mlxsw_sp, u16 fid, 3170 bool set) 3171 { 3172 u8 router_port = mlxsw_sp_router_port(mlxsw_sp); 3173 enum mlxsw_flood_table_type table_type; 3174 char *sftr_pl; 3175 u16 index; 3176 int err; 3177 3178 sftr_pl = kmalloc(MLXSW_REG_SFTR_LEN, GFP_KERNEL); 3179 if (!sftr_pl) 3180 return -ENOMEM; 3181 3182 table_type = mlxsw_sp_flood_table_type_get(fid); 3183 index = mlxsw_sp_flood_table_index_get(fid); 3184 mlxsw_reg_sftr_pack(sftr_pl, MLXSW_SP_FLOOD_TABLE_BC, index, table_type, 3185 1, router_port, set); 3186 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(sftr), sftr_pl); 3187 3188 kfree(sftr_pl); 3189 return err; 3190 } 3191 3192 static enum mlxsw_reg_ritr_if_type mlxsw_sp_rif_type_get(u16 fid) 3193 { 3194 if (mlxsw_sp_fid_is_vfid(fid)) 3195 return MLXSW_REG_RITR_FID_IF; 3196 else 3197 return MLXSW_REG_RITR_VLAN_IF; 3198 } 3199 3200 static int mlxsw_sp_rif_bridge_op(struct mlxsw_sp *mlxsw_sp, u16 vr_id, 3201 struct net_device *l3_dev, 3202 u16 fid, u16 rif, 3203 bool create) 3204 { 3205 enum mlxsw_reg_ritr_if_type rif_type; 3206 char ritr_pl[MLXSW_REG_RITR_LEN]; 3207 3208 rif_type = mlxsw_sp_rif_type_get(fid); 3209 mlxsw_reg_ritr_pack(ritr_pl, create, rif_type, rif, vr_id, l3_dev->mtu, 3210 l3_dev->dev_addr); 3211 mlxsw_reg_ritr_fid_set(ritr_pl, rif_type, fid); 3212 3213 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 3214 } 3215 3216 static int mlxsw_sp_rif_bridge_create(struct mlxsw_sp *mlxsw_sp, 3217 struct net_device *l3_dev, 3218 struct mlxsw_sp_fid *f) 3219 { 3220 u32 tb_id = l3mdev_fib_table(l3_dev); 3221 struct mlxsw_sp_rif *rif; 3222 struct mlxsw_sp_vr *vr; 3223 u16 rif_index; 3224 int err; 3225 3226 rif_index = mlxsw_sp_avail_rif_get(mlxsw_sp); 3227 if (rif_index == MLXSW_SP_INVALID_INDEX_RIF) 3228 return -ERANGE; 3229 3230 vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id ? : RT_TABLE_MAIN); 3231 if (IS_ERR(vr)) 3232 return PTR_ERR(vr); 3233 3234 err = mlxsw_sp_router_port_flood_set(mlxsw_sp, f->fid, true); 3235 if (err) 3236 goto err_port_flood_set; 3237 3238 err = mlxsw_sp_rif_bridge_op(mlxsw_sp, vr->id, l3_dev, f->fid, 3239 rif_index, true); 3240 if (err) 3241 goto err_rif_bridge_op; 3242 3243 err = mlxsw_sp_rif_fdb_op(mlxsw_sp, l3_dev->dev_addr, f->fid, true); 3244 if (err) 3245 goto err_rif_fdb_op; 3246 3247 rif = mlxsw_sp_rif_alloc(rif_index, vr->id, l3_dev, f); 3248 if (!rif) { 3249 err = -ENOMEM; 3250 goto err_rif_alloc; 3251 } 3252 3253 f->rif = rif; 3254 mlxsw_sp->rifs[rif_index] = rif; 3255 vr->rif_count++; 3256 3257 netdev_dbg(l3_dev, "RIF=%d created\n", rif_index); 3258 3259 return 0; 3260 3261 err_rif_alloc: 3262 mlxsw_sp_rif_fdb_op(mlxsw_sp, l3_dev->dev_addr, f->fid, false); 3263 err_rif_fdb_op: 3264 mlxsw_sp_rif_bridge_op(mlxsw_sp, vr->id, l3_dev, f->fid, rif_index, 3265 false); 3266 err_rif_bridge_op: 3267 mlxsw_sp_router_port_flood_set(mlxsw_sp, f->fid, false); 3268 err_port_flood_set: 3269 mlxsw_sp_vr_put(vr); 3270 return err; 3271 } 3272 3273 void mlxsw_sp_rif_bridge_destroy(struct mlxsw_sp *mlxsw_sp, 3274 struct mlxsw_sp_rif *rif) 3275 { 3276 struct mlxsw_sp_vr *vr = &mlxsw_sp->router.vrs[rif->vr_id]; 3277 struct net_device *l3_dev = rif->dev; 3278 struct mlxsw_sp_fid *f = rif->f; 3279 u16 rif_index = rif->rif_index; 3280 3281 mlxsw_sp_router_rif_gone_sync(mlxsw_sp, rif); 3282 3283 vr->rif_count--; 3284 mlxsw_sp->rifs[rif_index] = NULL; 3285 f->rif = NULL; 3286 3287 kfree(rif); 3288 3289 mlxsw_sp_rif_fdb_op(mlxsw_sp, l3_dev->dev_addr, f->fid, false); 3290 3291 mlxsw_sp_rif_bridge_op(mlxsw_sp, vr->id, l3_dev, f->fid, rif_index, 3292 false); 3293 3294 mlxsw_sp_router_port_flood_set(mlxsw_sp, f->fid, false); 3295 3296 mlxsw_sp_vr_put(vr); 3297 3298 netdev_dbg(l3_dev, "RIF=%d destroyed\n", rif_index); 3299 } 3300 3301 static int mlxsw_sp_inetaddr_bridge_event(struct net_device *l3_dev, 3302 struct net_device *br_dev, 3303 unsigned long event) 3304 { 3305 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_lower_get(l3_dev); 3306 struct mlxsw_sp_fid *f; 3307 3308 /* FID can either be an actual FID if the L3 device is the 3309 * VLAN-aware bridge or a VLAN device on top. Otherwise, the 3310 * L3 device is a VLAN-unaware bridge and we get a vFID. 3311 */ 3312 f = mlxsw_sp_bridge_fid_get(mlxsw_sp, l3_dev); 3313 if (WARN_ON(!f)) 3314 return -EINVAL; 3315 3316 switch (event) { 3317 case NETDEV_UP: 3318 return mlxsw_sp_rif_bridge_create(mlxsw_sp, l3_dev, f); 3319 case NETDEV_DOWN: 3320 mlxsw_sp_rif_bridge_destroy(mlxsw_sp, f->rif); 3321 break; 3322 } 3323 3324 return 0; 3325 } 3326 3327 static int mlxsw_sp_inetaddr_vlan_event(struct net_device *vlan_dev, 3328 unsigned long event) 3329 { 3330 struct net_device *real_dev = vlan_dev_real_dev(vlan_dev); 3331 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_lower_get(vlan_dev); 3332 u16 vid = vlan_dev_vlan_id(vlan_dev); 3333 3334 if (mlxsw_sp_port_dev_check(real_dev)) 3335 return mlxsw_sp_inetaddr_vport_event(vlan_dev, real_dev, event, 3336 vid); 3337 else if (netif_is_lag_master(real_dev)) 3338 return __mlxsw_sp_inetaddr_lag_event(vlan_dev, real_dev, event, 3339 vid); 3340 else if (netif_is_bridge_master(real_dev) && 3341 mlxsw_sp->master_bridge.dev == real_dev) 3342 return mlxsw_sp_inetaddr_bridge_event(vlan_dev, real_dev, 3343 event); 3344 3345 return 0; 3346 } 3347 3348 static int __mlxsw_sp_inetaddr_event(struct net_device *dev, 3349 unsigned long event) 3350 { 3351 if (mlxsw_sp_port_dev_check(dev)) 3352 return mlxsw_sp_inetaddr_port_event(dev, event); 3353 else if (netif_is_lag_master(dev)) 3354 return mlxsw_sp_inetaddr_lag_event(dev, event); 3355 else if (netif_is_bridge_master(dev)) 3356 return mlxsw_sp_inetaddr_bridge_event(dev, dev, event); 3357 else if (is_vlan_dev(dev)) 3358 return mlxsw_sp_inetaddr_vlan_event(dev, event); 3359 else 3360 return 0; 3361 } 3362 3363 int mlxsw_sp_inetaddr_event(struct notifier_block *unused, 3364 unsigned long event, void *ptr) 3365 { 3366 struct in_ifaddr *ifa = (struct in_ifaddr *) ptr; 3367 struct net_device *dev = ifa->ifa_dev->dev; 3368 struct mlxsw_sp *mlxsw_sp; 3369 struct mlxsw_sp_rif *rif; 3370 int err = 0; 3371 3372 mlxsw_sp = mlxsw_sp_lower_get(dev); 3373 if (!mlxsw_sp) 3374 goto out; 3375 3376 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 3377 if (!mlxsw_sp_rif_should_config(rif, ifa->ifa_dev, event)) 3378 goto out; 3379 3380 err = __mlxsw_sp_inetaddr_event(dev, event); 3381 out: 3382 return notifier_from_errno(err); 3383 } 3384 3385 static int mlxsw_sp_rif_edit(struct mlxsw_sp *mlxsw_sp, u16 rif_index, 3386 const char *mac, int mtu) 3387 { 3388 char ritr_pl[MLXSW_REG_RITR_LEN]; 3389 int err; 3390 3391 mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index); 3392 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 3393 if (err) 3394 return err; 3395 3396 mlxsw_reg_ritr_mtu_set(ritr_pl, mtu); 3397 mlxsw_reg_ritr_if_mac_memcpy_to(ritr_pl, mac); 3398 mlxsw_reg_ritr_op_set(ritr_pl, MLXSW_REG_RITR_RIF_CREATE); 3399 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 3400 } 3401 3402 int mlxsw_sp_netdevice_router_port_event(struct net_device *dev) 3403 { 3404 struct mlxsw_sp *mlxsw_sp; 3405 struct mlxsw_sp_rif *rif; 3406 int err; 3407 3408 mlxsw_sp = mlxsw_sp_lower_get(dev); 3409 if (!mlxsw_sp) 3410 return 0; 3411 3412 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 3413 if (!rif) 3414 return 0; 3415 3416 err = mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->addr, rif->f->fid, false); 3417 if (err) 3418 return err; 3419 3420 err = mlxsw_sp_rif_edit(mlxsw_sp, rif->rif_index, dev->dev_addr, 3421 dev->mtu); 3422 if (err) 3423 goto err_rif_edit; 3424 3425 err = mlxsw_sp_rif_fdb_op(mlxsw_sp, dev->dev_addr, rif->f->fid, true); 3426 if (err) 3427 goto err_rif_fdb_op; 3428 3429 ether_addr_copy(rif->addr, dev->dev_addr); 3430 rif->mtu = dev->mtu; 3431 3432 netdev_dbg(dev, "Updated RIF=%d\n", rif->rif_index); 3433 3434 return 0; 3435 3436 err_rif_fdb_op: 3437 mlxsw_sp_rif_edit(mlxsw_sp, rif->rif_index, rif->addr, rif->mtu); 3438 err_rif_edit: 3439 mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->addr, rif->f->fid, true); 3440 return err; 3441 } 3442 3443 static int mlxsw_sp_port_vrf_join(struct mlxsw_sp *mlxsw_sp, 3444 struct net_device *l3_dev) 3445 { 3446 struct mlxsw_sp_rif *rif; 3447 3448 /* If netdev is already associated with a RIF, then we need to 3449 * destroy it and create a new one with the new virtual router ID. 3450 */ 3451 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev); 3452 if (rif) 3453 __mlxsw_sp_inetaddr_event(l3_dev, NETDEV_DOWN); 3454 3455 return __mlxsw_sp_inetaddr_event(l3_dev, NETDEV_UP); 3456 } 3457 3458 static void mlxsw_sp_port_vrf_leave(struct mlxsw_sp *mlxsw_sp, 3459 struct net_device *l3_dev) 3460 { 3461 struct mlxsw_sp_rif *rif; 3462 3463 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev); 3464 if (!rif) 3465 return; 3466 __mlxsw_sp_inetaddr_event(l3_dev, NETDEV_DOWN); 3467 } 3468 3469 int mlxsw_sp_netdevice_vrf_event(struct net_device *l3_dev, unsigned long event, 3470 struct netdev_notifier_changeupper_info *info) 3471 { 3472 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_lower_get(l3_dev); 3473 int err = 0; 3474 3475 if (!mlxsw_sp) 3476 return 0; 3477 3478 switch (event) { 3479 case NETDEV_PRECHANGEUPPER: 3480 return 0; 3481 case NETDEV_CHANGEUPPER: 3482 if (info->linking) 3483 err = mlxsw_sp_port_vrf_join(mlxsw_sp, l3_dev); 3484 else 3485 mlxsw_sp_port_vrf_leave(mlxsw_sp, l3_dev); 3486 break; 3487 } 3488 3489 return err; 3490 } 3491 3492 static void mlxsw_sp_router_fib_dump_flush(struct notifier_block *nb) 3493 { 3494 struct mlxsw_sp *mlxsw_sp = container_of(nb, struct mlxsw_sp, fib_nb); 3495 3496 /* Flush pending FIB notifications and then flush the device's 3497 * table before requesting another dump. The FIB notification 3498 * block is unregistered, so no need to take RTNL. 3499 */ 3500 mlxsw_core_flush_owq(); 3501 mlxsw_sp_router_fib_flush(mlxsw_sp); 3502 } 3503 3504 static int __mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp) 3505 { 3506 char rgcr_pl[MLXSW_REG_RGCR_LEN]; 3507 u64 max_rifs; 3508 int err; 3509 3510 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_RIFS)) 3511 return -EIO; 3512 3513 max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); 3514 mlxsw_sp->rifs = kcalloc(max_rifs, sizeof(struct mlxsw_sp_rif *), 3515 GFP_KERNEL); 3516 if (!mlxsw_sp->rifs) 3517 return -ENOMEM; 3518 3519 mlxsw_reg_rgcr_pack(rgcr_pl, true); 3520 mlxsw_reg_rgcr_max_router_interfaces_set(rgcr_pl, max_rifs); 3521 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl); 3522 if (err) 3523 goto err_rgcr_fail; 3524 3525 return 0; 3526 3527 err_rgcr_fail: 3528 kfree(mlxsw_sp->rifs); 3529 return err; 3530 } 3531 3532 static void __mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp) 3533 { 3534 char rgcr_pl[MLXSW_REG_RGCR_LEN]; 3535 int i; 3536 3537 mlxsw_reg_rgcr_pack(rgcr_pl, false); 3538 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl); 3539 3540 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++) 3541 WARN_ON_ONCE(mlxsw_sp->rifs[i]); 3542 3543 kfree(mlxsw_sp->rifs); 3544 } 3545 3546 int mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp) 3547 { 3548 int err; 3549 3550 INIT_LIST_HEAD(&mlxsw_sp->router.nexthop_neighs_list); 3551 err = __mlxsw_sp_router_init(mlxsw_sp); 3552 if (err) 3553 return err; 3554 3555 err = rhashtable_init(&mlxsw_sp->router.nexthop_ht, 3556 &mlxsw_sp_nexthop_ht_params); 3557 if (err) 3558 goto err_nexthop_ht_init; 3559 3560 err = rhashtable_init(&mlxsw_sp->router.nexthop_group_ht, 3561 &mlxsw_sp_nexthop_group_ht_params); 3562 if (err) 3563 goto err_nexthop_group_ht_init; 3564 3565 err = mlxsw_sp_lpm_init(mlxsw_sp); 3566 if (err) 3567 goto err_lpm_init; 3568 3569 err = mlxsw_sp_vrs_init(mlxsw_sp); 3570 if (err) 3571 goto err_vrs_init; 3572 3573 err = mlxsw_sp_neigh_init(mlxsw_sp); 3574 if (err) 3575 goto err_neigh_init; 3576 3577 mlxsw_sp->fib_nb.notifier_call = mlxsw_sp_router_fib_event; 3578 err = register_fib_notifier(&mlxsw_sp->fib_nb, 3579 mlxsw_sp_router_fib_dump_flush); 3580 if (err) 3581 goto err_register_fib_notifier; 3582 3583 return 0; 3584 3585 err_register_fib_notifier: 3586 mlxsw_sp_neigh_fini(mlxsw_sp); 3587 err_neigh_init: 3588 mlxsw_sp_vrs_fini(mlxsw_sp); 3589 err_vrs_init: 3590 mlxsw_sp_lpm_fini(mlxsw_sp); 3591 err_lpm_init: 3592 rhashtable_destroy(&mlxsw_sp->router.nexthop_group_ht); 3593 err_nexthop_group_ht_init: 3594 rhashtable_destroy(&mlxsw_sp->router.nexthop_ht); 3595 err_nexthop_ht_init: 3596 __mlxsw_sp_router_fini(mlxsw_sp); 3597 return err; 3598 } 3599 3600 void mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp) 3601 { 3602 unregister_fib_notifier(&mlxsw_sp->fib_nb); 3603 mlxsw_sp_neigh_fini(mlxsw_sp); 3604 mlxsw_sp_vrs_fini(mlxsw_sp); 3605 mlxsw_sp_lpm_fini(mlxsw_sp); 3606 rhashtable_destroy(&mlxsw_sp->router.nexthop_group_ht); 3607 rhashtable_destroy(&mlxsw_sp->router.nexthop_ht); 3608 __mlxsw_sp_router_fini(mlxsw_sp); 3609 } 3610