1 // SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0 2 /* Copyright (c) 2016-2018 Mellanox Technologies. All rights reserved */ 3 4 #include <linux/kernel.h> 5 #include <linux/types.h> 6 #include <linux/rhashtable.h> 7 #include <linux/bitops.h> 8 #include <linux/in6.h> 9 #include <linux/notifier.h> 10 #include <linux/inetdevice.h> 11 #include <linux/netdevice.h> 12 #include <linux/if_bridge.h> 13 #include <linux/socket.h> 14 #include <linux/route.h> 15 #include <linux/gcd.h> 16 #include <linux/if_macvlan.h> 17 #include <linux/refcount.h> 18 #include <linux/jhash.h> 19 #include <linux/net_namespace.h> 20 #include <linux/mutex.h> 21 #include <net/netevent.h> 22 #include <net/neighbour.h> 23 #include <net/arp.h> 24 #include <net/ip_fib.h> 25 #include <net/ip6_fib.h> 26 #include <net/nexthop.h> 27 #include <net/fib_rules.h> 28 #include <net/ip_tunnels.h> 29 #include <net/l3mdev.h> 30 #include <net/addrconf.h> 31 #include <net/ndisc.h> 32 #include <net/ipv6.h> 33 #include <net/fib_notifier.h> 34 #include <net/switchdev.h> 35 36 #include "spectrum.h" 37 #include "core.h" 38 #include "reg.h" 39 #include "spectrum_cnt.h" 40 #include "spectrum_dpipe.h" 41 #include "spectrum_ipip.h" 42 #include "spectrum_mr.h" 43 #include "spectrum_mr_tcam.h" 44 #include "spectrum_router.h" 45 #include "spectrum_span.h" 46 47 struct mlxsw_sp_fib; 48 struct mlxsw_sp_vr; 49 struct mlxsw_sp_lpm_tree; 50 struct mlxsw_sp_rif_ops; 51 52 struct mlxsw_sp_rif { 53 struct list_head nexthop_list; 54 struct list_head neigh_list; 55 struct net_device *dev; /* NULL for underlay RIF */ 56 struct mlxsw_sp_fid *fid; 57 unsigned char addr[ETH_ALEN]; 58 int mtu; 59 u16 rif_index; 60 u16 vr_id; 61 const struct mlxsw_sp_rif_ops *ops; 62 struct mlxsw_sp *mlxsw_sp; 63 64 unsigned int counter_ingress; 65 bool counter_ingress_valid; 66 unsigned int counter_egress; 67 bool counter_egress_valid; 68 }; 69 70 struct mlxsw_sp_rif_params { 71 struct net_device *dev; 72 union { 73 u16 system_port; 74 u16 lag_id; 75 }; 76 u16 vid; 77 bool lag; 78 }; 79 80 struct mlxsw_sp_rif_subport { 81 struct mlxsw_sp_rif common; 82 refcount_t ref_count; 83 union { 84 u16 system_port; 85 u16 lag_id; 86 }; 87 u16 vid; 88 bool lag; 89 }; 90 91 struct mlxsw_sp_rif_ipip_lb { 92 struct mlxsw_sp_rif common; 93 struct mlxsw_sp_rif_ipip_lb_config lb_config; 94 u16 ul_vr_id; /* Reserved for Spectrum-2. */ 95 u16 ul_rif_id; /* Reserved for Spectrum. */ 96 }; 97 98 struct mlxsw_sp_rif_params_ipip_lb { 99 struct mlxsw_sp_rif_params common; 100 struct mlxsw_sp_rif_ipip_lb_config lb_config; 101 }; 102 103 struct mlxsw_sp_rif_ops { 104 enum mlxsw_sp_rif_type type; 105 size_t rif_size; 106 107 void (*setup)(struct mlxsw_sp_rif *rif, 108 const struct mlxsw_sp_rif_params *params); 109 int (*configure)(struct mlxsw_sp_rif *rif); 110 void (*deconfigure)(struct mlxsw_sp_rif *rif); 111 struct mlxsw_sp_fid * (*fid_get)(struct mlxsw_sp_rif *rif, 112 struct netlink_ext_ack *extack); 113 void (*fdb_del)(struct mlxsw_sp_rif *rif, const char *mac); 114 }; 115 116 static struct mlxsw_sp_rif * 117 mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp, 118 const struct net_device *dev); 119 static void mlxsw_sp_rif_destroy(struct mlxsw_sp_rif *rif); 120 static void mlxsw_sp_lpm_tree_hold(struct mlxsw_sp_lpm_tree *lpm_tree); 121 static void mlxsw_sp_lpm_tree_put(struct mlxsw_sp *mlxsw_sp, 122 struct mlxsw_sp_lpm_tree *lpm_tree); 123 static int mlxsw_sp_vr_lpm_tree_bind(struct mlxsw_sp *mlxsw_sp, 124 const struct mlxsw_sp_fib *fib, 125 u8 tree_id); 126 static int mlxsw_sp_vr_lpm_tree_unbind(struct mlxsw_sp *mlxsw_sp, 127 const struct mlxsw_sp_fib *fib); 128 129 static unsigned int * 130 mlxsw_sp_rif_p_counter_get(struct mlxsw_sp_rif *rif, 131 enum mlxsw_sp_rif_counter_dir dir) 132 { 133 switch (dir) { 134 case MLXSW_SP_RIF_COUNTER_EGRESS: 135 return &rif->counter_egress; 136 case MLXSW_SP_RIF_COUNTER_INGRESS: 137 return &rif->counter_ingress; 138 } 139 return NULL; 140 } 141 142 static bool 143 mlxsw_sp_rif_counter_valid_get(struct mlxsw_sp_rif *rif, 144 enum mlxsw_sp_rif_counter_dir dir) 145 { 146 switch (dir) { 147 case MLXSW_SP_RIF_COUNTER_EGRESS: 148 return rif->counter_egress_valid; 149 case MLXSW_SP_RIF_COUNTER_INGRESS: 150 return rif->counter_ingress_valid; 151 } 152 return false; 153 } 154 155 static void 156 mlxsw_sp_rif_counter_valid_set(struct mlxsw_sp_rif *rif, 157 enum mlxsw_sp_rif_counter_dir dir, 158 bool valid) 159 { 160 switch (dir) { 161 case MLXSW_SP_RIF_COUNTER_EGRESS: 162 rif->counter_egress_valid = valid; 163 break; 164 case MLXSW_SP_RIF_COUNTER_INGRESS: 165 rif->counter_ingress_valid = valid; 166 break; 167 } 168 } 169 170 static int mlxsw_sp_rif_counter_edit(struct mlxsw_sp *mlxsw_sp, u16 rif_index, 171 unsigned int counter_index, bool enable, 172 enum mlxsw_sp_rif_counter_dir dir) 173 { 174 char ritr_pl[MLXSW_REG_RITR_LEN]; 175 bool is_egress = false; 176 int err; 177 178 if (dir == MLXSW_SP_RIF_COUNTER_EGRESS) 179 is_egress = true; 180 mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index); 181 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 182 if (err) 183 return err; 184 185 mlxsw_reg_ritr_counter_pack(ritr_pl, counter_index, enable, 186 is_egress); 187 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 188 } 189 190 int mlxsw_sp_rif_counter_value_get(struct mlxsw_sp *mlxsw_sp, 191 struct mlxsw_sp_rif *rif, 192 enum mlxsw_sp_rif_counter_dir dir, u64 *cnt) 193 { 194 char ricnt_pl[MLXSW_REG_RICNT_LEN]; 195 unsigned int *p_counter_index; 196 bool valid; 197 int err; 198 199 valid = mlxsw_sp_rif_counter_valid_get(rif, dir); 200 if (!valid) 201 return -EINVAL; 202 203 p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir); 204 if (!p_counter_index) 205 return -EINVAL; 206 mlxsw_reg_ricnt_pack(ricnt_pl, *p_counter_index, 207 MLXSW_REG_RICNT_OPCODE_NOP); 208 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl); 209 if (err) 210 return err; 211 *cnt = mlxsw_reg_ricnt_good_unicast_packets_get(ricnt_pl); 212 return 0; 213 } 214 215 static int mlxsw_sp_rif_counter_clear(struct mlxsw_sp *mlxsw_sp, 216 unsigned int counter_index) 217 { 218 char ricnt_pl[MLXSW_REG_RICNT_LEN]; 219 220 mlxsw_reg_ricnt_pack(ricnt_pl, counter_index, 221 MLXSW_REG_RICNT_OPCODE_CLEAR); 222 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl); 223 } 224 225 int mlxsw_sp_rif_counter_alloc(struct mlxsw_sp *mlxsw_sp, 226 struct mlxsw_sp_rif *rif, 227 enum mlxsw_sp_rif_counter_dir dir) 228 { 229 unsigned int *p_counter_index; 230 int err; 231 232 p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir); 233 if (!p_counter_index) 234 return -EINVAL; 235 err = mlxsw_sp_counter_alloc(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF, 236 p_counter_index); 237 if (err) 238 return err; 239 240 err = mlxsw_sp_rif_counter_clear(mlxsw_sp, *p_counter_index); 241 if (err) 242 goto err_counter_clear; 243 244 err = mlxsw_sp_rif_counter_edit(mlxsw_sp, rif->rif_index, 245 *p_counter_index, true, dir); 246 if (err) 247 goto err_counter_edit; 248 mlxsw_sp_rif_counter_valid_set(rif, dir, true); 249 return 0; 250 251 err_counter_edit: 252 err_counter_clear: 253 mlxsw_sp_counter_free(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF, 254 *p_counter_index); 255 return err; 256 } 257 258 void mlxsw_sp_rif_counter_free(struct mlxsw_sp *mlxsw_sp, 259 struct mlxsw_sp_rif *rif, 260 enum mlxsw_sp_rif_counter_dir dir) 261 { 262 unsigned int *p_counter_index; 263 264 if (!mlxsw_sp_rif_counter_valid_get(rif, dir)) 265 return; 266 267 p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir); 268 if (WARN_ON(!p_counter_index)) 269 return; 270 mlxsw_sp_rif_counter_edit(mlxsw_sp, rif->rif_index, 271 *p_counter_index, false, dir); 272 mlxsw_sp_counter_free(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF, 273 *p_counter_index); 274 mlxsw_sp_rif_counter_valid_set(rif, dir, false); 275 } 276 277 static void mlxsw_sp_rif_counters_alloc(struct mlxsw_sp_rif *rif) 278 { 279 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 280 struct devlink *devlink; 281 282 devlink = priv_to_devlink(mlxsw_sp->core); 283 if (!devlink_dpipe_table_counter_enabled(devlink, 284 MLXSW_SP_DPIPE_TABLE_NAME_ERIF)) 285 return; 286 mlxsw_sp_rif_counter_alloc(mlxsw_sp, rif, MLXSW_SP_RIF_COUNTER_EGRESS); 287 } 288 289 static void mlxsw_sp_rif_counters_free(struct mlxsw_sp_rif *rif) 290 { 291 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 292 293 mlxsw_sp_rif_counter_free(mlxsw_sp, rif, MLXSW_SP_RIF_COUNTER_EGRESS); 294 } 295 296 #define MLXSW_SP_PREFIX_COUNT (sizeof(struct in6_addr) * BITS_PER_BYTE + 1) 297 298 struct mlxsw_sp_prefix_usage { 299 DECLARE_BITMAP(b, MLXSW_SP_PREFIX_COUNT); 300 }; 301 302 #define mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) \ 303 for_each_set_bit(prefix, (prefix_usage)->b, MLXSW_SP_PREFIX_COUNT) 304 305 static bool 306 mlxsw_sp_prefix_usage_eq(struct mlxsw_sp_prefix_usage *prefix_usage1, 307 struct mlxsw_sp_prefix_usage *prefix_usage2) 308 { 309 return !memcmp(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1)); 310 } 311 312 static void 313 mlxsw_sp_prefix_usage_cpy(struct mlxsw_sp_prefix_usage *prefix_usage1, 314 struct mlxsw_sp_prefix_usage *prefix_usage2) 315 { 316 memcpy(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1)); 317 } 318 319 static void 320 mlxsw_sp_prefix_usage_set(struct mlxsw_sp_prefix_usage *prefix_usage, 321 unsigned char prefix_len) 322 { 323 set_bit(prefix_len, prefix_usage->b); 324 } 325 326 static void 327 mlxsw_sp_prefix_usage_clear(struct mlxsw_sp_prefix_usage *prefix_usage, 328 unsigned char prefix_len) 329 { 330 clear_bit(prefix_len, prefix_usage->b); 331 } 332 333 struct mlxsw_sp_fib_key { 334 unsigned char addr[sizeof(struct in6_addr)]; 335 unsigned char prefix_len; 336 }; 337 338 enum mlxsw_sp_fib_entry_type { 339 MLXSW_SP_FIB_ENTRY_TYPE_REMOTE, 340 MLXSW_SP_FIB_ENTRY_TYPE_LOCAL, 341 MLXSW_SP_FIB_ENTRY_TYPE_TRAP, 342 MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE, 343 MLXSW_SP_FIB_ENTRY_TYPE_UNREACHABLE, 344 345 /* This is a special case of local delivery, where a packet should be 346 * decapsulated on reception. Note that there is no corresponding ENCAP, 347 * because that's a type of next hop, not of FIB entry. (There can be 348 * several next hops in a REMOTE entry, and some of them may be 349 * encapsulating entries.) 350 */ 351 MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP, 352 MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP, 353 }; 354 355 struct mlxsw_sp_nexthop_group_info; 356 struct mlxsw_sp_nexthop_group; 357 struct mlxsw_sp_fib_entry; 358 359 struct mlxsw_sp_fib_node { 360 struct mlxsw_sp_fib_entry *fib_entry; 361 struct list_head list; 362 struct rhash_head ht_node; 363 struct mlxsw_sp_fib *fib; 364 struct mlxsw_sp_fib_key key; 365 }; 366 367 struct mlxsw_sp_fib_entry_decap { 368 struct mlxsw_sp_ipip_entry *ipip_entry; 369 u32 tunnel_index; 370 }; 371 372 static struct mlxsw_sp_fib_entry_priv * 373 mlxsw_sp_fib_entry_priv_create(const struct mlxsw_sp_router_ll_ops *ll_ops) 374 { 375 struct mlxsw_sp_fib_entry_priv *priv; 376 377 if (!ll_ops->fib_entry_priv_size) 378 /* No need to have priv */ 379 return NULL; 380 381 priv = kzalloc(sizeof(*priv) + ll_ops->fib_entry_priv_size, GFP_KERNEL); 382 if (!priv) 383 return ERR_PTR(-ENOMEM); 384 refcount_set(&priv->refcnt, 1); 385 return priv; 386 } 387 388 static void 389 mlxsw_sp_fib_entry_priv_destroy(struct mlxsw_sp_fib_entry_priv *priv) 390 { 391 kfree(priv); 392 } 393 394 static void mlxsw_sp_fib_entry_priv_hold(struct mlxsw_sp_fib_entry_priv *priv) 395 { 396 refcount_inc(&priv->refcnt); 397 } 398 399 static void mlxsw_sp_fib_entry_priv_put(struct mlxsw_sp_fib_entry_priv *priv) 400 { 401 if (!priv || !refcount_dec_and_test(&priv->refcnt)) 402 return; 403 mlxsw_sp_fib_entry_priv_destroy(priv); 404 } 405 406 static void mlxsw_sp_fib_entry_op_ctx_priv_hold(struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 407 struct mlxsw_sp_fib_entry_priv *priv) 408 { 409 if (!priv) 410 return; 411 mlxsw_sp_fib_entry_priv_hold(priv); 412 list_add(&priv->list, &op_ctx->fib_entry_priv_list); 413 } 414 415 static void mlxsw_sp_fib_entry_op_ctx_priv_put_all(struct mlxsw_sp_fib_entry_op_ctx *op_ctx) 416 { 417 struct mlxsw_sp_fib_entry_priv *priv, *tmp; 418 419 list_for_each_entry_safe(priv, tmp, &op_ctx->fib_entry_priv_list, list) 420 mlxsw_sp_fib_entry_priv_put(priv); 421 INIT_LIST_HEAD(&op_ctx->fib_entry_priv_list); 422 } 423 424 struct mlxsw_sp_fib_entry { 425 struct mlxsw_sp_fib_node *fib_node; 426 enum mlxsw_sp_fib_entry_type type; 427 struct list_head nexthop_group_node; 428 struct mlxsw_sp_nexthop_group *nh_group; 429 struct mlxsw_sp_fib_entry_decap decap; /* Valid for decap entries. */ 430 struct mlxsw_sp_fib_entry_priv *priv; 431 }; 432 433 struct mlxsw_sp_fib4_entry { 434 struct mlxsw_sp_fib_entry common; 435 struct fib_info *fi; 436 u32 tb_id; 437 u8 tos; 438 u8 type; 439 }; 440 441 struct mlxsw_sp_fib6_entry { 442 struct mlxsw_sp_fib_entry common; 443 struct list_head rt6_list; 444 unsigned int nrt6; 445 }; 446 447 struct mlxsw_sp_rt6 { 448 struct list_head list; 449 struct fib6_info *rt; 450 }; 451 452 struct mlxsw_sp_lpm_tree { 453 u8 id; /* tree ID */ 454 unsigned int ref_count; 455 enum mlxsw_sp_l3proto proto; 456 unsigned long prefix_ref_count[MLXSW_SP_PREFIX_COUNT]; 457 struct mlxsw_sp_prefix_usage prefix_usage; 458 }; 459 460 struct mlxsw_sp_fib { 461 struct rhashtable ht; 462 struct list_head node_list; 463 struct mlxsw_sp_vr *vr; 464 struct mlxsw_sp_lpm_tree *lpm_tree; 465 enum mlxsw_sp_l3proto proto; 466 const struct mlxsw_sp_router_ll_ops *ll_ops; 467 }; 468 469 struct mlxsw_sp_vr { 470 u16 id; /* virtual router ID */ 471 u32 tb_id; /* kernel fib table id */ 472 unsigned int rif_count; 473 struct mlxsw_sp_fib *fib4; 474 struct mlxsw_sp_fib *fib6; 475 struct mlxsw_sp_mr_table *mr_table[MLXSW_SP_L3_PROTO_MAX]; 476 struct mlxsw_sp_rif *ul_rif; 477 refcount_t ul_rif_refcnt; 478 }; 479 480 static int mlxsw_sp_router_ll_basic_init(struct mlxsw_sp *mlxsw_sp, u16 vr_id, 481 enum mlxsw_sp_l3proto proto) 482 { 483 return 0; 484 } 485 486 static int mlxsw_sp_router_ll_basic_ralta_write(struct mlxsw_sp *mlxsw_sp, char *xralta_pl) 487 { 488 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), 489 xralta_pl + MLXSW_REG_XRALTA_RALTA_OFFSET); 490 } 491 492 static int mlxsw_sp_router_ll_basic_ralst_write(struct mlxsw_sp *mlxsw_sp, char *xralst_pl) 493 { 494 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralst), 495 xralst_pl + MLXSW_REG_XRALST_RALST_OFFSET); 496 } 497 498 static int mlxsw_sp_router_ll_basic_raltb_write(struct mlxsw_sp *mlxsw_sp, char *xraltb_pl) 499 { 500 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), 501 xraltb_pl + MLXSW_REG_XRALTB_RALTB_OFFSET); 502 } 503 504 static const struct rhashtable_params mlxsw_sp_fib_ht_params; 505 506 static struct mlxsw_sp_fib *mlxsw_sp_fib_create(struct mlxsw_sp *mlxsw_sp, 507 struct mlxsw_sp_vr *vr, 508 enum mlxsw_sp_l3proto proto) 509 { 510 const struct mlxsw_sp_router_ll_ops *ll_ops = mlxsw_sp->router->proto_ll_ops[proto]; 511 struct mlxsw_sp_lpm_tree *lpm_tree; 512 struct mlxsw_sp_fib *fib; 513 int err; 514 515 err = ll_ops->init(mlxsw_sp, vr->id, proto); 516 if (err) 517 return ERR_PTR(err); 518 519 lpm_tree = mlxsw_sp->router->lpm.proto_trees[proto]; 520 fib = kzalloc(sizeof(*fib), GFP_KERNEL); 521 if (!fib) 522 return ERR_PTR(-ENOMEM); 523 err = rhashtable_init(&fib->ht, &mlxsw_sp_fib_ht_params); 524 if (err) 525 goto err_rhashtable_init; 526 INIT_LIST_HEAD(&fib->node_list); 527 fib->proto = proto; 528 fib->vr = vr; 529 fib->lpm_tree = lpm_tree; 530 fib->ll_ops = ll_ops; 531 mlxsw_sp_lpm_tree_hold(lpm_tree); 532 err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib, lpm_tree->id); 533 if (err) 534 goto err_lpm_tree_bind; 535 return fib; 536 537 err_lpm_tree_bind: 538 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree); 539 err_rhashtable_init: 540 kfree(fib); 541 return ERR_PTR(err); 542 } 543 544 static void mlxsw_sp_fib_destroy(struct mlxsw_sp *mlxsw_sp, 545 struct mlxsw_sp_fib *fib) 546 { 547 mlxsw_sp_vr_lpm_tree_unbind(mlxsw_sp, fib); 548 mlxsw_sp_lpm_tree_put(mlxsw_sp, fib->lpm_tree); 549 WARN_ON(!list_empty(&fib->node_list)); 550 rhashtable_destroy(&fib->ht); 551 kfree(fib); 552 } 553 554 static struct mlxsw_sp_lpm_tree * 555 mlxsw_sp_lpm_tree_find_unused(struct mlxsw_sp *mlxsw_sp) 556 { 557 static struct mlxsw_sp_lpm_tree *lpm_tree; 558 int i; 559 560 for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) { 561 lpm_tree = &mlxsw_sp->router->lpm.trees[i]; 562 if (lpm_tree->ref_count == 0) 563 return lpm_tree; 564 } 565 return NULL; 566 } 567 568 static int mlxsw_sp_lpm_tree_alloc(struct mlxsw_sp *mlxsw_sp, 569 const struct mlxsw_sp_router_ll_ops *ll_ops, 570 struct mlxsw_sp_lpm_tree *lpm_tree) 571 { 572 char xralta_pl[MLXSW_REG_XRALTA_LEN]; 573 574 mlxsw_reg_xralta_pack(xralta_pl, true, 575 (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto, 576 lpm_tree->id); 577 return ll_ops->ralta_write(mlxsw_sp, xralta_pl); 578 } 579 580 static void mlxsw_sp_lpm_tree_free(struct mlxsw_sp *mlxsw_sp, 581 const struct mlxsw_sp_router_ll_ops *ll_ops, 582 struct mlxsw_sp_lpm_tree *lpm_tree) 583 { 584 char xralta_pl[MLXSW_REG_XRALTA_LEN]; 585 586 mlxsw_reg_xralta_pack(xralta_pl, false, 587 (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto, 588 lpm_tree->id); 589 ll_ops->ralta_write(mlxsw_sp, xralta_pl); 590 } 591 592 static int 593 mlxsw_sp_lpm_tree_left_struct_set(struct mlxsw_sp *mlxsw_sp, 594 const struct mlxsw_sp_router_ll_ops *ll_ops, 595 struct mlxsw_sp_prefix_usage *prefix_usage, 596 struct mlxsw_sp_lpm_tree *lpm_tree) 597 { 598 char xralst_pl[MLXSW_REG_XRALST_LEN]; 599 u8 root_bin = 0; 600 u8 prefix; 601 u8 last_prefix = MLXSW_REG_RALST_BIN_NO_CHILD; 602 603 mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) 604 root_bin = prefix; 605 606 mlxsw_reg_xralst_pack(xralst_pl, root_bin, lpm_tree->id); 607 mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) { 608 if (prefix == 0) 609 continue; 610 mlxsw_reg_xralst_bin_pack(xralst_pl, prefix, last_prefix, 611 MLXSW_REG_RALST_BIN_NO_CHILD); 612 last_prefix = prefix; 613 } 614 return ll_ops->ralst_write(mlxsw_sp, xralst_pl); 615 } 616 617 static struct mlxsw_sp_lpm_tree * 618 mlxsw_sp_lpm_tree_create(struct mlxsw_sp *mlxsw_sp, 619 const struct mlxsw_sp_router_ll_ops *ll_ops, 620 struct mlxsw_sp_prefix_usage *prefix_usage, 621 enum mlxsw_sp_l3proto proto) 622 { 623 struct mlxsw_sp_lpm_tree *lpm_tree; 624 int err; 625 626 lpm_tree = mlxsw_sp_lpm_tree_find_unused(mlxsw_sp); 627 if (!lpm_tree) 628 return ERR_PTR(-EBUSY); 629 lpm_tree->proto = proto; 630 err = mlxsw_sp_lpm_tree_alloc(mlxsw_sp, ll_ops, lpm_tree); 631 if (err) 632 return ERR_PTR(err); 633 634 err = mlxsw_sp_lpm_tree_left_struct_set(mlxsw_sp, ll_ops, prefix_usage, lpm_tree); 635 if (err) 636 goto err_left_struct_set; 637 memcpy(&lpm_tree->prefix_usage, prefix_usage, 638 sizeof(lpm_tree->prefix_usage)); 639 memset(&lpm_tree->prefix_ref_count, 0, 640 sizeof(lpm_tree->prefix_ref_count)); 641 lpm_tree->ref_count = 1; 642 return lpm_tree; 643 644 err_left_struct_set: 645 mlxsw_sp_lpm_tree_free(mlxsw_sp, ll_ops, lpm_tree); 646 return ERR_PTR(err); 647 } 648 649 static void mlxsw_sp_lpm_tree_destroy(struct mlxsw_sp *mlxsw_sp, 650 const struct mlxsw_sp_router_ll_ops *ll_ops, 651 struct mlxsw_sp_lpm_tree *lpm_tree) 652 { 653 mlxsw_sp_lpm_tree_free(mlxsw_sp, ll_ops, lpm_tree); 654 } 655 656 static struct mlxsw_sp_lpm_tree * 657 mlxsw_sp_lpm_tree_get(struct mlxsw_sp *mlxsw_sp, 658 struct mlxsw_sp_prefix_usage *prefix_usage, 659 enum mlxsw_sp_l3proto proto) 660 { 661 const struct mlxsw_sp_router_ll_ops *ll_ops = mlxsw_sp->router->proto_ll_ops[proto]; 662 struct mlxsw_sp_lpm_tree *lpm_tree; 663 int i; 664 665 for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) { 666 lpm_tree = &mlxsw_sp->router->lpm.trees[i]; 667 if (lpm_tree->ref_count != 0 && 668 lpm_tree->proto == proto && 669 mlxsw_sp_prefix_usage_eq(&lpm_tree->prefix_usage, 670 prefix_usage)) { 671 mlxsw_sp_lpm_tree_hold(lpm_tree); 672 return lpm_tree; 673 } 674 } 675 return mlxsw_sp_lpm_tree_create(mlxsw_sp, ll_ops, prefix_usage, proto); 676 } 677 678 static void mlxsw_sp_lpm_tree_hold(struct mlxsw_sp_lpm_tree *lpm_tree) 679 { 680 lpm_tree->ref_count++; 681 } 682 683 static void mlxsw_sp_lpm_tree_put(struct mlxsw_sp *mlxsw_sp, 684 struct mlxsw_sp_lpm_tree *lpm_tree) 685 { 686 const struct mlxsw_sp_router_ll_ops *ll_ops = 687 mlxsw_sp->router->proto_ll_ops[lpm_tree->proto]; 688 689 if (--lpm_tree->ref_count == 0) 690 mlxsw_sp_lpm_tree_destroy(mlxsw_sp, ll_ops, lpm_tree); 691 } 692 693 #define MLXSW_SP_LPM_TREE_MIN 1 /* tree 0 is reserved */ 694 695 static int mlxsw_sp_lpm_init(struct mlxsw_sp *mlxsw_sp) 696 { 697 struct mlxsw_sp_prefix_usage req_prefix_usage = {{ 0 } }; 698 struct mlxsw_sp_lpm_tree *lpm_tree; 699 u64 max_trees; 700 int err, i; 701 702 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_LPM_TREES)) 703 return -EIO; 704 705 max_trees = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_LPM_TREES); 706 mlxsw_sp->router->lpm.tree_count = max_trees - MLXSW_SP_LPM_TREE_MIN; 707 mlxsw_sp->router->lpm.trees = kcalloc(mlxsw_sp->router->lpm.tree_count, 708 sizeof(struct mlxsw_sp_lpm_tree), 709 GFP_KERNEL); 710 if (!mlxsw_sp->router->lpm.trees) 711 return -ENOMEM; 712 713 for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) { 714 lpm_tree = &mlxsw_sp->router->lpm.trees[i]; 715 lpm_tree->id = i + MLXSW_SP_LPM_TREE_MIN; 716 } 717 718 lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage, 719 MLXSW_SP_L3_PROTO_IPV4); 720 if (IS_ERR(lpm_tree)) { 721 err = PTR_ERR(lpm_tree); 722 goto err_ipv4_tree_get; 723 } 724 mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV4] = lpm_tree; 725 726 lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage, 727 MLXSW_SP_L3_PROTO_IPV6); 728 if (IS_ERR(lpm_tree)) { 729 err = PTR_ERR(lpm_tree); 730 goto err_ipv6_tree_get; 731 } 732 mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV6] = lpm_tree; 733 734 return 0; 735 736 err_ipv6_tree_get: 737 lpm_tree = mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV4]; 738 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree); 739 err_ipv4_tree_get: 740 kfree(mlxsw_sp->router->lpm.trees); 741 return err; 742 } 743 744 static void mlxsw_sp_lpm_fini(struct mlxsw_sp *mlxsw_sp) 745 { 746 struct mlxsw_sp_lpm_tree *lpm_tree; 747 748 lpm_tree = mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV6]; 749 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree); 750 751 lpm_tree = mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV4]; 752 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree); 753 754 kfree(mlxsw_sp->router->lpm.trees); 755 } 756 757 static bool mlxsw_sp_vr_is_used(const struct mlxsw_sp_vr *vr) 758 { 759 return !!vr->fib4 || !!vr->fib6 || 760 !!vr->mr_table[MLXSW_SP_L3_PROTO_IPV4] || 761 !!vr->mr_table[MLXSW_SP_L3_PROTO_IPV6]; 762 } 763 764 static struct mlxsw_sp_vr *mlxsw_sp_vr_find_unused(struct mlxsw_sp *mlxsw_sp) 765 { 766 struct mlxsw_sp_vr *vr; 767 int i; 768 769 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) { 770 vr = &mlxsw_sp->router->vrs[i]; 771 if (!mlxsw_sp_vr_is_used(vr)) 772 return vr; 773 } 774 return NULL; 775 } 776 777 static int mlxsw_sp_vr_lpm_tree_bind(struct mlxsw_sp *mlxsw_sp, 778 const struct mlxsw_sp_fib *fib, u8 tree_id) 779 { 780 char xraltb_pl[MLXSW_REG_XRALTB_LEN]; 781 782 mlxsw_reg_xraltb_pack(xraltb_pl, fib->vr->id, 783 (enum mlxsw_reg_ralxx_protocol) fib->proto, 784 tree_id); 785 return fib->ll_ops->raltb_write(mlxsw_sp, xraltb_pl); 786 } 787 788 static int mlxsw_sp_vr_lpm_tree_unbind(struct mlxsw_sp *mlxsw_sp, 789 const struct mlxsw_sp_fib *fib) 790 { 791 char xraltb_pl[MLXSW_REG_XRALTB_LEN]; 792 793 /* Bind to tree 0 which is default */ 794 mlxsw_reg_xraltb_pack(xraltb_pl, fib->vr->id, 795 (enum mlxsw_reg_ralxx_protocol) fib->proto, 0); 796 return fib->ll_ops->raltb_write(mlxsw_sp, xraltb_pl); 797 } 798 799 static u32 mlxsw_sp_fix_tb_id(u32 tb_id) 800 { 801 /* For our purpose, squash main, default and local tables into one */ 802 if (tb_id == RT_TABLE_LOCAL || tb_id == RT_TABLE_DEFAULT) 803 tb_id = RT_TABLE_MAIN; 804 return tb_id; 805 } 806 807 static struct mlxsw_sp_vr *mlxsw_sp_vr_find(struct mlxsw_sp *mlxsw_sp, 808 u32 tb_id) 809 { 810 struct mlxsw_sp_vr *vr; 811 int i; 812 813 tb_id = mlxsw_sp_fix_tb_id(tb_id); 814 815 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) { 816 vr = &mlxsw_sp->router->vrs[i]; 817 if (mlxsw_sp_vr_is_used(vr) && vr->tb_id == tb_id) 818 return vr; 819 } 820 return NULL; 821 } 822 823 int mlxsw_sp_router_tb_id_vr_id(struct mlxsw_sp *mlxsw_sp, u32 tb_id, 824 u16 *vr_id) 825 { 826 struct mlxsw_sp_vr *vr; 827 int err = 0; 828 829 mutex_lock(&mlxsw_sp->router->lock); 830 vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id); 831 if (!vr) { 832 err = -ESRCH; 833 goto out; 834 } 835 *vr_id = vr->id; 836 out: 837 mutex_unlock(&mlxsw_sp->router->lock); 838 return err; 839 } 840 841 static struct mlxsw_sp_fib *mlxsw_sp_vr_fib(const struct mlxsw_sp_vr *vr, 842 enum mlxsw_sp_l3proto proto) 843 { 844 switch (proto) { 845 case MLXSW_SP_L3_PROTO_IPV4: 846 return vr->fib4; 847 case MLXSW_SP_L3_PROTO_IPV6: 848 return vr->fib6; 849 } 850 return NULL; 851 } 852 853 static struct mlxsw_sp_vr *mlxsw_sp_vr_create(struct mlxsw_sp *mlxsw_sp, 854 u32 tb_id, 855 struct netlink_ext_ack *extack) 856 { 857 struct mlxsw_sp_mr_table *mr4_table, *mr6_table; 858 struct mlxsw_sp_fib *fib4; 859 struct mlxsw_sp_fib *fib6; 860 struct mlxsw_sp_vr *vr; 861 int err; 862 863 vr = mlxsw_sp_vr_find_unused(mlxsw_sp); 864 if (!vr) { 865 NL_SET_ERR_MSG_MOD(extack, "Exceeded number of supported virtual routers"); 866 return ERR_PTR(-EBUSY); 867 } 868 fib4 = mlxsw_sp_fib_create(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV4); 869 if (IS_ERR(fib4)) 870 return ERR_CAST(fib4); 871 fib6 = mlxsw_sp_fib_create(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV6); 872 if (IS_ERR(fib6)) { 873 err = PTR_ERR(fib6); 874 goto err_fib6_create; 875 } 876 mr4_table = mlxsw_sp_mr_table_create(mlxsw_sp, vr->id, 877 MLXSW_SP_L3_PROTO_IPV4); 878 if (IS_ERR(mr4_table)) { 879 err = PTR_ERR(mr4_table); 880 goto err_mr4_table_create; 881 } 882 mr6_table = mlxsw_sp_mr_table_create(mlxsw_sp, vr->id, 883 MLXSW_SP_L3_PROTO_IPV6); 884 if (IS_ERR(mr6_table)) { 885 err = PTR_ERR(mr6_table); 886 goto err_mr6_table_create; 887 } 888 889 vr->fib4 = fib4; 890 vr->fib6 = fib6; 891 vr->mr_table[MLXSW_SP_L3_PROTO_IPV4] = mr4_table; 892 vr->mr_table[MLXSW_SP_L3_PROTO_IPV6] = mr6_table; 893 vr->tb_id = tb_id; 894 return vr; 895 896 err_mr6_table_create: 897 mlxsw_sp_mr_table_destroy(mr4_table); 898 err_mr4_table_create: 899 mlxsw_sp_fib_destroy(mlxsw_sp, fib6); 900 err_fib6_create: 901 mlxsw_sp_fib_destroy(mlxsw_sp, fib4); 902 return ERR_PTR(err); 903 } 904 905 static void mlxsw_sp_vr_destroy(struct mlxsw_sp *mlxsw_sp, 906 struct mlxsw_sp_vr *vr) 907 { 908 mlxsw_sp_mr_table_destroy(vr->mr_table[MLXSW_SP_L3_PROTO_IPV6]); 909 vr->mr_table[MLXSW_SP_L3_PROTO_IPV6] = NULL; 910 mlxsw_sp_mr_table_destroy(vr->mr_table[MLXSW_SP_L3_PROTO_IPV4]); 911 vr->mr_table[MLXSW_SP_L3_PROTO_IPV4] = NULL; 912 mlxsw_sp_fib_destroy(mlxsw_sp, vr->fib6); 913 vr->fib6 = NULL; 914 mlxsw_sp_fib_destroy(mlxsw_sp, vr->fib4); 915 vr->fib4 = NULL; 916 } 917 918 static struct mlxsw_sp_vr *mlxsw_sp_vr_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id, 919 struct netlink_ext_ack *extack) 920 { 921 struct mlxsw_sp_vr *vr; 922 923 tb_id = mlxsw_sp_fix_tb_id(tb_id); 924 vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id); 925 if (!vr) 926 vr = mlxsw_sp_vr_create(mlxsw_sp, tb_id, extack); 927 return vr; 928 } 929 930 static void mlxsw_sp_vr_put(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_vr *vr) 931 { 932 if (!vr->rif_count && list_empty(&vr->fib4->node_list) && 933 list_empty(&vr->fib6->node_list) && 934 mlxsw_sp_mr_table_empty(vr->mr_table[MLXSW_SP_L3_PROTO_IPV4]) && 935 mlxsw_sp_mr_table_empty(vr->mr_table[MLXSW_SP_L3_PROTO_IPV6])) 936 mlxsw_sp_vr_destroy(mlxsw_sp, vr); 937 } 938 939 static bool 940 mlxsw_sp_vr_lpm_tree_should_replace(struct mlxsw_sp_vr *vr, 941 enum mlxsw_sp_l3proto proto, u8 tree_id) 942 { 943 struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto); 944 945 if (!mlxsw_sp_vr_is_used(vr)) 946 return false; 947 if (fib->lpm_tree->id == tree_id) 948 return true; 949 return false; 950 } 951 952 static int mlxsw_sp_vr_lpm_tree_replace(struct mlxsw_sp *mlxsw_sp, 953 struct mlxsw_sp_fib *fib, 954 struct mlxsw_sp_lpm_tree *new_tree) 955 { 956 struct mlxsw_sp_lpm_tree *old_tree = fib->lpm_tree; 957 int err; 958 959 fib->lpm_tree = new_tree; 960 mlxsw_sp_lpm_tree_hold(new_tree); 961 err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib, new_tree->id); 962 if (err) 963 goto err_tree_bind; 964 mlxsw_sp_lpm_tree_put(mlxsw_sp, old_tree); 965 return 0; 966 967 err_tree_bind: 968 mlxsw_sp_lpm_tree_put(mlxsw_sp, new_tree); 969 fib->lpm_tree = old_tree; 970 return err; 971 } 972 973 static int mlxsw_sp_vrs_lpm_tree_replace(struct mlxsw_sp *mlxsw_sp, 974 struct mlxsw_sp_fib *fib, 975 struct mlxsw_sp_lpm_tree *new_tree) 976 { 977 enum mlxsw_sp_l3proto proto = fib->proto; 978 struct mlxsw_sp_lpm_tree *old_tree; 979 u8 old_id, new_id = new_tree->id; 980 struct mlxsw_sp_vr *vr; 981 int i, err; 982 983 old_tree = mlxsw_sp->router->lpm.proto_trees[proto]; 984 old_id = old_tree->id; 985 986 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) { 987 vr = &mlxsw_sp->router->vrs[i]; 988 if (!mlxsw_sp_vr_lpm_tree_should_replace(vr, proto, old_id)) 989 continue; 990 err = mlxsw_sp_vr_lpm_tree_replace(mlxsw_sp, 991 mlxsw_sp_vr_fib(vr, proto), 992 new_tree); 993 if (err) 994 goto err_tree_replace; 995 } 996 997 memcpy(new_tree->prefix_ref_count, old_tree->prefix_ref_count, 998 sizeof(new_tree->prefix_ref_count)); 999 mlxsw_sp->router->lpm.proto_trees[proto] = new_tree; 1000 mlxsw_sp_lpm_tree_put(mlxsw_sp, old_tree); 1001 1002 return 0; 1003 1004 err_tree_replace: 1005 for (i--; i >= 0; i--) { 1006 if (!mlxsw_sp_vr_lpm_tree_should_replace(vr, proto, new_id)) 1007 continue; 1008 mlxsw_sp_vr_lpm_tree_replace(mlxsw_sp, 1009 mlxsw_sp_vr_fib(vr, proto), 1010 old_tree); 1011 } 1012 return err; 1013 } 1014 1015 static int mlxsw_sp_vrs_init(struct mlxsw_sp *mlxsw_sp) 1016 { 1017 struct mlxsw_sp_vr *vr; 1018 u64 max_vrs; 1019 int i; 1020 1021 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_VRS)) 1022 return -EIO; 1023 1024 max_vrs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); 1025 mlxsw_sp->router->vrs = kcalloc(max_vrs, sizeof(struct mlxsw_sp_vr), 1026 GFP_KERNEL); 1027 if (!mlxsw_sp->router->vrs) 1028 return -ENOMEM; 1029 1030 for (i = 0; i < max_vrs; i++) { 1031 vr = &mlxsw_sp->router->vrs[i]; 1032 vr->id = i; 1033 } 1034 1035 return 0; 1036 } 1037 1038 static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp); 1039 1040 static void mlxsw_sp_vrs_fini(struct mlxsw_sp *mlxsw_sp) 1041 { 1042 /* At this stage we're guaranteed not to have new incoming 1043 * FIB notifications and the work queue is free from FIBs 1044 * sitting on top of mlxsw netdevs. However, we can still 1045 * have other FIBs queued. Flush the queue before flushing 1046 * the device's tables. No need for locks, as we're the only 1047 * writer. 1048 */ 1049 mlxsw_core_flush_owq(); 1050 mlxsw_sp_router_fib_flush(mlxsw_sp); 1051 kfree(mlxsw_sp->router->vrs); 1052 } 1053 1054 static struct net_device * 1055 __mlxsw_sp_ipip_netdev_ul_dev_get(const struct net_device *ol_dev) 1056 { 1057 struct ip_tunnel *tun = netdev_priv(ol_dev); 1058 struct net *net = dev_net(ol_dev); 1059 1060 return dev_get_by_index_rcu(net, tun->parms.link); 1061 } 1062 1063 u32 mlxsw_sp_ipip_dev_ul_tb_id(const struct net_device *ol_dev) 1064 { 1065 struct net_device *d; 1066 u32 tb_id; 1067 1068 rcu_read_lock(); 1069 d = __mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev); 1070 if (d) 1071 tb_id = l3mdev_fib_table(d) ? : RT_TABLE_MAIN; 1072 else 1073 tb_id = RT_TABLE_MAIN; 1074 rcu_read_unlock(); 1075 1076 return tb_id; 1077 } 1078 1079 static struct mlxsw_sp_rif * 1080 mlxsw_sp_rif_create(struct mlxsw_sp *mlxsw_sp, 1081 const struct mlxsw_sp_rif_params *params, 1082 struct netlink_ext_ack *extack); 1083 1084 static struct mlxsw_sp_rif_ipip_lb * 1085 mlxsw_sp_ipip_ol_ipip_lb_create(struct mlxsw_sp *mlxsw_sp, 1086 enum mlxsw_sp_ipip_type ipipt, 1087 struct net_device *ol_dev, 1088 struct netlink_ext_ack *extack) 1089 { 1090 struct mlxsw_sp_rif_params_ipip_lb lb_params; 1091 const struct mlxsw_sp_ipip_ops *ipip_ops; 1092 struct mlxsw_sp_rif *rif; 1093 1094 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipipt]; 1095 lb_params = (struct mlxsw_sp_rif_params_ipip_lb) { 1096 .common.dev = ol_dev, 1097 .common.lag = false, 1098 .lb_config = ipip_ops->ol_loopback_config(mlxsw_sp, ol_dev), 1099 }; 1100 1101 rif = mlxsw_sp_rif_create(mlxsw_sp, &lb_params.common, extack); 1102 if (IS_ERR(rif)) 1103 return ERR_CAST(rif); 1104 return container_of(rif, struct mlxsw_sp_rif_ipip_lb, common); 1105 } 1106 1107 static struct mlxsw_sp_ipip_entry * 1108 mlxsw_sp_ipip_entry_alloc(struct mlxsw_sp *mlxsw_sp, 1109 enum mlxsw_sp_ipip_type ipipt, 1110 struct net_device *ol_dev) 1111 { 1112 const struct mlxsw_sp_ipip_ops *ipip_ops; 1113 struct mlxsw_sp_ipip_entry *ipip_entry; 1114 struct mlxsw_sp_ipip_entry *ret = NULL; 1115 1116 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipipt]; 1117 ipip_entry = kzalloc(sizeof(*ipip_entry), GFP_KERNEL); 1118 if (!ipip_entry) 1119 return ERR_PTR(-ENOMEM); 1120 1121 ipip_entry->ol_lb = mlxsw_sp_ipip_ol_ipip_lb_create(mlxsw_sp, ipipt, 1122 ol_dev, NULL); 1123 if (IS_ERR(ipip_entry->ol_lb)) { 1124 ret = ERR_CAST(ipip_entry->ol_lb); 1125 goto err_ol_ipip_lb_create; 1126 } 1127 1128 ipip_entry->ipipt = ipipt; 1129 ipip_entry->ol_dev = ol_dev; 1130 1131 switch (ipip_ops->ul_proto) { 1132 case MLXSW_SP_L3_PROTO_IPV4: 1133 ipip_entry->parms4 = mlxsw_sp_ipip_netdev_parms4(ol_dev); 1134 break; 1135 case MLXSW_SP_L3_PROTO_IPV6: 1136 WARN_ON(1); 1137 break; 1138 } 1139 1140 return ipip_entry; 1141 1142 err_ol_ipip_lb_create: 1143 kfree(ipip_entry); 1144 return ret; 1145 } 1146 1147 static void 1148 mlxsw_sp_ipip_entry_dealloc(struct mlxsw_sp_ipip_entry *ipip_entry) 1149 { 1150 mlxsw_sp_rif_destroy(&ipip_entry->ol_lb->common); 1151 kfree(ipip_entry); 1152 } 1153 1154 static bool 1155 mlxsw_sp_ipip_entry_saddr_matches(struct mlxsw_sp *mlxsw_sp, 1156 const enum mlxsw_sp_l3proto ul_proto, 1157 union mlxsw_sp_l3addr saddr, 1158 u32 ul_tb_id, 1159 struct mlxsw_sp_ipip_entry *ipip_entry) 1160 { 1161 u32 tun_ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ipip_entry->ol_dev); 1162 enum mlxsw_sp_ipip_type ipipt = ipip_entry->ipipt; 1163 union mlxsw_sp_l3addr tun_saddr; 1164 1165 if (mlxsw_sp->router->ipip_ops_arr[ipipt]->ul_proto != ul_proto) 1166 return false; 1167 1168 tun_saddr = mlxsw_sp_ipip_netdev_saddr(ul_proto, ipip_entry->ol_dev); 1169 return tun_ul_tb_id == ul_tb_id && 1170 mlxsw_sp_l3addr_eq(&tun_saddr, &saddr); 1171 } 1172 1173 static int 1174 mlxsw_sp_fib_entry_decap_init(struct mlxsw_sp *mlxsw_sp, 1175 struct mlxsw_sp_fib_entry *fib_entry, 1176 struct mlxsw_sp_ipip_entry *ipip_entry) 1177 { 1178 u32 tunnel_index; 1179 int err; 1180 1181 err = mlxsw_sp_kvdl_alloc(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1182 1, &tunnel_index); 1183 if (err) 1184 return err; 1185 1186 ipip_entry->decap_fib_entry = fib_entry; 1187 fib_entry->decap.ipip_entry = ipip_entry; 1188 fib_entry->decap.tunnel_index = tunnel_index; 1189 return 0; 1190 } 1191 1192 static void mlxsw_sp_fib_entry_decap_fini(struct mlxsw_sp *mlxsw_sp, 1193 struct mlxsw_sp_fib_entry *fib_entry) 1194 { 1195 /* Unlink this node from the IPIP entry that it's the decap entry of. */ 1196 fib_entry->decap.ipip_entry->decap_fib_entry = NULL; 1197 fib_entry->decap.ipip_entry = NULL; 1198 mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1199 1, fib_entry->decap.tunnel_index); 1200 } 1201 1202 static struct mlxsw_sp_fib_node * 1203 mlxsw_sp_fib_node_lookup(struct mlxsw_sp_fib *fib, const void *addr, 1204 size_t addr_len, unsigned char prefix_len); 1205 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp, 1206 struct mlxsw_sp_fib_entry *fib_entry); 1207 1208 static void 1209 mlxsw_sp_ipip_entry_demote_decap(struct mlxsw_sp *mlxsw_sp, 1210 struct mlxsw_sp_ipip_entry *ipip_entry) 1211 { 1212 struct mlxsw_sp_fib_entry *fib_entry = ipip_entry->decap_fib_entry; 1213 1214 mlxsw_sp_fib_entry_decap_fini(mlxsw_sp, fib_entry); 1215 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP; 1216 1217 mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry); 1218 } 1219 1220 static void 1221 mlxsw_sp_ipip_entry_promote_decap(struct mlxsw_sp *mlxsw_sp, 1222 struct mlxsw_sp_ipip_entry *ipip_entry, 1223 struct mlxsw_sp_fib_entry *decap_fib_entry) 1224 { 1225 if (mlxsw_sp_fib_entry_decap_init(mlxsw_sp, decap_fib_entry, 1226 ipip_entry)) 1227 return; 1228 decap_fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP; 1229 1230 if (mlxsw_sp_fib_entry_update(mlxsw_sp, decap_fib_entry)) 1231 mlxsw_sp_ipip_entry_demote_decap(mlxsw_sp, ipip_entry); 1232 } 1233 1234 static struct mlxsw_sp_fib_entry * 1235 mlxsw_sp_router_ip2me_fib_entry_find(struct mlxsw_sp *mlxsw_sp, u32 tb_id, 1236 enum mlxsw_sp_l3proto proto, 1237 const union mlxsw_sp_l3addr *addr, 1238 enum mlxsw_sp_fib_entry_type type) 1239 { 1240 struct mlxsw_sp_fib_node *fib_node; 1241 unsigned char addr_prefix_len; 1242 struct mlxsw_sp_fib *fib; 1243 struct mlxsw_sp_vr *vr; 1244 const void *addrp; 1245 size_t addr_len; 1246 u32 addr4; 1247 1248 vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id); 1249 if (!vr) 1250 return NULL; 1251 fib = mlxsw_sp_vr_fib(vr, proto); 1252 1253 switch (proto) { 1254 case MLXSW_SP_L3_PROTO_IPV4: 1255 addr4 = be32_to_cpu(addr->addr4); 1256 addrp = &addr4; 1257 addr_len = 4; 1258 addr_prefix_len = 32; 1259 break; 1260 case MLXSW_SP_L3_PROTO_IPV6: 1261 default: 1262 WARN_ON(1); 1263 return NULL; 1264 } 1265 1266 fib_node = mlxsw_sp_fib_node_lookup(fib, addrp, addr_len, 1267 addr_prefix_len); 1268 if (!fib_node || fib_node->fib_entry->type != type) 1269 return NULL; 1270 1271 return fib_node->fib_entry; 1272 } 1273 1274 /* Given an IPIP entry, find the corresponding decap route. */ 1275 static struct mlxsw_sp_fib_entry * 1276 mlxsw_sp_ipip_entry_find_decap(struct mlxsw_sp *mlxsw_sp, 1277 struct mlxsw_sp_ipip_entry *ipip_entry) 1278 { 1279 static struct mlxsw_sp_fib_node *fib_node; 1280 const struct mlxsw_sp_ipip_ops *ipip_ops; 1281 unsigned char saddr_prefix_len; 1282 union mlxsw_sp_l3addr saddr; 1283 struct mlxsw_sp_fib *ul_fib; 1284 struct mlxsw_sp_vr *ul_vr; 1285 const void *saddrp; 1286 size_t saddr_len; 1287 u32 ul_tb_id; 1288 u32 saddr4; 1289 1290 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt]; 1291 1292 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ipip_entry->ol_dev); 1293 ul_vr = mlxsw_sp_vr_find(mlxsw_sp, ul_tb_id); 1294 if (!ul_vr) 1295 return NULL; 1296 1297 ul_fib = mlxsw_sp_vr_fib(ul_vr, ipip_ops->ul_proto); 1298 saddr = mlxsw_sp_ipip_netdev_saddr(ipip_ops->ul_proto, 1299 ipip_entry->ol_dev); 1300 1301 switch (ipip_ops->ul_proto) { 1302 case MLXSW_SP_L3_PROTO_IPV4: 1303 saddr4 = be32_to_cpu(saddr.addr4); 1304 saddrp = &saddr4; 1305 saddr_len = 4; 1306 saddr_prefix_len = 32; 1307 break; 1308 default: 1309 WARN_ON(1); 1310 return NULL; 1311 } 1312 1313 fib_node = mlxsw_sp_fib_node_lookup(ul_fib, saddrp, saddr_len, 1314 saddr_prefix_len); 1315 if (!fib_node || 1316 fib_node->fib_entry->type != MLXSW_SP_FIB_ENTRY_TYPE_TRAP) 1317 return NULL; 1318 1319 return fib_node->fib_entry; 1320 } 1321 1322 static struct mlxsw_sp_ipip_entry * 1323 mlxsw_sp_ipip_entry_create(struct mlxsw_sp *mlxsw_sp, 1324 enum mlxsw_sp_ipip_type ipipt, 1325 struct net_device *ol_dev) 1326 { 1327 struct mlxsw_sp_ipip_entry *ipip_entry; 1328 1329 ipip_entry = mlxsw_sp_ipip_entry_alloc(mlxsw_sp, ipipt, ol_dev); 1330 if (IS_ERR(ipip_entry)) 1331 return ipip_entry; 1332 1333 list_add_tail(&ipip_entry->ipip_list_node, 1334 &mlxsw_sp->router->ipip_list); 1335 1336 return ipip_entry; 1337 } 1338 1339 static void 1340 mlxsw_sp_ipip_entry_destroy(struct mlxsw_sp *mlxsw_sp, 1341 struct mlxsw_sp_ipip_entry *ipip_entry) 1342 { 1343 list_del(&ipip_entry->ipip_list_node); 1344 mlxsw_sp_ipip_entry_dealloc(ipip_entry); 1345 } 1346 1347 static bool 1348 mlxsw_sp_ipip_entry_matches_decap(struct mlxsw_sp *mlxsw_sp, 1349 const struct net_device *ul_dev, 1350 enum mlxsw_sp_l3proto ul_proto, 1351 union mlxsw_sp_l3addr ul_dip, 1352 struct mlxsw_sp_ipip_entry *ipip_entry) 1353 { 1354 u32 ul_tb_id = l3mdev_fib_table(ul_dev) ? : RT_TABLE_MAIN; 1355 enum mlxsw_sp_ipip_type ipipt = ipip_entry->ipipt; 1356 1357 if (mlxsw_sp->router->ipip_ops_arr[ipipt]->ul_proto != ul_proto) 1358 return false; 1359 1360 return mlxsw_sp_ipip_entry_saddr_matches(mlxsw_sp, ul_proto, ul_dip, 1361 ul_tb_id, ipip_entry); 1362 } 1363 1364 /* Given decap parameters, find the corresponding IPIP entry. */ 1365 static struct mlxsw_sp_ipip_entry * 1366 mlxsw_sp_ipip_entry_find_by_decap(struct mlxsw_sp *mlxsw_sp, int ul_dev_ifindex, 1367 enum mlxsw_sp_l3proto ul_proto, 1368 union mlxsw_sp_l3addr ul_dip) 1369 { 1370 struct mlxsw_sp_ipip_entry *ipip_entry = NULL; 1371 struct net_device *ul_dev; 1372 1373 rcu_read_lock(); 1374 1375 ul_dev = dev_get_by_index_rcu(mlxsw_sp_net(mlxsw_sp), ul_dev_ifindex); 1376 if (!ul_dev) 1377 goto out_unlock; 1378 1379 list_for_each_entry(ipip_entry, &mlxsw_sp->router->ipip_list, 1380 ipip_list_node) 1381 if (mlxsw_sp_ipip_entry_matches_decap(mlxsw_sp, ul_dev, 1382 ul_proto, ul_dip, 1383 ipip_entry)) 1384 goto out_unlock; 1385 1386 rcu_read_unlock(); 1387 1388 return NULL; 1389 1390 out_unlock: 1391 rcu_read_unlock(); 1392 return ipip_entry; 1393 } 1394 1395 static bool mlxsw_sp_netdev_ipip_type(const struct mlxsw_sp *mlxsw_sp, 1396 const struct net_device *dev, 1397 enum mlxsw_sp_ipip_type *p_type) 1398 { 1399 struct mlxsw_sp_router *router = mlxsw_sp->router; 1400 const struct mlxsw_sp_ipip_ops *ipip_ops; 1401 enum mlxsw_sp_ipip_type ipipt; 1402 1403 for (ipipt = 0; ipipt < MLXSW_SP_IPIP_TYPE_MAX; ++ipipt) { 1404 ipip_ops = router->ipip_ops_arr[ipipt]; 1405 if (dev->type == ipip_ops->dev_type) { 1406 if (p_type) 1407 *p_type = ipipt; 1408 return true; 1409 } 1410 } 1411 return false; 1412 } 1413 1414 bool mlxsw_sp_netdev_is_ipip_ol(const struct mlxsw_sp *mlxsw_sp, 1415 const struct net_device *dev) 1416 { 1417 return mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, NULL); 1418 } 1419 1420 static struct mlxsw_sp_ipip_entry * 1421 mlxsw_sp_ipip_entry_find_by_ol_dev(struct mlxsw_sp *mlxsw_sp, 1422 const struct net_device *ol_dev) 1423 { 1424 struct mlxsw_sp_ipip_entry *ipip_entry; 1425 1426 list_for_each_entry(ipip_entry, &mlxsw_sp->router->ipip_list, 1427 ipip_list_node) 1428 if (ipip_entry->ol_dev == ol_dev) 1429 return ipip_entry; 1430 1431 return NULL; 1432 } 1433 1434 static struct mlxsw_sp_ipip_entry * 1435 mlxsw_sp_ipip_entry_find_by_ul_dev(const struct mlxsw_sp *mlxsw_sp, 1436 const struct net_device *ul_dev, 1437 struct mlxsw_sp_ipip_entry *start) 1438 { 1439 struct mlxsw_sp_ipip_entry *ipip_entry; 1440 1441 ipip_entry = list_prepare_entry(start, &mlxsw_sp->router->ipip_list, 1442 ipip_list_node); 1443 list_for_each_entry_continue(ipip_entry, &mlxsw_sp->router->ipip_list, 1444 ipip_list_node) { 1445 struct net_device *ol_dev = ipip_entry->ol_dev; 1446 struct net_device *ipip_ul_dev; 1447 1448 rcu_read_lock(); 1449 ipip_ul_dev = __mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev); 1450 rcu_read_unlock(); 1451 1452 if (ipip_ul_dev == ul_dev) 1453 return ipip_entry; 1454 } 1455 1456 return NULL; 1457 } 1458 1459 bool mlxsw_sp_netdev_is_ipip_ul(struct mlxsw_sp *mlxsw_sp, 1460 const struct net_device *dev) 1461 { 1462 bool is_ipip_ul; 1463 1464 mutex_lock(&mlxsw_sp->router->lock); 1465 is_ipip_ul = mlxsw_sp_ipip_entry_find_by_ul_dev(mlxsw_sp, dev, NULL); 1466 mutex_unlock(&mlxsw_sp->router->lock); 1467 1468 return is_ipip_ul; 1469 } 1470 1471 static bool mlxsw_sp_netdevice_ipip_can_offload(struct mlxsw_sp *mlxsw_sp, 1472 const struct net_device *ol_dev, 1473 enum mlxsw_sp_ipip_type ipipt) 1474 { 1475 const struct mlxsw_sp_ipip_ops *ops 1476 = mlxsw_sp->router->ipip_ops_arr[ipipt]; 1477 1478 return ops->can_offload(mlxsw_sp, ol_dev); 1479 } 1480 1481 static int mlxsw_sp_netdevice_ipip_ol_reg_event(struct mlxsw_sp *mlxsw_sp, 1482 struct net_device *ol_dev) 1483 { 1484 enum mlxsw_sp_ipip_type ipipt = MLXSW_SP_IPIP_TYPE_MAX; 1485 struct mlxsw_sp_ipip_entry *ipip_entry; 1486 enum mlxsw_sp_l3proto ul_proto; 1487 union mlxsw_sp_l3addr saddr; 1488 u32 ul_tb_id; 1489 1490 mlxsw_sp_netdev_ipip_type(mlxsw_sp, ol_dev, &ipipt); 1491 if (mlxsw_sp_netdevice_ipip_can_offload(mlxsw_sp, ol_dev, ipipt)) { 1492 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ol_dev); 1493 ul_proto = mlxsw_sp->router->ipip_ops_arr[ipipt]->ul_proto; 1494 saddr = mlxsw_sp_ipip_netdev_saddr(ul_proto, ol_dev); 1495 if (!mlxsw_sp_ipip_demote_tunnel_by_saddr(mlxsw_sp, ul_proto, 1496 saddr, ul_tb_id, 1497 NULL)) { 1498 ipip_entry = mlxsw_sp_ipip_entry_create(mlxsw_sp, ipipt, 1499 ol_dev); 1500 if (IS_ERR(ipip_entry)) 1501 return PTR_ERR(ipip_entry); 1502 } 1503 } 1504 1505 return 0; 1506 } 1507 1508 static void mlxsw_sp_netdevice_ipip_ol_unreg_event(struct mlxsw_sp *mlxsw_sp, 1509 struct net_device *ol_dev) 1510 { 1511 struct mlxsw_sp_ipip_entry *ipip_entry; 1512 1513 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev); 1514 if (ipip_entry) 1515 mlxsw_sp_ipip_entry_destroy(mlxsw_sp, ipip_entry); 1516 } 1517 1518 static void 1519 mlxsw_sp_ipip_entry_ol_up_event(struct mlxsw_sp *mlxsw_sp, 1520 struct mlxsw_sp_ipip_entry *ipip_entry) 1521 { 1522 struct mlxsw_sp_fib_entry *decap_fib_entry; 1523 1524 decap_fib_entry = mlxsw_sp_ipip_entry_find_decap(mlxsw_sp, ipip_entry); 1525 if (decap_fib_entry) 1526 mlxsw_sp_ipip_entry_promote_decap(mlxsw_sp, ipip_entry, 1527 decap_fib_entry); 1528 } 1529 1530 static int 1531 mlxsw_sp_rif_ipip_lb_op(struct mlxsw_sp_rif_ipip_lb *lb_rif, u16 ul_vr_id, 1532 u16 ul_rif_id, bool enable) 1533 { 1534 struct mlxsw_sp_rif_ipip_lb_config lb_cf = lb_rif->lb_config; 1535 struct mlxsw_sp_rif *rif = &lb_rif->common; 1536 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 1537 char ritr_pl[MLXSW_REG_RITR_LEN]; 1538 u32 saddr4; 1539 1540 switch (lb_cf.ul_protocol) { 1541 case MLXSW_SP_L3_PROTO_IPV4: 1542 saddr4 = be32_to_cpu(lb_cf.saddr.addr4); 1543 mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_LOOPBACK_IF, 1544 rif->rif_index, rif->vr_id, rif->dev->mtu); 1545 mlxsw_reg_ritr_loopback_ipip4_pack(ritr_pl, lb_cf.lb_ipipt, 1546 MLXSW_REG_RITR_LOOPBACK_IPIP_OPTIONS_GRE_KEY_PRESET, 1547 ul_vr_id, ul_rif_id, saddr4, lb_cf.okey); 1548 break; 1549 1550 case MLXSW_SP_L3_PROTO_IPV6: 1551 return -EAFNOSUPPORT; 1552 } 1553 1554 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 1555 } 1556 1557 static int mlxsw_sp_netdevice_ipip_ol_update_mtu(struct mlxsw_sp *mlxsw_sp, 1558 struct net_device *ol_dev) 1559 { 1560 struct mlxsw_sp_ipip_entry *ipip_entry; 1561 struct mlxsw_sp_rif_ipip_lb *lb_rif; 1562 int err = 0; 1563 1564 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev); 1565 if (ipip_entry) { 1566 lb_rif = ipip_entry->ol_lb; 1567 err = mlxsw_sp_rif_ipip_lb_op(lb_rif, lb_rif->ul_vr_id, 1568 lb_rif->ul_rif_id, true); 1569 if (err) 1570 goto out; 1571 lb_rif->common.mtu = ol_dev->mtu; 1572 } 1573 1574 out: 1575 return err; 1576 } 1577 1578 static void mlxsw_sp_netdevice_ipip_ol_up_event(struct mlxsw_sp *mlxsw_sp, 1579 struct net_device *ol_dev) 1580 { 1581 struct mlxsw_sp_ipip_entry *ipip_entry; 1582 1583 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev); 1584 if (ipip_entry) 1585 mlxsw_sp_ipip_entry_ol_up_event(mlxsw_sp, ipip_entry); 1586 } 1587 1588 static void 1589 mlxsw_sp_ipip_entry_ol_down_event(struct mlxsw_sp *mlxsw_sp, 1590 struct mlxsw_sp_ipip_entry *ipip_entry) 1591 { 1592 if (ipip_entry->decap_fib_entry) 1593 mlxsw_sp_ipip_entry_demote_decap(mlxsw_sp, ipip_entry); 1594 } 1595 1596 static void mlxsw_sp_netdevice_ipip_ol_down_event(struct mlxsw_sp *mlxsw_sp, 1597 struct net_device *ol_dev) 1598 { 1599 struct mlxsw_sp_ipip_entry *ipip_entry; 1600 1601 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev); 1602 if (ipip_entry) 1603 mlxsw_sp_ipip_entry_ol_down_event(mlxsw_sp, ipip_entry); 1604 } 1605 1606 static void mlxsw_sp_nexthop_rif_migrate(struct mlxsw_sp *mlxsw_sp, 1607 struct mlxsw_sp_rif *old_rif, 1608 struct mlxsw_sp_rif *new_rif); 1609 static int 1610 mlxsw_sp_ipip_entry_ol_lb_update(struct mlxsw_sp *mlxsw_sp, 1611 struct mlxsw_sp_ipip_entry *ipip_entry, 1612 bool keep_encap, 1613 struct netlink_ext_ack *extack) 1614 { 1615 struct mlxsw_sp_rif_ipip_lb *old_lb_rif = ipip_entry->ol_lb; 1616 struct mlxsw_sp_rif_ipip_lb *new_lb_rif; 1617 1618 new_lb_rif = mlxsw_sp_ipip_ol_ipip_lb_create(mlxsw_sp, 1619 ipip_entry->ipipt, 1620 ipip_entry->ol_dev, 1621 extack); 1622 if (IS_ERR(new_lb_rif)) 1623 return PTR_ERR(new_lb_rif); 1624 ipip_entry->ol_lb = new_lb_rif; 1625 1626 if (keep_encap) 1627 mlxsw_sp_nexthop_rif_migrate(mlxsw_sp, &old_lb_rif->common, 1628 &new_lb_rif->common); 1629 1630 mlxsw_sp_rif_destroy(&old_lb_rif->common); 1631 1632 return 0; 1633 } 1634 1635 static void mlxsw_sp_nexthop_rif_update(struct mlxsw_sp *mlxsw_sp, 1636 struct mlxsw_sp_rif *rif); 1637 1638 /** 1639 * __mlxsw_sp_ipip_entry_update_tunnel - Update offload related to IPIP entry. 1640 * @mlxsw_sp: mlxsw_sp. 1641 * @ipip_entry: IPIP entry. 1642 * @recreate_loopback: Recreates the associated loopback RIF. 1643 * @keep_encap: Updates next hops that use the tunnel netdevice. This is only 1644 * relevant when recreate_loopback is true. 1645 * @update_nexthops: Updates next hops, keeping the current loopback RIF. This 1646 * is only relevant when recreate_loopback is false. 1647 * @extack: extack. 1648 * 1649 * Return: Non-zero value on failure. 1650 */ 1651 int __mlxsw_sp_ipip_entry_update_tunnel(struct mlxsw_sp *mlxsw_sp, 1652 struct mlxsw_sp_ipip_entry *ipip_entry, 1653 bool recreate_loopback, 1654 bool keep_encap, 1655 bool update_nexthops, 1656 struct netlink_ext_ack *extack) 1657 { 1658 int err; 1659 1660 /* RIFs can't be edited, so to update loopback, we need to destroy and 1661 * recreate it. That creates a window of opportunity where RALUE and 1662 * RATR registers end up referencing a RIF that's already gone. RATRs 1663 * are handled in mlxsw_sp_ipip_entry_ol_lb_update(), and to take care 1664 * of RALUE, demote the decap route back. 1665 */ 1666 if (ipip_entry->decap_fib_entry) 1667 mlxsw_sp_ipip_entry_demote_decap(mlxsw_sp, ipip_entry); 1668 1669 if (recreate_loopback) { 1670 err = mlxsw_sp_ipip_entry_ol_lb_update(mlxsw_sp, ipip_entry, 1671 keep_encap, extack); 1672 if (err) 1673 return err; 1674 } else if (update_nexthops) { 1675 mlxsw_sp_nexthop_rif_update(mlxsw_sp, 1676 &ipip_entry->ol_lb->common); 1677 } 1678 1679 if (ipip_entry->ol_dev->flags & IFF_UP) 1680 mlxsw_sp_ipip_entry_ol_up_event(mlxsw_sp, ipip_entry); 1681 1682 return 0; 1683 } 1684 1685 static int mlxsw_sp_netdevice_ipip_ol_vrf_event(struct mlxsw_sp *mlxsw_sp, 1686 struct net_device *ol_dev, 1687 struct netlink_ext_ack *extack) 1688 { 1689 struct mlxsw_sp_ipip_entry *ipip_entry = 1690 mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev); 1691 1692 if (!ipip_entry) 1693 return 0; 1694 1695 return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry, 1696 true, false, false, extack); 1697 } 1698 1699 static int 1700 mlxsw_sp_netdevice_ipip_ul_vrf_event(struct mlxsw_sp *mlxsw_sp, 1701 struct mlxsw_sp_ipip_entry *ipip_entry, 1702 struct net_device *ul_dev, 1703 bool *demote_this, 1704 struct netlink_ext_ack *extack) 1705 { 1706 u32 ul_tb_id = l3mdev_fib_table(ul_dev) ? : RT_TABLE_MAIN; 1707 enum mlxsw_sp_l3proto ul_proto; 1708 union mlxsw_sp_l3addr saddr; 1709 1710 /* Moving underlay to a different VRF might cause local address 1711 * conflict, and the conflicting tunnels need to be demoted. 1712 */ 1713 ul_proto = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt]->ul_proto; 1714 saddr = mlxsw_sp_ipip_netdev_saddr(ul_proto, ipip_entry->ol_dev); 1715 if (mlxsw_sp_ipip_demote_tunnel_by_saddr(mlxsw_sp, ul_proto, 1716 saddr, ul_tb_id, 1717 ipip_entry)) { 1718 *demote_this = true; 1719 return 0; 1720 } 1721 1722 return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry, 1723 true, true, false, extack); 1724 } 1725 1726 static int 1727 mlxsw_sp_netdevice_ipip_ul_up_event(struct mlxsw_sp *mlxsw_sp, 1728 struct mlxsw_sp_ipip_entry *ipip_entry, 1729 struct net_device *ul_dev) 1730 { 1731 return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry, 1732 false, false, true, NULL); 1733 } 1734 1735 static int 1736 mlxsw_sp_netdevice_ipip_ul_down_event(struct mlxsw_sp *mlxsw_sp, 1737 struct mlxsw_sp_ipip_entry *ipip_entry, 1738 struct net_device *ul_dev) 1739 { 1740 /* A down underlay device causes encapsulated packets to not be 1741 * forwarded, but decap still works. So refresh next hops without 1742 * touching anything else. 1743 */ 1744 return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry, 1745 false, false, true, NULL); 1746 } 1747 1748 static int 1749 mlxsw_sp_netdevice_ipip_ol_change_event(struct mlxsw_sp *mlxsw_sp, 1750 struct net_device *ol_dev, 1751 struct netlink_ext_ack *extack) 1752 { 1753 const struct mlxsw_sp_ipip_ops *ipip_ops; 1754 struct mlxsw_sp_ipip_entry *ipip_entry; 1755 int err; 1756 1757 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev); 1758 if (!ipip_entry) 1759 /* A change might make a tunnel eligible for offloading, but 1760 * that is currently not implemented. What falls to slow path 1761 * stays there. 1762 */ 1763 return 0; 1764 1765 /* A change might make a tunnel not eligible for offloading. */ 1766 if (!mlxsw_sp_netdevice_ipip_can_offload(mlxsw_sp, ol_dev, 1767 ipip_entry->ipipt)) { 1768 mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry); 1769 return 0; 1770 } 1771 1772 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt]; 1773 err = ipip_ops->ol_netdev_change(mlxsw_sp, ipip_entry, extack); 1774 return err; 1775 } 1776 1777 void mlxsw_sp_ipip_entry_demote_tunnel(struct mlxsw_sp *mlxsw_sp, 1778 struct mlxsw_sp_ipip_entry *ipip_entry) 1779 { 1780 struct net_device *ol_dev = ipip_entry->ol_dev; 1781 1782 if (ol_dev->flags & IFF_UP) 1783 mlxsw_sp_ipip_entry_ol_down_event(mlxsw_sp, ipip_entry); 1784 mlxsw_sp_ipip_entry_destroy(mlxsw_sp, ipip_entry); 1785 } 1786 1787 /* The configuration where several tunnels have the same local address in the 1788 * same underlay table needs special treatment in the HW. That is currently not 1789 * implemented in the driver. This function finds and demotes the first tunnel 1790 * with a given source address, except the one passed in in the argument 1791 * `except'. 1792 */ 1793 bool 1794 mlxsw_sp_ipip_demote_tunnel_by_saddr(struct mlxsw_sp *mlxsw_sp, 1795 enum mlxsw_sp_l3proto ul_proto, 1796 union mlxsw_sp_l3addr saddr, 1797 u32 ul_tb_id, 1798 const struct mlxsw_sp_ipip_entry *except) 1799 { 1800 struct mlxsw_sp_ipip_entry *ipip_entry, *tmp; 1801 1802 list_for_each_entry_safe(ipip_entry, tmp, &mlxsw_sp->router->ipip_list, 1803 ipip_list_node) { 1804 if (ipip_entry != except && 1805 mlxsw_sp_ipip_entry_saddr_matches(mlxsw_sp, ul_proto, saddr, 1806 ul_tb_id, ipip_entry)) { 1807 mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry); 1808 return true; 1809 } 1810 } 1811 1812 return false; 1813 } 1814 1815 static void mlxsw_sp_ipip_demote_tunnel_by_ul_netdev(struct mlxsw_sp *mlxsw_sp, 1816 struct net_device *ul_dev) 1817 { 1818 struct mlxsw_sp_ipip_entry *ipip_entry, *tmp; 1819 1820 list_for_each_entry_safe(ipip_entry, tmp, &mlxsw_sp->router->ipip_list, 1821 ipip_list_node) { 1822 struct net_device *ol_dev = ipip_entry->ol_dev; 1823 struct net_device *ipip_ul_dev; 1824 1825 rcu_read_lock(); 1826 ipip_ul_dev = __mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev); 1827 rcu_read_unlock(); 1828 if (ipip_ul_dev == ul_dev) 1829 mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry); 1830 } 1831 } 1832 1833 int mlxsw_sp_netdevice_ipip_ol_event(struct mlxsw_sp *mlxsw_sp, 1834 struct net_device *ol_dev, 1835 unsigned long event, 1836 struct netdev_notifier_info *info) 1837 { 1838 struct netdev_notifier_changeupper_info *chup; 1839 struct netlink_ext_ack *extack; 1840 int err = 0; 1841 1842 mutex_lock(&mlxsw_sp->router->lock); 1843 switch (event) { 1844 case NETDEV_REGISTER: 1845 err = mlxsw_sp_netdevice_ipip_ol_reg_event(mlxsw_sp, ol_dev); 1846 break; 1847 case NETDEV_UNREGISTER: 1848 mlxsw_sp_netdevice_ipip_ol_unreg_event(mlxsw_sp, ol_dev); 1849 break; 1850 case NETDEV_UP: 1851 mlxsw_sp_netdevice_ipip_ol_up_event(mlxsw_sp, ol_dev); 1852 break; 1853 case NETDEV_DOWN: 1854 mlxsw_sp_netdevice_ipip_ol_down_event(mlxsw_sp, ol_dev); 1855 break; 1856 case NETDEV_CHANGEUPPER: 1857 chup = container_of(info, typeof(*chup), info); 1858 extack = info->extack; 1859 if (netif_is_l3_master(chup->upper_dev)) 1860 err = mlxsw_sp_netdevice_ipip_ol_vrf_event(mlxsw_sp, 1861 ol_dev, 1862 extack); 1863 break; 1864 case NETDEV_CHANGE: 1865 extack = info->extack; 1866 err = mlxsw_sp_netdevice_ipip_ol_change_event(mlxsw_sp, 1867 ol_dev, extack); 1868 break; 1869 case NETDEV_CHANGEMTU: 1870 err = mlxsw_sp_netdevice_ipip_ol_update_mtu(mlxsw_sp, ol_dev); 1871 break; 1872 } 1873 mutex_unlock(&mlxsw_sp->router->lock); 1874 return err; 1875 } 1876 1877 static int 1878 __mlxsw_sp_netdevice_ipip_ul_event(struct mlxsw_sp *mlxsw_sp, 1879 struct mlxsw_sp_ipip_entry *ipip_entry, 1880 struct net_device *ul_dev, 1881 bool *demote_this, 1882 unsigned long event, 1883 struct netdev_notifier_info *info) 1884 { 1885 struct netdev_notifier_changeupper_info *chup; 1886 struct netlink_ext_ack *extack; 1887 1888 switch (event) { 1889 case NETDEV_CHANGEUPPER: 1890 chup = container_of(info, typeof(*chup), info); 1891 extack = info->extack; 1892 if (netif_is_l3_master(chup->upper_dev)) 1893 return mlxsw_sp_netdevice_ipip_ul_vrf_event(mlxsw_sp, 1894 ipip_entry, 1895 ul_dev, 1896 demote_this, 1897 extack); 1898 break; 1899 1900 case NETDEV_UP: 1901 return mlxsw_sp_netdevice_ipip_ul_up_event(mlxsw_sp, ipip_entry, 1902 ul_dev); 1903 case NETDEV_DOWN: 1904 return mlxsw_sp_netdevice_ipip_ul_down_event(mlxsw_sp, 1905 ipip_entry, 1906 ul_dev); 1907 } 1908 return 0; 1909 } 1910 1911 int 1912 mlxsw_sp_netdevice_ipip_ul_event(struct mlxsw_sp *mlxsw_sp, 1913 struct net_device *ul_dev, 1914 unsigned long event, 1915 struct netdev_notifier_info *info) 1916 { 1917 struct mlxsw_sp_ipip_entry *ipip_entry = NULL; 1918 int err = 0; 1919 1920 mutex_lock(&mlxsw_sp->router->lock); 1921 while ((ipip_entry = mlxsw_sp_ipip_entry_find_by_ul_dev(mlxsw_sp, 1922 ul_dev, 1923 ipip_entry))) { 1924 struct mlxsw_sp_ipip_entry *prev; 1925 bool demote_this = false; 1926 1927 err = __mlxsw_sp_netdevice_ipip_ul_event(mlxsw_sp, ipip_entry, 1928 ul_dev, &demote_this, 1929 event, info); 1930 if (err) { 1931 mlxsw_sp_ipip_demote_tunnel_by_ul_netdev(mlxsw_sp, 1932 ul_dev); 1933 break; 1934 } 1935 1936 if (demote_this) { 1937 if (list_is_first(&ipip_entry->ipip_list_node, 1938 &mlxsw_sp->router->ipip_list)) 1939 prev = NULL; 1940 else 1941 /* This can't be cached from previous iteration, 1942 * because that entry could be gone now. 1943 */ 1944 prev = list_prev_entry(ipip_entry, 1945 ipip_list_node); 1946 mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry); 1947 ipip_entry = prev; 1948 } 1949 } 1950 mutex_unlock(&mlxsw_sp->router->lock); 1951 1952 return err; 1953 } 1954 1955 int mlxsw_sp_router_nve_promote_decap(struct mlxsw_sp *mlxsw_sp, u32 ul_tb_id, 1956 enum mlxsw_sp_l3proto ul_proto, 1957 const union mlxsw_sp_l3addr *ul_sip, 1958 u32 tunnel_index) 1959 { 1960 enum mlxsw_sp_fib_entry_type type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP; 1961 struct mlxsw_sp_router *router = mlxsw_sp->router; 1962 struct mlxsw_sp_fib_entry *fib_entry; 1963 int err = 0; 1964 1965 mutex_lock(&mlxsw_sp->router->lock); 1966 1967 if (WARN_ON_ONCE(router->nve_decap_config.valid)) { 1968 err = -EINVAL; 1969 goto out; 1970 } 1971 1972 router->nve_decap_config.ul_tb_id = ul_tb_id; 1973 router->nve_decap_config.tunnel_index = tunnel_index; 1974 router->nve_decap_config.ul_proto = ul_proto; 1975 router->nve_decap_config.ul_sip = *ul_sip; 1976 router->nve_decap_config.valid = true; 1977 1978 /* It is valid to create a tunnel with a local IP and only later 1979 * assign this IP address to a local interface 1980 */ 1981 fib_entry = mlxsw_sp_router_ip2me_fib_entry_find(mlxsw_sp, ul_tb_id, 1982 ul_proto, ul_sip, 1983 type); 1984 if (!fib_entry) 1985 goto out; 1986 1987 fib_entry->decap.tunnel_index = tunnel_index; 1988 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP; 1989 1990 err = mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry); 1991 if (err) 1992 goto err_fib_entry_update; 1993 1994 goto out; 1995 1996 err_fib_entry_update: 1997 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP; 1998 mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry); 1999 out: 2000 mutex_unlock(&mlxsw_sp->router->lock); 2001 return err; 2002 } 2003 2004 void mlxsw_sp_router_nve_demote_decap(struct mlxsw_sp *mlxsw_sp, u32 ul_tb_id, 2005 enum mlxsw_sp_l3proto ul_proto, 2006 const union mlxsw_sp_l3addr *ul_sip) 2007 { 2008 enum mlxsw_sp_fib_entry_type type = MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP; 2009 struct mlxsw_sp_router *router = mlxsw_sp->router; 2010 struct mlxsw_sp_fib_entry *fib_entry; 2011 2012 mutex_lock(&mlxsw_sp->router->lock); 2013 2014 if (WARN_ON_ONCE(!router->nve_decap_config.valid)) 2015 goto out; 2016 2017 router->nve_decap_config.valid = false; 2018 2019 fib_entry = mlxsw_sp_router_ip2me_fib_entry_find(mlxsw_sp, ul_tb_id, 2020 ul_proto, ul_sip, 2021 type); 2022 if (!fib_entry) 2023 goto out; 2024 2025 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP; 2026 mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry); 2027 out: 2028 mutex_unlock(&mlxsw_sp->router->lock); 2029 } 2030 2031 static bool mlxsw_sp_router_nve_is_decap(struct mlxsw_sp *mlxsw_sp, 2032 u32 ul_tb_id, 2033 enum mlxsw_sp_l3proto ul_proto, 2034 const union mlxsw_sp_l3addr *ul_sip) 2035 { 2036 struct mlxsw_sp_router *router = mlxsw_sp->router; 2037 2038 return router->nve_decap_config.valid && 2039 router->nve_decap_config.ul_tb_id == ul_tb_id && 2040 router->nve_decap_config.ul_proto == ul_proto && 2041 !memcmp(&router->nve_decap_config.ul_sip, ul_sip, 2042 sizeof(*ul_sip)); 2043 } 2044 2045 struct mlxsw_sp_neigh_key { 2046 struct neighbour *n; 2047 }; 2048 2049 struct mlxsw_sp_neigh_entry { 2050 struct list_head rif_list_node; 2051 struct rhash_head ht_node; 2052 struct mlxsw_sp_neigh_key key; 2053 u16 rif; 2054 bool connected; 2055 unsigned char ha[ETH_ALEN]; 2056 struct list_head nexthop_list; /* list of nexthops using 2057 * this neigh entry 2058 */ 2059 struct list_head nexthop_neighs_list_node; 2060 unsigned int counter_index; 2061 bool counter_valid; 2062 }; 2063 2064 static const struct rhashtable_params mlxsw_sp_neigh_ht_params = { 2065 .key_offset = offsetof(struct mlxsw_sp_neigh_entry, key), 2066 .head_offset = offsetof(struct mlxsw_sp_neigh_entry, ht_node), 2067 .key_len = sizeof(struct mlxsw_sp_neigh_key), 2068 }; 2069 2070 struct mlxsw_sp_neigh_entry * 2071 mlxsw_sp_rif_neigh_next(struct mlxsw_sp_rif *rif, 2072 struct mlxsw_sp_neigh_entry *neigh_entry) 2073 { 2074 if (!neigh_entry) { 2075 if (list_empty(&rif->neigh_list)) 2076 return NULL; 2077 else 2078 return list_first_entry(&rif->neigh_list, 2079 typeof(*neigh_entry), 2080 rif_list_node); 2081 } 2082 if (list_is_last(&neigh_entry->rif_list_node, &rif->neigh_list)) 2083 return NULL; 2084 return list_next_entry(neigh_entry, rif_list_node); 2085 } 2086 2087 int mlxsw_sp_neigh_entry_type(struct mlxsw_sp_neigh_entry *neigh_entry) 2088 { 2089 return neigh_entry->key.n->tbl->family; 2090 } 2091 2092 unsigned char * 2093 mlxsw_sp_neigh_entry_ha(struct mlxsw_sp_neigh_entry *neigh_entry) 2094 { 2095 return neigh_entry->ha; 2096 } 2097 2098 u32 mlxsw_sp_neigh4_entry_dip(struct mlxsw_sp_neigh_entry *neigh_entry) 2099 { 2100 struct neighbour *n; 2101 2102 n = neigh_entry->key.n; 2103 return ntohl(*((__be32 *) n->primary_key)); 2104 } 2105 2106 struct in6_addr * 2107 mlxsw_sp_neigh6_entry_dip(struct mlxsw_sp_neigh_entry *neigh_entry) 2108 { 2109 struct neighbour *n; 2110 2111 n = neigh_entry->key.n; 2112 return (struct in6_addr *) &n->primary_key; 2113 } 2114 2115 int mlxsw_sp_neigh_counter_get(struct mlxsw_sp *mlxsw_sp, 2116 struct mlxsw_sp_neigh_entry *neigh_entry, 2117 u64 *p_counter) 2118 { 2119 if (!neigh_entry->counter_valid) 2120 return -EINVAL; 2121 2122 return mlxsw_sp_flow_counter_get(mlxsw_sp, neigh_entry->counter_index, 2123 p_counter, NULL); 2124 } 2125 2126 static struct mlxsw_sp_neigh_entry * 2127 mlxsw_sp_neigh_entry_alloc(struct mlxsw_sp *mlxsw_sp, struct neighbour *n, 2128 u16 rif) 2129 { 2130 struct mlxsw_sp_neigh_entry *neigh_entry; 2131 2132 neigh_entry = kzalloc(sizeof(*neigh_entry), GFP_KERNEL); 2133 if (!neigh_entry) 2134 return NULL; 2135 2136 neigh_entry->key.n = n; 2137 neigh_entry->rif = rif; 2138 INIT_LIST_HEAD(&neigh_entry->nexthop_list); 2139 2140 return neigh_entry; 2141 } 2142 2143 static void mlxsw_sp_neigh_entry_free(struct mlxsw_sp_neigh_entry *neigh_entry) 2144 { 2145 kfree(neigh_entry); 2146 } 2147 2148 static int 2149 mlxsw_sp_neigh_entry_insert(struct mlxsw_sp *mlxsw_sp, 2150 struct mlxsw_sp_neigh_entry *neigh_entry) 2151 { 2152 return rhashtable_insert_fast(&mlxsw_sp->router->neigh_ht, 2153 &neigh_entry->ht_node, 2154 mlxsw_sp_neigh_ht_params); 2155 } 2156 2157 static void 2158 mlxsw_sp_neigh_entry_remove(struct mlxsw_sp *mlxsw_sp, 2159 struct mlxsw_sp_neigh_entry *neigh_entry) 2160 { 2161 rhashtable_remove_fast(&mlxsw_sp->router->neigh_ht, 2162 &neigh_entry->ht_node, 2163 mlxsw_sp_neigh_ht_params); 2164 } 2165 2166 static bool 2167 mlxsw_sp_neigh_counter_should_alloc(struct mlxsw_sp *mlxsw_sp, 2168 struct mlxsw_sp_neigh_entry *neigh_entry) 2169 { 2170 struct devlink *devlink; 2171 const char *table_name; 2172 2173 switch (mlxsw_sp_neigh_entry_type(neigh_entry)) { 2174 case AF_INET: 2175 table_name = MLXSW_SP_DPIPE_TABLE_NAME_HOST4; 2176 break; 2177 case AF_INET6: 2178 table_name = MLXSW_SP_DPIPE_TABLE_NAME_HOST6; 2179 break; 2180 default: 2181 WARN_ON(1); 2182 return false; 2183 } 2184 2185 devlink = priv_to_devlink(mlxsw_sp->core); 2186 return devlink_dpipe_table_counter_enabled(devlink, table_name); 2187 } 2188 2189 static void 2190 mlxsw_sp_neigh_counter_alloc(struct mlxsw_sp *mlxsw_sp, 2191 struct mlxsw_sp_neigh_entry *neigh_entry) 2192 { 2193 if (!mlxsw_sp_neigh_counter_should_alloc(mlxsw_sp, neigh_entry)) 2194 return; 2195 2196 if (mlxsw_sp_flow_counter_alloc(mlxsw_sp, &neigh_entry->counter_index)) 2197 return; 2198 2199 neigh_entry->counter_valid = true; 2200 } 2201 2202 static void 2203 mlxsw_sp_neigh_counter_free(struct mlxsw_sp *mlxsw_sp, 2204 struct mlxsw_sp_neigh_entry *neigh_entry) 2205 { 2206 if (!neigh_entry->counter_valid) 2207 return; 2208 mlxsw_sp_flow_counter_free(mlxsw_sp, 2209 neigh_entry->counter_index); 2210 neigh_entry->counter_valid = false; 2211 } 2212 2213 static struct mlxsw_sp_neigh_entry * 2214 mlxsw_sp_neigh_entry_create(struct mlxsw_sp *mlxsw_sp, struct neighbour *n) 2215 { 2216 struct mlxsw_sp_neigh_entry *neigh_entry; 2217 struct mlxsw_sp_rif *rif; 2218 int err; 2219 2220 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, n->dev); 2221 if (!rif) 2222 return ERR_PTR(-EINVAL); 2223 2224 neigh_entry = mlxsw_sp_neigh_entry_alloc(mlxsw_sp, n, rif->rif_index); 2225 if (!neigh_entry) 2226 return ERR_PTR(-ENOMEM); 2227 2228 err = mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry); 2229 if (err) 2230 goto err_neigh_entry_insert; 2231 2232 mlxsw_sp_neigh_counter_alloc(mlxsw_sp, neigh_entry); 2233 list_add(&neigh_entry->rif_list_node, &rif->neigh_list); 2234 2235 return neigh_entry; 2236 2237 err_neigh_entry_insert: 2238 mlxsw_sp_neigh_entry_free(neigh_entry); 2239 return ERR_PTR(err); 2240 } 2241 2242 static void 2243 mlxsw_sp_neigh_entry_destroy(struct mlxsw_sp *mlxsw_sp, 2244 struct mlxsw_sp_neigh_entry *neigh_entry) 2245 { 2246 list_del(&neigh_entry->rif_list_node); 2247 mlxsw_sp_neigh_counter_free(mlxsw_sp, neigh_entry); 2248 mlxsw_sp_neigh_entry_remove(mlxsw_sp, neigh_entry); 2249 mlxsw_sp_neigh_entry_free(neigh_entry); 2250 } 2251 2252 static struct mlxsw_sp_neigh_entry * 2253 mlxsw_sp_neigh_entry_lookup(struct mlxsw_sp *mlxsw_sp, struct neighbour *n) 2254 { 2255 struct mlxsw_sp_neigh_key key; 2256 2257 key.n = n; 2258 return rhashtable_lookup_fast(&mlxsw_sp->router->neigh_ht, 2259 &key, mlxsw_sp_neigh_ht_params); 2260 } 2261 2262 static void 2263 mlxsw_sp_router_neighs_update_interval_init(struct mlxsw_sp *mlxsw_sp) 2264 { 2265 unsigned long interval; 2266 2267 #if IS_ENABLED(CONFIG_IPV6) 2268 interval = min_t(unsigned long, 2269 NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME), 2270 NEIGH_VAR(&nd_tbl.parms, DELAY_PROBE_TIME)); 2271 #else 2272 interval = NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME); 2273 #endif 2274 mlxsw_sp->router->neighs_update.interval = jiffies_to_msecs(interval); 2275 } 2276 2277 static void mlxsw_sp_router_neigh_ent_ipv4_process(struct mlxsw_sp *mlxsw_sp, 2278 char *rauhtd_pl, 2279 int ent_index) 2280 { 2281 struct net_device *dev; 2282 struct neighbour *n; 2283 __be32 dipn; 2284 u32 dip; 2285 u16 rif; 2286 2287 mlxsw_reg_rauhtd_ent_ipv4_unpack(rauhtd_pl, ent_index, &rif, &dip); 2288 2289 if (!mlxsw_sp->router->rifs[rif]) { 2290 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Incorrect RIF in neighbour entry\n"); 2291 return; 2292 } 2293 2294 dipn = htonl(dip); 2295 dev = mlxsw_sp->router->rifs[rif]->dev; 2296 n = neigh_lookup(&arp_tbl, &dipn, dev); 2297 if (!n) 2298 return; 2299 2300 netdev_dbg(dev, "Updating neighbour with IP=%pI4h\n", &dip); 2301 neigh_event_send(n, NULL); 2302 neigh_release(n); 2303 } 2304 2305 #if IS_ENABLED(CONFIG_IPV6) 2306 static void mlxsw_sp_router_neigh_ent_ipv6_process(struct mlxsw_sp *mlxsw_sp, 2307 char *rauhtd_pl, 2308 int rec_index) 2309 { 2310 struct net_device *dev; 2311 struct neighbour *n; 2312 struct in6_addr dip; 2313 u16 rif; 2314 2315 mlxsw_reg_rauhtd_ent_ipv6_unpack(rauhtd_pl, rec_index, &rif, 2316 (char *) &dip); 2317 2318 if (!mlxsw_sp->router->rifs[rif]) { 2319 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Incorrect RIF in neighbour entry\n"); 2320 return; 2321 } 2322 2323 dev = mlxsw_sp->router->rifs[rif]->dev; 2324 n = neigh_lookup(&nd_tbl, &dip, dev); 2325 if (!n) 2326 return; 2327 2328 netdev_dbg(dev, "Updating neighbour with IP=%pI6c\n", &dip); 2329 neigh_event_send(n, NULL); 2330 neigh_release(n); 2331 } 2332 #else 2333 static void mlxsw_sp_router_neigh_ent_ipv6_process(struct mlxsw_sp *mlxsw_sp, 2334 char *rauhtd_pl, 2335 int rec_index) 2336 { 2337 } 2338 #endif 2339 2340 static void mlxsw_sp_router_neigh_rec_ipv4_process(struct mlxsw_sp *mlxsw_sp, 2341 char *rauhtd_pl, 2342 int rec_index) 2343 { 2344 u8 num_entries; 2345 int i; 2346 2347 num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl, 2348 rec_index); 2349 /* Hardware starts counting at 0, so add 1. */ 2350 num_entries++; 2351 2352 /* Each record consists of several neighbour entries. */ 2353 for (i = 0; i < num_entries; i++) { 2354 int ent_index; 2355 2356 ent_index = rec_index * MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC + i; 2357 mlxsw_sp_router_neigh_ent_ipv4_process(mlxsw_sp, rauhtd_pl, 2358 ent_index); 2359 } 2360 2361 } 2362 2363 static void mlxsw_sp_router_neigh_rec_ipv6_process(struct mlxsw_sp *mlxsw_sp, 2364 char *rauhtd_pl, 2365 int rec_index) 2366 { 2367 /* One record contains one entry. */ 2368 mlxsw_sp_router_neigh_ent_ipv6_process(mlxsw_sp, rauhtd_pl, 2369 rec_index); 2370 } 2371 2372 static void mlxsw_sp_router_neigh_rec_process(struct mlxsw_sp *mlxsw_sp, 2373 char *rauhtd_pl, int rec_index) 2374 { 2375 switch (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, rec_index)) { 2376 case MLXSW_REG_RAUHTD_TYPE_IPV4: 2377 mlxsw_sp_router_neigh_rec_ipv4_process(mlxsw_sp, rauhtd_pl, 2378 rec_index); 2379 break; 2380 case MLXSW_REG_RAUHTD_TYPE_IPV6: 2381 mlxsw_sp_router_neigh_rec_ipv6_process(mlxsw_sp, rauhtd_pl, 2382 rec_index); 2383 break; 2384 } 2385 } 2386 2387 static bool mlxsw_sp_router_rauhtd_is_full(char *rauhtd_pl) 2388 { 2389 u8 num_rec, last_rec_index, num_entries; 2390 2391 num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl); 2392 last_rec_index = num_rec - 1; 2393 2394 if (num_rec < MLXSW_REG_RAUHTD_REC_MAX_NUM) 2395 return false; 2396 if (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, last_rec_index) == 2397 MLXSW_REG_RAUHTD_TYPE_IPV6) 2398 return true; 2399 2400 num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl, 2401 last_rec_index); 2402 if (++num_entries == MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC) 2403 return true; 2404 return false; 2405 } 2406 2407 static int 2408 __mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp, 2409 char *rauhtd_pl, 2410 enum mlxsw_reg_rauhtd_type type) 2411 { 2412 int i, num_rec; 2413 int err; 2414 2415 /* Ensure the RIF we read from the device does not change mid-dump. */ 2416 mutex_lock(&mlxsw_sp->router->lock); 2417 do { 2418 mlxsw_reg_rauhtd_pack(rauhtd_pl, type); 2419 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(rauhtd), 2420 rauhtd_pl); 2421 if (err) { 2422 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Failed to dump neighbour table\n"); 2423 break; 2424 } 2425 num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl); 2426 for (i = 0; i < num_rec; i++) 2427 mlxsw_sp_router_neigh_rec_process(mlxsw_sp, rauhtd_pl, 2428 i); 2429 } while (mlxsw_sp_router_rauhtd_is_full(rauhtd_pl)); 2430 mutex_unlock(&mlxsw_sp->router->lock); 2431 2432 return err; 2433 } 2434 2435 static int mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp) 2436 { 2437 enum mlxsw_reg_rauhtd_type type; 2438 char *rauhtd_pl; 2439 int err; 2440 2441 rauhtd_pl = kmalloc(MLXSW_REG_RAUHTD_LEN, GFP_KERNEL); 2442 if (!rauhtd_pl) 2443 return -ENOMEM; 2444 2445 type = MLXSW_REG_RAUHTD_TYPE_IPV4; 2446 err = __mlxsw_sp_router_neighs_update_rauhtd(mlxsw_sp, rauhtd_pl, type); 2447 if (err) 2448 goto out; 2449 2450 type = MLXSW_REG_RAUHTD_TYPE_IPV6; 2451 err = __mlxsw_sp_router_neighs_update_rauhtd(mlxsw_sp, rauhtd_pl, type); 2452 out: 2453 kfree(rauhtd_pl); 2454 return err; 2455 } 2456 2457 static void mlxsw_sp_router_neighs_update_nh(struct mlxsw_sp *mlxsw_sp) 2458 { 2459 struct mlxsw_sp_neigh_entry *neigh_entry; 2460 2461 mutex_lock(&mlxsw_sp->router->lock); 2462 list_for_each_entry(neigh_entry, &mlxsw_sp->router->nexthop_neighs_list, 2463 nexthop_neighs_list_node) 2464 /* If this neigh have nexthops, make the kernel think this neigh 2465 * is active regardless of the traffic. 2466 */ 2467 neigh_event_send(neigh_entry->key.n, NULL); 2468 mutex_unlock(&mlxsw_sp->router->lock); 2469 } 2470 2471 static void 2472 mlxsw_sp_router_neighs_update_work_schedule(struct mlxsw_sp *mlxsw_sp) 2473 { 2474 unsigned long interval = mlxsw_sp->router->neighs_update.interval; 2475 2476 mlxsw_core_schedule_dw(&mlxsw_sp->router->neighs_update.dw, 2477 msecs_to_jiffies(interval)); 2478 } 2479 2480 static void mlxsw_sp_router_neighs_update_work(struct work_struct *work) 2481 { 2482 struct mlxsw_sp_router *router; 2483 int err; 2484 2485 router = container_of(work, struct mlxsw_sp_router, 2486 neighs_update.dw.work); 2487 err = mlxsw_sp_router_neighs_update_rauhtd(router->mlxsw_sp); 2488 if (err) 2489 dev_err(router->mlxsw_sp->bus_info->dev, "Could not update kernel for neigh activity"); 2490 2491 mlxsw_sp_router_neighs_update_nh(router->mlxsw_sp); 2492 2493 mlxsw_sp_router_neighs_update_work_schedule(router->mlxsw_sp); 2494 } 2495 2496 static void mlxsw_sp_router_probe_unresolved_nexthops(struct work_struct *work) 2497 { 2498 struct mlxsw_sp_neigh_entry *neigh_entry; 2499 struct mlxsw_sp_router *router; 2500 2501 router = container_of(work, struct mlxsw_sp_router, 2502 nexthop_probe_dw.work); 2503 /* Iterate over nexthop neighbours, find those who are unresolved and 2504 * send arp on them. This solves the chicken-egg problem when 2505 * the nexthop wouldn't get offloaded until the neighbor is resolved 2506 * but it wouldn't get resolved ever in case traffic is flowing in HW 2507 * using different nexthop. 2508 */ 2509 mutex_lock(&router->lock); 2510 list_for_each_entry(neigh_entry, &router->nexthop_neighs_list, 2511 nexthop_neighs_list_node) 2512 if (!neigh_entry->connected) 2513 neigh_event_send(neigh_entry->key.n, NULL); 2514 mutex_unlock(&router->lock); 2515 2516 mlxsw_core_schedule_dw(&router->nexthop_probe_dw, 2517 MLXSW_SP_UNRESOLVED_NH_PROBE_INTERVAL); 2518 } 2519 2520 static void 2521 mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp, 2522 struct mlxsw_sp_neigh_entry *neigh_entry, 2523 bool removing, bool dead); 2524 2525 static enum mlxsw_reg_rauht_op mlxsw_sp_rauht_op(bool adding) 2526 { 2527 return adding ? MLXSW_REG_RAUHT_OP_WRITE_ADD : 2528 MLXSW_REG_RAUHT_OP_WRITE_DELETE; 2529 } 2530 2531 static int 2532 mlxsw_sp_router_neigh_entry_op4(struct mlxsw_sp *mlxsw_sp, 2533 struct mlxsw_sp_neigh_entry *neigh_entry, 2534 enum mlxsw_reg_rauht_op op) 2535 { 2536 struct neighbour *n = neigh_entry->key.n; 2537 u32 dip = ntohl(*((__be32 *) n->primary_key)); 2538 char rauht_pl[MLXSW_REG_RAUHT_LEN]; 2539 2540 mlxsw_reg_rauht_pack4(rauht_pl, op, neigh_entry->rif, neigh_entry->ha, 2541 dip); 2542 if (neigh_entry->counter_valid) 2543 mlxsw_reg_rauht_pack_counter(rauht_pl, 2544 neigh_entry->counter_index); 2545 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl); 2546 } 2547 2548 static int 2549 mlxsw_sp_router_neigh_entry_op6(struct mlxsw_sp *mlxsw_sp, 2550 struct mlxsw_sp_neigh_entry *neigh_entry, 2551 enum mlxsw_reg_rauht_op op) 2552 { 2553 struct neighbour *n = neigh_entry->key.n; 2554 char rauht_pl[MLXSW_REG_RAUHT_LEN]; 2555 const char *dip = n->primary_key; 2556 2557 mlxsw_reg_rauht_pack6(rauht_pl, op, neigh_entry->rif, neigh_entry->ha, 2558 dip); 2559 if (neigh_entry->counter_valid) 2560 mlxsw_reg_rauht_pack_counter(rauht_pl, 2561 neigh_entry->counter_index); 2562 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl); 2563 } 2564 2565 bool mlxsw_sp_neigh_ipv6_ignore(struct mlxsw_sp_neigh_entry *neigh_entry) 2566 { 2567 struct neighbour *n = neigh_entry->key.n; 2568 2569 /* Packets with a link-local destination address are trapped 2570 * after LPM lookup and never reach the neighbour table, so 2571 * there is no need to program such neighbours to the device. 2572 */ 2573 if (ipv6_addr_type((struct in6_addr *) &n->primary_key) & 2574 IPV6_ADDR_LINKLOCAL) 2575 return true; 2576 return false; 2577 } 2578 2579 static void 2580 mlxsw_sp_neigh_entry_update(struct mlxsw_sp *mlxsw_sp, 2581 struct mlxsw_sp_neigh_entry *neigh_entry, 2582 bool adding) 2583 { 2584 enum mlxsw_reg_rauht_op op = mlxsw_sp_rauht_op(adding); 2585 int err; 2586 2587 if (!adding && !neigh_entry->connected) 2588 return; 2589 neigh_entry->connected = adding; 2590 if (neigh_entry->key.n->tbl->family == AF_INET) { 2591 err = mlxsw_sp_router_neigh_entry_op4(mlxsw_sp, neigh_entry, 2592 op); 2593 if (err) 2594 return; 2595 } else if (neigh_entry->key.n->tbl->family == AF_INET6) { 2596 if (mlxsw_sp_neigh_ipv6_ignore(neigh_entry)) 2597 return; 2598 err = mlxsw_sp_router_neigh_entry_op6(mlxsw_sp, neigh_entry, 2599 op); 2600 if (err) 2601 return; 2602 } else { 2603 WARN_ON_ONCE(1); 2604 return; 2605 } 2606 2607 if (adding) 2608 neigh_entry->key.n->flags |= NTF_OFFLOADED; 2609 else 2610 neigh_entry->key.n->flags &= ~NTF_OFFLOADED; 2611 } 2612 2613 void 2614 mlxsw_sp_neigh_entry_counter_update(struct mlxsw_sp *mlxsw_sp, 2615 struct mlxsw_sp_neigh_entry *neigh_entry, 2616 bool adding) 2617 { 2618 if (adding) 2619 mlxsw_sp_neigh_counter_alloc(mlxsw_sp, neigh_entry); 2620 else 2621 mlxsw_sp_neigh_counter_free(mlxsw_sp, neigh_entry); 2622 mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, true); 2623 } 2624 2625 struct mlxsw_sp_netevent_work { 2626 struct work_struct work; 2627 struct mlxsw_sp *mlxsw_sp; 2628 struct neighbour *n; 2629 }; 2630 2631 static void mlxsw_sp_router_neigh_event_work(struct work_struct *work) 2632 { 2633 struct mlxsw_sp_netevent_work *net_work = 2634 container_of(work, struct mlxsw_sp_netevent_work, work); 2635 struct mlxsw_sp *mlxsw_sp = net_work->mlxsw_sp; 2636 struct mlxsw_sp_neigh_entry *neigh_entry; 2637 struct neighbour *n = net_work->n; 2638 unsigned char ha[ETH_ALEN]; 2639 bool entry_connected; 2640 u8 nud_state, dead; 2641 2642 /* If these parameters are changed after we release the lock, 2643 * then we are guaranteed to receive another event letting us 2644 * know about it. 2645 */ 2646 read_lock_bh(&n->lock); 2647 memcpy(ha, n->ha, ETH_ALEN); 2648 nud_state = n->nud_state; 2649 dead = n->dead; 2650 read_unlock_bh(&n->lock); 2651 2652 mutex_lock(&mlxsw_sp->router->lock); 2653 mlxsw_sp_span_respin(mlxsw_sp); 2654 2655 entry_connected = nud_state & NUD_VALID && !dead; 2656 neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n); 2657 if (!entry_connected && !neigh_entry) 2658 goto out; 2659 if (!neigh_entry) { 2660 neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n); 2661 if (IS_ERR(neigh_entry)) 2662 goto out; 2663 } 2664 2665 memcpy(neigh_entry->ha, ha, ETH_ALEN); 2666 mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, entry_connected); 2667 mlxsw_sp_nexthop_neigh_update(mlxsw_sp, neigh_entry, !entry_connected, 2668 dead); 2669 2670 if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list)) 2671 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry); 2672 2673 out: 2674 mutex_unlock(&mlxsw_sp->router->lock); 2675 neigh_release(n); 2676 kfree(net_work); 2677 } 2678 2679 static int mlxsw_sp_mp_hash_init(struct mlxsw_sp *mlxsw_sp); 2680 2681 static void mlxsw_sp_router_mp_hash_event_work(struct work_struct *work) 2682 { 2683 struct mlxsw_sp_netevent_work *net_work = 2684 container_of(work, struct mlxsw_sp_netevent_work, work); 2685 struct mlxsw_sp *mlxsw_sp = net_work->mlxsw_sp; 2686 2687 mlxsw_sp_mp_hash_init(mlxsw_sp); 2688 kfree(net_work); 2689 } 2690 2691 static int __mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp); 2692 2693 static void mlxsw_sp_router_update_priority_work(struct work_struct *work) 2694 { 2695 struct mlxsw_sp_netevent_work *net_work = 2696 container_of(work, struct mlxsw_sp_netevent_work, work); 2697 struct mlxsw_sp *mlxsw_sp = net_work->mlxsw_sp; 2698 2699 __mlxsw_sp_router_init(mlxsw_sp); 2700 kfree(net_work); 2701 } 2702 2703 static int mlxsw_sp_router_schedule_work(struct net *net, 2704 struct notifier_block *nb, 2705 void (*cb)(struct work_struct *)) 2706 { 2707 struct mlxsw_sp_netevent_work *net_work; 2708 struct mlxsw_sp_router *router; 2709 2710 router = container_of(nb, struct mlxsw_sp_router, netevent_nb); 2711 if (!net_eq(net, mlxsw_sp_net(router->mlxsw_sp))) 2712 return NOTIFY_DONE; 2713 2714 net_work = kzalloc(sizeof(*net_work), GFP_ATOMIC); 2715 if (!net_work) 2716 return NOTIFY_BAD; 2717 2718 INIT_WORK(&net_work->work, cb); 2719 net_work->mlxsw_sp = router->mlxsw_sp; 2720 mlxsw_core_schedule_work(&net_work->work); 2721 return NOTIFY_DONE; 2722 } 2723 2724 static int mlxsw_sp_router_netevent_event(struct notifier_block *nb, 2725 unsigned long event, void *ptr) 2726 { 2727 struct mlxsw_sp_netevent_work *net_work; 2728 struct mlxsw_sp_port *mlxsw_sp_port; 2729 struct mlxsw_sp *mlxsw_sp; 2730 unsigned long interval; 2731 struct neigh_parms *p; 2732 struct neighbour *n; 2733 2734 switch (event) { 2735 case NETEVENT_DELAY_PROBE_TIME_UPDATE: 2736 p = ptr; 2737 2738 /* We don't care about changes in the default table. */ 2739 if (!p->dev || (p->tbl->family != AF_INET && 2740 p->tbl->family != AF_INET6)) 2741 return NOTIFY_DONE; 2742 2743 /* We are in atomic context and can't take RTNL mutex, 2744 * so use RCU variant to walk the device chain. 2745 */ 2746 mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(p->dev); 2747 if (!mlxsw_sp_port) 2748 return NOTIFY_DONE; 2749 2750 mlxsw_sp = mlxsw_sp_port->mlxsw_sp; 2751 interval = jiffies_to_msecs(NEIGH_VAR(p, DELAY_PROBE_TIME)); 2752 mlxsw_sp->router->neighs_update.interval = interval; 2753 2754 mlxsw_sp_port_dev_put(mlxsw_sp_port); 2755 break; 2756 case NETEVENT_NEIGH_UPDATE: 2757 n = ptr; 2758 2759 if (n->tbl->family != AF_INET && n->tbl->family != AF_INET6) 2760 return NOTIFY_DONE; 2761 2762 mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(n->dev); 2763 if (!mlxsw_sp_port) 2764 return NOTIFY_DONE; 2765 2766 net_work = kzalloc(sizeof(*net_work), GFP_ATOMIC); 2767 if (!net_work) { 2768 mlxsw_sp_port_dev_put(mlxsw_sp_port); 2769 return NOTIFY_BAD; 2770 } 2771 2772 INIT_WORK(&net_work->work, mlxsw_sp_router_neigh_event_work); 2773 net_work->mlxsw_sp = mlxsw_sp_port->mlxsw_sp; 2774 net_work->n = n; 2775 2776 /* Take a reference to ensure the neighbour won't be 2777 * destructed until we drop the reference in delayed 2778 * work. 2779 */ 2780 neigh_clone(n); 2781 mlxsw_core_schedule_work(&net_work->work); 2782 mlxsw_sp_port_dev_put(mlxsw_sp_port); 2783 break; 2784 case NETEVENT_IPV4_MPATH_HASH_UPDATE: 2785 case NETEVENT_IPV6_MPATH_HASH_UPDATE: 2786 return mlxsw_sp_router_schedule_work(ptr, nb, 2787 mlxsw_sp_router_mp_hash_event_work); 2788 2789 case NETEVENT_IPV4_FWD_UPDATE_PRIORITY_UPDATE: 2790 return mlxsw_sp_router_schedule_work(ptr, nb, 2791 mlxsw_sp_router_update_priority_work); 2792 } 2793 2794 return NOTIFY_DONE; 2795 } 2796 2797 static int mlxsw_sp_neigh_init(struct mlxsw_sp *mlxsw_sp) 2798 { 2799 int err; 2800 2801 err = rhashtable_init(&mlxsw_sp->router->neigh_ht, 2802 &mlxsw_sp_neigh_ht_params); 2803 if (err) 2804 return err; 2805 2806 /* Initialize the polling interval according to the default 2807 * table. 2808 */ 2809 mlxsw_sp_router_neighs_update_interval_init(mlxsw_sp); 2810 2811 /* Create the delayed works for the activity_update */ 2812 INIT_DELAYED_WORK(&mlxsw_sp->router->neighs_update.dw, 2813 mlxsw_sp_router_neighs_update_work); 2814 INIT_DELAYED_WORK(&mlxsw_sp->router->nexthop_probe_dw, 2815 mlxsw_sp_router_probe_unresolved_nexthops); 2816 mlxsw_core_schedule_dw(&mlxsw_sp->router->neighs_update.dw, 0); 2817 mlxsw_core_schedule_dw(&mlxsw_sp->router->nexthop_probe_dw, 0); 2818 return 0; 2819 } 2820 2821 static void mlxsw_sp_neigh_fini(struct mlxsw_sp *mlxsw_sp) 2822 { 2823 cancel_delayed_work_sync(&mlxsw_sp->router->neighs_update.dw); 2824 cancel_delayed_work_sync(&mlxsw_sp->router->nexthop_probe_dw); 2825 rhashtable_destroy(&mlxsw_sp->router->neigh_ht); 2826 } 2827 2828 static void mlxsw_sp_neigh_rif_gone_sync(struct mlxsw_sp *mlxsw_sp, 2829 struct mlxsw_sp_rif *rif) 2830 { 2831 struct mlxsw_sp_neigh_entry *neigh_entry, *tmp; 2832 2833 list_for_each_entry_safe(neigh_entry, tmp, &rif->neigh_list, 2834 rif_list_node) { 2835 mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, false); 2836 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry); 2837 } 2838 } 2839 2840 enum mlxsw_sp_nexthop_type { 2841 MLXSW_SP_NEXTHOP_TYPE_ETH, 2842 MLXSW_SP_NEXTHOP_TYPE_IPIP, 2843 }; 2844 2845 struct mlxsw_sp_nexthop_key { 2846 struct fib_nh *fib_nh; 2847 }; 2848 2849 struct mlxsw_sp_nexthop { 2850 struct list_head neigh_list_node; /* member of neigh entry list */ 2851 struct list_head rif_list_node; 2852 struct list_head router_list_node; 2853 struct mlxsw_sp_nexthop_group_info *nhgi; /* pointer back to the group 2854 * this nexthop belongs to 2855 */ 2856 struct rhash_head ht_node; 2857 struct neigh_table *neigh_tbl; 2858 struct mlxsw_sp_nexthop_key key; 2859 unsigned char gw_addr[sizeof(struct in6_addr)]; 2860 int ifindex; 2861 int nh_weight; 2862 int norm_nh_weight; 2863 int num_adj_entries; 2864 struct mlxsw_sp_rif *rif; 2865 u8 should_offload:1, /* set indicates this neigh is connected and 2866 * should be put to KVD linear area of this group. 2867 */ 2868 offloaded:1, /* set in case the neigh is actually put into 2869 * KVD linear area of this group. 2870 */ 2871 update:1, /* set indicates that MAC of this neigh should be 2872 * updated in HW 2873 */ 2874 discard:1; /* nexthop is programmed to discard packets */ 2875 enum mlxsw_sp_nexthop_type type; 2876 union { 2877 struct mlxsw_sp_neigh_entry *neigh_entry; 2878 struct mlxsw_sp_ipip_entry *ipip_entry; 2879 }; 2880 unsigned int counter_index; 2881 bool counter_valid; 2882 }; 2883 2884 enum mlxsw_sp_nexthop_group_type { 2885 MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4, 2886 MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6, 2887 MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ, 2888 }; 2889 2890 struct mlxsw_sp_nexthop_group_info { 2891 struct mlxsw_sp_nexthop_group *nh_grp; 2892 u32 adj_index; 2893 u16 ecmp_size; 2894 u16 count; 2895 int sum_norm_weight; 2896 u8 adj_index_valid:1, 2897 gateway:1; /* routes using the group use a gateway */ 2898 struct mlxsw_sp_nexthop nexthops[0]; 2899 #define nh_rif nexthops[0].rif 2900 }; 2901 2902 struct mlxsw_sp_nexthop_group_vr_key { 2903 u16 vr_id; 2904 enum mlxsw_sp_l3proto proto; 2905 }; 2906 2907 struct mlxsw_sp_nexthop_group_vr_entry { 2908 struct list_head list; /* member in vr_list */ 2909 struct rhash_head ht_node; /* member in vr_ht */ 2910 refcount_t ref_count; 2911 struct mlxsw_sp_nexthop_group_vr_key key; 2912 }; 2913 2914 struct mlxsw_sp_nexthop_group { 2915 struct rhash_head ht_node; 2916 struct list_head fib_list; /* list of fib entries that use this group */ 2917 union { 2918 struct { 2919 struct fib_info *fi; 2920 } ipv4; 2921 struct { 2922 u32 id; 2923 } obj; 2924 }; 2925 struct mlxsw_sp_nexthop_group_info *nhgi; 2926 struct list_head vr_list; 2927 struct rhashtable vr_ht; 2928 enum mlxsw_sp_nexthop_group_type type; 2929 bool can_destroy; 2930 }; 2931 2932 void mlxsw_sp_nexthop_counter_alloc(struct mlxsw_sp *mlxsw_sp, 2933 struct mlxsw_sp_nexthop *nh) 2934 { 2935 struct devlink *devlink; 2936 2937 devlink = priv_to_devlink(mlxsw_sp->core); 2938 if (!devlink_dpipe_table_counter_enabled(devlink, 2939 MLXSW_SP_DPIPE_TABLE_NAME_ADJ)) 2940 return; 2941 2942 if (mlxsw_sp_flow_counter_alloc(mlxsw_sp, &nh->counter_index)) 2943 return; 2944 2945 nh->counter_valid = true; 2946 } 2947 2948 void mlxsw_sp_nexthop_counter_free(struct mlxsw_sp *mlxsw_sp, 2949 struct mlxsw_sp_nexthop *nh) 2950 { 2951 if (!nh->counter_valid) 2952 return; 2953 mlxsw_sp_flow_counter_free(mlxsw_sp, nh->counter_index); 2954 nh->counter_valid = false; 2955 } 2956 2957 int mlxsw_sp_nexthop_counter_get(struct mlxsw_sp *mlxsw_sp, 2958 struct mlxsw_sp_nexthop *nh, u64 *p_counter) 2959 { 2960 if (!nh->counter_valid) 2961 return -EINVAL; 2962 2963 return mlxsw_sp_flow_counter_get(mlxsw_sp, nh->counter_index, 2964 p_counter, NULL); 2965 } 2966 2967 struct mlxsw_sp_nexthop *mlxsw_sp_nexthop_next(struct mlxsw_sp_router *router, 2968 struct mlxsw_sp_nexthop *nh) 2969 { 2970 if (!nh) { 2971 if (list_empty(&router->nexthop_list)) 2972 return NULL; 2973 else 2974 return list_first_entry(&router->nexthop_list, 2975 typeof(*nh), router_list_node); 2976 } 2977 if (list_is_last(&nh->router_list_node, &router->nexthop_list)) 2978 return NULL; 2979 return list_next_entry(nh, router_list_node); 2980 } 2981 2982 bool mlxsw_sp_nexthop_offload(struct mlxsw_sp_nexthop *nh) 2983 { 2984 return nh->offloaded; 2985 } 2986 2987 unsigned char *mlxsw_sp_nexthop_ha(struct mlxsw_sp_nexthop *nh) 2988 { 2989 if (!nh->offloaded) 2990 return NULL; 2991 return nh->neigh_entry->ha; 2992 } 2993 2994 int mlxsw_sp_nexthop_indexes(struct mlxsw_sp_nexthop *nh, u32 *p_adj_index, 2995 u32 *p_adj_size, u32 *p_adj_hash_index) 2996 { 2997 struct mlxsw_sp_nexthop_group_info *nhgi = nh->nhgi; 2998 u32 adj_hash_index = 0; 2999 int i; 3000 3001 if (!nh->offloaded || !nhgi->adj_index_valid) 3002 return -EINVAL; 3003 3004 *p_adj_index = nhgi->adj_index; 3005 *p_adj_size = nhgi->ecmp_size; 3006 3007 for (i = 0; i < nhgi->count; i++) { 3008 struct mlxsw_sp_nexthop *nh_iter = &nhgi->nexthops[i]; 3009 3010 if (nh_iter == nh) 3011 break; 3012 if (nh_iter->offloaded) 3013 adj_hash_index += nh_iter->num_adj_entries; 3014 } 3015 3016 *p_adj_hash_index = adj_hash_index; 3017 return 0; 3018 } 3019 3020 struct mlxsw_sp_rif *mlxsw_sp_nexthop_rif(struct mlxsw_sp_nexthop *nh) 3021 { 3022 return nh->rif; 3023 } 3024 3025 bool mlxsw_sp_nexthop_group_has_ipip(struct mlxsw_sp_nexthop *nh) 3026 { 3027 struct mlxsw_sp_nexthop_group_info *nhgi = nh->nhgi; 3028 int i; 3029 3030 for (i = 0; i < nhgi->count; i++) { 3031 struct mlxsw_sp_nexthop *nh_iter = &nhgi->nexthops[i]; 3032 3033 if (nh_iter->type == MLXSW_SP_NEXTHOP_TYPE_IPIP) 3034 return true; 3035 } 3036 return false; 3037 } 3038 3039 bool mlxsw_sp_nexthop_is_discard(const struct mlxsw_sp_nexthop *nh) 3040 { 3041 return nh->discard; 3042 } 3043 3044 static const struct rhashtable_params mlxsw_sp_nexthop_group_vr_ht_params = { 3045 .key_offset = offsetof(struct mlxsw_sp_nexthop_group_vr_entry, key), 3046 .head_offset = offsetof(struct mlxsw_sp_nexthop_group_vr_entry, ht_node), 3047 .key_len = sizeof(struct mlxsw_sp_nexthop_group_vr_key), 3048 .automatic_shrinking = true, 3049 }; 3050 3051 static struct mlxsw_sp_nexthop_group_vr_entry * 3052 mlxsw_sp_nexthop_group_vr_entry_lookup(struct mlxsw_sp_nexthop_group *nh_grp, 3053 const struct mlxsw_sp_fib *fib) 3054 { 3055 struct mlxsw_sp_nexthop_group_vr_key key; 3056 3057 memset(&key, 0, sizeof(key)); 3058 key.vr_id = fib->vr->id; 3059 key.proto = fib->proto; 3060 return rhashtable_lookup_fast(&nh_grp->vr_ht, &key, 3061 mlxsw_sp_nexthop_group_vr_ht_params); 3062 } 3063 3064 static int 3065 mlxsw_sp_nexthop_group_vr_entry_create(struct mlxsw_sp_nexthop_group *nh_grp, 3066 const struct mlxsw_sp_fib *fib) 3067 { 3068 struct mlxsw_sp_nexthop_group_vr_entry *vr_entry; 3069 int err; 3070 3071 vr_entry = kzalloc(sizeof(*vr_entry), GFP_KERNEL); 3072 if (!vr_entry) 3073 return -ENOMEM; 3074 3075 vr_entry->key.vr_id = fib->vr->id; 3076 vr_entry->key.proto = fib->proto; 3077 refcount_set(&vr_entry->ref_count, 1); 3078 3079 err = rhashtable_insert_fast(&nh_grp->vr_ht, &vr_entry->ht_node, 3080 mlxsw_sp_nexthop_group_vr_ht_params); 3081 if (err) 3082 goto err_hashtable_insert; 3083 3084 list_add(&vr_entry->list, &nh_grp->vr_list); 3085 3086 return 0; 3087 3088 err_hashtable_insert: 3089 kfree(vr_entry); 3090 return err; 3091 } 3092 3093 static void 3094 mlxsw_sp_nexthop_group_vr_entry_destroy(struct mlxsw_sp_nexthop_group *nh_grp, 3095 struct mlxsw_sp_nexthop_group_vr_entry *vr_entry) 3096 { 3097 list_del(&vr_entry->list); 3098 rhashtable_remove_fast(&nh_grp->vr_ht, &vr_entry->ht_node, 3099 mlxsw_sp_nexthop_group_vr_ht_params); 3100 kfree(vr_entry); 3101 } 3102 3103 static int 3104 mlxsw_sp_nexthop_group_vr_link(struct mlxsw_sp_nexthop_group *nh_grp, 3105 const struct mlxsw_sp_fib *fib) 3106 { 3107 struct mlxsw_sp_nexthop_group_vr_entry *vr_entry; 3108 3109 vr_entry = mlxsw_sp_nexthop_group_vr_entry_lookup(nh_grp, fib); 3110 if (vr_entry) { 3111 refcount_inc(&vr_entry->ref_count); 3112 return 0; 3113 } 3114 3115 return mlxsw_sp_nexthop_group_vr_entry_create(nh_grp, fib); 3116 } 3117 3118 static void 3119 mlxsw_sp_nexthop_group_vr_unlink(struct mlxsw_sp_nexthop_group *nh_grp, 3120 const struct mlxsw_sp_fib *fib) 3121 { 3122 struct mlxsw_sp_nexthop_group_vr_entry *vr_entry; 3123 3124 vr_entry = mlxsw_sp_nexthop_group_vr_entry_lookup(nh_grp, fib); 3125 if (WARN_ON_ONCE(!vr_entry)) 3126 return; 3127 3128 if (!refcount_dec_and_test(&vr_entry->ref_count)) 3129 return; 3130 3131 mlxsw_sp_nexthop_group_vr_entry_destroy(nh_grp, vr_entry); 3132 } 3133 3134 struct mlxsw_sp_nexthop_group_cmp_arg { 3135 enum mlxsw_sp_nexthop_group_type type; 3136 union { 3137 struct fib_info *fi; 3138 struct mlxsw_sp_fib6_entry *fib6_entry; 3139 u32 id; 3140 }; 3141 }; 3142 3143 static bool 3144 mlxsw_sp_nexthop6_group_has_nexthop(const struct mlxsw_sp_nexthop_group *nh_grp, 3145 const struct in6_addr *gw, int ifindex, 3146 int weight) 3147 { 3148 int i; 3149 3150 for (i = 0; i < nh_grp->nhgi->count; i++) { 3151 const struct mlxsw_sp_nexthop *nh; 3152 3153 nh = &nh_grp->nhgi->nexthops[i]; 3154 if (nh->ifindex == ifindex && nh->nh_weight == weight && 3155 ipv6_addr_equal(gw, (struct in6_addr *) nh->gw_addr)) 3156 return true; 3157 } 3158 3159 return false; 3160 } 3161 3162 static bool 3163 mlxsw_sp_nexthop6_group_cmp(const struct mlxsw_sp_nexthop_group *nh_grp, 3164 const struct mlxsw_sp_fib6_entry *fib6_entry) 3165 { 3166 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 3167 3168 if (nh_grp->nhgi->count != fib6_entry->nrt6) 3169 return false; 3170 3171 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) { 3172 struct fib6_nh *fib6_nh = mlxsw_sp_rt6->rt->fib6_nh; 3173 struct in6_addr *gw; 3174 int ifindex, weight; 3175 3176 ifindex = fib6_nh->fib_nh_dev->ifindex; 3177 weight = fib6_nh->fib_nh_weight; 3178 gw = &fib6_nh->fib_nh_gw6; 3179 if (!mlxsw_sp_nexthop6_group_has_nexthop(nh_grp, gw, ifindex, 3180 weight)) 3181 return false; 3182 } 3183 3184 return true; 3185 } 3186 3187 static int 3188 mlxsw_sp_nexthop_group_cmp(struct rhashtable_compare_arg *arg, const void *ptr) 3189 { 3190 const struct mlxsw_sp_nexthop_group_cmp_arg *cmp_arg = arg->key; 3191 const struct mlxsw_sp_nexthop_group *nh_grp = ptr; 3192 3193 if (nh_grp->type != cmp_arg->type) 3194 return 1; 3195 3196 switch (cmp_arg->type) { 3197 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4: 3198 return cmp_arg->fi != nh_grp->ipv4.fi; 3199 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6: 3200 return !mlxsw_sp_nexthop6_group_cmp(nh_grp, 3201 cmp_arg->fib6_entry); 3202 case MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ: 3203 return cmp_arg->id != nh_grp->obj.id; 3204 default: 3205 WARN_ON(1); 3206 return 1; 3207 } 3208 } 3209 3210 static u32 mlxsw_sp_nexthop_group_hash_obj(const void *data, u32 len, u32 seed) 3211 { 3212 const struct mlxsw_sp_nexthop_group *nh_grp = data; 3213 const struct mlxsw_sp_nexthop *nh; 3214 struct fib_info *fi; 3215 unsigned int val; 3216 int i; 3217 3218 switch (nh_grp->type) { 3219 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4: 3220 fi = nh_grp->ipv4.fi; 3221 return jhash(&fi, sizeof(fi), seed); 3222 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6: 3223 val = nh_grp->nhgi->count; 3224 for (i = 0; i < nh_grp->nhgi->count; i++) { 3225 nh = &nh_grp->nhgi->nexthops[i]; 3226 val ^= jhash(&nh->ifindex, sizeof(nh->ifindex), seed); 3227 val ^= jhash(&nh->gw_addr, sizeof(nh->gw_addr), seed); 3228 } 3229 return jhash(&val, sizeof(val), seed); 3230 case MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ: 3231 return jhash(&nh_grp->obj.id, sizeof(nh_grp->obj.id), seed); 3232 default: 3233 WARN_ON(1); 3234 return 0; 3235 } 3236 } 3237 3238 static u32 3239 mlxsw_sp_nexthop6_group_hash(struct mlxsw_sp_fib6_entry *fib6_entry, u32 seed) 3240 { 3241 unsigned int val = fib6_entry->nrt6; 3242 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 3243 3244 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) { 3245 struct fib6_nh *fib6_nh = mlxsw_sp_rt6->rt->fib6_nh; 3246 struct net_device *dev = fib6_nh->fib_nh_dev; 3247 struct in6_addr *gw = &fib6_nh->fib_nh_gw6; 3248 3249 val ^= jhash(&dev->ifindex, sizeof(dev->ifindex), seed); 3250 val ^= jhash(gw, sizeof(*gw), seed); 3251 } 3252 3253 return jhash(&val, sizeof(val), seed); 3254 } 3255 3256 static u32 3257 mlxsw_sp_nexthop_group_hash(const void *data, u32 len, u32 seed) 3258 { 3259 const struct mlxsw_sp_nexthop_group_cmp_arg *cmp_arg = data; 3260 3261 switch (cmp_arg->type) { 3262 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4: 3263 return jhash(&cmp_arg->fi, sizeof(cmp_arg->fi), seed); 3264 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6: 3265 return mlxsw_sp_nexthop6_group_hash(cmp_arg->fib6_entry, seed); 3266 case MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ: 3267 return jhash(&cmp_arg->id, sizeof(cmp_arg->id), seed); 3268 default: 3269 WARN_ON(1); 3270 return 0; 3271 } 3272 } 3273 3274 static const struct rhashtable_params mlxsw_sp_nexthop_group_ht_params = { 3275 .head_offset = offsetof(struct mlxsw_sp_nexthop_group, ht_node), 3276 .hashfn = mlxsw_sp_nexthop_group_hash, 3277 .obj_hashfn = mlxsw_sp_nexthop_group_hash_obj, 3278 .obj_cmpfn = mlxsw_sp_nexthop_group_cmp, 3279 }; 3280 3281 static int mlxsw_sp_nexthop_group_insert(struct mlxsw_sp *mlxsw_sp, 3282 struct mlxsw_sp_nexthop_group *nh_grp) 3283 { 3284 if (nh_grp->type == MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6 && 3285 !nh_grp->nhgi->gateway) 3286 return 0; 3287 3288 return rhashtable_insert_fast(&mlxsw_sp->router->nexthop_group_ht, 3289 &nh_grp->ht_node, 3290 mlxsw_sp_nexthop_group_ht_params); 3291 } 3292 3293 static void mlxsw_sp_nexthop_group_remove(struct mlxsw_sp *mlxsw_sp, 3294 struct mlxsw_sp_nexthop_group *nh_grp) 3295 { 3296 if (nh_grp->type == MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6 && 3297 !nh_grp->nhgi->gateway) 3298 return; 3299 3300 rhashtable_remove_fast(&mlxsw_sp->router->nexthop_group_ht, 3301 &nh_grp->ht_node, 3302 mlxsw_sp_nexthop_group_ht_params); 3303 } 3304 3305 static struct mlxsw_sp_nexthop_group * 3306 mlxsw_sp_nexthop4_group_lookup(struct mlxsw_sp *mlxsw_sp, 3307 struct fib_info *fi) 3308 { 3309 struct mlxsw_sp_nexthop_group_cmp_arg cmp_arg; 3310 3311 cmp_arg.type = MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4; 3312 cmp_arg.fi = fi; 3313 return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_group_ht, 3314 &cmp_arg, 3315 mlxsw_sp_nexthop_group_ht_params); 3316 } 3317 3318 static struct mlxsw_sp_nexthop_group * 3319 mlxsw_sp_nexthop6_group_lookup(struct mlxsw_sp *mlxsw_sp, 3320 struct mlxsw_sp_fib6_entry *fib6_entry) 3321 { 3322 struct mlxsw_sp_nexthop_group_cmp_arg cmp_arg; 3323 3324 cmp_arg.type = MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6; 3325 cmp_arg.fib6_entry = fib6_entry; 3326 return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_group_ht, 3327 &cmp_arg, 3328 mlxsw_sp_nexthop_group_ht_params); 3329 } 3330 3331 static const struct rhashtable_params mlxsw_sp_nexthop_ht_params = { 3332 .key_offset = offsetof(struct mlxsw_sp_nexthop, key), 3333 .head_offset = offsetof(struct mlxsw_sp_nexthop, ht_node), 3334 .key_len = sizeof(struct mlxsw_sp_nexthop_key), 3335 }; 3336 3337 static int mlxsw_sp_nexthop_insert(struct mlxsw_sp *mlxsw_sp, 3338 struct mlxsw_sp_nexthop *nh) 3339 { 3340 return rhashtable_insert_fast(&mlxsw_sp->router->nexthop_ht, 3341 &nh->ht_node, mlxsw_sp_nexthop_ht_params); 3342 } 3343 3344 static void mlxsw_sp_nexthop_remove(struct mlxsw_sp *mlxsw_sp, 3345 struct mlxsw_sp_nexthop *nh) 3346 { 3347 rhashtable_remove_fast(&mlxsw_sp->router->nexthop_ht, &nh->ht_node, 3348 mlxsw_sp_nexthop_ht_params); 3349 } 3350 3351 static struct mlxsw_sp_nexthop * 3352 mlxsw_sp_nexthop_lookup(struct mlxsw_sp *mlxsw_sp, 3353 struct mlxsw_sp_nexthop_key key) 3354 { 3355 return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_ht, &key, 3356 mlxsw_sp_nexthop_ht_params); 3357 } 3358 3359 static int mlxsw_sp_adj_index_mass_update_vr(struct mlxsw_sp *mlxsw_sp, 3360 enum mlxsw_sp_l3proto proto, 3361 u16 vr_id, 3362 u32 adj_index, u16 ecmp_size, 3363 u32 new_adj_index, 3364 u16 new_ecmp_size) 3365 { 3366 char raleu_pl[MLXSW_REG_RALEU_LEN]; 3367 3368 mlxsw_reg_raleu_pack(raleu_pl, 3369 (enum mlxsw_reg_ralxx_protocol) proto, vr_id, 3370 adj_index, ecmp_size, new_adj_index, 3371 new_ecmp_size); 3372 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raleu), raleu_pl); 3373 } 3374 3375 static int mlxsw_sp_adj_index_mass_update(struct mlxsw_sp *mlxsw_sp, 3376 struct mlxsw_sp_nexthop_group *nh_grp, 3377 u32 old_adj_index, u16 old_ecmp_size) 3378 { 3379 struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi; 3380 struct mlxsw_sp_nexthop_group_vr_entry *vr_entry; 3381 int err; 3382 3383 list_for_each_entry(vr_entry, &nh_grp->vr_list, list) { 3384 err = mlxsw_sp_adj_index_mass_update_vr(mlxsw_sp, 3385 vr_entry->key.proto, 3386 vr_entry->key.vr_id, 3387 old_adj_index, 3388 old_ecmp_size, 3389 nhgi->adj_index, 3390 nhgi->ecmp_size); 3391 if (err) 3392 goto err_mass_update_vr; 3393 } 3394 return 0; 3395 3396 err_mass_update_vr: 3397 list_for_each_entry_continue_reverse(vr_entry, &nh_grp->vr_list, list) 3398 mlxsw_sp_adj_index_mass_update_vr(mlxsw_sp, vr_entry->key.proto, 3399 vr_entry->key.vr_id, 3400 nhgi->adj_index, 3401 nhgi->ecmp_size, 3402 old_adj_index, old_ecmp_size); 3403 return err; 3404 } 3405 3406 static int __mlxsw_sp_nexthop_update(struct mlxsw_sp *mlxsw_sp, u32 adj_index, 3407 struct mlxsw_sp_nexthop *nh) 3408 { 3409 struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry; 3410 char ratr_pl[MLXSW_REG_RATR_LEN]; 3411 3412 mlxsw_reg_ratr_pack(ratr_pl, MLXSW_REG_RATR_OP_WRITE_WRITE_ENTRY, 3413 true, MLXSW_REG_RATR_TYPE_ETHERNET, 3414 adj_index, nh->rif->rif_index); 3415 if (nh->discard) 3416 mlxsw_reg_ratr_trap_action_set(ratr_pl, 3417 MLXSW_REG_RATR_TRAP_ACTION_DISCARD_ERRORS); 3418 else 3419 mlxsw_reg_ratr_eth_entry_pack(ratr_pl, neigh_entry->ha); 3420 if (nh->counter_valid) 3421 mlxsw_reg_ratr_counter_pack(ratr_pl, nh->counter_index, true); 3422 else 3423 mlxsw_reg_ratr_counter_pack(ratr_pl, 0, false); 3424 3425 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ratr), ratr_pl); 3426 } 3427 3428 int mlxsw_sp_nexthop_update(struct mlxsw_sp *mlxsw_sp, u32 adj_index, 3429 struct mlxsw_sp_nexthop *nh) 3430 { 3431 int i; 3432 3433 for (i = 0; i < nh->num_adj_entries; i++) { 3434 int err; 3435 3436 err = __mlxsw_sp_nexthop_update(mlxsw_sp, adj_index + i, nh); 3437 if (err) 3438 return err; 3439 } 3440 3441 return 0; 3442 } 3443 3444 static int __mlxsw_sp_nexthop_ipip_update(struct mlxsw_sp *mlxsw_sp, 3445 u32 adj_index, 3446 struct mlxsw_sp_nexthop *nh) 3447 { 3448 const struct mlxsw_sp_ipip_ops *ipip_ops; 3449 3450 ipip_ops = mlxsw_sp->router->ipip_ops_arr[nh->ipip_entry->ipipt]; 3451 return ipip_ops->nexthop_update(mlxsw_sp, adj_index, nh->ipip_entry); 3452 } 3453 3454 static int mlxsw_sp_nexthop_ipip_update(struct mlxsw_sp *mlxsw_sp, 3455 u32 adj_index, 3456 struct mlxsw_sp_nexthop *nh) 3457 { 3458 int i; 3459 3460 for (i = 0; i < nh->num_adj_entries; i++) { 3461 int err; 3462 3463 err = __mlxsw_sp_nexthop_ipip_update(mlxsw_sp, adj_index + i, 3464 nh); 3465 if (err) 3466 return err; 3467 } 3468 3469 return 0; 3470 } 3471 3472 static int 3473 mlxsw_sp_nexthop_group_update(struct mlxsw_sp *mlxsw_sp, 3474 struct mlxsw_sp_nexthop_group_info *nhgi, 3475 bool reallocate) 3476 { 3477 u32 adj_index = nhgi->adj_index; /* base */ 3478 struct mlxsw_sp_nexthop *nh; 3479 int i; 3480 3481 for (i = 0; i < nhgi->count; i++) { 3482 nh = &nhgi->nexthops[i]; 3483 3484 if (!nh->should_offload) { 3485 nh->offloaded = 0; 3486 continue; 3487 } 3488 3489 if (nh->update || reallocate) { 3490 int err = 0; 3491 3492 switch (nh->type) { 3493 case MLXSW_SP_NEXTHOP_TYPE_ETH: 3494 err = mlxsw_sp_nexthop_update 3495 (mlxsw_sp, adj_index, nh); 3496 break; 3497 case MLXSW_SP_NEXTHOP_TYPE_IPIP: 3498 err = mlxsw_sp_nexthop_ipip_update 3499 (mlxsw_sp, adj_index, nh); 3500 break; 3501 } 3502 if (err) 3503 return err; 3504 nh->update = 0; 3505 nh->offloaded = 1; 3506 } 3507 adj_index += nh->num_adj_entries; 3508 } 3509 return 0; 3510 } 3511 3512 static int 3513 mlxsw_sp_nexthop_fib_entries_update(struct mlxsw_sp *mlxsw_sp, 3514 struct mlxsw_sp_nexthop_group *nh_grp) 3515 { 3516 struct mlxsw_sp_fib_entry *fib_entry; 3517 int err; 3518 3519 list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) { 3520 err = mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry); 3521 if (err) 3522 return err; 3523 } 3524 return 0; 3525 } 3526 3527 static void mlxsw_sp_adj_grp_size_round_up(u16 *p_adj_grp_size) 3528 { 3529 /* Valid sizes for an adjacency group are: 3530 * 1-64, 512, 1024, 2048 and 4096. 3531 */ 3532 if (*p_adj_grp_size <= 64) 3533 return; 3534 else if (*p_adj_grp_size <= 512) 3535 *p_adj_grp_size = 512; 3536 else if (*p_adj_grp_size <= 1024) 3537 *p_adj_grp_size = 1024; 3538 else if (*p_adj_grp_size <= 2048) 3539 *p_adj_grp_size = 2048; 3540 else 3541 *p_adj_grp_size = 4096; 3542 } 3543 3544 static void mlxsw_sp_adj_grp_size_round_down(u16 *p_adj_grp_size, 3545 unsigned int alloc_size) 3546 { 3547 if (alloc_size >= 4096) 3548 *p_adj_grp_size = 4096; 3549 else if (alloc_size >= 2048) 3550 *p_adj_grp_size = 2048; 3551 else if (alloc_size >= 1024) 3552 *p_adj_grp_size = 1024; 3553 else if (alloc_size >= 512) 3554 *p_adj_grp_size = 512; 3555 } 3556 3557 static int mlxsw_sp_fix_adj_grp_size(struct mlxsw_sp *mlxsw_sp, 3558 u16 *p_adj_grp_size) 3559 { 3560 unsigned int alloc_size; 3561 int err; 3562 3563 /* Round up the requested group size to the next size supported 3564 * by the device and make sure the request can be satisfied. 3565 */ 3566 mlxsw_sp_adj_grp_size_round_up(p_adj_grp_size); 3567 err = mlxsw_sp_kvdl_alloc_count_query(mlxsw_sp, 3568 MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 3569 *p_adj_grp_size, &alloc_size); 3570 if (err) 3571 return err; 3572 /* It is possible the allocation results in more allocated 3573 * entries than requested. Try to use as much of them as 3574 * possible. 3575 */ 3576 mlxsw_sp_adj_grp_size_round_down(p_adj_grp_size, alloc_size); 3577 3578 return 0; 3579 } 3580 3581 static void 3582 mlxsw_sp_nexthop_group_normalize(struct mlxsw_sp_nexthop_group_info *nhgi) 3583 { 3584 int i, g = 0, sum_norm_weight = 0; 3585 struct mlxsw_sp_nexthop *nh; 3586 3587 for (i = 0; i < nhgi->count; i++) { 3588 nh = &nhgi->nexthops[i]; 3589 3590 if (!nh->should_offload) 3591 continue; 3592 if (g > 0) 3593 g = gcd(nh->nh_weight, g); 3594 else 3595 g = nh->nh_weight; 3596 } 3597 3598 for (i = 0; i < nhgi->count; i++) { 3599 nh = &nhgi->nexthops[i]; 3600 3601 if (!nh->should_offload) 3602 continue; 3603 nh->norm_nh_weight = nh->nh_weight / g; 3604 sum_norm_weight += nh->norm_nh_weight; 3605 } 3606 3607 nhgi->sum_norm_weight = sum_norm_weight; 3608 } 3609 3610 static void 3611 mlxsw_sp_nexthop_group_rebalance(struct mlxsw_sp_nexthop_group_info *nhgi) 3612 { 3613 int i, weight = 0, lower_bound = 0; 3614 int total = nhgi->sum_norm_weight; 3615 u16 ecmp_size = nhgi->ecmp_size; 3616 3617 for (i = 0; i < nhgi->count; i++) { 3618 struct mlxsw_sp_nexthop *nh = &nhgi->nexthops[i]; 3619 int upper_bound; 3620 3621 if (!nh->should_offload) 3622 continue; 3623 weight += nh->norm_nh_weight; 3624 upper_bound = DIV_ROUND_CLOSEST(ecmp_size * weight, total); 3625 nh->num_adj_entries = upper_bound - lower_bound; 3626 lower_bound = upper_bound; 3627 } 3628 } 3629 3630 static struct mlxsw_sp_nexthop * 3631 mlxsw_sp_rt6_nexthop(struct mlxsw_sp_nexthop_group *nh_grp, 3632 const struct mlxsw_sp_rt6 *mlxsw_sp_rt6); 3633 3634 static void 3635 mlxsw_sp_nexthop4_group_offload_refresh(struct mlxsw_sp *mlxsw_sp, 3636 struct mlxsw_sp_nexthop_group *nh_grp) 3637 { 3638 int i; 3639 3640 for (i = 0; i < nh_grp->nhgi->count; i++) { 3641 struct mlxsw_sp_nexthop *nh = &nh_grp->nhgi->nexthops[i]; 3642 3643 if (nh->offloaded) 3644 nh->key.fib_nh->fib_nh_flags |= RTNH_F_OFFLOAD; 3645 else 3646 nh->key.fib_nh->fib_nh_flags &= ~RTNH_F_OFFLOAD; 3647 } 3648 } 3649 3650 static void 3651 __mlxsw_sp_nexthop6_group_offload_refresh(struct mlxsw_sp_nexthop_group *nh_grp, 3652 struct mlxsw_sp_fib6_entry *fib6_entry) 3653 { 3654 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 3655 3656 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) { 3657 struct fib6_nh *fib6_nh = mlxsw_sp_rt6->rt->fib6_nh; 3658 struct mlxsw_sp_nexthop *nh; 3659 3660 nh = mlxsw_sp_rt6_nexthop(nh_grp, mlxsw_sp_rt6); 3661 if (nh && nh->offloaded) 3662 fib6_nh->fib_nh_flags |= RTNH_F_OFFLOAD; 3663 else 3664 fib6_nh->fib_nh_flags &= ~RTNH_F_OFFLOAD; 3665 } 3666 } 3667 3668 static void 3669 mlxsw_sp_nexthop6_group_offload_refresh(struct mlxsw_sp *mlxsw_sp, 3670 struct mlxsw_sp_nexthop_group *nh_grp) 3671 { 3672 struct mlxsw_sp_fib6_entry *fib6_entry; 3673 3674 /* Unfortunately, in IPv6 the route and the nexthop are described by 3675 * the same struct, so we need to iterate over all the routes using the 3676 * nexthop group and set / clear the offload indication for them. 3677 */ 3678 list_for_each_entry(fib6_entry, &nh_grp->fib_list, 3679 common.nexthop_group_node) 3680 __mlxsw_sp_nexthop6_group_offload_refresh(nh_grp, fib6_entry); 3681 } 3682 3683 static void 3684 mlxsw_sp_nexthop_obj_group_offload_refresh(struct mlxsw_sp *mlxsw_sp, 3685 struct mlxsw_sp_nexthop_group *nh_grp) 3686 { 3687 /* Do not update the flags if the nexthop group is being destroyed 3688 * since: 3689 * 1. The nexthop objects is being deleted, in which case the flags are 3690 * irrelevant. 3691 * 2. The nexthop group was replaced by a newer group, in which case 3692 * the flags of the nexthop object were already updated based on the 3693 * new group. 3694 */ 3695 if (nh_grp->can_destroy) 3696 return; 3697 3698 nexthop_set_hw_flags(mlxsw_sp_net(mlxsw_sp), nh_grp->obj.id, 3699 nh_grp->nhgi->adj_index_valid, false); 3700 } 3701 3702 static void 3703 mlxsw_sp_nexthop_group_offload_refresh(struct mlxsw_sp *mlxsw_sp, 3704 struct mlxsw_sp_nexthop_group *nh_grp) 3705 { 3706 switch (nh_grp->type) { 3707 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4: 3708 mlxsw_sp_nexthop4_group_offload_refresh(mlxsw_sp, nh_grp); 3709 break; 3710 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6: 3711 mlxsw_sp_nexthop6_group_offload_refresh(mlxsw_sp, nh_grp); 3712 break; 3713 case MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ: 3714 mlxsw_sp_nexthop_obj_group_offload_refresh(mlxsw_sp, nh_grp); 3715 break; 3716 } 3717 } 3718 3719 static int 3720 mlxsw_sp_nexthop_group_refresh(struct mlxsw_sp *mlxsw_sp, 3721 struct mlxsw_sp_nexthop_group *nh_grp) 3722 { 3723 struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi; 3724 u16 ecmp_size, old_ecmp_size; 3725 struct mlxsw_sp_nexthop *nh; 3726 bool offload_change = false; 3727 u32 adj_index; 3728 bool old_adj_index_valid; 3729 int i, err2, err = 0; 3730 u32 old_adj_index; 3731 3732 if (!nhgi->gateway) 3733 return mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp); 3734 3735 for (i = 0; i < nhgi->count; i++) { 3736 nh = &nhgi->nexthops[i]; 3737 3738 if (nh->should_offload != nh->offloaded) { 3739 offload_change = true; 3740 if (nh->should_offload) 3741 nh->update = 1; 3742 } 3743 } 3744 if (!offload_change) { 3745 /* Nothing was added or removed, so no need to reallocate. Just 3746 * update MAC on existing adjacency indexes. 3747 */ 3748 err = mlxsw_sp_nexthop_group_update(mlxsw_sp, nhgi, false); 3749 if (err) { 3750 dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n"); 3751 goto set_trap; 3752 } 3753 return 0; 3754 } 3755 mlxsw_sp_nexthop_group_normalize(nhgi); 3756 if (!nhgi->sum_norm_weight) 3757 /* No neigh of this group is connected so we just set 3758 * the trap and let everthing flow through kernel. 3759 */ 3760 goto set_trap; 3761 3762 ecmp_size = nhgi->sum_norm_weight; 3763 err = mlxsw_sp_fix_adj_grp_size(mlxsw_sp, &ecmp_size); 3764 if (err) 3765 /* No valid allocation size available. */ 3766 goto set_trap; 3767 3768 err = mlxsw_sp_kvdl_alloc(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 3769 ecmp_size, &adj_index); 3770 if (err) { 3771 /* We ran out of KVD linear space, just set the 3772 * trap and let everything flow through kernel. 3773 */ 3774 dev_warn(mlxsw_sp->bus_info->dev, "Failed to allocate KVD linear area for nexthop group.\n"); 3775 goto set_trap; 3776 } 3777 old_adj_index_valid = nhgi->adj_index_valid; 3778 old_adj_index = nhgi->adj_index; 3779 old_ecmp_size = nhgi->ecmp_size; 3780 nhgi->adj_index_valid = 1; 3781 nhgi->adj_index = adj_index; 3782 nhgi->ecmp_size = ecmp_size; 3783 mlxsw_sp_nexthop_group_rebalance(nhgi); 3784 err = mlxsw_sp_nexthop_group_update(mlxsw_sp, nhgi, true); 3785 if (err) { 3786 dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n"); 3787 goto set_trap; 3788 } 3789 3790 mlxsw_sp_nexthop_group_offload_refresh(mlxsw_sp, nh_grp); 3791 3792 if (!old_adj_index_valid) { 3793 /* The trap was set for fib entries, so we have to call 3794 * fib entry update to unset it and use adjacency index. 3795 */ 3796 err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp); 3797 if (err) { 3798 dev_warn(mlxsw_sp->bus_info->dev, "Failed to add adjacency index to fib entries.\n"); 3799 goto set_trap; 3800 } 3801 return 0; 3802 } 3803 3804 err = mlxsw_sp_adj_index_mass_update(mlxsw_sp, nh_grp, 3805 old_adj_index, old_ecmp_size); 3806 mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 3807 old_ecmp_size, old_adj_index); 3808 if (err) { 3809 dev_warn(mlxsw_sp->bus_info->dev, "Failed to mass-update adjacency index for nexthop group.\n"); 3810 goto set_trap; 3811 } 3812 3813 return 0; 3814 3815 set_trap: 3816 old_adj_index_valid = nhgi->adj_index_valid; 3817 nhgi->adj_index_valid = 0; 3818 for (i = 0; i < nhgi->count; i++) { 3819 nh = &nhgi->nexthops[i]; 3820 nh->offloaded = 0; 3821 } 3822 err2 = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp); 3823 if (err2) 3824 dev_warn(mlxsw_sp->bus_info->dev, "Failed to set traps for fib entries.\n"); 3825 mlxsw_sp_nexthop_group_offload_refresh(mlxsw_sp, nh_grp); 3826 if (old_adj_index_valid) 3827 mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 3828 nhgi->ecmp_size, nhgi->adj_index); 3829 return err; 3830 } 3831 3832 static void __mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp_nexthop *nh, 3833 bool removing) 3834 { 3835 if (!removing) 3836 nh->should_offload = 1; 3837 else 3838 nh->should_offload = 0; 3839 nh->update = 1; 3840 } 3841 3842 static int 3843 mlxsw_sp_nexthop_dead_neigh_replace(struct mlxsw_sp *mlxsw_sp, 3844 struct mlxsw_sp_neigh_entry *neigh_entry) 3845 { 3846 struct neighbour *n, *old_n = neigh_entry->key.n; 3847 struct mlxsw_sp_nexthop *nh; 3848 bool entry_connected; 3849 u8 nud_state, dead; 3850 int err; 3851 3852 nh = list_first_entry(&neigh_entry->nexthop_list, 3853 struct mlxsw_sp_nexthop, neigh_list_node); 3854 3855 n = neigh_lookup(nh->neigh_tbl, &nh->gw_addr, nh->rif->dev); 3856 if (!n) { 3857 n = neigh_create(nh->neigh_tbl, &nh->gw_addr, nh->rif->dev); 3858 if (IS_ERR(n)) 3859 return PTR_ERR(n); 3860 neigh_event_send(n, NULL); 3861 } 3862 3863 mlxsw_sp_neigh_entry_remove(mlxsw_sp, neigh_entry); 3864 neigh_entry->key.n = n; 3865 err = mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry); 3866 if (err) 3867 goto err_neigh_entry_insert; 3868 3869 read_lock_bh(&n->lock); 3870 nud_state = n->nud_state; 3871 dead = n->dead; 3872 read_unlock_bh(&n->lock); 3873 entry_connected = nud_state & NUD_VALID && !dead; 3874 3875 list_for_each_entry(nh, &neigh_entry->nexthop_list, 3876 neigh_list_node) { 3877 neigh_release(old_n); 3878 neigh_clone(n); 3879 __mlxsw_sp_nexthop_neigh_update(nh, !entry_connected); 3880 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp); 3881 } 3882 3883 neigh_release(n); 3884 3885 return 0; 3886 3887 err_neigh_entry_insert: 3888 neigh_entry->key.n = old_n; 3889 mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry); 3890 neigh_release(n); 3891 return err; 3892 } 3893 3894 static void 3895 mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp, 3896 struct mlxsw_sp_neigh_entry *neigh_entry, 3897 bool removing, bool dead) 3898 { 3899 struct mlxsw_sp_nexthop *nh; 3900 3901 if (list_empty(&neigh_entry->nexthop_list)) 3902 return; 3903 3904 if (dead) { 3905 int err; 3906 3907 err = mlxsw_sp_nexthop_dead_neigh_replace(mlxsw_sp, 3908 neigh_entry); 3909 if (err) 3910 dev_err(mlxsw_sp->bus_info->dev, "Failed to replace dead neigh\n"); 3911 return; 3912 } 3913 3914 list_for_each_entry(nh, &neigh_entry->nexthop_list, 3915 neigh_list_node) { 3916 __mlxsw_sp_nexthop_neigh_update(nh, removing); 3917 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp); 3918 } 3919 } 3920 3921 static void mlxsw_sp_nexthop_rif_init(struct mlxsw_sp_nexthop *nh, 3922 struct mlxsw_sp_rif *rif) 3923 { 3924 if (nh->rif) 3925 return; 3926 3927 nh->rif = rif; 3928 list_add(&nh->rif_list_node, &rif->nexthop_list); 3929 } 3930 3931 static void mlxsw_sp_nexthop_rif_fini(struct mlxsw_sp_nexthop *nh) 3932 { 3933 if (!nh->rif) 3934 return; 3935 3936 list_del(&nh->rif_list_node); 3937 nh->rif = NULL; 3938 } 3939 3940 static int mlxsw_sp_nexthop_neigh_init(struct mlxsw_sp *mlxsw_sp, 3941 struct mlxsw_sp_nexthop *nh) 3942 { 3943 struct mlxsw_sp_neigh_entry *neigh_entry; 3944 struct neighbour *n; 3945 u8 nud_state, dead; 3946 int err; 3947 3948 if (!nh->nhgi->gateway || nh->neigh_entry) 3949 return 0; 3950 3951 /* Take a reference of neigh here ensuring that neigh would 3952 * not be destructed before the nexthop entry is finished. 3953 * The reference is taken either in neigh_lookup() or 3954 * in neigh_create() in case n is not found. 3955 */ 3956 n = neigh_lookup(nh->neigh_tbl, &nh->gw_addr, nh->rif->dev); 3957 if (!n) { 3958 n = neigh_create(nh->neigh_tbl, &nh->gw_addr, nh->rif->dev); 3959 if (IS_ERR(n)) 3960 return PTR_ERR(n); 3961 neigh_event_send(n, NULL); 3962 } 3963 neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n); 3964 if (!neigh_entry) { 3965 neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n); 3966 if (IS_ERR(neigh_entry)) { 3967 err = -EINVAL; 3968 goto err_neigh_entry_create; 3969 } 3970 } 3971 3972 /* If that is the first nexthop connected to that neigh, add to 3973 * nexthop_neighs_list 3974 */ 3975 if (list_empty(&neigh_entry->nexthop_list)) 3976 list_add_tail(&neigh_entry->nexthop_neighs_list_node, 3977 &mlxsw_sp->router->nexthop_neighs_list); 3978 3979 nh->neigh_entry = neigh_entry; 3980 list_add_tail(&nh->neigh_list_node, &neigh_entry->nexthop_list); 3981 read_lock_bh(&n->lock); 3982 nud_state = n->nud_state; 3983 dead = n->dead; 3984 read_unlock_bh(&n->lock); 3985 __mlxsw_sp_nexthop_neigh_update(nh, !(nud_state & NUD_VALID && !dead)); 3986 3987 return 0; 3988 3989 err_neigh_entry_create: 3990 neigh_release(n); 3991 return err; 3992 } 3993 3994 static void mlxsw_sp_nexthop_neigh_fini(struct mlxsw_sp *mlxsw_sp, 3995 struct mlxsw_sp_nexthop *nh) 3996 { 3997 struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry; 3998 struct neighbour *n; 3999 4000 if (!neigh_entry) 4001 return; 4002 n = neigh_entry->key.n; 4003 4004 __mlxsw_sp_nexthop_neigh_update(nh, true); 4005 list_del(&nh->neigh_list_node); 4006 nh->neigh_entry = NULL; 4007 4008 /* If that is the last nexthop connected to that neigh, remove from 4009 * nexthop_neighs_list 4010 */ 4011 if (list_empty(&neigh_entry->nexthop_list)) 4012 list_del(&neigh_entry->nexthop_neighs_list_node); 4013 4014 if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list)) 4015 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry); 4016 4017 neigh_release(n); 4018 } 4019 4020 static bool mlxsw_sp_ipip_netdev_ul_up(struct net_device *ol_dev) 4021 { 4022 struct net_device *ul_dev; 4023 bool is_up; 4024 4025 rcu_read_lock(); 4026 ul_dev = __mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev); 4027 is_up = ul_dev ? (ul_dev->flags & IFF_UP) : true; 4028 rcu_read_unlock(); 4029 4030 return is_up; 4031 } 4032 4033 static void mlxsw_sp_nexthop_ipip_init(struct mlxsw_sp *mlxsw_sp, 4034 struct mlxsw_sp_nexthop *nh, 4035 struct mlxsw_sp_ipip_entry *ipip_entry) 4036 { 4037 bool removing; 4038 4039 if (!nh->nhgi->gateway || nh->ipip_entry) 4040 return; 4041 4042 nh->ipip_entry = ipip_entry; 4043 removing = !mlxsw_sp_ipip_netdev_ul_up(ipip_entry->ol_dev); 4044 __mlxsw_sp_nexthop_neigh_update(nh, removing); 4045 mlxsw_sp_nexthop_rif_init(nh, &ipip_entry->ol_lb->common); 4046 } 4047 4048 static void mlxsw_sp_nexthop_ipip_fini(struct mlxsw_sp *mlxsw_sp, 4049 struct mlxsw_sp_nexthop *nh) 4050 { 4051 struct mlxsw_sp_ipip_entry *ipip_entry = nh->ipip_entry; 4052 4053 if (!ipip_entry) 4054 return; 4055 4056 __mlxsw_sp_nexthop_neigh_update(nh, true); 4057 nh->ipip_entry = NULL; 4058 } 4059 4060 static bool mlxsw_sp_nexthop4_ipip_type(const struct mlxsw_sp *mlxsw_sp, 4061 const struct fib_nh *fib_nh, 4062 enum mlxsw_sp_ipip_type *p_ipipt) 4063 { 4064 struct net_device *dev = fib_nh->fib_nh_dev; 4065 4066 return dev && 4067 fib_nh->nh_parent->fib_type == RTN_UNICAST && 4068 mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, p_ipipt); 4069 } 4070 4071 static int mlxsw_sp_nexthop_type_init(struct mlxsw_sp *mlxsw_sp, 4072 struct mlxsw_sp_nexthop *nh, 4073 const struct net_device *dev) 4074 { 4075 const struct mlxsw_sp_ipip_ops *ipip_ops; 4076 struct mlxsw_sp_ipip_entry *ipip_entry; 4077 struct mlxsw_sp_rif *rif; 4078 int err; 4079 4080 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, dev); 4081 if (ipip_entry) { 4082 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt]; 4083 if (ipip_ops->can_offload(mlxsw_sp, dev)) { 4084 nh->type = MLXSW_SP_NEXTHOP_TYPE_IPIP; 4085 mlxsw_sp_nexthop_ipip_init(mlxsw_sp, nh, ipip_entry); 4086 return 0; 4087 } 4088 } 4089 4090 nh->type = MLXSW_SP_NEXTHOP_TYPE_ETH; 4091 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 4092 if (!rif) 4093 return 0; 4094 4095 mlxsw_sp_nexthop_rif_init(nh, rif); 4096 err = mlxsw_sp_nexthop_neigh_init(mlxsw_sp, nh); 4097 if (err) 4098 goto err_neigh_init; 4099 4100 return 0; 4101 4102 err_neigh_init: 4103 mlxsw_sp_nexthop_rif_fini(nh); 4104 return err; 4105 } 4106 4107 static void mlxsw_sp_nexthop_type_fini(struct mlxsw_sp *mlxsw_sp, 4108 struct mlxsw_sp_nexthop *nh) 4109 { 4110 switch (nh->type) { 4111 case MLXSW_SP_NEXTHOP_TYPE_ETH: 4112 mlxsw_sp_nexthop_neigh_fini(mlxsw_sp, nh); 4113 mlxsw_sp_nexthop_rif_fini(nh); 4114 break; 4115 case MLXSW_SP_NEXTHOP_TYPE_IPIP: 4116 mlxsw_sp_nexthop_rif_fini(nh); 4117 mlxsw_sp_nexthop_ipip_fini(mlxsw_sp, nh); 4118 break; 4119 } 4120 } 4121 4122 static int mlxsw_sp_nexthop4_init(struct mlxsw_sp *mlxsw_sp, 4123 struct mlxsw_sp_nexthop_group *nh_grp, 4124 struct mlxsw_sp_nexthop *nh, 4125 struct fib_nh *fib_nh) 4126 { 4127 struct net_device *dev = fib_nh->fib_nh_dev; 4128 struct in_device *in_dev; 4129 int err; 4130 4131 nh->nhgi = nh_grp->nhgi; 4132 nh->key.fib_nh = fib_nh; 4133 #ifdef CONFIG_IP_ROUTE_MULTIPATH 4134 nh->nh_weight = fib_nh->fib_nh_weight; 4135 #else 4136 nh->nh_weight = 1; 4137 #endif 4138 memcpy(&nh->gw_addr, &fib_nh->fib_nh_gw4, sizeof(fib_nh->fib_nh_gw4)); 4139 nh->neigh_tbl = &arp_tbl; 4140 err = mlxsw_sp_nexthop_insert(mlxsw_sp, nh); 4141 if (err) 4142 return err; 4143 4144 mlxsw_sp_nexthop_counter_alloc(mlxsw_sp, nh); 4145 list_add_tail(&nh->router_list_node, &mlxsw_sp->router->nexthop_list); 4146 4147 if (!dev) 4148 return 0; 4149 nh->ifindex = dev->ifindex; 4150 4151 rcu_read_lock(); 4152 in_dev = __in_dev_get_rcu(dev); 4153 if (in_dev && IN_DEV_IGNORE_ROUTES_WITH_LINKDOWN(in_dev) && 4154 fib_nh->fib_nh_flags & RTNH_F_LINKDOWN) { 4155 rcu_read_unlock(); 4156 return 0; 4157 } 4158 rcu_read_unlock(); 4159 4160 err = mlxsw_sp_nexthop_type_init(mlxsw_sp, nh, dev); 4161 if (err) 4162 goto err_nexthop_neigh_init; 4163 4164 return 0; 4165 4166 err_nexthop_neigh_init: 4167 mlxsw_sp_nexthop_remove(mlxsw_sp, nh); 4168 return err; 4169 } 4170 4171 static void mlxsw_sp_nexthop4_fini(struct mlxsw_sp *mlxsw_sp, 4172 struct mlxsw_sp_nexthop *nh) 4173 { 4174 mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh); 4175 list_del(&nh->router_list_node); 4176 mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh); 4177 mlxsw_sp_nexthop_remove(mlxsw_sp, nh); 4178 } 4179 4180 static void mlxsw_sp_nexthop4_event(struct mlxsw_sp *mlxsw_sp, 4181 unsigned long event, struct fib_nh *fib_nh) 4182 { 4183 struct mlxsw_sp_nexthop_key key; 4184 struct mlxsw_sp_nexthop *nh; 4185 4186 if (mlxsw_sp->router->aborted) 4187 return; 4188 4189 key.fib_nh = fib_nh; 4190 nh = mlxsw_sp_nexthop_lookup(mlxsw_sp, key); 4191 if (!nh) 4192 return; 4193 4194 switch (event) { 4195 case FIB_EVENT_NH_ADD: 4196 mlxsw_sp_nexthop_type_init(mlxsw_sp, nh, fib_nh->fib_nh_dev); 4197 break; 4198 case FIB_EVENT_NH_DEL: 4199 mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh); 4200 break; 4201 } 4202 4203 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp); 4204 } 4205 4206 static void mlxsw_sp_nexthop_rif_update(struct mlxsw_sp *mlxsw_sp, 4207 struct mlxsw_sp_rif *rif) 4208 { 4209 struct mlxsw_sp_nexthop *nh; 4210 bool removing; 4211 4212 list_for_each_entry(nh, &rif->nexthop_list, rif_list_node) { 4213 switch (nh->type) { 4214 case MLXSW_SP_NEXTHOP_TYPE_ETH: 4215 removing = false; 4216 break; 4217 case MLXSW_SP_NEXTHOP_TYPE_IPIP: 4218 removing = !mlxsw_sp_ipip_netdev_ul_up(rif->dev); 4219 break; 4220 default: 4221 WARN_ON(1); 4222 continue; 4223 } 4224 4225 __mlxsw_sp_nexthop_neigh_update(nh, removing); 4226 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp); 4227 } 4228 } 4229 4230 static void mlxsw_sp_nexthop_rif_migrate(struct mlxsw_sp *mlxsw_sp, 4231 struct mlxsw_sp_rif *old_rif, 4232 struct mlxsw_sp_rif *new_rif) 4233 { 4234 struct mlxsw_sp_nexthop *nh; 4235 4236 list_splice_init(&old_rif->nexthop_list, &new_rif->nexthop_list); 4237 list_for_each_entry(nh, &new_rif->nexthop_list, rif_list_node) 4238 nh->rif = new_rif; 4239 mlxsw_sp_nexthop_rif_update(mlxsw_sp, new_rif); 4240 } 4241 4242 static void mlxsw_sp_nexthop_rif_gone_sync(struct mlxsw_sp *mlxsw_sp, 4243 struct mlxsw_sp_rif *rif) 4244 { 4245 struct mlxsw_sp_nexthop *nh, *tmp; 4246 4247 list_for_each_entry_safe(nh, tmp, &rif->nexthop_list, rif_list_node) { 4248 mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh); 4249 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp); 4250 } 4251 } 4252 4253 static int 4254 mlxsw_sp_nexthop_obj_single_validate(struct mlxsw_sp *mlxsw_sp, 4255 const struct nh_notifier_single_info *nh, 4256 struct netlink_ext_ack *extack) 4257 { 4258 int err = -EINVAL; 4259 4260 if (nh->is_fdb) 4261 NL_SET_ERR_MSG_MOD(extack, "FDB nexthops are not supported"); 4262 else if (nh->has_encap) 4263 NL_SET_ERR_MSG_MOD(extack, "Encapsulating nexthops are not supported"); 4264 else 4265 err = 0; 4266 4267 return err; 4268 } 4269 4270 static int 4271 mlxsw_sp_nexthop_obj_group_validate(struct mlxsw_sp *mlxsw_sp, 4272 const struct nh_notifier_grp_info *nh_grp, 4273 struct netlink_ext_ack *extack) 4274 { 4275 int i; 4276 4277 if (nh_grp->is_fdb) { 4278 NL_SET_ERR_MSG_MOD(extack, "FDB nexthop groups are not supported"); 4279 return -EINVAL; 4280 } 4281 4282 for (i = 0; i < nh_grp->num_nh; i++) { 4283 const struct nh_notifier_single_info *nh; 4284 int err; 4285 4286 nh = &nh_grp->nh_entries[i].nh; 4287 err = mlxsw_sp_nexthop_obj_single_validate(mlxsw_sp, nh, 4288 extack); 4289 if (err) 4290 return err; 4291 4292 /* Device only nexthops with an IPIP device are programmed as 4293 * encapsulating adjacency entries. 4294 */ 4295 if (!nh->gw_family && !nh->is_reject && 4296 !mlxsw_sp_netdev_ipip_type(mlxsw_sp, nh->dev, NULL)) { 4297 NL_SET_ERR_MSG_MOD(extack, "Nexthop group entry does not have a gateway"); 4298 return -EINVAL; 4299 } 4300 } 4301 4302 return 0; 4303 } 4304 4305 static int mlxsw_sp_nexthop_obj_validate(struct mlxsw_sp *mlxsw_sp, 4306 unsigned long event, 4307 struct nh_notifier_info *info) 4308 { 4309 if (event != NEXTHOP_EVENT_REPLACE) 4310 return 0; 4311 4312 if (!info->is_grp) 4313 return mlxsw_sp_nexthop_obj_single_validate(mlxsw_sp, info->nh, 4314 info->extack); 4315 return mlxsw_sp_nexthop_obj_group_validate(mlxsw_sp, info->nh_grp, 4316 info->extack); 4317 } 4318 4319 static bool mlxsw_sp_nexthop_obj_is_gateway(struct mlxsw_sp *mlxsw_sp, 4320 const struct nh_notifier_info *info) 4321 { 4322 const struct net_device *dev; 4323 4324 if (info->is_grp) 4325 /* Already validated earlier. */ 4326 return true; 4327 4328 dev = info->nh->dev; 4329 return info->nh->gw_family || info->nh->is_reject || 4330 mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, NULL); 4331 } 4332 4333 static void mlxsw_sp_nexthop_obj_blackhole_init(struct mlxsw_sp *mlxsw_sp, 4334 struct mlxsw_sp_nexthop *nh) 4335 { 4336 u16 lb_rif_index = mlxsw_sp->router->lb_rif_index; 4337 4338 nh->discard = 1; 4339 nh->should_offload = 1; 4340 /* While nexthops that discard packets do not forward packets 4341 * via an egress RIF, they still need to be programmed using a 4342 * valid RIF, so use the loopback RIF created during init. 4343 */ 4344 nh->rif = mlxsw_sp->router->rifs[lb_rif_index]; 4345 } 4346 4347 static void mlxsw_sp_nexthop_obj_blackhole_fini(struct mlxsw_sp *mlxsw_sp, 4348 struct mlxsw_sp_nexthop *nh) 4349 { 4350 nh->rif = NULL; 4351 nh->should_offload = 0; 4352 } 4353 4354 static int 4355 mlxsw_sp_nexthop_obj_init(struct mlxsw_sp *mlxsw_sp, 4356 struct mlxsw_sp_nexthop_group *nh_grp, 4357 struct mlxsw_sp_nexthop *nh, 4358 struct nh_notifier_single_info *nh_obj, int weight) 4359 { 4360 struct net_device *dev = nh_obj->dev; 4361 int err; 4362 4363 nh->nhgi = nh_grp->nhgi; 4364 nh->nh_weight = weight; 4365 4366 switch (nh_obj->gw_family) { 4367 case AF_INET: 4368 memcpy(&nh->gw_addr, &nh_obj->ipv4, sizeof(nh_obj->ipv4)); 4369 nh->neigh_tbl = &arp_tbl; 4370 break; 4371 case AF_INET6: 4372 memcpy(&nh->gw_addr, &nh_obj->ipv6, sizeof(nh_obj->ipv6)); 4373 #if IS_ENABLED(CONFIG_IPV6) 4374 nh->neigh_tbl = &nd_tbl; 4375 #endif 4376 break; 4377 } 4378 4379 mlxsw_sp_nexthop_counter_alloc(mlxsw_sp, nh); 4380 list_add_tail(&nh->router_list_node, &mlxsw_sp->router->nexthop_list); 4381 nh->ifindex = dev->ifindex; 4382 4383 err = mlxsw_sp_nexthop_type_init(mlxsw_sp, nh, dev); 4384 if (err) 4385 goto err_type_init; 4386 4387 if (nh_obj->is_reject) 4388 mlxsw_sp_nexthop_obj_blackhole_init(mlxsw_sp, nh); 4389 4390 return 0; 4391 4392 err_type_init: 4393 list_del(&nh->router_list_node); 4394 mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh); 4395 return err; 4396 } 4397 4398 static void mlxsw_sp_nexthop_obj_fini(struct mlxsw_sp *mlxsw_sp, 4399 struct mlxsw_sp_nexthop *nh) 4400 { 4401 if (nh->discard) 4402 mlxsw_sp_nexthop_obj_blackhole_fini(mlxsw_sp, nh); 4403 mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh); 4404 list_del(&nh->router_list_node); 4405 mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh); 4406 } 4407 4408 static int 4409 mlxsw_sp_nexthop_obj_group_info_init(struct mlxsw_sp *mlxsw_sp, 4410 struct mlxsw_sp_nexthop_group *nh_grp, 4411 struct nh_notifier_info *info) 4412 { 4413 unsigned int nhs = info->is_grp ? info->nh_grp->num_nh : 1; 4414 struct mlxsw_sp_nexthop_group_info *nhgi; 4415 struct mlxsw_sp_nexthop *nh; 4416 int err, i; 4417 4418 nhgi = kzalloc(struct_size(nhgi, nexthops, nhs), GFP_KERNEL); 4419 if (!nhgi) 4420 return -ENOMEM; 4421 nh_grp->nhgi = nhgi; 4422 nhgi->nh_grp = nh_grp; 4423 nhgi->gateway = mlxsw_sp_nexthop_obj_is_gateway(mlxsw_sp, info); 4424 nhgi->count = nhs; 4425 for (i = 0; i < nhgi->count; i++) { 4426 struct nh_notifier_single_info *nh_obj; 4427 int weight; 4428 4429 nh = &nhgi->nexthops[i]; 4430 if (info->is_grp) { 4431 nh_obj = &info->nh_grp->nh_entries[i].nh; 4432 weight = info->nh_grp->nh_entries[i].weight; 4433 } else { 4434 nh_obj = info->nh; 4435 weight = 1; 4436 } 4437 err = mlxsw_sp_nexthop_obj_init(mlxsw_sp, nh_grp, nh, nh_obj, 4438 weight); 4439 if (err) 4440 goto err_nexthop_obj_init; 4441 } 4442 err = mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp); 4443 if (err) { 4444 NL_SET_ERR_MSG_MOD(info->extack, "Failed to write adjacency entries to the device"); 4445 goto err_group_refresh; 4446 } 4447 4448 return 0; 4449 4450 err_group_refresh: 4451 i = nhgi->count; 4452 err_nexthop_obj_init: 4453 for (i--; i >= 0; i--) { 4454 nh = &nhgi->nexthops[i]; 4455 mlxsw_sp_nexthop_obj_fini(mlxsw_sp, nh); 4456 } 4457 kfree(nhgi); 4458 return err; 4459 } 4460 4461 static void 4462 mlxsw_sp_nexthop_obj_group_info_fini(struct mlxsw_sp *mlxsw_sp, 4463 struct mlxsw_sp_nexthop_group *nh_grp) 4464 { 4465 struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi; 4466 int i; 4467 4468 for (i = nhgi->count - 1; i >= 0; i--) { 4469 struct mlxsw_sp_nexthop *nh = &nhgi->nexthops[i]; 4470 4471 mlxsw_sp_nexthop_obj_fini(mlxsw_sp, nh); 4472 } 4473 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp); 4474 WARN_ON_ONCE(nhgi->adj_index_valid); 4475 kfree(nhgi); 4476 } 4477 4478 static struct mlxsw_sp_nexthop_group * 4479 mlxsw_sp_nexthop_obj_group_create(struct mlxsw_sp *mlxsw_sp, 4480 struct nh_notifier_info *info) 4481 { 4482 struct mlxsw_sp_nexthop_group *nh_grp; 4483 int err; 4484 4485 nh_grp = kzalloc(sizeof(*nh_grp), GFP_KERNEL); 4486 if (!nh_grp) 4487 return ERR_PTR(-ENOMEM); 4488 INIT_LIST_HEAD(&nh_grp->vr_list); 4489 err = rhashtable_init(&nh_grp->vr_ht, 4490 &mlxsw_sp_nexthop_group_vr_ht_params); 4491 if (err) 4492 goto err_nexthop_group_vr_ht_init; 4493 INIT_LIST_HEAD(&nh_grp->fib_list); 4494 nh_grp->type = MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ; 4495 nh_grp->obj.id = info->id; 4496 4497 err = mlxsw_sp_nexthop_obj_group_info_init(mlxsw_sp, nh_grp, info); 4498 if (err) 4499 goto err_nexthop_group_info_init; 4500 4501 nh_grp->can_destroy = false; 4502 4503 return nh_grp; 4504 4505 err_nexthop_group_info_init: 4506 rhashtable_destroy(&nh_grp->vr_ht); 4507 err_nexthop_group_vr_ht_init: 4508 kfree(nh_grp); 4509 return ERR_PTR(err); 4510 } 4511 4512 static void 4513 mlxsw_sp_nexthop_obj_group_destroy(struct mlxsw_sp *mlxsw_sp, 4514 struct mlxsw_sp_nexthop_group *nh_grp) 4515 { 4516 if (!nh_grp->can_destroy) 4517 return; 4518 mlxsw_sp_nexthop_obj_group_info_fini(mlxsw_sp, nh_grp); 4519 WARN_ON_ONCE(!list_empty(&nh_grp->fib_list)); 4520 WARN_ON_ONCE(!list_empty(&nh_grp->vr_list)); 4521 rhashtable_destroy(&nh_grp->vr_ht); 4522 kfree(nh_grp); 4523 } 4524 4525 static struct mlxsw_sp_nexthop_group * 4526 mlxsw_sp_nexthop_obj_group_lookup(struct mlxsw_sp *mlxsw_sp, u32 id) 4527 { 4528 struct mlxsw_sp_nexthop_group_cmp_arg cmp_arg; 4529 4530 cmp_arg.type = MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ; 4531 cmp_arg.id = id; 4532 return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_group_ht, 4533 &cmp_arg, 4534 mlxsw_sp_nexthop_group_ht_params); 4535 } 4536 4537 static int mlxsw_sp_nexthop_obj_group_add(struct mlxsw_sp *mlxsw_sp, 4538 struct mlxsw_sp_nexthop_group *nh_grp) 4539 { 4540 return mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp); 4541 } 4542 4543 static int 4544 mlxsw_sp_nexthop_obj_group_replace(struct mlxsw_sp *mlxsw_sp, 4545 struct mlxsw_sp_nexthop_group *nh_grp, 4546 struct mlxsw_sp_nexthop_group *old_nh_grp, 4547 struct netlink_ext_ack *extack) 4548 { 4549 struct mlxsw_sp_nexthop_group_info *old_nhgi = old_nh_grp->nhgi; 4550 struct mlxsw_sp_nexthop_group_info *new_nhgi = nh_grp->nhgi; 4551 int err; 4552 4553 old_nh_grp->nhgi = new_nhgi; 4554 new_nhgi->nh_grp = old_nh_grp; 4555 nh_grp->nhgi = old_nhgi; 4556 old_nhgi->nh_grp = nh_grp; 4557 4558 if (old_nhgi->adj_index_valid && new_nhgi->adj_index_valid) { 4559 /* Both the old adjacency index and the new one are valid. 4560 * Routes are currently using the old one. Tell the device to 4561 * replace the old adjacency index with the new one. 4562 */ 4563 err = mlxsw_sp_adj_index_mass_update(mlxsw_sp, old_nh_grp, 4564 old_nhgi->adj_index, 4565 old_nhgi->ecmp_size); 4566 if (err) { 4567 NL_SET_ERR_MSG_MOD(extack, "Failed to replace old adjacency index with new one"); 4568 goto err_out; 4569 } 4570 } else if (old_nhgi->adj_index_valid && !new_nhgi->adj_index_valid) { 4571 /* The old adjacency index is valid, while the new one is not. 4572 * Iterate over all the routes using the group and change them 4573 * to trap packets to the CPU. 4574 */ 4575 err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, old_nh_grp); 4576 if (err) { 4577 NL_SET_ERR_MSG_MOD(extack, "Failed to update routes to trap packets"); 4578 goto err_out; 4579 } 4580 } else if (!old_nhgi->adj_index_valid && new_nhgi->adj_index_valid) { 4581 /* The old adjacency index is invalid, while the new one is. 4582 * Iterate over all the routes using the group and change them 4583 * to forward packets using the new valid index. 4584 */ 4585 err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, old_nh_grp); 4586 if (err) { 4587 NL_SET_ERR_MSG_MOD(extack, "Failed to update routes to forward packets"); 4588 goto err_out; 4589 } 4590 } 4591 4592 /* Make sure the flags are set / cleared based on the new nexthop group 4593 * information. 4594 */ 4595 mlxsw_sp_nexthop_obj_group_offload_refresh(mlxsw_sp, old_nh_grp); 4596 4597 /* At this point 'nh_grp' is just a shell that is not used by anyone 4598 * and its nexthop group info is the old info that was just replaced 4599 * with the new one. Remove it. 4600 */ 4601 nh_grp->can_destroy = true; 4602 mlxsw_sp_nexthop_obj_group_destroy(mlxsw_sp, nh_grp); 4603 4604 return 0; 4605 4606 err_out: 4607 old_nhgi->nh_grp = old_nh_grp; 4608 nh_grp->nhgi = new_nhgi; 4609 new_nhgi->nh_grp = nh_grp; 4610 old_nh_grp->nhgi = old_nhgi; 4611 return err; 4612 } 4613 4614 static int mlxsw_sp_nexthop_obj_new(struct mlxsw_sp *mlxsw_sp, 4615 struct nh_notifier_info *info) 4616 { 4617 struct mlxsw_sp_nexthop_group *nh_grp, *old_nh_grp; 4618 struct netlink_ext_ack *extack = info->extack; 4619 int err; 4620 4621 nh_grp = mlxsw_sp_nexthop_obj_group_create(mlxsw_sp, info); 4622 if (IS_ERR(nh_grp)) 4623 return PTR_ERR(nh_grp); 4624 4625 old_nh_grp = mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, info->id); 4626 if (!old_nh_grp) 4627 err = mlxsw_sp_nexthop_obj_group_add(mlxsw_sp, nh_grp); 4628 else 4629 err = mlxsw_sp_nexthop_obj_group_replace(mlxsw_sp, nh_grp, 4630 old_nh_grp, extack); 4631 4632 if (err) { 4633 nh_grp->can_destroy = true; 4634 mlxsw_sp_nexthop_obj_group_destroy(mlxsw_sp, nh_grp); 4635 } 4636 4637 return err; 4638 } 4639 4640 static void mlxsw_sp_nexthop_obj_del(struct mlxsw_sp *mlxsw_sp, 4641 struct nh_notifier_info *info) 4642 { 4643 struct mlxsw_sp_nexthop_group *nh_grp; 4644 4645 nh_grp = mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, info->id); 4646 if (!nh_grp) 4647 return; 4648 4649 nh_grp->can_destroy = true; 4650 mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp); 4651 4652 /* If the group still has routes using it, then defer the delete 4653 * operation until the last route using it is deleted. 4654 */ 4655 if (!list_empty(&nh_grp->fib_list)) 4656 return; 4657 mlxsw_sp_nexthop_obj_group_destroy(mlxsw_sp, nh_grp); 4658 } 4659 4660 static int mlxsw_sp_nexthop_obj_event(struct notifier_block *nb, 4661 unsigned long event, void *ptr) 4662 { 4663 struct nh_notifier_info *info = ptr; 4664 struct mlxsw_sp_router *router; 4665 int err = 0; 4666 4667 router = container_of(nb, struct mlxsw_sp_router, nexthop_nb); 4668 err = mlxsw_sp_nexthop_obj_validate(router->mlxsw_sp, event, info); 4669 if (err) 4670 goto out; 4671 4672 mutex_lock(&router->lock); 4673 4674 ASSERT_RTNL(); 4675 4676 switch (event) { 4677 case NEXTHOP_EVENT_REPLACE: 4678 err = mlxsw_sp_nexthop_obj_new(router->mlxsw_sp, info); 4679 break; 4680 case NEXTHOP_EVENT_DEL: 4681 mlxsw_sp_nexthop_obj_del(router->mlxsw_sp, info); 4682 break; 4683 default: 4684 break; 4685 } 4686 4687 mutex_unlock(&router->lock); 4688 4689 out: 4690 return notifier_from_errno(err); 4691 } 4692 4693 static bool mlxsw_sp_fi_is_gateway(const struct mlxsw_sp *mlxsw_sp, 4694 struct fib_info *fi) 4695 { 4696 const struct fib_nh *nh = fib_info_nh(fi, 0); 4697 4698 return nh->fib_nh_scope == RT_SCOPE_LINK || 4699 mlxsw_sp_nexthop4_ipip_type(mlxsw_sp, nh, NULL); 4700 } 4701 4702 static int 4703 mlxsw_sp_nexthop4_group_info_init(struct mlxsw_sp *mlxsw_sp, 4704 struct mlxsw_sp_nexthop_group *nh_grp) 4705 { 4706 unsigned int nhs = fib_info_num_path(nh_grp->ipv4.fi); 4707 struct mlxsw_sp_nexthop_group_info *nhgi; 4708 struct mlxsw_sp_nexthop *nh; 4709 int err, i; 4710 4711 nhgi = kzalloc(struct_size(nhgi, nexthops, nhs), GFP_KERNEL); 4712 if (!nhgi) 4713 return -ENOMEM; 4714 nh_grp->nhgi = nhgi; 4715 nhgi->nh_grp = nh_grp; 4716 nhgi->gateway = mlxsw_sp_fi_is_gateway(mlxsw_sp, nh_grp->ipv4.fi); 4717 nhgi->count = nhs; 4718 for (i = 0; i < nhgi->count; i++) { 4719 struct fib_nh *fib_nh; 4720 4721 nh = &nhgi->nexthops[i]; 4722 fib_nh = fib_info_nh(nh_grp->ipv4.fi, i); 4723 err = mlxsw_sp_nexthop4_init(mlxsw_sp, nh_grp, nh, fib_nh); 4724 if (err) 4725 goto err_nexthop4_init; 4726 } 4727 err = mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp); 4728 if (err) 4729 goto err_group_refresh; 4730 4731 return 0; 4732 4733 err_group_refresh: 4734 i = nhgi->count; 4735 err_nexthop4_init: 4736 for (i--; i >= 0; i--) { 4737 nh = &nhgi->nexthops[i]; 4738 mlxsw_sp_nexthop4_fini(mlxsw_sp, nh); 4739 } 4740 kfree(nhgi); 4741 return err; 4742 } 4743 4744 static void 4745 mlxsw_sp_nexthop4_group_info_fini(struct mlxsw_sp *mlxsw_sp, 4746 struct mlxsw_sp_nexthop_group *nh_grp) 4747 { 4748 struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi; 4749 int i; 4750 4751 for (i = nhgi->count - 1; i >= 0; i--) { 4752 struct mlxsw_sp_nexthop *nh = &nhgi->nexthops[i]; 4753 4754 mlxsw_sp_nexthop4_fini(mlxsw_sp, nh); 4755 } 4756 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp); 4757 WARN_ON_ONCE(nhgi->adj_index_valid); 4758 kfree(nhgi); 4759 } 4760 4761 static struct mlxsw_sp_nexthop_group * 4762 mlxsw_sp_nexthop4_group_create(struct mlxsw_sp *mlxsw_sp, struct fib_info *fi) 4763 { 4764 struct mlxsw_sp_nexthop_group *nh_grp; 4765 int err; 4766 4767 nh_grp = kzalloc(sizeof(*nh_grp), GFP_KERNEL); 4768 if (!nh_grp) 4769 return ERR_PTR(-ENOMEM); 4770 INIT_LIST_HEAD(&nh_grp->vr_list); 4771 err = rhashtable_init(&nh_grp->vr_ht, 4772 &mlxsw_sp_nexthop_group_vr_ht_params); 4773 if (err) 4774 goto err_nexthop_group_vr_ht_init; 4775 INIT_LIST_HEAD(&nh_grp->fib_list); 4776 nh_grp->type = MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4; 4777 nh_grp->ipv4.fi = fi; 4778 fib_info_hold(fi); 4779 4780 err = mlxsw_sp_nexthop4_group_info_init(mlxsw_sp, nh_grp); 4781 if (err) 4782 goto err_nexthop_group_info_init; 4783 4784 err = mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp); 4785 if (err) 4786 goto err_nexthop_group_insert; 4787 4788 nh_grp->can_destroy = true; 4789 4790 return nh_grp; 4791 4792 err_nexthop_group_insert: 4793 mlxsw_sp_nexthop4_group_info_fini(mlxsw_sp, nh_grp); 4794 err_nexthop_group_info_init: 4795 fib_info_put(fi); 4796 rhashtable_destroy(&nh_grp->vr_ht); 4797 err_nexthop_group_vr_ht_init: 4798 kfree(nh_grp); 4799 return ERR_PTR(err); 4800 } 4801 4802 static void 4803 mlxsw_sp_nexthop4_group_destroy(struct mlxsw_sp *mlxsw_sp, 4804 struct mlxsw_sp_nexthop_group *nh_grp) 4805 { 4806 if (!nh_grp->can_destroy) 4807 return; 4808 mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp); 4809 mlxsw_sp_nexthop4_group_info_fini(mlxsw_sp, nh_grp); 4810 fib_info_put(nh_grp->ipv4.fi); 4811 WARN_ON_ONCE(!list_empty(&nh_grp->vr_list)); 4812 rhashtable_destroy(&nh_grp->vr_ht); 4813 kfree(nh_grp); 4814 } 4815 4816 static int mlxsw_sp_nexthop4_group_get(struct mlxsw_sp *mlxsw_sp, 4817 struct mlxsw_sp_fib_entry *fib_entry, 4818 struct fib_info *fi) 4819 { 4820 struct mlxsw_sp_nexthop_group *nh_grp; 4821 4822 if (fi->nh) { 4823 nh_grp = mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, 4824 fi->nh->id); 4825 if (WARN_ON_ONCE(!nh_grp)) 4826 return -EINVAL; 4827 goto out; 4828 } 4829 4830 nh_grp = mlxsw_sp_nexthop4_group_lookup(mlxsw_sp, fi); 4831 if (!nh_grp) { 4832 nh_grp = mlxsw_sp_nexthop4_group_create(mlxsw_sp, fi); 4833 if (IS_ERR(nh_grp)) 4834 return PTR_ERR(nh_grp); 4835 } 4836 out: 4837 list_add_tail(&fib_entry->nexthop_group_node, &nh_grp->fib_list); 4838 fib_entry->nh_group = nh_grp; 4839 return 0; 4840 } 4841 4842 static void mlxsw_sp_nexthop4_group_put(struct mlxsw_sp *mlxsw_sp, 4843 struct mlxsw_sp_fib_entry *fib_entry) 4844 { 4845 struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group; 4846 4847 list_del(&fib_entry->nexthop_group_node); 4848 if (!list_empty(&nh_grp->fib_list)) 4849 return; 4850 4851 if (nh_grp->type == MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ) { 4852 mlxsw_sp_nexthop_obj_group_destroy(mlxsw_sp, nh_grp); 4853 return; 4854 } 4855 4856 mlxsw_sp_nexthop4_group_destroy(mlxsw_sp, nh_grp); 4857 } 4858 4859 static bool 4860 mlxsw_sp_fib4_entry_should_offload(const struct mlxsw_sp_fib_entry *fib_entry) 4861 { 4862 struct mlxsw_sp_fib4_entry *fib4_entry; 4863 4864 fib4_entry = container_of(fib_entry, struct mlxsw_sp_fib4_entry, 4865 common); 4866 return !fib4_entry->tos; 4867 } 4868 4869 static bool 4870 mlxsw_sp_fib_entry_should_offload(const struct mlxsw_sp_fib_entry *fib_entry) 4871 { 4872 struct mlxsw_sp_nexthop_group *nh_group = fib_entry->nh_group; 4873 4874 switch (fib_entry->fib_node->fib->proto) { 4875 case MLXSW_SP_L3_PROTO_IPV4: 4876 if (!mlxsw_sp_fib4_entry_should_offload(fib_entry)) 4877 return false; 4878 break; 4879 case MLXSW_SP_L3_PROTO_IPV6: 4880 break; 4881 } 4882 4883 switch (fib_entry->type) { 4884 case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE: 4885 return !!nh_group->nhgi->adj_index_valid; 4886 case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL: 4887 return !!nh_group->nhgi->nh_rif; 4888 case MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE: 4889 case MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP: 4890 case MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP: 4891 return true; 4892 default: 4893 return false; 4894 } 4895 } 4896 4897 static struct mlxsw_sp_nexthop * 4898 mlxsw_sp_rt6_nexthop(struct mlxsw_sp_nexthop_group *nh_grp, 4899 const struct mlxsw_sp_rt6 *mlxsw_sp_rt6) 4900 { 4901 int i; 4902 4903 for (i = 0; i < nh_grp->nhgi->count; i++) { 4904 struct mlxsw_sp_nexthop *nh = &nh_grp->nhgi->nexthops[i]; 4905 struct fib6_info *rt = mlxsw_sp_rt6->rt; 4906 4907 if (nh->rif && nh->rif->dev == rt->fib6_nh->fib_nh_dev && 4908 ipv6_addr_equal((const struct in6_addr *) &nh->gw_addr, 4909 &rt->fib6_nh->fib_nh_gw6)) 4910 return nh; 4911 continue; 4912 } 4913 4914 return NULL; 4915 } 4916 4917 static void 4918 mlxsw_sp_fib4_entry_hw_flags_set(struct mlxsw_sp *mlxsw_sp, 4919 struct mlxsw_sp_fib_entry *fib_entry) 4920 { 4921 u32 *p_dst = (u32 *) fib_entry->fib_node->key.addr; 4922 int dst_len = fib_entry->fib_node->key.prefix_len; 4923 struct mlxsw_sp_fib4_entry *fib4_entry; 4924 struct fib_rt_info fri; 4925 bool should_offload; 4926 4927 should_offload = mlxsw_sp_fib_entry_should_offload(fib_entry); 4928 fib4_entry = container_of(fib_entry, struct mlxsw_sp_fib4_entry, 4929 common); 4930 fri.fi = fib4_entry->fi; 4931 fri.tb_id = fib4_entry->tb_id; 4932 fri.dst = cpu_to_be32(*p_dst); 4933 fri.dst_len = dst_len; 4934 fri.tos = fib4_entry->tos; 4935 fri.type = fib4_entry->type; 4936 fri.offload = should_offload; 4937 fri.trap = !should_offload; 4938 fib_alias_hw_flags_set(mlxsw_sp_net(mlxsw_sp), &fri); 4939 } 4940 4941 static void 4942 mlxsw_sp_fib4_entry_hw_flags_clear(struct mlxsw_sp *mlxsw_sp, 4943 struct mlxsw_sp_fib_entry *fib_entry) 4944 { 4945 u32 *p_dst = (u32 *) fib_entry->fib_node->key.addr; 4946 int dst_len = fib_entry->fib_node->key.prefix_len; 4947 struct mlxsw_sp_fib4_entry *fib4_entry; 4948 struct fib_rt_info fri; 4949 4950 fib4_entry = container_of(fib_entry, struct mlxsw_sp_fib4_entry, 4951 common); 4952 fri.fi = fib4_entry->fi; 4953 fri.tb_id = fib4_entry->tb_id; 4954 fri.dst = cpu_to_be32(*p_dst); 4955 fri.dst_len = dst_len; 4956 fri.tos = fib4_entry->tos; 4957 fri.type = fib4_entry->type; 4958 fri.offload = false; 4959 fri.trap = false; 4960 fib_alias_hw_flags_set(mlxsw_sp_net(mlxsw_sp), &fri); 4961 } 4962 4963 static void 4964 mlxsw_sp_fib6_entry_hw_flags_set(struct mlxsw_sp *mlxsw_sp, 4965 struct mlxsw_sp_fib_entry *fib_entry) 4966 { 4967 struct mlxsw_sp_fib6_entry *fib6_entry; 4968 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 4969 bool should_offload; 4970 4971 should_offload = mlxsw_sp_fib_entry_should_offload(fib_entry); 4972 4973 /* In IPv6 a multipath route is represented using multiple routes, so 4974 * we need to set the flags on all of them. 4975 */ 4976 fib6_entry = container_of(fib_entry, struct mlxsw_sp_fib6_entry, 4977 common); 4978 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) 4979 fib6_info_hw_flags_set(mlxsw_sp_rt6->rt, should_offload, 4980 !should_offload); 4981 } 4982 4983 static void 4984 mlxsw_sp_fib6_entry_hw_flags_clear(struct mlxsw_sp *mlxsw_sp, 4985 struct mlxsw_sp_fib_entry *fib_entry) 4986 { 4987 struct mlxsw_sp_fib6_entry *fib6_entry; 4988 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 4989 4990 fib6_entry = container_of(fib_entry, struct mlxsw_sp_fib6_entry, 4991 common); 4992 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) 4993 fib6_info_hw_flags_set(mlxsw_sp_rt6->rt, false, false); 4994 } 4995 4996 static void 4997 mlxsw_sp_fib_entry_hw_flags_set(struct mlxsw_sp *mlxsw_sp, 4998 struct mlxsw_sp_fib_entry *fib_entry) 4999 { 5000 switch (fib_entry->fib_node->fib->proto) { 5001 case MLXSW_SP_L3_PROTO_IPV4: 5002 mlxsw_sp_fib4_entry_hw_flags_set(mlxsw_sp, fib_entry); 5003 break; 5004 case MLXSW_SP_L3_PROTO_IPV6: 5005 mlxsw_sp_fib6_entry_hw_flags_set(mlxsw_sp, fib_entry); 5006 break; 5007 } 5008 } 5009 5010 static void 5011 mlxsw_sp_fib_entry_hw_flags_clear(struct mlxsw_sp *mlxsw_sp, 5012 struct mlxsw_sp_fib_entry *fib_entry) 5013 { 5014 switch (fib_entry->fib_node->fib->proto) { 5015 case MLXSW_SP_L3_PROTO_IPV4: 5016 mlxsw_sp_fib4_entry_hw_flags_clear(mlxsw_sp, fib_entry); 5017 break; 5018 case MLXSW_SP_L3_PROTO_IPV6: 5019 mlxsw_sp_fib6_entry_hw_flags_clear(mlxsw_sp, fib_entry); 5020 break; 5021 } 5022 } 5023 5024 static void 5025 mlxsw_sp_fib_entry_hw_flags_refresh(struct mlxsw_sp *mlxsw_sp, 5026 struct mlxsw_sp_fib_entry *fib_entry, 5027 enum mlxsw_sp_fib_entry_op op) 5028 { 5029 switch (op) { 5030 case MLXSW_SP_FIB_ENTRY_OP_WRITE: 5031 case MLXSW_SP_FIB_ENTRY_OP_UPDATE: 5032 mlxsw_sp_fib_entry_hw_flags_set(mlxsw_sp, fib_entry); 5033 break; 5034 case MLXSW_SP_FIB_ENTRY_OP_DELETE: 5035 mlxsw_sp_fib_entry_hw_flags_clear(mlxsw_sp, fib_entry); 5036 break; 5037 default: 5038 break; 5039 } 5040 } 5041 5042 struct mlxsw_sp_fib_entry_op_ctx_basic { 5043 char ralue_pl[MLXSW_REG_RALUE_LEN]; 5044 }; 5045 5046 static void 5047 mlxsw_sp_router_ll_basic_fib_entry_pack(struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5048 enum mlxsw_sp_l3proto proto, 5049 enum mlxsw_sp_fib_entry_op op, 5050 u16 virtual_router, u8 prefix_len, 5051 unsigned char *addr, 5052 struct mlxsw_sp_fib_entry_priv *priv) 5053 { 5054 struct mlxsw_sp_fib_entry_op_ctx_basic *op_ctx_basic = (void *) op_ctx->ll_priv; 5055 enum mlxsw_reg_ralxx_protocol ralxx_proto; 5056 char *ralue_pl = op_ctx_basic->ralue_pl; 5057 enum mlxsw_reg_ralue_op ralue_op; 5058 5059 ralxx_proto = (enum mlxsw_reg_ralxx_protocol) proto; 5060 5061 switch (op) { 5062 case MLXSW_SP_FIB_ENTRY_OP_WRITE: 5063 case MLXSW_SP_FIB_ENTRY_OP_UPDATE: 5064 ralue_op = MLXSW_REG_RALUE_OP_WRITE_WRITE; 5065 break; 5066 case MLXSW_SP_FIB_ENTRY_OP_DELETE: 5067 ralue_op = MLXSW_REG_RALUE_OP_WRITE_DELETE; 5068 break; 5069 default: 5070 WARN_ON_ONCE(1); 5071 return; 5072 } 5073 5074 switch (proto) { 5075 case MLXSW_SP_L3_PROTO_IPV4: 5076 mlxsw_reg_ralue_pack4(ralue_pl, ralxx_proto, ralue_op, 5077 virtual_router, prefix_len, (u32 *) addr); 5078 break; 5079 case MLXSW_SP_L3_PROTO_IPV6: 5080 mlxsw_reg_ralue_pack6(ralue_pl, ralxx_proto, ralue_op, 5081 virtual_router, prefix_len, addr); 5082 break; 5083 } 5084 } 5085 5086 static void 5087 mlxsw_sp_router_ll_basic_fib_entry_act_remote_pack(struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5088 enum mlxsw_reg_ralue_trap_action trap_action, 5089 u16 trap_id, u32 adjacency_index, u16 ecmp_size) 5090 { 5091 struct mlxsw_sp_fib_entry_op_ctx_basic *op_ctx_basic = (void *) op_ctx->ll_priv; 5092 5093 mlxsw_reg_ralue_act_remote_pack(op_ctx_basic->ralue_pl, trap_action, 5094 trap_id, adjacency_index, ecmp_size); 5095 } 5096 5097 static void 5098 mlxsw_sp_router_ll_basic_fib_entry_act_local_pack(struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5099 enum mlxsw_reg_ralue_trap_action trap_action, 5100 u16 trap_id, u16 local_erif) 5101 { 5102 struct mlxsw_sp_fib_entry_op_ctx_basic *op_ctx_basic = (void *) op_ctx->ll_priv; 5103 5104 mlxsw_reg_ralue_act_local_pack(op_ctx_basic->ralue_pl, trap_action, 5105 trap_id, local_erif); 5106 } 5107 5108 static void 5109 mlxsw_sp_router_ll_basic_fib_entry_act_ip2me_pack(struct mlxsw_sp_fib_entry_op_ctx *op_ctx) 5110 { 5111 struct mlxsw_sp_fib_entry_op_ctx_basic *op_ctx_basic = (void *) op_ctx->ll_priv; 5112 5113 mlxsw_reg_ralue_act_ip2me_pack(op_ctx_basic->ralue_pl); 5114 } 5115 5116 static void 5117 mlxsw_sp_router_ll_basic_fib_entry_act_ip2me_tun_pack(struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5118 u32 tunnel_ptr) 5119 { 5120 struct mlxsw_sp_fib_entry_op_ctx_basic *op_ctx_basic = (void *) op_ctx->ll_priv; 5121 5122 mlxsw_reg_ralue_act_ip2me_tun_pack(op_ctx_basic->ralue_pl, tunnel_ptr); 5123 } 5124 5125 static int 5126 mlxsw_sp_router_ll_basic_fib_entry_commit(struct mlxsw_sp *mlxsw_sp, 5127 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5128 bool *postponed_for_bulk) 5129 { 5130 struct mlxsw_sp_fib_entry_op_ctx_basic *op_ctx_basic = (void *) op_ctx->ll_priv; 5131 5132 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), 5133 op_ctx_basic->ralue_pl); 5134 } 5135 5136 static bool 5137 mlxsw_sp_router_ll_basic_fib_entry_is_committed(struct mlxsw_sp_fib_entry_priv *priv) 5138 { 5139 return true; 5140 } 5141 5142 static void mlxsw_sp_fib_entry_pack(struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5143 struct mlxsw_sp_fib_entry *fib_entry, 5144 enum mlxsw_sp_fib_entry_op op) 5145 { 5146 struct mlxsw_sp_fib *fib = fib_entry->fib_node->fib; 5147 5148 mlxsw_sp_fib_entry_op_ctx_priv_hold(op_ctx, fib_entry->priv); 5149 fib->ll_ops->fib_entry_pack(op_ctx, fib->proto, op, fib->vr->id, 5150 fib_entry->fib_node->key.prefix_len, 5151 fib_entry->fib_node->key.addr, 5152 fib_entry->priv); 5153 } 5154 5155 static int mlxsw_sp_fib_entry_commit(struct mlxsw_sp *mlxsw_sp, 5156 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5157 const struct mlxsw_sp_router_ll_ops *ll_ops) 5158 { 5159 bool postponed_for_bulk = false; 5160 int err; 5161 5162 err = ll_ops->fib_entry_commit(mlxsw_sp, op_ctx, &postponed_for_bulk); 5163 if (!postponed_for_bulk) 5164 mlxsw_sp_fib_entry_op_ctx_priv_put_all(op_ctx); 5165 return err; 5166 } 5167 5168 static int mlxsw_sp_adj_discard_write(struct mlxsw_sp *mlxsw_sp) 5169 { 5170 enum mlxsw_reg_ratr_trap_action trap_action; 5171 char ratr_pl[MLXSW_REG_RATR_LEN]; 5172 int err; 5173 5174 if (mlxsw_sp->router->adj_discard_index_valid) 5175 return 0; 5176 5177 err = mlxsw_sp_kvdl_alloc(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1, 5178 &mlxsw_sp->router->adj_discard_index); 5179 if (err) 5180 return err; 5181 5182 trap_action = MLXSW_REG_RATR_TRAP_ACTION_TRAP; 5183 mlxsw_reg_ratr_pack(ratr_pl, MLXSW_REG_RATR_OP_WRITE_WRITE_ENTRY, true, 5184 MLXSW_REG_RATR_TYPE_ETHERNET, 5185 mlxsw_sp->router->adj_discard_index, 5186 mlxsw_sp->router->lb_rif_index); 5187 mlxsw_reg_ratr_trap_action_set(ratr_pl, trap_action); 5188 mlxsw_reg_ratr_trap_id_set(ratr_pl, MLXSW_TRAP_ID_RTR_EGRESS0); 5189 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ratr), ratr_pl); 5190 if (err) 5191 goto err_ratr_write; 5192 5193 mlxsw_sp->router->adj_discard_index_valid = true; 5194 5195 return 0; 5196 5197 err_ratr_write: 5198 mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1, 5199 mlxsw_sp->router->adj_discard_index); 5200 return err; 5201 } 5202 5203 static int mlxsw_sp_fib_entry_op_remote(struct mlxsw_sp *mlxsw_sp, 5204 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5205 struct mlxsw_sp_fib_entry *fib_entry, 5206 enum mlxsw_sp_fib_entry_op op) 5207 { 5208 const struct mlxsw_sp_router_ll_ops *ll_ops = fib_entry->fib_node->fib->ll_ops; 5209 struct mlxsw_sp_nexthop_group *nh_group = fib_entry->nh_group; 5210 struct mlxsw_sp_nexthop_group_info *nhgi = nh_group->nhgi; 5211 enum mlxsw_reg_ralue_trap_action trap_action; 5212 u16 trap_id = 0; 5213 u32 adjacency_index = 0; 5214 u16 ecmp_size = 0; 5215 int err; 5216 5217 /* In case the nexthop group adjacency index is valid, use it 5218 * with provided ECMP size. Otherwise, setup trap and pass 5219 * traffic to kernel. 5220 */ 5221 if (mlxsw_sp_fib_entry_should_offload(fib_entry)) { 5222 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP; 5223 adjacency_index = nhgi->adj_index; 5224 ecmp_size = nhgi->ecmp_size; 5225 } else if (!nhgi->adj_index_valid && nhgi->count && nhgi->nh_rif) { 5226 err = mlxsw_sp_adj_discard_write(mlxsw_sp); 5227 if (err) 5228 return err; 5229 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP; 5230 adjacency_index = mlxsw_sp->router->adj_discard_index; 5231 ecmp_size = 1; 5232 } else { 5233 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP; 5234 trap_id = MLXSW_TRAP_ID_RTR_INGRESS0; 5235 } 5236 5237 mlxsw_sp_fib_entry_pack(op_ctx, fib_entry, op); 5238 ll_ops->fib_entry_act_remote_pack(op_ctx, trap_action, trap_id, 5239 adjacency_index, ecmp_size); 5240 return mlxsw_sp_fib_entry_commit(mlxsw_sp, op_ctx, ll_ops); 5241 } 5242 5243 static int mlxsw_sp_fib_entry_op_local(struct mlxsw_sp *mlxsw_sp, 5244 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5245 struct mlxsw_sp_fib_entry *fib_entry, 5246 enum mlxsw_sp_fib_entry_op op) 5247 { 5248 const struct mlxsw_sp_router_ll_ops *ll_ops = fib_entry->fib_node->fib->ll_ops; 5249 struct mlxsw_sp_rif *rif = fib_entry->nh_group->nhgi->nh_rif; 5250 enum mlxsw_reg_ralue_trap_action trap_action; 5251 u16 trap_id = 0; 5252 u16 rif_index = 0; 5253 5254 if (mlxsw_sp_fib_entry_should_offload(fib_entry)) { 5255 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP; 5256 rif_index = rif->rif_index; 5257 } else { 5258 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP; 5259 trap_id = MLXSW_TRAP_ID_RTR_INGRESS0; 5260 } 5261 5262 mlxsw_sp_fib_entry_pack(op_ctx, fib_entry, op); 5263 ll_ops->fib_entry_act_local_pack(op_ctx, trap_action, trap_id, rif_index); 5264 return mlxsw_sp_fib_entry_commit(mlxsw_sp, op_ctx, ll_ops); 5265 } 5266 5267 static int mlxsw_sp_fib_entry_op_trap(struct mlxsw_sp *mlxsw_sp, 5268 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5269 struct mlxsw_sp_fib_entry *fib_entry, 5270 enum mlxsw_sp_fib_entry_op op) 5271 { 5272 const struct mlxsw_sp_router_ll_ops *ll_ops = fib_entry->fib_node->fib->ll_ops; 5273 5274 mlxsw_sp_fib_entry_pack(op_ctx, fib_entry, op); 5275 ll_ops->fib_entry_act_ip2me_pack(op_ctx); 5276 return mlxsw_sp_fib_entry_commit(mlxsw_sp, op_ctx, ll_ops); 5277 } 5278 5279 static int mlxsw_sp_fib_entry_op_blackhole(struct mlxsw_sp *mlxsw_sp, 5280 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5281 struct mlxsw_sp_fib_entry *fib_entry, 5282 enum mlxsw_sp_fib_entry_op op) 5283 { 5284 const struct mlxsw_sp_router_ll_ops *ll_ops = fib_entry->fib_node->fib->ll_ops; 5285 enum mlxsw_reg_ralue_trap_action trap_action; 5286 5287 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_DISCARD_ERROR; 5288 mlxsw_sp_fib_entry_pack(op_ctx, fib_entry, op); 5289 ll_ops->fib_entry_act_local_pack(op_ctx, trap_action, 0, 0); 5290 return mlxsw_sp_fib_entry_commit(mlxsw_sp, op_ctx, ll_ops); 5291 } 5292 5293 static int 5294 mlxsw_sp_fib_entry_op_unreachable(struct mlxsw_sp *mlxsw_sp, 5295 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5296 struct mlxsw_sp_fib_entry *fib_entry, 5297 enum mlxsw_sp_fib_entry_op op) 5298 { 5299 const struct mlxsw_sp_router_ll_ops *ll_ops = fib_entry->fib_node->fib->ll_ops; 5300 enum mlxsw_reg_ralue_trap_action trap_action; 5301 u16 trap_id; 5302 5303 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP; 5304 trap_id = MLXSW_TRAP_ID_RTR_INGRESS1; 5305 5306 mlxsw_sp_fib_entry_pack(op_ctx, fib_entry, op); 5307 ll_ops->fib_entry_act_local_pack(op_ctx, trap_action, trap_id, 0); 5308 return mlxsw_sp_fib_entry_commit(mlxsw_sp, op_ctx, ll_ops); 5309 } 5310 5311 static int 5312 mlxsw_sp_fib_entry_op_ipip_decap(struct mlxsw_sp *mlxsw_sp, 5313 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5314 struct mlxsw_sp_fib_entry *fib_entry, 5315 enum mlxsw_sp_fib_entry_op op) 5316 { 5317 const struct mlxsw_sp_router_ll_ops *ll_ops = fib_entry->fib_node->fib->ll_ops; 5318 struct mlxsw_sp_ipip_entry *ipip_entry = fib_entry->decap.ipip_entry; 5319 const struct mlxsw_sp_ipip_ops *ipip_ops; 5320 int err; 5321 5322 if (WARN_ON(!ipip_entry)) 5323 return -EINVAL; 5324 5325 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt]; 5326 err = ipip_ops->decap_config(mlxsw_sp, ipip_entry, 5327 fib_entry->decap.tunnel_index); 5328 if (err) 5329 return err; 5330 5331 mlxsw_sp_fib_entry_pack(op_ctx, fib_entry, op); 5332 ll_ops->fib_entry_act_ip2me_tun_pack(op_ctx, 5333 fib_entry->decap.tunnel_index); 5334 return mlxsw_sp_fib_entry_commit(mlxsw_sp, op_ctx, ll_ops); 5335 } 5336 5337 static int mlxsw_sp_fib_entry_op_nve_decap(struct mlxsw_sp *mlxsw_sp, 5338 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5339 struct mlxsw_sp_fib_entry *fib_entry, 5340 enum mlxsw_sp_fib_entry_op op) 5341 { 5342 const struct mlxsw_sp_router_ll_ops *ll_ops = fib_entry->fib_node->fib->ll_ops; 5343 5344 mlxsw_sp_fib_entry_pack(op_ctx, fib_entry, op); 5345 ll_ops->fib_entry_act_ip2me_tun_pack(op_ctx, 5346 fib_entry->decap.tunnel_index); 5347 return mlxsw_sp_fib_entry_commit(mlxsw_sp, op_ctx, ll_ops); 5348 } 5349 5350 static int __mlxsw_sp_fib_entry_op(struct mlxsw_sp *mlxsw_sp, 5351 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5352 struct mlxsw_sp_fib_entry *fib_entry, 5353 enum mlxsw_sp_fib_entry_op op) 5354 { 5355 switch (fib_entry->type) { 5356 case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE: 5357 return mlxsw_sp_fib_entry_op_remote(mlxsw_sp, op_ctx, fib_entry, op); 5358 case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL: 5359 return mlxsw_sp_fib_entry_op_local(mlxsw_sp, op_ctx, fib_entry, op); 5360 case MLXSW_SP_FIB_ENTRY_TYPE_TRAP: 5361 return mlxsw_sp_fib_entry_op_trap(mlxsw_sp, op_ctx, fib_entry, op); 5362 case MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE: 5363 return mlxsw_sp_fib_entry_op_blackhole(mlxsw_sp, op_ctx, fib_entry, op); 5364 case MLXSW_SP_FIB_ENTRY_TYPE_UNREACHABLE: 5365 return mlxsw_sp_fib_entry_op_unreachable(mlxsw_sp, op_ctx, fib_entry, op); 5366 case MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP: 5367 return mlxsw_sp_fib_entry_op_ipip_decap(mlxsw_sp, op_ctx, fib_entry, op); 5368 case MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP: 5369 return mlxsw_sp_fib_entry_op_nve_decap(mlxsw_sp, op_ctx, fib_entry, op); 5370 } 5371 return -EINVAL; 5372 } 5373 5374 static int mlxsw_sp_fib_entry_op(struct mlxsw_sp *mlxsw_sp, 5375 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5376 struct mlxsw_sp_fib_entry *fib_entry, 5377 enum mlxsw_sp_fib_entry_op op) 5378 { 5379 int err = __mlxsw_sp_fib_entry_op(mlxsw_sp, op_ctx, fib_entry, op); 5380 5381 if (err) 5382 return err; 5383 5384 mlxsw_sp_fib_entry_hw_flags_refresh(mlxsw_sp, fib_entry, op); 5385 5386 return err; 5387 } 5388 5389 static int __mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp, 5390 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5391 struct mlxsw_sp_fib_entry *fib_entry, 5392 bool is_new) 5393 { 5394 return mlxsw_sp_fib_entry_op(mlxsw_sp, op_ctx, fib_entry, 5395 is_new ? MLXSW_SP_FIB_ENTRY_OP_WRITE : 5396 MLXSW_SP_FIB_ENTRY_OP_UPDATE); 5397 } 5398 5399 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp, 5400 struct mlxsw_sp_fib_entry *fib_entry) 5401 { 5402 struct mlxsw_sp_fib_entry_op_ctx *op_ctx = mlxsw_sp->router->ll_op_ctx; 5403 5404 mlxsw_sp_fib_entry_op_ctx_clear(op_ctx); 5405 return __mlxsw_sp_fib_entry_update(mlxsw_sp, op_ctx, fib_entry, false); 5406 } 5407 5408 static int mlxsw_sp_fib_entry_del(struct mlxsw_sp *mlxsw_sp, 5409 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5410 struct mlxsw_sp_fib_entry *fib_entry) 5411 { 5412 const struct mlxsw_sp_router_ll_ops *ll_ops = fib_entry->fib_node->fib->ll_ops; 5413 5414 if (!ll_ops->fib_entry_is_committed(fib_entry->priv)) 5415 return 0; 5416 return mlxsw_sp_fib_entry_op(mlxsw_sp, op_ctx, fib_entry, 5417 MLXSW_SP_FIB_ENTRY_OP_DELETE); 5418 } 5419 5420 static int 5421 mlxsw_sp_fib4_entry_type_set(struct mlxsw_sp *mlxsw_sp, 5422 const struct fib_entry_notifier_info *fen_info, 5423 struct mlxsw_sp_fib_entry *fib_entry) 5424 { 5425 struct mlxsw_sp_nexthop_group_info *nhgi = fib_entry->nh_group->nhgi; 5426 union mlxsw_sp_l3addr dip = { .addr4 = htonl(fen_info->dst) }; 5427 struct mlxsw_sp_router *router = mlxsw_sp->router; 5428 u32 tb_id = mlxsw_sp_fix_tb_id(fen_info->tb_id); 5429 int ifindex = nhgi->nexthops[0].ifindex; 5430 struct mlxsw_sp_ipip_entry *ipip_entry; 5431 5432 switch (fen_info->type) { 5433 case RTN_LOCAL: 5434 ipip_entry = mlxsw_sp_ipip_entry_find_by_decap(mlxsw_sp, ifindex, 5435 MLXSW_SP_L3_PROTO_IPV4, dip); 5436 if (ipip_entry && ipip_entry->ol_dev->flags & IFF_UP) { 5437 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP; 5438 return mlxsw_sp_fib_entry_decap_init(mlxsw_sp, 5439 fib_entry, 5440 ipip_entry); 5441 } 5442 if (mlxsw_sp_router_nve_is_decap(mlxsw_sp, tb_id, 5443 MLXSW_SP_L3_PROTO_IPV4, 5444 &dip)) { 5445 u32 tunnel_index; 5446 5447 tunnel_index = router->nve_decap_config.tunnel_index; 5448 fib_entry->decap.tunnel_index = tunnel_index; 5449 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP; 5450 return 0; 5451 } 5452 fallthrough; 5453 case RTN_BROADCAST: 5454 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP; 5455 return 0; 5456 case RTN_BLACKHOLE: 5457 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE; 5458 return 0; 5459 case RTN_UNREACHABLE: 5460 case RTN_PROHIBIT: 5461 /* Packets hitting these routes need to be trapped, but 5462 * can do so with a lower priority than packets directed 5463 * at the host, so use action type local instead of trap. 5464 */ 5465 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_UNREACHABLE; 5466 return 0; 5467 case RTN_UNICAST: 5468 if (nhgi->gateway) 5469 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_REMOTE; 5470 else 5471 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL; 5472 return 0; 5473 default: 5474 return -EINVAL; 5475 } 5476 } 5477 5478 static void 5479 mlxsw_sp_fib4_entry_type_unset(struct mlxsw_sp *mlxsw_sp, 5480 struct mlxsw_sp_fib_entry *fib_entry) 5481 { 5482 switch (fib_entry->type) { 5483 case MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP: 5484 mlxsw_sp_fib_entry_decap_fini(mlxsw_sp, fib_entry); 5485 break; 5486 default: 5487 break; 5488 } 5489 } 5490 5491 static struct mlxsw_sp_fib4_entry * 5492 mlxsw_sp_fib4_entry_create(struct mlxsw_sp *mlxsw_sp, 5493 struct mlxsw_sp_fib_node *fib_node, 5494 const struct fib_entry_notifier_info *fen_info) 5495 { 5496 struct mlxsw_sp_fib4_entry *fib4_entry; 5497 struct mlxsw_sp_fib_entry *fib_entry; 5498 int err; 5499 5500 fib4_entry = kzalloc(sizeof(*fib4_entry), GFP_KERNEL); 5501 if (!fib4_entry) 5502 return ERR_PTR(-ENOMEM); 5503 fib_entry = &fib4_entry->common; 5504 5505 fib_entry->priv = mlxsw_sp_fib_entry_priv_create(fib_node->fib->ll_ops); 5506 if (IS_ERR(fib_entry->priv)) { 5507 err = PTR_ERR(fib_entry->priv); 5508 goto err_fib_entry_priv_create; 5509 } 5510 5511 err = mlxsw_sp_nexthop4_group_get(mlxsw_sp, fib_entry, fen_info->fi); 5512 if (err) 5513 goto err_nexthop4_group_get; 5514 5515 err = mlxsw_sp_nexthop_group_vr_link(fib_entry->nh_group, 5516 fib_node->fib); 5517 if (err) 5518 goto err_nexthop_group_vr_link; 5519 5520 err = mlxsw_sp_fib4_entry_type_set(mlxsw_sp, fen_info, fib_entry); 5521 if (err) 5522 goto err_fib4_entry_type_set; 5523 5524 fib4_entry->fi = fen_info->fi; 5525 fib_info_hold(fib4_entry->fi); 5526 fib4_entry->tb_id = fen_info->tb_id; 5527 fib4_entry->type = fen_info->type; 5528 fib4_entry->tos = fen_info->tos; 5529 5530 fib_entry->fib_node = fib_node; 5531 5532 return fib4_entry; 5533 5534 err_fib4_entry_type_set: 5535 mlxsw_sp_nexthop_group_vr_unlink(fib_entry->nh_group, fib_node->fib); 5536 err_nexthop_group_vr_link: 5537 mlxsw_sp_nexthop4_group_put(mlxsw_sp, &fib4_entry->common); 5538 err_nexthop4_group_get: 5539 mlxsw_sp_fib_entry_priv_put(fib_entry->priv); 5540 err_fib_entry_priv_create: 5541 kfree(fib4_entry); 5542 return ERR_PTR(err); 5543 } 5544 5545 static void mlxsw_sp_fib4_entry_destroy(struct mlxsw_sp *mlxsw_sp, 5546 struct mlxsw_sp_fib4_entry *fib4_entry) 5547 { 5548 struct mlxsw_sp_fib_node *fib_node = fib4_entry->common.fib_node; 5549 5550 fib_info_put(fib4_entry->fi); 5551 mlxsw_sp_fib4_entry_type_unset(mlxsw_sp, &fib4_entry->common); 5552 mlxsw_sp_nexthop_group_vr_unlink(fib4_entry->common.nh_group, 5553 fib_node->fib); 5554 mlxsw_sp_nexthop4_group_put(mlxsw_sp, &fib4_entry->common); 5555 mlxsw_sp_fib_entry_priv_put(fib4_entry->common.priv); 5556 kfree(fib4_entry); 5557 } 5558 5559 static struct mlxsw_sp_fib4_entry * 5560 mlxsw_sp_fib4_entry_lookup(struct mlxsw_sp *mlxsw_sp, 5561 const struct fib_entry_notifier_info *fen_info) 5562 { 5563 struct mlxsw_sp_fib4_entry *fib4_entry; 5564 struct mlxsw_sp_fib_node *fib_node; 5565 struct mlxsw_sp_fib *fib; 5566 struct mlxsw_sp_vr *vr; 5567 5568 vr = mlxsw_sp_vr_find(mlxsw_sp, fen_info->tb_id); 5569 if (!vr) 5570 return NULL; 5571 fib = mlxsw_sp_vr_fib(vr, MLXSW_SP_L3_PROTO_IPV4); 5572 5573 fib_node = mlxsw_sp_fib_node_lookup(fib, &fen_info->dst, 5574 sizeof(fen_info->dst), 5575 fen_info->dst_len); 5576 if (!fib_node) 5577 return NULL; 5578 5579 fib4_entry = container_of(fib_node->fib_entry, 5580 struct mlxsw_sp_fib4_entry, common); 5581 if (fib4_entry->tb_id == fen_info->tb_id && 5582 fib4_entry->tos == fen_info->tos && 5583 fib4_entry->type == fen_info->type && 5584 fib4_entry->fi == fen_info->fi) 5585 return fib4_entry; 5586 5587 return NULL; 5588 } 5589 5590 static const struct rhashtable_params mlxsw_sp_fib_ht_params = { 5591 .key_offset = offsetof(struct mlxsw_sp_fib_node, key), 5592 .head_offset = offsetof(struct mlxsw_sp_fib_node, ht_node), 5593 .key_len = sizeof(struct mlxsw_sp_fib_key), 5594 .automatic_shrinking = true, 5595 }; 5596 5597 static int mlxsw_sp_fib_node_insert(struct mlxsw_sp_fib *fib, 5598 struct mlxsw_sp_fib_node *fib_node) 5599 { 5600 return rhashtable_insert_fast(&fib->ht, &fib_node->ht_node, 5601 mlxsw_sp_fib_ht_params); 5602 } 5603 5604 static void mlxsw_sp_fib_node_remove(struct mlxsw_sp_fib *fib, 5605 struct mlxsw_sp_fib_node *fib_node) 5606 { 5607 rhashtable_remove_fast(&fib->ht, &fib_node->ht_node, 5608 mlxsw_sp_fib_ht_params); 5609 } 5610 5611 static struct mlxsw_sp_fib_node * 5612 mlxsw_sp_fib_node_lookup(struct mlxsw_sp_fib *fib, const void *addr, 5613 size_t addr_len, unsigned char prefix_len) 5614 { 5615 struct mlxsw_sp_fib_key key; 5616 5617 memset(&key, 0, sizeof(key)); 5618 memcpy(key.addr, addr, addr_len); 5619 key.prefix_len = prefix_len; 5620 return rhashtable_lookup_fast(&fib->ht, &key, mlxsw_sp_fib_ht_params); 5621 } 5622 5623 static struct mlxsw_sp_fib_node * 5624 mlxsw_sp_fib_node_create(struct mlxsw_sp_fib *fib, const void *addr, 5625 size_t addr_len, unsigned char prefix_len) 5626 { 5627 struct mlxsw_sp_fib_node *fib_node; 5628 5629 fib_node = kzalloc(sizeof(*fib_node), GFP_KERNEL); 5630 if (!fib_node) 5631 return NULL; 5632 5633 list_add(&fib_node->list, &fib->node_list); 5634 memcpy(fib_node->key.addr, addr, addr_len); 5635 fib_node->key.prefix_len = prefix_len; 5636 5637 return fib_node; 5638 } 5639 5640 static void mlxsw_sp_fib_node_destroy(struct mlxsw_sp_fib_node *fib_node) 5641 { 5642 list_del(&fib_node->list); 5643 kfree(fib_node); 5644 } 5645 5646 static int mlxsw_sp_fib_lpm_tree_link(struct mlxsw_sp *mlxsw_sp, 5647 struct mlxsw_sp_fib_node *fib_node) 5648 { 5649 struct mlxsw_sp_prefix_usage req_prefix_usage; 5650 struct mlxsw_sp_fib *fib = fib_node->fib; 5651 struct mlxsw_sp_lpm_tree *lpm_tree; 5652 int err; 5653 5654 lpm_tree = mlxsw_sp->router->lpm.proto_trees[fib->proto]; 5655 if (lpm_tree->prefix_ref_count[fib_node->key.prefix_len] != 0) 5656 goto out; 5657 5658 mlxsw_sp_prefix_usage_cpy(&req_prefix_usage, &lpm_tree->prefix_usage); 5659 mlxsw_sp_prefix_usage_set(&req_prefix_usage, fib_node->key.prefix_len); 5660 lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage, 5661 fib->proto); 5662 if (IS_ERR(lpm_tree)) 5663 return PTR_ERR(lpm_tree); 5664 5665 err = mlxsw_sp_vrs_lpm_tree_replace(mlxsw_sp, fib, lpm_tree); 5666 if (err) 5667 goto err_lpm_tree_replace; 5668 5669 out: 5670 lpm_tree->prefix_ref_count[fib_node->key.prefix_len]++; 5671 return 0; 5672 5673 err_lpm_tree_replace: 5674 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree); 5675 return err; 5676 } 5677 5678 static void mlxsw_sp_fib_lpm_tree_unlink(struct mlxsw_sp *mlxsw_sp, 5679 struct mlxsw_sp_fib_node *fib_node) 5680 { 5681 struct mlxsw_sp_lpm_tree *lpm_tree = fib_node->fib->lpm_tree; 5682 struct mlxsw_sp_prefix_usage req_prefix_usage; 5683 struct mlxsw_sp_fib *fib = fib_node->fib; 5684 int err; 5685 5686 if (--lpm_tree->prefix_ref_count[fib_node->key.prefix_len] != 0) 5687 return; 5688 /* Try to construct a new LPM tree from the current prefix usage 5689 * minus the unused one. If we fail, continue using the old one. 5690 */ 5691 mlxsw_sp_prefix_usage_cpy(&req_prefix_usage, &lpm_tree->prefix_usage); 5692 mlxsw_sp_prefix_usage_clear(&req_prefix_usage, 5693 fib_node->key.prefix_len); 5694 lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage, 5695 fib->proto); 5696 if (IS_ERR(lpm_tree)) 5697 return; 5698 5699 err = mlxsw_sp_vrs_lpm_tree_replace(mlxsw_sp, fib, lpm_tree); 5700 if (err) 5701 goto err_lpm_tree_replace; 5702 5703 return; 5704 5705 err_lpm_tree_replace: 5706 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree); 5707 } 5708 5709 static int mlxsw_sp_fib_node_init(struct mlxsw_sp *mlxsw_sp, 5710 struct mlxsw_sp_fib_node *fib_node, 5711 struct mlxsw_sp_fib *fib) 5712 { 5713 int err; 5714 5715 err = mlxsw_sp_fib_node_insert(fib, fib_node); 5716 if (err) 5717 return err; 5718 fib_node->fib = fib; 5719 5720 err = mlxsw_sp_fib_lpm_tree_link(mlxsw_sp, fib_node); 5721 if (err) 5722 goto err_fib_lpm_tree_link; 5723 5724 return 0; 5725 5726 err_fib_lpm_tree_link: 5727 fib_node->fib = NULL; 5728 mlxsw_sp_fib_node_remove(fib, fib_node); 5729 return err; 5730 } 5731 5732 static void mlxsw_sp_fib_node_fini(struct mlxsw_sp *mlxsw_sp, 5733 struct mlxsw_sp_fib_node *fib_node) 5734 { 5735 struct mlxsw_sp_fib *fib = fib_node->fib; 5736 5737 mlxsw_sp_fib_lpm_tree_unlink(mlxsw_sp, fib_node); 5738 fib_node->fib = NULL; 5739 mlxsw_sp_fib_node_remove(fib, fib_node); 5740 } 5741 5742 static struct mlxsw_sp_fib_node * 5743 mlxsw_sp_fib_node_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id, const void *addr, 5744 size_t addr_len, unsigned char prefix_len, 5745 enum mlxsw_sp_l3proto proto) 5746 { 5747 struct mlxsw_sp_fib_node *fib_node; 5748 struct mlxsw_sp_fib *fib; 5749 struct mlxsw_sp_vr *vr; 5750 int err; 5751 5752 vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id, NULL); 5753 if (IS_ERR(vr)) 5754 return ERR_CAST(vr); 5755 fib = mlxsw_sp_vr_fib(vr, proto); 5756 5757 fib_node = mlxsw_sp_fib_node_lookup(fib, addr, addr_len, prefix_len); 5758 if (fib_node) 5759 return fib_node; 5760 5761 fib_node = mlxsw_sp_fib_node_create(fib, addr, addr_len, prefix_len); 5762 if (!fib_node) { 5763 err = -ENOMEM; 5764 goto err_fib_node_create; 5765 } 5766 5767 err = mlxsw_sp_fib_node_init(mlxsw_sp, fib_node, fib); 5768 if (err) 5769 goto err_fib_node_init; 5770 5771 return fib_node; 5772 5773 err_fib_node_init: 5774 mlxsw_sp_fib_node_destroy(fib_node); 5775 err_fib_node_create: 5776 mlxsw_sp_vr_put(mlxsw_sp, vr); 5777 return ERR_PTR(err); 5778 } 5779 5780 static void mlxsw_sp_fib_node_put(struct mlxsw_sp *mlxsw_sp, 5781 struct mlxsw_sp_fib_node *fib_node) 5782 { 5783 struct mlxsw_sp_vr *vr = fib_node->fib->vr; 5784 5785 if (fib_node->fib_entry) 5786 return; 5787 mlxsw_sp_fib_node_fini(mlxsw_sp, fib_node); 5788 mlxsw_sp_fib_node_destroy(fib_node); 5789 mlxsw_sp_vr_put(mlxsw_sp, vr); 5790 } 5791 5792 static int mlxsw_sp_fib_node_entry_link(struct mlxsw_sp *mlxsw_sp, 5793 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5794 struct mlxsw_sp_fib_entry *fib_entry) 5795 { 5796 struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node; 5797 bool is_new = !fib_node->fib_entry; 5798 int err; 5799 5800 fib_node->fib_entry = fib_entry; 5801 5802 err = __mlxsw_sp_fib_entry_update(mlxsw_sp, op_ctx, fib_entry, is_new); 5803 if (err) 5804 goto err_fib_entry_update; 5805 5806 return 0; 5807 5808 err_fib_entry_update: 5809 fib_node->fib_entry = NULL; 5810 return err; 5811 } 5812 5813 static int __mlxsw_sp_fib_node_entry_unlink(struct mlxsw_sp *mlxsw_sp, 5814 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5815 struct mlxsw_sp_fib_entry *fib_entry) 5816 { 5817 struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node; 5818 int err; 5819 5820 err = mlxsw_sp_fib_entry_del(mlxsw_sp, op_ctx, fib_entry); 5821 fib_node->fib_entry = NULL; 5822 return err; 5823 } 5824 5825 static void mlxsw_sp_fib_node_entry_unlink(struct mlxsw_sp *mlxsw_sp, 5826 struct mlxsw_sp_fib_entry *fib_entry) 5827 { 5828 struct mlxsw_sp_fib_entry_op_ctx *op_ctx = mlxsw_sp->router->ll_op_ctx; 5829 5830 mlxsw_sp_fib_entry_op_ctx_clear(op_ctx); 5831 __mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, op_ctx, fib_entry); 5832 } 5833 5834 static bool mlxsw_sp_fib4_allow_replace(struct mlxsw_sp_fib4_entry *fib4_entry) 5835 { 5836 struct mlxsw_sp_fib_node *fib_node = fib4_entry->common.fib_node; 5837 struct mlxsw_sp_fib4_entry *fib4_replaced; 5838 5839 if (!fib_node->fib_entry) 5840 return true; 5841 5842 fib4_replaced = container_of(fib_node->fib_entry, 5843 struct mlxsw_sp_fib4_entry, common); 5844 if (fib4_entry->tb_id == RT_TABLE_MAIN && 5845 fib4_replaced->tb_id == RT_TABLE_LOCAL) 5846 return false; 5847 5848 return true; 5849 } 5850 5851 static int 5852 mlxsw_sp_router_fib4_replace(struct mlxsw_sp *mlxsw_sp, 5853 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5854 const struct fib_entry_notifier_info *fen_info) 5855 { 5856 struct mlxsw_sp_fib4_entry *fib4_entry, *fib4_replaced; 5857 struct mlxsw_sp_fib_entry *replaced; 5858 struct mlxsw_sp_fib_node *fib_node; 5859 int err; 5860 5861 if (mlxsw_sp->router->aborted) 5862 return 0; 5863 5864 fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, fen_info->tb_id, 5865 &fen_info->dst, sizeof(fen_info->dst), 5866 fen_info->dst_len, 5867 MLXSW_SP_L3_PROTO_IPV4); 5868 if (IS_ERR(fib_node)) { 5869 dev_warn(mlxsw_sp->bus_info->dev, "Failed to get FIB node\n"); 5870 return PTR_ERR(fib_node); 5871 } 5872 5873 fib4_entry = mlxsw_sp_fib4_entry_create(mlxsw_sp, fib_node, fen_info); 5874 if (IS_ERR(fib4_entry)) { 5875 dev_warn(mlxsw_sp->bus_info->dev, "Failed to create FIB entry\n"); 5876 err = PTR_ERR(fib4_entry); 5877 goto err_fib4_entry_create; 5878 } 5879 5880 if (!mlxsw_sp_fib4_allow_replace(fib4_entry)) { 5881 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry); 5882 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 5883 return 0; 5884 } 5885 5886 replaced = fib_node->fib_entry; 5887 err = mlxsw_sp_fib_node_entry_link(mlxsw_sp, op_ctx, &fib4_entry->common); 5888 if (err) { 5889 dev_warn(mlxsw_sp->bus_info->dev, "Failed to link FIB entry to node\n"); 5890 goto err_fib_node_entry_link; 5891 } 5892 5893 /* Nothing to replace */ 5894 if (!replaced) 5895 return 0; 5896 5897 mlxsw_sp_fib_entry_hw_flags_clear(mlxsw_sp, replaced); 5898 fib4_replaced = container_of(replaced, struct mlxsw_sp_fib4_entry, 5899 common); 5900 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_replaced); 5901 5902 return 0; 5903 5904 err_fib_node_entry_link: 5905 fib_node->fib_entry = replaced; 5906 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry); 5907 err_fib4_entry_create: 5908 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 5909 return err; 5910 } 5911 5912 static int mlxsw_sp_router_fib4_del(struct mlxsw_sp *mlxsw_sp, 5913 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5914 struct fib_entry_notifier_info *fen_info) 5915 { 5916 struct mlxsw_sp_fib4_entry *fib4_entry; 5917 struct mlxsw_sp_fib_node *fib_node; 5918 int err; 5919 5920 if (mlxsw_sp->router->aborted) 5921 return 0; 5922 5923 fib4_entry = mlxsw_sp_fib4_entry_lookup(mlxsw_sp, fen_info); 5924 if (!fib4_entry) 5925 return 0; 5926 fib_node = fib4_entry->common.fib_node; 5927 5928 err = __mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, op_ctx, &fib4_entry->common); 5929 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry); 5930 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 5931 return err; 5932 } 5933 5934 static bool mlxsw_sp_fib6_rt_should_ignore(const struct fib6_info *rt) 5935 { 5936 /* Multicast routes aren't supported, so ignore them. Neighbour 5937 * Discovery packets are specifically trapped. 5938 */ 5939 if (ipv6_addr_type(&rt->fib6_dst.addr) & IPV6_ADDR_MULTICAST) 5940 return true; 5941 5942 /* Cloned routes are irrelevant in the forwarding path. */ 5943 if (rt->fib6_flags & RTF_CACHE) 5944 return true; 5945 5946 return false; 5947 } 5948 5949 static struct mlxsw_sp_rt6 *mlxsw_sp_rt6_create(struct fib6_info *rt) 5950 { 5951 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 5952 5953 mlxsw_sp_rt6 = kzalloc(sizeof(*mlxsw_sp_rt6), GFP_KERNEL); 5954 if (!mlxsw_sp_rt6) 5955 return ERR_PTR(-ENOMEM); 5956 5957 /* In case of route replace, replaced route is deleted with 5958 * no notification. Take reference to prevent accessing freed 5959 * memory. 5960 */ 5961 mlxsw_sp_rt6->rt = rt; 5962 fib6_info_hold(rt); 5963 5964 return mlxsw_sp_rt6; 5965 } 5966 5967 #if IS_ENABLED(CONFIG_IPV6) 5968 static void mlxsw_sp_rt6_release(struct fib6_info *rt) 5969 { 5970 fib6_info_release(rt); 5971 } 5972 #else 5973 static void mlxsw_sp_rt6_release(struct fib6_info *rt) 5974 { 5975 } 5976 #endif 5977 5978 static void mlxsw_sp_rt6_destroy(struct mlxsw_sp_rt6 *mlxsw_sp_rt6) 5979 { 5980 struct fib6_nh *fib6_nh = mlxsw_sp_rt6->rt->fib6_nh; 5981 5982 if (!mlxsw_sp_rt6->rt->nh) 5983 fib6_nh->fib_nh_flags &= ~RTNH_F_OFFLOAD; 5984 mlxsw_sp_rt6_release(mlxsw_sp_rt6->rt); 5985 kfree(mlxsw_sp_rt6); 5986 } 5987 5988 static struct fib6_info * 5989 mlxsw_sp_fib6_entry_rt(const struct mlxsw_sp_fib6_entry *fib6_entry) 5990 { 5991 return list_first_entry(&fib6_entry->rt6_list, struct mlxsw_sp_rt6, 5992 list)->rt; 5993 } 5994 5995 static struct mlxsw_sp_rt6 * 5996 mlxsw_sp_fib6_entry_rt_find(const struct mlxsw_sp_fib6_entry *fib6_entry, 5997 const struct fib6_info *rt) 5998 { 5999 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 6000 6001 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) { 6002 if (mlxsw_sp_rt6->rt == rt) 6003 return mlxsw_sp_rt6; 6004 } 6005 6006 return NULL; 6007 } 6008 6009 static bool mlxsw_sp_nexthop6_ipip_type(const struct mlxsw_sp *mlxsw_sp, 6010 const struct fib6_info *rt, 6011 enum mlxsw_sp_ipip_type *ret) 6012 { 6013 return rt->fib6_nh->fib_nh_dev && 6014 mlxsw_sp_netdev_ipip_type(mlxsw_sp, rt->fib6_nh->fib_nh_dev, ret); 6015 } 6016 6017 static int mlxsw_sp_nexthop6_init(struct mlxsw_sp *mlxsw_sp, 6018 struct mlxsw_sp_nexthop_group *nh_grp, 6019 struct mlxsw_sp_nexthop *nh, 6020 const struct fib6_info *rt) 6021 { 6022 struct net_device *dev = rt->fib6_nh->fib_nh_dev; 6023 6024 nh->nhgi = nh_grp->nhgi; 6025 nh->nh_weight = rt->fib6_nh->fib_nh_weight; 6026 memcpy(&nh->gw_addr, &rt->fib6_nh->fib_nh_gw6, sizeof(nh->gw_addr)); 6027 #if IS_ENABLED(CONFIG_IPV6) 6028 nh->neigh_tbl = &nd_tbl; 6029 #endif 6030 mlxsw_sp_nexthop_counter_alloc(mlxsw_sp, nh); 6031 6032 list_add_tail(&nh->router_list_node, &mlxsw_sp->router->nexthop_list); 6033 6034 if (!dev) 6035 return 0; 6036 nh->ifindex = dev->ifindex; 6037 6038 return mlxsw_sp_nexthop_type_init(mlxsw_sp, nh, dev); 6039 } 6040 6041 static void mlxsw_sp_nexthop6_fini(struct mlxsw_sp *mlxsw_sp, 6042 struct mlxsw_sp_nexthop *nh) 6043 { 6044 mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh); 6045 list_del(&nh->router_list_node); 6046 mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh); 6047 } 6048 6049 static bool mlxsw_sp_rt6_is_gateway(const struct mlxsw_sp *mlxsw_sp, 6050 const struct fib6_info *rt) 6051 { 6052 return rt->fib6_nh->fib_nh_gw_family || 6053 mlxsw_sp_nexthop6_ipip_type(mlxsw_sp, rt, NULL); 6054 } 6055 6056 static int 6057 mlxsw_sp_nexthop6_group_info_init(struct mlxsw_sp *mlxsw_sp, 6058 struct mlxsw_sp_nexthop_group *nh_grp, 6059 struct mlxsw_sp_fib6_entry *fib6_entry) 6060 { 6061 struct mlxsw_sp_nexthop_group_info *nhgi; 6062 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 6063 struct mlxsw_sp_nexthop *nh; 6064 int err, i; 6065 6066 nhgi = kzalloc(struct_size(nhgi, nexthops, fib6_entry->nrt6), 6067 GFP_KERNEL); 6068 if (!nhgi) 6069 return -ENOMEM; 6070 nh_grp->nhgi = nhgi; 6071 nhgi->nh_grp = nh_grp; 6072 mlxsw_sp_rt6 = list_first_entry(&fib6_entry->rt6_list, 6073 struct mlxsw_sp_rt6, list); 6074 nhgi->gateway = mlxsw_sp_rt6_is_gateway(mlxsw_sp, mlxsw_sp_rt6->rt); 6075 nhgi->count = fib6_entry->nrt6; 6076 for (i = 0; i < nhgi->count; i++) { 6077 struct fib6_info *rt = mlxsw_sp_rt6->rt; 6078 6079 nh = &nhgi->nexthops[i]; 6080 err = mlxsw_sp_nexthop6_init(mlxsw_sp, nh_grp, nh, rt); 6081 if (err) 6082 goto err_nexthop6_init; 6083 mlxsw_sp_rt6 = list_next_entry(mlxsw_sp_rt6, list); 6084 } 6085 nh_grp->nhgi = nhgi; 6086 err = mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp); 6087 if (err) 6088 goto err_group_refresh; 6089 6090 return 0; 6091 6092 err_group_refresh: 6093 i = nhgi->count; 6094 err_nexthop6_init: 6095 for (i--; i >= 0; i--) { 6096 nh = &nhgi->nexthops[i]; 6097 mlxsw_sp_nexthop6_fini(mlxsw_sp, nh); 6098 } 6099 kfree(nhgi); 6100 return err; 6101 } 6102 6103 static void 6104 mlxsw_sp_nexthop6_group_info_fini(struct mlxsw_sp *mlxsw_sp, 6105 struct mlxsw_sp_nexthop_group *nh_grp) 6106 { 6107 struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi; 6108 int i; 6109 6110 for (i = nhgi->count - 1; i >= 0; i--) { 6111 struct mlxsw_sp_nexthop *nh = &nhgi->nexthops[i]; 6112 6113 mlxsw_sp_nexthop6_fini(mlxsw_sp, nh); 6114 } 6115 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp); 6116 WARN_ON_ONCE(nhgi->adj_index_valid); 6117 kfree(nhgi); 6118 } 6119 6120 static struct mlxsw_sp_nexthop_group * 6121 mlxsw_sp_nexthop6_group_create(struct mlxsw_sp *mlxsw_sp, 6122 struct mlxsw_sp_fib6_entry *fib6_entry) 6123 { 6124 struct mlxsw_sp_nexthop_group *nh_grp; 6125 int err; 6126 6127 nh_grp = kzalloc(sizeof(*nh_grp), GFP_KERNEL); 6128 if (!nh_grp) 6129 return ERR_PTR(-ENOMEM); 6130 INIT_LIST_HEAD(&nh_grp->vr_list); 6131 err = rhashtable_init(&nh_grp->vr_ht, 6132 &mlxsw_sp_nexthop_group_vr_ht_params); 6133 if (err) 6134 goto err_nexthop_group_vr_ht_init; 6135 INIT_LIST_HEAD(&nh_grp->fib_list); 6136 nh_grp->type = MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6; 6137 6138 err = mlxsw_sp_nexthop6_group_info_init(mlxsw_sp, nh_grp, fib6_entry); 6139 if (err) 6140 goto err_nexthop_group_info_init; 6141 6142 err = mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp); 6143 if (err) 6144 goto err_nexthop_group_insert; 6145 6146 nh_grp->can_destroy = true; 6147 6148 return nh_grp; 6149 6150 err_nexthop_group_insert: 6151 mlxsw_sp_nexthop6_group_info_fini(mlxsw_sp, nh_grp); 6152 err_nexthop_group_info_init: 6153 rhashtable_destroy(&nh_grp->vr_ht); 6154 err_nexthop_group_vr_ht_init: 6155 kfree(nh_grp); 6156 return ERR_PTR(err); 6157 } 6158 6159 static void 6160 mlxsw_sp_nexthop6_group_destroy(struct mlxsw_sp *mlxsw_sp, 6161 struct mlxsw_sp_nexthop_group *nh_grp) 6162 { 6163 if (!nh_grp->can_destroy) 6164 return; 6165 mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp); 6166 mlxsw_sp_nexthop6_group_info_fini(mlxsw_sp, nh_grp); 6167 WARN_ON_ONCE(!list_empty(&nh_grp->vr_list)); 6168 rhashtable_destroy(&nh_grp->vr_ht); 6169 kfree(nh_grp); 6170 } 6171 6172 static int mlxsw_sp_nexthop6_group_get(struct mlxsw_sp *mlxsw_sp, 6173 struct mlxsw_sp_fib6_entry *fib6_entry) 6174 { 6175 struct fib6_info *rt = mlxsw_sp_fib6_entry_rt(fib6_entry); 6176 struct mlxsw_sp_nexthop_group *nh_grp; 6177 6178 if (rt->nh) { 6179 nh_grp = mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, 6180 rt->nh->id); 6181 if (WARN_ON_ONCE(!nh_grp)) 6182 return -EINVAL; 6183 goto out; 6184 } 6185 6186 nh_grp = mlxsw_sp_nexthop6_group_lookup(mlxsw_sp, fib6_entry); 6187 if (!nh_grp) { 6188 nh_grp = mlxsw_sp_nexthop6_group_create(mlxsw_sp, fib6_entry); 6189 if (IS_ERR(nh_grp)) 6190 return PTR_ERR(nh_grp); 6191 } 6192 6193 /* The route and the nexthop are described by the same struct, so we 6194 * need to the update the nexthop offload indication for the new route. 6195 */ 6196 __mlxsw_sp_nexthop6_group_offload_refresh(nh_grp, fib6_entry); 6197 6198 out: 6199 list_add_tail(&fib6_entry->common.nexthop_group_node, 6200 &nh_grp->fib_list); 6201 fib6_entry->common.nh_group = nh_grp; 6202 6203 return 0; 6204 } 6205 6206 static void mlxsw_sp_nexthop6_group_put(struct mlxsw_sp *mlxsw_sp, 6207 struct mlxsw_sp_fib_entry *fib_entry) 6208 { 6209 struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group; 6210 6211 list_del(&fib_entry->nexthop_group_node); 6212 if (!list_empty(&nh_grp->fib_list)) 6213 return; 6214 6215 if (nh_grp->type == MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ) { 6216 mlxsw_sp_nexthop_obj_group_destroy(mlxsw_sp, nh_grp); 6217 return; 6218 } 6219 6220 mlxsw_sp_nexthop6_group_destroy(mlxsw_sp, nh_grp); 6221 } 6222 6223 static int mlxsw_sp_nexthop6_group_update(struct mlxsw_sp *mlxsw_sp, 6224 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 6225 struct mlxsw_sp_fib6_entry *fib6_entry) 6226 { 6227 struct mlxsw_sp_nexthop_group *old_nh_grp = fib6_entry->common.nh_group; 6228 struct mlxsw_sp_fib_node *fib_node = fib6_entry->common.fib_node; 6229 int err; 6230 6231 mlxsw_sp_nexthop_group_vr_unlink(old_nh_grp, fib_node->fib); 6232 fib6_entry->common.nh_group = NULL; 6233 list_del(&fib6_entry->common.nexthop_group_node); 6234 6235 err = mlxsw_sp_nexthop6_group_get(mlxsw_sp, fib6_entry); 6236 if (err) 6237 goto err_nexthop6_group_get; 6238 6239 err = mlxsw_sp_nexthop_group_vr_link(fib6_entry->common.nh_group, 6240 fib_node->fib); 6241 if (err) 6242 goto err_nexthop_group_vr_link; 6243 6244 /* In case this entry is offloaded, then the adjacency index 6245 * currently associated with it in the device's table is that 6246 * of the old group. Start using the new one instead. 6247 */ 6248 err = __mlxsw_sp_fib_entry_update(mlxsw_sp, op_ctx, 6249 &fib6_entry->common, false); 6250 if (err) 6251 goto err_fib_entry_update; 6252 6253 if (list_empty(&old_nh_grp->fib_list)) 6254 mlxsw_sp_nexthop6_group_destroy(mlxsw_sp, old_nh_grp); 6255 6256 return 0; 6257 6258 err_fib_entry_update: 6259 mlxsw_sp_nexthop_group_vr_unlink(fib6_entry->common.nh_group, 6260 fib_node->fib); 6261 err_nexthop_group_vr_link: 6262 mlxsw_sp_nexthop6_group_put(mlxsw_sp, &fib6_entry->common); 6263 err_nexthop6_group_get: 6264 list_add_tail(&fib6_entry->common.nexthop_group_node, 6265 &old_nh_grp->fib_list); 6266 fib6_entry->common.nh_group = old_nh_grp; 6267 mlxsw_sp_nexthop_group_vr_link(old_nh_grp, fib_node->fib); 6268 return err; 6269 } 6270 6271 static int 6272 mlxsw_sp_fib6_entry_nexthop_add(struct mlxsw_sp *mlxsw_sp, 6273 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 6274 struct mlxsw_sp_fib6_entry *fib6_entry, 6275 struct fib6_info **rt_arr, unsigned int nrt6) 6276 { 6277 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 6278 int err, i; 6279 6280 for (i = 0; i < nrt6; i++) { 6281 mlxsw_sp_rt6 = mlxsw_sp_rt6_create(rt_arr[i]); 6282 if (IS_ERR(mlxsw_sp_rt6)) { 6283 err = PTR_ERR(mlxsw_sp_rt6); 6284 goto err_rt6_create; 6285 } 6286 6287 list_add_tail(&mlxsw_sp_rt6->list, &fib6_entry->rt6_list); 6288 fib6_entry->nrt6++; 6289 } 6290 6291 err = mlxsw_sp_nexthop6_group_update(mlxsw_sp, op_ctx, fib6_entry); 6292 if (err) 6293 goto err_nexthop6_group_update; 6294 6295 return 0; 6296 6297 err_nexthop6_group_update: 6298 i = nrt6; 6299 err_rt6_create: 6300 for (i--; i >= 0; i--) { 6301 fib6_entry->nrt6--; 6302 mlxsw_sp_rt6 = list_last_entry(&fib6_entry->rt6_list, 6303 struct mlxsw_sp_rt6, list); 6304 list_del(&mlxsw_sp_rt6->list); 6305 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6); 6306 } 6307 return err; 6308 } 6309 6310 static void 6311 mlxsw_sp_fib6_entry_nexthop_del(struct mlxsw_sp *mlxsw_sp, 6312 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 6313 struct mlxsw_sp_fib6_entry *fib6_entry, 6314 struct fib6_info **rt_arr, unsigned int nrt6) 6315 { 6316 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 6317 int i; 6318 6319 for (i = 0; i < nrt6; i++) { 6320 mlxsw_sp_rt6 = mlxsw_sp_fib6_entry_rt_find(fib6_entry, 6321 rt_arr[i]); 6322 if (WARN_ON_ONCE(!mlxsw_sp_rt6)) 6323 continue; 6324 6325 fib6_entry->nrt6--; 6326 list_del(&mlxsw_sp_rt6->list); 6327 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6); 6328 } 6329 6330 mlxsw_sp_nexthop6_group_update(mlxsw_sp, op_ctx, fib6_entry); 6331 } 6332 6333 static void mlxsw_sp_fib6_entry_type_set(struct mlxsw_sp *mlxsw_sp, 6334 struct mlxsw_sp_fib_entry *fib_entry, 6335 const struct fib6_info *rt) 6336 { 6337 if (rt->fib6_flags & (RTF_LOCAL | RTF_ANYCAST)) 6338 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP; 6339 else if (rt->fib6_type == RTN_BLACKHOLE) 6340 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE; 6341 else if (rt->fib6_flags & RTF_REJECT) 6342 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_UNREACHABLE; 6343 else if (fib_entry->nh_group->nhgi->gateway) 6344 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_REMOTE; 6345 else 6346 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL; 6347 } 6348 6349 static void 6350 mlxsw_sp_fib6_entry_rt_destroy_all(struct mlxsw_sp_fib6_entry *fib6_entry) 6351 { 6352 struct mlxsw_sp_rt6 *mlxsw_sp_rt6, *tmp; 6353 6354 list_for_each_entry_safe(mlxsw_sp_rt6, tmp, &fib6_entry->rt6_list, 6355 list) { 6356 fib6_entry->nrt6--; 6357 list_del(&mlxsw_sp_rt6->list); 6358 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6); 6359 } 6360 } 6361 6362 static struct mlxsw_sp_fib6_entry * 6363 mlxsw_sp_fib6_entry_create(struct mlxsw_sp *mlxsw_sp, 6364 struct mlxsw_sp_fib_node *fib_node, 6365 struct fib6_info **rt_arr, unsigned int nrt6) 6366 { 6367 struct mlxsw_sp_fib6_entry *fib6_entry; 6368 struct mlxsw_sp_fib_entry *fib_entry; 6369 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 6370 int err, i; 6371 6372 fib6_entry = kzalloc(sizeof(*fib6_entry), GFP_KERNEL); 6373 if (!fib6_entry) 6374 return ERR_PTR(-ENOMEM); 6375 fib_entry = &fib6_entry->common; 6376 6377 fib_entry->priv = mlxsw_sp_fib_entry_priv_create(fib_node->fib->ll_ops); 6378 if (IS_ERR(fib_entry->priv)) { 6379 err = PTR_ERR(fib_entry->priv); 6380 goto err_fib_entry_priv_create; 6381 } 6382 6383 INIT_LIST_HEAD(&fib6_entry->rt6_list); 6384 6385 for (i = 0; i < nrt6; i++) { 6386 mlxsw_sp_rt6 = mlxsw_sp_rt6_create(rt_arr[i]); 6387 if (IS_ERR(mlxsw_sp_rt6)) { 6388 err = PTR_ERR(mlxsw_sp_rt6); 6389 goto err_rt6_create; 6390 } 6391 list_add_tail(&mlxsw_sp_rt6->list, &fib6_entry->rt6_list); 6392 fib6_entry->nrt6++; 6393 } 6394 6395 err = mlxsw_sp_nexthop6_group_get(mlxsw_sp, fib6_entry); 6396 if (err) 6397 goto err_nexthop6_group_get; 6398 6399 err = mlxsw_sp_nexthop_group_vr_link(fib_entry->nh_group, 6400 fib_node->fib); 6401 if (err) 6402 goto err_nexthop_group_vr_link; 6403 6404 mlxsw_sp_fib6_entry_type_set(mlxsw_sp, fib_entry, rt_arr[0]); 6405 6406 fib_entry->fib_node = fib_node; 6407 6408 return fib6_entry; 6409 6410 err_nexthop_group_vr_link: 6411 mlxsw_sp_nexthop6_group_put(mlxsw_sp, fib_entry); 6412 err_nexthop6_group_get: 6413 i = nrt6; 6414 err_rt6_create: 6415 for (i--; i >= 0; i--) { 6416 fib6_entry->nrt6--; 6417 mlxsw_sp_rt6 = list_last_entry(&fib6_entry->rt6_list, 6418 struct mlxsw_sp_rt6, list); 6419 list_del(&mlxsw_sp_rt6->list); 6420 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6); 6421 } 6422 mlxsw_sp_fib_entry_priv_put(fib_entry->priv); 6423 err_fib_entry_priv_create: 6424 kfree(fib6_entry); 6425 return ERR_PTR(err); 6426 } 6427 6428 static void mlxsw_sp_fib6_entry_destroy(struct mlxsw_sp *mlxsw_sp, 6429 struct mlxsw_sp_fib6_entry *fib6_entry) 6430 { 6431 struct mlxsw_sp_fib_node *fib_node = fib6_entry->common.fib_node; 6432 6433 mlxsw_sp_nexthop_group_vr_unlink(fib6_entry->common.nh_group, 6434 fib_node->fib); 6435 mlxsw_sp_nexthop6_group_put(mlxsw_sp, &fib6_entry->common); 6436 mlxsw_sp_fib6_entry_rt_destroy_all(fib6_entry); 6437 WARN_ON(fib6_entry->nrt6); 6438 mlxsw_sp_fib_entry_priv_put(fib6_entry->common.priv); 6439 kfree(fib6_entry); 6440 } 6441 6442 static struct mlxsw_sp_fib6_entry * 6443 mlxsw_sp_fib6_entry_lookup(struct mlxsw_sp *mlxsw_sp, 6444 const struct fib6_info *rt) 6445 { 6446 struct mlxsw_sp_fib6_entry *fib6_entry; 6447 struct mlxsw_sp_fib_node *fib_node; 6448 struct mlxsw_sp_fib *fib; 6449 struct fib6_info *cmp_rt; 6450 struct mlxsw_sp_vr *vr; 6451 6452 vr = mlxsw_sp_vr_find(mlxsw_sp, rt->fib6_table->tb6_id); 6453 if (!vr) 6454 return NULL; 6455 fib = mlxsw_sp_vr_fib(vr, MLXSW_SP_L3_PROTO_IPV6); 6456 6457 fib_node = mlxsw_sp_fib_node_lookup(fib, &rt->fib6_dst.addr, 6458 sizeof(rt->fib6_dst.addr), 6459 rt->fib6_dst.plen); 6460 if (!fib_node) 6461 return NULL; 6462 6463 fib6_entry = container_of(fib_node->fib_entry, 6464 struct mlxsw_sp_fib6_entry, common); 6465 cmp_rt = mlxsw_sp_fib6_entry_rt(fib6_entry); 6466 if (rt->fib6_table->tb6_id == cmp_rt->fib6_table->tb6_id && 6467 rt->fib6_metric == cmp_rt->fib6_metric && 6468 mlxsw_sp_fib6_entry_rt_find(fib6_entry, rt)) 6469 return fib6_entry; 6470 6471 return NULL; 6472 } 6473 6474 static bool mlxsw_sp_fib6_allow_replace(struct mlxsw_sp_fib6_entry *fib6_entry) 6475 { 6476 struct mlxsw_sp_fib_node *fib_node = fib6_entry->common.fib_node; 6477 struct mlxsw_sp_fib6_entry *fib6_replaced; 6478 struct fib6_info *rt, *rt_replaced; 6479 6480 if (!fib_node->fib_entry) 6481 return true; 6482 6483 fib6_replaced = container_of(fib_node->fib_entry, 6484 struct mlxsw_sp_fib6_entry, 6485 common); 6486 rt = mlxsw_sp_fib6_entry_rt(fib6_entry); 6487 rt_replaced = mlxsw_sp_fib6_entry_rt(fib6_replaced); 6488 if (rt->fib6_table->tb6_id == RT_TABLE_MAIN && 6489 rt_replaced->fib6_table->tb6_id == RT_TABLE_LOCAL) 6490 return false; 6491 6492 return true; 6493 } 6494 6495 static int mlxsw_sp_router_fib6_replace(struct mlxsw_sp *mlxsw_sp, 6496 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 6497 struct fib6_info **rt_arr, unsigned int nrt6) 6498 { 6499 struct mlxsw_sp_fib6_entry *fib6_entry, *fib6_replaced; 6500 struct mlxsw_sp_fib_entry *replaced; 6501 struct mlxsw_sp_fib_node *fib_node; 6502 struct fib6_info *rt = rt_arr[0]; 6503 int err; 6504 6505 if (mlxsw_sp->router->aborted) 6506 return 0; 6507 6508 if (rt->fib6_src.plen) 6509 return -EINVAL; 6510 6511 if (mlxsw_sp_fib6_rt_should_ignore(rt)) 6512 return 0; 6513 6514 fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, rt->fib6_table->tb6_id, 6515 &rt->fib6_dst.addr, 6516 sizeof(rt->fib6_dst.addr), 6517 rt->fib6_dst.plen, 6518 MLXSW_SP_L3_PROTO_IPV6); 6519 if (IS_ERR(fib_node)) 6520 return PTR_ERR(fib_node); 6521 6522 fib6_entry = mlxsw_sp_fib6_entry_create(mlxsw_sp, fib_node, rt_arr, 6523 nrt6); 6524 if (IS_ERR(fib6_entry)) { 6525 err = PTR_ERR(fib6_entry); 6526 goto err_fib6_entry_create; 6527 } 6528 6529 if (!mlxsw_sp_fib6_allow_replace(fib6_entry)) { 6530 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry); 6531 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 6532 return 0; 6533 } 6534 6535 replaced = fib_node->fib_entry; 6536 err = mlxsw_sp_fib_node_entry_link(mlxsw_sp, op_ctx, &fib6_entry->common); 6537 if (err) 6538 goto err_fib_node_entry_link; 6539 6540 /* Nothing to replace */ 6541 if (!replaced) 6542 return 0; 6543 6544 mlxsw_sp_fib_entry_hw_flags_clear(mlxsw_sp, replaced); 6545 fib6_replaced = container_of(replaced, struct mlxsw_sp_fib6_entry, 6546 common); 6547 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_replaced); 6548 6549 return 0; 6550 6551 err_fib_node_entry_link: 6552 fib_node->fib_entry = replaced; 6553 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry); 6554 err_fib6_entry_create: 6555 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 6556 return err; 6557 } 6558 6559 static int mlxsw_sp_router_fib6_append(struct mlxsw_sp *mlxsw_sp, 6560 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 6561 struct fib6_info **rt_arr, unsigned int nrt6) 6562 { 6563 struct mlxsw_sp_fib6_entry *fib6_entry; 6564 struct mlxsw_sp_fib_node *fib_node; 6565 struct fib6_info *rt = rt_arr[0]; 6566 int err; 6567 6568 if (mlxsw_sp->router->aborted) 6569 return 0; 6570 6571 if (rt->fib6_src.plen) 6572 return -EINVAL; 6573 6574 if (mlxsw_sp_fib6_rt_should_ignore(rt)) 6575 return 0; 6576 6577 fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, rt->fib6_table->tb6_id, 6578 &rt->fib6_dst.addr, 6579 sizeof(rt->fib6_dst.addr), 6580 rt->fib6_dst.plen, 6581 MLXSW_SP_L3_PROTO_IPV6); 6582 if (IS_ERR(fib_node)) 6583 return PTR_ERR(fib_node); 6584 6585 if (WARN_ON_ONCE(!fib_node->fib_entry)) { 6586 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 6587 return -EINVAL; 6588 } 6589 6590 fib6_entry = container_of(fib_node->fib_entry, 6591 struct mlxsw_sp_fib6_entry, common); 6592 err = mlxsw_sp_fib6_entry_nexthop_add(mlxsw_sp, op_ctx, fib6_entry, rt_arr, nrt6); 6593 if (err) 6594 goto err_fib6_entry_nexthop_add; 6595 6596 return 0; 6597 6598 err_fib6_entry_nexthop_add: 6599 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 6600 return err; 6601 } 6602 6603 static int mlxsw_sp_router_fib6_del(struct mlxsw_sp *mlxsw_sp, 6604 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 6605 struct fib6_info **rt_arr, unsigned int nrt6) 6606 { 6607 struct mlxsw_sp_fib6_entry *fib6_entry; 6608 struct mlxsw_sp_fib_node *fib_node; 6609 struct fib6_info *rt = rt_arr[0]; 6610 int err; 6611 6612 if (mlxsw_sp->router->aborted) 6613 return 0; 6614 6615 if (mlxsw_sp_fib6_rt_should_ignore(rt)) 6616 return 0; 6617 6618 /* Multipath routes are first added to the FIB trie and only then 6619 * notified. If we vetoed the addition, we will get a delete 6620 * notification for a route we do not have. Therefore, do not warn if 6621 * route was not found. 6622 */ 6623 fib6_entry = mlxsw_sp_fib6_entry_lookup(mlxsw_sp, rt); 6624 if (!fib6_entry) 6625 return 0; 6626 6627 /* If not all the nexthops are deleted, then only reduce the nexthop 6628 * group. 6629 */ 6630 if (nrt6 != fib6_entry->nrt6) { 6631 mlxsw_sp_fib6_entry_nexthop_del(mlxsw_sp, op_ctx, fib6_entry, rt_arr, nrt6); 6632 return 0; 6633 } 6634 6635 fib_node = fib6_entry->common.fib_node; 6636 6637 err = __mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, op_ctx, &fib6_entry->common); 6638 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry); 6639 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 6640 return err; 6641 } 6642 6643 static int __mlxsw_sp_router_set_abort_trap(struct mlxsw_sp *mlxsw_sp, 6644 enum mlxsw_sp_l3proto proto, 6645 u8 tree_id) 6646 { 6647 const struct mlxsw_sp_router_ll_ops *ll_ops = mlxsw_sp->router->proto_ll_ops[proto]; 6648 enum mlxsw_reg_ralxx_protocol ralxx_proto = 6649 (enum mlxsw_reg_ralxx_protocol) proto; 6650 struct mlxsw_sp_fib_entry_priv *priv; 6651 char xralta_pl[MLXSW_REG_XRALTA_LEN]; 6652 char xralst_pl[MLXSW_REG_XRALST_LEN]; 6653 int i, err; 6654 6655 mlxsw_reg_xralta_pack(xralta_pl, true, ralxx_proto, tree_id); 6656 err = ll_ops->ralta_write(mlxsw_sp, xralta_pl); 6657 if (err) 6658 return err; 6659 6660 mlxsw_reg_xralst_pack(xralst_pl, 0xff, tree_id); 6661 err = ll_ops->ralst_write(mlxsw_sp, xralst_pl); 6662 if (err) 6663 return err; 6664 6665 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) { 6666 struct mlxsw_sp_fib_entry_op_ctx *op_ctx = mlxsw_sp->router->ll_op_ctx; 6667 struct mlxsw_sp_vr *vr = &mlxsw_sp->router->vrs[i]; 6668 char xraltb_pl[MLXSW_REG_XRALTB_LEN]; 6669 6670 mlxsw_sp_fib_entry_op_ctx_clear(op_ctx); 6671 mlxsw_reg_xraltb_pack(xraltb_pl, vr->id, ralxx_proto, tree_id); 6672 err = ll_ops->raltb_write(mlxsw_sp, xraltb_pl); 6673 if (err) 6674 return err; 6675 6676 priv = mlxsw_sp_fib_entry_priv_create(ll_ops); 6677 if (IS_ERR(priv)) 6678 return PTR_ERR(priv); 6679 6680 ll_ops->fib_entry_pack(op_ctx, proto, MLXSW_SP_FIB_ENTRY_OP_WRITE, 6681 vr->id, 0, NULL, priv); 6682 ll_ops->fib_entry_act_ip2me_pack(op_ctx); 6683 err = ll_ops->fib_entry_commit(mlxsw_sp, op_ctx, NULL); 6684 mlxsw_sp_fib_entry_priv_put(priv); 6685 if (err) 6686 return err; 6687 } 6688 6689 return 0; 6690 } 6691 6692 static struct mlxsw_sp_mr_table * 6693 mlxsw_sp_router_fibmr_family_to_table(struct mlxsw_sp_vr *vr, int family) 6694 { 6695 if (family == RTNL_FAMILY_IPMR) 6696 return vr->mr_table[MLXSW_SP_L3_PROTO_IPV4]; 6697 else 6698 return vr->mr_table[MLXSW_SP_L3_PROTO_IPV6]; 6699 } 6700 6701 static int mlxsw_sp_router_fibmr_add(struct mlxsw_sp *mlxsw_sp, 6702 struct mfc_entry_notifier_info *men_info, 6703 bool replace) 6704 { 6705 struct mlxsw_sp_mr_table *mrt; 6706 struct mlxsw_sp_vr *vr; 6707 6708 if (mlxsw_sp->router->aborted) 6709 return 0; 6710 6711 vr = mlxsw_sp_vr_get(mlxsw_sp, men_info->tb_id, NULL); 6712 if (IS_ERR(vr)) 6713 return PTR_ERR(vr); 6714 6715 mrt = mlxsw_sp_router_fibmr_family_to_table(vr, men_info->info.family); 6716 return mlxsw_sp_mr_route_add(mrt, men_info->mfc, replace); 6717 } 6718 6719 static void mlxsw_sp_router_fibmr_del(struct mlxsw_sp *mlxsw_sp, 6720 struct mfc_entry_notifier_info *men_info) 6721 { 6722 struct mlxsw_sp_mr_table *mrt; 6723 struct mlxsw_sp_vr *vr; 6724 6725 if (mlxsw_sp->router->aborted) 6726 return; 6727 6728 vr = mlxsw_sp_vr_find(mlxsw_sp, men_info->tb_id); 6729 if (WARN_ON(!vr)) 6730 return; 6731 6732 mrt = mlxsw_sp_router_fibmr_family_to_table(vr, men_info->info.family); 6733 mlxsw_sp_mr_route_del(mrt, men_info->mfc); 6734 mlxsw_sp_vr_put(mlxsw_sp, vr); 6735 } 6736 6737 static int 6738 mlxsw_sp_router_fibmr_vif_add(struct mlxsw_sp *mlxsw_sp, 6739 struct vif_entry_notifier_info *ven_info) 6740 { 6741 struct mlxsw_sp_mr_table *mrt; 6742 struct mlxsw_sp_rif *rif; 6743 struct mlxsw_sp_vr *vr; 6744 6745 if (mlxsw_sp->router->aborted) 6746 return 0; 6747 6748 vr = mlxsw_sp_vr_get(mlxsw_sp, ven_info->tb_id, NULL); 6749 if (IS_ERR(vr)) 6750 return PTR_ERR(vr); 6751 6752 mrt = mlxsw_sp_router_fibmr_family_to_table(vr, ven_info->info.family); 6753 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, ven_info->dev); 6754 return mlxsw_sp_mr_vif_add(mrt, ven_info->dev, 6755 ven_info->vif_index, 6756 ven_info->vif_flags, rif); 6757 } 6758 6759 static void 6760 mlxsw_sp_router_fibmr_vif_del(struct mlxsw_sp *mlxsw_sp, 6761 struct vif_entry_notifier_info *ven_info) 6762 { 6763 struct mlxsw_sp_mr_table *mrt; 6764 struct mlxsw_sp_vr *vr; 6765 6766 if (mlxsw_sp->router->aborted) 6767 return; 6768 6769 vr = mlxsw_sp_vr_find(mlxsw_sp, ven_info->tb_id); 6770 if (WARN_ON(!vr)) 6771 return; 6772 6773 mrt = mlxsw_sp_router_fibmr_family_to_table(vr, ven_info->info.family); 6774 mlxsw_sp_mr_vif_del(mrt, ven_info->vif_index); 6775 mlxsw_sp_vr_put(mlxsw_sp, vr); 6776 } 6777 6778 static int mlxsw_sp_router_set_abort_trap(struct mlxsw_sp *mlxsw_sp) 6779 { 6780 enum mlxsw_sp_l3proto proto = MLXSW_SP_L3_PROTO_IPV4; 6781 int err; 6782 6783 err = __mlxsw_sp_router_set_abort_trap(mlxsw_sp, proto, 6784 MLXSW_SP_LPM_TREE_MIN); 6785 if (err) 6786 return err; 6787 6788 /* The multicast router code does not need an abort trap as by default, 6789 * packets that don't match any routes are trapped to the CPU. 6790 */ 6791 6792 proto = MLXSW_SP_L3_PROTO_IPV6; 6793 return __mlxsw_sp_router_set_abort_trap(mlxsw_sp, proto, 6794 MLXSW_SP_LPM_TREE_MIN + 1); 6795 } 6796 6797 static void mlxsw_sp_fib4_node_flush(struct mlxsw_sp *mlxsw_sp, 6798 struct mlxsw_sp_fib_node *fib_node) 6799 { 6800 struct mlxsw_sp_fib4_entry *fib4_entry; 6801 6802 fib4_entry = container_of(fib_node->fib_entry, 6803 struct mlxsw_sp_fib4_entry, common); 6804 mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, fib_node->fib_entry); 6805 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry); 6806 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 6807 } 6808 6809 static void mlxsw_sp_fib6_node_flush(struct mlxsw_sp *mlxsw_sp, 6810 struct mlxsw_sp_fib_node *fib_node) 6811 { 6812 struct mlxsw_sp_fib6_entry *fib6_entry; 6813 6814 fib6_entry = container_of(fib_node->fib_entry, 6815 struct mlxsw_sp_fib6_entry, common); 6816 mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, fib_node->fib_entry); 6817 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry); 6818 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 6819 } 6820 6821 static void mlxsw_sp_fib_node_flush(struct mlxsw_sp *mlxsw_sp, 6822 struct mlxsw_sp_fib_node *fib_node) 6823 { 6824 switch (fib_node->fib->proto) { 6825 case MLXSW_SP_L3_PROTO_IPV4: 6826 mlxsw_sp_fib4_node_flush(mlxsw_sp, fib_node); 6827 break; 6828 case MLXSW_SP_L3_PROTO_IPV6: 6829 mlxsw_sp_fib6_node_flush(mlxsw_sp, fib_node); 6830 break; 6831 } 6832 } 6833 6834 static void mlxsw_sp_vr_fib_flush(struct mlxsw_sp *mlxsw_sp, 6835 struct mlxsw_sp_vr *vr, 6836 enum mlxsw_sp_l3proto proto) 6837 { 6838 struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto); 6839 struct mlxsw_sp_fib_node *fib_node, *tmp; 6840 6841 list_for_each_entry_safe(fib_node, tmp, &fib->node_list, list) { 6842 bool do_break = &tmp->list == &fib->node_list; 6843 6844 mlxsw_sp_fib_node_flush(mlxsw_sp, fib_node); 6845 if (do_break) 6846 break; 6847 } 6848 } 6849 6850 static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp) 6851 { 6852 int i, j; 6853 6854 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) { 6855 struct mlxsw_sp_vr *vr = &mlxsw_sp->router->vrs[i]; 6856 6857 if (!mlxsw_sp_vr_is_used(vr)) 6858 continue; 6859 6860 for (j = 0; j < MLXSW_SP_L3_PROTO_MAX; j++) 6861 mlxsw_sp_mr_table_flush(vr->mr_table[j]); 6862 mlxsw_sp_vr_fib_flush(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV4); 6863 6864 /* If virtual router was only used for IPv4, then it's no 6865 * longer used. 6866 */ 6867 if (!mlxsw_sp_vr_is_used(vr)) 6868 continue; 6869 mlxsw_sp_vr_fib_flush(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV6); 6870 } 6871 6872 /* After flushing all the routes, it is not possible anyone is still 6873 * using the adjacency index that is discarding packets, so free it in 6874 * case it was allocated. 6875 */ 6876 if (!mlxsw_sp->router->adj_discard_index_valid) 6877 return; 6878 mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1, 6879 mlxsw_sp->router->adj_discard_index); 6880 mlxsw_sp->router->adj_discard_index_valid = false; 6881 } 6882 6883 static void mlxsw_sp_router_fib_abort(struct mlxsw_sp *mlxsw_sp) 6884 { 6885 int err; 6886 6887 if (mlxsw_sp->router->aborted) 6888 return; 6889 dev_warn(mlxsw_sp->bus_info->dev, "FIB abort triggered. Note that FIB entries are no longer being offloaded to this device.\n"); 6890 mlxsw_sp_router_fib_flush(mlxsw_sp); 6891 mlxsw_sp->router->aborted = true; 6892 err = mlxsw_sp_router_set_abort_trap(mlxsw_sp); 6893 if (err) 6894 dev_warn(mlxsw_sp->bus_info->dev, "Failed to set abort trap.\n"); 6895 } 6896 6897 struct mlxsw_sp_fib6_event { 6898 struct fib6_info **rt_arr; 6899 unsigned int nrt6; 6900 }; 6901 6902 struct mlxsw_sp_fib_event { 6903 struct list_head list; /* node in fib queue */ 6904 union { 6905 struct mlxsw_sp_fib6_event fib6_event; 6906 struct fib_entry_notifier_info fen_info; 6907 struct fib_rule_notifier_info fr_info; 6908 struct fib_nh_notifier_info fnh_info; 6909 struct mfc_entry_notifier_info men_info; 6910 struct vif_entry_notifier_info ven_info; 6911 }; 6912 struct mlxsw_sp *mlxsw_sp; 6913 unsigned long event; 6914 int family; 6915 }; 6916 6917 static int 6918 mlxsw_sp_router_fib6_event_init(struct mlxsw_sp_fib6_event *fib6_event, 6919 struct fib6_entry_notifier_info *fen6_info) 6920 { 6921 struct fib6_info *rt = fen6_info->rt; 6922 struct fib6_info **rt_arr; 6923 struct fib6_info *iter; 6924 unsigned int nrt6; 6925 int i = 0; 6926 6927 nrt6 = fen6_info->nsiblings + 1; 6928 6929 rt_arr = kcalloc(nrt6, sizeof(struct fib6_info *), GFP_ATOMIC); 6930 if (!rt_arr) 6931 return -ENOMEM; 6932 6933 fib6_event->rt_arr = rt_arr; 6934 fib6_event->nrt6 = nrt6; 6935 6936 rt_arr[0] = rt; 6937 fib6_info_hold(rt); 6938 6939 if (!fen6_info->nsiblings) 6940 return 0; 6941 6942 list_for_each_entry(iter, &rt->fib6_siblings, fib6_siblings) { 6943 if (i == fen6_info->nsiblings) 6944 break; 6945 6946 rt_arr[i + 1] = iter; 6947 fib6_info_hold(iter); 6948 i++; 6949 } 6950 WARN_ON_ONCE(i != fen6_info->nsiblings); 6951 6952 return 0; 6953 } 6954 6955 static void 6956 mlxsw_sp_router_fib6_event_fini(struct mlxsw_sp_fib6_event *fib6_event) 6957 { 6958 int i; 6959 6960 for (i = 0; i < fib6_event->nrt6; i++) 6961 mlxsw_sp_rt6_release(fib6_event->rt_arr[i]); 6962 kfree(fib6_event->rt_arr); 6963 } 6964 6965 static void mlxsw_sp_router_fib4_event_process(struct mlxsw_sp *mlxsw_sp, 6966 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 6967 struct mlxsw_sp_fib_event *fib_event) 6968 { 6969 int err; 6970 6971 mlxsw_sp_span_respin(mlxsw_sp); 6972 6973 switch (fib_event->event) { 6974 case FIB_EVENT_ENTRY_REPLACE: 6975 err = mlxsw_sp_router_fib4_replace(mlxsw_sp, op_ctx, &fib_event->fen_info); 6976 if (err) { 6977 mlxsw_sp_fib_entry_op_ctx_priv_put_all(op_ctx); 6978 mlxsw_sp_router_fib_abort(mlxsw_sp); 6979 } 6980 fib_info_put(fib_event->fen_info.fi); 6981 break; 6982 case FIB_EVENT_ENTRY_DEL: 6983 err = mlxsw_sp_router_fib4_del(mlxsw_sp, op_ctx, &fib_event->fen_info); 6984 if (err) 6985 mlxsw_sp_fib_entry_op_ctx_priv_put_all(op_ctx); 6986 fib_info_put(fib_event->fen_info.fi); 6987 break; 6988 case FIB_EVENT_NH_ADD: 6989 case FIB_EVENT_NH_DEL: 6990 mlxsw_sp_nexthop4_event(mlxsw_sp, fib_event->event, fib_event->fnh_info.fib_nh); 6991 fib_info_put(fib_event->fnh_info.fib_nh->nh_parent); 6992 break; 6993 } 6994 } 6995 6996 static void mlxsw_sp_router_fib6_event_process(struct mlxsw_sp *mlxsw_sp, 6997 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 6998 struct mlxsw_sp_fib_event *fib_event) 6999 { 7000 int err; 7001 7002 mlxsw_sp_span_respin(mlxsw_sp); 7003 7004 switch (fib_event->event) { 7005 case FIB_EVENT_ENTRY_REPLACE: 7006 err = mlxsw_sp_router_fib6_replace(mlxsw_sp, op_ctx, fib_event->fib6_event.rt_arr, 7007 fib_event->fib6_event.nrt6); 7008 if (err) { 7009 mlxsw_sp_fib_entry_op_ctx_priv_put_all(op_ctx); 7010 mlxsw_sp_router_fib_abort(mlxsw_sp); 7011 } 7012 mlxsw_sp_router_fib6_event_fini(&fib_event->fib6_event); 7013 break; 7014 case FIB_EVENT_ENTRY_APPEND: 7015 err = mlxsw_sp_router_fib6_append(mlxsw_sp, op_ctx, fib_event->fib6_event.rt_arr, 7016 fib_event->fib6_event.nrt6); 7017 if (err) { 7018 mlxsw_sp_fib_entry_op_ctx_priv_put_all(op_ctx); 7019 mlxsw_sp_router_fib_abort(mlxsw_sp); 7020 } 7021 mlxsw_sp_router_fib6_event_fini(&fib_event->fib6_event); 7022 break; 7023 case FIB_EVENT_ENTRY_DEL: 7024 err = mlxsw_sp_router_fib6_del(mlxsw_sp, op_ctx, fib_event->fib6_event.rt_arr, 7025 fib_event->fib6_event.nrt6); 7026 if (err) 7027 mlxsw_sp_fib_entry_op_ctx_priv_put_all(op_ctx); 7028 mlxsw_sp_router_fib6_event_fini(&fib_event->fib6_event); 7029 break; 7030 } 7031 } 7032 7033 static void mlxsw_sp_router_fibmr_event_process(struct mlxsw_sp *mlxsw_sp, 7034 struct mlxsw_sp_fib_event *fib_event) 7035 { 7036 bool replace; 7037 int err; 7038 7039 rtnl_lock(); 7040 mutex_lock(&mlxsw_sp->router->lock); 7041 switch (fib_event->event) { 7042 case FIB_EVENT_ENTRY_REPLACE: 7043 case FIB_EVENT_ENTRY_ADD: 7044 replace = fib_event->event == FIB_EVENT_ENTRY_REPLACE; 7045 7046 err = mlxsw_sp_router_fibmr_add(mlxsw_sp, &fib_event->men_info, replace); 7047 if (err) 7048 mlxsw_sp_router_fib_abort(mlxsw_sp); 7049 mr_cache_put(fib_event->men_info.mfc); 7050 break; 7051 case FIB_EVENT_ENTRY_DEL: 7052 mlxsw_sp_router_fibmr_del(mlxsw_sp, &fib_event->men_info); 7053 mr_cache_put(fib_event->men_info.mfc); 7054 break; 7055 case FIB_EVENT_VIF_ADD: 7056 err = mlxsw_sp_router_fibmr_vif_add(mlxsw_sp, 7057 &fib_event->ven_info); 7058 if (err) 7059 mlxsw_sp_router_fib_abort(mlxsw_sp); 7060 dev_put(fib_event->ven_info.dev); 7061 break; 7062 case FIB_EVENT_VIF_DEL: 7063 mlxsw_sp_router_fibmr_vif_del(mlxsw_sp, &fib_event->ven_info); 7064 dev_put(fib_event->ven_info.dev); 7065 break; 7066 } 7067 mutex_unlock(&mlxsw_sp->router->lock); 7068 rtnl_unlock(); 7069 } 7070 7071 static void mlxsw_sp_router_fib_event_work(struct work_struct *work) 7072 { 7073 struct mlxsw_sp_router *router = container_of(work, struct mlxsw_sp_router, fib_event_work); 7074 struct mlxsw_sp_fib_entry_op_ctx *op_ctx = router->ll_op_ctx; 7075 struct mlxsw_sp *mlxsw_sp = router->mlxsw_sp; 7076 struct mlxsw_sp_fib_event *next_fib_event; 7077 struct mlxsw_sp_fib_event *fib_event; 7078 int last_family = AF_UNSPEC; 7079 LIST_HEAD(fib_event_queue); 7080 7081 spin_lock_bh(&router->fib_event_queue_lock); 7082 list_splice_init(&router->fib_event_queue, &fib_event_queue); 7083 spin_unlock_bh(&router->fib_event_queue_lock); 7084 7085 /* Router lock is held here to make sure per-instance 7086 * operation context is not used in between FIB4/6 events 7087 * processing. 7088 */ 7089 mutex_lock(&router->lock); 7090 mlxsw_sp_fib_entry_op_ctx_clear(op_ctx); 7091 list_for_each_entry_safe(fib_event, next_fib_event, 7092 &fib_event_queue, list) { 7093 /* Check if the next entry in the queue exists and it is 7094 * of the same type (family and event) as the currect one. 7095 * In that case it is permitted to do the bulking 7096 * of multiple FIB entries to a single register write. 7097 */ 7098 op_ctx->bulk_ok = !list_is_last(&fib_event->list, &fib_event_queue) && 7099 fib_event->family == next_fib_event->family && 7100 fib_event->event == next_fib_event->event; 7101 op_ctx->event = fib_event->event; 7102 7103 /* In case family of this and the previous entry are different, context 7104 * reinitialization is going to be needed now, indicate that. 7105 * Note that since last_family is initialized to AF_UNSPEC, this is always 7106 * going to happen for the first entry processed in the work. 7107 */ 7108 if (fib_event->family != last_family) 7109 op_ctx->initialized = false; 7110 7111 switch (fib_event->family) { 7112 case AF_INET: 7113 mlxsw_sp_router_fib4_event_process(mlxsw_sp, op_ctx, 7114 fib_event); 7115 break; 7116 case AF_INET6: 7117 mlxsw_sp_router_fib6_event_process(mlxsw_sp, op_ctx, 7118 fib_event); 7119 break; 7120 case RTNL_FAMILY_IP6MR: 7121 case RTNL_FAMILY_IPMR: 7122 /* Unlock here as inside FIBMR the lock is taken again 7123 * under RTNL. The per-instance operation context 7124 * is not used by FIBMR. 7125 */ 7126 mutex_unlock(&router->lock); 7127 mlxsw_sp_router_fibmr_event_process(mlxsw_sp, 7128 fib_event); 7129 mutex_lock(&router->lock); 7130 break; 7131 default: 7132 WARN_ON_ONCE(1); 7133 } 7134 last_family = fib_event->family; 7135 kfree(fib_event); 7136 cond_resched(); 7137 } 7138 WARN_ON_ONCE(!list_empty(&router->ll_op_ctx->fib_entry_priv_list)); 7139 mutex_unlock(&router->lock); 7140 } 7141 7142 static void mlxsw_sp_router_fib4_event(struct mlxsw_sp_fib_event *fib_event, 7143 struct fib_notifier_info *info) 7144 { 7145 struct fib_entry_notifier_info *fen_info; 7146 struct fib_nh_notifier_info *fnh_info; 7147 7148 switch (fib_event->event) { 7149 case FIB_EVENT_ENTRY_REPLACE: 7150 case FIB_EVENT_ENTRY_DEL: 7151 fen_info = container_of(info, struct fib_entry_notifier_info, 7152 info); 7153 fib_event->fen_info = *fen_info; 7154 /* Take reference on fib_info to prevent it from being 7155 * freed while event is queued. Release it afterwards. 7156 */ 7157 fib_info_hold(fib_event->fen_info.fi); 7158 break; 7159 case FIB_EVENT_NH_ADD: 7160 case FIB_EVENT_NH_DEL: 7161 fnh_info = container_of(info, struct fib_nh_notifier_info, 7162 info); 7163 fib_event->fnh_info = *fnh_info; 7164 fib_info_hold(fib_event->fnh_info.fib_nh->nh_parent); 7165 break; 7166 } 7167 } 7168 7169 static int mlxsw_sp_router_fib6_event(struct mlxsw_sp_fib_event *fib_event, 7170 struct fib_notifier_info *info) 7171 { 7172 struct fib6_entry_notifier_info *fen6_info; 7173 int err; 7174 7175 switch (fib_event->event) { 7176 case FIB_EVENT_ENTRY_REPLACE: 7177 case FIB_EVENT_ENTRY_APPEND: 7178 case FIB_EVENT_ENTRY_DEL: 7179 fen6_info = container_of(info, struct fib6_entry_notifier_info, 7180 info); 7181 err = mlxsw_sp_router_fib6_event_init(&fib_event->fib6_event, 7182 fen6_info); 7183 if (err) 7184 return err; 7185 break; 7186 } 7187 7188 return 0; 7189 } 7190 7191 static void 7192 mlxsw_sp_router_fibmr_event(struct mlxsw_sp_fib_event *fib_event, 7193 struct fib_notifier_info *info) 7194 { 7195 switch (fib_event->event) { 7196 case FIB_EVENT_ENTRY_REPLACE: 7197 case FIB_EVENT_ENTRY_ADD: 7198 case FIB_EVENT_ENTRY_DEL: 7199 memcpy(&fib_event->men_info, info, sizeof(fib_event->men_info)); 7200 mr_cache_hold(fib_event->men_info.mfc); 7201 break; 7202 case FIB_EVENT_VIF_ADD: 7203 case FIB_EVENT_VIF_DEL: 7204 memcpy(&fib_event->ven_info, info, sizeof(fib_event->ven_info)); 7205 dev_hold(fib_event->ven_info.dev); 7206 break; 7207 } 7208 } 7209 7210 static int mlxsw_sp_router_fib_rule_event(unsigned long event, 7211 struct fib_notifier_info *info, 7212 struct mlxsw_sp *mlxsw_sp) 7213 { 7214 struct netlink_ext_ack *extack = info->extack; 7215 struct fib_rule_notifier_info *fr_info; 7216 struct fib_rule *rule; 7217 int err = 0; 7218 7219 /* nothing to do at the moment */ 7220 if (event == FIB_EVENT_RULE_DEL) 7221 return 0; 7222 7223 if (mlxsw_sp->router->aborted) 7224 return 0; 7225 7226 fr_info = container_of(info, struct fib_rule_notifier_info, info); 7227 rule = fr_info->rule; 7228 7229 /* Rule only affects locally generated traffic */ 7230 if (rule->iifindex == mlxsw_sp_net(mlxsw_sp)->loopback_dev->ifindex) 7231 return 0; 7232 7233 switch (info->family) { 7234 case AF_INET: 7235 if (!fib4_rule_default(rule) && !rule->l3mdev) 7236 err = -EOPNOTSUPP; 7237 break; 7238 case AF_INET6: 7239 if (!fib6_rule_default(rule) && !rule->l3mdev) 7240 err = -EOPNOTSUPP; 7241 break; 7242 case RTNL_FAMILY_IPMR: 7243 if (!ipmr_rule_default(rule) && !rule->l3mdev) 7244 err = -EOPNOTSUPP; 7245 break; 7246 case RTNL_FAMILY_IP6MR: 7247 if (!ip6mr_rule_default(rule) && !rule->l3mdev) 7248 err = -EOPNOTSUPP; 7249 break; 7250 } 7251 7252 if (err < 0) 7253 NL_SET_ERR_MSG_MOD(extack, "FIB rules not supported"); 7254 7255 return err; 7256 } 7257 7258 /* Called with rcu_read_lock() */ 7259 static int mlxsw_sp_router_fib_event(struct notifier_block *nb, 7260 unsigned long event, void *ptr) 7261 { 7262 struct mlxsw_sp_fib_event *fib_event; 7263 struct fib_notifier_info *info = ptr; 7264 struct mlxsw_sp_router *router; 7265 int err; 7266 7267 if ((info->family != AF_INET && info->family != AF_INET6 && 7268 info->family != RTNL_FAMILY_IPMR && 7269 info->family != RTNL_FAMILY_IP6MR)) 7270 return NOTIFY_DONE; 7271 7272 router = container_of(nb, struct mlxsw_sp_router, fib_nb); 7273 7274 switch (event) { 7275 case FIB_EVENT_RULE_ADD: 7276 case FIB_EVENT_RULE_DEL: 7277 err = mlxsw_sp_router_fib_rule_event(event, info, 7278 router->mlxsw_sp); 7279 return notifier_from_errno(err); 7280 case FIB_EVENT_ENTRY_ADD: 7281 case FIB_EVENT_ENTRY_REPLACE: 7282 case FIB_EVENT_ENTRY_APPEND: 7283 if (router->aborted) { 7284 NL_SET_ERR_MSG_MOD(info->extack, "FIB offload was aborted. Not configuring route"); 7285 return notifier_from_errno(-EINVAL); 7286 } 7287 if (info->family == AF_INET) { 7288 struct fib_entry_notifier_info *fen_info = ptr; 7289 7290 if (fen_info->fi->fib_nh_is_v6) { 7291 NL_SET_ERR_MSG_MOD(info->extack, "IPv6 gateway with IPv4 route is not supported"); 7292 return notifier_from_errno(-EINVAL); 7293 } 7294 } 7295 break; 7296 } 7297 7298 fib_event = kzalloc(sizeof(*fib_event), GFP_ATOMIC); 7299 if (!fib_event) 7300 return NOTIFY_BAD; 7301 7302 fib_event->mlxsw_sp = router->mlxsw_sp; 7303 fib_event->event = event; 7304 fib_event->family = info->family; 7305 7306 switch (info->family) { 7307 case AF_INET: 7308 mlxsw_sp_router_fib4_event(fib_event, info); 7309 break; 7310 case AF_INET6: 7311 err = mlxsw_sp_router_fib6_event(fib_event, info); 7312 if (err) 7313 goto err_fib_event; 7314 break; 7315 case RTNL_FAMILY_IP6MR: 7316 case RTNL_FAMILY_IPMR: 7317 mlxsw_sp_router_fibmr_event(fib_event, info); 7318 break; 7319 } 7320 7321 /* Enqueue the event and trigger the work */ 7322 spin_lock_bh(&router->fib_event_queue_lock); 7323 list_add_tail(&fib_event->list, &router->fib_event_queue); 7324 spin_unlock_bh(&router->fib_event_queue_lock); 7325 mlxsw_core_schedule_work(&router->fib_event_work); 7326 7327 return NOTIFY_DONE; 7328 7329 err_fib_event: 7330 kfree(fib_event); 7331 return NOTIFY_BAD; 7332 } 7333 7334 static struct mlxsw_sp_rif * 7335 mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp, 7336 const struct net_device *dev) 7337 { 7338 int i; 7339 7340 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++) 7341 if (mlxsw_sp->router->rifs[i] && 7342 mlxsw_sp->router->rifs[i]->dev == dev) 7343 return mlxsw_sp->router->rifs[i]; 7344 7345 return NULL; 7346 } 7347 7348 bool mlxsw_sp_rif_exists(struct mlxsw_sp *mlxsw_sp, 7349 const struct net_device *dev) 7350 { 7351 struct mlxsw_sp_rif *rif; 7352 7353 mutex_lock(&mlxsw_sp->router->lock); 7354 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 7355 mutex_unlock(&mlxsw_sp->router->lock); 7356 7357 return rif; 7358 } 7359 7360 u16 mlxsw_sp_rif_vid(struct mlxsw_sp *mlxsw_sp, const struct net_device *dev) 7361 { 7362 struct mlxsw_sp_rif *rif; 7363 u16 vid = 0; 7364 7365 mutex_lock(&mlxsw_sp->router->lock); 7366 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 7367 if (!rif) 7368 goto out; 7369 7370 /* We only return the VID for VLAN RIFs. Otherwise we return an 7371 * invalid value (0). 7372 */ 7373 if (rif->ops->type != MLXSW_SP_RIF_TYPE_VLAN) 7374 goto out; 7375 7376 vid = mlxsw_sp_fid_8021q_vid(rif->fid); 7377 7378 out: 7379 mutex_unlock(&mlxsw_sp->router->lock); 7380 return vid; 7381 } 7382 7383 static int mlxsw_sp_router_rif_disable(struct mlxsw_sp *mlxsw_sp, u16 rif) 7384 { 7385 char ritr_pl[MLXSW_REG_RITR_LEN]; 7386 int err; 7387 7388 mlxsw_reg_ritr_rif_pack(ritr_pl, rif); 7389 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 7390 if (err) 7391 return err; 7392 7393 mlxsw_reg_ritr_enable_set(ritr_pl, false); 7394 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 7395 } 7396 7397 static void mlxsw_sp_router_rif_gone_sync(struct mlxsw_sp *mlxsw_sp, 7398 struct mlxsw_sp_rif *rif) 7399 { 7400 mlxsw_sp_router_rif_disable(mlxsw_sp, rif->rif_index); 7401 mlxsw_sp_nexthop_rif_gone_sync(mlxsw_sp, rif); 7402 mlxsw_sp_neigh_rif_gone_sync(mlxsw_sp, rif); 7403 } 7404 7405 static bool 7406 mlxsw_sp_rif_should_config(struct mlxsw_sp_rif *rif, struct net_device *dev, 7407 unsigned long event) 7408 { 7409 struct inet6_dev *inet6_dev; 7410 bool addr_list_empty = true; 7411 struct in_device *idev; 7412 7413 switch (event) { 7414 case NETDEV_UP: 7415 return rif == NULL; 7416 case NETDEV_DOWN: 7417 rcu_read_lock(); 7418 idev = __in_dev_get_rcu(dev); 7419 if (idev && idev->ifa_list) 7420 addr_list_empty = false; 7421 7422 inet6_dev = __in6_dev_get(dev); 7423 if (addr_list_empty && inet6_dev && 7424 !list_empty(&inet6_dev->addr_list)) 7425 addr_list_empty = false; 7426 rcu_read_unlock(); 7427 7428 /* macvlans do not have a RIF, but rather piggy back on the 7429 * RIF of their lower device. 7430 */ 7431 if (netif_is_macvlan(dev) && addr_list_empty) 7432 return true; 7433 7434 if (rif && addr_list_empty && 7435 !netif_is_l3_slave(rif->dev)) 7436 return true; 7437 /* It is possible we already removed the RIF ourselves 7438 * if it was assigned to a netdev that is now a bridge 7439 * or LAG slave. 7440 */ 7441 return false; 7442 } 7443 7444 return false; 7445 } 7446 7447 static enum mlxsw_sp_rif_type 7448 mlxsw_sp_dev_rif_type(const struct mlxsw_sp *mlxsw_sp, 7449 const struct net_device *dev) 7450 { 7451 enum mlxsw_sp_fid_type type; 7452 7453 if (mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, NULL)) 7454 return MLXSW_SP_RIF_TYPE_IPIP_LB; 7455 7456 /* Otherwise RIF type is derived from the type of the underlying FID. */ 7457 if (is_vlan_dev(dev) && netif_is_bridge_master(vlan_dev_real_dev(dev))) 7458 type = MLXSW_SP_FID_TYPE_8021Q; 7459 else if (netif_is_bridge_master(dev) && br_vlan_enabled(dev)) 7460 type = MLXSW_SP_FID_TYPE_8021Q; 7461 else if (netif_is_bridge_master(dev)) 7462 type = MLXSW_SP_FID_TYPE_8021D; 7463 else 7464 type = MLXSW_SP_FID_TYPE_RFID; 7465 7466 return mlxsw_sp_fid_type_rif_type(mlxsw_sp, type); 7467 } 7468 7469 static int mlxsw_sp_rif_index_alloc(struct mlxsw_sp *mlxsw_sp, u16 *p_rif_index) 7470 { 7471 int i; 7472 7473 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++) { 7474 if (!mlxsw_sp->router->rifs[i]) { 7475 *p_rif_index = i; 7476 return 0; 7477 } 7478 } 7479 7480 return -ENOBUFS; 7481 } 7482 7483 static struct mlxsw_sp_rif *mlxsw_sp_rif_alloc(size_t rif_size, u16 rif_index, 7484 u16 vr_id, 7485 struct net_device *l3_dev) 7486 { 7487 struct mlxsw_sp_rif *rif; 7488 7489 rif = kzalloc(rif_size, GFP_KERNEL); 7490 if (!rif) 7491 return NULL; 7492 7493 INIT_LIST_HEAD(&rif->nexthop_list); 7494 INIT_LIST_HEAD(&rif->neigh_list); 7495 if (l3_dev) { 7496 ether_addr_copy(rif->addr, l3_dev->dev_addr); 7497 rif->mtu = l3_dev->mtu; 7498 rif->dev = l3_dev; 7499 } 7500 rif->vr_id = vr_id; 7501 rif->rif_index = rif_index; 7502 7503 return rif; 7504 } 7505 7506 struct mlxsw_sp_rif *mlxsw_sp_rif_by_index(const struct mlxsw_sp *mlxsw_sp, 7507 u16 rif_index) 7508 { 7509 return mlxsw_sp->router->rifs[rif_index]; 7510 } 7511 7512 u16 mlxsw_sp_rif_index(const struct mlxsw_sp_rif *rif) 7513 { 7514 return rif->rif_index; 7515 } 7516 7517 u16 mlxsw_sp_ipip_lb_rif_index(const struct mlxsw_sp_rif_ipip_lb *lb_rif) 7518 { 7519 return lb_rif->common.rif_index; 7520 } 7521 7522 u16 mlxsw_sp_ipip_lb_ul_vr_id(const struct mlxsw_sp_rif_ipip_lb *lb_rif) 7523 { 7524 u32 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(lb_rif->common.dev); 7525 struct mlxsw_sp_vr *ul_vr; 7526 7527 ul_vr = mlxsw_sp_vr_get(lb_rif->common.mlxsw_sp, ul_tb_id, NULL); 7528 if (WARN_ON(IS_ERR(ul_vr))) 7529 return 0; 7530 7531 return ul_vr->id; 7532 } 7533 7534 u16 mlxsw_sp_ipip_lb_ul_rif_id(const struct mlxsw_sp_rif_ipip_lb *lb_rif) 7535 { 7536 return lb_rif->ul_rif_id; 7537 } 7538 7539 int mlxsw_sp_rif_dev_ifindex(const struct mlxsw_sp_rif *rif) 7540 { 7541 return rif->dev->ifindex; 7542 } 7543 7544 const struct net_device *mlxsw_sp_rif_dev(const struct mlxsw_sp_rif *rif) 7545 { 7546 return rif->dev; 7547 } 7548 7549 static struct mlxsw_sp_rif * 7550 mlxsw_sp_rif_create(struct mlxsw_sp *mlxsw_sp, 7551 const struct mlxsw_sp_rif_params *params, 7552 struct netlink_ext_ack *extack) 7553 { 7554 u32 tb_id = l3mdev_fib_table(params->dev); 7555 const struct mlxsw_sp_rif_ops *ops; 7556 struct mlxsw_sp_fid *fid = NULL; 7557 enum mlxsw_sp_rif_type type; 7558 struct mlxsw_sp_rif *rif; 7559 struct mlxsw_sp_vr *vr; 7560 u16 rif_index; 7561 int i, err; 7562 7563 type = mlxsw_sp_dev_rif_type(mlxsw_sp, params->dev); 7564 ops = mlxsw_sp->rif_ops_arr[type]; 7565 7566 vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id ? : RT_TABLE_MAIN, extack); 7567 if (IS_ERR(vr)) 7568 return ERR_CAST(vr); 7569 vr->rif_count++; 7570 7571 err = mlxsw_sp_rif_index_alloc(mlxsw_sp, &rif_index); 7572 if (err) { 7573 NL_SET_ERR_MSG_MOD(extack, "Exceeded number of supported router interfaces"); 7574 goto err_rif_index_alloc; 7575 } 7576 7577 rif = mlxsw_sp_rif_alloc(ops->rif_size, rif_index, vr->id, params->dev); 7578 if (!rif) { 7579 err = -ENOMEM; 7580 goto err_rif_alloc; 7581 } 7582 dev_hold(rif->dev); 7583 mlxsw_sp->router->rifs[rif_index] = rif; 7584 rif->mlxsw_sp = mlxsw_sp; 7585 rif->ops = ops; 7586 7587 if (ops->fid_get) { 7588 fid = ops->fid_get(rif, extack); 7589 if (IS_ERR(fid)) { 7590 err = PTR_ERR(fid); 7591 goto err_fid_get; 7592 } 7593 rif->fid = fid; 7594 } 7595 7596 if (ops->setup) 7597 ops->setup(rif, params); 7598 7599 err = ops->configure(rif); 7600 if (err) 7601 goto err_configure; 7602 7603 for (i = 0; i < MLXSW_SP_L3_PROTO_MAX; i++) { 7604 err = mlxsw_sp_mr_rif_add(vr->mr_table[i], rif); 7605 if (err) 7606 goto err_mr_rif_add; 7607 } 7608 7609 mlxsw_sp_rif_counters_alloc(rif); 7610 7611 return rif; 7612 7613 err_mr_rif_add: 7614 for (i--; i >= 0; i--) 7615 mlxsw_sp_mr_rif_del(vr->mr_table[i], rif); 7616 ops->deconfigure(rif); 7617 err_configure: 7618 if (fid) 7619 mlxsw_sp_fid_put(fid); 7620 err_fid_get: 7621 mlxsw_sp->router->rifs[rif_index] = NULL; 7622 dev_put(rif->dev); 7623 kfree(rif); 7624 err_rif_alloc: 7625 err_rif_index_alloc: 7626 vr->rif_count--; 7627 mlxsw_sp_vr_put(mlxsw_sp, vr); 7628 return ERR_PTR(err); 7629 } 7630 7631 static void mlxsw_sp_rif_destroy(struct mlxsw_sp_rif *rif) 7632 { 7633 const struct mlxsw_sp_rif_ops *ops = rif->ops; 7634 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 7635 struct mlxsw_sp_fid *fid = rif->fid; 7636 struct mlxsw_sp_vr *vr; 7637 int i; 7638 7639 mlxsw_sp_router_rif_gone_sync(mlxsw_sp, rif); 7640 vr = &mlxsw_sp->router->vrs[rif->vr_id]; 7641 7642 mlxsw_sp_rif_counters_free(rif); 7643 for (i = 0; i < MLXSW_SP_L3_PROTO_MAX; i++) 7644 mlxsw_sp_mr_rif_del(vr->mr_table[i], rif); 7645 ops->deconfigure(rif); 7646 if (fid) 7647 /* Loopback RIFs are not associated with a FID. */ 7648 mlxsw_sp_fid_put(fid); 7649 mlxsw_sp->router->rifs[rif->rif_index] = NULL; 7650 dev_put(rif->dev); 7651 kfree(rif); 7652 vr->rif_count--; 7653 mlxsw_sp_vr_put(mlxsw_sp, vr); 7654 } 7655 7656 void mlxsw_sp_rif_destroy_by_dev(struct mlxsw_sp *mlxsw_sp, 7657 struct net_device *dev) 7658 { 7659 struct mlxsw_sp_rif *rif; 7660 7661 mutex_lock(&mlxsw_sp->router->lock); 7662 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 7663 if (!rif) 7664 goto out; 7665 mlxsw_sp_rif_destroy(rif); 7666 out: 7667 mutex_unlock(&mlxsw_sp->router->lock); 7668 } 7669 7670 static void 7671 mlxsw_sp_rif_subport_params_init(struct mlxsw_sp_rif_params *params, 7672 struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan) 7673 { 7674 struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port; 7675 7676 params->vid = mlxsw_sp_port_vlan->vid; 7677 params->lag = mlxsw_sp_port->lagged; 7678 if (params->lag) 7679 params->lag_id = mlxsw_sp_port->lag_id; 7680 else 7681 params->system_port = mlxsw_sp_port->local_port; 7682 } 7683 7684 static struct mlxsw_sp_rif_subport * 7685 mlxsw_sp_rif_subport_rif(const struct mlxsw_sp_rif *rif) 7686 { 7687 return container_of(rif, struct mlxsw_sp_rif_subport, common); 7688 } 7689 7690 static struct mlxsw_sp_rif * 7691 mlxsw_sp_rif_subport_get(struct mlxsw_sp *mlxsw_sp, 7692 const struct mlxsw_sp_rif_params *params, 7693 struct netlink_ext_ack *extack) 7694 { 7695 struct mlxsw_sp_rif_subport *rif_subport; 7696 struct mlxsw_sp_rif *rif; 7697 7698 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, params->dev); 7699 if (!rif) 7700 return mlxsw_sp_rif_create(mlxsw_sp, params, extack); 7701 7702 rif_subport = mlxsw_sp_rif_subport_rif(rif); 7703 refcount_inc(&rif_subport->ref_count); 7704 return rif; 7705 } 7706 7707 static void mlxsw_sp_rif_subport_put(struct mlxsw_sp_rif *rif) 7708 { 7709 struct mlxsw_sp_rif_subport *rif_subport; 7710 7711 rif_subport = mlxsw_sp_rif_subport_rif(rif); 7712 if (!refcount_dec_and_test(&rif_subport->ref_count)) 7713 return; 7714 7715 mlxsw_sp_rif_destroy(rif); 7716 } 7717 7718 static int 7719 __mlxsw_sp_port_vlan_router_join(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan, 7720 struct net_device *l3_dev, 7721 struct netlink_ext_ack *extack) 7722 { 7723 struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port; 7724 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp; 7725 struct mlxsw_sp_rif_params params = { 7726 .dev = l3_dev, 7727 }; 7728 u16 vid = mlxsw_sp_port_vlan->vid; 7729 struct mlxsw_sp_rif *rif; 7730 struct mlxsw_sp_fid *fid; 7731 int err; 7732 7733 mlxsw_sp_rif_subport_params_init(¶ms, mlxsw_sp_port_vlan); 7734 rif = mlxsw_sp_rif_subport_get(mlxsw_sp, ¶ms, extack); 7735 if (IS_ERR(rif)) 7736 return PTR_ERR(rif); 7737 7738 /* FID was already created, just take a reference */ 7739 fid = rif->ops->fid_get(rif, extack); 7740 err = mlxsw_sp_fid_port_vid_map(fid, mlxsw_sp_port, vid); 7741 if (err) 7742 goto err_fid_port_vid_map; 7743 7744 err = mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, false); 7745 if (err) 7746 goto err_port_vid_learning_set; 7747 7748 err = mlxsw_sp_port_vid_stp_set(mlxsw_sp_port, vid, 7749 BR_STATE_FORWARDING); 7750 if (err) 7751 goto err_port_vid_stp_set; 7752 7753 mlxsw_sp_port_vlan->fid = fid; 7754 7755 return 0; 7756 7757 err_port_vid_stp_set: 7758 mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, true); 7759 err_port_vid_learning_set: 7760 mlxsw_sp_fid_port_vid_unmap(fid, mlxsw_sp_port, vid); 7761 err_fid_port_vid_map: 7762 mlxsw_sp_fid_put(fid); 7763 mlxsw_sp_rif_subport_put(rif); 7764 return err; 7765 } 7766 7767 static void 7768 __mlxsw_sp_port_vlan_router_leave(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan) 7769 { 7770 struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port; 7771 struct mlxsw_sp_fid *fid = mlxsw_sp_port_vlan->fid; 7772 struct mlxsw_sp_rif *rif = mlxsw_sp_fid_rif(fid); 7773 u16 vid = mlxsw_sp_port_vlan->vid; 7774 7775 if (WARN_ON(mlxsw_sp_fid_type(fid) != MLXSW_SP_FID_TYPE_RFID)) 7776 return; 7777 7778 mlxsw_sp_port_vlan->fid = NULL; 7779 mlxsw_sp_port_vid_stp_set(mlxsw_sp_port, vid, BR_STATE_BLOCKING); 7780 mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, true); 7781 mlxsw_sp_fid_port_vid_unmap(fid, mlxsw_sp_port, vid); 7782 mlxsw_sp_fid_put(fid); 7783 mlxsw_sp_rif_subport_put(rif); 7784 } 7785 7786 int 7787 mlxsw_sp_port_vlan_router_join(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan, 7788 struct net_device *l3_dev, 7789 struct netlink_ext_ack *extack) 7790 { 7791 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port_vlan->mlxsw_sp_port->mlxsw_sp; 7792 struct mlxsw_sp_rif *rif; 7793 int err = 0; 7794 7795 mutex_lock(&mlxsw_sp->router->lock); 7796 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev); 7797 if (!rif) 7798 goto out; 7799 7800 err = __mlxsw_sp_port_vlan_router_join(mlxsw_sp_port_vlan, l3_dev, 7801 extack); 7802 out: 7803 mutex_unlock(&mlxsw_sp->router->lock); 7804 return err; 7805 } 7806 7807 void 7808 mlxsw_sp_port_vlan_router_leave(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan) 7809 { 7810 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port_vlan->mlxsw_sp_port->mlxsw_sp; 7811 7812 mutex_lock(&mlxsw_sp->router->lock); 7813 __mlxsw_sp_port_vlan_router_leave(mlxsw_sp_port_vlan); 7814 mutex_unlock(&mlxsw_sp->router->lock); 7815 } 7816 7817 static int mlxsw_sp_inetaddr_port_vlan_event(struct net_device *l3_dev, 7818 struct net_device *port_dev, 7819 unsigned long event, u16 vid, 7820 struct netlink_ext_ack *extack) 7821 { 7822 struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(port_dev); 7823 struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan; 7824 7825 mlxsw_sp_port_vlan = mlxsw_sp_port_vlan_find_by_vid(mlxsw_sp_port, vid); 7826 if (WARN_ON(!mlxsw_sp_port_vlan)) 7827 return -EINVAL; 7828 7829 switch (event) { 7830 case NETDEV_UP: 7831 return __mlxsw_sp_port_vlan_router_join(mlxsw_sp_port_vlan, 7832 l3_dev, extack); 7833 case NETDEV_DOWN: 7834 __mlxsw_sp_port_vlan_router_leave(mlxsw_sp_port_vlan); 7835 break; 7836 } 7837 7838 return 0; 7839 } 7840 7841 static int mlxsw_sp_inetaddr_port_event(struct net_device *port_dev, 7842 unsigned long event, 7843 struct netlink_ext_ack *extack) 7844 { 7845 if (netif_is_bridge_port(port_dev) || 7846 netif_is_lag_port(port_dev) || 7847 netif_is_ovs_port(port_dev)) 7848 return 0; 7849 7850 return mlxsw_sp_inetaddr_port_vlan_event(port_dev, port_dev, event, 7851 MLXSW_SP_DEFAULT_VID, extack); 7852 } 7853 7854 static int __mlxsw_sp_inetaddr_lag_event(struct net_device *l3_dev, 7855 struct net_device *lag_dev, 7856 unsigned long event, u16 vid, 7857 struct netlink_ext_ack *extack) 7858 { 7859 struct net_device *port_dev; 7860 struct list_head *iter; 7861 int err; 7862 7863 netdev_for_each_lower_dev(lag_dev, port_dev, iter) { 7864 if (mlxsw_sp_port_dev_check(port_dev)) { 7865 err = mlxsw_sp_inetaddr_port_vlan_event(l3_dev, 7866 port_dev, 7867 event, vid, 7868 extack); 7869 if (err) 7870 return err; 7871 } 7872 } 7873 7874 return 0; 7875 } 7876 7877 static int mlxsw_sp_inetaddr_lag_event(struct net_device *lag_dev, 7878 unsigned long event, 7879 struct netlink_ext_ack *extack) 7880 { 7881 if (netif_is_bridge_port(lag_dev)) 7882 return 0; 7883 7884 return __mlxsw_sp_inetaddr_lag_event(lag_dev, lag_dev, event, 7885 MLXSW_SP_DEFAULT_VID, extack); 7886 } 7887 7888 static int mlxsw_sp_inetaddr_bridge_event(struct mlxsw_sp *mlxsw_sp, 7889 struct net_device *l3_dev, 7890 unsigned long event, 7891 struct netlink_ext_ack *extack) 7892 { 7893 struct mlxsw_sp_rif_params params = { 7894 .dev = l3_dev, 7895 }; 7896 struct mlxsw_sp_rif *rif; 7897 7898 switch (event) { 7899 case NETDEV_UP: 7900 if (netif_is_bridge_master(l3_dev) && br_vlan_enabled(l3_dev)) { 7901 u16 proto; 7902 7903 br_vlan_get_proto(l3_dev, &proto); 7904 if (proto == ETH_P_8021AD) { 7905 NL_SET_ERR_MSG_MOD(extack, "Adding an IP address to 802.1ad bridge is not supported"); 7906 return -EOPNOTSUPP; 7907 } 7908 } 7909 rif = mlxsw_sp_rif_create(mlxsw_sp, ¶ms, extack); 7910 if (IS_ERR(rif)) 7911 return PTR_ERR(rif); 7912 break; 7913 case NETDEV_DOWN: 7914 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev); 7915 mlxsw_sp_rif_destroy(rif); 7916 break; 7917 } 7918 7919 return 0; 7920 } 7921 7922 static int mlxsw_sp_inetaddr_vlan_event(struct mlxsw_sp *mlxsw_sp, 7923 struct net_device *vlan_dev, 7924 unsigned long event, 7925 struct netlink_ext_ack *extack) 7926 { 7927 struct net_device *real_dev = vlan_dev_real_dev(vlan_dev); 7928 u16 vid = vlan_dev_vlan_id(vlan_dev); 7929 7930 if (netif_is_bridge_port(vlan_dev)) 7931 return 0; 7932 7933 if (mlxsw_sp_port_dev_check(real_dev)) 7934 return mlxsw_sp_inetaddr_port_vlan_event(vlan_dev, real_dev, 7935 event, vid, extack); 7936 else if (netif_is_lag_master(real_dev)) 7937 return __mlxsw_sp_inetaddr_lag_event(vlan_dev, real_dev, event, 7938 vid, extack); 7939 else if (netif_is_bridge_master(real_dev) && br_vlan_enabled(real_dev)) 7940 return mlxsw_sp_inetaddr_bridge_event(mlxsw_sp, vlan_dev, event, 7941 extack); 7942 7943 return 0; 7944 } 7945 7946 static bool mlxsw_sp_rif_macvlan_is_vrrp4(const u8 *mac) 7947 { 7948 u8 vrrp4[ETH_ALEN] = { 0x00, 0x00, 0x5e, 0x00, 0x01, 0x00 }; 7949 u8 mask[ETH_ALEN] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0x00 }; 7950 7951 return ether_addr_equal_masked(mac, vrrp4, mask); 7952 } 7953 7954 static bool mlxsw_sp_rif_macvlan_is_vrrp6(const u8 *mac) 7955 { 7956 u8 vrrp6[ETH_ALEN] = { 0x00, 0x00, 0x5e, 0x00, 0x02, 0x00 }; 7957 u8 mask[ETH_ALEN] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0x00 }; 7958 7959 return ether_addr_equal_masked(mac, vrrp6, mask); 7960 } 7961 7962 static int mlxsw_sp_rif_vrrp_op(struct mlxsw_sp *mlxsw_sp, u16 rif_index, 7963 const u8 *mac, bool adding) 7964 { 7965 char ritr_pl[MLXSW_REG_RITR_LEN]; 7966 u8 vrrp_id = adding ? mac[5] : 0; 7967 int err; 7968 7969 if (!mlxsw_sp_rif_macvlan_is_vrrp4(mac) && 7970 !mlxsw_sp_rif_macvlan_is_vrrp6(mac)) 7971 return 0; 7972 7973 mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index); 7974 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 7975 if (err) 7976 return err; 7977 7978 if (mlxsw_sp_rif_macvlan_is_vrrp4(mac)) 7979 mlxsw_reg_ritr_if_vrrp_id_ipv4_set(ritr_pl, vrrp_id); 7980 else 7981 mlxsw_reg_ritr_if_vrrp_id_ipv6_set(ritr_pl, vrrp_id); 7982 7983 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 7984 } 7985 7986 static int mlxsw_sp_rif_macvlan_add(struct mlxsw_sp *mlxsw_sp, 7987 const struct net_device *macvlan_dev, 7988 struct netlink_ext_ack *extack) 7989 { 7990 struct macvlan_dev *vlan = netdev_priv(macvlan_dev); 7991 struct mlxsw_sp_rif *rif; 7992 int err; 7993 7994 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, vlan->lowerdev); 7995 if (!rif) { 7996 NL_SET_ERR_MSG_MOD(extack, "macvlan is only supported on top of router interfaces"); 7997 return -EOPNOTSUPP; 7998 } 7999 8000 err = mlxsw_sp_rif_fdb_op(mlxsw_sp, macvlan_dev->dev_addr, 8001 mlxsw_sp_fid_index(rif->fid), true); 8002 if (err) 8003 return err; 8004 8005 err = mlxsw_sp_rif_vrrp_op(mlxsw_sp, rif->rif_index, 8006 macvlan_dev->dev_addr, true); 8007 if (err) 8008 goto err_rif_vrrp_add; 8009 8010 /* Make sure the bridge driver does not have this MAC pointing at 8011 * some other port. 8012 */ 8013 if (rif->ops->fdb_del) 8014 rif->ops->fdb_del(rif, macvlan_dev->dev_addr); 8015 8016 return 0; 8017 8018 err_rif_vrrp_add: 8019 mlxsw_sp_rif_fdb_op(mlxsw_sp, macvlan_dev->dev_addr, 8020 mlxsw_sp_fid_index(rif->fid), false); 8021 return err; 8022 } 8023 8024 static void __mlxsw_sp_rif_macvlan_del(struct mlxsw_sp *mlxsw_sp, 8025 const struct net_device *macvlan_dev) 8026 { 8027 struct macvlan_dev *vlan = netdev_priv(macvlan_dev); 8028 struct mlxsw_sp_rif *rif; 8029 8030 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, vlan->lowerdev); 8031 /* If we do not have a RIF, then we already took care of 8032 * removing the macvlan's MAC during RIF deletion. 8033 */ 8034 if (!rif) 8035 return; 8036 mlxsw_sp_rif_vrrp_op(mlxsw_sp, rif->rif_index, macvlan_dev->dev_addr, 8037 false); 8038 mlxsw_sp_rif_fdb_op(mlxsw_sp, macvlan_dev->dev_addr, 8039 mlxsw_sp_fid_index(rif->fid), false); 8040 } 8041 8042 void mlxsw_sp_rif_macvlan_del(struct mlxsw_sp *mlxsw_sp, 8043 const struct net_device *macvlan_dev) 8044 { 8045 mutex_lock(&mlxsw_sp->router->lock); 8046 __mlxsw_sp_rif_macvlan_del(mlxsw_sp, macvlan_dev); 8047 mutex_unlock(&mlxsw_sp->router->lock); 8048 } 8049 8050 static int mlxsw_sp_inetaddr_macvlan_event(struct mlxsw_sp *mlxsw_sp, 8051 struct net_device *macvlan_dev, 8052 unsigned long event, 8053 struct netlink_ext_ack *extack) 8054 { 8055 switch (event) { 8056 case NETDEV_UP: 8057 return mlxsw_sp_rif_macvlan_add(mlxsw_sp, macvlan_dev, extack); 8058 case NETDEV_DOWN: 8059 __mlxsw_sp_rif_macvlan_del(mlxsw_sp, macvlan_dev); 8060 break; 8061 } 8062 8063 return 0; 8064 } 8065 8066 static int mlxsw_sp_router_port_check_rif_addr(struct mlxsw_sp *mlxsw_sp, 8067 struct net_device *dev, 8068 const unsigned char *dev_addr, 8069 struct netlink_ext_ack *extack) 8070 { 8071 struct mlxsw_sp_rif *rif; 8072 int i; 8073 8074 /* A RIF is not created for macvlan netdevs. Their MAC is used to 8075 * populate the FDB 8076 */ 8077 if (netif_is_macvlan(dev) || netif_is_l3_master(dev)) 8078 return 0; 8079 8080 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++) { 8081 rif = mlxsw_sp->router->rifs[i]; 8082 if (rif && rif->ops && 8083 rif->ops->type == MLXSW_SP_RIF_TYPE_IPIP_LB) 8084 continue; 8085 if (rif && rif->dev && rif->dev != dev && 8086 !ether_addr_equal_masked(rif->dev->dev_addr, dev_addr, 8087 mlxsw_sp->mac_mask)) { 8088 NL_SET_ERR_MSG_MOD(extack, "All router interface MAC addresses must have the same prefix"); 8089 return -EINVAL; 8090 } 8091 } 8092 8093 return 0; 8094 } 8095 8096 static int __mlxsw_sp_inetaddr_event(struct mlxsw_sp *mlxsw_sp, 8097 struct net_device *dev, 8098 unsigned long event, 8099 struct netlink_ext_ack *extack) 8100 { 8101 if (mlxsw_sp_port_dev_check(dev)) 8102 return mlxsw_sp_inetaddr_port_event(dev, event, extack); 8103 else if (netif_is_lag_master(dev)) 8104 return mlxsw_sp_inetaddr_lag_event(dev, event, extack); 8105 else if (netif_is_bridge_master(dev)) 8106 return mlxsw_sp_inetaddr_bridge_event(mlxsw_sp, dev, event, 8107 extack); 8108 else if (is_vlan_dev(dev)) 8109 return mlxsw_sp_inetaddr_vlan_event(mlxsw_sp, dev, event, 8110 extack); 8111 else if (netif_is_macvlan(dev)) 8112 return mlxsw_sp_inetaddr_macvlan_event(mlxsw_sp, dev, event, 8113 extack); 8114 else 8115 return 0; 8116 } 8117 8118 static int mlxsw_sp_inetaddr_event(struct notifier_block *nb, 8119 unsigned long event, void *ptr) 8120 { 8121 struct in_ifaddr *ifa = (struct in_ifaddr *) ptr; 8122 struct net_device *dev = ifa->ifa_dev->dev; 8123 struct mlxsw_sp_router *router; 8124 struct mlxsw_sp_rif *rif; 8125 int err = 0; 8126 8127 /* NETDEV_UP event is handled by mlxsw_sp_inetaddr_valid_event */ 8128 if (event == NETDEV_UP) 8129 return NOTIFY_DONE; 8130 8131 router = container_of(nb, struct mlxsw_sp_router, inetaddr_nb); 8132 mutex_lock(&router->lock); 8133 rif = mlxsw_sp_rif_find_by_dev(router->mlxsw_sp, dev); 8134 if (!mlxsw_sp_rif_should_config(rif, dev, event)) 8135 goto out; 8136 8137 err = __mlxsw_sp_inetaddr_event(router->mlxsw_sp, dev, event, NULL); 8138 out: 8139 mutex_unlock(&router->lock); 8140 return notifier_from_errno(err); 8141 } 8142 8143 int mlxsw_sp_inetaddr_valid_event(struct notifier_block *unused, 8144 unsigned long event, void *ptr) 8145 { 8146 struct in_validator_info *ivi = (struct in_validator_info *) ptr; 8147 struct net_device *dev = ivi->ivi_dev->dev; 8148 struct mlxsw_sp *mlxsw_sp; 8149 struct mlxsw_sp_rif *rif; 8150 int err = 0; 8151 8152 mlxsw_sp = mlxsw_sp_lower_get(dev); 8153 if (!mlxsw_sp) 8154 return NOTIFY_DONE; 8155 8156 mutex_lock(&mlxsw_sp->router->lock); 8157 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 8158 if (!mlxsw_sp_rif_should_config(rif, dev, event)) 8159 goto out; 8160 8161 err = mlxsw_sp_router_port_check_rif_addr(mlxsw_sp, dev, dev->dev_addr, 8162 ivi->extack); 8163 if (err) 8164 goto out; 8165 8166 err = __mlxsw_sp_inetaddr_event(mlxsw_sp, dev, event, ivi->extack); 8167 out: 8168 mutex_unlock(&mlxsw_sp->router->lock); 8169 return notifier_from_errno(err); 8170 } 8171 8172 struct mlxsw_sp_inet6addr_event_work { 8173 struct work_struct work; 8174 struct mlxsw_sp *mlxsw_sp; 8175 struct net_device *dev; 8176 unsigned long event; 8177 }; 8178 8179 static void mlxsw_sp_inet6addr_event_work(struct work_struct *work) 8180 { 8181 struct mlxsw_sp_inet6addr_event_work *inet6addr_work = 8182 container_of(work, struct mlxsw_sp_inet6addr_event_work, work); 8183 struct mlxsw_sp *mlxsw_sp = inet6addr_work->mlxsw_sp; 8184 struct net_device *dev = inet6addr_work->dev; 8185 unsigned long event = inet6addr_work->event; 8186 struct mlxsw_sp_rif *rif; 8187 8188 rtnl_lock(); 8189 mutex_lock(&mlxsw_sp->router->lock); 8190 8191 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 8192 if (!mlxsw_sp_rif_should_config(rif, dev, event)) 8193 goto out; 8194 8195 __mlxsw_sp_inetaddr_event(mlxsw_sp, dev, event, NULL); 8196 out: 8197 mutex_unlock(&mlxsw_sp->router->lock); 8198 rtnl_unlock(); 8199 dev_put(dev); 8200 kfree(inet6addr_work); 8201 } 8202 8203 /* Called with rcu_read_lock() */ 8204 static int mlxsw_sp_inet6addr_event(struct notifier_block *nb, 8205 unsigned long event, void *ptr) 8206 { 8207 struct inet6_ifaddr *if6 = (struct inet6_ifaddr *) ptr; 8208 struct mlxsw_sp_inet6addr_event_work *inet6addr_work; 8209 struct net_device *dev = if6->idev->dev; 8210 struct mlxsw_sp_router *router; 8211 8212 /* NETDEV_UP event is handled by mlxsw_sp_inet6addr_valid_event */ 8213 if (event == NETDEV_UP) 8214 return NOTIFY_DONE; 8215 8216 inet6addr_work = kzalloc(sizeof(*inet6addr_work), GFP_ATOMIC); 8217 if (!inet6addr_work) 8218 return NOTIFY_BAD; 8219 8220 router = container_of(nb, struct mlxsw_sp_router, inet6addr_nb); 8221 INIT_WORK(&inet6addr_work->work, mlxsw_sp_inet6addr_event_work); 8222 inet6addr_work->mlxsw_sp = router->mlxsw_sp; 8223 inet6addr_work->dev = dev; 8224 inet6addr_work->event = event; 8225 dev_hold(dev); 8226 mlxsw_core_schedule_work(&inet6addr_work->work); 8227 8228 return NOTIFY_DONE; 8229 } 8230 8231 int mlxsw_sp_inet6addr_valid_event(struct notifier_block *unused, 8232 unsigned long event, void *ptr) 8233 { 8234 struct in6_validator_info *i6vi = (struct in6_validator_info *) ptr; 8235 struct net_device *dev = i6vi->i6vi_dev->dev; 8236 struct mlxsw_sp *mlxsw_sp; 8237 struct mlxsw_sp_rif *rif; 8238 int err = 0; 8239 8240 mlxsw_sp = mlxsw_sp_lower_get(dev); 8241 if (!mlxsw_sp) 8242 return NOTIFY_DONE; 8243 8244 mutex_lock(&mlxsw_sp->router->lock); 8245 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 8246 if (!mlxsw_sp_rif_should_config(rif, dev, event)) 8247 goto out; 8248 8249 err = mlxsw_sp_router_port_check_rif_addr(mlxsw_sp, dev, dev->dev_addr, 8250 i6vi->extack); 8251 if (err) 8252 goto out; 8253 8254 err = __mlxsw_sp_inetaddr_event(mlxsw_sp, dev, event, i6vi->extack); 8255 out: 8256 mutex_unlock(&mlxsw_sp->router->lock); 8257 return notifier_from_errno(err); 8258 } 8259 8260 static int mlxsw_sp_rif_edit(struct mlxsw_sp *mlxsw_sp, u16 rif_index, 8261 const char *mac, int mtu) 8262 { 8263 char ritr_pl[MLXSW_REG_RITR_LEN]; 8264 int err; 8265 8266 mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index); 8267 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 8268 if (err) 8269 return err; 8270 8271 mlxsw_reg_ritr_mtu_set(ritr_pl, mtu); 8272 mlxsw_reg_ritr_if_mac_memcpy_to(ritr_pl, mac); 8273 mlxsw_reg_ritr_op_set(ritr_pl, MLXSW_REG_RITR_RIF_CREATE); 8274 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 8275 } 8276 8277 static int 8278 mlxsw_sp_router_port_change_event(struct mlxsw_sp *mlxsw_sp, 8279 struct mlxsw_sp_rif *rif) 8280 { 8281 struct net_device *dev = rif->dev; 8282 u16 fid_index; 8283 int err; 8284 8285 fid_index = mlxsw_sp_fid_index(rif->fid); 8286 8287 err = mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->addr, fid_index, false); 8288 if (err) 8289 return err; 8290 8291 err = mlxsw_sp_rif_edit(mlxsw_sp, rif->rif_index, dev->dev_addr, 8292 dev->mtu); 8293 if (err) 8294 goto err_rif_edit; 8295 8296 err = mlxsw_sp_rif_fdb_op(mlxsw_sp, dev->dev_addr, fid_index, true); 8297 if (err) 8298 goto err_rif_fdb_op; 8299 8300 if (rif->mtu != dev->mtu) { 8301 struct mlxsw_sp_vr *vr; 8302 int i; 8303 8304 /* The RIF is relevant only to its mr_table instance, as unlike 8305 * unicast routing, in multicast routing a RIF cannot be shared 8306 * between several multicast routing tables. 8307 */ 8308 vr = &mlxsw_sp->router->vrs[rif->vr_id]; 8309 for (i = 0; i < MLXSW_SP_L3_PROTO_MAX; i++) 8310 mlxsw_sp_mr_rif_mtu_update(vr->mr_table[i], 8311 rif, dev->mtu); 8312 } 8313 8314 ether_addr_copy(rif->addr, dev->dev_addr); 8315 rif->mtu = dev->mtu; 8316 8317 netdev_dbg(dev, "Updated RIF=%d\n", rif->rif_index); 8318 8319 return 0; 8320 8321 err_rif_fdb_op: 8322 mlxsw_sp_rif_edit(mlxsw_sp, rif->rif_index, rif->addr, rif->mtu); 8323 err_rif_edit: 8324 mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->addr, fid_index, true); 8325 return err; 8326 } 8327 8328 static int mlxsw_sp_router_port_pre_changeaddr_event(struct mlxsw_sp_rif *rif, 8329 struct netdev_notifier_pre_changeaddr_info *info) 8330 { 8331 struct netlink_ext_ack *extack; 8332 8333 extack = netdev_notifier_info_to_extack(&info->info); 8334 return mlxsw_sp_router_port_check_rif_addr(rif->mlxsw_sp, rif->dev, 8335 info->dev_addr, extack); 8336 } 8337 8338 int mlxsw_sp_netdevice_router_port_event(struct net_device *dev, 8339 unsigned long event, void *ptr) 8340 { 8341 struct mlxsw_sp *mlxsw_sp; 8342 struct mlxsw_sp_rif *rif; 8343 int err = 0; 8344 8345 mlxsw_sp = mlxsw_sp_lower_get(dev); 8346 if (!mlxsw_sp) 8347 return 0; 8348 8349 mutex_lock(&mlxsw_sp->router->lock); 8350 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 8351 if (!rif) 8352 goto out; 8353 8354 switch (event) { 8355 case NETDEV_CHANGEMTU: 8356 case NETDEV_CHANGEADDR: 8357 err = mlxsw_sp_router_port_change_event(mlxsw_sp, rif); 8358 break; 8359 case NETDEV_PRE_CHANGEADDR: 8360 err = mlxsw_sp_router_port_pre_changeaddr_event(rif, ptr); 8361 break; 8362 } 8363 8364 out: 8365 mutex_unlock(&mlxsw_sp->router->lock); 8366 return err; 8367 } 8368 8369 static int mlxsw_sp_port_vrf_join(struct mlxsw_sp *mlxsw_sp, 8370 struct net_device *l3_dev, 8371 struct netlink_ext_ack *extack) 8372 { 8373 struct mlxsw_sp_rif *rif; 8374 8375 /* If netdev is already associated with a RIF, then we need to 8376 * destroy it and create a new one with the new virtual router ID. 8377 */ 8378 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev); 8379 if (rif) 8380 __mlxsw_sp_inetaddr_event(mlxsw_sp, l3_dev, NETDEV_DOWN, 8381 extack); 8382 8383 return __mlxsw_sp_inetaddr_event(mlxsw_sp, l3_dev, NETDEV_UP, extack); 8384 } 8385 8386 static void mlxsw_sp_port_vrf_leave(struct mlxsw_sp *mlxsw_sp, 8387 struct net_device *l3_dev) 8388 { 8389 struct mlxsw_sp_rif *rif; 8390 8391 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev); 8392 if (!rif) 8393 return; 8394 __mlxsw_sp_inetaddr_event(mlxsw_sp, l3_dev, NETDEV_DOWN, NULL); 8395 } 8396 8397 int mlxsw_sp_netdevice_vrf_event(struct net_device *l3_dev, unsigned long event, 8398 struct netdev_notifier_changeupper_info *info) 8399 { 8400 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_lower_get(l3_dev); 8401 int err = 0; 8402 8403 /* We do not create a RIF for a macvlan, but only use it to 8404 * direct more MAC addresses to the router. 8405 */ 8406 if (!mlxsw_sp || netif_is_macvlan(l3_dev)) 8407 return 0; 8408 8409 mutex_lock(&mlxsw_sp->router->lock); 8410 switch (event) { 8411 case NETDEV_PRECHANGEUPPER: 8412 break; 8413 case NETDEV_CHANGEUPPER: 8414 if (info->linking) { 8415 struct netlink_ext_ack *extack; 8416 8417 extack = netdev_notifier_info_to_extack(&info->info); 8418 err = mlxsw_sp_port_vrf_join(mlxsw_sp, l3_dev, extack); 8419 } else { 8420 mlxsw_sp_port_vrf_leave(mlxsw_sp, l3_dev); 8421 } 8422 break; 8423 } 8424 mutex_unlock(&mlxsw_sp->router->lock); 8425 8426 return err; 8427 } 8428 8429 static int __mlxsw_sp_rif_macvlan_flush(struct net_device *dev, 8430 struct netdev_nested_priv *priv) 8431 { 8432 struct mlxsw_sp_rif *rif = (struct mlxsw_sp_rif *)priv->data; 8433 8434 if (!netif_is_macvlan(dev)) 8435 return 0; 8436 8437 return mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, dev->dev_addr, 8438 mlxsw_sp_fid_index(rif->fid), false); 8439 } 8440 8441 static int mlxsw_sp_rif_macvlan_flush(struct mlxsw_sp_rif *rif) 8442 { 8443 struct netdev_nested_priv priv = { 8444 .data = (void *)rif, 8445 }; 8446 8447 if (!netif_is_macvlan_port(rif->dev)) 8448 return 0; 8449 8450 netdev_warn(rif->dev, "Router interface is deleted. Upper macvlans will not work\n"); 8451 return netdev_walk_all_upper_dev_rcu(rif->dev, 8452 __mlxsw_sp_rif_macvlan_flush, &priv); 8453 } 8454 8455 static void mlxsw_sp_rif_subport_setup(struct mlxsw_sp_rif *rif, 8456 const struct mlxsw_sp_rif_params *params) 8457 { 8458 struct mlxsw_sp_rif_subport *rif_subport; 8459 8460 rif_subport = mlxsw_sp_rif_subport_rif(rif); 8461 refcount_set(&rif_subport->ref_count, 1); 8462 rif_subport->vid = params->vid; 8463 rif_subport->lag = params->lag; 8464 if (params->lag) 8465 rif_subport->lag_id = params->lag_id; 8466 else 8467 rif_subport->system_port = params->system_port; 8468 } 8469 8470 static int mlxsw_sp_rif_subport_op(struct mlxsw_sp_rif *rif, bool enable) 8471 { 8472 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 8473 struct mlxsw_sp_rif_subport *rif_subport; 8474 char ritr_pl[MLXSW_REG_RITR_LEN]; 8475 8476 rif_subport = mlxsw_sp_rif_subport_rif(rif); 8477 mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_SP_IF, 8478 rif->rif_index, rif->vr_id, rif->dev->mtu); 8479 mlxsw_reg_ritr_mac_pack(ritr_pl, rif->dev->dev_addr); 8480 mlxsw_reg_ritr_sp_if_pack(ritr_pl, rif_subport->lag, 8481 rif_subport->lag ? rif_subport->lag_id : 8482 rif_subport->system_port, 8483 rif_subport->vid); 8484 8485 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 8486 } 8487 8488 static int mlxsw_sp_rif_subport_configure(struct mlxsw_sp_rif *rif) 8489 { 8490 int err; 8491 8492 err = mlxsw_sp_rif_subport_op(rif, true); 8493 if (err) 8494 return err; 8495 8496 err = mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr, 8497 mlxsw_sp_fid_index(rif->fid), true); 8498 if (err) 8499 goto err_rif_fdb_op; 8500 8501 mlxsw_sp_fid_rif_set(rif->fid, rif); 8502 return 0; 8503 8504 err_rif_fdb_op: 8505 mlxsw_sp_rif_subport_op(rif, false); 8506 return err; 8507 } 8508 8509 static void mlxsw_sp_rif_subport_deconfigure(struct mlxsw_sp_rif *rif) 8510 { 8511 struct mlxsw_sp_fid *fid = rif->fid; 8512 8513 mlxsw_sp_fid_rif_set(fid, NULL); 8514 mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr, 8515 mlxsw_sp_fid_index(fid), false); 8516 mlxsw_sp_rif_macvlan_flush(rif); 8517 mlxsw_sp_rif_subport_op(rif, false); 8518 } 8519 8520 static struct mlxsw_sp_fid * 8521 mlxsw_sp_rif_subport_fid_get(struct mlxsw_sp_rif *rif, 8522 struct netlink_ext_ack *extack) 8523 { 8524 return mlxsw_sp_fid_rfid_get(rif->mlxsw_sp, rif->rif_index); 8525 } 8526 8527 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_subport_ops = { 8528 .type = MLXSW_SP_RIF_TYPE_SUBPORT, 8529 .rif_size = sizeof(struct mlxsw_sp_rif_subport), 8530 .setup = mlxsw_sp_rif_subport_setup, 8531 .configure = mlxsw_sp_rif_subport_configure, 8532 .deconfigure = mlxsw_sp_rif_subport_deconfigure, 8533 .fid_get = mlxsw_sp_rif_subport_fid_get, 8534 }; 8535 8536 static int mlxsw_sp_rif_vlan_fid_op(struct mlxsw_sp_rif *rif, 8537 enum mlxsw_reg_ritr_if_type type, 8538 u16 vid_fid, bool enable) 8539 { 8540 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 8541 char ritr_pl[MLXSW_REG_RITR_LEN]; 8542 8543 mlxsw_reg_ritr_pack(ritr_pl, enable, type, rif->rif_index, rif->vr_id, 8544 rif->dev->mtu); 8545 mlxsw_reg_ritr_mac_pack(ritr_pl, rif->dev->dev_addr); 8546 mlxsw_reg_ritr_fid_set(ritr_pl, type, vid_fid); 8547 8548 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 8549 } 8550 8551 u8 mlxsw_sp_router_port(const struct mlxsw_sp *mlxsw_sp) 8552 { 8553 return mlxsw_core_max_ports(mlxsw_sp->core) + 1; 8554 } 8555 8556 static int mlxsw_sp_rif_fid_configure(struct mlxsw_sp_rif *rif) 8557 { 8558 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 8559 u16 fid_index = mlxsw_sp_fid_index(rif->fid); 8560 int err; 8561 8562 err = mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index, 8563 true); 8564 if (err) 8565 return err; 8566 8567 err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC, 8568 mlxsw_sp_router_port(mlxsw_sp), true); 8569 if (err) 8570 goto err_fid_mc_flood_set; 8571 8572 err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC, 8573 mlxsw_sp_router_port(mlxsw_sp), true); 8574 if (err) 8575 goto err_fid_bc_flood_set; 8576 8577 err = mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr, 8578 mlxsw_sp_fid_index(rif->fid), true); 8579 if (err) 8580 goto err_rif_fdb_op; 8581 8582 mlxsw_sp_fid_rif_set(rif->fid, rif); 8583 return 0; 8584 8585 err_rif_fdb_op: 8586 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC, 8587 mlxsw_sp_router_port(mlxsw_sp), false); 8588 err_fid_bc_flood_set: 8589 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC, 8590 mlxsw_sp_router_port(mlxsw_sp), false); 8591 err_fid_mc_flood_set: 8592 mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index, false); 8593 return err; 8594 } 8595 8596 static void mlxsw_sp_rif_fid_deconfigure(struct mlxsw_sp_rif *rif) 8597 { 8598 u16 fid_index = mlxsw_sp_fid_index(rif->fid); 8599 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 8600 struct mlxsw_sp_fid *fid = rif->fid; 8601 8602 mlxsw_sp_fid_rif_set(fid, NULL); 8603 mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr, 8604 mlxsw_sp_fid_index(fid), false); 8605 mlxsw_sp_rif_macvlan_flush(rif); 8606 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC, 8607 mlxsw_sp_router_port(mlxsw_sp), false); 8608 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC, 8609 mlxsw_sp_router_port(mlxsw_sp), false); 8610 mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index, false); 8611 } 8612 8613 static struct mlxsw_sp_fid * 8614 mlxsw_sp_rif_fid_fid_get(struct mlxsw_sp_rif *rif, 8615 struct netlink_ext_ack *extack) 8616 { 8617 return mlxsw_sp_fid_8021d_get(rif->mlxsw_sp, rif->dev->ifindex); 8618 } 8619 8620 static void mlxsw_sp_rif_fid_fdb_del(struct mlxsw_sp_rif *rif, const char *mac) 8621 { 8622 struct switchdev_notifier_fdb_info info; 8623 struct net_device *dev; 8624 8625 dev = br_fdb_find_port(rif->dev, mac, 0); 8626 if (!dev) 8627 return; 8628 8629 info.addr = mac; 8630 info.vid = 0; 8631 call_switchdev_notifiers(SWITCHDEV_FDB_DEL_TO_BRIDGE, dev, &info.info, 8632 NULL); 8633 } 8634 8635 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_fid_ops = { 8636 .type = MLXSW_SP_RIF_TYPE_FID, 8637 .rif_size = sizeof(struct mlxsw_sp_rif), 8638 .configure = mlxsw_sp_rif_fid_configure, 8639 .deconfigure = mlxsw_sp_rif_fid_deconfigure, 8640 .fid_get = mlxsw_sp_rif_fid_fid_get, 8641 .fdb_del = mlxsw_sp_rif_fid_fdb_del, 8642 }; 8643 8644 static struct mlxsw_sp_fid * 8645 mlxsw_sp_rif_vlan_fid_get(struct mlxsw_sp_rif *rif, 8646 struct netlink_ext_ack *extack) 8647 { 8648 struct net_device *br_dev; 8649 u16 vid; 8650 int err; 8651 8652 if (is_vlan_dev(rif->dev)) { 8653 vid = vlan_dev_vlan_id(rif->dev); 8654 br_dev = vlan_dev_real_dev(rif->dev); 8655 if (WARN_ON(!netif_is_bridge_master(br_dev))) 8656 return ERR_PTR(-EINVAL); 8657 } else { 8658 err = br_vlan_get_pvid(rif->dev, &vid); 8659 if (err < 0 || !vid) { 8660 NL_SET_ERR_MSG_MOD(extack, "Couldn't determine bridge PVID"); 8661 return ERR_PTR(-EINVAL); 8662 } 8663 } 8664 8665 return mlxsw_sp_fid_8021q_get(rif->mlxsw_sp, vid); 8666 } 8667 8668 static void mlxsw_sp_rif_vlan_fdb_del(struct mlxsw_sp_rif *rif, const char *mac) 8669 { 8670 u16 vid = mlxsw_sp_fid_8021q_vid(rif->fid); 8671 struct switchdev_notifier_fdb_info info; 8672 struct net_device *br_dev; 8673 struct net_device *dev; 8674 8675 br_dev = is_vlan_dev(rif->dev) ? vlan_dev_real_dev(rif->dev) : rif->dev; 8676 dev = br_fdb_find_port(br_dev, mac, vid); 8677 if (!dev) 8678 return; 8679 8680 info.addr = mac; 8681 info.vid = vid; 8682 call_switchdev_notifiers(SWITCHDEV_FDB_DEL_TO_BRIDGE, dev, &info.info, 8683 NULL); 8684 } 8685 8686 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_vlan_emu_ops = { 8687 .type = MLXSW_SP_RIF_TYPE_VLAN, 8688 .rif_size = sizeof(struct mlxsw_sp_rif), 8689 .configure = mlxsw_sp_rif_fid_configure, 8690 .deconfigure = mlxsw_sp_rif_fid_deconfigure, 8691 .fid_get = mlxsw_sp_rif_vlan_fid_get, 8692 .fdb_del = mlxsw_sp_rif_vlan_fdb_del, 8693 }; 8694 8695 static struct mlxsw_sp_rif_ipip_lb * 8696 mlxsw_sp_rif_ipip_lb_rif(struct mlxsw_sp_rif *rif) 8697 { 8698 return container_of(rif, struct mlxsw_sp_rif_ipip_lb, common); 8699 } 8700 8701 static void 8702 mlxsw_sp_rif_ipip_lb_setup(struct mlxsw_sp_rif *rif, 8703 const struct mlxsw_sp_rif_params *params) 8704 { 8705 struct mlxsw_sp_rif_params_ipip_lb *params_lb; 8706 struct mlxsw_sp_rif_ipip_lb *rif_lb; 8707 8708 params_lb = container_of(params, struct mlxsw_sp_rif_params_ipip_lb, 8709 common); 8710 rif_lb = mlxsw_sp_rif_ipip_lb_rif(rif); 8711 rif_lb->lb_config = params_lb->lb_config; 8712 } 8713 8714 static int 8715 mlxsw_sp1_rif_ipip_lb_configure(struct mlxsw_sp_rif *rif) 8716 { 8717 struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif); 8718 u32 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(rif->dev); 8719 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 8720 struct mlxsw_sp_vr *ul_vr; 8721 int err; 8722 8723 ul_vr = mlxsw_sp_vr_get(mlxsw_sp, ul_tb_id, NULL); 8724 if (IS_ERR(ul_vr)) 8725 return PTR_ERR(ul_vr); 8726 8727 err = mlxsw_sp_rif_ipip_lb_op(lb_rif, ul_vr->id, 0, true); 8728 if (err) 8729 goto err_loopback_op; 8730 8731 lb_rif->ul_vr_id = ul_vr->id; 8732 lb_rif->ul_rif_id = 0; 8733 ++ul_vr->rif_count; 8734 return 0; 8735 8736 err_loopback_op: 8737 mlxsw_sp_vr_put(mlxsw_sp, ul_vr); 8738 return err; 8739 } 8740 8741 static void mlxsw_sp1_rif_ipip_lb_deconfigure(struct mlxsw_sp_rif *rif) 8742 { 8743 struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif); 8744 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 8745 struct mlxsw_sp_vr *ul_vr; 8746 8747 ul_vr = &mlxsw_sp->router->vrs[lb_rif->ul_vr_id]; 8748 mlxsw_sp_rif_ipip_lb_op(lb_rif, ul_vr->id, 0, false); 8749 8750 --ul_vr->rif_count; 8751 mlxsw_sp_vr_put(mlxsw_sp, ul_vr); 8752 } 8753 8754 static const struct mlxsw_sp_rif_ops mlxsw_sp1_rif_ipip_lb_ops = { 8755 .type = MLXSW_SP_RIF_TYPE_IPIP_LB, 8756 .rif_size = sizeof(struct mlxsw_sp_rif_ipip_lb), 8757 .setup = mlxsw_sp_rif_ipip_lb_setup, 8758 .configure = mlxsw_sp1_rif_ipip_lb_configure, 8759 .deconfigure = mlxsw_sp1_rif_ipip_lb_deconfigure, 8760 }; 8761 8762 const struct mlxsw_sp_rif_ops *mlxsw_sp1_rif_ops_arr[] = { 8763 [MLXSW_SP_RIF_TYPE_SUBPORT] = &mlxsw_sp_rif_subport_ops, 8764 [MLXSW_SP_RIF_TYPE_VLAN] = &mlxsw_sp_rif_vlan_emu_ops, 8765 [MLXSW_SP_RIF_TYPE_FID] = &mlxsw_sp_rif_fid_ops, 8766 [MLXSW_SP_RIF_TYPE_IPIP_LB] = &mlxsw_sp1_rif_ipip_lb_ops, 8767 }; 8768 8769 static int 8770 mlxsw_sp_rif_ipip_lb_ul_rif_op(struct mlxsw_sp_rif *ul_rif, bool enable) 8771 { 8772 struct mlxsw_sp *mlxsw_sp = ul_rif->mlxsw_sp; 8773 char ritr_pl[MLXSW_REG_RITR_LEN]; 8774 8775 mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_LOOPBACK_IF, 8776 ul_rif->rif_index, ul_rif->vr_id, IP_MAX_MTU); 8777 mlxsw_reg_ritr_loopback_protocol_set(ritr_pl, 8778 MLXSW_REG_RITR_LOOPBACK_GENERIC); 8779 8780 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 8781 } 8782 8783 static struct mlxsw_sp_rif * 8784 mlxsw_sp_ul_rif_create(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_vr *vr, 8785 struct netlink_ext_ack *extack) 8786 { 8787 struct mlxsw_sp_rif *ul_rif; 8788 u16 rif_index; 8789 int err; 8790 8791 err = mlxsw_sp_rif_index_alloc(mlxsw_sp, &rif_index); 8792 if (err) { 8793 NL_SET_ERR_MSG_MOD(extack, "Exceeded number of supported router interfaces"); 8794 return ERR_PTR(err); 8795 } 8796 8797 ul_rif = mlxsw_sp_rif_alloc(sizeof(*ul_rif), rif_index, vr->id, NULL); 8798 if (!ul_rif) 8799 return ERR_PTR(-ENOMEM); 8800 8801 mlxsw_sp->router->rifs[rif_index] = ul_rif; 8802 ul_rif->mlxsw_sp = mlxsw_sp; 8803 err = mlxsw_sp_rif_ipip_lb_ul_rif_op(ul_rif, true); 8804 if (err) 8805 goto ul_rif_op_err; 8806 8807 return ul_rif; 8808 8809 ul_rif_op_err: 8810 mlxsw_sp->router->rifs[rif_index] = NULL; 8811 kfree(ul_rif); 8812 return ERR_PTR(err); 8813 } 8814 8815 static void mlxsw_sp_ul_rif_destroy(struct mlxsw_sp_rif *ul_rif) 8816 { 8817 struct mlxsw_sp *mlxsw_sp = ul_rif->mlxsw_sp; 8818 8819 mlxsw_sp_rif_ipip_lb_ul_rif_op(ul_rif, false); 8820 mlxsw_sp->router->rifs[ul_rif->rif_index] = NULL; 8821 kfree(ul_rif); 8822 } 8823 8824 static struct mlxsw_sp_rif * 8825 mlxsw_sp_ul_rif_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id, 8826 struct netlink_ext_ack *extack) 8827 { 8828 struct mlxsw_sp_vr *vr; 8829 int err; 8830 8831 vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id, extack); 8832 if (IS_ERR(vr)) 8833 return ERR_CAST(vr); 8834 8835 if (refcount_inc_not_zero(&vr->ul_rif_refcnt)) 8836 return vr->ul_rif; 8837 8838 vr->ul_rif = mlxsw_sp_ul_rif_create(mlxsw_sp, vr, extack); 8839 if (IS_ERR(vr->ul_rif)) { 8840 err = PTR_ERR(vr->ul_rif); 8841 goto err_ul_rif_create; 8842 } 8843 8844 vr->rif_count++; 8845 refcount_set(&vr->ul_rif_refcnt, 1); 8846 8847 return vr->ul_rif; 8848 8849 err_ul_rif_create: 8850 mlxsw_sp_vr_put(mlxsw_sp, vr); 8851 return ERR_PTR(err); 8852 } 8853 8854 static void mlxsw_sp_ul_rif_put(struct mlxsw_sp_rif *ul_rif) 8855 { 8856 struct mlxsw_sp *mlxsw_sp = ul_rif->mlxsw_sp; 8857 struct mlxsw_sp_vr *vr; 8858 8859 vr = &mlxsw_sp->router->vrs[ul_rif->vr_id]; 8860 8861 if (!refcount_dec_and_test(&vr->ul_rif_refcnt)) 8862 return; 8863 8864 vr->rif_count--; 8865 mlxsw_sp_ul_rif_destroy(ul_rif); 8866 mlxsw_sp_vr_put(mlxsw_sp, vr); 8867 } 8868 8869 int mlxsw_sp_router_ul_rif_get(struct mlxsw_sp *mlxsw_sp, u32 ul_tb_id, 8870 u16 *ul_rif_index) 8871 { 8872 struct mlxsw_sp_rif *ul_rif; 8873 int err = 0; 8874 8875 mutex_lock(&mlxsw_sp->router->lock); 8876 ul_rif = mlxsw_sp_ul_rif_get(mlxsw_sp, ul_tb_id, NULL); 8877 if (IS_ERR(ul_rif)) { 8878 err = PTR_ERR(ul_rif); 8879 goto out; 8880 } 8881 *ul_rif_index = ul_rif->rif_index; 8882 out: 8883 mutex_unlock(&mlxsw_sp->router->lock); 8884 return err; 8885 } 8886 8887 void mlxsw_sp_router_ul_rif_put(struct mlxsw_sp *mlxsw_sp, u16 ul_rif_index) 8888 { 8889 struct mlxsw_sp_rif *ul_rif; 8890 8891 mutex_lock(&mlxsw_sp->router->lock); 8892 ul_rif = mlxsw_sp->router->rifs[ul_rif_index]; 8893 if (WARN_ON(!ul_rif)) 8894 goto out; 8895 8896 mlxsw_sp_ul_rif_put(ul_rif); 8897 out: 8898 mutex_unlock(&mlxsw_sp->router->lock); 8899 } 8900 8901 static int 8902 mlxsw_sp2_rif_ipip_lb_configure(struct mlxsw_sp_rif *rif) 8903 { 8904 struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif); 8905 u32 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(rif->dev); 8906 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 8907 struct mlxsw_sp_rif *ul_rif; 8908 int err; 8909 8910 ul_rif = mlxsw_sp_ul_rif_get(mlxsw_sp, ul_tb_id, NULL); 8911 if (IS_ERR(ul_rif)) 8912 return PTR_ERR(ul_rif); 8913 8914 err = mlxsw_sp_rif_ipip_lb_op(lb_rif, 0, ul_rif->rif_index, true); 8915 if (err) 8916 goto err_loopback_op; 8917 8918 lb_rif->ul_vr_id = 0; 8919 lb_rif->ul_rif_id = ul_rif->rif_index; 8920 8921 return 0; 8922 8923 err_loopback_op: 8924 mlxsw_sp_ul_rif_put(ul_rif); 8925 return err; 8926 } 8927 8928 static void mlxsw_sp2_rif_ipip_lb_deconfigure(struct mlxsw_sp_rif *rif) 8929 { 8930 struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif); 8931 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 8932 struct mlxsw_sp_rif *ul_rif; 8933 8934 ul_rif = mlxsw_sp_rif_by_index(mlxsw_sp, lb_rif->ul_rif_id); 8935 mlxsw_sp_rif_ipip_lb_op(lb_rif, 0, lb_rif->ul_rif_id, false); 8936 mlxsw_sp_ul_rif_put(ul_rif); 8937 } 8938 8939 static const struct mlxsw_sp_rif_ops mlxsw_sp2_rif_ipip_lb_ops = { 8940 .type = MLXSW_SP_RIF_TYPE_IPIP_LB, 8941 .rif_size = sizeof(struct mlxsw_sp_rif_ipip_lb), 8942 .setup = mlxsw_sp_rif_ipip_lb_setup, 8943 .configure = mlxsw_sp2_rif_ipip_lb_configure, 8944 .deconfigure = mlxsw_sp2_rif_ipip_lb_deconfigure, 8945 }; 8946 8947 const struct mlxsw_sp_rif_ops *mlxsw_sp2_rif_ops_arr[] = { 8948 [MLXSW_SP_RIF_TYPE_SUBPORT] = &mlxsw_sp_rif_subport_ops, 8949 [MLXSW_SP_RIF_TYPE_VLAN] = &mlxsw_sp_rif_vlan_emu_ops, 8950 [MLXSW_SP_RIF_TYPE_FID] = &mlxsw_sp_rif_fid_ops, 8951 [MLXSW_SP_RIF_TYPE_IPIP_LB] = &mlxsw_sp2_rif_ipip_lb_ops, 8952 }; 8953 8954 static int mlxsw_sp_rifs_init(struct mlxsw_sp *mlxsw_sp) 8955 { 8956 u64 max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); 8957 8958 mlxsw_sp->router->rifs = kcalloc(max_rifs, 8959 sizeof(struct mlxsw_sp_rif *), 8960 GFP_KERNEL); 8961 if (!mlxsw_sp->router->rifs) 8962 return -ENOMEM; 8963 8964 return 0; 8965 } 8966 8967 static void mlxsw_sp_rifs_fini(struct mlxsw_sp *mlxsw_sp) 8968 { 8969 int i; 8970 8971 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++) 8972 WARN_ON_ONCE(mlxsw_sp->router->rifs[i]); 8973 8974 kfree(mlxsw_sp->router->rifs); 8975 } 8976 8977 static int 8978 mlxsw_sp_ipip_config_tigcr(struct mlxsw_sp *mlxsw_sp) 8979 { 8980 char tigcr_pl[MLXSW_REG_TIGCR_LEN]; 8981 8982 mlxsw_reg_tigcr_pack(tigcr_pl, true, 0); 8983 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(tigcr), tigcr_pl); 8984 } 8985 8986 static int mlxsw_sp_ipips_init(struct mlxsw_sp *mlxsw_sp) 8987 { 8988 int err; 8989 8990 mlxsw_sp->router->ipip_ops_arr = mlxsw_sp_ipip_ops_arr; 8991 INIT_LIST_HEAD(&mlxsw_sp->router->ipip_list); 8992 8993 err = mlxsw_sp_ipip_ecn_encap_init(mlxsw_sp); 8994 if (err) 8995 return err; 8996 err = mlxsw_sp_ipip_ecn_decap_init(mlxsw_sp); 8997 if (err) 8998 return err; 8999 9000 return mlxsw_sp_ipip_config_tigcr(mlxsw_sp); 9001 } 9002 9003 static void mlxsw_sp_ipips_fini(struct mlxsw_sp *mlxsw_sp) 9004 { 9005 WARN_ON(!list_empty(&mlxsw_sp->router->ipip_list)); 9006 } 9007 9008 static void mlxsw_sp_router_fib_dump_flush(struct notifier_block *nb) 9009 { 9010 struct mlxsw_sp_router *router; 9011 9012 /* Flush pending FIB notifications and then flush the device's 9013 * table before requesting another dump. The FIB notification 9014 * block is unregistered, so no need to take RTNL. 9015 */ 9016 mlxsw_core_flush_owq(); 9017 router = container_of(nb, struct mlxsw_sp_router, fib_nb); 9018 mlxsw_sp_router_fib_flush(router->mlxsw_sp); 9019 } 9020 9021 #ifdef CONFIG_IP_ROUTE_MULTIPATH 9022 static void mlxsw_sp_mp_hash_header_set(char *recr2_pl, int header) 9023 { 9024 mlxsw_reg_recr2_outer_header_enables_set(recr2_pl, header, true); 9025 } 9026 9027 static void mlxsw_sp_mp_hash_field_set(char *recr2_pl, int field) 9028 { 9029 mlxsw_reg_recr2_outer_header_fields_enable_set(recr2_pl, field, true); 9030 } 9031 9032 static void mlxsw_sp_mp4_hash_init(struct mlxsw_sp *mlxsw_sp, char *recr2_pl) 9033 { 9034 struct net *net = mlxsw_sp_net(mlxsw_sp); 9035 bool only_l3 = !net->ipv4.sysctl_fib_multipath_hash_policy; 9036 9037 mlxsw_sp_mp_hash_header_set(recr2_pl, 9038 MLXSW_REG_RECR2_IPV4_EN_NOT_TCP_NOT_UDP); 9039 mlxsw_sp_mp_hash_header_set(recr2_pl, MLXSW_REG_RECR2_IPV4_EN_TCP_UDP); 9040 mlxsw_reg_recr2_ipv4_sip_enable(recr2_pl); 9041 mlxsw_reg_recr2_ipv4_dip_enable(recr2_pl); 9042 if (only_l3) 9043 return; 9044 mlxsw_sp_mp_hash_header_set(recr2_pl, MLXSW_REG_RECR2_TCP_UDP_EN_IPV4); 9045 mlxsw_sp_mp_hash_field_set(recr2_pl, MLXSW_REG_RECR2_IPV4_PROTOCOL); 9046 mlxsw_sp_mp_hash_field_set(recr2_pl, MLXSW_REG_RECR2_TCP_UDP_SPORT); 9047 mlxsw_sp_mp_hash_field_set(recr2_pl, MLXSW_REG_RECR2_TCP_UDP_DPORT); 9048 } 9049 9050 static void mlxsw_sp_mp6_hash_init(struct mlxsw_sp *mlxsw_sp, char *recr2_pl) 9051 { 9052 bool only_l3 = !ip6_multipath_hash_policy(mlxsw_sp_net(mlxsw_sp)); 9053 9054 mlxsw_sp_mp_hash_header_set(recr2_pl, 9055 MLXSW_REG_RECR2_IPV6_EN_NOT_TCP_NOT_UDP); 9056 mlxsw_sp_mp_hash_header_set(recr2_pl, MLXSW_REG_RECR2_IPV6_EN_TCP_UDP); 9057 mlxsw_reg_recr2_ipv6_sip_enable(recr2_pl); 9058 mlxsw_reg_recr2_ipv6_dip_enable(recr2_pl); 9059 mlxsw_sp_mp_hash_field_set(recr2_pl, MLXSW_REG_RECR2_IPV6_NEXT_HEADER); 9060 if (only_l3) { 9061 mlxsw_sp_mp_hash_field_set(recr2_pl, 9062 MLXSW_REG_RECR2_IPV6_FLOW_LABEL); 9063 } else { 9064 mlxsw_sp_mp_hash_header_set(recr2_pl, 9065 MLXSW_REG_RECR2_TCP_UDP_EN_IPV6); 9066 mlxsw_sp_mp_hash_field_set(recr2_pl, 9067 MLXSW_REG_RECR2_TCP_UDP_SPORT); 9068 mlxsw_sp_mp_hash_field_set(recr2_pl, 9069 MLXSW_REG_RECR2_TCP_UDP_DPORT); 9070 } 9071 } 9072 9073 static int mlxsw_sp_mp_hash_init(struct mlxsw_sp *mlxsw_sp) 9074 { 9075 char recr2_pl[MLXSW_REG_RECR2_LEN]; 9076 u32 seed; 9077 9078 seed = jhash(mlxsw_sp->base_mac, sizeof(mlxsw_sp->base_mac), 0); 9079 mlxsw_reg_recr2_pack(recr2_pl, seed); 9080 mlxsw_sp_mp4_hash_init(mlxsw_sp, recr2_pl); 9081 mlxsw_sp_mp6_hash_init(mlxsw_sp, recr2_pl); 9082 9083 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(recr2), recr2_pl); 9084 } 9085 #else 9086 static int mlxsw_sp_mp_hash_init(struct mlxsw_sp *mlxsw_sp) 9087 { 9088 return 0; 9089 } 9090 #endif 9091 9092 static int mlxsw_sp_dscp_init(struct mlxsw_sp *mlxsw_sp) 9093 { 9094 char rdpm_pl[MLXSW_REG_RDPM_LEN]; 9095 unsigned int i; 9096 9097 MLXSW_REG_ZERO(rdpm, rdpm_pl); 9098 9099 /* HW is determining switch priority based on DSCP-bits, but the 9100 * kernel is still doing that based on the ToS. Since there's a 9101 * mismatch in bits we need to make sure to translate the right 9102 * value ToS would observe, skipping the 2 least-significant ECN bits. 9103 */ 9104 for (i = 0; i < MLXSW_REG_RDPM_DSCP_ENTRY_REC_MAX_COUNT; i++) 9105 mlxsw_reg_rdpm_pack(rdpm_pl, i, rt_tos2priority(i << 2)); 9106 9107 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rdpm), rdpm_pl); 9108 } 9109 9110 static int __mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp) 9111 { 9112 struct net *net = mlxsw_sp_net(mlxsw_sp); 9113 bool usp = net->ipv4.sysctl_ip_fwd_update_priority; 9114 char rgcr_pl[MLXSW_REG_RGCR_LEN]; 9115 u64 max_rifs; 9116 9117 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_RIFS)) 9118 return -EIO; 9119 max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); 9120 9121 mlxsw_reg_rgcr_pack(rgcr_pl, true, true); 9122 mlxsw_reg_rgcr_max_router_interfaces_set(rgcr_pl, max_rifs); 9123 mlxsw_reg_rgcr_usp_set(rgcr_pl, usp); 9124 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl); 9125 } 9126 9127 static void __mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp) 9128 { 9129 char rgcr_pl[MLXSW_REG_RGCR_LEN]; 9130 9131 mlxsw_reg_rgcr_pack(rgcr_pl, false, false); 9132 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl); 9133 } 9134 9135 static const struct mlxsw_sp_router_ll_ops mlxsw_sp_router_ll_basic_ops = { 9136 .init = mlxsw_sp_router_ll_basic_init, 9137 .ralta_write = mlxsw_sp_router_ll_basic_ralta_write, 9138 .ralst_write = mlxsw_sp_router_ll_basic_ralst_write, 9139 .raltb_write = mlxsw_sp_router_ll_basic_raltb_write, 9140 .fib_entry_op_ctx_size = sizeof(struct mlxsw_sp_fib_entry_op_ctx_basic), 9141 .fib_entry_pack = mlxsw_sp_router_ll_basic_fib_entry_pack, 9142 .fib_entry_act_remote_pack = mlxsw_sp_router_ll_basic_fib_entry_act_remote_pack, 9143 .fib_entry_act_local_pack = mlxsw_sp_router_ll_basic_fib_entry_act_local_pack, 9144 .fib_entry_act_ip2me_pack = mlxsw_sp_router_ll_basic_fib_entry_act_ip2me_pack, 9145 .fib_entry_act_ip2me_tun_pack = mlxsw_sp_router_ll_basic_fib_entry_act_ip2me_tun_pack, 9146 .fib_entry_commit = mlxsw_sp_router_ll_basic_fib_entry_commit, 9147 .fib_entry_is_committed = mlxsw_sp_router_ll_basic_fib_entry_is_committed, 9148 }; 9149 9150 static int mlxsw_sp_router_ll_op_ctx_init(struct mlxsw_sp_router *router) 9151 { 9152 size_t max_size = 0; 9153 int i; 9154 9155 for (i = 0; i < MLXSW_SP_L3_PROTO_MAX; i++) { 9156 size_t size = router->proto_ll_ops[i]->fib_entry_op_ctx_size; 9157 9158 if (size > max_size) 9159 max_size = size; 9160 } 9161 router->ll_op_ctx = kzalloc(sizeof(*router->ll_op_ctx) + max_size, 9162 GFP_KERNEL); 9163 if (!router->ll_op_ctx) 9164 return -ENOMEM; 9165 INIT_LIST_HEAD(&router->ll_op_ctx->fib_entry_priv_list); 9166 return 0; 9167 } 9168 9169 static void mlxsw_sp_router_ll_op_ctx_fini(struct mlxsw_sp_router *router) 9170 { 9171 WARN_ON(!list_empty(&router->ll_op_ctx->fib_entry_priv_list)); 9172 kfree(router->ll_op_ctx); 9173 } 9174 9175 static int mlxsw_sp_lb_rif_init(struct mlxsw_sp *mlxsw_sp) 9176 { 9177 u16 lb_rif_index; 9178 int err; 9179 9180 /* Create a generic loopback RIF associated with the main table 9181 * (default VRF). Any table can be used, but the main table exists 9182 * anyway, so we do not waste resources. 9183 */ 9184 err = mlxsw_sp_router_ul_rif_get(mlxsw_sp, RT_TABLE_MAIN, 9185 &lb_rif_index); 9186 if (err) 9187 return err; 9188 9189 mlxsw_sp->router->lb_rif_index = lb_rif_index; 9190 9191 return 0; 9192 } 9193 9194 static void mlxsw_sp_lb_rif_fini(struct mlxsw_sp *mlxsw_sp) 9195 { 9196 mlxsw_sp_router_ul_rif_put(mlxsw_sp, mlxsw_sp->router->lb_rif_index); 9197 } 9198 9199 int mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp, 9200 struct netlink_ext_ack *extack) 9201 { 9202 struct mlxsw_sp_router *router; 9203 int err; 9204 9205 router = kzalloc(sizeof(*mlxsw_sp->router), GFP_KERNEL); 9206 if (!router) 9207 return -ENOMEM; 9208 mutex_init(&router->lock); 9209 mlxsw_sp->router = router; 9210 router->mlxsw_sp = mlxsw_sp; 9211 9212 err = mlxsw_sp_router_xm_init(mlxsw_sp); 9213 if (err) 9214 goto err_xm_init; 9215 9216 router->proto_ll_ops[MLXSW_SP_L3_PROTO_IPV4] = mlxsw_sp_router_xm_ipv4_is_supported(mlxsw_sp) ? 9217 &mlxsw_sp_router_ll_xm_ops : 9218 &mlxsw_sp_router_ll_basic_ops; 9219 router->proto_ll_ops[MLXSW_SP_L3_PROTO_IPV6] = &mlxsw_sp_router_ll_basic_ops; 9220 9221 err = mlxsw_sp_router_ll_op_ctx_init(router); 9222 if (err) 9223 goto err_ll_op_ctx_init; 9224 9225 INIT_LIST_HEAD(&mlxsw_sp->router->nexthop_neighs_list); 9226 err = __mlxsw_sp_router_init(mlxsw_sp); 9227 if (err) 9228 goto err_router_init; 9229 9230 err = mlxsw_sp_rifs_init(mlxsw_sp); 9231 if (err) 9232 goto err_rifs_init; 9233 9234 err = mlxsw_sp_ipips_init(mlxsw_sp); 9235 if (err) 9236 goto err_ipips_init; 9237 9238 err = rhashtable_init(&mlxsw_sp->router->nexthop_ht, 9239 &mlxsw_sp_nexthop_ht_params); 9240 if (err) 9241 goto err_nexthop_ht_init; 9242 9243 err = rhashtable_init(&mlxsw_sp->router->nexthop_group_ht, 9244 &mlxsw_sp_nexthop_group_ht_params); 9245 if (err) 9246 goto err_nexthop_group_ht_init; 9247 9248 INIT_LIST_HEAD(&mlxsw_sp->router->nexthop_list); 9249 err = mlxsw_sp_lpm_init(mlxsw_sp); 9250 if (err) 9251 goto err_lpm_init; 9252 9253 err = mlxsw_sp_mr_init(mlxsw_sp, &mlxsw_sp_mr_tcam_ops); 9254 if (err) 9255 goto err_mr_init; 9256 9257 err = mlxsw_sp_vrs_init(mlxsw_sp); 9258 if (err) 9259 goto err_vrs_init; 9260 9261 err = mlxsw_sp_lb_rif_init(mlxsw_sp); 9262 if (err) 9263 goto err_lb_rif_init; 9264 9265 err = mlxsw_sp_neigh_init(mlxsw_sp); 9266 if (err) 9267 goto err_neigh_init; 9268 9269 err = mlxsw_sp_mp_hash_init(mlxsw_sp); 9270 if (err) 9271 goto err_mp_hash_init; 9272 9273 err = mlxsw_sp_dscp_init(mlxsw_sp); 9274 if (err) 9275 goto err_dscp_init; 9276 9277 INIT_WORK(&router->fib_event_work, mlxsw_sp_router_fib_event_work); 9278 INIT_LIST_HEAD(&router->fib_event_queue); 9279 spin_lock_init(&router->fib_event_queue_lock); 9280 9281 router->inetaddr_nb.notifier_call = mlxsw_sp_inetaddr_event; 9282 err = register_inetaddr_notifier(&router->inetaddr_nb); 9283 if (err) 9284 goto err_register_inetaddr_notifier; 9285 9286 router->inet6addr_nb.notifier_call = mlxsw_sp_inet6addr_event; 9287 err = register_inet6addr_notifier(&router->inet6addr_nb); 9288 if (err) 9289 goto err_register_inet6addr_notifier; 9290 9291 mlxsw_sp->router->netevent_nb.notifier_call = 9292 mlxsw_sp_router_netevent_event; 9293 err = register_netevent_notifier(&mlxsw_sp->router->netevent_nb); 9294 if (err) 9295 goto err_register_netevent_notifier; 9296 9297 mlxsw_sp->router->nexthop_nb.notifier_call = 9298 mlxsw_sp_nexthop_obj_event; 9299 err = register_nexthop_notifier(mlxsw_sp_net(mlxsw_sp), 9300 &mlxsw_sp->router->nexthop_nb, 9301 extack); 9302 if (err) 9303 goto err_register_nexthop_notifier; 9304 9305 mlxsw_sp->router->fib_nb.notifier_call = mlxsw_sp_router_fib_event; 9306 err = register_fib_notifier(mlxsw_sp_net(mlxsw_sp), 9307 &mlxsw_sp->router->fib_nb, 9308 mlxsw_sp_router_fib_dump_flush, extack); 9309 if (err) 9310 goto err_register_fib_notifier; 9311 9312 return 0; 9313 9314 err_register_fib_notifier: 9315 unregister_nexthop_notifier(mlxsw_sp_net(mlxsw_sp), 9316 &mlxsw_sp->router->nexthop_nb); 9317 err_register_nexthop_notifier: 9318 unregister_netevent_notifier(&mlxsw_sp->router->netevent_nb); 9319 err_register_netevent_notifier: 9320 unregister_inet6addr_notifier(&router->inet6addr_nb); 9321 err_register_inet6addr_notifier: 9322 unregister_inetaddr_notifier(&router->inetaddr_nb); 9323 err_register_inetaddr_notifier: 9324 mlxsw_core_flush_owq(); 9325 WARN_ON(!list_empty(&router->fib_event_queue)); 9326 err_dscp_init: 9327 err_mp_hash_init: 9328 mlxsw_sp_neigh_fini(mlxsw_sp); 9329 err_neigh_init: 9330 mlxsw_sp_lb_rif_fini(mlxsw_sp); 9331 err_lb_rif_init: 9332 mlxsw_sp_vrs_fini(mlxsw_sp); 9333 err_vrs_init: 9334 mlxsw_sp_mr_fini(mlxsw_sp); 9335 err_mr_init: 9336 mlxsw_sp_lpm_fini(mlxsw_sp); 9337 err_lpm_init: 9338 rhashtable_destroy(&mlxsw_sp->router->nexthop_group_ht); 9339 err_nexthop_group_ht_init: 9340 rhashtable_destroy(&mlxsw_sp->router->nexthop_ht); 9341 err_nexthop_ht_init: 9342 mlxsw_sp_ipips_fini(mlxsw_sp); 9343 err_ipips_init: 9344 mlxsw_sp_rifs_fini(mlxsw_sp); 9345 err_rifs_init: 9346 __mlxsw_sp_router_fini(mlxsw_sp); 9347 err_router_init: 9348 mlxsw_sp_router_ll_op_ctx_fini(router); 9349 err_ll_op_ctx_init: 9350 mlxsw_sp_router_xm_fini(mlxsw_sp); 9351 err_xm_init: 9352 mutex_destroy(&mlxsw_sp->router->lock); 9353 kfree(mlxsw_sp->router); 9354 return err; 9355 } 9356 9357 void mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp) 9358 { 9359 unregister_fib_notifier(mlxsw_sp_net(mlxsw_sp), 9360 &mlxsw_sp->router->fib_nb); 9361 unregister_nexthop_notifier(mlxsw_sp_net(mlxsw_sp), 9362 &mlxsw_sp->router->nexthop_nb); 9363 unregister_netevent_notifier(&mlxsw_sp->router->netevent_nb); 9364 unregister_inet6addr_notifier(&mlxsw_sp->router->inet6addr_nb); 9365 unregister_inetaddr_notifier(&mlxsw_sp->router->inetaddr_nb); 9366 mlxsw_core_flush_owq(); 9367 WARN_ON(!list_empty(&mlxsw_sp->router->fib_event_queue)); 9368 mlxsw_sp_neigh_fini(mlxsw_sp); 9369 mlxsw_sp_lb_rif_fini(mlxsw_sp); 9370 mlxsw_sp_vrs_fini(mlxsw_sp); 9371 mlxsw_sp_mr_fini(mlxsw_sp); 9372 mlxsw_sp_lpm_fini(mlxsw_sp); 9373 rhashtable_destroy(&mlxsw_sp->router->nexthop_group_ht); 9374 rhashtable_destroy(&mlxsw_sp->router->nexthop_ht); 9375 mlxsw_sp_ipips_fini(mlxsw_sp); 9376 mlxsw_sp_rifs_fini(mlxsw_sp); 9377 __mlxsw_sp_router_fini(mlxsw_sp); 9378 mlxsw_sp_router_ll_op_ctx_fini(mlxsw_sp->router); 9379 mlxsw_sp_router_xm_fini(mlxsw_sp); 9380 mutex_destroy(&mlxsw_sp->router->lock); 9381 kfree(mlxsw_sp->router); 9382 } 9383